def test_hikey620_uarts(self):
with open(
os.path.join(os.path.dirname(__file__), "devices", "hi6220-hikey-01.jinja2")
) as hikey:
data = hikey.read()
self.assertIsNotNone(data)
job_ctx = {}
self.assertTrue(self.validate_data("hi6220-hikey-01", data))
template_dict = prepare_jinja_template(
"staging-hikey-01", data, job_ctx=job_ctx, raw=False
)
validate_device(template_dict)
self.assertIsNotNone(template_dict)
self.assertIn("commands", template_dict)
self.assertNotIn("connect", template_dict["commands"])
self.assertIn("connections", template_dict["commands"])
self.assertIn("uart0", template_dict["commands"]["connections"])
self.assertIn("uart1", template_dict["commands"]["connections"])
self.assertIn("tags", template_dict["commands"]["connections"]["uart1"])
self.assertIn(
"primary", template_dict["commands"]["connections"]["uart1"]["tags"]
)
self.assertNotIn("tags", template_dict["commands"]["connections"]["uart0"])
self.assertEqual(
template_dict["commands"]["connections"]["uart0"]["connect"],
"telnet localhost 4002",
)
self.assertEqual(
template_dict["commands"]["connections"]["uart1"]["connect"],
"telnet 192.168.1.200 8001",
)
def get_pipeline_device_config(self, device_hostname):
"""
Name
----
`get_pipeline_device_config` (`device_hostname`)
Description
-----------
Get the pipeline device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(
400, "Bad request: Device hostname was not "
"specified.")
try:
device = Device.objects.get(hostname=device_hostname)
except Device.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device not found.")
config = device.load_configuration(output_format="yaml")
# validate against the device schema
validate_device(yaml.load(config))
return xmlrpclib.Binary(config.encode('UTF-8'))
def test_hikey620_uarts(self):
with open(
os.path.join(os.path.dirname(__file__), 'devices',
'hi6220-hikey-01.jinja2')) as hikey:
data = hikey.read()
self.assertIsNotNone(data)
job_ctx = {}
self.assertTrue(self.validate_data('hi6220-hikey-01', data))
template_dict = prepare_jinja_template('staging-hikey-01',
data,
job_ctx=job_ctx,
raw=False)
validate_device(template_dict)
self.assertIsNotNone(template_dict)
self.assertIn('commands', template_dict)
self.assertNotIn('connect', template_dict['commands'])
self.assertIn('connections', template_dict['commands'])
self.assertIn('uart0', template_dict['commands']['connections'])
self.assertIn('uart1', template_dict['commands']['connections'])
self.assertIn('tags',
template_dict['commands']['connections']['uart1'])
self.assertIn(
'primary',
template_dict['commands']['connections']['uart1']['tags'])
self.assertNotIn('tags',
template_dict['commands']['connections']['uart0'])
self.assertEqual(
template_dict['commands']['connections']['uart0']['connect'],
'telnet localhost 4002')
self.assertEqual(
template_dict['commands']['connections']['uart1']['connect'],
'telnet 192.168.1.200 8001')
def test_jinja_string_templates(self):
jinja2_path = jinja_template_path(system=False)
self.assertTrue(os.path.exists(jinja2_path))
device_dictionary = {
'usb_label': 'SanDisk_Ultra',
'sata_label': 'ST160LM003',
'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0",
'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184",
'connection_command': 'telnet localhost 6002',
'console_device': 'ttyfake1',
'baud_rate': 56
}
data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.jinja2')
template = prepare_jinja_template('cubie', data, system_path=False, path=jinja2_path)
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
self.assertIn('timeouts', yaml_data)
self.assertIn('parameters', yaml_data)
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('media', yaml_data['parameters'])
self.assertIn('usb', yaml_data['parameters']['media'])
self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb'])
self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']])
self.assertEqual(
yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'],
device_dictionary['usb_uuid']
)
self.assertIn('commands', yaml_data)
self.assertIn('connect', yaml_data['commands'])
self.assertEqual(
device_dictionary['connection_command'],
yaml_data['commands']['connect'])
ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot']['ramdisk']
self.assertIn('commands', ramdisk_args)
self.assertIn('boot', ramdisk_args['commands'])
self.assertIn(
"setenv bootargs 'console=ttyfake1,56n8 root=/dev/ram0 ip=dhcp'",
ramdisk_args['commands'])
device_dictionary.update(
{
'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08",
'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08",
'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08"
}
)
data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.jinja2')
template = prepare_jinja_template('bbb', data, system_path=False, path=jinja2_path)
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
device = PipelineDevice(yaml_data, 'bbb')
self.assertIn('power_state', device)
# bbb has power_on_command defined above
self.assertEqual(device.power_state, 'off')
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def test_jinja_string_templates(self):
jinja2_path = jinja_template_path(system=False)
self.assertTrue(os.path.exists(jinja2_path))
device_dictionary = {
'usb_label': 'SanDisk_Ultra',
'sata_label': 'ST160LM003',
'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0",
'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184",
'connection_command': 'telnet localhost 6002',
'console_device': 'ttyfake1',
'baud_rate': 56
}
data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.jinja2')
template = prepare_jinja_template('cubie', data, system_path=False, path=jinja2_path)
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
self.assertIn('timeouts', yaml_data)
self.assertIn('parameters', yaml_data)
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('media', yaml_data['parameters'])
self.assertIn('usb', yaml_data['parameters']['media'])
self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb'])
self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']])
self.assertEqual(
yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'],
device_dictionary['usb_uuid']
)
self.assertIn('commands', yaml_data)
self.assertIn('connect', yaml_data['commands'])
self.assertEqual(
device_dictionary['connection_command'],
yaml_data['commands']['connect'])
ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot']['ramdisk']
self.assertIn('commands', ramdisk_args)
self.assertIn('boot', ramdisk_args['commands'])
self.assertIn(
"setenv bootargs 'console=ttyfake1,56n8 root=/dev/ram0 ip=dhcp'",
ramdisk_args['commands'])
device_dictionary.update(
{
'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08",
'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08",
'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08"
}
)
data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.jinja2')
template = prepare_jinja_template('bbb', data, system_path=False, path=jinja2_path)
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
device = PipelineDevice(yaml_data, 'bbb')
self.assertIn('power_state', device)
# bbb has power_on_command defined above
self.assertEqual(device.power_state, 'off')
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def test_all_template_connections(self):
path = os.path.dirname(CONFIG_PATH)
templates = glob.glob(os.path.join(path, 'device-types', '*.jinja2'))
self.assertNotEqual([], templates)
# keep this out of the loop, as creating the environment is slow.
path = os.path.dirname(CONFIG_PATH)
fs_loader = jinja2.FileSystemLoader([os.path.join(path, 'device-types')])
env = jinja2.Environment(loader=fs_loader, trim_blocks=True, autoescape=False) # nosec - YAML, not HTML, no XSS scope.
for template in templates:
name = os.path.basename(template)
data = "{%% extends '%s' %%}" % os.path.basename(template)
data += "{% set connection_command = 'telnet calvin 6080' %}"
test_template = env.from_string(data)
rendered = test_template.render()
template_dict = yaml.load(rendered, Loader=yaml.CSafeLoader) # nosec - safe_load implemented directly
validate_device(template_dict)
self.assertIn('connect', template_dict['commands'])
self.assertNotIn(
'connections',
template_dict['commands'], msg="%s - failed support for connection_list syntax" % name)
data = "{%% extends '%s' %%}" % os.path.basename(template)
data += "{% set connection_list = ['uart0'] %}"
data += "{% set connection_commands = {'uart1': 'telnet calvin 6080'} %}"
data += "{% set connection_tags = {'uart1': ['primary']} %}"
test_template = env.from_string(data)
rendered = test_template.render()
template_dict = yaml.load(rendered, Loader=yaml.CSafeLoader) # nosec - safe_load implemented directly
validate_device(template_dict)
self.assertNotIn('connect', template_dict['commands'])
self.assertIn(
'connections',
template_dict['commands'], msg="%s - missing connection_list syntax" % name)
def get_device_config(self, device_hostname, context=None):
"""
New in api_version 2 - see system.api_version()
Name
----
`get_device_config` (`device_hostname`, context=None)
Description
-----------
Get the device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Some device templates need a context specified when processing the
device-type template. This can be specified as a YAML string:
`get_device_config` `('qemu01', '{arch: amd64}')`
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpc.client.Fault(
400, "Bad request: Device hostname was not specified."
)
job_ctx = None
if context is not None:
try:
job_ctx = yaml_safe_load(context)
except yaml.YAMLError as exc:
raise xmlrpc.client.Fault(
400, "Job context '%s' is not valid. %s" % (context, exc)
)
try:
device = Device.objects.get(hostname=device_hostname)
except Device.DoesNotExist:
raise xmlrpc.client.Fault(404, "Specified device was not found.")
if not device.can_view(self.user):
raise xmlrpc.client.Fault(
401, "Permission denied for user to device %s" % device.hostname
)
config = device.load_configuration(job_ctx=job_ctx, output_format="yaml")
# validate against the device schema
validate_device(yaml_safe_load(config))
return xmlrpc.client.Binary(config.encode("UTF-8"))
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import'] is not None:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] is not None or options['review'] is not None:
element = DeviceDictionary.get(hostname)
data = None
if element is None:
self.stderr.write(
"Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters, element.parameters['extends'])
if options['review'] is None:
self.stdout.write(data)
else:
string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data})
type_loader = jinja2.FileSystemLoader(
[os.path.join(options['path'], 'device-types')])
env = jinja2.Environment(loader=jinja2.ChoiceLoader(
[string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % hostname)
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write(
"Please specify one of --import, --export or --review")
sys.exit(1)
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import']:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] or options['review']:
element = DeviceDictionary.get(hostname)
if element is None:
self.stderr.write(
"Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters, element.parameters['extends'])
if not options['review']:
self.stdout.write(data)
else:
template = prepare_jinja_template(hostname,
data,
system_path=False,
path=options['path'])
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write(
"Please specify one of --import, --export or --review")
sys.exit(1)
def get_device_config(self, device_hostname, context=None):
"""
New in api_version 2 - see system.api_version()
Name
----
`get_device_config` (`device_hostname`, context=None)
Description
-----------
Get the device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Some device templates need a context specified when processing the
device-type template. This can be specified as a YAML string:
`get_device_config` `('qemu01', '{arch: amd64}')`
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(400, "Bad request: Device hostname was not "
"specified.")
job_ctx = None
if context is not None:
try:
job_ctx = yaml.load(context)
except yaml.YAMLError as exc:
raise xmlrpclib.Fault(
400,
"Job context '%s' is not valid. %s" % (context, exc))
try:
device = Device.objects.get(hostname=device_hostname)
except Device.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device was not found.")
config = device.load_configuration(job_ctx=job_ctx, output_format="yaml")
# validate against the device schema
validate_device(yaml.load(config))
return xmlrpclib.Binary(config.encode('UTF-8'))
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import'] is not None:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] is not None or options['review'] is not None:
element = DeviceDictionary.get(hostname)
data = None
if element is None:
self.stderr.write("Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters,
element.parameters['extends']
)
if options['review'] is None:
self.stdout.write(data)
else:
template = prepare_jinja_template(hostname, data, system_path=False, path=options['path'])
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write("Please specify one of --import, --export or --review")
sys.exit(1)
def test_primary_connection_power_commands(self):
data = """{% extends 'x86.jinja2' %}
{% set power_off_command = '/usr/bin/pduclient --command off' %}
{% set hard_reset_command = '/usr/bin/pduclient --command reset' %}
{% set power_on_command = '/usr/bin/pduclient --command on' %}
{% set connection_command = 'telnet localhost 7302' %}"""
device_dict = self.render_device_dictionary("staging-x86-01", data)
self.assertTrue(validate_device(yaml_safe_load(device_dict)))
def test_primary_connection_power_commands_empty_ssh_host(self): # pylint: disable=invalid-name
data = """{% extends 'x86.jinja2' %}
{% set power_off_command = '/usr/bin/pduclient --command off' %}
{% set power_on_command = '/usr/bin/pduclient --command on' %}
{% set hard_reset_command = '/usr/bin/pduclient --command reset' %}
{% set connection_command = 'telnet localhost 7302' %}
{% set ssh_host = '' %}"""
device_dict = self.render_device_dictionary('staging-x86-01', data)
self.assertTrue(validate_device(yaml.safe_load(device_dict)))
def test_all_templates(self):
path = os.path.dirname(CONFIG_PATH)
templates = glob.glob(os.path.join(path, 'device-types', '*.jinja2'))
self.assertNotEqual([], templates)
# keep this out of the loop, as creating the environment is slow.
path = os.path.dirname(CONFIG_PATH)
fs_loader = jinja2.FileSystemLoader([os.path.join(path, 'device-types')])
env = jinja2.Environment(loader=fs_loader, trim_blocks=True, autoescape=False) # nosec - YAML, not HTML, no XSS scope.
for template in templates:
data = "{%% extends '%s' %%}" % os.path.basename(template)
try:
test_template = env.from_string(data)
rendered = test_template.render()
template_dict = yaml.load(rendered, Loader=yaml.CLoader) # nosec - safe_load implemented directly
validate_device(template_dict)
except AssertionError as exc:
self.fail("Template %s failed: %s" % (os.path.basename(template), exc))
def validate_data(self, hostname, data, job_ctx=None):
rendered = self.render_device_dictionary(hostname, data, job_ctx, raw=True)
try:
ret = validate_device(yaml.load(rendered, Loader=yaml.CSafeLoader)) # nosec - safe_load implemented directly
except SubmissionException as exc:
print('#######')
print(rendered)
print('#######')
self.fail(exc)
return ret
def get_pipeline_device_config(self, device_hostname):
"""
Name
----
`get_pipeline_device_config` (`device_hostname`)
Description
-----------
Get the pipeline device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(
400, "Bad request: Device hostname was not "
"specified.")
element = DeviceDictionary.get(device_hostname)
if element is None:
raise xmlrpclib.Fault(404, "Specified device not found.")
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
string_loader = jinja2.DictLoader({'%s.yaml' % device_hostname: data})
type_loader = jinja2.FileSystemLoader(
[os.path.join(jinja_template_path(), 'device-types')])
env = jinja2.Environment(loader=jinja2.ChoiceLoader(
[string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % device_hostname)
device_configuration = template.render()
# validate against the device schema
validate_device(device_configuration)
return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def test_jinja_postgres_loader(self):
# path used for the device_type template
jinja2_path = jinja_template_path(system=False)
self.assertTrue(os.path.exists(jinja2_path))
device_type = 'cubietruck'
# pretend this was already imported into the database and use for comparison later.
device_dictionary = {
'usb_label': 'SanDisk_Ultra',
'sata_label': 'ST160LM003',
'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0",
'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184",
'connection_command': 'telnet localhost 6002'
}
# create a DeviceDictionary for this test
cubie = DeviceDictionary(hostname='cubie')
cubie.parameters = device_dictionary
cubie.save()
jinja_data = devicedictionary_to_jinja2(cubie.parameters, '%s.jinja2' % device_type)
dict_loader = jinja2.DictLoader({'cubie.jinja2': jinja_data})
type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')])
env = jinja2.Environment(
loader=jinja2.ChoiceLoader([dict_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.jinja2" % 'cubie')
# pylint gets this wrong from jinja
device_configuration = template.render() # pylint: disable=no-member
chk_template = prepare_jinja_template('cubie', jinja_data, system_path=False, path=jinja2_path)
self.assertEqual(template.render(), chk_template.render()) # pylint: disable=no-member
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
self.assertIn('timeouts', yaml_data)
self.assertIn('parameters', yaml_data)
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('media', yaml_data['parameters'])
self.assertIn('usb', yaml_data['parameters']['media'])
self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb'])
self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']])
self.assertEqual(
yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'],
device_dictionary['usb_uuid']
)
self.assertIn('commands', yaml_data)
self.assertIn('connect', yaml_data['commands'])
self.assertEqual(
device_dictionary['connection_command'],
yaml_data['commands']['connect'])
device = PipelineDevice(yaml_data, 'cubie')
self.assertIn('power_state', device)
# cubie1 has no power_on_command defined
self.assertEqual(device.power_state, '')
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def test_jinja_postgres_loader(self):
# path used for the device_type template
jinja2_path = jinja_template_path(system=False)
self.assertTrue(os.path.exists(jinja2_path))
device_type = 'cubietruck'
# pretend this was already imported into the database and use for comparison later.
device_dictionary = {
'usb_label': 'SanDisk_Ultra',
'sata_label': 'ST160LM003',
'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0",
'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184",
'connection_command': 'telnet localhost 6002'
}
# create a DeviceDictionary for this test
cubie = DeviceDictionary(hostname='cubie')
cubie.parameters = device_dictionary
cubie.save()
jinja_data = devicedictionary_to_jinja2(cubie.parameters, '%s.jinja2' % device_type)
dict_loader = jinja2.DictLoader({'cubie.jinja2': jinja_data})
type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')])
env = jinja2.Environment(
loader=jinja2.ChoiceLoader([dict_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.jinja2" % 'cubie')
# pylint gets this wrong from jinja
device_configuration = template.render() # pylint: disable=no-member
chk_template = prepare_jinja_template('cubie', jinja_data, system_path=False, path=jinja2_path)
self.assertEqual(template.render(), chk_template.render()) # pylint: disable=no-member
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
self.assertIn('timeouts', yaml_data)
self.assertIn('parameters', yaml_data)
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('media', yaml_data['parameters'])
self.assertIn('usb', yaml_data['parameters']['media'])
self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb'])
self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']])
self.assertEqual(
yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'],
device_dictionary['usb_uuid']
)
self.assertIn('commands', yaml_data)
self.assertIn('connect', yaml_data['commands'])
self.assertEqual(
device_dictionary['connection_command'],
yaml_data['commands']['connect'])
device = PipelineDevice(yaml_data, 'cubie')
self.assertIn('power_state', device)
# cubie1 has no power_on_command defined
self.assertEqual(device.power_state, '')
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def test_all_template_connections(self):
templates = glob.glob(os.path.join(settings.DEVICE_TYPES_PATH, "*.jinja2"))
self.assertNotEqual([], templates)
# keep this out of the loop, as creating the environment is slow.
fs_loader = jinja2.FileSystemLoader([settings.DEVICE_TYPES_PATH])
env = jinja2.Environment( # nosec - YAML, not HTML, no XSS scope.
loader=fs_loader, trim_blocks=True, autoescape=False
)
for template in templates:
name = os.path.basename(template)
data = "{%% extends '%s' %%}" % os.path.basename(template)
data += "{% set connection_command = 'telnet calvin 6080' %}"
test_template = env.from_string(data)
rendered = test_template.render()
template_dict = yaml.load( # nosec - safe_load implemented directly
rendered, Loader=yaml.CSafeLoader
)
validate_device(template_dict)
self.assertIn("connect", template_dict["commands"])
self.assertNotIn(
"connections",
template_dict["commands"],
msg="%s - failed support for connection_list syntax" % name,
)
data = "{%% extends '%s' %%}" % os.path.basename(template)
data += "{% set connection_list = ['uart0'] %}"
data += "{% set connection_commands = {'uart1': 'telnet calvin 6080'} %}"
data += "{% set connection_tags = {'uart1': ['primary']} %}"
test_template = env.from_string(data)
rendered = test_template.render()
template_dict = yaml.load( # nosec - safe_load implemented directly
rendered, Loader=yaml.CSafeLoader
)
validate_device(template_dict)
self.assertNotIn("connect", template_dict["commands"])
self.assertIn(
"connections",
template_dict["commands"],
msg="%s - missing connection_list syntax" % name,
)
def validate_data(self, hostname, data, job_ctx=None):
rendered = self.render_device_dictionary(hostname,
data,
job_ctx,
raw=True)
try:
ret = validate_device(yaml_safe_load(rendered))
except SubmissionException as exc:
print("#######")
print(rendered)
print("#######")
self.fail(exc)
return ret
def test_all_templates(self):
templates = glob.glob(
os.path.join(settings.DEVICE_TYPES_PATH, "*.jinja2"))
self.assertNotEqual([], templates)
# keep this out of the loop, as creating the environment is slow.
fs_loader = jinja2.FileSystemLoader([settings.DEVICE_TYPES_PATH])
env = jinja2.Environment( # nosec - YAML, not HTML, no XSS scope.
loader=fs_loader,
trim_blocks=True,
autoescape=False)
for template in templates:
data = "{%% extends '%s' %%}" % os.path.basename(template)
try:
test_template = env.from_string(data)
rendered = test_template.render()
template_dict = yaml_safe_load(rendered)
validate_device(template_dict)
except AssertionError as exc:
self.fail("Template %s failed: %s" %
(os.path.basename(template), exc))
def test_submission_schema(self):
files = []
path = os.path.normpath(os.path.dirname(__file__))
for name in os.listdir(path):
if name.endswith('.yaml'):
files.append(name)
device_files = [
# device files supporting unit tests
'bbb-01.yaml'
]
# these files have already been split by utils as multinode sub_id jobs.
# FIXME: validate the schema of split files using lava-dispatcher.
split_files = [
'kvm-multinode-client.yaml', 'kvm-multinode-server.yaml',
'qemu-ssh-guest-1.yaml', 'qemu-ssh-guest-2.yaml',
'qemu-ssh-parent.yaml'
]
for filename in files:
# some files are dispatcher-level test files, e.g. after the multinode split
try:
yaml_data = yaml.safe_load(
open(os.path.join(path, filename), 'r'))
except yaml.YAMLError as exc:
raise RuntimeError("Decoding YAML job submission failed: %s." %
exc)
if filename in device_files:
validate_device(yaml_data)
continue
if filename in split_files:
self.assertRaises(SubmissionException, validate_submission,
yaml_data)
else:
try:
ret = validate_submission(yaml_data)
self.assertTrue(ret)
except SubmissionException as exc:
msg = '########## %s ###########\n%s' % (filename, exc)
self.fail(msg)
def validate_data(self, hostname, data, job_ctx=None):
"""
Needs to be passed a device dictionary (jinja2 format)
"""
rendered = self.render_device_dictionary(hostname, data, job_ctx)
try:
ret = validate_device(yaml.safe_load(rendered))
except (SubmissionException, ConfigurationError) as exc:
print('#######')
print(rendered)
print('#######')
self.fail(exc)
return ret
def test_submission_schema(self):
files = []
path = os.path.normpath(os.path.dirname(__file__))
for name in os.listdir(path):
if name.endswith('.yaml'):
files.append(name)
device_files = [
# device files supporting unit tests
'bbb-01.yaml'
]
# these files have already been split by utils as multinode sub_id jobs.
# FIXME: validate the schema of split files using lava-dispatcher.
split_files = [
'kvm-multinode-client.yaml',
'kvm-multinode-server.yaml',
'qemu-ssh-guest-1.yaml',
'qemu-ssh-guest-2.yaml',
'qemu-ssh-parent.yaml'
]
for filename in files:
# some files are dispatcher-level test files, e.g. after the multinode split
try:
yaml_data = yaml.load(open(os.path.join(path, filename), 'r'))
except yaml.YAMLError as exc:
raise RuntimeError("Decoding YAML job submission failed: %s." % exc)
if filename in device_files:
validate_device(yaml_data)
continue
if filename in split_files:
self.assertRaises(SubmissionException, validate_submission, yaml_data)
else:
try:
ret = validate_submission(yaml_data)
self.assertTrue(ret)
except SubmissionException as exc:
msg = '########## %s ###########\n%s' % (filename, exc)
self.fail(msg)
def get_pipeline_device_config(self, device_hostname):
"""
Name
----
`get_pipeline_device_config` (`device_hostname`)
Description
-----------
Get the pipeline device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(400, "Bad request: Device hostname was not "
"specified.")
element = DeviceDictionary.get(device_hostname)
if element is None:
raise xmlrpclib.Fault(404, "Specified device not found.")
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
template = prepare_jinja_template(device_hostname, data, system_path=True)
device_configuration = template.render()
# validate against the device schema
validate_device(yaml.load(device_configuration))
return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def get_pipeline_device_config(self, device_hostname):
"""
Name
----
`get_pipeline_device_config` (`device_hostname`)
Description
-----------
Get the pipeline device configuration for given device hostname.
Arguments
---------
`device_hostname`: string
Device hostname for which the configuration is required.
Return value
------------
This function returns an XML-RPC binary data of output file.
"""
if not device_hostname:
raise xmlrpclib.Fault(400, "Bad request: Device hostname was not "
"specified.")
element = DeviceDictionary.get(device_hostname)
if element is None:
raise xmlrpclib.Fault(404, "Specified device not found.")
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
template = prepare_jinja_template(device_hostname, data, system_path=True)
device_configuration = template.render()
# validate against the device schema
validate_device(yaml.load(device_configuration))
return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def validate_data(self, hostname, data):
test_template = prepare_jinja_template(hostname, data, system_path=False)
rendered = test_template.render()
if self.debug:
print('#######')
print(rendered)
print('#######')
try:
ret = validate_device(yaml.load(rendered))
except SubmissionException as exc:
print('#######')
print(rendered)
print('#######')
self.fail(exc)
return ret
def validate_data(self, hostname, data, job_ctx=None):
if not job_ctx:
job_ctx = {}
test_template = prepare_jinja_template(hostname,
data,
system_path=self.system)
rendered = test_template.render(**job_ctx)
if self.debug:
print('#######')
print(rendered)
print('#######')
try:
ret = validate_device(yaml.load(rendered))
except SubmissionException as exc:
print('#######')
print(rendered)
print('#######')
self.fail(exc)
return ret
def validate_pipeline_devices(self, name=None):
"""
Name
----
`validate_pipeline_device` [`name`]
Description
-----------
Validate that the device dictionary and device-type template
together create a valid YAML file which matches the pipeline
device schema.
Retired devices are ignored.
See also get_pipeline_device_config
Arguments
---------
`name`: string
Can be device hostname or device type name.
If name is specified, method will search for either a matching device
hostname or matching device type name in which case it will only
validate that(those) device(s).
If not specified, this method will validate all non-retired devices
in the system.
Return value
------------
This function returns an XML-RPC structure of results with the
following fields.
`device_hostname`: {'Valid': null}
or
`device_hostname`: {'Invalid': message}
`
"""
if not name:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED))
else:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(
device_type__name=name))
if not devices:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(hostname=name))
if not devices and name:
raise xmlrpclib.Fault(
404,
"No devices found with hostname or device type name %s" % name
)
if not devices and not name:
raise xmlrpclib.Fault(
404, "No pipeline device found on this instance."
)
results = {}
for device in devices:
key = str(device.hostname)
element = DeviceDictionary.get(device.hostname)
if element is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
if data is None:
results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'}
continue
try:
template = prepare_jinja_template(device.hostname, data, system_path=True)
device_configuration = template.render()
except jinja2.TemplateError as exc:
results[key] = {'Invalid': exc}
continue
try:
# validate against the device schema
validate_device(yaml.load(device_configuration))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
results[key] = {'Valid': None}
return xmlrpclib.Binary(yaml.dump(results))
def validate_pipeline_devices(self, name=None):
"""
Name
----
`validate_pipeline_device` [`name`]
Description
-----------
Validate that the device dictionary and device-type template
together create a valid YAML file which matches the pipeline
device schema.
Retired devices are ignored.
See also get_pipeline_device_config
Arguments
---------
`name`: string
Can be device hostname or device type name.
If name is specified, method will search for either a matching device
hostname or matching device type name in which case it will only
validate that(those) device(s).
If not specified, this method will validate all non-retired devices
in the system.
Return value
------------
This function returns an XML-RPC structure of results with the
following fields.
`device_hostname`: {'Valid': null}
or
`device_hostname`: {'Invalid': message}
`
"""
if not name:
devices = Device.objects.exclude(health=Device.HEALTH_RETIRED)
else:
devices = Device.objects.exclude(health=Device.HEALTH_RETIRED).filter(device_type__name=name)
if not devices:
devices = Device.objects.exclude(health=Device.HEALTH_RETIRED).filter(hostname=name)
if not devices and name:
raise xmlrpclib.Fault(
404,
"No devices found with hostname or device type name %s" % name
)
if not devices and not name:
raise xmlrpclib.Fault(
404, "No pipeline device found on this instance."
)
results = {}
for device in devices:
key = str(device.hostname)
config = device.load_configuration(output_format="yaml")
if config is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
try:
# validate against the device schema
validate_device(yaml.load(config))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
results[key] = {'Valid': None}
return xmlrpclib.Binary(yaml.dump(results).encode('UTF-8'))
def test_jinja_string_templates(self):
jinja2_path = os.path.realpath(
os.path.join(__file__, '..', '..', '..', 'etc',
'dispatcher-config'))
self.assertTrue(os.path.exists(jinja2_path))
device_dictionary = {
'usb_label': 'SanDisk_Ultra',
'sata_label': 'ST160LM003',
'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0",
'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184",
'connection_command': 'telnet localhost 6002',
'console_device': 'ttyfake1',
'baud_rate': 56
}
data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.yaml')
string_loader = jinja2.DictLoader({'cubie.yaml': data})
type_loader = jinja2.FileSystemLoader(
[os.path.join(jinja2_path, 'device-types')])
env = jinja2.Environment(loader=jinja2.ChoiceLoader(
[string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % 'cubie')
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
self.assertIn('timeouts', yaml_data)
self.assertIn('parameters', yaml_data)
self.assertIn('bootz', yaml_data['parameters'])
self.assertIn('media', yaml_data['parameters'])
self.assertIn('usb', yaml_data['parameters']['media'])
self.assertIn(device_dictionary['usb_label'],
yaml_data['parameters']['media']['usb'])
self.assertIn(
'uuid', yaml_data['parameters']['media']['usb'][
device_dictionary['usb_label']])
self.assertEqual(
yaml_data['parameters']['media']['usb'][
device_dictionary['usb_label']]['uuid'],
device_dictionary['usb_uuid'])
self.assertIn('commands', yaml_data)
self.assertIn('connect', yaml_data['commands'])
self.assertEqual(device_dictionary['connection_command'],
yaml_data['commands']['connect'])
ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot'][
'ramdisk']
self.assertIn('commands', ramdisk_args)
self.assertIn('boot', ramdisk_args['commands'])
self.assertIn(
"setenv bootargs 'console=ttyfake1,56 debug rw root=/dev/ram0 ip=dhcp'",
ramdisk_args['commands'])
device_dictionary.update({
'hard_reset_command':
"/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08",
'power_off_command':
"/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08",
'power_on_command':
"/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08"
})
data = devicedictionary_to_jinja2(device_dictionary,
'beaglebone-black.yaml')
string_loader = jinja2.DictLoader({'bbb.yaml': data})
type_loader = jinja2.FileSystemLoader(
[os.path.join(jinja2_path, 'device-types')])
env = jinja2.Environment(loader=jinja2.ChoiceLoader(
[string_loader, type_loader]),
trim_blocks=True)
template = env.get_template("%s.yaml" % 'bbb')
device_configuration = template.render()
yaml_data = yaml.load(device_configuration)
self.assertTrue(validate_device(yaml_data))
device = PipelineDevice(yaml_data, 'bbb')
self.assertIn('power_state', device)
# bbb has power_on_command defined above
self.assertEqual(device.power_state, 'off')
self.assertTrue(hasattr(device, 'power_state'))
self.assertFalse(hasattr(device, 'hostname'))
self.assertIn('hostname', device)
def validate_pipeline_devices(self, name=None):
"""
Name
----
`validate_pipeline_device` [`name`]
Description
-----------
Validate that the device dictionary and device-type template
together create a valid YAML file which matches the pipeline
device schema.
Retired devices are ignored.
See also get_pipeline_device_config
Arguments
---------
`name`: string
Can be device hostname or device type name.
If name is specified, method will search for either a matching device
hostname or matching device type name in which case it will only
validate that(those) device(s).
If not specified, this method will validate all non-retired devices
in the system.
Return value
------------
This function returns an XML-RPC structure of results with the
following fields.
`device_hostname`: {'Valid': null}
or
`device_hostname`: {'Invalid': message}
`
"""
if not name:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED))
else:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(
device_type__name=name))
if not devices:
devices = Device.objects.filter(
Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(hostname=name))
if not devices and name:
raise xmlrpclib.Fault(
404,
"No devices found with hostname or device type name %s" % name
)
if not devices and not name:
raise xmlrpclib.Fault(
404, "No pipeline device found on this instance."
)
results = {}
for device in devices:
key = str(device.hostname)
element = DeviceDictionary.get(device.hostname)
if element is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
if data is None:
results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'}
continue
try:
template = prepare_jinja_template(device.hostname, data, system_path=True)
device_configuration = template.render()
except jinja2.TemplateError as exc:
results[key] = {'Invalid': exc}
continue
try:
# validate against the device schema
validate_device(yaml.load(device_configuration))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
results[key] = {'Valid': None}
return xmlrpclib.Binary(yaml.dump(results))
def validate_pipeline_devices(self, name=None):
"""
Name
----
`validate_pipeline_device` [`name`]
Description
-----------
Validate that the device dictionary and device-type template
together create a valid YAML file which matches the pipeline
device schema.
Retired devices are ignored.
See also get_pipeline_device_config
Arguments
---------
`name`: string
Can be device hostname or device type name.
If name is specified, method will search for either a matching device
hostname or matching device type name in which case it will only
validate that(those) device(s).
If not specified, this method will validate all non-retired devices
in the system.
Return value
------------
This function returns an XML-RPC structure of results with the
following fields.
`device_hostname`: {'Valid': null}
or
`device_hostname`: {'Invalid': message}
`
"""
if not name:
devices = Device.objects.exclude(health=Device.HEALTH_RETIRED)
else:
devices = Device.objects.exclude(health=Device.HEALTH_RETIRED).filter(device_type__name=name)
if not devices:
devices = Device.objects.exclude(health=Device.HEALTH_RETIRED).filter(hostname=name)
if not devices and name:
raise xmlrpc.client.Fault(
404,
"No devices found with hostname or device type name %s" % name
)
if not devices and not name:
raise xmlrpc.client.Fault(
404, "No pipeline device found on this instance."
)
results = {}
for device in devices:
key = str(device.hostname)
config = device.load_configuration(output_format="yaml")
if config is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
try:
# validate against the device schema
validate_device(yaml.safe_load(config))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
results[key] = {'Valid': None}
return xmlrpc.client.Binary(yaml.dump(results).encode('UTF-8'))
def validate_pipeline_devices(self, hostname=None):
"""
Name
----
`validate_pipeline_device` [`device_hostname`]
Description
-----------
Validate that the device dictionary and device-type template
together create a valid YAML file which matches the pipeline
device schema.
See also get_pipeline_device_config
Arguments
---------
`device_hostname`: string
Device hostname to validate.
If a device hostname is not specified, all pipeline devices
are checked.
Return value
------------
This function returns an XML-RPC structure of results with the
following fields.
`device_hostname`: {'Valid': null}
or
`device_hostname`: {'Invalid': message}
`
"""
if not hostname:
devices = Device.objects.filter(is_pipeline=True)
else:
devices = Device.objects.filter(is_pipeline=True, hostname=hostname)
if not devices and hostname:
raise xmlrpclib.Fault(
404, "No pipeline device found with hostname %s" % hostname
)
if not devices and not hostname:
raise xmlrpclib.Fault(
404, "No pipeline device found on this instance."
)
results = {}
for device in devices:
key = str(device.hostname)
element = DeviceDictionary.get(device.hostname)
if element is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
if data is None:
results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'}
continue
try:
template = prepare_jinja_template(device.hostname, data, system_path=True)
device_configuration = template.render()
except jinja2.TemplateError as exc:
results[key] = {'Invalid': exc}
continue
try:
# validate against the device schema
validate_device(yaml.load(device_configuration))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
results[key] = {'Valid': None}
return xmlrpclib.Binary(yaml.dump(results))
