def test_hikey620_uarts(self): with open( os.path.join(os.path.dirname(__file__), "devices", "hi6220-hikey-01.jinja2") ) as hikey: data = hikey.read() self.assertIsNotNone(data) job_ctx = {} self.assertTrue(self.validate_data("hi6220-hikey-01", data)) template_dict = prepare_jinja_template( "staging-hikey-01", data, job_ctx=job_ctx, raw=False ) validate_device(template_dict) self.assertIsNotNone(template_dict) self.assertIn("commands", template_dict) self.assertNotIn("connect", template_dict["commands"]) self.assertIn("connections", template_dict["commands"]) self.assertIn("uart0", template_dict["commands"]["connections"]) self.assertIn("uart1", template_dict["commands"]["connections"]) self.assertIn("tags", template_dict["commands"]["connections"]["uart1"]) self.assertIn( "primary", template_dict["commands"]["connections"]["uart1"]["tags"] ) self.assertNotIn("tags", template_dict["commands"]["connections"]["uart0"]) self.assertEqual( template_dict["commands"]["connections"]["uart0"]["connect"], "telnet localhost 4002", ) self.assertEqual( template_dict["commands"]["connections"]["uart1"]["connect"], "telnet 192.168.1.200 8001", )
def get_pipeline_device_config(self, device_hostname): """ Name ---- `get_pipeline_device_config` (`device_hostname`) Description ----------- Get the pipeline device configuration for given device hostname. Arguments --------- `device_hostname`: string Device hostname for which the configuration is required. Return value ------------ This function returns an XML-RPC binary data of output file. """ if not device_hostname: raise xmlrpclib.Fault( 400, "Bad request: Device hostname was not " "specified.") try: device = Device.objects.get(hostname=device_hostname) except Device.DoesNotExist: raise xmlrpclib.Fault(404, "Specified device not found.") config = device.load_configuration(output_format="yaml") # validate against the device schema validate_device(yaml.load(config)) return xmlrpclib.Binary(config.encode('UTF-8'))
def test_hikey620_uarts(self): with open( os.path.join(os.path.dirname(__file__), 'devices', 'hi6220-hikey-01.jinja2')) as hikey: data = hikey.read() self.assertIsNotNone(data) job_ctx = {} self.assertTrue(self.validate_data('hi6220-hikey-01', data)) template_dict = prepare_jinja_template('staging-hikey-01', data, job_ctx=job_ctx, raw=False) validate_device(template_dict) self.assertIsNotNone(template_dict) self.assertIn('commands', template_dict) self.assertNotIn('connect', template_dict['commands']) self.assertIn('connections', template_dict['commands']) self.assertIn('uart0', template_dict['commands']['connections']) self.assertIn('uart1', template_dict['commands']['connections']) self.assertIn('tags', template_dict['commands']['connections']['uart1']) self.assertIn( 'primary', template_dict['commands']['connections']['uart1']['tags']) self.assertNotIn('tags', template_dict['commands']['connections']['uart0']) self.assertEqual( template_dict['commands']['connections']['uart0']['connect'], 'telnet localhost 4002') self.assertEqual( template_dict['commands']['connections']['uart1']['connect'], 'telnet 192.168.1.200 8001')
def test_jinja_string_templates(self): jinja2_path = jinja_template_path(system=False) self.assertTrue(os.path.exists(jinja2_path)) device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002', 'console_device': 'ttyfake1', 'baud_rate': 56 } data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.jinja2') template = prepare_jinja_template('cubie', data, system_path=False, path=jinja2_path) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot']['ramdisk'] self.assertIn('commands', ramdisk_args) self.assertIn('boot', ramdisk_args['commands']) self.assertIn( "setenv bootargs 'console=ttyfake1,56n8 root=/dev/ram0 ip=dhcp'", ramdisk_args['commands']) device_dictionary.update( { 'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08", 'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08", 'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08" } ) data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.jinja2') template = prepare_jinja_template('bbb', data, system_path=False, path=jinja2_path) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) device = PipelineDevice(yaml_data, 'bbb') self.assertIn('power_state', device) # bbb has power_on_command defined above self.assertEqual(device.power_state, 'off') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_all_template_connections(self): path = os.path.dirname(CONFIG_PATH) templates = glob.glob(os.path.join(path, 'device-types', '*.jinja2')) self.assertNotEqual([], templates) # keep this out of the loop, as creating the environment is slow. path = os.path.dirname(CONFIG_PATH) fs_loader = jinja2.FileSystemLoader([os.path.join(path, 'device-types')]) env = jinja2.Environment(loader=fs_loader, trim_blocks=True, autoescape=False) # nosec - YAML, not HTML, no XSS scope. for template in templates: name = os.path.basename(template) data = "{%% extends '%s' %%}" % os.path.basename(template) data += "{% set connection_command = 'telnet calvin 6080' %}" test_template = env.from_string(data) rendered = test_template.render() template_dict = yaml.load(rendered, Loader=yaml.CSafeLoader) # nosec - safe_load implemented directly validate_device(template_dict) self.assertIn('connect', template_dict['commands']) self.assertNotIn( 'connections', template_dict['commands'], msg="%s - failed support for connection_list syntax" % name) data = "{%% extends '%s' %%}" % os.path.basename(template) data += "{% set connection_list = ['uart0'] %}" data += "{% set connection_commands = {'uart1': 'telnet calvin 6080'} %}" data += "{% set connection_tags = {'uart1': ['primary']} %}" test_template = env.from_string(data) rendered = test_template.render() template_dict = yaml.load(rendered, Loader=yaml.CSafeLoader) # nosec - safe_load implemented directly validate_device(template_dict) self.assertNotIn('connect', template_dict['commands']) self.assertIn( 'connections', template_dict['commands'], msg="%s - missing connection_list syntax" % name)
def get_device_config(self, device_hostname, context=None): """ New in api_version 2 - see system.api_version() Name ---- `get_device_config` (`device_hostname`, context=None) Description ----------- Get the device configuration for given device hostname. Arguments --------- `device_hostname`: string Device hostname for which the configuration is required. Some device templates need a context specified when processing the device-type template. This can be specified as a YAML string: `get_device_config` `('qemu01', '{arch: amd64}')` Return value ------------ This function returns an XML-RPC binary data of output file. """ if not device_hostname: raise xmlrpc.client.Fault( 400, "Bad request: Device hostname was not specified." ) job_ctx = None if context is not None: try: job_ctx = yaml_safe_load(context) except yaml.YAMLError as exc: raise xmlrpc.client.Fault( 400, "Job context '%s' is not valid. %s" % (context, exc) ) try: device = Device.objects.get(hostname=device_hostname) except Device.DoesNotExist: raise xmlrpc.client.Fault(404, "Specified device was not found.") if not device.can_view(self.user): raise xmlrpc.client.Fault( 401, "Permission denied for user to device %s" % device.hostname ) config = device.load_configuration(job_ctx=job_ctx, output_format="yaml") # validate against the device schema validate_device(yaml_safe_load(config)) return xmlrpc.client.Binary(config.encode("UTF-8"))
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import'] is not None: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] is not None or options['review'] is not None: element = DeviceDictionary.get(hostname) data = None if element is None: self.stderr.write( "Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends']) if options['review'] is None: self.stdout.write(data) else: string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data}) type_loader = jinja2.FileSystemLoader( [os.path.join(options['path'], 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % hostname) device_configuration = template.render() # validate against the device schema try: validate_device(yaml.load(device_configuration)) except (yaml.YAMLError, SubmissionException) as exc: self.stderr.write("Invalid template: %s" % exc) self.stdout.write(device_configuration) else: self.stderr.write( "Please specify one of --import, --export or --review") sys.exit(1)
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import']: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] or options['review']: element = DeviceDictionary.get(hostname) if element is None: self.stderr.write( "Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends']) if not options['review']: self.stdout.write(data) else: template = prepare_jinja_template(hostname, data, system_path=False, path=options['path']) device_configuration = template.render() # validate against the device schema try: validate_device(yaml.load(device_configuration)) except (yaml.YAMLError, SubmissionException) as exc: self.stderr.write("Invalid template: %s" % exc) self.stdout.write(device_configuration) else: self.stderr.write( "Please specify one of --import, --export or --review") sys.exit(1)
def get_device_config(self, device_hostname, context=None): """ New in api_version 2 - see system.api_version() Name ---- `get_device_config` (`device_hostname`, context=None) Description ----------- Get the device configuration for given device hostname. Arguments --------- `device_hostname`: string Device hostname for which the configuration is required. Some device templates need a context specified when processing the device-type template. This can be specified as a YAML string: `get_device_config` `('qemu01', '{arch: amd64}')` Return value ------------ This function returns an XML-RPC binary data of output file. """ if not device_hostname: raise xmlrpclib.Fault(400, "Bad request: Device hostname was not " "specified.") job_ctx = None if context is not None: try: job_ctx = yaml.load(context) except yaml.YAMLError as exc: raise xmlrpclib.Fault( 400, "Job context '%s' is not valid. %s" % (context, exc)) try: device = Device.objects.get(hostname=device_hostname) except Device.DoesNotExist: raise xmlrpclib.Fault(404, "Specified device was not found.") config = device.load_configuration(job_ctx=job_ctx, output_format="yaml") # validate against the device schema validate_device(yaml.load(config)) return xmlrpclib.Binary(config.encode('UTF-8'))
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import'] is not None: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] is not None or options['review'] is not None: element = DeviceDictionary.get(hostname) data = None if element is None: self.stderr.write("Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends'] ) if options['review'] is None: self.stdout.write(data) else: template = prepare_jinja_template(hostname, data, system_path=False, path=options['path']) device_configuration = template.render() # validate against the device schema try: validate_device(yaml.load(device_configuration)) except (yaml.YAMLError, SubmissionException) as exc: self.stderr.write("Invalid template: %s" % exc) self.stdout.write(device_configuration) else: self.stderr.write("Please specify one of --import, --export or --review") sys.exit(1)
def test_primary_connection_power_commands(self): data = """{% extends 'x86.jinja2' %} {% set power_off_command = '/usr/bin/pduclient --command off' %} {% set hard_reset_command = '/usr/bin/pduclient --command reset' %} {% set power_on_command = '/usr/bin/pduclient --command on' %} {% set connection_command = 'telnet localhost 7302' %}""" device_dict = self.render_device_dictionary("staging-x86-01", data) self.assertTrue(validate_device(yaml_safe_load(device_dict)))
def test_primary_connection_power_commands_empty_ssh_host(self): # pylint: disable=invalid-name data = """{% extends 'x86.jinja2' %} {% set power_off_command = '/usr/bin/pduclient --command off' %} {% set power_on_command = '/usr/bin/pduclient --command on' %} {% set hard_reset_command = '/usr/bin/pduclient --command reset' %} {% set connection_command = 'telnet localhost 7302' %} {% set ssh_host = '' %}""" device_dict = self.render_device_dictionary('staging-x86-01', data) self.assertTrue(validate_device(yaml.safe_load(device_dict)))
def test_all_templates(self): path = os.path.dirname(CONFIG_PATH) templates = glob.glob(os.path.join(path, 'device-types', '*.jinja2')) self.assertNotEqual([], templates) # keep this out of the loop, as creating the environment is slow. path = os.path.dirname(CONFIG_PATH) fs_loader = jinja2.FileSystemLoader([os.path.join(path, 'device-types')]) env = jinja2.Environment(loader=fs_loader, trim_blocks=True, autoescape=False) # nosec - YAML, not HTML, no XSS scope. for template in templates: data = "{%% extends '%s' %%}" % os.path.basename(template) try: test_template = env.from_string(data) rendered = test_template.render() template_dict = yaml.load(rendered, Loader=yaml.CLoader) # nosec - safe_load implemented directly validate_device(template_dict) except AssertionError as exc: self.fail("Template %s failed: %s" % (os.path.basename(template), exc))
def validate_data(self, hostname, data, job_ctx=None): rendered = self.render_device_dictionary(hostname, data, job_ctx, raw=True) try: ret = validate_device(yaml.load(rendered, Loader=yaml.CSafeLoader)) # nosec - safe_load implemented directly except SubmissionException as exc: print('#######') print(rendered) print('#######') self.fail(exc) return ret
def get_pipeline_device_config(self, device_hostname): """ Name ---- `get_pipeline_device_config` (`device_hostname`) Description ----------- Get the pipeline device configuration for given device hostname. Arguments --------- `device_hostname`: string Device hostname for which the configuration is required. Return value ------------ This function returns an XML-RPC binary data of output file. """ if not device_hostname: raise xmlrpclib.Fault( 400, "Bad request: Device hostname was not " "specified.") element = DeviceDictionary.get(device_hostname) if element is None: raise xmlrpclib.Fault(404, "Specified device not found.") data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) string_loader = jinja2.DictLoader({'%s.yaml' % device_hostname: data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja_template_path(), 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % device_hostname) device_configuration = template.render() # validate against the device schema validate_device(device_configuration) return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def test_jinja_postgres_loader(self): # path used for the device_type template jinja2_path = jinja_template_path(system=False) self.assertTrue(os.path.exists(jinja2_path)) device_type = 'cubietruck' # pretend this was already imported into the database and use for comparison later. device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002' } # create a DeviceDictionary for this test cubie = DeviceDictionary(hostname='cubie') cubie.parameters = device_dictionary cubie.save() jinja_data = devicedictionary_to_jinja2(cubie.parameters, '%s.jinja2' % device_type) dict_loader = jinja2.DictLoader({'cubie.jinja2': jinja_data}) type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment( loader=jinja2.ChoiceLoader([dict_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.jinja2" % 'cubie') # pylint gets this wrong from jinja device_configuration = template.render() # pylint: disable=no-member chk_template = prepare_jinja_template('cubie', jinja_data, system_path=False, path=jinja2_path) self.assertEqual(template.render(), chk_template.render()) # pylint: disable=no-member yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) device = PipelineDevice(yaml_data, 'cubie') self.assertIn('power_state', device) # cubie1 has no power_on_command defined self.assertEqual(device.power_state, '') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_all_template_connections(self): templates = glob.glob(os.path.join(settings.DEVICE_TYPES_PATH, "*.jinja2")) self.assertNotEqual([], templates) # keep this out of the loop, as creating the environment is slow. fs_loader = jinja2.FileSystemLoader([settings.DEVICE_TYPES_PATH]) env = jinja2.Environment( # nosec - YAML, not HTML, no XSS scope. loader=fs_loader, trim_blocks=True, autoescape=False ) for template in templates: name = os.path.basename(template) data = "{%% extends '%s' %%}" % os.path.basename(template) data += "{% set connection_command = 'telnet calvin 6080' %}" test_template = env.from_string(data) rendered = test_template.render() template_dict = yaml.load( # nosec - safe_load implemented directly rendered, Loader=yaml.CSafeLoader ) validate_device(template_dict) self.assertIn("connect", template_dict["commands"]) self.assertNotIn( "connections", template_dict["commands"], msg="%s - failed support for connection_list syntax" % name, ) data = "{%% extends '%s' %%}" % os.path.basename(template) data += "{% set connection_list = ['uart0'] %}" data += "{% set connection_commands = {'uart1': 'telnet calvin 6080'} %}" data += "{% set connection_tags = {'uart1': ['primary']} %}" test_template = env.from_string(data) rendered = test_template.render() template_dict = yaml.load( # nosec - safe_load implemented directly rendered, Loader=yaml.CSafeLoader ) validate_device(template_dict) self.assertNotIn("connect", template_dict["commands"]) self.assertIn( "connections", template_dict["commands"], msg="%s - missing connection_list syntax" % name, )
def validate_data(self, hostname, data, job_ctx=None): rendered = self.render_device_dictionary(hostname, data, job_ctx, raw=True) try: ret = validate_device(yaml_safe_load(rendered)) except SubmissionException as exc: print("#######") print(rendered) print("#######") self.fail(exc) return ret
def test_all_templates(self): templates = glob.glob( os.path.join(settings.DEVICE_TYPES_PATH, "*.jinja2")) self.assertNotEqual([], templates) # keep this out of the loop, as creating the environment is slow. fs_loader = jinja2.FileSystemLoader([settings.DEVICE_TYPES_PATH]) env = jinja2.Environment( # nosec - YAML, not HTML, no XSS scope. loader=fs_loader, trim_blocks=True, autoescape=False) for template in templates: data = "{%% extends '%s' %%}" % os.path.basename(template) try: test_template = env.from_string(data) rendered = test_template.render() template_dict = yaml_safe_load(rendered) validate_device(template_dict) except AssertionError as exc: self.fail("Template %s failed: %s" % (os.path.basename(template), exc))
def test_submission_schema(self): files = [] path = os.path.normpath(os.path.dirname(__file__)) for name in os.listdir(path): if name.endswith('.yaml'): files.append(name) device_files = [ # device files supporting unit tests 'bbb-01.yaml' ] # these files have already been split by utils as multinode sub_id jobs. # FIXME: validate the schema of split files using lava-dispatcher. split_files = [ 'kvm-multinode-client.yaml', 'kvm-multinode-server.yaml', 'qemu-ssh-guest-1.yaml', 'qemu-ssh-guest-2.yaml', 'qemu-ssh-parent.yaml' ] for filename in files: # some files are dispatcher-level test files, e.g. after the multinode split try: yaml_data = yaml.safe_load( open(os.path.join(path, filename), 'r')) except yaml.YAMLError as exc: raise RuntimeError("Decoding YAML job submission failed: %s." % exc) if filename in device_files: validate_device(yaml_data) continue if filename in split_files: self.assertRaises(SubmissionException, validate_submission, yaml_data) else: try: ret = validate_submission(yaml_data) self.assertTrue(ret) except SubmissionException as exc: msg = '########## %s ###########\n%s' % (filename, exc) self.fail(msg)
def validate_data(self, hostname, data, job_ctx=None): """ Needs to be passed a device dictionary (jinja2 format) """ rendered = self.render_device_dictionary(hostname, data, job_ctx) try: ret = validate_device(yaml.safe_load(rendered)) except (SubmissionException, ConfigurationError) as exc: print('#######') print(rendered) print('#######') self.fail(exc) return ret
def test_submission_schema(self): files = [] path = os.path.normpath(os.path.dirname(__file__)) for name in os.listdir(path): if name.endswith('.yaml'): files.append(name) device_files = [ # device files supporting unit tests 'bbb-01.yaml' ] # these files have already been split by utils as multinode sub_id jobs. # FIXME: validate the schema of split files using lava-dispatcher. split_files = [ 'kvm-multinode-client.yaml', 'kvm-multinode-server.yaml', 'qemu-ssh-guest-1.yaml', 'qemu-ssh-guest-2.yaml', 'qemu-ssh-parent.yaml' ] for filename in files: # some files are dispatcher-level test files, e.g. after the multinode split try: yaml_data = yaml.load(open(os.path.join(path, filename), 'r')) except yaml.YAMLError as exc: raise RuntimeError("Decoding YAML job submission failed: %s." % exc) if filename in device_files: validate_device(yaml_data) continue if filename in split_files: self.assertRaises(SubmissionException, validate_submission, yaml_data) else: try: ret = validate_submission(yaml_data) self.assertTrue(ret) except SubmissionException as exc: msg = '########## %s ###########\n%s' % (filename, exc) self.fail(msg)
def get_pipeline_device_config(self, device_hostname): """ Name ---- `get_pipeline_device_config` (`device_hostname`) Description ----------- Get the pipeline device configuration for given device hostname. Arguments --------- `device_hostname`: string Device hostname for which the configuration is required. Return value ------------ This function returns an XML-RPC binary data of output file. """ if not device_hostname: raise xmlrpclib.Fault(400, "Bad request: Device hostname was not " "specified.") element = DeviceDictionary.get(device_hostname) if element is None: raise xmlrpclib.Fault(404, "Specified device not found.") data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) template = prepare_jinja_template(device_hostname, data, system_path=True) device_configuration = template.render() # validate against the device schema validate_device(yaml.load(device_configuration)) return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def validate_data(self, hostname, data): test_template = prepare_jinja_template(hostname, data, system_path=False) rendered = test_template.render() if self.debug: print('#######') print(rendered) print('#######') try: ret = validate_device(yaml.load(rendered)) except SubmissionException as exc: print('#######') print(rendered) print('#######') self.fail(exc) return ret
def validate_data(self, hostname, data, job_ctx=None): if not job_ctx: job_ctx = {} test_template = prepare_jinja_template(hostname, data, system_path=self.system) rendered = test_template.render(**job_ctx) if self.debug: print('#######') print(rendered) print('#######') try: ret = validate_device(yaml.load(rendered)) except SubmissionException as exc: print('#######') print(rendered) print('#######') self.fail(exc) return ret
def validate_pipeline_devices(self, name=None): """ Name ---- `validate_pipeline_device` [`name`] Description ----------- Validate that the device dictionary and device-type template together create a valid YAML file which matches the pipeline device schema. Retired devices are ignored. See also get_pipeline_device_config Arguments --------- `name`: string Can be device hostname or device type name. If name is specified, method will search for either a matching device hostname or matching device type name in which case it will only validate that(those) device(s). If not specified, this method will validate all non-retired devices in the system. Return value ------------ This function returns an XML-RPC structure of results with the following fields. `device_hostname`: {'Valid': null} or `device_hostname`: {'Invalid': message} ` """ if not name: devices = Device.objects.filter( Q(is_pipeline=True) & ~Q(status=Device.RETIRED)) else: devices = Device.objects.filter( Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q( device_type__name=name)) if not devices: devices = Device.objects.filter( Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(hostname=name)) if not devices and name: raise xmlrpclib.Fault( 404, "No devices found with hostname or device type name %s" % name ) if not devices and not name: raise xmlrpclib.Fault( 404, "No pipeline device found on this instance." ) results = {} for device in devices: key = str(device.hostname) element = DeviceDictionary.get(device.hostname) if element is None: results[key] = {'Invalid': "Missing device dictionary"} continue data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) if data is None: results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'} continue try: template = prepare_jinja_template(device.hostname, data, system_path=True) device_configuration = template.render() except jinja2.TemplateError as exc: results[key] = {'Invalid': exc} continue try: # validate against the device schema validate_device(yaml.load(device_configuration)) except SubmissionException as exc: results[key] = {'Invalid': exc} continue results[key] = {'Valid': None} return xmlrpclib.Binary(yaml.dump(results))
def validate_pipeline_devices(self, name=None): """ Name ---- `validate_pipeline_device` [`name`] Description ----------- Validate that the device dictionary and device-type template together create a valid YAML file which matches the pipeline device schema. Retired devices are ignored. See also get_pipeline_device_config Arguments --------- `name`: string Can be device hostname or device type name. If name is specified, method will search for either a matching device hostname or matching device type name in which case it will only validate that(those) device(s). If not specified, this method will validate all non-retired devices in the system. Return value ------------ This function returns an XML-RPC structure of results with the following fields. `device_hostname`: {'Valid': null} or `device_hostname`: {'Invalid': message} ` """ if not name: devices = Device.objects.exclude(health=Device.HEALTH_RETIRED) else: devices = Device.objects.exclude(health=Device.HEALTH_RETIRED).filter(device_type__name=name) if not devices: devices = Device.objects.exclude(health=Device.HEALTH_RETIRED).filter(hostname=name) if not devices and name: raise xmlrpclib.Fault( 404, "No devices found with hostname or device type name %s" % name ) if not devices and not name: raise xmlrpclib.Fault( 404, "No pipeline device found on this instance." ) results = {} for device in devices: key = str(device.hostname) config = device.load_configuration(output_format="yaml") if config is None: results[key] = {'Invalid': "Missing device dictionary"} continue try: # validate against the device schema validate_device(yaml.load(config)) except SubmissionException as exc: results[key] = {'Invalid': exc} continue results[key] = {'Valid': None} return xmlrpclib.Binary(yaml.dump(results).encode('UTF-8'))
def test_jinja_string_templates(self): jinja2_path = os.path.realpath( os.path.join(__file__, '..', '..', '..', 'etc', 'dispatcher-config')) self.assertTrue(os.path.exists(jinja2_path)) device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002', 'console_device': 'ttyfake1', 'baud_rate': 56 } data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.yaml') string_loader = jinja2.DictLoader({'cubie.yaml': data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'cubie') device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn( 'uuid', yaml_data['parameters']['media']['usb'][ device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][ device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid']) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual(device_dictionary['connection_command'], yaml_data['commands']['connect']) ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot'][ 'ramdisk'] self.assertIn('commands', ramdisk_args) self.assertIn('boot', ramdisk_args['commands']) self.assertIn( "setenv bootargs 'console=ttyfake1,56 debug rw root=/dev/ram0 ip=dhcp'", ramdisk_args['commands']) device_dictionary.update({ 'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08", 'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08", 'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08" }) data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.yaml') string_loader = jinja2.DictLoader({'bbb.yaml': data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'bbb') device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) device = PipelineDevice(yaml_data, 'bbb') self.assertIn('power_state', device) # bbb has power_on_command defined above self.assertEqual(device.power_state, 'off') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def validate_pipeline_devices(self, name=None): """ Name ---- `validate_pipeline_device` [`name`] Description ----------- Validate that the device dictionary and device-type template together create a valid YAML file which matches the pipeline device schema. Retired devices are ignored. See also get_pipeline_device_config Arguments --------- `name`: string Can be device hostname or device type name. If name is specified, method will search for either a matching device hostname or matching device type name in which case it will only validate that(those) device(s). If not specified, this method will validate all non-retired devices in the system. Return value ------------ This function returns an XML-RPC structure of results with the following fields. `device_hostname`: {'Valid': null} or `device_hostname`: {'Invalid': message} ` """ if not name: devices = Device.objects.exclude(health=Device.HEALTH_RETIRED) else: devices = Device.objects.exclude(health=Device.HEALTH_RETIRED).filter(device_type__name=name) if not devices: devices = Device.objects.exclude(health=Device.HEALTH_RETIRED).filter(hostname=name) if not devices and name: raise xmlrpc.client.Fault( 404, "No devices found with hostname or device type name %s" % name ) if not devices and not name: raise xmlrpc.client.Fault( 404, "No pipeline device found on this instance." ) results = {} for device in devices: key = str(device.hostname) config = device.load_configuration(output_format="yaml") if config is None: results[key] = {'Invalid': "Missing device dictionary"} continue try: # validate against the device schema validate_device(yaml.safe_load(config)) except SubmissionException as exc: results[key] = {'Invalid': exc} continue results[key] = {'Valid': None} return xmlrpc.client.Binary(yaml.dump(results).encode('UTF-8'))
def validate_pipeline_devices(self, hostname=None): """ Name ---- `validate_pipeline_device` [`device_hostname`] Description ----------- Validate that the device dictionary and device-type template together create a valid YAML file which matches the pipeline device schema. See also get_pipeline_device_config Arguments --------- `device_hostname`: string Device hostname to validate. If a device hostname is not specified, all pipeline devices are checked. Return value ------------ This function returns an XML-RPC structure of results with the following fields. `device_hostname`: {'Valid': null} or `device_hostname`: {'Invalid': message} ` """ if not hostname: devices = Device.objects.filter(is_pipeline=True) else: devices = Device.objects.filter(is_pipeline=True, hostname=hostname) if not devices and hostname: raise xmlrpclib.Fault( 404, "No pipeline device found with hostname %s" % hostname ) if not devices and not hostname: raise xmlrpclib.Fault( 404, "No pipeline device found on this instance." ) results = {} for device in devices: key = str(device.hostname) element = DeviceDictionary.get(device.hostname) if element is None: results[key] = {'Invalid': "Missing device dictionary"} continue data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) if data is None: results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'} continue try: template = prepare_jinja_template(device.hostname, data, system_path=True) device_configuration = template.render() except jinja2.TemplateError as exc: results[key] = {'Invalid': exc} continue try: # validate against the device schema validate_device(yaml.load(device_configuration)) except SubmissionException as exc: results[key] = {'Invalid': exc} continue results[key] = {'Valid': None} return xmlrpclib.Binary(yaml.dump(results))