def test_jinja_string_templates(self): jinja2_path = jinja_template_path(system=False) self.assertTrue(os.path.exists(jinja2_path)) device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002', 'console_device': 'ttyfake1', 'baud_rate': 56 } data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.jinja2') template = prepare_jinja_template('cubie', data, system_path=False, path=jinja2_path) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot']['ramdisk'] self.assertIn('commands', ramdisk_args) self.assertIn('boot', ramdisk_args['commands']) self.assertIn( "setenv bootargs 'console=ttyfake1,56n8 root=/dev/ram0 ip=dhcp'", ramdisk_args['commands']) device_dictionary.update( { 'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08", 'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08", 'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08" } ) data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.jinja2') template = prepare_jinja_template('bbb', data, system_path=False, path=jinja2_path) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) device = PipelineDevice(yaml_data, 'bbb') self.assertIn('power_state', device) # bbb has power_on_command defined above self.assertEqual(device.power_state, 'off') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_vland_jinja2(self): """ Test complex device dictionary values The reference data can cross lines but cannot be indented as the pprint object in utils uses indent=0, width=80 for YAML compatibility. The strings read in from config files can have indenting spaces, these are removed in the pprint. """ data = """{% extends 'vland.jinja2' %} {% set interfaces = ['eth0', 'eth1'] %} {% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0', 'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %} {% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %} {% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %} {% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %} """ result = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1', 'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0' }, 'extends': 'vland.jinja2', 'mac_addr': { 'eth1': '00:24:d7:9b:c0:8c', 'eth0': 'f0:de:f1:46:8c:21' }, 'tags': { 'eth1': ['1G'], 'eth0': ['1G', '10G'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } dictionary = jinja2_to_devicedictionary(data_dict=data) self.assertEqual(result, dictionary) jinja2_str = devicedictionary_to_jinja2(data_dict=dictionary, extends='vland.jinja2') # ordering within the dict can change but each line needs to still appear for line in str(data).split('\n'): self.assertIn(line, str(jinja2_str)) # create a DeviceDictionary for this test vlan = DeviceDictionary(hostname='vlanned1') vlan.parameters = dictionary vlan.save() del vlan vlan = DeviceDictionary.get('vlanned1') cmp_str = str(devicedictionary_to_jinja2(vlan.parameters, 'vland.jinja2')) for line in str(data).split('\n'): self.assertIn(line, cmp_str)
def test_jinja_template(self): jinja2_path = os.path.realpath( os.path.join(__file__, '..', '..', '..', 'etc', 'dispatcher-config')) self.assertTrue(os.path.exists(jinja2_path)) device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': ['1G', '10G'], 'eth1': ['1G'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } # {% map = '{'eth1': {'3': 8}, 'eth0': {'3': 19}}' %} device_dict.save() data = devicedictionary_to_jinja2(device_dict.parameters, 'beaglebone-black.jinja2') check_str = """{% extends 'beaglebone-black.jinja2' %} {% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %} {% set interfaces = ['eth0', 'eth1'] %} {% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0', 'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %} {% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %} {% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %} """ self.assertEqual(check_str, data) template = prepare_jinja_template(self.factory.bbb1.hostname, data, system_path=False) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertIn('parameters', yaml_data) self.assertIn('interfaces', yaml_data['parameters']) self.assertIn('bootm', yaml_data['parameters']) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('actions', yaml_data) self.assertIn('eth0', yaml_data['parameters']['interfaces']) self.assertIn('eth1', yaml_data['parameters']['interfaces']) self.assertIn('sysfs', yaml_data['parameters']['interfaces']['eth0']) self.assertEqual( '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1', yaml_data['parameters']['interfaces']['eth1']['sysfs'])
def test_menu_context(self): job_ctx = { 'menu_early_printk': '', 'menu_interrupt_prompt': 'Default boot will start in', 'base_ip_args': 'ip=dhcp' } hostname = self.factory.make_fake_mustang_device() device_dict = DeviceDictionary.get(hostname) device_data = devicedictionary_to_jinja2( device_dict.parameters, device_dict.parameters['extends']) template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path) config_str = template.render(**job_ctx) self.assertIsNotNone(config_str) config = yaml.load(config_str) self.assertIsNotNone( config['actions']['boot']['methods']['uefi-menu']['nfs']) menu_data = config['actions']['boot']['methods']['uefi-menu'] # assert that menu_interrupt_prompt replaces the default 'The default boot selection will start in' self.assertEqual(menu_data['parameters']['interrupt_prompt'], job_ctx['menu_interrupt_prompt']) # assert that menu_early_printk replaces the default earlyprintk default self.assertEqual( [ e for e in menu_data['nfs'] if 'enter' in e['select'] and 'new Entry' in e['select']['wait'] ][0]['select']['enter'], 'console=ttyS0,115200 debug root=/dev/nfs rw nfsroot={NFS_SERVER_IP}:{NFSROOTFS},tcp,hard,intr ip=dhcp' )
def test_menu_device(self): job_ctx = {} hostname = 'mustang01' device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() device = self.factory.make_device(self.device_type, hostname) self.assertEqual(device.device_type.name, 'mustang-uefi') device_data = devicedictionary_to_jinja2( device_dict.parameters, device_dict.parameters['extends']) template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path) config_str = template.render(**job_ctx) self.assertIsNotNone(config_str) config = yaml.load(config_str) self.assertIsNotNone(config) self.assertEqual(config['device_type'], self.device_type.name) self.assertNotIn('parameters', config) self.assertIsNotNone( config['actions']['boot']['methods']['uefi-menu']['nfs']) menu_data = config['actions']['boot']['methods']['uefi-menu']['nfs'] tftp_menu = [ item for item in menu_data if 'items' in item['select'] and 'TFTP' in item['select']['items'][0] ][0] tftp_mac = self.conf['tftp_mac'] # value from device dictionary correctly replaces device type default self.assertIn(tftp_mac, tftp_menu['select']['items'][0])
def test_menu_device(self): job_ctx = {} hostname = 'mustang01' device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() device = self.factory.make_device(self.device_type, hostname) self.assertEqual(device.device_type.name, 'mustang-uefi') device_data = devicedictionary_to_jinja2( device_dict.parameters, device_dict.parameters['extends'] ) template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path) config_str = template.render(**job_ctx) self.assertIsNotNone(config_str) config = yaml.load(config_str) self.assertIsNotNone(config) self.assertEqual(config['device_type'], self.device_type.name) self.assertIsNotNone(config['parameters']) self.assertIsNotNone(config['actions']['boot']['methods']['uefi-menu']['nfs']) menu_data = config['actions']['boot']['methods']['uefi-menu']['nfs'] tftp_menu = [item for item in menu_data if 'items' in item['select'] and 'TFTP' in item['select']['items'][0]][0] tftp_mac = self.conf['tftp_mac'] # value from device dictionary correctly replaces device type default self.assertIn(tftp_mac, tftp_menu['select']['items'][0])
def test_menu_context(self): job_ctx = { 'menu_early_printk': '', 'menu_interrupt_prompt': 'Default boot will start in' } hostname = self.factory.make_fake_mustang_device() device_dict = DeviceDictionary.get(hostname) device_data = devicedictionary_to_jinja2( device_dict.parameters, device_dict.parameters['extends'] ) template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path) config_str = template.render(**job_ctx) self.assertIsNotNone(config_str) config = yaml.load(config_str) self.assertIsNotNone(config['actions']['boot']['methods']['uefi-menu']['nfs']) menu_data = config['actions']['boot']['methods']['uefi-menu'] # assert that menu_interrupt_prompt replaces the default 'The default boot selection will start in' self.assertEqual( menu_data['parameters']['interrupt_prompt'], job_ctx['menu_interrupt_prompt'] ) # assert that menu_early_printk replaces the default earlyprintk default self.assertEqual( [e for e in menu_data['nfs'] if 'enter' in e['select'] and 'new Entry' in e['select']['wait']][0]['select']['enter'], 'console=ttyS0,115200 debug root=/dev/nfs rw nfsroot={NFS_SERVER_IP}:{NFSROOTFS},tcp,hard,intr ip=dhcp' )
def test_jinja_postgres_loader(self): # path used for the device_type template jinja2_path = os.path.realpath(os.path.join(__file__, '..', '..', '..', 'etc', 'dispatcher-config')) self.assertTrue(os.path.exists(jinja2_path)) device_type = 'cubietruck' # pretend this was already imported into the database and use for comparison later. device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002' } # create a DeviceDictionary for this test cubie = DeviceDictionary(hostname='cubie') cubie.parameters = device_dictionary cubie.save() dict_loader = jinja2.DictLoader( { 'cubie.yaml': devicedictionary_to_jinja2(cubie.parameters, '%s.yaml' % device_type) } ) type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment( loader=jinja2.ChoiceLoader([dict_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'cubie') device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) device = PipelineDevice(yaml_data, 'cubie') self.assertIn('power_state', device) # cubie1 has no power_on_command defined self.assertEqual(device.power_state, '') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_jinja_postgres_loader(self): # path used for the device_type template jinja2_path = jinja_template_path(system=False) self.assertTrue(os.path.exists(jinja2_path)) device_type = 'cubietruck' # pretend this was already imported into the database and use for comparison later. device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002' } # create a DeviceDictionary for this test cubie = DeviceDictionary(hostname='cubie') cubie.parameters = device_dictionary cubie.save() jinja_data = devicedictionary_to_jinja2(cubie.parameters, '%s.jinja2' % device_type) dict_loader = jinja2.DictLoader({'cubie.jinja2': jinja_data}) type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment( loader=jinja2.ChoiceLoader([dict_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.jinja2" % 'cubie') # pylint gets this wrong from jinja device_configuration = template.render() # pylint: disable=no-member chk_template = prepare_jinja_template('cubie', jinja_data, system_path=False, path=jinja2_path) self.assertEqual(template.render(), chk_template.render()) # pylint: disable=no-member yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) device = PipelineDevice(yaml_data, 'cubie') self.assertIn('power_state', device) # cubie1 has no power_on_command defined self.assertEqual(device.power_state, '') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import'] is not None: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] is not None or options['review'] is not None: element = DeviceDictionary.get(hostname) data = None if element is None: self.stderr.write( "Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends']) if options['review'] is None: self.stdout.write(data) else: string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data}) type_loader = jinja2.FileSystemLoader( [os.path.join(options['path'], 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % hostname) device_configuration = template.render() # validate against the device schema try: validate_device(yaml.load(device_configuration)) except (yaml.YAMLError, SubmissionException) as exc: self.stderr.write("Invalid template: %s" % exc) self.stdout.write(device_configuration) else: self.stderr.write( "Please specify one of --import, --export or --review") sys.exit(1)
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import']: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] or options['review']: element = DeviceDictionary.get(hostname) if element is None: self.stderr.write( "Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends']) if not options['review']: self.stdout.write(data) else: template = prepare_jinja_template(hostname, data, system_path=False, path=options['path']) device_configuration = template.render() # validate against the device schema try: validate_device(yaml.load(device_configuration)) except (yaml.YAMLError, SubmissionException) as exc: self.stderr.write("Invalid template: %s" % exc) self.stdout.write(device_configuration) else: self.stderr.write( "Please specify one of --import, --export or --review") sys.exit(1)
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import'] is not None: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] is not None or options['review'] is not None: element = DeviceDictionary.get(hostname) data = None if element is None: self.stderr.write("Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends'] ) if options['review'] is None: self.stdout.write(data) else: template = prepare_jinja_template(hostname, data, system_path=False, path=options['path']) device_configuration = template.render() # validate against the device schema try: validate_device(yaml.load(device_configuration)) except (yaml.YAMLError, SubmissionException) as exc: self.stderr.write("Invalid template: %s" % exc) self.stdout.write(device_configuration) else: self.stderr.write("Please specify one of --import, --export or --review") sys.exit(1)
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import'] is not None: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] is not None or options['review'] is not None: element = DeviceDictionary.get(hostname) data = None if element is None: self.stderr.write("Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends'] ) if options['review'] is None: self.stdout.write(data) else: string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data}) type_loader = jinja2.FileSystemLoader([ os.path.join(options['path'], 'device-types')]) env = jinja2.Environment( loader=jinja2.ChoiceLoader([string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % hostname) device_configuration = template.render() self.stdout.write(device_configuration) else: self.stderr.write("Please specify one of --import, --export or --review") sys.exit(1)
def export_device_dictionary(self, hostname): """ Name ---- `export_device_dictionary` (`device_hostname`) Description ----------- [superuser only] Export the device dictionary key value store for a pipeline device. See also get_pipeline_device_config Arguments --------- `device_hostname`: string Device hostname to update. Return value ------------ This function returns an XML-RPC binary data of output file. """ self._authenticate() if not self.user.is_superuser: raise xmlrpclib.Fault( 403, "User '%s' is not superuser." % self.user.username ) try: device = Device.objects.get(hostname=hostname) except DeviceType.DoesNotExist: raise xmlrpclib.Fault( 404, "Device '%s' was not found." % hostname ) if not device.is_pipeline: raise xmlrpclib.Fault( 400, "Device '%s' is not a pipeline device" % hostname ) device_dict = DeviceDictionary.get(hostname) if not device_dict: raise xmlrpclib.Fault( 404, "Device '%s' does not have a device dictionary" % hostname ) device_dict = device_dict.to_dict() jinja_str = devicedictionary_to_jinja2(device_dict['parameters'], device_dict['parameters']['extends']) return xmlrpclib.Binary(jinja_str.encode('UTF-8'))
def get_pipeline_device_config(self, device_hostname): """ Name ---- `get_pipeline_device_config` (`device_hostname`) Description ----------- Get the pipeline device configuration for given device hostname. Arguments --------- `device_hostname`: string Device hostname for which the configuration is required. Return value ------------ This function returns an XML-RPC binary data of output file. """ if not device_hostname: raise xmlrpclib.Fault( 400, "Bad request: Device hostname was not " "specified.") element = DeviceDictionary.get(device_hostname) if element is None: raise xmlrpclib.Fault(404, "Specified device not found.") data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) string_loader = jinja2.DictLoader({'%s.yaml' % device_hostname: data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja_template_path(), 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % device_hostname) device_configuration = template.render() # validate against the device schema validate_device(device_configuration) return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def test_jinja_template(self): jinja2_path = os.path.realpath(os.path.join( __file__, '..', '..', '..', 'etc', 'dispatcher-config')) self.assertTrue(os.path.exists(jinja2_path)) device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"}, 'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"}, 'tags': {'eth0': ['1G', '10G'], 'eth1': ['1G']}, 'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} } # {% map = '{'eth1': {'3': 8}, 'eth0': {'3': 19}}' %} device_dict.save() data = devicedictionary_to_jinja2(device_dict.parameters, 'beaglebone-black.jinja2') check_str = """{% extends 'beaglebone-black.jinja2' %} {% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %} {% set interfaces = ['eth0', 'eth1'] %} {% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0', 'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %} {% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %} {% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %} """ self.assertEqual(check_str, data) template = prepare_jinja_template(self.factory.bbb1.hostname, data, system_path=False) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertIn('parameters', yaml_data) self.assertIn('interfaces', yaml_data['parameters']) self.assertIn('bootm', yaml_data['parameters']) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('actions', yaml_data) self.assertIn('eth0', yaml_data['parameters']['interfaces']) self.assertIn('eth1', yaml_data['parameters']['interfaces']) self.assertIn('sysfs', yaml_data['parameters']['interfaces']['eth0']) self.assertEqual( '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1', yaml_data['parameters']['interfaces']['eth1']['sysfs'] )
def get_pipeline_device_config(self, device_hostname): """ Name ---- `get_pipeline_device_config` (`device_hostname`) Description ----------- Get the pipeline device configuration for given device hostname. Arguments --------- `device_hostname`: string Device hostname for which the configuration is required. Return value ------------ This function returns an XML-RPC binary data of output file. """ if not device_hostname: raise xmlrpclib.Fault(400, "Bad request: Device hostname was not " "specified.") element = DeviceDictionary.get(device_hostname) if element is None: raise xmlrpclib.Fault(404, "Specified device not found.") data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) string_loader = jinja2.DictLoader({'%s.yaml' % device_hostname: data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja_template_path(), 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader([string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % device_hostname) device_configuration = template.render() return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def get_pipeline_device_config(self, device_hostname): """ Name ---- `get_pipeline_device_config` (`device_hostname`) Description ----------- Get the pipeline device configuration for given device hostname. Arguments --------- `device_hostname`: string Device hostname for which the configuration is required. Return value ------------ This function returns an XML-RPC binary data of output file. """ if not device_hostname: raise xmlrpclib.Fault(400, "Bad request: Device hostname was not " "specified.") element = DeviceDictionary.get(device_hostname) if element is None: raise xmlrpclib.Fault(404, "Specified device not found.") data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) template = prepare_jinja_template(device_hostname, data, system_path=True) device_configuration = template.render() # validate against the device schema validate_device(yaml.load(device_configuration)) return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
def validate_pipeline_devices(self, hostname=None): """ Name ---- `validate_pipeline_device` [`device_hostname`] Description ----------- Validate that the device dictionary and device-type template together create a valid YAML file which matches the pipeline device schema. See also get_pipeline_device_config Arguments --------- `device_hostname`: string Device hostname to validate. If a device hostname is not specified, all pipeline devices are checked. Return value ------------ This function returns an XML-RPC structure of results with the following fields. `device_hostname`: {'Valid': null} or `device_hostname`: {'Invalid': message} ` """ if not hostname: devices = Device.objects.filter(is_pipeline=True) else: devices = Device.objects.filter(is_pipeline=True, hostname=hostname) if not devices and hostname: raise xmlrpclib.Fault( 404, "No pipeline device found with hostname %s" % hostname ) if not devices and not hostname: raise xmlrpclib.Fault( 404, "No pipeline device found on this instance." ) results = {} for device in devices: key = str(device.hostname) element = DeviceDictionary.get(device.hostname) if element is None: results[key] = {'Invalid': "Missing device dictionary"} continue data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) if data is None: results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'} continue try: template = prepare_jinja_template(device.hostname, data, system_path=True) device_configuration = template.render() except jinja2.TemplateError as exc: results[key] = {'Invalid': exc} continue try: # validate against the device schema validate_device(yaml.load(device_configuration)) except SubmissionException as exc: results[key] = {'Invalid': exc} continue results[key] = {'Valid': None} return xmlrpclib.Binary(yaml.dump(results))
def validate_pipeline_devices(self, name=None): """ Name ---- `validate_pipeline_device` [`name`] Description ----------- Validate that the device dictionary and device-type template together create a valid YAML file which matches the pipeline device schema. Retired devices are ignored. See also get_pipeline_device_config Arguments --------- `name`: string Can be device hostname or device type name. If name is specified, method will search for either a matching device hostname or matching device type name in which case it will only validate that(those) device(s). If not specified, this method will validate all non-retired devices in the system. Return value ------------ This function returns an XML-RPC structure of results with the following fields. `device_hostname`: {'Valid': null} or `device_hostname`: {'Invalid': message} ` """ if not name: devices = Device.objects.filter( Q(is_pipeline=True) & ~Q(status=Device.RETIRED)) else: devices = Device.objects.filter( Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q( device_type__name=name)) if not devices: devices = Device.objects.filter( Q(is_pipeline=True) & ~Q(status=Device.RETIRED) & Q(hostname=name)) if not devices and name: raise xmlrpclib.Fault( 404, "No devices found with hostname or device type name %s" % name ) if not devices and not name: raise xmlrpclib.Fault( 404, "No pipeline device found on this instance." ) results = {} for device in devices: key = str(device.hostname) element = DeviceDictionary.get(device.hostname) if element is None: results[key] = {'Invalid': "Missing device dictionary"} continue data = devicedictionary_to_jinja2(element.parameters, element.parameters['extends']) if data is None: results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'} continue try: template = prepare_jinja_template(device.hostname, data, system_path=True) device_configuration = template.render() except jinja2.TemplateError as exc: results[key] = {'Invalid': exc} continue try: # validate against the device schema validate_device(yaml.load(device_configuration)) except SubmissionException as exc: results[key] = {'Invalid': exc} continue results[key] = {'Valid': None} return xmlrpclib.Binary(yaml.dump(results))
def test_jinja_string_templates(self): jinja2_path = os.path.realpath(os.path.join( __file__, '..', '..', '..', 'etc', 'dispatcher-config')) self.assertTrue(os.path.exists(jinja2_path)) device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002', 'console_device': 'ttyfake1', 'baud_rate': 56 } data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.yaml') string_loader = jinja2.DictLoader({'cubie.yaml': data}) type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment( loader=jinja2.ChoiceLoader([string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'cubie') device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot']['ramdisk'] self.assertIn('commands', ramdisk_args) self.assertIn('boot', ramdisk_args['commands']) self.assertIn( "setenv bootargs 'console=ttyfake1,56 debug rw root=/dev/ram0 ip=dhcp'", ramdisk_args['commands']) device_dictionary.update( { 'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08", 'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08", 'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08" } ) data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.yaml') string_loader = jinja2.DictLoader({'bbb.yaml': data}) type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment( loader=jinja2.ChoiceLoader([string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'bbb') device_configuration = template.render() yaml_data = yaml.load(device_configuration) device = PipelineDevice(yaml_data, 'bbb') self.assertIn('power_state', device) # bbb has power_on_command defined above self.assertEqual(device.power_state, 'off') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_jinja_string_templates(self): jinja2_path = os.path.realpath( os.path.join(__file__, '..', '..', '..', 'etc', 'dispatcher-config')) self.assertTrue(os.path.exists(jinja2_path)) device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002', 'console_device': 'ttyfake1', 'baud_rate': 56 } data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.yaml') string_loader = jinja2.DictLoader({'cubie.yaml': data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'cubie') device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn( 'uuid', yaml_data['parameters']['media']['usb'][ device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][ device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid']) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual(device_dictionary['connection_command'], yaml_data['commands']['connect']) ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot'][ 'ramdisk'] self.assertIn('commands', ramdisk_args) self.assertIn('boot', ramdisk_args['commands']) self.assertIn( "setenv bootargs 'console=ttyfake1,56 debug rw root=/dev/ram0 ip=dhcp'", ramdisk_args['commands']) device_dictionary.update({ 'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08", 'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08", 'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08" }) data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.yaml') string_loader = jinja2.DictLoader({'bbb.yaml': data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'bbb') device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) device = PipelineDevice(yaml_data, 'bbb') self.assertIn('power_state', device) # bbb has power_on_command defined above self.assertEqual(device.power_state, 'off') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)