def test_match_devices_without_map(self): """ Without a map, there is no support for knowing which interfaces to put onto a VLAN, so these devices cannot be assigned to a VLAN testjob See http://localhost/static/docs/v2/vland.html#vland-and-interface-tags-in-lava """ devices = Device.objects.filter( status=Device.IDLE).order_by('is_public') self.factory.ensure_tag('usb-eth') self.factory.ensure_tag('sata') self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth') self.factory.bbb1.save() self.factory.cubie1.tags = Tag.objects.filter(name='sata') self.factory.cubie1.save() device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) self.assertIsNone(device_dict.to_dict()['parameters']) device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname) self.assertIsNone(device_dict.to_dict()['parameters']) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertIsNone(device) # no map defined self.assertFalse( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertIsNone(assignments['client']) self.assertIsNone(assignments['server'])
def test_jinja_template(self): jinja2_path = os.path.realpath( os.path.join(__file__, '..', '..', '..', 'etc', 'dispatcher-config')) self.assertTrue(os.path.exists(jinja2_path)) device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': ['1G', '10G'], 'eth1': ['1G'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } # {% map = '{'eth1': {'3': 8}, 'eth0': {'3': 19}}' %} device_dict.save() data = devicedictionary_to_jinja2(device_dict.parameters, 'beaglebone-black.jinja2') check_str = """{% extends 'beaglebone-black.jinja2' %} {% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %} {% set interfaces = ['eth0', 'eth1'] %} {% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0', 'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %} {% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %} {% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %} """ self.assertEqual(check_str, data) template = prepare_jinja_template(self.factory.bbb1.hostname, data, system_path=False) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertIn('parameters', yaml_data) self.assertIn('interfaces', yaml_data['parameters']) self.assertIn('bootm', yaml_data['parameters']) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('actions', yaml_data) self.assertIn('eth0', yaml_data['parameters']['interfaces']) self.assertIn('eth1', yaml_data['parameters']['interfaces']) self.assertIn('sysfs', yaml_data['parameters']['interfaces']['eth0']) self.assertEqual( '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1', yaml_data['parameters']['interfaces']['eth1']['sysfs'])
def test_select_device(self): self.restart() hostname = 'fakeqemu3' device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() device = self.factory.make_device(self.device_type, hostname) job = TestJob.from_yaml_and_user( self.factory.make_job_yaml(), self.factory.make_user()) # this uses the system jinja2 path - local changes to the qemu.jinja2 # will not be available. selected = select_device(job, self.dispatchers) self.assertIsNone(selected) job.actual_device = device selected = select_device(job, self.dispatchers) self.assertIsNone(selected) device.worker_host = self.worker selected = select_device(job, self.dispatchers) self.assertIsNone(selected) # device needs to be in reserved state # fake up the assignment which needs a separate test job.actual_device = device job.save() device.current_job = job device.status = Device.RESERVED device.save() selected = select_device(job, self.dispatchers) self.assertEqual(selected, device)
def test_pipeline_device(self): foo = DeviceDictionary(hostname='foo') foo.parameters = { 'bootz': { 'kernel': '0x4700000', 'ramdisk': '0x4800000', 'dtb': '0x4300000' }, 'media': { 'usb': { 'UUID-required': True, 'SanDisk_Ultra': { 'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0', 'device_id': 0 }, 'sata': { 'UUID-required': False } } } } device = PipelineDevice(foo.parameters, 'foo') self.assertEqual(device.target, 'foo') self.assertIn('power_state', device) self.assertEqual(device.power_state, '') # there is no power_on_command for this device, so the property is '' self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def handle_add(self, hostname, device_type, worker_name, description, dictionary, pipeline, public, online): try: dt = DeviceType.objects.get(name=device_type) except DeviceType.DoesNotExist: self.stderr.write("Unable to find device-type '%s'" % device_type) sys.exit(1) try: worker = Worker.objects.get(hostname=worker_name) except Worker.DoesNotExist: self.stderr.write("Unable to find worker '%s'" % worker_name) sys.exit(1) status = Device.IDLE if online else Device.OFFLINE Device.objects.create(hostname=hostname, device_type=dt, description=description, worker_host=worker, is_pipeline=pipeline, status=status, is_public=public) if dictionary is not None: data = jinja2_to_devicedictionary(dictionary.read()) if data is None: self.stderr.write("Invalid device dictionary") sys.exit(1) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save()
def test_menu_device(self): job_ctx = {} hostname = 'mustang01' device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() device = self.factory.make_device(self.device_type, hostname) self.assertEqual(device.device_type.name, 'mustang-uefi') device_data = devicedictionary_to_jinja2( device_dict.parameters, device_dict.parameters['extends']) template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path) config_str = template.render(**job_ctx) self.assertIsNotNone(config_str) config = yaml.load(config_str) self.assertIsNotNone(config) self.assertEqual(config['device_type'], self.device_type.name) self.assertNotIn('parameters', config) self.assertIsNotNone( config['actions']['boot']['methods']['uefi-menu']['nfs']) menu_data = config['actions']['boot']['methods']['uefi-menu']['nfs'] tftp_menu = [ item for item in menu_data if 'items' in item['select'] and 'TFTP' in item['select']['items'][0] ][0] tftp_mac = self.conf['tftp_mac'] # value from device dictionary correctly replaces device type default self.assertIn(tftp_mac, tftp_menu['select']['items'][0])
def test_make_device(self): hostname = 'fakeqemu3' device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() device = self.factory.make_device(self.device_type, hostname) self.assertEqual(device.device_type.name, 'qemu') job = self.factory.make_job_yaml() self.assertIsNotNone(job)
def test_new_dictionary(self): foo = JobPipeline.get('foo') self.assertIsNone(foo) foo = DeviceDictionary(hostname='foo') foo.save() self.assertEqual(foo.hostname, 'foo') self.assertIsInstance(foo, DeviceDictionary) foo = DeviceDictionary.get('foo') self.assertIsNotNone(foo)
def test_vland_jinja2(self): """ Test complex device dictionary values The reference data can cross lines but cannot be indented as the pprint object in utils uses indent=0, width=80 for YAML compatibility. The strings read in from config files can have indenting spaces, these are removed in the pprint. """ data = """{% extends 'vland.jinja2' %} {% set interfaces = ['eth0', 'eth1'] %} {% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0', 'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %} {% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %} {% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %} {% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %} """ result = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1', 'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0' }, 'extends': 'vland.jinja2', 'mac_addr': { 'eth1': '00:24:d7:9b:c0:8c', 'eth0': 'f0:de:f1:46:8c:21' }, 'tags': { 'eth1': ['1G'], 'eth0': ['1G', '10G'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } dictionary = jinja2_to_devicedictionary(data_dict=data) self.assertEqual(result, dictionary) jinja2_str = devicedictionary_to_jinja2(data_dict=dictionary, extends='vland.jinja2') # ordering within the dict can change but each line needs to still appear for line in str(data).split('\n'): self.assertIn(line, str(jinja2_str)) # create a DeviceDictionary for this test vlan = DeviceDictionary(hostname='vlanned1') vlan.parameters = dictionary vlan.save() del vlan vlan = DeviceDictionary.get('vlanned1') cmp_str = str(devicedictionary_to_jinja2(vlan.parameters, 'vland.jinja2')) for line in str(data).split('\n'): self.assertIn(line, cmp_str)
def test_jinja_postgres_loader(self): # path used for the device_type template jinja2_path = jinja_template_path(system=False) self.assertTrue(os.path.exists(jinja2_path)) device_type = 'cubietruck' # pretend this was already imported into the database and use for comparison later. device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002' } # create a DeviceDictionary for this test cubie = DeviceDictionary(hostname='cubie') cubie.parameters = device_dictionary cubie.save() jinja_data = devicedictionary_to_jinja2(cubie.parameters, '%s.jinja2' % device_type) dict_loader = jinja2.DictLoader({'cubie.jinja2': jinja_data}) type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment( loader=jinja2.ChoiceLoader([dict_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.jinja2" % 'cubie') # pylint gets this wrong from jinja device_configuration = template.render() # pylint: disable=no-member chk_template = prepare_jinja_template('cubie', jinja_data, system_path=False, path=jinja2_path) self.assertEqual(template.render(), chk_template.render()) # pylint: disable=no-member yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) device = PipelineDevice(yaml_data, 'cubie') self.assertIn('power_state', device) # cubie1 has no power_on_command defined self.assertEqual(device.power_state, '') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import'] is not None: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] is not None or options['review'] is not None: element = DeviceDictionary.get(hostname) data = None if element is None: self.stderr.write( "Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends']) if options['review'] is None: self.stdout.write(data) else: string_loader = jinja2.DictLoader({'%s.yaml' % hostname: data}) type_loader = jinja2.FileSystemLoader( [os.path.join(options['path'], 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % hostname) device_configuration = template.render() # validate against the device schema try: validate_device(yaml.load(device_configuration)) except (yaml.YAMLError, SubmissionException) as exc: self.stderr.write("Invalid template: %s" % exc) self.stdout.write(device_configuration) else: self.stderr.write( "Please specify one of --import, --export or --review") sys.exit(1)
def handle(self, *args, **options): """ Accept options via lava-server manage which provides access to the database. """ hostname = options['hostname'] if hostname is None: self.stderr.write("Please specify a hostname") sys.exit(2) if options['import']: data = parse_template(options['import']) element = DeviceDictionary.get(hostname) if element is None: self.stdout.write("Adding new device dictionary for %s" % hostname) element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() self.stdout.write("Device dictionary updated for %s" % hostname) elif options['export'] or options['review']: element = DeviceDictionary.get(hostname) if element is None: self.stderr.write( "Unable to export - no dictionary found for '%s'" % hostname) sys.exit(2) else: data = devicedictionary_to_jinja2( element.parameters, element.parameters['extends']) if not options['review']: self.stdout.write(data) else: template = prepare_jinja_template(hostname, data, system_path=False, path=options['path']) device_configuration = template.render() # validate against the device schema try: validate_device(yaml.load(device_configuration)) except (yaml.YAMLError, SubmissionException) as exc: self.stderr.write("Invalid template: %s" % exc) self.stdout.write(device_configuration) else: self.stderr.write( "Please specify one of --import, --export or --review") sys.exit(1)
def test_host_role(self): # need a full job to properly test the multinode YAML split hostname = 'fakeqemu3' self.factory.make_device(self.device_type, hostname) device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() # create a new device to allow the submission to reach the multinode YAML test. hostname = 'fakeqemu4' self.factory.make_device(self.device_type, hostname) data = yaml.load(self.factory.make_job_json()) data['protocols']['lava-multinode']['roles']['host']['count'] = 2 self.assertRaises(SubmissionException, TestJob.from_yaml_and_user, yaml.dump(data), self.factory.make_user())
def handle_set(self, options): """ Set device properties """ hostname = options["hostname"] try: device = Device.objects.get(hostname=hostname) except Device.DoesNotExist: self.stderr.write("Unable to find device '%s'" % hostname) sys.exit(1) status = options["status"] if status is not None: device.status = self.device_status[status] health = options["health"] if health is not None: device.health_status = self.health_status[health] description = options["description"] if description is not None: device.description = description worker_name = options["worker"] if worker_name is not None: try: worker = Worker.objects.get(hostname=worker_name) device.worker_host = worker except Worker.DoesNotExist: self.stderr.write("Unable to find worker '%s'" % worker_name) sys.exit(1) public = options["public"] if public is not None: device.is_public = public dictionary = options["dictionary"] if dictionary is not None: data = jinja2_to_devicedictionary(dictionary.read()) if data is None: self.stderr.write("Invalid device dictionary") sys.exit(1) element = DeviceDictionary.get(hostname) if element is None: element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = data element.save() # Save the modifications device.save()
def test_exclusivity(self): device = Device.objects.get(hostname="fakeqemu1") self.assertTrue(device.is_pipeline) self.assertFalse(device.is_exclusive) self.assertIsNotNone(DeviceDictionary.get(device.hostname)) device_dict = DeviceDictionary(hostname=device.hostname) device_dict.save() device_dict = DeviceDictionary.get(device.hostname) self.assertTrue(device.is_pipeline) self.assertFalse(device.is_exclusive) update = device_dict.to_dict() update.update({'exclusive': 'True'}) device_dict.parameters = update device_dict.save() self.assertTrue(device.is_pipeline) self.assertTrue(device.is_exclusive)
def test_dictionary_remove(self): foo = DeviceDictionary(hostname='foo') foo.parameters = { 'bootz': { 'kernel': '0x4700000', 'ramdisk': '0x4800000', 'dtb': '0x4300000' }, } foo.save() baz = DeviceDictionary.get('foo') self.assertEqual(baz.parameters, foo.parameters) baz.delete() self.assertIsInstance(baz, DeviceDictionary) baz = DeviceDictionary.get('foo') self.assertIsNone(baz)
def test_from_json_rejects_exclusive(self): panda_type = self.factory.ensure_device_type(name='panda') panda_board = self.factory.make_device(device_type=panda_type, hostname='panda01') self.assertFalse(panda_board.is_exclusive) job = TestJob.from_json_and_user( self.factory.make_job_json(device_type='panda'), self.factory.make_user()) self.assertEqual(panda_type, job.requested_device_type) device_dict = DeviceDictionary.get(panda_board.hostname) self.assertIsNone(device_dict) device_dict = DeviceDictionary(hostname=panda_board.hostname) device_dict.parameters = {'exclusive': 'True'} device_dict.save() self.assertTrue(panda_board.is_exclusive) self.assertRaises(DevicesUnavailableException, _check_exclusivity, [panda_board], pipeline=False)
def test_find_nonexclusive_device(self): """ test that exclusive devices are not assigned JSON jobs """ self.assertFalse(self.panda01.is_exclusive) device_dict = DeviceDictionary.get(self.panda01.hostname) self.assertIsNone(device_dict) device_dict = DeviceDictionary(hostname=self.panda01.hostname) device_dict.parameters = {'exclusive': 'True'} device_dict.save() self.assertTrue(self.panda01.is_exclusive) self.assertRaises(DevicesUnavailableException, self.submit_job, target='panda01', device_type='panda') job = self.submit_job(device_type='panda') devices = [self.panda02, self.panda01] self.assertEqual(find_device_for_job(job, devices), self.panda02) device_dict.delete() self.assertFalse(self.panda01.is_exclusive)
def test_job_handlers(self): self.restart() hostname = 'fakeqemu3' device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() device = self.factory.make_device(self.device_type, hostname) job = TestJob.from_yaml_and_user( self.factory.make_job_yaml(), self.factory.make_user()) selected = select_device(job, self.dispatchers) self.assertIsNone(selected) job.actual_device = device selected = select_device(job, self.dispatchers) self.assertIsNone(selected) device.worker_host = self.worker selected = select_device(job, self.dispatchers) self.assertIsNone(selected) create_job(job, device) self.assertEqual(job.actual_device, device) self.assertEqual(device.status, Device.RESERVED)
def test_broken_link_yaml(self): hostname = 'fakeqemu3' self.factory.make_device(self.device_type, hostname) device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() # create a new device to allow the submission to reach the multinode YAML test. hostname = 'fakeqemu4' self.factory.make_device(self.device_type, hostname) data = yaml.load(self.factory.make_job_json()) deploy = [action['deploy'] for action in data['actions'] if 'deploy' in action] # replace working image with a broken URL for block in deploy: block['image'] = 'http://localhost/unknown/invalid.gz' try: jobs = TestJob.from_yaml_and_user( self.factory.make_job_json(), self.factory.make_user()) except DevicesUnavailableException as exc: self.fail(exc) self.assertEqual( jobs[0].sub_id, "%d.%d" % (int(jobs[0].id), 0))
def test_dictionary_parameters(self): foo = DeviceDictionary(hostname='foo') foo.parameters = { 'bootz': { 'kernel': '0x4700000', 'ramdisk': '0x4800000', 'dtb': '0x4300000' }, 'media': { 'usb': { 'UUID-required': True, 'SanDisk_Ultra': { 'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0', 'device_id': 0 }, 'sata': { 'UUID-required': False } } } } foo.save() bar = DeviceDictionary.get('foo') self.assertEqual(bar.parameters, foo.parameters)
def test_match_devices_with_map(self): devices = Device.objects.filter( status=Device.IDLE).order_by('is_public') device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { # client, RJ45 10M only 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } device_dict.save() device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname) device_dict.parameters = { # server includes 100M 'extends': 'cubietruck.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M'] }, 'map': { 'eth0': { '192.168.0.2': 4 }, 'eth1': { '192.168.0.2': 6 } } } device_dict.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) del (data['protocols']['lava-multinode']['roles']['client']['tags']) del (data['protocols']['lava-multinode']['roles']['server']['tags']) interfaces = [] job_dict = split_multinode_yaml(data, 'abcdefg123456789') client_job = job_dict['client'][0] device_dict = DeviceDictionary.get( self.factory.bbb1.hostname).to_dict() self.assertIsNotNone(device_dict) tag_list = client_job['protocols']['lava-vland']['vlan_one']['tags'] for interface, tags in device_dict['parameters']['tags'].iteritems(): if set(tags) & set(tag_list) == set( tag_list) and interface not in interfaces: interfaces.append(interface) break self.assertEqual(['eth1'], interfaces) self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys())) interfaces = [] server_job = job_dict['server'][0] device_dict = DeviceDictionary.get( self.factory.cubie1.hostname).to_dict() self.assertIsNotNone(device_dict) tag_list = server_job['protocols']['lava-vland']['vlan_two']['tags'] for interface, tags in device_dict['parameters']['tags'].iteritems(): if set(tags) & set(tag_list) == set( tag_list) and interface not in interfaces: interfaces.append(interface) break self.assertEqual(['eth1'], interfaces) self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys())) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) # vlan_one: client role. RJ45 10M. bbb device type # vlan_two: server role. RJ45 100M. cubie device type. assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_same_type_devices_with_map(self): device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { # client, RJ45 10M 100M 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:22", 'eth1': "00:24:d7:9b:c0:8b" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } device_dict.save() bbb2 = self.factory.make_device(self.factory.bbb_type, hostname='bbb2') device_dict = DeviceDictionary(hostname=bbb2.hostname) device_dict.parameters = { # server, RJ45 10M 100M 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M'] }, 'map': { 'eth0': { '192.168.0.2': 7 }, 'eth1': { '192.168.0.2': 9 } } } device_dict.save() devices = list( Device.objects.filter(status=Device.IDLE).order_by('is_public')) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-bbb-vland-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device if device in devices: devices.remove(device) assign_jobs() self.factory.bbb1.refresh_from_db() bbb2.refresh_from_db() self.assertIsNotNone(self.factory.bbb1.current_job) self.assertIsNotNone(bbb2.current_job) self.assertIsNotNone(self.factory.bbb1.current_job.actual_device) self.assertIsNotNone(bbb2.current_job.actual_device) self.assertNotEqual(self.factory.bbb1.current_job, bbb2.current_job) self.assertNotEqual(self.factory.bbb1.current_job.actual_device, bbb2.current_job.actual_device)
def test_new_dictionary(self): foo = DeviceDictionary(hostname='foo') foo.save() self.assertEqual(foo.hostname, 'foo')
def test_match_devices_with_map_and_tags(self): devices = Device.objects.filter( status=Device.IDLE).order_by('is_public') self.factory.ensure_tag('usb-eth') self.factory.ensure_tag('sata') self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth') self.factory.bbb1.save() self.factory.cubie1.tags = Tag.objects.filter(name='sata') self.factory.cubie1.save() device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } device_dict.save() device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '100M'] }, 'map': { 'eth0': { '192.168.0.2': 4 }, 'eth1': { '192.168.0.2': 6 } } } device_dict.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def import_device_dictionary(self, hostname, jinja_str): """ Name ---- `import_device_dictionary` (`device_hostname`, `jinja_string`) Description ----------- [superuser only] Import or update the device dictionary key value store for a pipeline device. This action will be logged. Arguments --------- `device_hostname`: string Device hostname to update. `jinja_str`: string Jinja2 settings to store in the DeviceDictionary Return value ------------ This function returns an XML-RPC binary data of output file. """ self._authenticate() if not self.user.is_superuser: raise xmlrpclib.Fault( 403, "User '%s' is not superuser." % self.user.username ) try: device = Device.objects.get(hostname=hostname) except DeviceType.DoesNotExist: raise xmlrpclib.Fault( 404, "Device '%s' was not found." % hostname ) try: device_data = jinja2_to_devicedictionary(jinja_str) except (ValueError, KeyError, TypeError): raise xmlrpclib.Fault( 400, "Unable to parse specified jinja string" ) if not device_data or 'extends' not in device_data: raise xmlrpclib.Fault( 400, "Invalid device dictionary content - %s - not updating." % jinja_str ) try: template = prepare_jinja_template(hostname, jinja_str, system_path=True) except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: raise xmlrpclib.Fault( 400, "Template error: %s" % exc ) if not template: raise xmlrpclib.Fault(400, "Empty template") element = DeviceDictionary.get(hostname) msg = '' if element is None: msg = "Adding new device dictionary for %s\n" % hostname element = DeviceDictionary(hostname=hostname) element.hostname = hostname element.parameters = device_data element.save() msg += "Device dictionary updated for %s\n" % hostname device.log_admin_entry(self.user, msg) return msg
def test_make_ssh_guest_yaml(self): hostname = 'fakeqemu3' device_dict = DeviceDictionary(hostname=hostname) device_dict.parameters = self.conf device_dict.save() device = self.factory.make_device(self.device_type, hostname) try: jobs = TestJob.from_yaml_and_user( self.factory.make_job_json(), self.factory.make_user()) except DevicesUnavailableException as exc: self.fail(exc) sub_id = [] group_size = 0 path = os.path.dirname(os.path.join(__file__)) host_role = [] for job in jobs: data = yaml.load(job.definition) params = data['protocols']['lava-multinode'] params['target_group'] = 'replaced' if not group_size: group_size = params['group_size'] if job.sub_id.endswith('.0'): self.assertFalse(job.dynamic_connection) self.assertEqual(job.requested_device_type.name, device.device_type.name) self.assertEqual(params['sub_id'], 0) sub_id.append(params['sub_id']) self.assertEqual( data, yaml.load(open(os.path.join(path, 'qemu-ssh-parent.yaml'), 'r').read()) ) self.assertEqual(job.device_role, 'host') host_role.append(job.device_role) else: self.assertTrue(job.dynamic_connection) self.assertNotIn(sub_id, params['sub_id']) sub_id.append(params['sub_id']) self.assertIsNone(job.requested_device_type) deploy = [action for action in data['actions'] if 'deploy' in action][0] self.assertEqual(deploy['deploy']['connection'], 'ssh') # validate each job if params['sub_id'] == 1: self.assertEqual( data, yaml.load(open(os.path.join(path, 'qemu-ssh-guest-1.yaml'), 'r').read()) ) elif params['sub_id'] == 2: self.assertEqual( data, yaml.load(open(os.path.join(path, 'qemu-ssh-guest-2.yaml'), 'r').read()) ) else: self.fail("Unexpected sub_id parameter") self.assertIsNone(job.requested_device_type) self.assertIsNone(job.actual_device) self.assertIsNone(job.requested_device) host_role.append(data['host_role']) self.assertFalse(any(role for role in host_role if role != 'host')) self.assertEqual(len(sub_id), group_size) self.assertEqual(sub_id, range(0, group_size))
def make_fake_mustang_device(self, hostname='fakemustang1'): # pylint: disable=no-self-use mustang = DeviceDictionary(hostname=hostname) mustang.parameters = {'extends': 'mustang-uefi.jinja2'} mustang.save() return hostname
def test_job_protocols(self): self.factory.ensure_tag('usb-eth') self.factory.ensure_tag('sata') self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth') self.factory.bbb1.save() self.factory.cubie1.tags = Tag.objects.filter(name='sata') self.factory.cubie1.save() device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': ['1G', '10G'], 'eth1': ['1G'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } device_dict.save() device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': ['1G', '10G'], 'eth1': ['1G'] }, 'map': { 'eth0': { '192.168.0.2': 4 }, 'eth1': { '192.168.0.2': 6 } } } device_dict.save() target_group = "unit-test-only" job_dict = split_multinode_yaml(self.factory.make_vland_job(), target_group) client_job = job_dict['client'][0] client_handle, client_file_name = tempfile.mkstemp() yaml.dump(client_job, open(client_file_name, 'w')) # YAML device file, as required by lava-dispatch --target device_yaml_file = os.path.realpath( os.path.join(os.path.dirname(__file__), 'bbb-01.yaml')) self.assertTrue(os.path.exists(device_yaml_file)) parser = JobParser() bbb_device = NewDevice(device_yaml_file) with open(client_file_name) as sample_job_data: bbb_job = parser.parse(sample_job_data, bbb_device, 4212, None, None, None, output_dir='/tmp/') os.close(client_handle) os.unlink(client_file_name) self.assertIn('protocols', bbb_job.parameters) self.assertIn(VlandProtocol.name, bbb_job.parameters['protocols']) self.assertIn(MultinodeProtocol.name, bbb_job.parameters['protocols'])
def make_fake_qemu_device(self, hostname='fakeqemu1'): # pylint: disable=no-self-use qemu = DeviceDictionary(hostname=hostname) qemu.parameters = {'extends': 'qemu.yaml', 'arch': 'amd64'} qemu.save()