def test_find_device_for_job_with_multiple_tags(self): """ test that tags are used to set which device is selected choose black02 and never black01 due to the presence of both the common tag and the unique tag only with black02. """ job = self.submit_job(device_type='beaglebone', tags=[ self.common_tag.name, self.unique_tag.name ]) devices = [self.panda01, self.black01, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black02, chosen_device) try: job = self.submit_job(device_type='panda', tags=[ self.common_tag.name, self.unique_tag.name ]) except DevicesUnavailableException: pass else: self.fail("Offered a panda when no pandas support the requested tags") devices = [self.black01, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black02, chosen_device) devices = [self.arndale02, self.panda02, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black02, chosen_device)
def test_find_device_for_job_with_multiple_tags(self): """ test that tags are used to set which device is selected choose black02 and never black01 due to the presence of both the common tag and the unique tag only with black02. """ job = self.submit_job(device_type='beaglebone', tags=[ self.common_tag.name, self.unique_tag.name ]) devices = [self.panda01, self.black01, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black02, chosen_device) try: job = self.submit_job(device_type='panda', tags=[ self.common_tag.name, self.unique_tag.name ]) except DevicesUnavailableException: pass else: self.fail("Offered a panda when no pandas support the requested tags") devices = [self.black01, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black02, chosen_device) devices = [self.arndale02, self.panda02, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black02, chosen_device)
def test_same_type_devices_with_map(self): bbb2 = self.factory.make_device(self.factory.bbb_type, hostname='bbb-02') devices = list(Device.objects.filter(status=Device.IDLE).order_by('is_public')) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'bbb-bbb-vland-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertIsNotNone(device) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device if device in devices: devices.remove(device) assign_jobs() self.factory.bbb1.refresh_from_db() bbb2.refresh_from_db() self.assertIsNotNone(self.factory.bbb1.current_job) self.assertIsNotNone(bbb2.current_job) self.assertIsNotNone(self.factory.bbb1.current_job.actual_device) self.assertIsNotNone(bbb2.current_job.actual_device) # pylint: disable=no-member self.assertNotEqual(self.factory.bbb1.current_job, bbb2.current_job) self.assertNotEqual( self.factory.bbb1.current_job.actual_device, bbb2.current_job.actual_device) # pylint: disable=no-member
def test_pipeline_health_assignment(self): user = self.factory.make_user() device1 = Device.objects.get(hostname='fakeqemu1') self.factory.make_fake_qemu_device(hostname="fakeqemu1") self.assertTrue(device1.is_pipeline) device2 = self.factory.make_device(device_type=device1.device_type, hostname='fakeqemu2') self.factory.make_fake_qemu_device(hostname="fakeqemu2") self.assertTrue(device2.is_pipeline) device3 = self.factory.make_device(device_type=device1.device_type, hostname='fakeqemu3') self.factory.make_fake_qemu_device(hostname="fakeqemu3") self.assertTrue(device3.is_pipeline) job1 = testjob_submission(self.factory.make_job_yaml(), user, check_device=device1) job2 = testjob_submission(self.factory.make_job_yaml(), user, check_device=device2) self.assertEqual(user, job1.submitter) self.assertTrue(job1.health_check) self.assertEqual(job1.requested_device, device1) self.assertEqual(job2.requested_device, device2) self.assertIsNone(job1.actual_device) self.assertIsNone(job2.actual_device) device_list = Device.objects.filter(device_type=device1.device_type) count = 0 while True: device_list.reverse() assigned = find_device_for_job(job2, device_list) if assigned != device2: self.fail("[%d] invalid device assigment in health check." % count) count += 1 if count > 100: break self.assertGreater(count, 100)
def test_match_devices_without_map(self): devices = Device.objects.filter( status=Device.IDLE).order_by('is_public') self.factory.ensure_tag('usb-eth') self.factory.ensure_tag('sata') self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth') self.factory.bbb1.save() self.factory.cubie1.tags = Tag.objects.filter(name='sata') self.factory.cubie1.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # no map defined self.assertFalse( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_find_device_with_single_job_tag(self): """ tests handling of jobs with less tags than supported but still choosing one tag which only applies to one device in the set. """ job = self.submit_job(device_type='beaglebone', tags=[self.unique_tag.name]) devices = [self.panda02, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black02, chosen_device) job = self.submit_job(device_type='beaglebone', tags=[self.exclusion_tag.name]) devices = [self.panda02, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black03, chosen_device)
def test_match_devices_without_map(self): """ Without a map, there is no support for knowing which interfaces to put onto a VLAN, so these devices cannot be assigned to a VLAN testjob See http://localhost/static/docs/v2/vland.html#vland-and-interface-tags-in-lava """ self.bbb3 = self.factory.make_device(self.factory.bbb_type, hostname='bbb-03') self.cubie2 = self.factory.make_device(self.factory.cubie_type, hostname='cubie2') devices = [self.bbb3, self.cubie2] self.factory.ensure_tag('usb-eth') self.factory.ensure_tag('sata') self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth') self.factory.bbb1.save() self.factory.cubie1.tags = Tag.objects.filter(name='sata') self.factory.cubie1.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertIsNone(device) # no map defined self.assertFalse(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertIsNone(assignments['client']) self.assertIsNone(assignments['server'])
def test_find_device_for_job(self): """ tests that find_device_for_job gives preference to matching by requested _device_ over matching by requested device _type_. """ job = self.submit_job(target='panda01', device_type='panda') devices = [self.panda02, self.panda01] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.panda01, chosen_device)
def test_find_device_with_single_job_tag(self): """ tests handling of jobs with less tags than supported but still choosing one tag which only applies to one device in the set. """ job = self.submit_job(device_type='beaglebone', tags=[ self.unique_tag.name ]) devices = [self.panda02, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black02, chosen_device) job = self.submit_job(device_type='beaglebone', tags=[ self.exclusion_tag.name ]) devices = [self.panda02, self.black02, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black03, chosen_device)
def test_find_device_for_job(self): """ tests that find_device_for_job gives preference to matching by requested _device_ over matching by requested device _type_. """ job = self.submit_job(target='panda01', device_type='panda') devices = [self.panda02, self.panda01] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.panda01, chosen_device)
def test_find_device_for_job_with_tag(self): """ test that tags are used to set which device is selected panda should be excluded by device_type black03 should be excluded as it does not have the common tag black02 would also match but is not included in the device check """ job = self.submit_job(device_type='beaglebone', tags=[self.common_tag.name]) devices = [self.panda01, self.arndale02, self.black01, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black01, chosen_device)
def test_queueing(self): """ uses stderr to avoid buffered prints Expect the test itself to take <30s and the gap between jobs submitted and end being ~500ms Most of the time is spent setting up the database and submitting all the test jobs. """ print >> sys.stderr, timezone.now(), "start" user = self.factory.ensure_user('test', '*****@*****.**', 'test') user.user_permissions.add( Permission.objects.get(codename='add_testjob')) user.save() device_type = self.factory.make_device_type('beaglebone-black') device = self.factory.make_device(device_type=device_type, hostname="black01") device.save() device_type = self.factory.make_device_type('wandboard') count = 1 while count < 100: suffix = "{:02d}".format(count) device = self.factory.make_device(device_type=device_type, hostname="imx6q-%s" % suffix) device.save() count += 1 print >> sys.stderr, timezone.now(), "%d dummy devices created" % count device_list = list(get_available_devices()) print >> sys.stderr, timezone.now( ), "%d available devices" % len(device_list) filename = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'master-check.json') self.assertTrue(os.path.exists(filename)) with open(filename, 'r') as json_file: definition = json_file.read() count = 0 # each 1000 more can take ~15s in the test. while count < 1000: # simulate API submission job = testjob_submission(definition, user) self.assertFalse(job.health_check) count += 1 print >> sys.stderr, timezone.now(), "%d jobs submitted" % count jobs = list(get_job_queue()) self.assertIsNotNone(jobs) print >> sys.stderr, timezone.now(), "Finding devices for jobs." for job in jobs: # this needs to stay as a tight loop to cope with load device = find_device_for_job(job, device_list) if device: print >> sys.stderr, timezone.now(), "[%d] allocated %s" % ( job.id, device) device_list.remove(device) print >> sys.stderr, timezone.now(), "end"
def test_find_device_for_job_with_tag(self): """ test that tags are used to set which device is selected panda should be excluded by device_type black03 should be excluded as it does not have the common tag black02 would also match but is not included in the device check """ job = self.submit_job(device_type='beaglebone', tags=[ self.common_tag.name ]) devices = [self.panda01, self.arndale02, self.black01, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.black01, chosen_device)
def test_match_devices_with_map(self): devices = Device.objects.filter(status=Device.IDLE).order_by('is_public') user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) del(data['protocols']['lava-multinode']['roles']['client']['tags']) del(data['protocols']['lava-multinode']['roles']['server']['tags']) interfaces = [] job_dict = split_multinode_yaml(data, 'abcdefg123456789') client_job = job_dict['client'][0] device_dict = self.factory.bbb1.load_configuration() self.assertIsNotNone(device_dict) tag_list = client_job['protocols']['lava-vland']['vlan_one']['tags'] for interface in device_dict['parameters']['interfaces']: tags = device_dict['parameters']['interfaces'][interface]['tags'] if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces: interfaces.append(interface) break self.assertEqual(['eth1'], interfaces) self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys())) interfaces = [] server_job = job_dict['server'][0] device_dict = self.factory.cubie1.load_configuration() self.assertIsNotNone(device_dict) tag_list = server_job['protocols']['lava-vland']['vlan_two']['tags'] for interface in device_dict['parameters']['interfaces']: tags = device_dict['parameters']['interfaces'][interface]['tags'] if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces: interfaces.append(interface) break self.assertEqual(['eth1'], interfaces) self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys())) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) # vlan_one: client role. RJ45 10M. bbb device type # vlan_two: server role. RJ45 100M. cubie device type. assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_find_nonexclusive_device(self): """ test that exclusive devices are not assigned JSON jobs """ self.panda03 = self.factory.make_device(device_type=self.panda, hostname='panda03') self.assertTrue(self.panda03.is_exclusive) self.assertRaises(DevicesUnavailableException, self.submit_job, target='panda03', device_type='panda') job = self.submit_job(device_type='panda') devices = [self.panda02, self.panda01] self.assertEqual(find_device_for_job(job, devices), self.panda02)
def test_same_type_devices_with_map(self): device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { # client, RJ45 10M 100M 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"}, 'mac_addr': {'eth0': "f0:de:f1:46:8c:22", 'eth1': "00:24:d7:9b:c0:8b"}, 'tags': {'eth0': [], 'eth1': ['RJ45', '10M', '100M']}, 'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} } device_dict.save() bbb2 = self.factory.make_device(self.factory.bbb_type, hostname='bbb2') device_dict = DeviceDictionary(hostname=bbb2.hostname) device_dict.parameters = { # server, RJ45 10M 100M 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"}, 'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"}, 'tags': {'eth0': [], 'eth1': ['RJ45', '10M', '100M']}, 'map': {'eth0': {'192.168.0.2': 7}, 'eth1': {'192.168.0.2': 9}} } device_dict.save() devices = list(Device.objects.filter(status=Device.IDLE).order_by('is_public')) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-bbb-vland-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device if device in devices: devices.remove(device) assign_jobs() self.factory.bbb1.refresh_from_db() bbb2.refresh_from_db() self.assertIsNotNone(self.factory.bbb1.current_job) self.assertIsNotNone(bbb2.current_job) self.assertIsNotNone(self.factory.bbb1.current_job.actual_device) self.assertIsNotNone(bbb2.current_job.actual_device) self.assertNotEqual(self.factory.bbb1.current_job, bbb2.current_job) self.assertNotEqual(self.factory.bbb1.current_job.actual_device, bbb2.current_job.actual_device)
def test_queueing(self): """ uses stderr to avoid buffered prints Expect the test itself to take <30s and the gap between jobs submitted and end being ~500ms Most of the time is spent setting up the database and submitting all the test jobs. """ print >> sys.stderr, timezone.now(), "start" user = self.factory.ensure_user('test', '*****@*****.**', 'test') user.user_permissions.add( Permission.objects.get(codename='add_testjob')) user.save() device_type = self.factory.make_device_type('beaglebone-black') device = self.factory.make_device(device_type=device_type, hostname="black01") device.save() device_type = self.factory.make_device_type('wandboard') count = 1 while count < 100: suffix = "{:02d}".format(count) device = self.factory.make_device(device_type=device_type, hostname="imx6q-%s" % suffix) device.save() count += 1 print >> sys.stderr, timezone.now(), "%d dummy devices created" % count device_list = list(get_available_devices()) print >> sys.stderr, timezone.now(), "%d available devices" % len(device_list) filename = os.path.join(os.path.dirname(__file__), 'master-check.json') self.assertTrue(os.path.exists(filename)) with open(filename, 'r') as json_file: definition = json_file.read() count = 0 # each 1000 more can take ~15s in the test. while count < 1000: # simulate API submission job = testjob_submission(definition, user) self.assertFalse(job.health_check) count += 1 print >> sys.stderr, timezone.now(), "%d jobs submitted" % count jobs = list(get_job_queue()) self.assertIsNotNone(jobs) print >> sys.stderr, timezone.now(), "Finding devices for jobs." for job in jobs: # this needs to stay as a tight loop to cope with load device = find_device_for_job(job, device_list) if device: print >> sys.stderr, timezone.now(), "[%d] allocated %s" % (job.id, device) device_list.remove(device) print >> sys.stderr, timezone.now(), "end"
def test_find_device_for_devices_without_tags(self): """ ensure that tags do not interfere with finding devices of unrelated types """ job = self.submit_job(device_type='arndale', tags=[]) devices = [self.panda01, self.arndale02, self.black01, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.arndale02, chosen_device) try: self.submit_job(device_type='arndale', tags=[ self.common_tag.name ]) except DevicesUnavailableException: pass else: self.fail("Offered an arndale when no arndale support the requested tags")
def test_find_device_for_devices_without_tags(self): """ ensure that tags do not interfere with finding devices of unrelated types """ job = self.submit_job(device_type='arndale', tags=[]) devices = [self.panda01, self.arndale02, self.black01, self.black03] chosen_device = find_device_for_job(job, devices) self.assertEqual(self.arndale02, chosen_device) try: self.submit_job(device_type='arndale', tags=[ self.common_tag.name ]) except DevicesUnavailableException: pass else: self.fail("Offered an arndale when no arndale support the requested tags")
def test_match_devices_with_map_and_tags(self): devices = Device.objects.filter(status=Device.IDLE).order_by('is_public') self.factory.ensure_tag('usb-eth') self.factory.ensure_tag('sata') self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth') self.factory.bbb1.save() self.factory.cubie1.tags = Tag.objects.filter(name='sata') self.factory.cubie1.save() device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"}, 'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"}, 'tags': {'eth0': [], 'eth1': ['RJ45', '10M']}, 'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} } device_dict.save() device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"}, 'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"}, 'tags': {'eth0': [], 'eth1': ['RJ45', '100M']}, 'map': {'eth0': {'192.168.0.2': 4}, 'eth1': {'192.168.0.2': 6}} } device_dict.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_find_nonexclusive_device(self): """ test that exclusive devices are not assigned JSON jobs """ self.assertFalse(self.panda01.is_exclusive) device_dict = DeviceDictionary.get(self.panda01.hostname) self.assertIsNone(device_dict) device_dict = DeviceDictionary(hostname=self.panda01.hostname) device_dict.parameters = {'exclusive': 'True'} device_dict.save() self.assertTrue(self.panda01.is_exclusive) self.assertRaises(DevicesUnavailableException, self.submit_job, target='panda01', device_type='panda') job = self.submit_job(device_type='panda') devices = [self.panda02, self.panda01] self.assertEqual(find_device_for_job(job, devices), self.panda02) device_dict.delete() self.assertFalse(self.panda01.is_exclusive)
def test_match_devices_without_map(self): devices = Device.objects.filter(status=Device.IDLE).order_by('is_public') self.factory.ensure_tag('usb-eth') self.factory.ensure_tag('sata') self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth') self.factory.bbb1.save() self.factory.cubie1.tags = Tag.objects.filter(name='sata') self.factory.cubie1.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # no map defined self.assertFalse(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_find_nonexclusive_device(self): """ test that exclusive devices are not assigned JSON jobs """ self.assertFalse(self.panda01.is_exclusive) device_dict = DeviceDictionary.get(self.panda01.hostname) self.assertIsNone(device_dict) device_dict = DeviceDictionary(hostname=self.panda01.hostname) device_dict.parameters = {'exclusive': 'True'} device_dict.save() self.assertTrue(self.panda01.is_exclusive) self.assertRaises( DevicesUnavailableException, self.submit_job, target='panda01', device_type='panda') job = self.submit_job(device_type='panda') devices = [self.panda02, self.panda01] self.assertEqual( find_device_for_job(job, devices), self.panda02 ) device_dict.delete() self.assertFalse(self.panda01.is_exclusive)
def test_match_devices_with_map_and_tags(self): devices = Device.objects.filter( status=Device.IDLE).order_by('is_public') self.factory.ensure_tag('usb-eth') self.factory.ensure_tag('sata') self.factory.bbb1.tags = Tag.objects.filter(name='usb-eth') self.factory.bbb1.save() self.factory.cubie1.tags = Tag.objects.filter(name='sata') self.factory.cubie1.save() device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } device_dict.save() device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname) device_dict.parameters = { 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '100M'] }, 'map': { 'eth0': { '192.168.0.2': 4 }, 'eth1': { '192.168.0.2': 6 } } } device_dict.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_same_type_devices_with_map(self): device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { # client, RJ45 10M 100M 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:22", 'eth1': "00:24:d7:9b:c0:8b" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } device_dict.save() bbb2 = self.factory.make_device(self.factory.bbb_type, hostname='bbb2') device_dict = DeviceDictionary(hostname=bbb2.hostname) device_dict.parameters = { # server, RJ45 10M 100M 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M'] }, 'map': { 'eth0': { '192.168.0.2': 7 }, 'eth1': { '192.168.0.2': 9 } } } device_dict.save() devices = list( Device.objects.filter(status=Device.IDLE).order_by('is_public')) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-bbb-vland-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device if device in devices: devices.remove(device) assign_jobs() self.factory.bbb1.refresh_from_db() bbb2.refresh_from_db() self.assertIsNotNone(self.factory.bbb1.current_job) self.assertIsNotNone(bbb2.current_job) self.assertIsNotNone(self.factory.bbb1.current_job.actual_device) self.assertIsNotNone(bbb2.current_job.actual_device) self.assertNotEqual(self.factory.bbb1.current_job, bbb2.current_job) self.assertNotEqual(self.factory.bbb1.current_job.actual_device, bbb2.current_job.actual_device)
def test_match_devices_with_map(self): devices = Device.objects.filter( status=Device.IDLE).order_by('is_public') device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { # client, RJ45 10M only 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 } } } device_dict.save() device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname) device_dict.parameters = { # server includes 100M 'extends': 'cubietruck.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M'] }, 'map': { 'eth0': { '192.168.0.2': 4 }, 'eth1': { '192.168.0.2': 6 } } } device_dict.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) del (data['protocols']['lava-multinode']['roles']['client']['tags']) del (data['protocols']['lava-multinode']['roles']['server']['tags']) interfaces = [] job_dict = split_multinode_yaml(data, 'abcdefg123456789') client_job = job_dict['client'][0] device_dict = DeviceDictionary.get( self.factory.bbb1.hostname).to_dict() self.assertIsNotNone(device_dict) tag_list = client_job['protocols']['lava-vland']['vlan_one']['tags'] for interface, tags in device_dict['parameters']['tags'].iteritems(): if set(tags) & set(tag_list) == set( tag_list) and interface not in interfaces: interfaces.append(interface) break self.assertEqual(['eth1'], interfaces) self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys())) interfaces = [] server_job = job_dict['server'][0] device_dict = DeviceDictionary.get( self.factory.cubie1.hostname).to_dict() self.assertIsNotNone(device_dict) tag_list = server_job['protocols']['lava-vland']['vlan_two']['tags'] for interface, tags in device_dict['parameters']['tags'].iteritems(): if set(tags) & set(tag_list) == set( tag_list) and interface not in interfaces: interfaces.append(interface) break self.assertEqual(['eth1'], interfaces) self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys())) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) # vlan_one: client role. RJ45 10M. bbb device type # vlan_two: server role. RJ45 100M. cubie device type. assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)
def test_differing_vlan_tags(self): """ More devices of the requested type than needed by the test with some devices having unsuitable vland interface tags. """ x86 = self.factory.make_device_type('x86') x86_1 = self.factory.make_device(device_type=x86, hostname='x86-01') x86_2 = self.factory.make_device(device_type=x86, hostname='x86-02') x86_3 = self.factory.make_device(device_type=x86, hostname='x86-03') x86_4 = self.factory.make_device(device_type=x86, hostname='x86-04') devices = list(Device.objects.filter(status=Device.IDLE).order_by('is_public')) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'x86-vlan.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertIsNotNone(device) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device if device in devices: devices.remove(device) assign_jobs() # reset state, pretend the assigned jobs have completed. for job in TestJob.objects.all(): job.status = TestJob.COMPLETE job.actual_device.status = Device.IDLE job.actual_device.current_job = None job.actual_device.save(update_fields=['status', 'current_job']) job.save(update_fields=['status']) # take x86_1 offline, forcing the idle list to include x86_3 for evaluation x86_1.status = Device.OFFLINE x86_1.save(update_fields=['status']) x86_1.refresh_from_db() devices = list(Device.objects.filter(status=Device.IDLE).order_by('is_public')) self.assertNotIn(x86_1, devices) self.assertIn(x86_2, devices) self.assertIn(x86_3, devices) self.assertIn(x86_4, devices) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'x86-vlan.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertIsNotNone(device) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device if device in devices: devices.remove(device) assign_jobs() x86_1.refresh_from_db() x86_2.refresh_from_db() x86_3.refresh_from_db() x86_4.refresh_from_db() self.assertEqual(Device.STATUS_CHOICES[Device.OFFLINE], Device.STATUS_CHOICES[x86_1.status]) self.assertEqual(Device.STATUS_CHOICES[Device.RESERVED], Device.STATUS_CHOICES[x86_2.status]) self.assertEqual(Device.STATUS_CHOICES[Device.IDLE], Device.STATUS_CHOICES[x86_3.status]) self.assertEqual(Device.STATUS_CHOICES[Device.RESERVED], Device.STATUS_CHOICES[x86_4.status])
def test_differing_vlan_tags(self): """ More devices of the requested type than needed by the test with some devices having unsuitable vland interface tags. """ x86 = self.factory.make_device_type('x86') x86_1 = self.factory.make_device(device_type=x86, hostname='x86-1') device_dict = DeviceDictionary(hostname=x86_1.hostname) device_dict.parameters = { # client, RJ45 10M 100M with separate 10G 'extends': 'x86.jinja2', 'interfaces': ['eth0', 'eth1', 'eth2'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1", 'eth2': "/sys/devices/pci0000:00/0000:00:1c.2/0000:04:00.0/net/eth2" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:22", 'eth1': "00:24:d7:9b:c0:8b", 'eth2': "00:24:d7:9b:c0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M', '1G'], 'eth2': ['SFP+', '10G'] }, 'map': { 'eth0': { '192.168.0.2': 5 }, 'eth1': { '192.168.0.2': 7 }, 'eth2': { '192.168.0.2': 12 } } } device_dict.save() x86_2 = self.factory.make_device(device_type=x86, hostname='x86-2') device_dict = DeviceDictionary(hostname=x86_2.hostname) device_dict.parameters = { # client, RJ45 10M 100M with separate 10G 'extends': 'x86.jinja2', 'interfaces': ['eth0', 'eth1', 'eth2'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1", 'eth2': "/sys/devices/pci0000:00/0000:00:1c.2/0000:04:00.0/net/eth2" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:22", 'eth1': "00:24:d7:9b:d0:8b", 'eth2': "00:24:d7:9b:d0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M', '1G'], 'eth2': ['SFP+', '10G'] }, 'map': { 'eth0': { '192.168.0.2': 14 }, 'eth1': { '192.168.0.2': 17 }, 'eth2': { '192.168.0.2': 22 } } } device_dict.save() x86_3 = self.factory.make_device(device_type=x86, hostname='x86-3') device_dict = DeviceDictionary(hostname=x86_3.hostname) device_dict.parameters = { # client, 40G, not 10G 'extends': 'x86.jinja2', 'interfaces': ['eth0', 'eth1', 'eth2'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1", 'eth2': "/sys/devices/pci0000:00/0000:00:1c.2/0000:04:00.0/net/eth2" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:22", 'eth1': "00:24:d7:9b:d0:8b", 'eth2': "00:24:d7:9b:d0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M', '1G'], 'eth2': ['QSFP+', '40G'] }, 'map': { 'eth0': { '192.168.0.2': 15 }, 'eth1': { '192.168.0.2': 16 }, 'eth2': { '192.168.0.2': 20 } } } device_dict.save() x86_4 = self.factory.make_device(device_type=x86, hostname='x86-4') device_dict = DeviceDictionary(hostname=x86_4.hostname) device_dict.parameters = { # client, RJ45 10M 100M with separate 10G 'extends': 'x86.jinja2', 'interfaces': ['eth0', 'eth1', 'eth2'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1", 'eth2': "/sys/devices/pci0000:00/0000:00:1c.2/0000:04:00.0/net/eth2" }, 'mac_addr': { 'eth0': "f0:de:f1:46:8c:22", 'eth1': "00:24:d7:9b:d0:8b", 'eth2': "00:24:d7:9b:d0:8c" }, 'tags': { 'eth0': [], 'eth1': ['RJ45', '10M', '100M', '1G'], 'eth2': ['SFP+', '10G'] }, 'map': { 'eth0': { '192.168.0.2': 14 }, 'eth1': { '192.168.0.2': 17 }, 'eth2': { '192.168.0.2': 22 } } } device_dict.save() devices = list( Device.objects.filter(status=Device.IDLE).order_by('is_public')) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'x86-vlan.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertIsNotNone(device) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device if device in devices: devices.remove(device) assign_jobs() # reset state, pretend the assigned jobs have completed. for job in TestJob.objects.all(): job.status = TestJob.COMPLETE job.actual_device.status = Device.IDLE job.actual_device.current_job = None job.actual_device.save(update_fields=['status', 'current_job']) job.save(update_fields=['status']) # take x86_1 offline, forcing the idle list to include x86_3 for evaluation x86_1.status = Device.OFFLINE x86_1.save(update_fields=['status']) x86_1.refresh_from_db() devices = list( Device.objects.filter(status=Device.IDLE).order_by('is_public')) self.assertNotIn(x86_1, devices) self.assertIn(x86_2, devices) self.assertIn(x86_3, devices) self.assertIn(x86_4, devices) user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'x86-vlan.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertIsNotNone(device) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue( match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device if device in devices: devices.remove(device) assign_jobs() x86_1.refresh_from_db() x86_2.refresh_from_db() x86_3.refresh_from_db() x86_4.refresh_from_db() self.assertEqual(Device.STATUS_CHOICES[Device.OFFLINE], Device.STATUS_CHOICES[x86_1.status]) self.assertEqual(Device.STATUS_CHOICES[Device.RESERVED], Device.STATUS_CHOICES[x86_2.status]) self.assertEqual(Device.STATUS_CHOICES[Device.IDLE], Device.STATUS_CHOICES[x86_3.status]) self.assertEqual(Device.STATUS_CHOICES[Device.RESERVED], Device.STATUS_CHOICES[x86_4.status])
def test_match_devices_with_map(self): devices = Device.objects.filter(status=Device.IDLE).order_by('is_public') device_dict = DeviceDictionary(hostname=self.factory.bbb1.hostname) device_dict.parameters = { # client, RJ45 10M only 'extends': 'beaglebone-black.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"}, 'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"}, 'tags': {'eth0': [], 'eth1': ['RJ45', '10M']}, 'map': {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} } device_dict.save() device_dict = DeviceDictionary(hostname=self.factory.cubie1.hostname) device_dict.parameters = { # server includes 100M 'extends': 'cubietruck.jinja2', 'interfaces': ['eth0', 'eth1'], 'sysfs': { 'eth0': "/sys/devices/pci0000:00/0000:00:19.0/net/eth0", 'eth1': "/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1"}, 'mac_addr': {'eth0': "f0:de:f1:46:8c:21", 'eth1': "00:24:d7:9b:c0:8c"}, 'tags': {'eth0': [], 'eth1': ['RJ45', '10M', '100M']}, 'map': {'eth0': {'192.168.0.2': 4}, 'eth1': {'192.168.0.2': 6}} } device_dict.save() user = self.factory.make_user() sample_job_file = os.path.join(os.path.dirname(__file__), 'bbb-cubie-vlan-group.yaml') with open(sample_job_file, 'r') as test_support: data = yaml.load(test_support) del(data['protocols']['lava-multinode']['roles']['client']['tags']) del(data['protocols']['lava-multinode']['roles']['server']['tags']) interfaces = [] job_dict = split_multinode_yaml(data, 'abcdefg123456789') client_job = job_dict['client'][0] device_dict = DeviceDictionary.get(self.factory.bbb1.hostname).to_dict() self.assertIsNotNone(device_dict) tag_list = client_job['protocols']['lava-vland']['vlan_one']['tags'] for interface, tags in device_dict['parameters']['tags'].iteritems(): if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces: interfaces.append(interface) break self.assertEqual(['eth1'], interfaces) self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys())) interfaces = [] server_job = job_dict['server'][0] device_dict = DeviceDictionary.get(self.factory.cubie1.hostname).to_dict() self.assertIsNotNone(device_dict) tag_list = server_job['protocols']['lava-vland']['vlan_two']['tags'] for interface, tags in device_dict['parameters']['tags'].iteritems(): if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces: interfaces.append(interface) break self.assertEqual(['eth1'], interfaces) self.assertEqual(len(interfaces), len(client_job['protocols']['lava-vland'].keys())) vlan_job = TestJob.from_yaml_and_user(yaml.dump(data), user) # vlan_one: client role. RJ45 10M. bbb device type # vlan_two: server role. RJ45 100M. cubie device type. assignments = {} for job in vlan_job: device = find_device_for_job(job, devices) self.assertEqual(device.device_type, job.requested_device_type) # map has been defined self.assertTrue(match_vlan_interface(device, yaml.load(job.definition))) assignments[job.device_role] = device self.assertEqual(assignments['client'].hostname, self.factory.bbb1.hostname) self.assertEqual(assignments['server'].hostname, self.factory.cubie1.hostname)