def test_multinode_definition(self): user = self.factory.make_user() self.device_type = self.factory.make_device_type() submission = yaml.load( open(os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) self.factory.make_device(self.device_type, 'fakeqemu1') self.factory.make_device(self.device_type, 'fakeqemu2') tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(self.device_type, 'fakeqemu3', tags=tag_list) job_object_list = _pipeline_protocols(submission, user, None) for job in job_object_list: self.assertIsNotNone(job.multinode_definition) self.assertNotIn('#', job.multinode_definition) with open( os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r') as source: yaml_str = source.read() self.assertIn('# unit test support comment', yaml_str) job_object_list = _pipeline_protocols(submission, user, yaml_str) for job in job_object_list: self.assertIsNotNone(job.multinode_definition) self.assertIn('# unit test support comment', job.multinode_definition)
def test_multinode_protocols(self): user = self.factory.make_user() self.device_type = self.factory.make_device_type() submission = yaml.load( open(os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) # no devices defined for the specified type self.assertRaises(DevicesUnavailableException, _pipeline_protocols, submission, user, yaml_data=None) self.factory.make_device(self.device_type, 'fakeqemu1') # specified tags do not exist self.assertRaises(yaml.YAMLError, _pipeline_protocols, submission, user) tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(self.device_type, 'fakeqemu2') # no devices which have the required tags applied self.assertRaises(DevicesUnavailableException, _pipeline_protocols, submission, user, yaml_data=None) self.factory.make_device(self.device_type, 'fakeqemu3', tags=tag_list) job_object_list = _pipeline_protocols(submission, user) self.assertEqual(len(job_object_list), 2) for job in job_object_list: self.assertEqual(list(job.sub_jobs_list), job_object_list) check = yaml.load(job.definition) if check['protocols']['lava-multinode']['role'] == 'client': self.assertEqual(check['protocols']['lava-multinode']['tags'], ['usb-flash', 'usb-eth']) self.assertEqual( check['protocols']['lava-multinode']['interfaces'], [{ 'vlan': 'name_two', 'tags': ['10G'] }, { 'vlan': 'name_two', 'tags': ['1G'] }]) self.assertEqual(set(tag_list), set(job.tags.all())) if check['protocols']['lava-multinode']['role'] == 'server': self.assertNotIn('tags', check['protocols']['lava-multinode']) self.assertNotIn('interfaces', check['protocols']['lava-multinode']) self.assertEqual(set([]), set(job.tags.all()))
def test_multinode_definition(self): user = self.factory.make_user() device_type = self.factory.make_device_type() submission = yaml.load(open( os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) self.factory.make_device(device_type, 'fakeqemu1') self.factory.make_device(device_type, 'fakeqemu2') tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(device_type, 'fakeqemu3', tags=tag_list) job_object_list = _pipeline_protocols(submission, user, None) for job in job_object_list: self.assertIsNotNone(job.multinode_definition) self.assertNotIn('#', job.multinode_definition) with open(os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r') as source: yaml_str = source.read() self.assertIn('# unit test support comment', yaml_str) job_object_list = _pipeline_protocols(submission, user, yaml_str) for job in job_object_list: self.assertIsNotNone(job.multinode_definition) self.assertIn('# unit test support comment', job.multinode_definition)
def test_multinode_protocols(self): user = self.factory.make_user() device_type = self.factory.make_device_type() Device.objects.filter(device_type=device_type).delete() Tag.objects.all().delete() submission = yaml.load(open( os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) # no devices defined for the specified type self.assertRaises(DevicesUnavailableException, _pipeline_protocols, submission, user, yaml_data=None) self.factory.make_device(device_type, 'fakeqemu1') # specified tags do not exist self.assertRaises(yaml.YAMLError, _pipeline_protocols, submission, user) tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(device_type, 'fakeqemu2') # no devices which have the required tags applied self.assertRaises(DevicesUnavailableException, _pipeline_protocols, submission, user, yaml_data=None) self.factory.make_device(device_type, 'fakeqemu3', tags=tag_list) job_object_list = _pipeline_protocols(submission, user) self.assertEqual(len(job_object_list), 2) for job in job_object_list: self.assertEqual(list(job.sub_jobs_list), job_object_list) check = yaml.load(job.definition) if check['protocols']['lava-multinode']['role'] == 'client': self.assertEqual( check['protocols']['lava-multinode']['tags'], ['usb-flash', 'usb-eth']) self.assertEqual( check['protocols']['lava-multinode']['interfaces'], [{'vlan': 'name_two', 'tags': ['10G']}, {'vlan': 'name_two', 'tags': ['1G']}] ) self.assertEqual(set(tag_list), set(job.tags.all())) if check['protocols']['lava-multinode']['role'] == 'server': self.assertNotIn('tags', check['protocols']['lava-multinode']) self.assertNotIn('interfaces', check['protocols']['lava-multinode']) self.assertEqual(set([]), set(job.tags.all()))
def test_multinode_group(self): user = self.factory.make_user() device_type = self.factory.make_device_type() submission = yaml.load(open( os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) self.factory.make_device(device_type, 'fakeqemu1') tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(device_type, 'fakeqemu2', tags=tag_list) job_object_list = _pipeline_protocols(submission, user, yaml.dump(submission)) for job in job_object_list: self.assertEqual(list(job.sub_jobs_list), job_object_list) check_one = yaml.load(job_object_list[0].definition) check_two = yaml.load(job_object_list[1].definition) self.assertEqual( job_object_list[0].target_group, job_object_list[1].target_group ) # job 0 needs to have a sub_id of <id>.0 # job 1 needs to have a sub_id of <id>.1 despite job 1 id being <id>+1 self.assertEqual(int(job_object_list[0].id), int(float(job_object_list[0].sub_id))) self.assertEqual(int(job_object_list[0].id) + 1, int(job_object_list[1].id)) self.assertEqual( job_object_list[0].sub_id, "%d.%d" % (int(job_object_list[0].id), 0)) self.assertEqual( job_object_list[1].sub_id, "%d.%d" % (int(job_object_list[0].id), 1)) self.assertNotEqual( job_object_list[1].sub_id, "%d.%d" % (int(job_object_list[1].id), 0)) self.assertNotEqual( job_object_list[1].sub_id, "%d.%d" % (int(job_object_list[1].id), 1)) self.assertNotEqual( check_one['protocols']['lava-multinode']['role'], check_two['protocols']['lava-multinode']['role'] )
def test_multinode_group(self): user = self.factory.make_user() self.device_type = self.factory.make_device_type() submission = yaml.load( open(os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) self.factory.make_device(self.device_type, 'fakeqemu1') tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(self.device_type, 'fakeqemu2', tags=tag_list) job_object_list = _pipeline_protocols(submission, user, yaml.dump(submission)) for job in job_object_list: self.assertEqual(list(job.sub_jobs_list), job_object_list) check_one = yaml.load(job_object_list[0].definition) check_two = yaml.load(job_object_list[1].definition) self.assertEqual(job_object_list[0].target_group, job_object_list[1].target_group) # job 0 needs to have a sub_id of <id>.0 # job 1 needs to have a sub_id of <id>.1 despite job 1 id being <id>+1 self.assertEqual(int(job_object_list[0].id), int(float(job_object_list[0].sub_id))) self.assertEqual( int(job_object_list[0].id) + 1, int(job_object_list[1].id)) self.assertEqual(job_object_list[0].sub_id, "%d.%d" % (int(job_object_list[0].id), 0)) self.assertEqual(job_object_list[1].sub_id, "%d.%d" % (int(job_object_list[0].id), 1)) self.assertNotEqual(job_object_list[1].sub_id, "%d.%d" % (int(job_object_list[1].id), 0)) self.assertNotEqual(job_object_list[1].sub_id, "%d.%d" % (int(job_object_list[1].id), 1)) self.assertNotEqual(check_one['protocols']['lava-multinode']['role'], check_two['protocols']['lava-multinode']['role'])
def test_invalid_multinode(self): # pylint: disable=too-many-locals user = self.factory.make_user() device_type = self.factory.make_device_type() submission = yaml.load(open( os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(device_type, 'fakeqemu1') self.factory.make_device(device_type, 'fakeqemu2') self.factory.make_device(device_type, 'fakeqemu3', tags=tag_list) deploy = [action['deploy'] for action in submission['actions'] if 'deploy' in action] # replace working image with a broken URL for block in deploy: block['images'] = { 'rootfs': { 'url': 'http://localhost/unknown/invalid.gz', 'image_arg': '{rootfs}' } } job_object_list = _pipeline_protocols(submission, user, yaml.dump(submission)) self.assertEqual(len(job_object_list), 2) self.assertEqual( job_object_list[0].sub_id, "%d.%d" % (int(job_object_list[0].id), 0)) # FIXME: dispatcher master needs to make this kind of test more accessible. for job in job_object_list: definition = yaml.load(job.definition) self.assertNotEqual(definition['protocols']['lava-multinode']['sub_id'], '') job.actual_device = Device.objects.get(hostname='fakeqemu1') job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) parser = JobParser() device = None device_object = None if not job.dynamic_connection: device = job.actual_device try: device_config = device.load_device_configuration(job_ctx, system=False) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: # FIXME: report the exceptions as useful user messages self.fail("[%d] jinja2 error: %s" % (job.id, exc)) if not device_config or not isinstance(device_config, dict): # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname self.fail('[%d] device-dictionary error: %s' % (job.id, msg)) device_object = PipelineDevice(device_config, device.hostname) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname validate_list = job.sub_jobs_list if job.is_multinode else [job] for check_job in validate_list: parser_device = None if job.dynamic_connection else device_object try: # pass (unused) output_dir just for validation as there is no zmq socket either. pipeline_job = parser.parse( check_job.definition, parser_device, check_job.id, None, None, None, output_dir=check_job.output_dir) except (AttributeError, JobError, NotImplementedError, KeyError, TypeError) as exc: self.fail('[%s] parser error: %s' % (check_job.sub_id, exc)) with TestCase.assertRaises(self, (JobError, InfrastructureError)) as check: pipeline_job.pipeline.validate_actions() check_missing_path(self, check, 'qemu-system-x86_64') for job in job_object_list: job = TestJob.objects.get(id=job.id) self.assertNotEqual(job.sub_id, '')
def test_invalid_multinode(self): user = self.factory.make_user() self.device_type = self.factory.make_device_type() submission = yaml.load( open(os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(self.device_type, 'fakeqemu1') self.factory.make_device(self.device_type, 'fakeqemu2') self.factory.make_device(self.device_type, 'fakeqemu3', tags=tag_list) deploy = [ action['deploy'] for action in submission['actions'] if 'deploy' in action ] # replace working image with a broken URL for block in deploy: block['image'] = 'http://localhost/unknown/invalid.gz' job_object_list = _pipeline_protocols(submission, user, yaml.dump(submission)) self.assertEqual(len(job_object_list), 2) self.assertEqual(job_object_list[0].sub_id, "%d.%d" % (int(job_object_list[0].id), 0)) # FIXME: dispatcher master needs to make this kind of test more accessible. for job in job_object_list: definition = yaml.load(job.definition) self.assertNotEqual( definition['protocols']['lava-multinode']['sub_id'], '') job.actual_device = Device.objects.get(hostname='fakeqemu1') job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) parser = JobParser() device = None device_object = None if not job.dynamic_connection: device = job.actual_device try: device_config = device.load_device_configuration( job_ctx) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: # FIXME: report the exceptions as useful user messages self.fail("[%d] jinja2 error: %s" % (job.id, exc)) if not device_config or type(device_config) is not dict: # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname self.fail('[%d] device-dictionary error: %s' % (job.id, msg)) device_object = PipelineDevice( device_config, device.hostname ) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname validate_list = job.sub_jobs_list if job.is_multinode else [job] for check_job in validate_list: parser_device = None if job.dynamic_connection else device_object try: # pass (unused) output_dir just for validation as there is no zmq socket either. pipeline_job = parser.parse( check_job.definition, parser_device, check_job.id, None, output_dir=check_job.output_dir) except (AttributeError, JobError, NotImplementedError, KeyError, TypeError) as exc: self.fail('[%s] parser error: %s' % (check_job.sub_id, exc)) if os.path.exists( '/dev/loop0' ): # rather than skipping the entire test, just the validation. self.assertRaises(JobError, pipeline_job.pipeline.validate_actions) for job in job_object_list: job = TestJob.objects.get(id=job.id) self.assertNotEqual(job.sub_id, '')