def test_compatibility(self): # pylint: disable=too-many-locals user = self.factory.make_user() # public set in the YAML yaml_str = self.factory.make_job_json() yaml_data = yaml.load(yaml_str) job = TestJob.from_yaml_and_user( yaml_str, user) self.assertTrue(job.is_public) self.assertTrue(job.can_view(user)) # initial state prior to validation self.assertEqual(job.pipeline_compatibility, 0) self.assertNotIn('compatibility', yaml_data) # FIXME: dispatcher master needs to make this kind of test more accessible. definition = yaml.load(job.definition) self.assertNotIn('protocols', definition) job.actual_device = Device.objects.get(hostname='fakeqemu1') job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) parser = JobParser() device = job.actual_device try: device_config = device.load_device_configuration(job_ctx, system=False) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: # FIXME: report the exceptions as useful user messages self.fail("[%d] jinja2 error: %s" % (job.id, exc)) if not device_config or not isinstance(device_config, dict): # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname self.fail('[%d] device-dictionary error: %s' % (job.id, msg)) device_object = PipelineDevice(device_config, device.hostname) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname parser_device = device_object try: # pass (unused) output_dir just for validation as there is no zmq socket either. pipeline_job = parser.parse( job.definition, parser_device, job.id, None, None, None, output_dir=job.output_dir) except (AttributeError, JobError, NotImplementedError, KeyError, TypeError) as exc: self.fail('[%s] parser error: %s' % (job.sub_id, exc)) description = pipeline_job.describe() self.assertIn('compatibility', description) self.assertGreaterEqual(description['compatibility'], BootQEMU.compatibility)
def test_pipeline_device(self): foo = DeviceDictionary(hostname='foo') foo.parameters = { 'bootz': { 'kernel': '0x4700000', 'ramdisk': '0x4800000', 'dtb': '0x4300000' }, 'media': { 'usb': { 'UUID-required': True, 'SanDisk_Ultra': { 'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0', 'device_id': 0 }, 'sata': { 'UUID-required': False } } } } device = PipelineDevice(foo.parameters, 'foo') self.assertEqual(device.target, 'foo') self.assertIn('power_state', device) self.assertEqual(device.power_state, '') # there is no power_on_command for this device, so the property is '' self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_job_multi(self): MetaType.objects.all().delete() multi_test_file = os.path.join(os.path.dirname(__file__), 'multi-test.yaml') self.assertTrue(os.path.exists(multi_test_file)) with open(multi_test_file, 'r') as test_support: data = test_support.read() job = TestJob.from_yaml_and_user(data, self.user) job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) device = Device.objects.get(hostname='fakeqemu1') device_config = device.load_device_configuration( job_ctx, system=False) # raw dict parser = JobParser() obj = PipelineDevice(device_config, device.hostname) pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp') allow_missing_path(pipeline_job.pipeline.validate_actions, self, 'qemu-system-x86_64') pipeline = pipeline_job.describe() map_metadata(yaml.dump(pipeline), job)
def test_jinja_string_templates(self): jinja2_path = jinja_template_path(system=False) self.assertTrue(os.path.exists(jinja2_path)) device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002', 'console_device': 'ttyfake1', 'baud_rate': 56 } data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.jinja2') template = prepare_jinja_template('cubie', data, system_path=False, path=jinja2_path) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot']['ramdisk'] self.assertIn('commands', ramdisk_args) self.assertIn('boot', ramdisk_args['commands']) self.assertIn( "setenv bootargs 'console=ttyfake1,56n8 root=/dev/ram0 ip=dhcp'", ramdisk_args['commands']) device_dictionary.update( { 'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08", 'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08", 'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08" } ) data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.jinja2') template = prepare_jinja_template('bbb', data, system_path=False, path=jinja2_path) device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) device = PipelineDevice(yaml_data, 'bbb') self.assertIn('power_state', device) # bbb has power_on_command defined above self.assertEqual(device.power_state, 'off') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_repositories(self): # pylint: disable=too-many-locals job = TestJob.from_yaml_and_user(self.factory.make_job_yaml(), self.user) job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) job_ctx.update( {'no_kvm': True}) # override to allow unit tests on all types of systems device = Device.objects.get(hostname='fakeqemu1') device_config = device.load_device_configuration( job_ctx, system=False) # raw dict parser = JobParser() obj = PipelineDevice(device_config, device.hostname) pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp') allow_missing_path(pipeline_job.pipeline.validate_actions, self, 'qemu-system-x86_64') pipeline = pipeline_job.describe() device_values = _get_device_metadata(pipeline['device']) self.assertEqual(device_values, {'target.device_type': 'qemu'}) del pipeline['device']['device_type'] self.assertNotIn('device_type', pipeline['device']) device_values = _get_device_metadata(pipeline['device']) try: testdata, _ = TestData.objects.get_or_create(testjob=job) except (MultipleObjectsReturned): self.fail('multiple objects') for key, value in device_values.items(): if not key or not value: continue testdata.attributes.create(name=key, value=value) retval = _get_job_metadata(pipeline['job']['actions']) if 'lava-server-version' in retval: del retval['lava-server-version'] self.assertEqual( retval, { 'test.1.common.definition.from': 'git', 'test.0.common.definition.repository': 'git://git.linaro.org/qa/test-definitions.git', 'test.0.common.definition.name': 'smoke-tests', 'test.1.common.definition.repository': 'http://git.linaro.org/lava-team/lava-functional-tests.git', 'boot.0.common.method': 'qemu', 'test.1.common.definition.name': 'singlenode-advanced', 'test.0.common.definition.from': 'git', 'test.0.common.definition.path': 'ubuntu/smoke-tests-basic.yaml', 'test.1.common.definition.path': 'lava-test-shell/single-node/singlenode03.yaml' })
def test_jinja_postgres_loader(self): # path used for the device_type template jinja2_path = jinja_template_path(system=False) self.assertTrue(os.path.exists(jinja2_path)) device_type = 'cubietruck' # pretend this was already imported into the database and use for comparison later. device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002' } # create a DeviceDictionary for this test cubie = DeviceDictionary(hostname='cubie') cubie.parameters = device_dictionary cubie.save() jinja_data = devicedictionary_to_jinja2(cubie.parameters, '%s.jinja2' % device_type) dict_loader = jinja2.DictLoader({'cubie.jinja2': jinja_data}) type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment( loader=jinja2.ChoiceLoader([dict_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.jinja2" % 'cubie') # pylint gets this wrong from jinja device_configuration = template.render() # pylint: disable=no-member chk_template = prepare_jinja_template('cubie', jinja_data, system_path=False, path=jinja2_path) self.assertEqual(template.render(), chk_template.render()) # pylint: disable=no-member yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid'] ) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual( device_dictionary['connection_command'], yaml_data['commands']['connect']) device = PipelineDevice(yaml_data, 'cubie') self.assertIn('power_state', device) # cubie1 has no power_on_command defined self.assertEqual(device.power_state, '') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_job(self): MetaType.objects.all().delete() TestJob.objects.all().delete() job = TestJob.from_yaml_and_user(self.factory.make_job_yaml(), self.user) job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) job_ctx.update( {'no_kvm': True}) # override to allow unit tests on all types of systems device = Device.objects.get(hostname='fakeqemu1') device_config = device.load_device_configuration( job_ctx, system=False) # raw dict parser = JobParser() obj = PipelineDevice(device_config, device.hostname) pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp') allow_missing_path(pipeline_job.pipeline.validate_actions, self, 'qemu-system-x86_64') pipeline = pipeline_job.describe() map_metadata(yaml.dump(pipeline), job) self.assertEqual( MetaType.objects.filter(metatype=MetaType.DEPLOY_TYPE).count(), 1) self.assertEqual( MetaType.objects.filter(metatype=MetaType.BOOT_TYPE).count(), 1) count = ActionData.objects.all().count() self.assertEqual(TestData.objects.all().count(), 1) testdata = TestData.objects.all()[0] self.assertEqual(testdata.testjob, job) for actionlevel in ActionData.objects.all(): self.assertEqual(actionlevel.testdata, testdata) action_levels = [] for testdata in job.testdata_set.all(): action_levels.extend(testdata.actionlevels.all()) self.assertEqual(count, len(action_levels)) count = ActionData.objects.filter( meta_type__metatype=MetaType.DEPLOY_TYPE).count() self.assertNotEqual( ActionData.objects.filter( meta_type__metatype=MetaType.BOOT_TYPE).count(), 0) self.assertEqual( ActionData.objects.filter( meta_type__metatype=MetaType.UNKNOWN_TYPE).count(), 0) for actionlevel in ActionData.objects.filter( meta_type__metatype=MetaType.BOOT_TYPE): self.assertEqual(actionlevel.testdata.testjob.id, job.id) self.assertEqual( ActionData.objects.filter(meta_type__metatype=MetaType.DEPLOY_TYPE, testdata__testjob=job).count(), count)
def test_parameter_support(self): # pylint: disable=too-many-locals data = self.factory.make_job_data() test_block = [block for block in data['actions'] if 'test' in block][0] smoke = test_block['test']['definitions'][0] smoke['parameters'] = { 'VARIABLE_NAME_1': "first variable value", 'VARIABLE_NAME_2': "second value" } job = TestJob.from_yaml_and_user(yaml.dump(data), self.user) job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) job_ctx.update( {'no_kvm': True}) # override to allow unit tests on all types of systems device = Device.objects.get(hostname='fakeqemu1') device_config = device.load_device_configuration( job_ctx, system=False) # raw dict parser = JobParser() obj = PipelineDevice(device_config, device.hostname) pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp') allow_missing_path(pipeline_job.pipeline.validate_actions, self, 'qemu-system-x86_64') pipeline = pipeline_job.describe() device_values = _get_device_metadata(pipeline['device']) try: testdata, _ = TestData.objects.get_or_create(testjob=job) except (MultipleObjectsReturned): self.fail('multiple objects') for key, value in device_values.items(): if not key or not value: continue testdata.attributes.create(name=key, value=value) retval = _get_job_metadata(pipeline['job']['actions']) self.assertIn('test.0.common.definition.parameters.VARIABLE_NAME_2', retval) self.assertIn('test.0.common.definition.parameters.VARIABLE_NAME_1', retval) self.assertEqual( retval['test.0.common.definition.parameters.VARIABLE_NAME_1'], 'first variable value') self.assertEqual( retval['test.0.common.definition.parameters.VARIABLE_NAME_2'], 'second value')
def test_inline(self): """ Test inline can be parsed without run steps """ data = self.factory.make_job_data() test_block = [block for block in data['actions'] if 'test' in block][0] smoke = [{ "path": "inline/smoke-tests-basic.yaml", "from": "inline", "name": "smoke-tests-inline", "repository": { "install": { "steps": [ "apt", ] }, "metadata": { "description": "Basic system test command for Linaro Ubuntu images", "format": "Lava-Test Test Definition 1.0", "name": "smoke-tests-basic" } } }] test_block['test']['definitions'] = smoke job = TestJob.from_yaml_and_user(yaml.dump(data), self.user) job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) job_ctx.update( {'no_kvm': True}) # override to allow unit tests on all types of systems device = Device.objects.get(hostname='fakeqemu1') device_config = device.load_device_configuration( job_ctx, system=False) # raw dict parser = JobParser() obj = PipelineDevice(device_config, device.hostname) pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp') allow_missing_path(pipeline_job.pipeline.validate_actions, self, 'qemu-system-x86_64') pipeline = pipeline_job.describe() map_metadata(yaml.dump(pipeline), job)
def test_job(self): user = self.factory.make_user() job = TestJob.from_yaml_and_user(self.factory.make_job_yaml(), user) job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) device = Device.objects.get(hostname='fakeqemu1') device_config = device.load_device_configuration(job_ctx) # raw dict parser = JobParser() obj = PipelineDevice(device_config, device.hostname) pipeline_job = parser.parse(job.definition, obj, job.id, None, output_dir='/tmp') pipeline_job.pipeline.validate_actions() pipeline = pipeline_job.describe() map_metadata(yaml.dump(pipeline), job) self.assertEqual( MetaType.objects.filter(metatype=MetaType.DEPLOY_TYPE).count(), 1) self.assertEqual( MetaType.objects.filter(metatype=MetaType.BOOT_TYPE).count(), 1) count = ActionData.objects.all().count() self.assertEqual(TestData.objects.all().count(), 1) testdata = TestData.objects.all()[0] self.assertEqual(testdata.testjob, job) for actionlevel in ActionData.objects.all(): self.assertEqual(actionlevel.testdata, testdata) action_levels = [] for testdata in job.test_data.all(): action_levels.extend(testdata.actionlevels.all()) self.assertEqual(count, len(action_levels)) count = ActionData.objects.filter( meta_type__metatype=MetaType.DEPLOY_TYPE).count() self.assertNotEqual( ActionData.objects.filter( meta_type__metatype=MetaType.BOOT_TYPE).count(), 0) self.assertEqual( ActionData.objects.filter( meta_type__metatype=MetaType.UNKNOWN_TYPE).count(), 0) for actionlevel in ActionData.objects.filter( meta_type__metatype=MetaType.BOOT_TYPE): self.assertEqual(actionlevel.testdata.testjob.id, job.id) self.assertEqual( ActionData.objects.filter(meta_type__metatype=MetaType.DEPLOY_TYPE, testdata__testjob=job).count(), count)
def test_invalid_device(self): user = self.factory.make_user() job = TestJob.from_yaml_and_user(self.factory.make_job_json(), user) job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) device = Device.objects.get(hostname='fakeqemu1') device_config = device.load_device_configuration(job_ctx) # raw dict del device_config['device_type'] parser = JobParser() obj = PipelineDevice( device_config, device.hostname ) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. self.assertRaises(KeyError, parser.parse, job.definition, obj, job.id, None, output_dir='/tmp')
def select_device(job): """ Transitioning a device from Idle to Reserved is the responsibility of the scheduler_daemon (currently). This function just checks that the reserved device is valid for this job. Jobs will only enter this function if a device is already reserved for that job. Stores the pipeline description To prevent cycling between lava_scheduler_daemon:assign_jobs and here, if a job fails validation, the job is incomplete. Issues with this need to be fixed using device tags. """ logger = logging.getLogger('dispatcher-master') if not job.dynamic_connection: if not job.actual_device: return None if job.actual_device.status is not Device.RESERVED: # should not happen logger.error("[%d] device [%s] not in reserved state", job.id, job.actual_device) return None if job.actual_device.worker_host is None: fail_msg = "Misconfigured device configuration for %s - missing worker_host" % job.actual_device fail_job(job, fail_msg=fail_msg) logger.error(fail_msg) if job.is_multinode: # inject the actual group hostnames into the roles for the dispatcher to populate in the overlay. devices = {} for multinode_job in job.sub_jobs_list: # build a list of all devices in this group definition = yaml.load(multinode_job.definition) # devices are not necessarily assigned to all jobs in a group at the same time # check all jobs in this multinode group before allowing any to start. if multinode_job.dynamic_connection: logger.debug("[%s] dynamic connection job", multinode_job.sub_id) continue if not multinode_job.actual_device: logger.debug("[%s] job has no device yet", multinode_job.sub_id) return None devices[str(multinode_job.actual_device.hostname )] = definition['protocols']['lava-multinode']['role'] for multinode_job in job.sub_jobs_list: # apply the complete list to all jobs in this group definition = yaml.load(multinode_job.definition) definition['protocols']['lava-multinode']['roles'] = devices multinode_job.definition = yaml.dump(definition) multinode_job.save() # Load job definition to get the variables for template rendering job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) parser = JobParser() device = None device_object = None if not job.dynamic_connection: device = job.actual_device try: device_config = device.load_device_configuration( job_ctx) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: # FIXME: report the exceptions as useful user messages logger.error("[%d] jinja2 error: %s" % (job.id, exc)) return None if not device_config or type(device_config) is not dict: # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname logger.error('[%d] device-dictionary error: %s' % (job.id, msg)) # as we don't control the scheduler, yet, this has to be an error and an incomplete job. # the scheduler_daemon sorts by a fixed order, so this would otherwise just keep on repeating. fail_job(job, fail_msg=msg) return None device_object = PipelineDevice( device_config, device.hostname ) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname validate_list = job.sub_jobs_list if job.is_multinode else [job] for check_job in validate_list: parser_device = None if job.dynamic_connection else device_object try: logger.debug("[%d] parsing definition" % check_job.id) # pass (unused) output_dir just for validation as there is no zmq socket either. pipeline_job = parser.parse(check_job.definition, parser_device, check_job.id, None, output_dir=check_job.output_dir) except (AttributeError, JobError, NotImplementedError, KeyError, TypeError) as exc: logger.error('[%d] parser error: %s' % (check_job.id, exc)) fail_job(check_job, fail_msg=exc) return None try: logger.debug("[%d] validating actions" % check_job.id) pipeline_job.pipeline.validate_actions() except (AttributeError, JobError, KeyError, TypeError) as exc: logger.error({device: exc}) fail_job(check_job, fail_msg=exc) return None if pipeline_job: pipeline = pipeline_job.describe() # write the pipeline description to the job output directory. if not os.path.exists(check_job.output_dir): os.makedirs(check_job.output_dir) with open(os.path.join(check_job.output_dir, 'description.yaml'), 'w') as describe_yaml: describe_yaml.write(yaml.dump(pipeline)) map_metadata(yaml.dump(pipeline), job) return device
def select_device(job, dispatchers): """ Transitioning a device from Idle to Reserved is the responsibility of the scheduler_daemon (currently). This function just checks that the reserved device is valid for this job. Jobs will only enter this function if a device is already reserved for that job. Stores the pipeline description To prevent cycling between lava_scheduler_daemon:assign_jobs and here, if a job fails validation, the job is incomplete. Issues with this need to be fixed using device tags. """ # FIXME: split out dynamic_connection, multinode and validation logger = logging.getLogger('dispatcher-master') if not job.dynamic_connection: if not job.actual_device: return None if job.actual_device.status is not Device.RESERVED: # should not happen logger.error("[%d] device [%s] not in reserved state", job.id, job.actual_device) return None if job.actual_device.worker_host is None: fail_msg = "Misconfigured device configuration for %s - missing worker_host" % job.actual_device fail_job(job, fail_msg=fail_msg) logger.error(fail_msg) return None if job.is_multinode: # inject the actual group hostnames into the roles for the dispatcher to populate in the overlay. devices = {} for multinode_job in job.sub_jobs_list: # build a list of all devices in this group definition = yaml.load(multinode_job.definition) # devices are not necessarily assigned to all jobs in a group at the same time # check all jobs in this multinode group before allowing any to start. if multinode_job.dynamic_connection: logger.debug("[%s] dynamic connection job", multinode_job.sub_id) continue if not multinode_job.actual_device: logger.debug("[%s] job has no device yet", multinode_job.sub_id) return None devices[str(multinode_job.actual_device.hostname)] = definition['protocols']['lava-multinode']['role'] for multinode_job in job.sub_jobs_list: # apply the complete list to all jobs in this group definition = yaml.load(multinode_job.definition) definition['protocols']['lava-multinode']['roles'] = devices multinode_job.definition = yaml.dump(definition) multinode_job.save() # Load job definition to get the variables for template rendering job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) parser = JobParser() device = None device_object = None if not job.dynamic_connection: device = job.actual_device try: device_config = device.load_device_configuration(job_ctx) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: logger.error("[%d] jinja2 error: %s" % (job.id, exc)) msg = "Administrative error. Unable to parse '%s'" % exc fail_job(job, fail_msg=msg) return None if not device_config or type(device_config) is not dict: # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname logger.error('[%d] device-dictionary error: %s' % (job.id, msg)) # as we don't control the scheduler, yet, this has to be an error and an incomplete job. # the scheduler_daemon sorts by a fixed order, so this would otherwise just keep on repeating. fail_job(job, fail_msg=msg) return None if not device.worker_host or not device.worker_host.hostname: msg = "Administrative error. Device '%s' has no worker host." % device.hostname logger.error('[%d] worker host error: %s', job.id, msg) fail_job(job, fail_msg=msg) return None if device.worker_host.hostname not in dispatchers: # a configured worker has not called in to this master # likely that the worker is misconfigured - polling the wrong master # or simply not running at all. msg = """Administrative error. Device '{0}' has a worker_host setting of '{1}' but no slave has registered with this master using that FQDN.""".format(device.hostname, device.worker_host.hostname) logger.error('[%d] worker-hostname error: %s', job.id, msg) fail_job(job, fail_msg=msg) return None device_object = PipelineDevice(device_config, device.hostname) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname validate_list = job.sub_jobs_list if job.is_multinode else [job] for check_job in validate_list: parser_device = None if job.dynamic_connection else device_object try: logger.info("[%d] Parsing definition" % check_job.id) # pass (unused) output_dir just for validation as there is no zmq socket either. pipeline_job = parser.parse( check_job.definition, parser_device, check_job.id, None, output_dir=check_job.output_dir) except (AttributeError, JobError, NotImplementedError, KeyError, TypeError) as exc: logger.error('[%d] parser error: %s' % (check_job.id, exc)) fail_job(check_job, fail_msg=exc) return None try: logger.info("[%d] Validating actions" % check_job.id) pipeline_job.pipeline.validate_actions() except (AttributeError, JobError, KeyError, TypeError) as exc: logger.error({device: exc}) fail_job(check_job, fail_msg=exc) return None if pipeline_job: pipeline = pipeline_job.describe() # write the pipeline description to the job output directory. if not os.path.exists(check_job.output_dir): os.makedirs(check_job.output_dir) with open(os.path.join(check_job.output_dir, 'description.yaml'), 'w') as describe_yaml: describe_yaml.write(yaml.dump(pipeline)) map_metadata(yaml.dump(pipeline), job) # add the compatibility result from the master to the definition for comparison on the slave. if 'compatibility' in pipeline: try: compat = int(pipeline['compatibility']) except ValueError: logger.error("[%d] Unable to parse job compatibility: %s", check_job.id, pipeline['compatibility']) compat = 0 check_job.pipeline_compatibility = compat check_job.save(update_fields=['pipeline_compatibility']) else: logger.error("[%d] Unable to identify job compatibility.", check_job.id) fail_job(check_job, fail_msg='Unknown compatibility') return None return device
def test_jinja_string_templates(self): jinja2_path = os.path.realpath( os.path.join(__file__, '..', '..', '..', 'etc', 'dispatcher-config')) self.assertTrue(os.path.exists(jinja2_path)) device_dictionary = { 'usb_label': 'SanDisk_Ultra', 'sata_label': 'ST160LM003', 'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0", 'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184", 'connection_command': 'telnet localhost 6002', 'console_device': 'ttyfake1', 'baud_rate': 56 } data = devicedictionary_to_jinja2(device_dictionary, 'cubietruck.yaml') string_loader = jinja2.DictLoader({'cubie.yaml': data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'cubie') device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) self.assertIn('timeouts', yaml_data) self.assertIn('parameters', yaml_data) self.assertIn('bootz', yaml_data['parameters']) self.assertIn('media', yaml_data['parameters']) self.assertIn('usb', yaml_data['parameters']['media']) self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb']) self.assertIn( 'uuid', yaml_data['parameters']['media']['usb'][ device_dictionary['usb_label']]) self.assertEqual( yaml_data['parameters']['media']['usb'][ device_dictionary['usb_label']]['uuid'], device_dictionary['usb_uuid']) self.assertIn('commands', yaml_data) self.assertIn('connect', yaml_data['commands']) self.assertEqual(device_dictionary['connection_command'], yaml_data['commands']['connect']) ramdisk_args = yaml_data['actions']['boot']['methods']['u-boot'][ 'ramdisk'] self.assertIn('commands', ramdisk_args) self.assertIn('boot', ramdisk_args['commands']) self.assertIn( "setenv bootargs 'console=ttyfake1,56 debug rw root=/dev/ram0 ip=dhcp'", ramdisk_args['commands']) device_dictionary.update({ 'hard_reset_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command reboot --port 08", 'power_off_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command off --port 08", 'power_on_command': "/usr/bin/pduclient --daemon localhost --hostname pdu --command on --port 08" }) data = devicedictionary_to_jinja2(device_dictionary, 'beaglebone-black.yaml') string_loader = jinja2.DictLoader({'bbb.yaml': data}) type_loader = jinja2.FileSystemLoader( [os.path.join(jinja2_path, 'device-types')]) env = jinja2.Environment(loader=jinja2.ChoiceLoader( [string_loader, type_loader]), trim_blocks=True) template = env.get_template("%s.yaml" % 'bbb') device_configuration = template.render() yaml_data = yaml.load(device_configuration) self.assertTrue(validate_device(yaml_data)) device = PipelineDevice(yaml_data, 'bbb') self.assertIn('power_state', device) # bbb has power_on_command defined above self.assertEqual(device.power_state, 'off') self.assertTrue(hasattr(device, 'power_state')) self.assertFalse(hasattr(device, 'hostname')) self.assertIn('hostname', device)
def test_invalid_multinode(self): user = self.factory.make_user() self.device_type = self.factory.make_device_type() submission = yaml.load( open(os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(self.device_type, 'fakeqemu1') self.factory.make_device(self.device_type, 'fakeqemu2') self.factory.make_device(self.device_type, 'fakeqemu3', tags=tag_list) deploy = [ action['deploy'] for action in submission['actions'] if 'deploy' in action ] # replace working image with a broken URL for block in deploy: block['image'] = 'http://localhost/unknown/invalid.gz' job_object_list = _pipeline_protocols(submission, user, yaml.dump(submission)) self.assertEqual(len(job_object_list), 2) self.assertEqual(job_object_list[0].sub_id, "%d.%d" % (int(job_object_list[0].id), 0)) # FIXME: dispatcher master needs to make this kind of test more accessible. for job in job_object_list: definition = yaml.load(job.definition) self.assertNotEqual( definition['protocols']['lava-multinode']['sub_id'], '') job.actual_device = Device.objects.get(hostname='fakeqemu1') job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) parser = JobParser() device = None device_object = None if not job.dynamic_connection: device = job.actual_device try: device_config = device.load_device_configuration( job_ctx) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: # FIXME: report the exceptions as useful user messages self.fail("[%d] jinja2 error: %s" % (job.id, exc)) if not device_config or type(device_config) is not dict: # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname self.fail('[%d] device-dictionary error: %s' % (job.id, msg)) device_object = PipelineDevice( device_config, device.hostname ) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname validate_list = job.sub_jobs_list if job.is_multinode else [job] for check_job in validate_list: parser_device = None if job.dynamic_connection else device_object try: # pass (unused) output_dir just for validation as there is no zmq socket either. pipeline_job = parser.parse( check_job.definition, parser_device, check_job.id, None, output_dir=check_job.output_dir) except (AttributeError, JobError, NotImplementedError, KeyError, TypeError) as exc: self.fail('[%s] parser error: %s' % (check_job.sub_id, exc)) if os.path.exists( '/dev/loop0' ): # rather than skipping the entire test, just the validation. self.assertRaises(JobError, pipeline_job.pipeline.validate_actions) for job in job_object_list: job = TestJob.objects.get(id=job.id) self.assertNotEqual(job.sub_id, '')
def select_device(job): """ Transitioning a device from Idle to Reserved is the responsibility of the scheduler_daemon (currently). This function just checks that the reserved device is valid for this job. Jobs will only enter this function if a device is already reserved for that job. Storse the pipeline description To prevent cycling between lava_scheduler_daemon:assign_jobs and here, if a job fails validation, the job is incomplete. Issues with this need to be fixed using device tags. """ logger = logging.getLogger('dispatcher-master') if not job.actual_device: # should not happen. logger.error("[%d] no device reserved", job.id) return None if job.actual_device.status is not Device.RESERVED: # should not happen logger.error("[%d] device [%s] not in reserved state", job.id, job.actual_device) return None if job.actual_device.worker_host is None: fail_msg = "Misconfigured device configuration for %s - missing worker_host" % job.actual_device end_job(job, fail_msg=fail_msg, job_status=TestJob.INCOMPLETE) logger.error(fail_msg) if job.is_multinode: # inject the actual group hostnames into the roles for the dispatcher to populate in the overlay. devices = {} for multinode_job in job.sub_jobs_list: # build a list of all devices in this group definition = yaml.load(multinode_job.definition) # devices are not necessarily assigned to all jobs in a group at the same time # check all jobs in this multinode group before allowing any to start. if not multinode_job.actual_device: logger.debug("[%s] job has no device yet", multinode_job.sub_id) return None devices[str(multinode_job.actual_device.hostname)] = definition['protocols']['lava-multinode']['role'] for multinode_job in job.sub_jobs_list: # apply the complete list to all jobs in this group definition = yaml.load(multinode_job.definition) definition['protocols']['lava-multinode']['roles'] = devices multinode_job.definition = yaml.dump(definition) multinode_job.save() # Load job definition to get the variables for template rendering job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) device = job.actual_device try: device_config = device.load_device_configuration(job_ctx) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: # FIXME: report the exceptions as useful user messages logger.error({'jinja2': exc}) return None if not device_config or type(device_config) is not dict: # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname logger.error({'device-dictionary': msg}) # as we don't control the scheduler, yet, this has to be an error and an incomplete job. # the scheduler_daemon sorts by a fixed order, so this would otherwise just keep on repeating. end_job(job, fail_msg=msg, job_status=TestJob.INCOMPLETE) return None parser = JobParser() obj = PipelineDevice(device_config, device.hostname) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in obj: obj.target = device.hostname obj['hostname'] = device.hostname # pass (unused) output_dir just for validation as there is no zmq socket either. try: pipeline_job = parser.parse(job.definition, obj, job.id, None, output_dir='/tmp') except (JobError, AttributeError, NotImplementedError, KeyError, TypeError) as exc: logger.error({'parser': exc}) end_job(job, fail_msg=exc, job_status=TestJob.INCOMPLETE) return None try: pipeline_job.pipeline.validate_actions() except (AttributeError, JobError, KeyError, TypeError) as exc: logger.error({device: exc}) end_job(job, fail_msg=exc, job_status=TestJob.INCOMPLETE) return None if pipeline_job: pipeline = pipeline_job.describe() # write the pipeline description to the job output directory. if not os.path.exists(job.output_dir): os.makedirs(job.output_dir) with open(os.path.join(job.output_dir, 'description.yaml'), 'w') as describe_yaml: describe_yaml.write(yaml.dump(pipeline)) map_metadata(yaml.dump(pipeline), job) return device
def select_device(job, dispatchers): # pylint: disable=too-many-return-statements """ Transitioning a device from Idle to Reserved is the responsibility of the scheduler_daemon (currently). This function just checks that the reserved device is valid for this job. Jobs will only enter this function if a device is already reserved for that job. Stores the pipeline description To prevent cycling between lava_scheduler_daemon:assign_jobs and here, if a job fails validation, the job is incomplete. Issues with this need to be fixed using device tags. """ # FIXME: split out dynamic_connection, multinode and validation logger = logging.getLogger('dispatcher-master') if not job.dynamic_connection: if not job.actual_device: return None if job.actual_device.status is not Device.RESERVED: # should not happen logger.error("[%d] device [%s] not in reserved state", job.id, job.actual_device) return None if job.actual_device.worker_host is None: fail_msg = "Misconfigured device configuration for %s - missing worker_host" % job.actual_device fail_job(job, fail_msg=fail_msg) logger.error(fail_msg) return None if job.is_multinode: # inject the actual group hostnames into the roles for the dispatcher to populate in the overlay. devices = {} for multinode_job in job.sub_jobs_list: # build a list of all devices in this group definition = yaml.load(multinode_job.definition) # devices are not necessarily assigned to all jobs in a group at the same time # check all jobs in this multinode group before allowing any to start. if multinode_job.dynamic_connection: logger.debug("[%s] dynamic connection job", multinode_job.sub_id) continue if not multinode_job.actual_device: logger.debug("[%s] job has no device yet", multinode_job.sub_id) return None devices[str(multinode_job.actual_device.hostname)] = definition['protocols']['lava-multinode']['role'] for multinode_job in job.sub_jobs_list: # apply the complete list to all jobs in this group definition = yaml.load(multinode_job.definition) definition['protocols']['lava-multinode']['roles'] = devices multinode_job.definition = yaml.dump(definition) multinode_job.save() # Load job definition to get the variables for template rendering job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) device = None if not job.dynamic_connection: device = job.actual_device try: device_config = device.load_device_configuration(job_ctx) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: logger.error("[%d] jinja2 error: %s", job.id, exc) msg = "Administrative error. Unable to parse device configuration: '%s'" % exc fail_job(job, fail_msg=msg) return None if not device_config or not isinstance(device_config, dict): # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname logger.error('[%d] device-dictionary error: %s', job.id, msg) # as we don't control the scheduler, yet, this has to be an error and an incomplete job. # the scheduler_daemon sorts by a fixed order, so this would otherwise just keep on repeating. fail_job(job, fail_msg=msg) return None if not device.worker_host or not device.worker_host.hostname: msg = "Administrative error. Device '%s' has no worker host." % device.hostname logger.error('[%d] worker host error: %s', job.id, msg) fail_job(job, fail_msg=msg) return None if device.worker_host.hostname not in dispatchers: # A configured worker has not (yet) called in to this master. # It is likely that the worker is misconfigured - polling the wrong master # or simply not running at all. There is also a possible race condition # here when the master gets restarted with a queue of jobs and has not yet # received polls from all slaves, so do not fail the job. msg = "Device '{0}' has a worker_host setting of " \ "'{1}' but no slave has yet registered with this master " \ "using that FQDN.".format(device.hostname, device.worker_host.hostname) logger.info('[%d] worker-hostname not seen: %s', job.id, msg) return None device_object = PipelineDevice(device_config, device.hostname) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname return device
def test_invalid_multinode(self): # pylint: disable=too-many-locals user = self.factory.make_user() device_type = self.factory.make_device_type() submission = yaml.load(open( os.path.join(os.path.dirname(__file__), 'kvm-multinode.yaml'), 'r')) tag_list = [ self.factory.ensure_tag('usb-flash'), self.factory.ensure_tag('usb-eth') ] self.factory.make_device(device_type, 'fakeqemu1') self.factory.make_device(device_type, 'fakeqemu2') self.factory.make_device(device_type, 'fakeqemu3', tags=tag_list) deploy = [action['deploy'] for action in submission['actions'] if 'deploy' in action] # replace working image with a broken URL for block in deploy: block['images'] = { 'rootfs': { 'url': 'http://localhost/unknown/invalid.gz', 'image_arg': '{rootfs}' } } job_object_list = _pipeline_protocols(submission, user, yaml.dump(submission)) self.assertEqual(len(job_object_list), 2) self.assertEqual( job_object_list[0].sub_id, "%d.%d" % (int(job_object_list[0].id), 0)) # FIXME: dispatcher master needs to make this kind of test more accessible. for job in job_object_list: definition = yaml.load(job.definition) self.assertNotEqual(definition['protocols']['lava-multinode']['sub_id'], '') job.actual_device = Device.objects.get(hostname='fakeqemu1') job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) parser = JobParser() device = None device_object = None if not job.dynamic_connection: device = job.actual_device try: device_config = device.load_device_configuration(job_ctx, system=False) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: # FIXME: report the exceptions as useful user messages self.fail("[%d] jinja2 error: %s" % (job.id, exc)) if not device_config or not isinstance(device_config, dict): # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname self.fail('[%d] device-dictionary error: %s' % (job.id, msg)) device_object = PipelineDevice(device_config, device.hostname) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname validate_list = job.sub_jobs_list if job.is_multinode else [job] for check_job in validate_list: parser_device = None if job.dynamic_connection else device_object try: # pass (unused) output_dir just for validation as there is no zmq socket either. pipeline_job = parser.parse( check_job.definition, parser_device, check_job.id, None, None, None, output_dir=check_job.output_dir) except (AttributeError, JobError, NotImplementedError, KeyError, TypeError) as exc: self.fail('[%s] parser error: %s' % (check_job.sub_id, exc)) with TestCase.assertRaises(self, (JobError, InfrastructureError)) as check: pipeline_job.pipeline.validate_actions() check_missing_path(self, check, 'qemu-system-x86_64') for job in job_object_list: job = TestJob.objects.get(id=job.id) self.assertNotEqual(job.sub_id, '')
def select_device(job, dispatchers): # pylint: disable=too-many-return-statements """ Transitioning a device from Idle to Reserved is the responsibility of the scheduler_daemon (currently). This function just checks that the reserved device is valid for this job. Jobs will only enter this function if a device is already reserved for that job. Stores the pipeline description To prevent cycling between lava_scheduler_daemon:assign_jobs and here, if a job fails validation, the job is incomplete. Issues with this need to be fixed using device tags. """ # FIXME: split out dynamic_connection, multinode and validation logger = logging.getLogger('dispatcher-master') if not job.dynamic_connection: if not job.actual_device: return None if job.actual_device.status is not Device.RESERVED: # should not happen logger.error("[%d] device [%s] not in reserved state", job.id, job.actual_device) return None if job.actual_device.worker_host is None: fail_msg = "Misconfigured device configuration for %s - missing worker_host" % job.actual_device fail_job(job, fail_msg=fail_msg) logger.error(fail_msg) return None if job.is_multinode: # inject the actual group hostnames into the roles for the dispatcher to populate in the overlay. devices = {} for multinode_job in job.sub_jobs_list: # build a list of all devices in this group definition = yaml.load(multinode_job.definition) # devices are not necessarily assigned to all jobs in a group at the same time # check all jobs in this multinode group before allowing any to start. if multinode_job.dynamic_connection: logger.debug("[%s] dynamic connection job", multinode_job.sub_id) continue if not multinode_job.actual_device: logger.debug("[%s] job has no device yet", multinode_job.sub_id) return None devices[str(multinode_job.actual_device.hostname )] = definition['protocols']['lava-multinode']['role'] for multinode_job in job.sub_jobs_list: # apply the complete list to all jobs in this group definition = yaml.load(multinode_job.definition) definition['protocols']['lava-multinode']['roles'] = devices multinode_job.definition = yaml.dump(definition) multinode_job.save() # Load job definition to get the variables for template rendering job_def = yaml.load(job.definition) job_ctx = job_def.get('context', {}) device = None if not job.dynamic_connection: device = job.actual_device try: device_config = device.load_device_configuration( job_ctx) # raw dict except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc: logger.error("[%d] jinja2 error: %s", job.id, exc) msg = "Administrative error. Unable to parse device configuration: '%s'" % exc fail_job(job, fail_msg=msg) return None if not device_config or not isinstance(device_config, dict): # it is an error to have a pipeline device without a device dictionary as it will never get any jobs. msg = "Administrative error. Device '%s' has no device dictionary." % device.hostname logger.error('[%d] device-dictionary error: %s', job.id, msg) # as we don't control the scheduler, yet, this has to be an error and an incomplete job. # the scheduler_daemon sorts by a fixed order, so this would otherwise just keep on repeating. fail_job(job, fail_msg=msg) return None if not device.worker_host or not device.worker_host.hostname: msg = "Administrative error. Device '%s' has no worker host." % device.hostname logger.error('[%d] worker host error: %s', job.id, msg) fail_job(job, fail_msg=msg) return None if device.worker_host.hostname not in dispatchers: # A configured worker has not (yet) called in to this master. # It is likely that the worker is misconfigured - polling the wrong master # or simply not running at all. There is also a possible race condition # here when the master gets restarted with a queue of jobs and has not yet # received polls from all slaves, so do not fail the job. msg = "Device '{0}' has a worker_host setting of " \ "'{1}' but no slave has yet registered with this master " \ "using that FQDN.".format(device.hostname, device.worker_host.hostname) logger.info('[%d] worker-hostname not seen: %s', job.id, msg) return None device_object = PipelineDevice( device_config, device.hostname ) # equivalent of the NewDevice in lava-dispatcher, without .yaml file. # FIXME: drop this nasty hack once 'target' is dropped as a parameter if 'target' not in device_object: device_object.target = device.hostname device_object['hostname'] = device.hostname return device