def test_pipeline_health_assignment(self): user = self.factory.make_user() device1 = Device.objects.get(hostname='fakeqemu1') self.factory.make_fake_qemu_device(hostname="fakeqemu1") self.assertTrue(device1.is_pipeline) device2 = self.factory.make_device(device_type=device1.device_type, hostname='fakeqemu2') self.factory.make_fake_qemu_device(hostname="fakeqemu2") self.assertTrue(device2.is_pipeline) device3 = self.factory.make_device(device_type=device1.device_type, hostname='fakeqemu3') self.factory.make_fake_qemu_device(hostname="fakeqemu3") self.assertTrue(device3.is_pipeline) job1 = testjob_submission(self.factory.make_job_yaml(), user, check_device=device1) job2 = testjob_submission(self.factory.make_job_yaml(), user, check_device=device2) self.assertEqual(user, job1.submitter) self.assertTrue(job1.health_check) self.assertEqual(job1.requested_device, device1) self.assertEqual(job2.requested_device, device2) self.assertIsNone(job1.actual_device) self.assertIsNone(job2.actual_device) device_list = Device.objects.filter(device_type=device1.device_type) count = 0 while True: device_list.reverse() assigned = find_device_for_job(job2, device_list) if assigned != device2: self.fail("[%d] invalid device assigment in health check." % count) count += 1 if count > 100: break self.assertGreater(count, 100)
def test_pipeline_health_check(self): user = self.factory.make_user() device = Device.objects.get(hostname='fakeqemu1') job = testjob_submission(self.factory.make_job_yaml(), user, check_device=None) self.assertEqual(user, job.submitter) self.assertFalse(job.health_check) self.assertIsNone(job.requested_device) job = testjob_submission(self.factory.make_job_yaml(), user, check_device=device) self.assertEqual(user, job.submitter) self.assertTrue(job.health_check) self.assertEqual(job.requested_device, device) self.assertIsNone(job.actual_device)
def test_device_type_alias(self): self.factory.cleanup() user = self.factory.make_user() dt = self.factory.make_device_type(name="qemu") device = self.factory.make_device(device_type=dt, hostname="qemu-1") device.save() alias_name = "qemu_foo" alias = self.factory.make_device_type_alias(dt, name=alias_name) definition = self.factory.make_job_data_from_file("qemu-foo.yaml") job = testjob_submission(definition, user, None) self.assertEqual(job.requested_device_type, dt)
def test_queueing(self): """ uses stderr to avoid buffered prints Expect the test itself to take <30s and the gap between jobs submitted and end being ~500ms Most of the time is spent setting up the database and submitting all the test jobs. """ print >> sys.stderr, timezone.now(), "start" user = self.factory.ensure_user('test', '*****@*****.**', 'test') user.user_permissions.add( Permission.objects.get(codename='add_testjob')) user.save() device_type = self.factory.make_device_type('beaglebone-black') device = self.factory.make_device(device_type=device_type, hostname="black01") device.save() device_type = self.factory.make_device_type('wandboard') count = 1 while count < 100: suffix = "{:02d}".format(count) device = self.factory.make_device(device_type=device_type, hostname="imx6q-%s" % suffix) device.save() count += 1 print >> sys.stderr, timezone.now(), "%d dummy devices created" % count device_list = list(get_available_devices()) print >> sys.stderr, timezone.now( ), "%d available devices" % len(device_list) filename = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'master-check.json') self.assertTrue(os.path.exists(filename)) with open(filename, 'r') as json_file: definition = json_file.read() count = 0 # each 1000 more can take ~15s in the test. while count < 1000: # simulate API submission job = testjob_submission(definition, user) self.assertFalse(job.health_check) count += 1 print >> sys.stderr, timezone.now(), "%d jobs submitted" % count jobs = list(get_job_queue()) self.assertIsNotNone(jobs) print >> sys.stderr, timezone.now(), "Finding devices for jobs." for job in jobs: # this needs to stay as a tight loop to cope with load device = find_device_for_job(job, device_list) if device: print >> sys.stderr, timezone.now(), "[%d] allocated %s" % ( job.id, device) device_list.remove(device) print >> sys.stderr, timezone.now(), "end"
def submit_job(self, job_data): """ Name ---- `submit_job` (`job_data`) Description ----------- Submit the given job data which is in LAVA job JSON or YAML format as a new job to LAVA scheduler. Arguments --------- `job_data`: string Job JSON or YAML string. Return value ------------ This function returns an XML-RPC integer which is the newly created job's id, provided the user is authenticated with an username and token. If the job is a multinode job, this function returns the list of created job IDs. """ self._authenticate() if not self.user.has_perm("lava_scheduler_app.add_testjob"): raise xmlrpc.client.Fault( 403, "Permission denied. User %r does not have the " "'lava_scheduler_app.add_testjob' permission. Contact " "the administrators." % self.user.username, ) try: job = testjob_submission(job_data, self.user) except SubmissionException as exc: raise xmlrpc.client.Fault( 400, "Problem with submitted job data: %s" % exc) # FIXME: json error is not needed anymore except (JSONDataError, JSONDecodeError, ValueError) as exc: raise xmlrpc.client.Fault( 400, "Decoding job submission failed: %s." % exc) except (Device.DoesNotExist, DeviceType.DoesNotExist): raise xmlrpc.client.Fault( 404, "Specified device or device type not found.") except DevicesUnavailableException as exc: raise xmlrpc.client.Fault(400, "Device unavailable: %s" % str(exc)) if isinstance(job, list): return [j.sub_id for j in job] else: return job.id
def test_queueing(self): """ uses stderr to avoid buffered prints Expect the test itself to take <30s and the gap between jobs submitted and end being ~500ms Most of the time is spent setting up the database and submitting all the test jobs. """ print >> sys.stderr, timezone.now(), "start" user = self.factory.ensure_user('test', '*****@*****.**', 'test') user.user_permissions.add( Permission.objects.get(codename='add_testjob')) user.save() device_type = self.factory.make_device_type('beaglebone-black') device = self.factory.make_device(device_type=device_type, hostname="black01") device.save() device_type = self.factory.make_device_type('wandboard') count = 1 while count < 100: suffix = "{:02d}".format(count) device = self.factory.make_device(device_type=device_type, hostname="imx6q-%s" % suffix) device.save() count += 1 print >> sys.stderr, timezone.now(), "%d dummy devices created" % count device_list = list(get_available_devices()) print >> sys.stderr, timezone.now(), "%d available devices" % len(device_list) filename = os.path.join(os.path.dirname(__file__), 'master-check.json') self.assertTrue(os.path.exists(filename)) with open(filename, 'r') as json_file: definition = json_file.read() count = 0 # each 1000 more can take ~15s in the test. while count < 1000: # simulate API submission job = testjob_submission(definition, user) self.assertFalse(job.health_check) count += 1 print >> sys.stderr, timezone.now(), "%d jobs submitted" % count jobs = list(get_job_queue()) self.assertIsNotNone(jobs) print >> sys.stderr, timezone.now(), "Finding devices for jobs." for job in jobs: # this needs to stay as a tight loop to cope with load device = find_device_for_job(job, device_list) if device: print >> sys.stderr, timezone.now(), "[%d] allocated %s" % (job.id, device) device_list.remove(device) print >> sys.stderr, timezone.now(), "end"
def test_json_yaml(self): self.factory.cleanup() user = self.factory.make_user() dt = self.factory.make_device_type(name="qemu") device = self.factory.make_device(device_type=dt, hostname="qemu-1") device.save() definition = self.factory.make_job_data_from_file( "qemu-pipeline-first-job.yaml" ) # convert content of file to JSON string json_def = json.dumps(yaml_safe_load(definition)) job = testjob_submission(json_def, user, None) # check that submitted JSON is now YAML self.assertRaises(json.decoder.JSONDecodeError, json.loads, job.definition) yaml_safe_load(job.definition) self.assertIsInstance(job.definition, str)
def test_job_data(self): self.factory.cleanup() user = self.factory.make_user() dt = self.factory.make_device_type(name="qemu") device = self.factory.make_device(device_type=dt, hostname="qemu-1") device.save() definition = self.factory.make_job_data_from_file( "qemu-pipeline-first-job.yaml") job = testjob_submission(definition, user, None) data = job.create_job_data() self.assertIsNotNone(data) self.assertIn("start_time", data) # job has not started but job_data explicitly turns start_time into str() self.assertEqual(data["start_time"], "None") self.assertEqual(data["state_string"], TestJob.STATE_CHOICES[TestJob.STATE_SUBMITTED][1])
def submit_job(self, job_data): """ Name ---- `submit_job` (`job_data`) Description ----------- Submit the given job data which is in LAVA job JSON or YAML format as a new job to LAVA scheduler. Arguments --------- `job_data`: string Job JSON or YAML string. Return value ------------ This function returns an XML-RPC integer which is the newly created job's id, provided the user is authenticated with an username and token. If the job is a multinode job, this function returns the list of created job IDs. """ self._authenticate() if not self.user.has_perm('lava_scheduler_app.add_testjob'): raise xmlrpclib.Fault( 403, "Permission denied. User %r does not have the " "'lava_scheduler_app.add_testjob' permission. Contact " "the administrators." % self.user.username) try: job = testjob_submission(job_data, self.user) except SubmissionException as exc: raise xmlrpclib.Fault(400, "Problem with submitted job data: %s" % exc) # FIXME: json error is not needed anymore except (JSONDataError, JSONDecodeError, ValueError) as exc: raise xmlrpclib.Fault(400, "Decoding job submission failed: %s." % exc) except (Device.DoesNotExist, DeviceType.DoesNotExist): raise xmlrpclib.Fault(404, "Specified device or device type not found.") except DevicesUnavailableException as exc: raise xmlrpclib.Fault(400, "Device unavailable: %s" % str(exc)) if isinstance(job, type(list())): return [j.sub_id for j in job] else: return job.id
def test_job_data(self): self.factory.cleanup() user = self.factory.make_user() user.user_permissions.add( Permission.objects.get(codename='add_testjob')) user.save() dt = self.factory.make_device_type(name='qemu') device = self.factory.make_device(device_type=dt, hostname='qemu-1') device.save() definition = self.factory.make_job_data_from_file( 'qemu-pipeline-first-job.yaml') job = testjob_submission(definition, user, None) data = job.create_job_data() self.assertIsNotNone(data) self.assertIn('start_time', data) # job has not started but job_data explicitly turns start_time into str() self.assertEqual(data['start_time'], "None") self.assertEqual(data['state_string'], TestJob.STATE_CHOICES[TestJob.STATE_SUBMITTED][1])
def submit_job(self, job_data): """ Name ---- `submit_job` (`job_data`) Description ----------- Submit the given job data which is in LAVA job JSON or YAML format as a new job to LAVA scheduler. Arguments --------- `job_data`: string Job JSON or YAML string. Return value ------------ This function returns an XML-RPC integer which is the newly created job's id, provided the user is authenticated with an username and token. If the job is a multinode job, this function returns the list of created job IDs. """ self._authenticate() try: job = testjob_submission(job_data, self.user) except SubmissionException as exc: raise xmlrpc.client.Fault( 400, "Problem with submitted job data: %s" % exc) except ValueError as exc: raise xmlrpc.client.Fault( 400, "Decoding job submission failed: %s." % exc) except yaml.YAMLError as exc: raise xmlrpc.client.Fault(400, "Invalid job definition: %s." % exc) except (Device.DoesNotExist, DeviceType.DoesNotExist): raise xmlrpc.client.Fault( 404, "Specified device or device type not found.") except DevicesUnavailableException as exc: raise xmlrpc.client.Fault(400, "Device unavailable: %s" % str(exc)) if isinstance(job, list): return [j.sub_id for j in job] else: return job.id
def create(self, request, **kwargs): serializer = serializers.TestJobSerializer( data=request.data, context={"request": request}) serializer.is_valid(raise_exception=True) definition = serializer.validated_data["definition"] try: job = testjob_submission(definition, self.request.user) except SubmissionException as exc: return Response( {"message": "Problem with submitted job data: %s" % exc}, status=status.HTTP_400_BAD_REQUEST, ) except (ValueError, KeyError) as exc: return Response( {"message": "job submission failed: %s." % exc}, status=status.HTTP_400_BAD_REQUEST, ) except (Device.DoesNotExist, DeviceType.DoesNotExist): return Response( {"message": "Specified device or device type not found."}, status=status.HTTP_400_BAD_REQUEST, ) except DevicesUnavailableException as exc: return Response( {"message": "Devices unavailable: %s" % exc}, status=status.HTTP_400_BAD_REQUEST, ) if isinstance(job, list): job_ids = [j.sub_id for j in job] else: job_ids = [job.id] return Response( { "message": "job(s) successfully submitted", "job_ids": job_ids }, status=status.HTTP_201_CREATED, )
def resubmit(self, request, **kwargs): if self.get_object().is_multinode: definition = self.get_object().multinode_definition else: definition = self.get_object().definition try: job = testjob_submission(definition, self.request.user) except SubmissionException as exc: return Response( {"message": "Problem with submitted job data: %s" % exc}, status=status.HTTP_400_BAD_REQUEST, ) except (ValueError, KeyError) as exc: return Response( {"message": "job submission failed: %s." % exc}, status=status.HTTP_400_BAD_REQUEST, ) except (Device.DoesNotExist, DeviceType.DoesNotExist): return Response( {"message": "Specified device or device type not found."}, status=status.HTTP_400_BAD_REQUEST, ) except DevicesUnavailableException as exc: return Response( {"message": "Devices unavailable: %s" % exc}, status=status.HTTP_400_BAD_REQUEST, ) if isinstance(job, list): job_ids = [j.sub_id for j in job] else: job_ids = [job.id] return Response( { "message": "job(s) successfully submitted", "job_ids": job_ids }, status=status.HTTP_201_CREATED, )
def test_health_determination(self): # pylint: disable=too-many-statements user = self.factory.ensure_user('test', '*****@*****.**', 'test') user.user_permissions.add( Permission.objects.get(codename='add_testjob')) user.save() device_type = self.factory.make_device_type('beaglebone-black') device = self.factory.make_device(device_type=device_type, hostname="black01") device.save() filename = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'master-check.json') self.assertTrue(os.path.exists(filename)) with open(filename, 'r') as json_file: definition = json_file.read() # simulate UI submission job = self.factory.make_testjob(definition=definition, submitter=user) self.assertFalse(job.health_check) job.save(update_fields=['health_check', 'requested_device']) self.assertFalse(job.health_check) job.delete() # simulate API submission job = testjob_submission(definition, user) self.assertFalse(job.health_check) self.assertIsNone(job.requested_device) job.delete() job = testjob_submission(definition, user, check_device=None) self.assertFalse(job.health_check) self.assertIsNone(job.requested_device) job.delete() # simulate initiating a health check job = testjob_submission(definition, user, check_device=device) self.assertTrue(job.health_check) self.assertEqual(job.requested_device.hostname, device.hostname) job.delete() # modify definition to use the deprecated target support device2 = self.factory.make_device(device_type=device_type, hostname="black02") device2.save() def_dict = json.loads(definition) self.assertNotIn('target', def_dict) def_dict['target'] = device2.hostname definition = json.dumps(def_dict) # simulate API submission with target set job = testjob_submission(definition, user, check_device=None) self.assertFalse(job.health_check) self.assertEqual(job.requested_device.hostname, device2.hostname) job.delete() # healthcheck designation overrides target (although this is itself an admin error) job = testjob_submission(definition, user, check_device=device) self.assertTrue(job.health_check) self.assertEqual(job.requested_device.hostname, device.hostname) job.delete() # check malformed JSON self.assertRaises(SubmissionException, testjob_submission, definition[:100], user) # check non-existent targets def_dict['target'] = 'nosuchdevice' definition = json.dumps(def_dict) self.assertRaises(Device.DoesNotExist, testjob_submission, definition, user) # check multinode API submission. bug #2130 filename = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'master-multinode.json') self.assertTrue(os.path.exists(filename)) with open(filename, 'r') as json_file: definition = json_file.read() job_list = testjob_submission(definition, user) self.assertIsInstance(job_list, list) for job in job_list: self.assertIsNotNone(job.vm_group) self.assertFalse(job.health_check) if job.requested_device_type == device_type: self.assertIsNone(job.requested_device) else: self.assertIsNotNone(job.requested_device) self.assertIsInstance(job.requested_device, TemporaryDevice) job.requested_device.delete() job.delete()
def test_health_determination(self): # pylint: disable=too-many-statements user = self.factory.ensure_user('test', '*****@*****.**', 'test') user.user_permissions.add( Permission.objects.get(codename='add_testjob')) user.save() device_type = self.factory.make_device_type('beaglebone-black') device = self.factory.make_device(device_type=device_type, hostname="black01") device.save() filename = os.path.join(os.path.dirname(__file__), 'master-check.json') self.assertTrue(os.path.exists(filename)) with open(filename, 'r') as json_file: definition = json_file.read() # simulate UI submission job = self.factory.make_testjob(definition=definition, submitter=user) self.assertFalse(job.health_check) job.save(update_fields=['health_check', 'requested_device']) self.assertFalse(job.health_check) job.delete() # simulate API submission job = testjob_submission(definition, user) self.assertFalse(job.health_check) self.assertIsNone(job.requested_device) job.delete() job = testjob_submission(definition, user, check_device=None) self.assertFalse(job.health_check) self.assertIsNone(job.requested_device) job.delete() # simulate initiating a health check job = testjob_submission(definition, user, check_device=device) self.assertTrue(job.health_check) self.assertEqual(job.requested_device.hostname, device.hostname) job.delete() # modify definition to use the deprecated target support device2 = self.factory.make_device(device_type=device_type, hostname="black02") device2.save() def_dict = json.loads(definition) self.assertNotIn('target', def_dict) def_dict['target'] = device2.hostname definition = json.dumps(def_dict) # simulate API submission with target set job = testjob_submission(definition, user, check_device=None) self.assertFalse(job.health_check) self.assertEqual(job.requested_device.hostname, device2.hostname) job.delete() # healthcheck designation overrides target (although this is itself an admin error) job = testjob_submission(definition, user, check_device=device) self.assertTrue(job.health_check) self.assertEqual(job.requested_device.hostname, device.hostname) job.delete() # check malformed JSON self.assertRaises(SubmissionException, testjob_submission, definition[:100], user) # check non-existent targets def_dict['target'] = 'nosuchdevice' definition = json.dumps(def_dict) self.assertRaises(Device.DoesNotExist, testjob_submission, definition, user) # check multinode API submission. bug #2130 filename = os.path.join(os.path.dirname(__file__), 'master-multinode.json') self.assertTrue(os.path.exists(filename)) with open(filename, 'r') as json_file: definition = json_file.read() job_list = testjob_submission(definition, user) self.assertIsInstance(job_list, list) for job in job_list: self.assertIsNotNone(job.vm_group) self.assertFalse(job.health_check) if job.requested_device_type == device_type: self.assertIsNone(job.requested_device) else: self.assertIsNotNone(job.requested_device) self.assertIsInstance(job.requested_device, TemporaryDevice) job.requested_device.delete() job.delete()