def test_pipeline_health_assignment(self):
user = self.factory.make_user()
device1 = Device.objects.get(hostname='fakeqemu1')
self.factory.make_fake_qemu_device(hostname="fakeqemu1")
self.assertTrue(device1.is_pipeline)
device2 = self.factory.make_device(device_type=device1.device_type, hostname='fakeqemu2')
self.factory.make_fake_qemu_device(hostname="fakeqemu2")
self.assertTrue(device2.is_pipeline)
device3 = self.factory.make_device(device_type=device1.device_type, hostname='fakeqemu3')
self.factory.make_fake_qemu_device(hostname="fakeqemu3")
self.assertTrue(device3.is_pipeline)
job1 = testjob_submission(self.factory.make_job_yaml(), user, check_device=device1)
job2 = testjob_submission(self.factory.make_job_yaml(), user, check_device=device2)
self.assertEqual(user, job1.submitter)
self.assertTrue(job1.health_check)
self.assertEqual(job1.requested_device, device1)
self.assertEqual(job2.requested_device, device2)
self.assertIsNone(job1.actual_device)
self.assertIsNone(job2.actual_device)
device_list = Device.objects.filter(device_type=device1.device_type)
count = 0
while True:
device_list.reverse()
assigned = find_device_for_job(job2, device_list)
if assigned != device2:
self.fail("[%d] invalid device assigment in health check." % count)
count += 1
if count > 100:
break
self.assertGreater(count, 100)
def test_pipeline_health_check(self):
user = self.factory.make_user()
device = Device.objects.get(hostname='fakeqemu1')
job = testjob_submission(self.factory.make_job_yaml(), user, check_device=None)
self.assertEqual(user, job.submitter)
self.assertFalse(job.health_check)
self.assertIsNone(job.requested_device)
job = testjob_submission(self.factory.make_job_yaml(), user, check_device=device)
self.assertEqual(user, job.submitter)
self.assertTrue(job.health_check)
self.assertEqual(job.requested_device, device)
self.assertIsNone(job.actual_device)
def test_device_type_alias(self):
self.factory.cleanup()
user = self.factory.make_user()
dt = self.factory.make_device_type(name="qemu")
device = self.factory.make_device(device_type=dt, hostname="qemu-1")
device.save()
alias_name = "qemu_foo"
alias = self.factory.make_device_type_alias(dt, name=alias_name)
definition = self.factory.make_job_data_from_file("qemu-foo.yaml")
job = testjob_submission(definition, user, None)
self.assertEqual(job.requested_device_type, dt)
def test_queueing(self):
"""
uses stderr to avoid buffered prints
Expect the test itself to take <30s and
the gap between jobs submitted and end being ~500ms
Most of the time is spent setting up the database
and submitting all the test jobs.
"""
print >> sys.stderr, timezone.now(), "start"
user = self.factory.ensure_user('test', '*****@*****.**', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
device_type = self.factory.make_device_type('beaglebone-black')
device = self.factory.make_device(device_type=device_type,
hostname="black01")
device.save()
device_type = self.factory.make_device_type('wandboard')
count = 1
while count < 100:
suffix = "{:02d}".format(count)
device = self.factory.make_device(device_type=device_type,
hostname="imx6q-%s" % suffix)
device.save()
count += 1
print >> sys.stderr, timezone.now(), "%d dummy devices created" % count
device_list = list(get_available_devices())
print >> sys.stderr, timezone.now(
), "%d available devices" % len(device_list)
filename = os.path.join(os.path.dirname(__file__), 'sample_jobs',
'master-check.json')
self.assertTrue(os.path.exists(filename))
with open(filename, 'r') as json_file:
definition = json_file.read()
count = 0
# each 1000 more can take ~15s in the test.
while count < 1000:
# simulate API submission
job = testjob_submission(definition, user)
self.assertFalse(job.health_check)
count += 1
print >> sys.stderr, timezone.now(), "%d jobs submitted" % count
jobs = list(get_job_queue())
self.assertIsNotNone(jobs)
print >> sys.stderr, timezone.now(), "Finding devices for jobs."
for job in jobs:
# this needs to stay as a tight loop to cope with load
device = find_device_for_job(job, device_list)
if device:
print >> sys.stderr, timezone.now(), "[%d] allocated %s" % (
job.id, device)
device_list.remove(device)
print >> sys.stderr, timezone.now(), "end"
def submit_job(self, job_data):
"""
Name
----
`submit_job` (`job_data`)
Description
-----------
Submit the given job data which is in LAVA job JSON or YAML format as a
new job to LAVA scheduler.
Arguments
---------
`job_data`: string
Job JSON or YAML string.
Return value
------------
This function returns an XML-RPC integer which is the newly created
job's id, provided the user is authenticated with an username and token.
If the job is a multinode job, this function returns the list of created
job IDs.
"""
self._authenticate()
if not self.user.has_perm("lava_scheduler_app.add_testjob"):
raise xmlrpc.client.Fault(
403,
"Permission denied. User %r does not have the "
"'lava_scheduler_app.add_testjob' permission. Contact "
"the administrators." % self.user.username,
)
try:
job = testjob_submission(job_data, self.user)
except SubmissionException as exc:
raise xmlrpc.client.Fault(
400, "Problem with submitted job data: %s" % exc)
# FIXME: json error is not needed anymore
except (JSONDataError, JSONDecodeError, ValueError) as exc:
raise xmlrpc.client.Fault(
400, "Decoding job submission failed: %s." % exc)
except (Device.DoesNotExist, DeviceType.DoesNotExist):
raise xmlrpc.client.Fault(
404, "Specified device or device type not found.")
except DevicesUnavailableException as exc:
raise xmlrpc.client.Fault(400, "Device unavailable: %s" % str(exc))
if isinstance(job, list):
return [j.sub_id for j in job]
else:
return job.id
def test_queueing(self):
"""
uses stderr to avoid buffered prints
Expect the test itself to take <30s and
the gap between jobs submitted and end being ~500ms
Most of the time is spent setting up the database
and submitting all the test jobs.
"""
print >> sys.stderr, timezone.now(), "start"
user = self.factory.ensure_user('test', '*****@*****.**', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
device_type = self.factory.make_device_type('beaglebone-black')
device = self.factory.make_device(device_type=device_type, hostname="black01")
device.save()
device_type = self.factory.make_device_type('wandboard')
count = 1
while count < 100:
suffix = "{:02d}".format(count)
device = self.factory.make_device(device_type=device_type, hostname="imx6q-%s" % suffix)
device.save()
count += 1
print >> sys.stderr, timezone.now(), "%d dummy devices created" % count
device_list = list(get_available_devices())
print >> sys.stderr, timezone.now(), "%d available devices" % len(device_list)
filename = os.path.join(os.path.dirname(__file__), 'master-check.json')
self.assertTrue(os.path.exists(filename))
with open(filename, 'r') as json_file:
definition = json_file.read()
count = 0
# each 1000 more can take ~15s in the test.
while count < 1000:
# simulate API submission
job = testjob_submission(definition, user)
self.assertFalse(job.health_check)
count += 1
print >> sys.stderr, timezone.now(), "%d jobs submitted" % count
jobs = list(get_job_queue())
self.assertIsNotNone(jobs)
print >> sys.stderr, timezone.now(), "Finding devices for jobs."
for job in jobs:
# this needs to stay as a tight loop to cope with load
device = find_device_for_job(job, device_list)
if device:
print >> sys.stderr, timezone.now(), "[%d] allocated %s" % (job.id, device)
device_list.remove(device)
print >> sys.stderr, timezone.now(), "end"
def test_json_yaml(self):
self.factory.cleanup()
user = self.factory.make_user()
dt = self.factory.make_device_type(name="qemu")
device = self.factory.make_device(device_type=dt, hostname="qemu-1")
device.save()
definition = self.factory.make_job_data_from_file(
"qemu-pipeline-first-job.yaml"
)
# convert content of file to JSON string
json_def = json.dumps(yaml_safe_load(definition))
job = testjob_submission(json_def, user, None)
# check that submitted JSON is now YAML
self.assertRaises(json.decoder.JSONDecodeError, json.loads, job.definition)
yaml_safe_load(job.definition)
self.assertIsInstance(job.definition, str)
def test_job_data(self):
self.factory.cleanup()
user = self.factory.make_user()
dt = self.factory.make_device_type(name="qemu")
device = self.factory.make_device(device_type=dt, hostname="qemu-1")
device.save()
definition = self.factory.make_job_data_from_file(
"qemu-pipeline-first-job.yaml")
job = testjob_submission(definition, user, None)
data = job.create_job_data()
self.assertIsNotNone(data)
self.assertIn("start_time", data)
# job has not started but job_data explicitly turns start_time into str()
self.assertEqual(data["start_time"], "None")
self.assertEqual(data["state_string"],
TestJob.STATE_CHOICES[TestJob.STATE_SUBMITTED][1])
def submit_job(self, job_data):
"""
Name
----
`submit_job` (`job_data`)
Description
-----------
Submit the given job data which is in LAVA job JSON or YAML format as a
new job to LAVA scheduler.
Arguments
---------
`job_data`: string
Job JSON or YAML string.
Return value
------------
This function returns an XML-RPC integer which is the newly created
job's id, provided the user is authenticated with an username and token.
If the job is a multinode job, this function returns the list of created
job IDs.
"""
self._authenticate()
if not self.user.has_perm('lava_scheduler_app.add_testjob'):
raise xmlrpclib.Fault(
403, "Permission denied. User %r does not have the "
"'lava_scheduler_app.add_testjob' permission. Contact "
"the administrators." % self.user.username)
try:
job = testjob_submission(job_data, self.user)
except SubmissionException as exc:
raise xmlrpclib.Fault(400, "Problem with submitted job data: %s" % exc)
# FIXME: json error is not needed anymore
except (JSONDataError, JSONDecodeError, ValueError) as exc:
raise xmlrpclib.Fault(400, "Decoding job submission failed: %s." % exc)
except (Device.DoesNotExist, DeviceType.DoesNotExist):
raise xmlrpclib.Fault(404, "Specified device or device type not found.")
except DevicesUnavailableException as exc:
raise xmlrpclib.Fault(400, "Device unavailable: %s" % str(exc))
if isinstance(job, type(list())):
return [j.sub_id for j in job]
else:
return job.id
def test_job_data(self):
self.factory.cleanup()
user = self.factory.make_user()
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
dt = self.factory.make_device_type(name='qemu')
device = self.factory.make_device(device_type=dt, hostname='qemu-1')
device.save()
definition = self.factory.make_job_data_from_file(
'qemu-pipeline-first-job.yaml')
job = testjob_submission(definition, user, None)
data = job.create_job_data()
self.assertIsNotNone(data)
self.assertIn('start_time', data)
# job has not started but job_data explicitly turns start_time into str()
self.assertEqual(data['start_time'], "None")
self.assertEqual(data['state_string'],
TestJob.STATE_CHOICES[TestJob.STATE_SUBMITTED][1])
def submit_job(self, job_data):
"""
Name
----
`submit_job` (`job_data`)
Description
-----------
Submit the given job data which is in LAVA job JSON or YAML format as a
new job to LAVA scheduler.
Arguments
---------
`job_data`: string
Job JSON or YAML string.
Return value
------------
This function returns an XML-RPC integer which is the newly created
job's id, provided the user is authenticated with an username and token.
If the job is a multinode job, this function returns the list of created
job IDs.
"""
self._authenticate()
try:
job = testjob_submission(job_data, self.user)
except SubmissionException as exc:
raise xmlrpc.client.Fault(
400, "Problem with submitted job data: %s" % exc)
except ValueError as exc:
raise xmlrpc.client.Fault(
400, "Decoding job submission failed: %s." % exc)
except yaml.YAMLError as exc:
raise xmlrpc.client.Fault(400, "Invalid job definition: %s." % exc)
except (Device.DoesNotExist, DeviceType.DoesNotExist):
raise xmlrpc.client.Fault(
404, "Specified device or device type not found.")
except DevicesUnavailableException as exc:
raise xmlrpc.client.Fault(400, "Device unavailable: %s" % str(exc))
if isinstance(job, list):
return [j.sub_id for j in job]
else:
return job.id
def create(self, request, **kwargs):
serializer = serializers.TestJobSerializer(
data=request.data, context={"request": request})
serializer.is_valid(raise_exception=True)
definition = serializer.validated_data["definition"]
try:
job = testjob_submission(definition, self.request.user)
except SubmissionException as exc:
return Response(
{"message": "Problem with submitted job data: %s" % exc},
status=status.HTTP_400_BAD_REQUEST,
)
except (ValueError, KeyError) as exc:
return Response(
{"message": "job submission failed: %s." % exc},
status=status.HTTP_400_BAD_REQUEST,
)
except (Device.DoesNotExist, DeviceType.DoesNotExist):
return Response(
{"message": "Specified device or device type not found."},
status=status.HTTP_400_BAD_REQUEST,
)
except DevicesUnavailableException as exc:
return Response(
{"message": "Devices unavailable: %s" % exc},
status=status.HTTP_400_BAD_REQUEST,
)
if isinstance(job, list):
job_ids = [j.sub_id for j in job]
else:
job_ids = [job.id]
return Response(
{
"message": "job(s) successfully submitted",
"job_ids": job_ids
},
status=status.HTTP_201_CREATED,
)
def resubmit(self, request, **kwargs):
if self.get_object().is_multinode:
definition = self.get_object().multinode_definition
else:
definition = self.get_object().definition
try:
job = testjob_submission(definition, self.request.user)
except SubmissionException as exc:
return Response(
{"message": "Problem with submitted job data: %s" % exc},
status=status.HTTP_400_BAD_REQUEST,
)
except (ValueError, KeyError) as exc:
return Response(
{"message": "job submission failed: %s." % exc},
status=status.HTTP_400_BAD_REQUEST,
)
except (Device.DoesNotExist, DeviceType.DoesNotExist):
return Response(
{"message": "Specified device or device type not found."},
status=status.HTTP_400_BAD_REQUEST,
)
except DevicesUnavailableException as exc:
return Response(
{"message": "Devices unavailable: %s" % exc},
status=status.HTTP_400_BAD_REQUEST,
)
if isinstance(job, list):
job_ids = [j.sub_id for j in job]
else:
job_ids = [job.id]
return Response(
{
"message": "job(s) successfully submitted",
"job_ids": job_ids
},
status=status.HTTP_201_CREATED,
)
def test_health_determination(self): # pylint: disable=too-many-statements
user = self.factory.ensure_user('test', '*****@*****.**', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
device_type = self.factory.make_device_type('beaglebone-black')
device = self.factory.make_device(device_type=device_type,
hostname="black01")
device.save()
filename = os.path.join(os.path.dirname(__file__), 'sample_jobs',
'master-check.json')
self.assertTrue(os.path.exists(filename))
with open(filename, 'r') as json_file:
definition = json_file.read()
# simulate UI submission
job = self.factory.make_testjob(definition=definition, submitter=user)
self.assertFalse(job.health_check)
job.save(update_fields=['health_check', 'requested_device'])
self.assertFalse(job.health_check)
job.delete()
# simulate API submission
job = testjob_submission(definition, user)
self.assertFalse(job.health_check)
self.assertIsNone(job.requested_device)
job.delete()
job = testjob_submission(definition, user, check_device=None)
self.assertFalse(job.health_check)
self.assertIsNone(job.requested_device)
job.delete()
# simulate initiating a health check
job = testjob_submission(definition, user, check_device=device)
self.assertTrue(job.health_check)
self.assertEqual(job.requested_device.hostname, device.hostname)
job.delete()
# modify definition to use the deprecated target support
device2 = self.factory.make_device(device_type=device_type,
hostname="black02")
device2.save()
def_dict = json.loads(definition)
self.assertNotIn('target', def_dict)
def_dict['target'] = device2.hostname
definition = json.dumps(def_dict)
# simulate API submission with target set
job = testjob_submission(definition, user, check_device=None)
self.assertFalse(job.health_check)
self.assertEqual(job.requested_device.hostname, device2.hostname)
job.delete()
# healthcheck designation overrides target (although this is itself an admin error)
job = testjob_submission(definition, user, check_device=device)
self.assertTrue(job.health_check)
self.assertEqual(job.requested_device.hostname, device.hostname)
job.delete()
# check malformed JSON
self.assertRaises(SubmissionException, testjob_submission,
definition[:100], user)
# check non-existent targets
def_dict['target'] = 'nosuchdevice'
definition = json.dumps(def_dict)
self.assertRaises(Device.DoesNotExist, testjob_submission, definition,
user)
# check multinode API submission. bug #2130
filename = os.path.join(os.path.dirname(__file__), 'sample_jobs',
'master-multinode.json')
self.assertTrue(os.path.exists(filename))
with open(filename, 'r') as json_file:
definition = json_file.read()
job_list = testjob_submission(definition, user)
self.assertIsInstance(job_list, list)
for job in job_list:
self.assertIsNotNone(job.vm_group)
self.assertFalse(job.health_check)
if job.requested_device_type == device_type:
self.assertIsNone(job.requested_device)
else:
self.assertIsNotNone(job.requested_device)
self.assertIsInstance(job.requested_device, TemporaryDevice)
job.requested_device.delete()
job.delete()
def test_health_determination(self): # pylint: disable=too-many-statements
user = self.factory.ensure_user('test', '*****@*****.**', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
device_type = self.factory.make_device_type('beaglebone-black')
device = self.factory.make_device(device_type=device_type, hostname="black01")
device.save()
filename = os.path.join(os.path.dirname(__file__), 'master-check.json')
self.assertTrue(os.path.exists(filename))
with open(filename, 'r') as json_file:
definition = json_file.read()
# simulate UI submission
job = self.factory.make_testjob(definition=definition, submitter=user)
self.assertFalse(job.health_check)
job.save(update_fields=['health_check', 'requested_device'])
self.assertFalse(job.health_check)
job.delete()
# simulate API submission
job = testjob_submission(definition, user)
self.assertFalse(job.health_check)
self.assertIsNone(job.requested_device)
job.delete()
job = testjob_submission(definition, user, check_device=None)
self.assertFalse(job.health_check)
self.assertIsNone(job.requested_device)
job.delete()
# simulate initiating a health check
job = testjob_submission(definition, user, check_device=device)
self.assertTrue(job.health_check)
self.assertEqual(job.requested_device.hostname, device.hostname)
job.delete()
# modify definition to use the deprecated target support
device2 = self.factory.make_device(device_type=device_type, hostname="black02")
device2.save()
def_dict = json.loads(definition)
self.assertNotIn('target', def_dict)
def_dict['target'] = device2.hostname
definition = json.dumps(def_dict)
# simulate API submission with target set
job = testjob_submission(definition, user, check_device=None)
self.assertFalse(job.health_check)
self.assertEqual(job.requested_device.hostname, device2.hostname)
job.delete()
# healthcheck designation overrides target (although this is itself an admin error)
job = testjob_submission(definition, user, check_device=device)
self.assertTrue(job.health_check)
self.assertEqual(job.requested_device.hostname, device.hostname)
job.delete()
# check malformed JSON
self.assertRaises(SubmissionException, testjob_submission, definition[:100], user)
# check non-existent targets
def_dict['target'] = 'nosuchdevice'
definition = json.dumps(def_dict)
self.assertRaises(Device.DoesNotExist, testjob_submission, definition, user)
# check multinode API submission. bug #2130
filename = os.path.join(os.path.dirname(__file__), 'master-multinode.json')
self.assertTrue(os.path.exists(filename))
with open(filename, 'r') as json_file:
definition = json_file.read()
job_list = testjob_submission(definition, user)
self.assertIsInstance(job_list, list)
for job in job_list:
self.assertIsNotNone(job.vm_group)
self.assertFalse(job.health_check)
if job.requested_device_type == device_type:
self.assertIsNone(job.requested_device)
else:
self.assertIsNotNone(job.requested_device)
self.assertIsInstance(job.requested_device, TemporaryDevice)
job.requested_device.delete()
job.delete()
