def test_from_json_and_user_reuses_tag_objects(self):
device_type = self.factory.ensure_device_type(name='panda')
self.factory.ensure_tag('tag')
tags = list(Tag.objects.filter(name='tag'))
self.factory.make_device(device_type=device_type,
hostname="panda3",
tags=tags)
job1 = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']), self.factory.make_user())
job2 = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']), self.factory.make_user())
self.assertEqual(set(tag.pk for tag in job1.tags.all()),
set(tag.pk for tag in job2.tags.all()))
def test_from_json_and_user_reuses_tag_objects(self):
device_type = self.factory.ensure_device_type(name='panda')
self.factory.ensure_tag('tag')
tags = list(Tag.objects.filter(name='tag'))
self.factory.make_device(device_type=device_type, hostname="panda3", tags=tags)
job1 = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']),
self.factory.make_user())
job2 = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']),
self.factory.make_user())
self.assertEqual(
set(tag.pk for tag in job1.tags.all()),
set(tag.pk for tag in job2.tags.all()))
def test_from_json_and_user_sets_target(self):
panda_board = self.factory.make_device(hostname='panda01')
job = TestJob.from_json_and_user(
self.factory.make_job_json(target='panda01'),
self.factory.make_user())
self.assertEqual(panda_board, job.requested_device)
self.factory.cleanup()
def test_from_json_and_user_repeat_parameter_zero(self):
device_type = self.factory.make_device_type('base')
device = self.factory.make_device(device_type=device_type, hostname="generic")
repeat = 0
job_data = {
'timeout': 1,
'target': device.hostname,
'actions': [
{
'command': 'lava_test_shell',
'parameters': {
'repeat': repeat,
'testdef_repos': [
{
'git-repo': 'git://server/test.git',
'testdef': 'testdef.yaml'
}
],
}
}
],
}
job_json = simplejson.dumps(job_data, sort_keys=True, indent=4 * ' ')
job = TestJob.from_json_and_user(job_json, self.factory.make_user())
definition_data = simplejson.loads(job.definition)
self.assertEqual(len(definition_data['actions']), 1)
self.assertNotIn('repeat_count', definition_data['actions'][0]['parameters'])
self.assertNotIn('repeat', definition_data['actions'][0]['parameters'])
self.assertEqual(job.status, TestJob.SUBMITTED)
self.factory.cleanup()
def test_from_json_and_user_matches_available_tags(self):
"""
Test that with more than one device of the requested type supporting
tags, that the tag list set for the TestJob matches the list requested,
not a shorter list from a different device or a combined list of multiple
devices.
"""
device_type = self.factory.ensure_device_type(name='panda')
tag_list = [
self.factory.ensure_tag('common_tag1'),
self.factory.ensure_tag('common_tag2')
]
self.factory.make_device(device_type=device_type,
hostname="panda4",
tags=tag_list)
tag_list.append(self.factory.ensure_tag('unique_tag'))
self.factory.make_device(device_type=device_type,
hostname="panda5",
tags=tag_list)
job = TestJob.from_json_and_user(
self.factory.make_job_json(
tags=['common_tag1', 'common_tag2', 'unique_tag']),
self.factory.make_user())
self.assertEqual(set(tag for tag in job.tags.all()), set(tag_list))
self.factory.cleanup()
def test_from_json_and_user_sets_device_type(self):
panda_type = self.factory.ensure_device_type(name='panda')
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
self.factory.cleanup()
def test_from_json_and_user_sets_date_submitted(self):
before = datetime.datetime.now()
job = TestJob.from_json_and_user(
self.factory.make_job_json(),
self.factory.make_user())
after = datetime.datetime.now()
self.assertTrue(before < job.submit_time < after)
def test_from_json_and_user_repeat_parameter_zero(self):
device_type = self.factory.make_device_type('base')
device = self.factory.make_device(device_type=device_type,
hostname="generic")
repeat = 0
job_data = {
'timeout':
1,
'target':
device.hostname,
'actions': [{
'command': 'lava_test_shell',
'parameters': {
'repeat':
repeat,
'testdef_repos': [{
'git-repo': 'git://server/test.git',
'testdef': 'testdef.yaml'
}],
}
}],
}
job_json = simplejson.dumps(job_data, sort_keys=True, indent=4 * ' ')
job = TestJob.from_json_and_user(job_json, self.factory.make_user())
definition_data = simplejson.loads(job.definition)
self.assertEqual(len(definition_data['actions']), 1)
self.assertNotIn('repeat_count',
definition_data['actions'][0]['parameters'])
self.assertNotIn('repeat', definition_data['actions'][0]['parameters'])
self.assertEqual(job.status, TestJob.SUBMITTED)
self.factory.cleanup()
def test_from_json_and_user_sets_device_type(self):
panda_type = self.factory.ensure_device_type(name='panda')
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
self.factory.cleanup()
def test_from_json_and_user_sets_target(self):
panda_board = self.factory.make_device(hostname='panda01')
job = TestJob.from_json_and_user(
self.factory.make_job_json(target='panda01'),
self.factory.make_user())
self.assertEqual(panda_board, job.requested_device)
self.factory.cleanup()
def test_from_json_and_user_does_not_set_device_type_from_target(self):
panda_type = self.factory.ensure_device_type(name='panda')
self.factory.make_device(device_type=panda_type, hostname='panda01')
job = TestJob.from_json_and_user(
self.factory.make_job_json(target='panda01'),
self.factory.make_user())
self.assertEqual(None, job.requested_device_type)
def test_from_json_and_user_sets_outputdir(self):
definition = self.factory.make_job_json()
job = TestJob.from_json_and_user(definition, self.factory.make_user())
dir_list = job.output_dir.split('/')
self.assertIn("%02d" % job.submit_time.year, dir_list)
self.assertIn("%02d" % job.submit_time.month, dir_list)
self.assertIn("%02d" % job.submit_time.day, dir_list)
self.assertIn(str(job.id), dir_list)
def test_from_json_and_user_errors_on_unsupported_tags(self):
"""
Tests that tags which do exist but are not defined for the
any of the devices of the requested type cause the submission
to be rejected with Devices Unavailable.
"""
device_type = self.factory.ensure_device_type(name='panda')
self.factory.make_device(device_type=device_type, hostname="panda2")
self.factory.ensure_tag('tag1')
self.factory.ensure_tag('tag2')
try:
TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag1', 'tag2']),
self.factory.make_user())
except DevicesUnavailableException:
pass
else:
self.fail("Device tags failure: job submitted without any devices supporting the requested tags")
def test_from_json_and_user_can_submit_to_anonymous(self):
user = self.factory.make_user()
anon_user = User.objects.get_or_create(username="******")[0]
b = BundleStream.objects.create(
slug='anonymous', is_anonymous=True, user=anon_user,
is_public=True)
b.save()
j = self.make_job_json_for_stream_name('/anonymous/anonymous/')
job = TestJob.from_json_and_user(j, user)
self.assertEqual(user, job.submitter)
def make_testjob(self, definition=None, submitter=None, **kwargs):
if definition is None:
definition = self.make_job_json()
if submitter is None:
submitter = self.make_user()
if 'user' not in kwargs:
kwargs['user'] = submitter
testjob = TestJob.from_json_and_user(definition, submitter)
testjob.save()
return testjob
def test_from_json_and_user_sets_is_public_from_bundlestream(self):
group = Group.objects.create(name='group')
user = self.factory.make_user()
user.groups.add(group)
b = BundleStream.objects.create(
group=group, slug='blah', is_public=False, is_anonymous=False)
b.save()
j = self.make_job_json_for_stream_name(b.pathname)
job = TestJob.from_json_and_user(j, user)
self.assertEqual(False, job.is_public)
def testjob_submission(job_definition, user, check_device=None, original_job=None):
"""
Single submission frontend for JSON or YAML
:param job_definition: string of the job submission
:param user: user attempting the submission
:param check_device: set specified device as the target
**and** thereby set job as a health check job. (JSON only)
:return: a job or a list of jobs
:raises: SubmissionException, Device.DoesNotExist,
DeviceType.DoesNotExist, DevicesUnavailableException,
JSONDataError, JSONDecodeError, ValueError
"""
if is_deprecated_json(job_definition):
allow_health = False
job_json = simplejson.loads(job_definition)
target_device = None
if 'target' in job_json:
target_device = Device.objects.get(hostname=job_json['target'])
if check_device:
job_json['target'] = check_device.hostname
job_json['health-check'] = True
job_definition = simplejson.dumps(job_json)
allow_health = True
try:
# returns a single job or a list (not a QuerySet) of job objects.
job = TestJob.from_json_and_user(job_definition, user, health_check=allow_health)
if isinstance(job, list):
# multinode health checks not supported
return job
job.health_check = allow_health
if check_device:
job.requested_device = check_device
elif target_device:
job.requested_device = target_device
job.save(update_fields=['health_check', 'requested_device'])
except (JSONDataError, ValueError) as exc:
if check_device:
check_device.put_into_maintenance_mode(
user, "Job submission failed for health job for %s: %s" % (check_device, exc))
raise JSONDataError("Health check job submission failed for %s: %s" % (check_device, exc))
else:
raise JSONDataError("Job submission failed: %s" % exc)
else:
validate_job(job_definition)
# returns a single job or a list (not a QuerySet) of job objects.
job = TestJob.from_yaml_and_user(job_definition, user, original_job=original_job)
if check_device and isinstance(check_device, Device) and not isinstance(job, list):
# the slave must neither know nor care if this is a health check,
# only the master cares and that has the database connection.
job.health_check = True
job.requested_device = check_device
job.save(update_fields=['health_check', 'requested_device'])
return job
def testjob_submission(job_definition, user, check_device=None):
"""
Single submission frontend for JSON or YAML
:param job_definition: string of the job submission
:param user: user attempting the submission
:param check_device: set specified device as the target
**and** thereby set job as a health check job. (JSON only)
:return: a job or a list of jobs
:raises: SubmissionException, Device.DoesNotExist,
DeviceType.DoesNotExist, DevicesUnavailableException,
JSONDataError, JSONDecodeError, ValueError
"""
if is_deprecated_json(job_definition):
allow_health = False
job_json = simplejson.loads(job_definition)
target_device = None
if 'target' in job_json:
target_device = Device.objects.get(hostname=job_json['target'])
if check_device:
job_json['target'] = check_device.hostname
job_json['health-check'] = True
job_definition = simplejson.dumps(job_json)
allow_health = True
try:
# returns a single job or a list (not a QuerySet) of job objects.
job = TestJob.from_json_and_user(job_definition, user, health_check=allow_health)
if isinstance(job, list):
# multinode health checks not supported
return job
job.health_check = allow_health
if check_device:
job.requested_device = check_device
elif target_device:
job.requested_device = target_device
job.save(update_fields=['health_check', 'requested_device'])
except (JSONDataError, ValueError) as exc:
if check_device:
check_device.put_into_maintenance_mode(
user, "Job submission failed for health job for %s: %s" % (check_device, exc))
raise JSONDataError("Health check job submission failed for %s: %s" % (check_device, exc))
else:
raise JSONDataError("Job submission failed: %s" % exc)
else:
validate_job(job_definition)
# returns a single job or a list (not a QuerySet) of job objects.
job = TestJob.from_yaml_and_user(job_definition, user)
if check_device and isinstance(check_device, Device) and not isinstance(job, list):
# the slave must neither know nor care if this is a health check,
# only the master cares and that has the database connection.
job.health_check = True
job.requested_device = check_device
job.save(update_fields=['health_check', 'requested_device'])
return job
def test_from_json_and_user_sets_tag_from_device_tags(self):
device_type = self.factory.ensure_device_type(name='panda')
self.factory.ensure_tag('tag')
tags = list(Tag.objects.filter(name='tag'))
self.factory.make_device(device_type=device_type, hostname="panda1", tags=tags)
job = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']),
self.factory.make_user())
self.assertEqual(
set(tag.name for tag in job.tags.all()), {'tag'})
self.factory.cleanup()
def test_from_json_and_user_sets_tag_from_device_tags(self):
device_type = self.factory.ensure_device_type(name='panda')
self.factory.ensure_tag('tag')
tags = list(Tag.objects.filter(name='tag'))
self.factory.make_device(device_type=device_type,
hostname="panda1",
tags=tags)
job = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']), self.factory.make_user())
self.assertEqual(set(tag.name for tag in job.tags.all()), {'tag'})
self.factory.cleanup()
def test_from_json_and_user_sets_multiple_tag_from_device_tags(self):
device_type = self.factory.ensure_device_type(name='panda')
tag_list = [
self.factory.ensure_tag('tag1'),
self.factory.ensure_tag('tag2')
]
self.factory.make_device(device_type=device_type, hostname="panda2", tags=tag_list)
job = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag1', 'tag2']),
self.factory.make_user())
self.assertEqual(
set(tag.name for tag in job.tags.all()), {'tag1', 'tag2'})
def test_from_json_rejects_exclusive(self):
panda_type = self.factory.ensure_device_type(name='panda')
panda_board = self.factory.make_device(device_type=panda_type, hostname='panda03')
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
self.assertTrue(panda_board.is_exclusive)
self.assertRaises(
DevicesUnavailableException, _check_exclusivity, [panda_board], pipeline=False
)
self.factory.cleanup()
def test_from_json_and_user_can_submit_to_group_stream(self):
user = self.factory.make_user()
anon_user = User.objects.get_or_create(username="******")[0]
group = Group.objects.get_or_create(name="owner")[0]
group.user_set.add(user)
b = BundleStream.objects.create(slug='basic',
is_anonymous=False,
group=group,
is_public=True)
b.save()
self.assertEqual(b.pathname, "/public/team/owner/basic/")
j = self.make_job_json_for_stream_name(b.pathname)
job = TestJob.from_json_and_user(j, user)
self.assertEqual(user, job.submitter)
self.assertEqual(True, job.is_public)
self.assertRaises(ValueError, TestJob.from_json_and_user, j, anon_user)
def submit_job(self, job_data):
"""
Name
----
`submit_job` (`job_data`)
Description
-----------
Submit the given job data which is in LAVA job JSON format as a new
job to LAVA scheduler.
Arguments
---------
`job_data`: string
Job JSON string.
Return value
------------
This function returns an XML-RPC integer which is the newly created
job's id, provided the user is authenticated with an username and
token.
"""
if not self.user:
raise xmlrpclib.Fault(
401, "Authentication with user and token required for this "
"API.")
if not self.user.has_perm('lava_scheduler_app.add_testjob'):
raise xmlrpclib.Fault(
403, "Permission denied. User %r does not have the "
"'lava_scheduler_app.add_testjob' permission. Contact "
"the administrators." % self.user.username)
try:
job = TestJob.from_json_and_user(job_data, self.user)
except JSONDecodeError as e:
raise xmlrpclib.Fault(400, "Decoding JSON failed: %s." % e)
except (JSONDataError, ValueError) as e:
raise xmlrpclib.Fault(400, str(e))
except Device.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device not found.")
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device type not found.")
except DevicesUnavailableException as e:
raise xmlrpclib.Fault(400, str(e))
if isinstance(job, type(list())):
return [j.sub_id for j in job]
else:
return job.id
def submit_job(self, job_data):
"""
Name
----
`submit_job` (`job_data`)
Description
-----------
Submit the given job data which is in LAVA job JSON format as a new
job to LAVA scheduler.
Arguments
---------
`job_data`: string
Job JSON string.
Return value
------------
This function returns an XML-RPC integer which is the newly created
job's id, provided the user is authenticated with an username and
token.
"""
if not self.user:
raise xmlrpclib.Fault(
401, "Authentication with user and token required for this "
"API.")
if not self.user.has_perm('lava_scheduler_app.add_testjob'):
raise xmlrpclib.Fault(
403, "Permission denied. User %r does not have the "
"'lava_scheduler_app.add_testjob' permission. Contact "
"the administrators." % self.user.username)
try:
job = TestJob.from_json_and_user(job_data, self.user)
except JSONDecodeError as e:
raise xmlrpclib.Fault(400, "Decoding JSON failed: %s." % e)
except (JSONDataError, ValueError) as e:
raise xmlrpclib.Fault(400, str(e))
except Device.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device not found.")
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device type not found.")
except DevicesUnavailableException as e:
raise xmlrpclib.Fault(400, str(e))
if isinstance(job, type(list())):
return [j.sub_id for j in job]
else:
return job.id
def test_from_json_and_user_can_submit_to_group_stream(self):
user = self.factory.make_user()
anon_user = User.objects.get_or_create(username="******")[0]
group = Group.objects.get_or_create(name="owner")[0]
group.user_set.add(user)
b = BundleStream.objects.create(
slug='basic',
is_anonymous=False,
group=group,
is_public=True)
b.save()
self.assertEqual(b.pathname, "/public/team/owner/basic/")
j = self.make_job_json_for_stream_name(b.pathname)
job = TestJob.from_json_and_user(j, user)
self.assertEqual(user, job.submitter)
self.assertEqual(True, job.is_public)
self.assertRaises(ValueError, TestJob.from_json_and_user, j, anon_user)
def test_from_json_rejects_exclusive(self):
panda_type = self.factory.ensure_device_type(name='panda')
panda_board = self.factory.make_device(device_type=panda_type, hostname='panda01')
self.assertFalse(panda_board.is_exclusive)
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
device_dict = DeviceDictionary.get(panda_board.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=panda_board.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(panda_board.is_exclusive)
self.assertRaises(
DevicesUnavailableException, _check_exclusivity, [panda_board], pipeline=False
)
def test_from_json_rejects_exclusive(self):
panda_type = self.factory.ensure_device_type(name='panda')
panda_board = self.factory.make_device(device_type=panda_type,
hostname='panda01')
self.assertFalse(panda_board.is_exclusive)
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
device_dict = DeviceDictionary.get(panda_board.hostname)
self.assertIsNone(device_dict)
device_dict = DeviceDictionary(hostname=panda_board.hostname)
device_dict.parameters = {'exclusive': 'True'}
device_dict.save()
self.assertTrue(panda_board.is_exclusive)
self.assertRaises(DevicesUnavailableException,
_check_exclusivity, [panda_board],
pipeline=False)
def test_device_type_with_target(self):
"""
See https://bugs.launchpad.net/lava-server/+bug/1318579
Check that a submission with device_type and target
results in the device_type being dropped.
"""
user = self.factory.make_user()
group = self.factory.make_group()
group.user_set.add(user)
b = BundleStream.objects.create(slug='hidden',
is_anonymous=False,
group=group,
is_public=True)
b.save()
device_type = self.factory.make_device_type('base')
device = self.factory.make_device(device_type=device_type,
hostname="generic")
job_data = {
"device_type":
"broken",
"target":
device.hostname,
"timeout":
1,
"health_check":
False,
'actions': [{
'command': 'submit_results',
'parameters': {
'server': 'http://localhost/RPC2',
'stream': b.pathname,
}
}],
}
job_json = simplejson.dumps(job_data, sort_keys=True, indent=4 * ' ')
job = TestJob.from_json_and_user(job_json, user)
self.assertNotIn("device_type", job.definition)
self.assertIn('device_type', job_data)
definition_data = simplejson.loads(job.definition)
self.assertEqual(definition_data['target'], job_data['target'])
self.assertEqual(definition_data['timeout'], job_data['timeout'])
self.assertEqual(definition_data['health_check'],
job_data['health_check'])
self.factory.cleanup()
def test_hidden_submitted_job_is_hidden(self):
user = self.factory.make_user()
anon_user = User.objects.get_or_create(username="******")[0]
device_type = self.factory.make_hidden_device_type('hide_me_now')
device = self.factory.make_device(device_type=device_type, hostname="hidden1")
device.user = user
device.is_public = False
device.save()
b = BundleStream.objects.create(
slug='hidden', is_anonymous=False, user=user,
is_public=False)
b.save()
self.assertEqual(b.is_public, False)
j = self.make_job_json_for_stream_name('/private/personal/generic-1/hidden/', target='hidden1')
job = TestJob.from_json_and_user(j, user)
self.assertEqual(job.user, device.user)
self.assertEqual(job.is_public, False)
self.assertEqual(device.is_public, False)
self.assertRaises(DevicesUnavailableException, TestJob.from_json_and_user, j, anon_user)
def test_from_json_and_user_matches_available_tags(self):
"""
Test that with more than one device of the requested type supporting
tags, that the tag list set for the TestJob matches the list requested,
not a shorter list from a different device or a combined list of multiple
devices.
"""
device_type = self.factory.ensure_device_type(name='panda')
tag_list = [
self.factory.ensure_tag('common_tag1'),
self.factory.ensure_tag('common_tag2')
]
self.factory.make_device(device_type=device_type, hostname="panda4", tags=tag_list)
tag_list.append(self.factory.ensure_tag('unique_tag'))
self.factory.make_device(device_type=device_type, hostname="panda5", tags=tag_list)
job = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['common_tag1', 'common_tag2', 'unique_tag']),
self.factory.make_user())
self.assertEqual(
set(tag for tag in job.tags.all()),
set(tag_list)
)
def test_device_type_with_target(self):
"""
See https://bugs.launchpad.net/lava-server/+bug/1318579
Check that a submission with device_type and target
results in the device_type being dropped.
"""
user = self.factory.make_user()
group = self.factory.make_group()
group.user_set.add(user)
b = BundleStream.objects.create(
slug='hidden', is_anonymous=False, group=group,
is_public=True)
b.save()
device_type = self.factory.make_device_type('base')
device = self.factory.make_device(device_type=device_type, hostname="generic")
job_data = {
"device_type": "broken",
"target": device.hostname,
"timeout": 1,
"health_check": False,
'actions': [
{
'command': 'submit_results',
'parameters': {
'server': 'http://localhost/RPC2',
'stream': b.pathname,
}
}
],
}
job_json = simplejson.dumps(job_data, sort_keys=True, indent=4 * ' ')
job = TestJob.from_json_and_user(job_json, user)
self.assertNotIn("device_type", job.definition)
self.assertIn('device_type', job_data)
definition_data = simplejson.loads(job.definition)
self.assertEqual(definition_data['target'], job_data['target'])
self.assertEqual(definition_data['timeout'], job_data['timeout'])
self.assertEqual(definition_data['health_check'], job_data['health_check'])
self.factory.cleanup()
def submit_job(self, job_data):
"""
Name
----
`submit_job` (`job_data`)
Description
-----------
Submit the given job data which is in LAVA job JSON format as a new
job to LAVA scheduler.
Arguments
---------
`job_data`: string
Job JSON string.
Return value
------------
This function returns an XML-RPC integer which is the newly created
job's id, provided the user is authenticated with an username and
token.
"""
self._authenticate()
if not self.user.has_perm('lava_scheduler_app.add_testjob'):
raise xmlrpclib.Fault(
403, "Permission denied. User %r does not have the "
"'lava_scheduler_app.add_testjob' permission. Contact "
"the administrators." % self.user.username)
is_json = True
is_yaml = False
try:
json.loads(job_data)
except (AttributeError, JSONDecodeError, ValueError) as exc:
is_json = False
try:
# only try YAML if this is not JSON
# YAML can parse JSON as YAML, JSON cannot parse YAML at all
yaml_data = yaml.load(job_data)
except yaml.YAMLError as exc:
# neither yaml nor json loaders were able to process the submission.
raise xmlrpclib.Fault(
400, "Loading job submission failed: %s." % exc)
# validate against the submission schema.
is_yaml = validate_submission(
yaml_data) # raises SubmissionException if invalid.
try:
if is_json:
job = TestJob.from_json_and_user(job_data, self.user)
elif is_yaml:
job = TestJob.from_yaml_and_user(job_data, self.user)
else:
raise xmlrpclib.Fault(
400, "Unable to determine whether job is JSON or YAML.")
except (JSONDataError, JSONDecodeError, ValueError) as exc:
raise xmlrpclib.Fault(400,
"Decoding job submission failed: %s." % exc)
except Device.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device not found.")
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device type not found.")
except DevicesUnavailableException as e:
raise xmlrpclib.Fault(400, str(e))
if isinstance(job, type(list())):
return [j.sub_id for j in job]
else:
return job.id
def test_restricted_submitted_job_with_group_bundle_and_multinode(self):
"""
Need to expand this into a MultiNode test class / file with factory
functions and add the rest of the MultiNode tests.
"""
superuser = self.factory.make_user()
superuser.is_superuser = True
user = self.factory.make_user()
anon_user = User.objects.get_or_create(username="******")[0]
group = Group.objects.get_or_create(name="owner")[0]
group.user_set.add(user)
device_type1 = self.factory.make_device_type('hide_me_now')
device_type2 = self.factory.make_device_type('secretive')
device1 = self.factory.make_device(device_type=device_type1, hostname="hidden1", group=group, is_public=False)
device1.save()
device2 = self.factory.make_device(device_type=device_type2, hostname="terces", group=group, is_public=False)
device2.save()
self.assertEqual(device1.is_public, False)
self.assertEqual(device2.is_public, False)
b = BundleStream.objects.create(
slug='hidden', is_anonymous=False, group=group,
is_public=True)
b.save()
self.assertEqual(b.is_public, True)
self.assertEqual(b.user, None)
self.assertEqual(b.pathname, '/public/team/owner/hidden/')
# singlenode must pass
j = self.make_job_json_for_stream_name(b.pathname, target="hidden1")
self.assertRaises(DevicesUnavailableException, TestJob.from_json_and_user, j, anon_user)
job = TestJob.from_json_and_user(j, user)
self.assertEqual(job.user, device1.user)
self.assertEqual(job.group, device1.group)
self.assertEqual(job.user, device2.user)
self.assertEqual(job.group, device2.group)
self.assertEqual(job.is_public, True)
self.assertEqual(device1.is_public, False)
# multinode must pass
job_data = {
'actions': [
{
'command': 'submit_results',
'parameters': {
'server': 'http://localhost/RPC2',
'stream': b.pathname,
}
}
],
'device_group': [
{
"role": "felix",
"count": 1,
"device_type": device_type1.name
},
{
"role": "rex",
"count": 1,
"device_type": device_type2.name
}
],
}
job_data.update({'timeout': 1, 'health_check': False})
job_json = simplejson.dumps(job_data, sort_keys=True, indent=4 * ' ')
job = TestJob.from_json_and_user(job_json, user)
self.assertEqual(len(job), 2)
self.assertEqual(job[0].is_public, True)
self.assertEqual(job[1].is_public, True)
def submit_job(self, job_data):
"""
Name
----
`submit_job` (`job_data`)
Description
-----------
Submit the given job data which is in LAVA job JSON format as a new
job to LAVA scheduler.
Arguments
---------
`job_data`: string
Job JSON string.
Return value
------------
This function returns an XML-RPC integer which is the newly created
job's id, provided the user is authenticated with an username and
token.
"""
self._authenticate()
if not self.user.has_perm('lava_scheduler_app.add_testjob'):
raise xmlrpclib.Fault(
403, "Permission denied. User %r does not have the "
"'lava_scheduler_app.add_testjob' permission. Contact "
"the administrators." % self.user.username)
is_json = True
is_yaml = False
try:
json.loads(job_data)
except (AttributeError, JSONDecodeError, ValueError) as exc:
is_json = False
try:
# only try YAML if this is not JSON
# YAML can parse JSON as YAML, JSON cannot parse YAML at all
yaml_data = yaml.load(job_data)
except yaml.YAMLError:
raise xmlrpclib.Fault(400, "Decoding job submission failed: %s." % exc)
if type(yaml_data) is not dict or 'actions' not in yaml_data:
raise xmlrpclib.Fault(400, "Decoding job submission failed.")
actions = [item for item in yaml_data['actions']]
# pipeline jobs only accept deploy, boot or test actions
# but only have to include one of the possible actions.
is_yaml = any(
[True for item in actions
if 'deploy' in item or 'boot' in item or 'test' in item])
try:
if is_json:
job = TestJob.from_json_and_user(job_data, self.user)
elif is_yaml:
job = TestJob.from_yaml_and_user(job_data, self.user)
else:
raise xmlrpclib.Fault(400, "Decoding job submission failed")
except (JSONDataError, JSONDecodeError, ValueError) as exc:
raise xmlrpclib.Fault(400, "Decoding job submission failed: %s." % exc)
except Device.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device not found.")
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device type not found.")
except DevicesUnavailableException as e:
raise xmlrpclib.Fault(400, str(e))
if isinstance(job, type(list())):
return [j.sub_id for j in job]
else:
return job.id
def submit_job(self, **kw):
job_definition = self.factory.make_job_json(**kw)
return TestJob.from_json_and_user(job_definition, self.user)
def submit_job(self, job_data):
"""
Name
----
`submit_job` (`job_data`)
Description
-----------
Submit the given job data which is in LAVA job JSON format as a new
job to LAVA scheduler.
Arguments
---------
`job_data`: string
Job JSON string.
Return value
------------
This function returns an XML-RPC integer which is the newly created
job's id, provided the user is authenticated with an username and
token.
"""
self._authenticate()
if not self.user.has_perm('lava_scheduler_app.add_testjob'):
raise xmlrpclib.Fault(
403, "Permission denied. User %r does not have the "
"'lava_scheduler_app.add_testjob' permission. Contact "
"the administrators." % self.user.username)
is_json = True
is_yaml = False
try:
json.loads(job_data)
except (AttributeError, JSONDecodeError, ValueError) as exc:
is_json = False
try:
# only try YAML if this is not JSON
# YAML can parse JSON as YAML, JSON cannot parse YAML at all
yaml_data = yaml.load(job_data)
except yaml.YAMLError as exc:
# neither yaml nor json loaders were able to process the submission.
raise xmlrpclib.Fault(400, "Loading job submission failed: %s." % exc)
# validate against the submission schema.
is_yaml = validate_submission(yaml_data) # raises SubmissionException if invalid.
try:
if is_json:
job = TestJob.from_json_and_user(job_data, self.user)
elif is_yaml:
job = TestJob.from_yaml_and_user(job_data, self.user)
else:
raise xmlrpclib.Fault(400, "Unable to determine whether job is JSON or YAML.")
except (JSONDataError, JSONDecodeError, ValueError) as exc:
raise xmlrpclib.Fault(400, "Decoding job submission failed: %s." % exc)
except Device.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device not found.")
except DeviceType.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device type not found.")
except DevicesUnavailableException as exc:
raise xmlrpclib.Fault(400, str(exc))
if isinstance(job, type(list())):
return [j.sub_id for j in job]
else:
return job.id
def test_from_json_and_user_sets_no_tags_if_no_tags(self):
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_tags=[]),
self.factory.make_user())
self.assertEqual(set(job.tags.all()), set([]))
def test_from_json_and_user_sets_status_to_SUBMITTED(self):
job = TestJob.from_json_and_user(
self.factory.make_job_json(),
self.factory.make_user())
self.assertEqual(job.status, TestJob.SUBMITTED)
def submit_job(self, **kw):
job_definition = self.factory.make_job_json(**kw)
return TestJob.from_json_and_user(job_definition, self.user)
def test_from_json_and_user_sets_submitter(self):
user = self.factory.make_user()
job = TestJob.from_json_and_user(
self.factory.make_job_json(), user)
self.assertEqual(user, job.submitter)
def test_from_json_and_user_sets_definition(self):
definition = self.factory.make_job_json()
job = TestJob.from_json_and_user(definition, self.factory.make_user())
self.assertEqual(definition, job.definition)
