def validate_yaml(yaml_data):
if "notify" in yaml_data:
if "recipients" in yaml_data["notify"]:
for recipient in yaml_data["notify"]["recipients"]:
if recipient["to"][
"method"] == NotificationRecipient.EMAIL_STR:
if "email" not in recipient[
"to"] and "user" not in recipient["to"]:
raise SubmissionException(
"No valid user or email address specified.")
else:
if ("handle" not in recipient["to"]
and "user" not in recipient["to"]):
raise SubmissionException(
"No valid user or IRC handle specified.")
if "user" in recipient["to"]:
try:
User.objects.get(username=recipient["to"]["user"])
except User.DoesNotExist:
raise SubmissionException(
"%r is not an existing user in LAVA." %
recipient["to"]["user"])
elif "email" in recipient["to"]:
try:
validate_email(recipient["to"]["email"])
except ValidationError:
raise SubmissionException(
"%r is not a valid email address." %
recipient["to"]["email"])
if ("compare" in yaml_data["notify"]
and "query" in yaml_data["notify"]["compare"]):
query_yaml_data = yaml_data["notify"]["compare"]["query"]
if "username" in query_yaml_data:
try:
query = Query.objects.get(
owner__username=query_yaml_data["username"],
name=query_yaml_data["name"],
)
if query.content_type.model_class() != TestJob:
raise SubmissionException(
"Only TestJob queries allowed.")
except Query.DoesNotExist:
raise SubmissionException(
"Query ~%s/%s does not exist" %
(query_yaml_data["username"], query_yaml_data["name"]))
else: # Custom query.
if query_yaml_data["entity"] != "testjob":
raise SubmissionException("Only TestJob queries allowed.")
try:
conditions = None
if "conditions" in query_yaml_data:
conditions = query_yaml_data["conditions"]
Query.validate_custom_query(query_yaml_data["entity"],
conditions)
except Exception as e:
raise SubmissionException(e)
def _split_multinode_vland(submission, jobs):
for role, _ in jobs.items():
# populate the lava-vland protocol metadata
if len(jobs[role]) != 1:
raise SubmissionException("vland protocol only supports one device per role.")
jobs[role][0]['protocols'].update({'lava-vland': submission['protocols']['lava-vland'][role]})
return jobs
def validate_job(data):
try:
yaml_data = yaml_safe_load(data)
except yaml.YAMLError as exc:
raise SubmissionException("Loading job submission failed: %s." % exc)
# validate against the submission schema.
validate_submission(yaml_data) # raises SubmissionException if invalid.
validate_yaml(yaml_data) # raises SubmissionException if invalid.
def split_multinode_yaml(submission, target_group):
"""
Handles the lava-multinode protocol requirements.
Uses the multinode protocol requirements to generate as many YAML
snippets as are required to create TestJobs for the multinode submission.
Each multinode YAML submission is only split once for all roles and all sub jobs.
parameters:
submission - the dictionary of the submission YAML.
device_dict - the dictionary mapping device hostnames to roles
target_group - the uuid of the multinode group
return:
None on error or a dictionary of job roles.
key: role
value: list of jobs to be created for that role.
"""
# the list of devices cannot be definite here, only after devices have been reserved
# FIXME: needs a Protocol base class in the server and protocol-specific split handlers
copies = [
"job_name",
"timeouts",
"priority",
"visibility",
"notify",
"metadata",
"reboot_to_fastboot",
]
skip = ["role", "roles"]
scheduling = [
"device_type",
"connection",
"host_role",
"context",
] # top level values to be preserved
maps = ["count"] # elements to be matched but not listed at top level.
roles = {}
actions = {}
subid = 0
# FIXME: check structure using a schema
role_data = submission["protocols"]["lava-multinode"]["roles"]
group_size = sum([
role_data[count]["count"] for count in role_data
if "count" in role_data[count]
])
# populate the lava-multinode protocol metadata
for role, value in submission["protocols"]["lava-multinode"][
"roles"].items():
roles[role] = {}
for item in copies:
if item in submission:
roles[role][item] = submission[item]
for name in maps:
if name in value:
roles[role][name] = value[name]
for name in scheduling:
if name in value:
roles[role][name] = value[name]
tags = set(value) - set(maps) - set(scheduling)
params = {
"target_group": target_group,
"role": role,
"group_size": group_size,
"sub_id": subid,
}
if "essential" in value:
params["essential"] = value
for tag in tags:
params[tag] = value[tag]
roles[role].update({"protocols": {"lava-multinode": params}})
subid += 1
# split the submission based on the roles specified for the actions, retaining order.
for role in roles:
for action in submission["actions"]:
for key, value in action.items():
try:
value["role"]
except (KeyError, TypeError):
raise SubmissionException(
"Invalid YAML - Did not find a role in action '%s', check for consistent use of whitespace indents."
% next(iter(action.keys())))
if role in value["role"]:
actions.setdefault(role, {"actions": []})
actions[role]["actions"].append(
{copy.deepcopy(key): copy.deepcopy(value)})
# add other parameters from the lava-multinode protocol
for key, value in submission["protocols"]["lava-multinode"].items():
if key in skip:
continue
for role in roles:
roles[role]["protocols"]["lava-multinode"][key] = value
# set the role for each action to the role of the job instead of the original list..
for role in actions:
for action in actions[role]["actions"]:
for key, value in action.items():
value["role"] = role
# jobs dictionary lists the jobs per role,
jobs = {}
# check the count of the host_roles
check_count = None
for role in roles:
if "host_role" in roles[role]:
check_count = roles[role]["host_role"]
for role in roles:
if role == check_count:
if roles[role]["count"] != 1:
raise SubmissionException(
"The count for a role designated as a host_role must be 1."
)
sub_id_count = 0
for role in roles:
jobs[role] = []
for sub in range(0, roles[role]["count"]):
job = {}
job.update(actions[role])
job.update(roles[role])
# only here do multiple jobs for the same role differ
params = job["protocols"]["lava-multinode"]
params.update({"sub_id": sub_id_count})
job["protocols"]["lava-multinode"].update(params)
del params
for item in maps:
if item in job:
del job[item]
jobs[role].append(copy.deepcopy(job))
sub_id_count += 1
# populate the lava-vland protocol metadata
if "lava-vland" in submission["protocols"]:
_split_multinode_vland(submission, jobs)
# populate the lava-lxc protocol data
if "lava-lxc" in submission["protocols"]:
for role, _ in jobs.items():
if role not in submission["protocols"]["lava-lxc"]:
continue
# populate the lava-lxc protocol metadata
for job in jobs[role]:
job["protocols"].update(
{"lava-lxc": submission["protocols"]["lava-lxc"][role]})
return jobs
def split_multinode_yaml(submission, target_group): # pylint: disable=too-many-branches,too-many-locals,too-many-statements
"""
Handles the lava-multinode protocol requirements.
Uses the multinode protocol requirements to generate as many YAML
snippets as are required to create TestJobs for the multinode submission.
Each multinode YAML submission is only split once for all roles and all sub jobs.
parameters:
submission - the dictionary of the submission YAML.
device_dict - the dictionary mapping device hostnames to roles
target_group - the uuid of the multinode group
return:
None on error or a dictionary of job roles.
key: role
value: list of jobs to be created for that role.
"""
# the list of devices cannot be definite here, only after devices have been reserved
# FIXME: needs a Protocol base class in the server and protocol-specific split handlers
copies = [
'job_name',
'timeouts',
'priority',
'visibility',
'notify',
'metadata',
'reboot_to_fastboot',
]
skip = ['role', 'roles']
scheduling = ['device_type', 'connection', 'host_role', 'context'] # top level values to be preserved
maps = ['count'] # elements to be matched but not listed at top level.
roles = {}
actions = {}
subid = 0
# FIXME: check structure using a schema
role_data = submission['protocols']['lava-multinode']['roles']
group_size = sum(
[role_data[count]['count'] for count in role_data if 'count' in role_data[count]]
)
# populate the lava-multinode protocol metadata
for role, value in submission['protocols']['lava-multinode']['roles'].items():
roles[role] = {}
for item in copies:
if item in submission:
roles[role][item] = submission[item]
for name in maps:
if name in value:
roles[role][name] = value[name]
for name in scheduling:
if name in value:
roles[role][name] = value[name]
tags = set(value) - set(maps) - set(scheduling)
params = {
'target_group': target_group,
'role': role,
'group_size': group_size,
'sub_id': subid,
}
if 'essential' in value:
params['essential'] = value
for tag in tags:
params[tag] = value[tag]
roles[role].update({'protocols': {'lava-multinode': params}})
subid += 1
# split the submission based on the roles specified for the actions, retaining order.
for role in roles:
for action in submission['actions']:
for key, value in action.items():
try:
value['role']
except (KeyError, TypeError):
raise SubmissionException("Invalid YAML - Did not find a role in action '%s', check for consistent use of whitespace indents." % action.keys()[0])
if role in value['role']:
actions.setdefault(role, {'actions': []})
actions[role]['actions'].append({copy.deepcopy(key): copy.deepcopy(value)})
# add other parameters from the lava-multinode protocol
for key, value in submission['protocols']['lava-multinode'].items():
if key in skip:
continue
for role in roles:
roles[role]['protocols']['lava-multinode'][key] = value
# set the role for each action to the role of the job instead of the original list..
for role in actions:
for action in actions[role]['actions']:
for key, value in action.items():
value['role'] = role
# jobs dictionary lists the jobs per role,
jobs = {}
# check the count of the host_roles
check_count = None
for role in roles:
if 'host_role' in roles[role]:
check_count = roles[role]['host_role']
for role in roles:
if role == check_count:
if roles[role]['count'] != 1:
raise SubmissionException('The count for a role designated as a host_role must be 1.')
sub_id_count = 0
for role in roles:
jobs[role] = []
for sub in range(0, roles[role]['count']):
job = {}
job.update(actions[role])
job.update(roles[role])
# only here do multiple jobs for the same role differ
params = job['protocols']['lava-multinode']
params.update({'sub_id': sub_id_count})
job['protocols']['lava-multinode'].update(params)
del params
for item in maps:
if item in job:
del job[item]
jobs[role].append(copy.deepcopy(job))
sub_id_count += 1
# populate the lava-vland protocol metadata
if 'lava-vland' in submission['protocols']:
_split_multinode_vland(submission, jobs)
# populate the lava-lxc protocol data
if 'lava-lxc' in submission['protocols']:
for role, _ in jobs.items():
if role not in submission['protocols']['lava-lxc']:
continue
# populate the lava-lxc protocol metadata
for job in jobs[role]:
job['protocols'].update({'lava-lxc': submission['protocols']['lava-lxc'][role]})
return jobs