def validate(self, definition, strict=False):
"""
Name
----
`scheduler.jobs.validate` (`definition`, `strict=False`)
Description
-----------
Validate the given job definition against the schema validator.
Arguments
---------
`definition`: string
Job YAML string.
`strict`: boolean
If set to True, the validator will reject any extra keys that are
present in the job definition but not defined in the schema.
Return value
------------
This function returns None if the job definition is valid. Returns a
dictionary in case of error with the key and msg.
"""
data = yaml_safe_load(definition)
try:
schemas.validate(
data,
strict=strict,
extra_context_variables=settings.EXTRA_CONTEXT_VARIABLES,
)
return {}
except voluptuous.Invalid as exc:
return {"path": str(exc.path), "msg": exc.msg}
def check_health_checks(app_configs, **kwargs):
errors = []
for device in Device.objects.all():
ht = device.get_health_check()
ht_disabled = device.device_type.disable_health_check
# All active devices should have a health check,
# provided that health checks are not disabled for this device type.
if device.health == Device.HEALTH_RETIRED:
continue
if ht is None:
if not ht_disabled:
errors.append(
Warning(
"No health check for '%s'" % device.hostname,
obj="health-checks",
)
)
continue
# check the health-check YAML syntax
try:
data = yaml.safe_load(ht)
except yaml.YAMLError as exc:
errors.append(
Error(
"Invalid YAML syntax for '%s': '%s'" % (device.hostname, exc),
obj="health-checks",
)
)
continue
# check the schema
try:
validate(
data,
strict=False,
extra_context_variables=settings.EXTRA_CONTEXT_VARIABLES,
)
except Invalid as exc:
errors.append(
Error(
"Invalid schema for '%s': '%s'" % (device.hostname, exc),
obj="health-checks",
)
)
try:
validate_job(ht)
except SubmissionException as exc:
errors.append(
Error(
"Invalid schema for '%s': '%s'" % (device.hostname, exc),
obj="health-checks",
)
)
return errors
def check_job(data, options, prefix=""):
data = yaml.safe_load(data)
try:
validate(data, options.strict)
except v.Invalid as exc:
print("%sInvalid job definition:" % prefix)
print("%skey: %s" % (prefix, exc.path))
print("%smgs: %s" % (prefix, exc.msg))
return 1
return 0
def handle_validate(self, newer_than, submitter, strict,
should_mail_admins):
jobs = TestJob.objects.all().order_by("id")
if newer_than is not None:
pattern = re.compile(r"^(?P<time>\d+)(?P<unit>(h|d))$")
match = pattern.match(newer_than)
if match is None:
raise CommandError("Invalid newer-than format")
if match.groupdict()["unit"] == "d":
delta = datetime.timedelta(days=int(match.groupdict()["time"]))
else:
delta = datetime.timedelta(
hours=int(match.groupdict()["time"]))
jobs = jobs.filter(end_time__gt=(timezone.now() - delta))
if submitter is not None:
try:
user = User.objects.get(username=submitter)
except User.DoesNotExist:
raise CommandError("Unable to find submitter '%s'" % submitter)
jobs = jobs.filter(submitter=user)
invalid = {}
for job in jobs:
if job.is_multinode:
definition = job.multinode_definition
else:
definition = job.original_definition
data = yaml.safe_load(definition)
try:
validate(data, strict, settings.EXTRA_CONTEXT_VARIABLES)
print("* %s" % job.id)
except voluptuous.Invalid as exc:
invalid[job.id] = {
"submitter": job.submitter,
"dt": job.requested_device_type,
"key": exc.path,
"msg": exc.msg,
}
print("* %s Invalid job definition" % job.id)
print(" submitter: %s" % job.submitter)
print(" device-type: %s" % job.requested_device_type)
print(" key: %s" % exc.path)
print(" msg: %s" % exc.msg)
if invalid:
if should_mail_admins:
body = "Hello,\n\nthe following jobs schema are invalid:\n"
for job_id in invalid.keys():
body += "* %s\n" % job_id
body += " submitter: {submitter}\n device-type: {dt}\n key: {key}\n msg: {msg}\n".format(
**invalid[job_id])
body += "\n-- \nlava-server manage jobs validate"
mail_admins("Invalid jobs", body)
raise CommandError("Some jobs are invalid")
def validate(self, request, **kwargs):
definition = request.data.get("definition", None)
strict = request.data.get("strict", False)
if not definition:
raise ValidationError({"definition": "Test job definition is required."})
data = yaml_safe_load(definition)
try:
schemas.validate(
data,
strict=strict,
extra_context_variables=settings.EXTRA_CONTEXT_VARIABLES,
)
return Response({"message": "Job valid."}, status=status.HTTP_200_OK)
except voluptuous.Invalid as exc:
return Response(
{"message": "Job invalid: %s" % exc.msg}, status=status.HTTP_200_OK
)
def handle_validate(self, newer_than, submitter, strict):
jobs = TestJob.objects.all().order_by("id")
if newer_than is not None:
pattern = re.compile(r"^(?P<time>\d+)(?P<unit>(h|d))$")
match = pattern.match(newer_than)
if match is None:
raise CommandError("Invalid newer-than format")
if match.groupdict()["unit"] == "d":
delta = datetime.timedelta(days=int(match.groupdict()["time"]))
else:
delta = datetime.timedelta(
hours=int(match.groupdict()["time"]))
jobs = jobs.filter(end_time__gt=(timezone.now() - delta))
if submitter is not None:
try:
user = User.objects.get(username=submitter)
except User.DoesNotExist:
raise CommandError("Unable to find submitter '%s'" % submitter)
jobs = jobs.filter(submitter=user)
invalid = False
for job in jobs:
data = yaml.safe_load(job.original_definition)
try:
validate(data, strict)
print("[%s] Passed " % job.id)
except voluptuous.Invalid as exc:
invalid = True
print("\n[%s] Invalid job definition," % job.id)
print(" submitted by %s" % job.submitter)
print(" for %s:" % job.requested_device_type)
print(" key: %s" % exc.path)
print(" msg: %s" % exc.msg)
if invalid:
raise CommandError("Some jobs are invalid")
