def validate(config, jobs):
for job in jobs:
label = job['name']
validate_schema(
beetmover_cdns_description_schema, job,
"In cdns-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
def validate(config, jobs):
for job in jobs:
label = job['label']
validate_schema(
job_description_schema, job,
"In repackage-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
validate_schema(
checksums_signing_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
def configure_taskdesc_for_run(config, job, taskdesc, worker_implementation):
"""
Run the appropriate function for this job against the given task
description.
This will raise an appropriate error if no function exists, or if the job's
run is not valid according to the schema.
"""
run_using = job['run']['using']
if run_using not in registry:
raise Exception("no functions for run.using {!r}".format(run_using))
if worker_implementation not in registry[run_using]:
raise Exception("no functions for run.using {!r} on {!r}".format(
run_using, worker_implementation))
func, schema, defaults = registry[run_using][worker_implementation]
for k, v in defaults.items():
job['run'].setdefault(k, v)
if schema:
validate_schema(
schema, job['run'],
"In job.run using {!r}/{!r} for job {!r}:".format(
job['run']['using'], worker_implementation, job['label']))
func(config, job, taskdesc)
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
validate_schema(
balrog_description_schema, job,
"In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
for job in jobs:
label = job.get('label', '?no-label?')
validate_schema(
description_schema, job,
"In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
)
yield job
def validate_jobs_schema_transform_partial(config, jobs):
for job in jobs:
label = job.get('label', '?no-label?')
validate_schema(
push_apk_description_schema, job,
"In PushApk ({!r} kind) task for {!r}:".format(config.kind, label)
)
yield job
def validate(config, jobs):
for job in jobs:
label = job['name']
validate_schema(
beetmover_push_to_release_description_schema, job,
"In beetmover-push-to-release ({!r} kind) task for {!r}:".format(
config.kind, label))
yield job
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency',
object).__dict__.get('label', '?no-label?')
validate_schema(
release_generate_checksums_beetmover_schema, job,
"In ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency',
object).__dict__.get('label', '?no-label?')
validate_schema(
signing_description_schema, job,
"In signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
def validate(config, jobs):
for job in jobs:
label = job['label']
validate_schema(
job_description_schema, job,
"In repackage-signing ({!r} kind) task for {!r}:".format(
config.kind, label))
yield job
def validate(config, jobs):
for job in jobs:
label = job['name']
validate_schema(
beetmover_cdns_description_schema, job,
"In cdns-signing ({!r} kind) task for {!r}:".format(
config.kind, label))
yield job
def validate_jobs_schema_transform_partial(description_schema, transform_type,
config, jobs):
for job in jobs:
label = job.get('label', '?no-label?')
validate_schema(
description_schema, job,
"In {} ({!r} kind) task for {!r}:".format(transform_type,
config.kind, label))
yield job
def get_manifest():
manifest_paths = glob.glob(os.path.join(MANIFEST_DIR, "*.yml"))
all_manifests = {}
for path in manifest_paths:
rw_manifest = yaml.load_yaml(path)
manifest_name = os.path.basename(path).replace(".yml", "")
rw_manifest["manifest_name"] = manifest_name
validate_schema(base_schema, deepcopy(rw_manifest), "Invalid manifest:")
check_manifest(deepcopy(rw_manifest))
assert manifest_name not in all_manifests
all_manifests[manifest_name] = rw_manifest
return ReadOnlyDict(all_manifests)
def validate(config, tasks):
for task in tasks:
validate_schema(
task_description_schema,
task,
"In task {!r}:".format(task.get("label", "?no-label?")),
)
validate_schema(
payload_builders[task["worker"]["implementation"]].schema,
task["worker"],
"In task.run {!r}:".format(task.get("label", "?no-label?")),
)
yield task
def get_manifest():
manifest_paths = glob.glob(os.path.join(MANIFEST_DIR, "*.yml"))
all_manifests = {}
graph_config = load_graph_config(ROOT)
for path in manifest_paths:
rw_manifest = yaml.load_yaml(path)
manifest_name = os.path.basename(path).replace(".yml", "")
rw_manifest["manifest_name"] = manifest_name
validate_schema(base_schema, deepcopy(rw_manifest), "Invalid manifest:")
check_manifest(deepcopy(rw_manifest), graph_config)
rw_manifest["artifacts"] = tuple(rw_manifest["artifacts"])
assert manifest_name not in all_manifests
all_manifests[manifest_name] = ReadOnlyDict(rw_manifest)
return ReadOnlyDict(all_manifests)
def test_worker_caches(task, transform):
config, job, taskdesc, impl = transform(task)
add_cache(job, taskdesc, 'cache1', '/cache1')
add_cache(job, taskdesc, 'cache2', '/cache2', skip_untrusted=True)
if impl not in ('docker-worker', 'generic-worker'):
pytest.xfail("caches not implemented for '{}'".format(impl))
key = 'caches' if impl == 'docker-worker' else 'mounts'
assert key in taskdesc['worker']
assert len(taskdesc['worker'][key]) == 2
# Create a new schema object with just the part relevant to caches.
partial_schema = Schema(payload_builders[impl].schema.schema[key])
validate_schema(partial_schema, taskdesc['worker'][key], "validation error")
def test_worker_caches(task, transform):
config, job, taskdesc, impl = transform(task)
add_cache(job, taskdesc, "cache1", "/cache1")
add_cache(job, taskdesc, "cache2", "/cache2", skip_untrusted=True)
if impl not in ("docker-worker", "generic-worker"):
pytest.xfail("caches not implemented for '{}'".format(impl))
key = "caches" if impl == "docker-worker" else "mounts"
assert key in taskdesc["worker"]
assert len(taskdesc["worker"][key]) == 2
# Create a new schema object with just the part relevant to caches.
partial_schema = Schema(payload_builders[impl].schema.schema[key])
validate_schema(partial_schema, taskdesc["worker"][key],
"validation error")
def configure_taskdesc_for_run(config, job, taskdesc):
"""
Run the appropriate function for this job against the given task
description.
This will raise an appropriate error if no function exists, or if the job's
run is not valid according to the schema.
"""
run_using = job['run']['using']
if run_using not in registry:
raise Exception("no functions for run.using {!r}".format(run_using))
worker_implementation = job['worker']['implementation']
if worker_implementation not in registry[run_using]:
raise Exception("no functions for run.using {!r} on {!r}".format(
run_using, worker_implementation))
func, schema = registry[run_using][worker_implementation]
if schema:
job['run'] = validate_schema(
schema, job['run'],
"In job.run using {!r} for job {!r}:".format(
job['run']['using'], job['label']))
func(config, job, taskdesc)
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task',
object).__dict__.get('label', '?no-label?')
yield validate_schema(
checksums_signing_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(
config.kind, label))
def validate(config, tasks):
for task in tasks:
yield validate_schema(
task_description_schema, task,
"In task {!r}:".format(task.get('label', '?no-label?')))
def validate_again(config, jobs):
for job in jobs:
yield validate_schema(l10n_description_schema, job,
"In job {!r}:".format(job.get('name', 'unknown')))
def test_invalid(self):
try:
validate_schema(schema, {'x': 'not-int'}, "pfx")
self.fail("no exception raised")
except Exception, e:
self.failUnless(str(e).startswith("pfx\n"))
def test_valid(self):
validate_schema(schema, {'x': 10, 'y': 'foo'}, "pfx")
def test_invalid(self):
try:
validate_schema(schema, {'x': 'not-int'}, "pfx")
self.fail("no exception raised")
except Exception as e:
self.assertTrue(str(e).startswith("pfx\n"))
def check(self):
schema = base_schema if self.strict else base_schema.extend(
{}, extra=ALLOW_EXTRA)
validate_schema(schema, self.copy(), 'Invalid parameters:')
def validate(config, tests):
for test in tests:
validate_schema(test_description_schema, test,
"In test {!r}:".format(test['test-name']))
yield test
def check(self):
schema = Schema(base_schema,
extra=PREVENT_EXTRA if self.strict else ALLOW_EXTRA)
validate_schema(schema, self.copy(), 'Invalid parameters:')
def validate(config, jobs):
for job in jobs:
validate_schema(source_test_description_schema, job,
"In job {!r}:".format(job['name']))
yield job
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
yield validate_schema(
beetmover_description_schema, job,
"In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
def validate(cron_yml):
validate_schema(cron_yml_schema, cron_yml, "Invalid .cron.yml:")
def validate(config, tasks):
for task in tasks:
validate_schema(
diff_description_schema, task,
"In diff task {!r}:".format(task.get('name', 'unknown')))
yield task
def validate(config, tasks):
for task in tasks:
validate_schema(
docker_image_schema, task,
"In docker image {!r}:".format(task.get('name', 'unknown')))
yield task
def validate(config, jobs):
for job in jobs:
yield validate_schema(source_test_description_schema, job,
"In job {!r}:".format(job['name']))
def validate(config, jobs):
for job in jobs:
yield validate_schema(job_description_schema, job,
"In job {!r}:".format(job['name']))
def validate(config, jobs):
for job in jobs:
validate_schema(job_description_schema, job,
"In job {!r}:".format(job.get('name', job.get('label'))))
yield job
def test_valid(self):
validate_schema(schema, {'x': 10, 'y': 'foo'}, "pfx")
def validate_again(config, jobs):
for job in jobs:
yield validate_schema(
l10n_description_schema, job,
"In job {!r}:".format(job.get('name', 'unknown')))
def validate(config, tests):
for test in tests:
yield validate_schema(test_description_schema, test,
"In test {!r}:".format(test['test-name']))
def validate(config, tasks):
for task in tasks:
validate_schema(
diff_description_schema, task,
"In diff task {!r}:".format(task.get('name', 'unknown')))
yield task
def test_valid(self):
validate_schema(schema, {"x": 10, "y": "foo"}, "pfx")
def validate(config, jobs):
for job in jobs:
yield validate_schema(
job_description_schema, job,
"In job {!r}:".format(job.get('name', job.get('label'))))
def validate(config, tasks):
for task in tasks:
yield validate_schema(
task_description_schema, task,
"In task {!r}:".format(task.get('label', '?no-label?')))
def validate(cron_yml):
validate_schema(cron_yml_schema, cron_yml, "Invalid .cron.yml:")
def validate(config, tasks):
for task in tasks:
validate_schema(
docker_image_schema, task,
"In docker image {!r}:".format(task.get('name', 'unknown')))
yield task
