def _generate_job_ids2(depends_on, pjob_id, cparsed_template, ctemplate,
group_name, child):
so_far = set()
autofill_values = get_autofill_values(child, raise_err=False)
job_id_data = prep_crossproduct(cparsed_template, depends_on,
autofill_values, pjob_id)
# Build job_ids using crossproduct of all components
for job_id_data in crossproduct(job_id_data):
cjob_id = ctemplate.format(dependency_group_name=group_name,
**dict(zip(cparsed_template, job_id_data)))
if cjob_id not in so_far:
so_far.add(cjob_id)
yield (child, cjob_id)
def _generate_job_ids2(depends_on, pjob_id,
cparsed_template, ctemplate, group_name, child):
so_far = set()
autofill_values = get_autofill_values(child, raise_err=False)
job_id_data = prep_crossproduct(
cparsed_template, depends_on, autofill_values, pjob_id)
# Build job_ids using crossproduct of all components
for job_id_data in crossproduct(job_id_data):
cjob_id = ctemplate.format(
dependency_group_name=group_name,
**dict(zip(cparsed_template, job_id_data)))
if cjob_id not in so_far:
so_far.add(cjob_id)
yield (child, cjob_id)
def _get_parent_job_ids(group_name, depends_on, child_app_name, child_job_id,
ld):
"""
Yield the parent app_name and derived job_id for each parent listed in
depends_on metadata
If there is extra job_id criteria that doesn't apply to a
particular parent app's job_id template, ignore it.
"""
for parent_app_name in depends_on['app_name']:
depends_on = dict(depends_on) # shallow copy to change the keys
_inplace_modify_depends_on(depends_on, child_app_name, child_job_id,
parent_app_name, ld)
# are there specific job_ids the child would inherit from?
if 'job_id' in depends_on:
for rv in _iter_job_ids(dep_group=depends_on,
group_name=group_name,
parent_app_name=parent_app_name,
ld=ld):
yield rv
else:
# try to fill in the parent's job_id template and yield it
template, parsed_template = get_job_id_template(parent_app_name)
so_far = set()
_, cparsed_template = get_job_id_template(child_app_name)
autofill_values = get_autofill_values(child_app_name,
raise_err=False)
pjob_id = parse_job_id(child_app_name, child_job_id)
job_id_data = prep_crossproduct(cparsed_template, depends_on,
autofill_values, pjob_id)
for job_id_data in crossproduct(
[depends_on[_key] for _key in parsed_template]):
_pjob_id = dict(zip(parsed_template, job_id_data))
parent_job_id = template.format(
dependency_group_name=group_name, **_pjob_id)
if parent_job_id not in so_far:
so_far.add(parent_job_id)
yield (parent_app_name, parent_job_id)
def _get_parent_job_ids(group_name, depends_on,
child_app_name, child_job_id, ld):
"""
Yield the parent app_name and derived job_id for each parent listed in
depends_on metadata
If there is extra job_id criteria that doesn't apply to a
particular parent app's job_id template, ignore it.
"""
for parent_app_name in depends_on['app_name']:
depends_on = dict(depends_on) # shallow copy to change the keys
_inplace_modify_depends_on(
depends_on, child_app_name, child_job_id, parent_app_name, ld)
# are there specific job_ids the child would inherit from?
if 'job_id' in depends_on:
for rv in _iter_job_ids(
dep_group=depends_on, group_name=group_name,
parent_app_name=parent_app_name, ld=ld):
yield rv
else:
# try to fill in the parent's job_id template and yield it
template, parsed_template = get_job_id_template(parent_app_name)
so_far = set()
_, cparsed_template = get_job_id_template(child_app_name)
autofill_values = get_autofill_values(
child_app_name, raise_err=False)
pjob_id = parse_job_id(child_app_name, child_job_id)
job_id_data = prep_crossproduct(
cparsed_template, depends_on, autofill_values, pjob_id)
for job_id_data in crossproduct([depends_on[_key]
for _key in parsed_template]):
_pjob_id = dict(zip(parsed_template, job_id_data))
parent_job_id = template.format(
dependency_group_name=group_name, **_pjob_id)
if parent_job_id not in so_far:
so_far.add(parent_job_id)
yield (parent_app_name, parent_job_id)