def test_format_with_phs(self):
project = {
'variables[0]':
{'storage_root_dir': 'root_dir'}
}
workflow = {
'jobs': {
'raw_data_job': {'name': 'raw_data123'}
}
}
yaml = format_yaml("""
{
"name": "OUTPUT_BASE_DIR",
"value": "${project.variables[0].storage_root_dir}/raw_data/${workflow.jobs.raw_data_job.name}"
}
""", project=project, workflow=workflow)
self.assertEqual(yaml, """
{
"name": "OUTPUT_BASE_DIR",
"value": "root_dir/raw_data/raw_data123"
}
""")
self.assertEqual(format_yaml('$project.variables[0].storage_root_dir',
project=project),
project['variables[0]']['storage_root_dir'])
def test_format_yaml_unknown_ph(self):
x = {'y': 123}
with self.assertRaises(RuntimeError) as cm:
format_yaml('$x.y is $i.j.k', x=x)
self.assertEqual(str(cm.exception), 'Unknown placeholder: i.j.k')
with self.assertRaises(RuntimeError) as cm:
format_yaml('$x.y is ${i.j}', x=x)
self.assertEqual(str(cm.exception), 'Unknown placeholder: i.j')
def _schedule_job(self, job_id):
job = Job.query.get(job_id)
assert job is not None, 'Job %d not found' % job_id
if job.state != JobState.WAITING:
return job.state
deps = JobDependency.query.filter(
JobDependency.dst_job_id == job.id).all()
for dep in deps:
src_job = Job.query.get(dep.src_job_id)
assert src_job is not None, 'Job %d not found' % dep.src_job_id
if not src_job.is_complete():
return job.state
k8s_client = get_client()
system_dict = {
'basic_envs':
os.environ.get(
'BASIC_ENVS', '{"name": "SYSTEM_BASIC_ENVS_DEFAULT",'
'"value": ""}')
}
workflow = job.workflow.to_dict()
workflow['variables'] = self._make_variables_dict(
job.workflow.get_config().variables)
workflow['jobs'] = {}
for j in job.workflow.get_jobs():
variables = self._make_variables_dict(j.get_config().variables)
j_dic = j.to_dict()
j_dic['variables'] = variables
workflow['jobs'][j.get_config().name] = j_dic
project = job.project.to_dict()
project['variables'] = self._make_variables_dict(
job.project.get_config().variables)
yaml = format_yaml(job.yaml_template,
workflow=workflow,
project=project,
system=system_dict)
yaml = json.loads(yaml)
try:
k8s_client.create_or_replace_custom_object(CrdKind.FLAPP, yaml)
except RuntimeError as e:
logging.error('Start job %d has Runtime error msg: %s', job_id,
e.args)
return job.state
job.start()
db.session.commit()
return job.state
def test_format_with_no_ph(self):
self.assertEqual(format_yaml('{a: 123, b: 234}'),
'{a: 123, b: 234}')