def update_pipe(self, pipe):
for db_pipe in self.dbm.get_all_pipeline_templates():
db_json = json.loads(db_pipe.json_template)
# update pipeline if already present in db
if db_json['name'].lower() == pipe['name'].lower():
# Do everything relative from pipeline definition file path.
oldwd = os.getcwd()
os.chdir(self.src_pipe_template_path)
logging.info('Updated pipeline: {}'.format(db_json['name']))
for pe_j in pipe['elements']:
if 'script' in pe_j:
element_j = pe_j['script']
script = parse_script(element_j)
db_script = self.dbm.get_script(
name=self._get_script_name(script))
script_arguments = get_default_script_arguments(
script.path)
script_envs = get_default_script_envs(script.path)
script_resources = get_default_script_resources(
script.path)
if 'arguments' in element_j:
for arg in element_j['arguments']:
if arg not in script_arguments:
logging.error(
"Invalid argument >> {} << in pipeline definition json"
.format(arg))
valid_args = ""
for v_arg in script_arguments:
valid_args += ">> {} <<\n".format(
v_arg)
logging.error(
"Valid arguments are: \n{}".format(
valid_args[:-1]))
raise Exception(
'Invalid arguments. Start Cleanup')
if db_script is None:
self.dbm.add(script)
self.dbm.commit()
script_out_path = os.path.join(
self.dst_pipe_template_path, script.path)
script.path = self.file_man.make_path_relative(
script_out_path)
script.arguments = json.dumps(script_arguments)
script.envs = json.dumps(script_envs)
script.resources = json.dumps(script_resources)
self.dbm.save_obj(script)
logging.info("Added script to database")
else:
script_out_path = os.path.join(
self.dst_pipe_template_path, script.path)
db_script.path = self.file_man.make_path_relative(
script_out_path)
db_script.arguments = json.dumps(script_arguments)
db_script.envs = json.dumps(script_envs)
db_script.description = script.description
db_script.resources = json.dumps(script_resources)
self.dbm.save_obj(db_script)
logging.info('Updated script: {}'.format(
db_script.name))
# self._fix_sia_config(pe_j)
db_pipe.json_template = json.dumps(pipe)
self.dbm.save_obj(db_pipe)
os.chdir(oldwd) # Change dir back to old working directory.
return True
# import pipe if not already present in database
self.import_pipe(pipe)
def import_pipe(self, pipe):
try:
logging.info('\n---\n')
# Do everything relative from pipeline definition file path.
oldwd = os.getcwd()
os.chdir(self.src_pipe_template_path)
for db_pipe in self.dbm.get_all_pipeline_templates():
db_json = json.loads(db_pipe.json_template)
if db_json['name'].lower() == pipe['name'].lower():
logging.warning("PipeTemplate in database.")
logging.warning("Name of this template is: %s" %
(pipe['name'], ))
logging.warning("Will not import PipeTemplate.")
return db_pipe.idx
for pe_j in pipe['elements']:
if 'script' in pe_j:
element_j = pe_j['script']
script = parse_script(element_j)
db_script = self.dbm.get_script(
name=self._get_script_name(script))
script_arguments = get_default_script_arguments(
script.path)
script_envs = get_default_script_envs(script.path)
script_resources = get_default_script_resources(
script.path)
if 'arguments' in element_j:
for arg in element_j['arguments']:
if arg not in script_arguments:
logging.error(
"Invalid argument >> {} << in pipeline definition json"
.format(arg))
valid_args = ""
for v_arg in script_arguments:
valid_args += ">> {} <<\n".format(v_arg)
logging.error(
"Valid arguments are: \n{}".format(
valid_args[:-1]))
raise Exception(
'Invalid arguments. Start Cleanup')
if db_script is None:
self.dbm.add(script)
self.dbm.commit()
script_out_path = os.path.join(
self.dst_pipe_template_path, script.path)
script.path = self.file_man.make_path_relative(
script_out_path)
script.arguments = json.dumps(script_arguments)
script.envs = json.dumps(script_envs)
script.resources = json.dumps(script_resources)
self.dbm.save_obj(script)
logging.info("Added script to database\n")
else:
logging.warning(
"Script is already present in database.\n")
logging.warning((str(db_script.idx), db_script.name,
db_script.path))
# self._fix_sia_config(pe_j)
pipe_temp = model.PipeTemplate(json_template=json.dumps(pipe),
timestamp=datetime.now())
self.dbm.save_obj(pipe_temp)
logging.info("Added Pipeline: *** %s ***" % (pipe['name'], ))
os.chdir(oldwd) # Change dir back to old working directory.
return pipe_temp.idx
except Exception as e:
logging.error(e, exc_info=True)
if not self.forTest:
self.remove_pipe_project()
logging.error('Cleanup successful. Removed buggy pipeline.')