def put(self, run_id, action):
"""
Delete / stop / start a run
"""
parser = reqparse.RequestParser()
parser.add_argument('user', type=str, default=None)
parser.add_argument('force', action='store_true', default=False)
args = parser.parse_args()
arg_user = auth_get_username(request.authorization,
args.get('user'))
arg_force = args.get('force')
errors = ''
query = {'run_id': run_id}
keys = {'work_dir': 1, 'output_dir': 1, 'status': 1, 'user': 1}
pipeline = db.pipelines.find_one(query, keys)
if not pipeline:
errors = 'ERROR: Run ID %s not found' % str(run_id)
elif pipeline['user'] != arg_user:
errors = 'ERROR: cannot modify pipeline %s: permission denied' % str(
run_id)
elif action.lower() == "delete":
if pipeline['status'] != JOB_STATUS.FAILED:
errors = 'ERROR: Run status is %s: cannot delete' % pipeline[
'status'].upper()
else:
db.steps.delete_many({'run_id': run_id})
db.pipelines.find_one_and_delete({'run_id': run_id})
errors = pm.delete_pipeline(run_id, arg_user,
pipeline.get('output_dir'),
pipeline.get('work_dir'))
elif action.lower() == "resume":
if pipeline['status'] != JOB_STATUS.INTERRUPTED:
errors = 'ERROR: Run status is %s: cannot resume' % pipeline[
'status'].upper()
else:
db.pipelines.update(
{'run_id': run_id},
{'$set': {
'status': JOB_STATUS.QUEUED
}})
errors = pm.resume_pipeline(run_id, arg_user,
pipeline.get('work_dir'))
elif action.lower() == "pause":
if pipeline['status'] != JOB_STATUS.RUNNING:
errors = 'ERROR: Run status is %s: cannot pause' % pipeline[
'status'].upper()
else:
errors = pm.pause_pipeline(run_id, arg_user)
else:
errors = "Uknown action requested: %s" % str(action)
if errors:
ut.pretty_print(errors)
return errors, 400
else:
return 200
def put(self, run_id, action):
"""
Delete / stop / start a run
"""
parser = reqparse.RequestParser()
parser.add_argument('user', type=str, default=None)
parser.add_argument('force', action='store_true', default=False)
args = parser.parse_args()
arg_user = auth_get_username(request.authorization, args.get('user'))
arg_force = args.get('force')
errors = ''
query = {'run_id' : run_id}
keys = {'work_dir':1, 'output_dir':1, 'status':1, 'user':1}
pipeline = db.pipelines.find_one(query, keys)
if not pipeline:
errors = 'ERROR: Run ID %s not found' % str(run_id)
elif pipeline['user'] != arg_user:
errors = 'ERROR: cannot modify pipeline %s: permission denied' % str(run_id)
elif action.lower() == "delete":
if pipeline['status'] != JOB_STATUS.FAILED:
errors = 'ERROR: Run status is %s: cannot delete' % pipeline['status'].upper()
else:
db.steps.delete_many({'run_id': run_id})
db.pipelines.find_one_and_delete({'run_id': run_id})
errors = pm.delete_pipeline(run_id, arg_user,
pipeline.get('output_dir'), pipeline.get('work_dir')
)
elif action.lower() == "resume":
if pipeline['status'] != JOB_STATUS.INTERRUPTED:
errors = 'ERROR: Run status is %s: cannot resume' % pipeline['status'].upper()
else:
db.pipelines.update({'run_id': run_id},
{'$set': {'status': JOB_STATUS.QUEUED}})
errors = pm.resume_pipeline(run_id, arg_user, pipeline.get('work_dir'))
elif action.lower() == "pause":
if pipeline['status'] != JOB_STATUS.RUNNING:
errors = 'ERROR: Run status is %s: cannot pause' % pipeline['status'].upper()
else:
errors = pm.pause_pipeline(run_id, arg_user)
else:
errors = "Uknown action requested: %s" % str(action)
if errors:
ut.pretty_print(errors)
return errors, 400
else:
return 200
def put(self):
"""
Queue the specific pipeline
"""
data = request.get_json(force=True)
config = data.get('config')
user = auth_get_username(request.authorization, data.get('user'))
errors = None # Pipeline.validate_config(config, user)
if not errors:
config = Pipeline.load_cfg(config)
# Get id from DB
db_info = dbmodel.PipelineDb(config['name'], config, Pipeline.ordered_steps(config), user)
config['run_id'] = db_info.run_id
ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" % (config['label'], config['run_id'], user))
return pm.add_pipeline(config, user)
else:
return errors, 400
def put(self):
"""
Queue the specific pipeline
"""
data = request.get_json(force=True)
config = data.get('config')
user = auth_get_username(request.authorization, data.get('user'))
errors = None # Pipeline.validate_config(config, user)
if not errors:
config = Pipeline.load_cfg(config)
# Get id from DB
db_info = dbmodel.PipelineDb(config['name'], config,
Pipeline.ordered_steps(config),
user)
config['run_id'] = db_info.run_id
ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" %
(config['label'], config['run_id'], user))
return pm.add_pipeline(config, user)
else:
return errors, 400
def post(self, run_id):
"""
Pushes files into iRODS
"""
data = request.get_json(force=True)
runmeta = data.get('meta')
selection = data.get('selection')
user = auth_get_username(request.authorization, data.get('user'))
npdis = dbmodel.get_npdi_projects()
npdi = runmeta.get('Project NPDI ID', '')
study_nickname = runmeta.get('Study nickname', 'Required field missing')
if (npdi + study_nickname) not in npdis:
return {'pipeline': {
'Project': '%s (%s)' %(npdi, study_nickname)
}}, 400
run = db.pipelines.find_one({'run_id': run_id}, {'meta':1, 'run_id':1})
steps_names = selection.keys()
steps = list(db.steps.find(
{"run_id":run_id, "name": {'$in': steps_names}, "jobs": {"$elemMatch": {"outputs": {"$exists": True}}}},
{"name":1, "jobs":1, "outputs.output_dir": 1, "step_config": 1}))
outputs = {}
for step in steps:
if step.get('step_config', {}):
s = Step.load_step(step['step_config'])
output_files = {}
for job_id, job in enumerate(step['jobs']):
for key in job['outputs']:
if key in s.keys(key_groups='outputs', key_filter={'type':'file'}):
for i, filename in enumerate(job['outputs'][key]):
filemeta = {'step': step['name'], 'job_id': job_id}
ext = os.path.splitext(filename)[1][1:].upper()
for key in job.get('meta', {}):
meta = job['meta'][key]
if key == 'sample_id':
okey = 'Operational sample accession'
else:
okey = key
if isinstance(meta, list):
filemeta[okey] = meta[i]
else:
filemeta[okey] = meta
filemeta['File type'] = 'Processed data file'
filemeta['File format'] = ext
output_files[filename] = filemeta
if output_files:
outputs[step['name']] = output_files
input_files = []
meta_data = []
for step_name, step_selection in selection.iteritems():
for filepath in step_selection:
input_files.append(filepath)
filemeta = outputs[step_name][filepath]
filemeta.update(runmeta)
meta_data.append(filemeta)
cfg = Pipeline.load_cfg(pipeline_specs['irods_lz'])
cfg['config']['steps']['irods_mvtolz'] = {
'input_files' : input_files,
'meta_data' : meta_data
}
cfg['config']['steps']['irods_monitorlz'] = {
'prun_id' : run['run_id']
}
cfg['config']['pipeline']['project_name'] = run['meta']['project_name']
cfg['config']['pipeline']['description'] = 'Archive data for run %s' %run['run_id']
cfg['config']['pipeline']['output_dir'] = '/scratch/cgi/irods'
# Get id from DB
db_info = dbmodel.PipelineDb(cfg['name'], cfg, Pipeline.ordered_steps(cfg), user)
cfg['run_id'] = db_info.run_id
ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" % (cfg['label'], cfg['run_id'], user))
return pm.add_pipeline(cfg, user)
def post(self, run_id):
"""
Pushes files into iRODS
"""
data = request.get_json(force=True)
runmeta = data.get('meta')
selection = data.get('selection')
user = auth_get_username(request.authorization, data.get('user'))
npdis = dbmodel.get_npdi_projects()
npdi = runmeta.get('Project NPDI ID', '')
study_nickname = runmeta.get('Study nickname',
'Required field missing')
if (npdi + study_nickname) not in npdis:
return {
'pipeline': {
'Project': '%s (%s)' % (npdi, study_nickname)
}
}, 400
run = db.pipelines.find_one({'run_id': run_id}, {
'meta': 1,
'run_id': 1
})
steps_names = selection.keys()
steps = list(
db.steps.find(
{
"run_id": run_id,
"name": {
'$in': steps_names
},
"jobs": {
"$elemMatch": {
"outputs": {
"$exists": True
}
}
}
}, {
"name": 1,
"jobs": 1,
"outputs.output_dir": 1,
"step_config": 1
}))
outputs = {}
for step in steps:
if step.get('step_config', {}):
s = Step.load_step(step['step_config'])
output_files = {}
for job_id, job in enumerate(step['jobs']):
for key in job['outputs']:
if key in s.keys(key_groups='outputs',
key_filter={'type': 'file'}):
for i, filename in enumerate(
job['outputs'][key]):
filemeta = {
'step': step['name'],
'job_id': job_id
}
ext = os.path.splitext(
filename)[1][1:].upper()
for key in job.get('meta', {}):
meta = job['meta'][key]
if key == 'sample_id':
okey = 'Operational sample accession'
else:
okey = key
if isinstance(meta, list):
filemeta[okey] = meta[i]
else:
filemeta[okey] = meta
filemeta[
'File type'] = 'Processed data file'
filemeta['File format'] = ext
output_files[filename] = filemeta
if output_files:
outputs[step['name']] = output_files
input_files = []
meta_data = []
for step_name, step_selection in selection.iteritems():
for filepath in step_selection:
input_files.append(filepath)
filemeta = outputs[step_name][filepath]
filemeta.update(runmeta)
meta_data.append(filemeta)
cfg = Pipeline.load_cfg(pipeline_specs['irods_lz'])
cfg['config']['steps']['irods_mvtolz'] = {
'input_files': input_files,
'meta_data': meta_data
}
cfg['config']['steps']['irods_monitorlz'] = {
'prun_id': run['run_id']
}
cfg['config']['pipeline']['project_name'] = run['meta'][
'project_name']
cfg['config']['pipeline'][
'description'] = 'Archive data for run %s' % run['run_id']
cfg['config']['pipeline']['output_dir'] = '/scratch/cgi/irods'
# Get id from DB
db_info = dbmodel.PipelineDb(cfg['name'], cfg,
Pipeline.ordered_steps(cfg), user)
cfg['run_id'] = db_info.run_id
ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" %
(cfg['label'], cfg['run_id'], user))
return pm.add_pipeline(cfg, user)
