def put(self):
"""
Queue the specific pipeline
"""
data = request.get_json(force=True)
config = data.get('config')
user = auth_get_username(request.authorization, data.get('user'))
errors = None # Pipeline.validate_config(config, user)
if not errors:
config = Pipeline.load_cfg(config)
# Get id from DB
db_info = dbmodel.PipelineDb(config['name'], config, Pipeline.ordered_steps(config), user)
config['run_id'] = db_info.run_id
ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" % (config['label'], config['run_id'], user))
return pm.add_pipeline(config, user)
else:
return errors, 400
def put(self):
"""
Queue the specific pipeline
"""
data = request.get_json(force=True)
config = data.get('config')
user = auth_get_username(request.authorization, data.get('user'))
errors = None # Pipeline.validate_config(config, user)
if not errors:
config = Pipeline.load_cfg(config)
# Get id from DB
db_info = dbmodel.PipelineDb(config['name'], config,
Pipeline.ordered_steps(config),
user)
config['run_id'] = db_info.run_id
ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" %
(config['label'], config['run_id'], user))
return pm.add_pipeline(config, user)
else:
return errors, 400
def post(self, run_id):
"""
Pushes files into iRODS
"""
data = request.get_json(force=True)
runmeta = data.get('meta')
selection = data.get('selection')
user = auth_get_username(request.authorization, data.get('user'))
npdis = dbmodel.get_npdi_projects()
npdi = runmeta.get('Project NPDI ID', '')
study_nickname = runmeta.get('Study nickname', 'Required field missing')
if (npdi + study_nickname) not in npdis:
return {'pipeline': {
'Project': '%s (%s)' %(npdi, study_nickname)
}}, 400
run = db.pipelines.find_one({'run_id': run_id}, {'meta':1, 'run_id':1})
steps_names = selection.keys()
steps = list(db.steps.find(
{"run_id":run_id, "name": {'$in': steps_names}, "jobs": {"$elemMatch": {"outputs": {"$exists": True}}}},
{"name":1, "jobs":1, "outputs.output_dir": 1, "step_config": 1}))
outputs = {}
for step in steps:
if step.get('step_config', {}):
s = Step.load_step(step['step_config'])
output_files = {}
for job_id, job in enumerate(step['jobs']):
for key in job['outputs']:
if key in s.keys(key_groups='outputs', key_filter={'type':'file'}):
for i, filename in enumerate(job['outputs'][key]):
filemeta = {'step': step['name'], 'job_id': job_id}
ext = os.path.splitext(filename)[1][1:].upper()
for key in job.get('meta', {}):
meta = job['meta'][key]
if key == 'sample_id':
okey = 'Operational sample accession'
else:
okey = key
if isinstance(meta, list):
filemeta[okey] = meta[i]
else:
filemeta[okey] = meta
filemeta['File type'] = 'Processed data file'
filemeta['File format'] = ext
output_files[filename] = filemeta
if output_files:
outputs[step['name']] = output_files
input_files = []
meta_data = []
for step_name, step_selection in selection.iteritems():
for filepath in step_selection:
input_files.append(filepath)
filemeta = outputs[step_name][filepath]
filemeta.update(runmeta)
meta_data.append(filemeta)
cfg = Pipeline.load_cfg(pipeline_specs['irods_lz'])
cfg['config']['steps']['irods_mvtolz'] = {
'input_files' : input_files,
'meta_data' : meta_data
}
cfg['config']['steps']['irods_monitorlz'] = {
'prun_id' : run['run_id']
}
cfg['config']['pipeline']['project_name'] = run['meta']['project_name']
cfg['config']['pipeline']['description'] = 'Archive data for run %s' %run['run_id']
cfg['config']['pipeline']['output_dir'] = '/scratch/cgi/irods'
# Get id from DB
db_info = dbmodel.PipelineDb(cfg['name'], cfg, Pipeline.ordered_steps(cfg), user)
cfg['run_id'] = db_info.run_id
ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" % (cfg['label'], cfg['run_id'], user))
return pm.add_pipeline(cfg, user)
def post(self, run_id):
"""
Pushes files into iRODS
"""
data = request.get_json(force=True)
runmeta = data.get('meta')
selection = data.get('selection')
user = auth_get_username(request.authorization, data.get('user'))
npdis = dbmodel.get_npdi_projects()
npdi = runmeta.get('Project NPDI ID', '')
study_nickname = runmeta.get('Study nickname',
'Required field missing')
if (npdi + study_nickname) not in npdis:
return {
'pipeline': {
'Project': '%s (%s)' % (npdi, study_nickname)
}
}, 400
run = db.pipelines.find_one({'run_id': run_id}, {
'meta': 1,
'run_id': 1
})
steps_names = selection.keys()
steps = list(
db.steps.find(
{
"run_id": run_id,
"name": {
'$in': steps_names
},
"jobs": {
"$elemMatch": {
"outputs": {
"$exists": True
}
}
}
}, {
"name": 1,
"jobs": 1,
"outputs.output_dir": 1,
"step_config": 1
}))
outputs = {}
for step in steps:
if step.get('step_config', {}):
s = Step.load_step(step['step_config'])
output_files = {}
for job_id, job in enumerate(step['jobs']):
for key in job['outputs']:
if key in s.keys(key_groups='outputs',
key_filter={'type': 'file'}):
for i, filename in enumerate(
job['outputs'][key]):
filemeta = {
'step': step['name'],
'job_id': job_id
}
ext = os.path.splitext(
filename)[1][1:].upper()
for key in job.get('meta', {}):
meta = job['meta'][key]
if key == 'sample_id':
okey = 'Operational sample accession'
else:
okey = key
if isinstance(meta, list):
filemeta[okey] = meta[i]
else:
filemeta[okey] = meta
filemeta[
'File type'] = 'Processed data file'
filemeta['File format'] = ext
output_files[filename] = filemeta
if output_files:
outputs[step['name']] = output_files
input_files = []
meta_data = []
for step_name, step_selection in selection.iteritems():
for filepath in step_selection:
input_files.append(filepath)
filemeta = outputs[step_name][filepath]
filemeta.update(runmeta)
meta_data.append(filemeta)
cfg = Pipeline.load_cfg(pipeline_specs['irods_lz'])
cfg['config']['steps']['irods_mvtolz'] = {
'input_files': input_files,
'meta_data': meta_data
}
cfg['config']['steps']['irods_monitorlz'] = {
'prun_id': run['run_id']
}
cfg['config']['pipeline']['project_name'] = run['meta'][
'project_name']
cfg['config']['pipeline'][
'description'] = 'Archive data for run %s' % run['run_id']
cfg['config']['pipeline']['output_dir'] = '/scratch/cgi/irods'
# Get id from DB
db_info = dbmodel.PipelineDb(cfg['name'], cfg,
Pipeline.ordered_steps(cfg), user)
cfg['run_id'] = db_info.run_id
ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" %
(cfg['label'], cfg['run_id'], user))
return pm.add_pipeline(cfg, user)
