def put(self, run_id, action): """ Delete / stop / start a run """ parser = reqparse.RequestParser() parser.add_argument('user', type=str, default=None) parser.add_argument('force', action='store_true', default=False) args = parser.parse_args() arg_user = auth_get_username(request.authorization, args.get('user')) arg_force = args.get('force') errors = '' query = {'run_id': run_id} keys = {'work_dir': 1, 'output_dir': 1, 'status': 1, 'user': 1} pipeline = db.pipelines.find_one(query, keys) if not pipeline: errors = 'ERROR: Run ID %s not found' % str(run_id) elif pipeline['user'] != arg_user: errors = 'ERROR: cannot modify pipeline %s: permission denied' % str( run_id) elif action.lower() == "delete": if pipeline['status'] != JOB_STATUS.FAILED: errors = 'ERROR: Run status is %s: cannot delete' % pipeline[ 'status'].upper() else: db.steps.delete_many({'run_id': run_id}) db.pipelines.find_one_and_delete({'run_id': run_id}) errors = pm.delete_pipeline(run_id, arg_user, pipeline.get('output_dir'), pipeline.get('work_dir')) elif action.lower() == "resume": if pipeline['status'] != JOB_STATUS.INTERRUPTED: errors = 'ERROR: Run status is %s: cannot resume' % pipeline[ 'status'].upper() else: db.pipelines.update( {'run_id': run_id}, {'$set': { 'status': JOB_STATUS.QUEUED }}) errors = pm.resume_pipeline(run_id, arg_user, pipeline.get('work_dir')) elif action.lower() == "pause": if pipeline['status'] != JOB_STATUS.RUNNING: errors = 'ERROR: Run status is %s: cannot pause' % pipeline[ 'status'].upper() else: errors = pm.pause_pipeline(run_id, arg_user) else: errors = "Uknown action requested: %s" % str(action) if errors: ut.pretty_print(errors) return errors, 400 else: return 200
def put(self, run_id, action): """ Delete / stop / start a run """ parser = reqparse.RequestParser() parser.add_argument('user', type=str, default=None) parser.add_argument('force', action='store_true', default=False) args = parser.parse_args() arg_user = auth_get_username(request.authorization, args.get('user')) arg_force = args.get('force') errors = '' query = {'run_id' : run_id} keys = {'work_dir':1, 'output_dir':1, 'status':1, 'user':1} pipeline = db.pipelines.find_one(query, keys) if not pipeline: errors = 'ERROR: Run ID %s not found' % str(run_id) elif pipeline['user'] != arg_user: errors = 'ERROR: cannot modify pipeline %s: permission denied' % str(run_id) elif action.lower() == "delete": if pipeline['status'] != JOB_STATUS.FAILED: errors = 'ERROR: Run status is %s: cannot delete' % pipeline['status'].upper() else: db.steps.delete_many({'run_id': run_id}) db.pipelines.find_one_and_delete({'run_id': run_id}) errors = pm.delete_pipeline(run_id, arg_user, pipeline.get('output_dir'), pipeline.get('work_dir') ) elif action.lower() == "resume": if pipeline['status'] != JOB_STATUS.INTERRUPTED: errors = 'ERROR: Run status is %s: cannot resume' % pipeline['status'].upper() else: db.pipelines.update({'run_id': run_id}, {'$set': {'status': JOB_STATUS.QUEUED}}) errors = pm.resume_pipeline(run_id, arg_user, pipeline.get('work_dir')) elif action.lower() == "pause": if pipeline['status'] != JOB_STATUS.RUNNING: errors = 'ERROR: Run status is %s: cannot pause' % pipeline['status'].upper() else: errors = pm.pause_pipeline(run_id, arg_user) else: errors = "Uknown action requested: %s" % str(action) if errors: ut.pretty_print(errors) return errors, 400 else: return 200
def put(self): """ Queue the specific pipeline """ data = request.get_json(force=True) config = data.get('config') user = auth_get_username(request.authorization, data.get('user')) errors = None # Pipeline.validate_config(config, user) if not errors: config = Pipeline.load_cfg(config) # Get id from DB db_info = dbmodel.PipelineDb(config['name'], config, Pipeline.ordered_steps(config), user) config['run_id'] = db_info.run_id ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" % (config['label'], config['run_id'], user)) return pm.add_pipeline(config, user) else: return errors, 400
def post(self, run_id): """ Pushes files into iRODS """ data = request.get_json(force=True) runmeta = data.get('meta') selection = data.get('selection') user = auth_get_username(request.authorization, data.get('user')) npdis = dbmodel.get_npdi_projects() npdi = runmeta.get('Project NPDI ID', '') study_nickname = runmeta.get('Study nickname', 'Required field missing') if (npdi + study_nickname) not in npdis: return {'pipeline': { 'Project': '%s (%s)' %(npdi, study_nickname) }}, 400 run = db.pipelines.find_one({'run_id': run_id}, {'meta':1, 'run_id':1}) steps_names = selection.keys() steps = list(db.steps.find( {"run_id":run_id, "name": {'$in': steps_names}, "jobs": {"$elemMatch": {"outputs": {"$exists": True}}}}, {"name":1, "jobs":1, "outputs.output_dir": 1, "step_config": 1})) outputs = {} for step in steps: if step.get('step_config', {}): s = Step.load_step(step['step_config']) output_files = {} for job_id, job in enumerate(step['jobs']): for key in job['outputs']: if key in s.keys(key_groups='outputs', key_filter={'type':'file'}): for i, filename in enumerate(job['outputs'][key]): filemeta = {'step': step['name'], 'job_id': job_id} ext = os.path.splitext(filename)[1][1:].upper() for key in job.get('meta', {}): meta = job['meta'][key] if key == 'sample_id': okey = 'Operational sample accession' else: okey = key if isinstance(meta, list): filemeta[okey] = meta[i] else: filemeta[okey] = meta filemeta['File type'] = 'Processed data file' filemeta['File format'] = ext output_files[filename] = filemeta if output_files: outputs[step['name']] = output_files input_files = [] meta_data = [] for step_name, step_selection in selection.iteritems(): for filepath in step_selection: input_files.append(filepath) filemeta = outputs[step_name][filepath] filemeta.update(runmeta) meta_data.append(filemeta) cfg = Pipeline.load_cfg(pipeline_specs['irods_lz']) cfg['config']['steps']['irods_mvtolz'] = { 'input_files' : input_files, 'meta_data' : meta_data } cfg['config']['steps']['irods_monitorlz'] = { 'prun_id' : run['run_id'] } cfg['config']['pipeline']['project_name'] = run['meta']['project_name'] cfg['config']['pipeline']['description'] = 'Archive data for run %s' %run['run_id'] cfg['config']['pipeline']['output_dir'] = '/scratch/cgi/irods' # Get id from DB db_info = dbmodel.PipelineDb(cfg['name'], cfg, Pipeline.ordered_steps(cfg), user) cfg['run_id'] = db_info.run_id ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" % (cfg['label'], cfg['run_id'], user)) return pm.add_pipeline(cfg, user)
def post(self, run_id): """ Pushes files into iRODS """ data = request.get_json(force=True) runmeta = data.get('meta') selection = data.get('selection') user = auth_get_username(request.authorization, data.get('user')) npdis = dbmodel.get_npdi_projects() npdi = runmeta.get('Project NPDI ID', '') study_nickname = runmeta.get('Study nickname', 'Required field missing') if (npdi + study_nickname) not in npdis: return { 'pipeline': { 'Project': '%s (%s)' % (npdi, study_nickname) } }, 400 run = db.pipelines.find_one({'run_id': run_id}, { 'meta': 1, 'run_id': 1 }) steps_names = selection.keys() steps = list( db.steps.find( { "run_id": run_id, "name": { '$in': steps_names }, "jobs": { "$elemMatch": { "outputs": { "$exists": True } } } }, { "name": 1, "jobs": 1, "outputs.output_dir": 1, "step_config": 1 })) outputs = {} for step in steps: if step.get('step_config', {}): s = Step.load_step(step['step_config']) output_files = {} for job_id, job in enumerate(step['jobs']): for key in job['outputs']: if key in s.keys(key_groups='outputs', key_filter={'type': 'file'}): for i, filename in enumerate( job['outputs'][key]): filemeta = { 'step': step['name'], 'job_id': job_id } ext = os.path.splitext( filename)[1][1:].upper() for key in job.get('meta', {}): meta = job['meta'][key] if key == 'sample_id': okey = 'Operational sample accession' else: okey = key if isinstance(meta, list): filemeta[okey] = meta[i] else: filemeta[okey] = meta filemeta[ 'File type'] = 'Processed data file' filemeta['File format'] = ext output_files[filename] = filemeta if output_files: outputs[step['name']] = output_files input_files = [] meta_data = [] for step_name, step_selection in selection.iteritems(): for filepath in step_selection: input_files.append(filepath) filemeta = outputs[step_name][filepath] filemeta.update(runmeta) meta_data.append(filemeta) cfg = Pipeline.load_cfg(pipeline_specs['irods_lz']) cfg['config']['steps']['irods_mvtolz'] = { 'input_files': input_files, 'meta_data': meta_data } cfg['config']['steps']['irods_monitorlz'] = { 'prun_id': run['run_id'] } cfg['config']['pipeline']['project_name'] = run['meta'][ 'project_name'] cfg['config']['pipeline'][ 'description'] = 'Archive data for run %s' % run['run_id'] cfg['config']['pipeline']['output_dir'] = '/scratch/cgi/irods' # Get id from DB db_info = dbmodel.PipelineDb(cfg['name'], cfg, Pipeline.ordered_steps(cfg), user) cfg['run_id'] = db_info.run_id ut.pretty_print("Submitting pipeline %s (ID %d) for user %s" % (cfg['label'], cfg['run_id'], user)) return pm.add_pipeline(cfg, user)