def insert_output(self, output, default_type, module_name): """ Parses the output dict of a completed module and stores the data and information within the WaspLink object """ filesets = [] for outtype, outvalue in output.items(): name = '{}_{}'.format(module_name, outtype) if not type(outvalue) is list: outvalue = [outvalue] ## Store default output if default_type == outtype: if isinstance(outvalue[0], asmtypes.FileSet): self['default_output'] = outvalue else: # Files self['default_output'] = asmtypes.set_factory(outtype, [asmtypes.FileInfo(f) for f in outvalue], name=name) ## Store all outputs and values outputs = [] are_files = False for out in outvalue: try: if os.path.exists(out): # These are files, convert to FileInfo format outputs.append(asmtypes.FileInfo(out)) are_files = True else: raise Exception('Not a file') except Exception as e: # Not a file outputs = outvalue break if are_files: filesets.append(asmtypes.set_factory(outtype, outputs, name=name)) else: self['info'][outtype] = outputs if not len(outputs) == 1 else outputs[0] self['data'] = asmtypes.FileSetContainer(filesets)
def func_on_contigs(*wasplinks): contigs = [] for w in wasplinks: contigs += w['default_output'].files newlink = wasp.WaspLink() output = func(contigs) newlink['default_output'] = asmtypes.set_factory('contigs', [output]) return newlink
def wasp_data(self): """ Compatibility layer for wasp data types. Scans self for certain data types and populates a FileSetContainer """ all_sets = [] #### Convert Old Reads Format to ReadSets for set_type in ['reads', 'reference', 'contigs']: if set_type in self: for fs in self[set_type]: ### Get supported set attributes (ins, std, etc) kwargs = {} for key in ['insert', 'stdev', 'tags']: if key in fs: kwargs[key] = fs[key] all_sets.append( asmtypes.set_factory( fs['type'], [asmtypes.FileInfo(f) for f in fs['files']], **kwargs)) #### Convert final_contigs from pipeline mode if 'final_contigs' in self: if self['final_contigs']: ## Not empty ## Remove left over contigs del (self['contigs']) for contig_data in self['final_contigs']: all_sets.append( asmtypes.set_factory( 'contigs', [ asmtypes.FileInfo(fs, ) for fs in contig_data['files'] ], #{'name':contig_data['name']})) name=contig_data['name'])) #### Convert Contig/Ref format # for set_type in ['contigs', 'reference']: # if set_type in self: # all_sets.append(asmtypes.set_factory(set_type, [asmtypes.FileInfo(fs) for fs in self[set_type]])) return asmtypes.FileSetContainer(all_sets)
def insert_output(self, output, default_type, module_name): """ Parses the output dict of a completed module and stores the data and information within the WaspLink object """ filesets = [] for outtype, outvalue in output.items(): name = '{}_{}'.format(module_name, outtype) if not type(outvalue) is list: outvalue = [outvalue] ## Store default output if default_type == outtype: if isinstance(outvalue[0], asmtypes.FileSet): for out in outvalue: out['tags'].append(module_name) self['default_output'] = outvalue else: # Files self['default_output'] = asmtypes.set_factory(outtype, [asmtypes.FileInfo(f) for f in outvalue], name=name) self['default_output']['tags'].append(module_name) ## Store all outputs and values outputs = [] are_files = False for out in outvalue: try: if os.path.exists(out): # These are files, convert to FileInfo format outputs.append(asmtypes.FileInfo(out)) are_files = True else: raise Exception('Not a file') except Exception as e: # Not a file outputs = outvalue break if are_files: filesets.append(asmtypes.set_factory(outtype, outputs, name=name)) else: self['info'][outtype] = outputs if not len(outputs) == 1 else outputs[0] self['data'] = asmtypes.FileSetContainer(filesets)
def wasp_data(self): """ Compatibility layer for wasp data types. Scans self for certain data types and populates a FileSetContainer """ all_sets = [] #### Convert Old Reads Format to ReadSets for set_type in ['reads', 'reference', 'contigs']: if set_type in self: for fs in self[set_type]: ### Get supported set attributes (ins, std, etc) kwargs = {} for key in ['insert', 'stdev', 'platform', 'tags']: if key in fs: kwargs[key] = fs[key] print fs['files'], fs['type'] all_sets.append(asmtypes.set_factory(fs['type'], [asmtypes.FileInfo(f) for f in fs['files']], **kwargs)) #### Convert final_contigs from pipeline mode if 'final_contigs' in self: if self['final_contigs']: ## Not empty ## Remove left over contigs del(self['contigs']) for contig_data in self['final_contigs']: all_sets.append(asmtypes.set_factory('contigs', [asmtypes.FileInfo(fs,) for fs in contig_data['files']], #{'name':contig_data['name']})) name=contig_data['name'])) #### Convert Contig/Ref format # for set_type in ['contigs', 'reference']: # if set_type in self: # all_sets.append(asmtypes.set_factory(set_type, [asmtypes.FileInfo(fs) for fs in self[set_type]])) return asmtypes.FileSetContainer(all_sets)
def wasp_run(self): #### Save and restore insert data, handle extra output orig_sets = copy.deepcopy(self.data.readsets) output = self.run() if output['reads']: if type(output['reads'][0]) is list: ## Multiple Libraries for i, readset in enumerate(output['reads']): orig_sets[i].update_files(readset) orig_sets[i]['name'] = '{}_reads'.format(self.name) else: orig_sets[0].update_files(output['reads']) orig_sets[0]['name'] = '{}_reads'.format(self.name) readsets = orig_sets try: readsets.append(asmtypes.set_factory('single', output['extra'], name='{}_single'.format(self.name))) except:pass return {'reads': readsets}
def run(self): contigsets = self.data.contigsets if len(contigsets) < 2: raise Exception("Fewer than 2 contig sets") merged_files = [] if len(contigsets) == 2: mfile = self.merge(contigsets[0], contigsets[1]) if mfile: merged_files.append(mfile) elif len(contigsets) > 2: mfile = contigsets[0] for i, cset in enumerate(contigsets[1:]): mfile = asmtypes.set_factory( "contigs", [self.merge(mfile, cset)], name="merged{}_contigs".format(i + 1) ) merged_files = mfile.files return {"contigs": merged_files}
def run(self): contigsets = self.data.contigsets if len(contigsets) < 2: raise Exception('Fewer than 2 contig sets') merged_files = [] if len(contigsets) == 2: mfile = self.merge(contigsets[0], contigsets[1]) if mfile: merged_files.append(mfile) elif len(contigsets) > 2: mfile = contigsets[0] for i, cset in enumerate(contigsets[1:]): mfile = asmtypes.set_factory( 'contigs', [self.merge(mfile, cset)], name='merged{}_contigs'.format(i + 1)) merged_files = mfile.files return {'contigs': merged_files}
def wasp_run(self): #### Save and restore insert data, handle extra output orig_sets = copy.deepcopy(self.data.readsets) output = self.run() if output['reads']: if type(output['reads'][0]) is list: ## Multiple Libraries for i, readset in enumerate(output['reads']): orig_sets[i].update_files(readset) orig_sets[i]['name'] = '{}_reads'.format(self.name) else: orig_sets[0].update_files(output['reads']) orig_sets[0]['name'] = '{}_reads'.format(self.name) else: return readsets = orig_sets try: for extra in output['extra']: readsets.append( asmtypes.set_factory('single', extra, name='{}_single'.format(self.name))) except Exception as e: logger.warn('Wasp handles extra output: {}'.format(e)) return {'reads': readsets}
def compute(self, body): self.job_list_lock.acquire() try: job_data = self.prepare_job_data(body) self.job_list.append(job_data) except: logger.error("Error in adding new job to job_list") raise finally: self.job_list_lock.release() status = '' logger.debug('job_data = {}'.format(job_data)) params = json.loads(body) job_id = params['job_id'] data_id = params['data_id'] uid = params['_id'] user = params['ARASTUSER'] token = params['oauth_token'] pipelines = params.get('pipeline') recipe = params.get('recipe') wasp_in = params.get('wasp') jobpath = os.path.join(self.datapath, user, str(data_id), str(job_id)) url = shock.verify_shock_url(self.shockurl) self.start_time = time.time() timer_thread = UpdateTimer(self.metadata, 29, time.time(), uid, self.done_flag) timer_thread.start() #### Parse pipeline to wasp exp reload(recipes) if recipe: try: wasp_exp = recipes.get(recipe[0], job_id) except AttributeError: raise Exception('"{}" recipe not found.'.format(recipe[0])) elif wasp_in: wasp_exp = wasp_in[0] elif not pipelines: wasp_exp = recipes.get('auto', job_id) elif pipelines: ## Legacy client if pipelines[0] == 'auto': wasp_exp = recipes.get('auto', job_id) ########## else: if type(pipelines[0]) is not list: # --assemblers pipelines = [pipelines] all_pipes = [] for p in pipelines: all_pipes += self.pmanager.parse_input(p) logger.debug("pipelines = {}".format(all_pipes)) wasp_exp = wasp.pipelines_to_exp(all_pipes, params['job_id']) else: raise asmtypes.ArastClientRequestError('Malformed job request.') logger.debug('Wasp Expression: {}'.format(wasp_exp)) w_engine = wasp.WaspEngine(self.pmanager, job_data, self.metadata) ###### Run Job try: w_engine.run_expression(wasp_exp, job_data) ###### Upload all result files and place them into appropriate tags uploaded_fsets = job_data.upload_results(url, token) # Format report new_report = open('{}.tmp'.format(self.out_report_name), 'w') ### Log errors if len(job_data['errors']) > 0: new_report.write('PIPELINE ERRORS\n') for i,e in enumerate(job_data['errors']): new_report.write('{}: {}\n'.format(i, e)) try: ## Get Quast output quast_report = job_data['wasp_chain'].find_module('quast')['data'].find_type('report')[0].files[0] with open(quast_report) as q: new_report.write(q.read()) except: new_report.write('No Summary File Generated!\n\n\n') self.out_report.close() with open(self.out_report_name) as old: new_report.write(old.read()) for log in job_data['logfiles']: new_report.write('\n{1} {0} {1}\n'.format(os.path.basename(log), '='*20)) with open(log) as l: new_report.write(l.read()) ### Log tracebacks if len(job_data['tracebacks']) > 0: new_report.write('EXCEPTION TRACEBACKS\n') for i,e in enumerate(job_data['tracebacks']): new_report.write('{}: {}\n'.format(i, e)) new_report.close() os.remove(self.out_report_name) shutil.move(new_report.name, self.out_report_name) res = self.upload(url, user, token, self.out_report_name) report_info = asmtypes.FileInfo(self.out_report_name, shock_url=url, shock_id=res['data']['id']) self.metadata.update_job(uid, 'report', [asmtypes.set_factory('report', [report_info])]) status = 'Complete with errors' if job_data.get('errors') else 'Complete' ## Make compatible with JSON dumps() del job_data['out_report'] del job_data['initial_reads'] del job_data['raw_reads'] self.metadata.update_job(uid, 'data', job_data) self.metadata.update_job(uid, 'result_data', uploaded_fsets) ###### Legacy Support ####### filesets = uploaded_fsets.append(asmtypes.set_factory('report', [report_info])) contigsets = [fset for fset in uploaded_fsets if fset.type == 'contigs' or fset.type == 'scaffolds'] download_ids = {fi['filename']: fi['shock_id'] for fset in uploaded_fsets for fi in fset['file_infos']} contig_ids = {fi['filename']: fi['shock_id'] for fset in contigsets for fi in fset['file_infos']} self.metadata.update_job(uid, 'result_data_legacy', [download_ids]) self.metadata.update_job(uid, 'contig_ids', [contig_ids]) ################### sys.stdout.flush() touch(os.path.join(jobpath, "_DONE_")) logger.info('============== JOB COMPLETE ===============') except asmtypes.ArastUserInterrupt: status = 'Terminated by user' sys.stdout.flush() touch(os.path.join(jobpath, "_CANCELLED__")) logger.info('============== JOB KILLED ===============') finally: self.remove_job_from_lists(job_data) logger.debug('Reinitialize plugin manager...') # Reinitialize to get live changes self.pmanager = ModuleManager(self.threads, self.kill_list, self.kill_list_lock, self.job_list, self.binpath, self.modulebin) self.metadata.update_job(uid, 'status', status)
def eval(x, env): "Evaluate an expression in an environment." if isa(x, Symbol): # variable reference try: return env.find(x)[x] except: raise Exception('Module "{}" not found'.format(x)) elif not isa(x, list): # constant literal return x elif x[0] == 'quote': # (quote exp) (_, exp) = x return exp ####### Casting to FileSet Types elif x[0] in ['contigs', 'paired', 'single', 'reference']: wlink = WaspLink() eval_files = [] try: for exp in x[1:]: eval_files += eval(exp, env).files wlink['default_output'] = asmtypes.set_factory(x[0], eval_files, name='{}_override'.format(x[0])) except Exception as e: wlink['default_output'] = asmtypes.set_factory(x[0], x[1:]) return wlink ################################## elif x[0] == 'if': # (if test conseq alt) if len(x) == 4: (_, test, conseq, alt) = x elif len(x) == 3: (_, test, conseq) = x alt = None if eval(test, env): return eval(conseq, env) elif alt: return eval(alt, env) elif x[0] == 'set!': # (set! var exp) (_, var, exp) = x env.find(var)[var] = eval(exp, env) elif x[0] == 'setparam': (_, param, value) = x try: env.parameters[param] = env.find(value)[value] except: env.parameters[param] = value elif x[0] == 'define': # (define var exp) (_, var, exp) = x try: env[var] = eval(exp, env) except Exception as e: logger.warning('Failed to evaluate definition of "{}": {}'.format(var, e)) logger.debug(traceback.format_exc()) env[var] = None elif x[0] == 'sort': seq = [link for link in eval(x[1], env) if link is not None and link.output] logger.debug(seq) if len(seq) == 1: return seq try: pred = x[2] except: pred = '<' try: k = x[3] assert k == ':key' lam = x[4] eval(['define', 'sort_func', lam], env) except: lam = None rev = pred == '>' if lam: l = sorted(seq, key=lambda n: eval(['sort_func', n], env), reverse=rev) else: l = sorted(seq, reverse=rev) return l elif x[0] == 'lambda': # (lambda (var*) exp) (_, vars, exp) = x return lambda *args: eval(exp, Env(vars, args, env)) elif x[0] == 'upload': # (upload exp) Store each intermediate for return (_, exp) = x try: val = eval(exp, env) results = val except Exception as e: logger.warn('Failed to evaluate upload of "{}": {}'. format(to_string(exp), e)) logger.debug(traceback.format_exc()) env.errors.append(e) env.exceptions.append(traceback.format_exc()) results = None if type(results) is list: for r in results: env.emissions.append(r) elif results: env.emissions.append(results) return results elif x[0] == 'get': (_, key, exp) = x chain = eval(exp, env) assert type(chain) is WaspLink val = chain.get_value(key) if isinstance(val, asmtypes.FileSet): chain['default_output'] = val return chain else: # A value return val elif x[0] == 'all_files': ## Gets all data from module directory (_, exp) = x chain = eval(exp, env) assert type(chain) is WaspLink all_files = utils.ls_recursive(chain['outpath']) module = chain['module'] chain['default_output'] = asmtypes.set_factory('misc', all_files, name='{}.all_files'.format(module), keep_name=True) return chain elif x[0] == 'tar': ## Tar outputs from WaspLink(s) bare_exp, kwargs = extract_kwargs(x) wlinks = [eval(exp, env) for exp in bare_exp[1:]] ### Format tarball name if 'name' in kwargs: tar_name = '{}.tar.gz'.format(kwargs['name']) else: # Generate Tar Name tar_name = '{}.tar.gz'.format('_'.join([w['module'] for w in wlinks])) ### Tag the tarball fileset tag = kwargs.get('tag') tags = [tag] if tag else [] ### Create new link chain = WaspLink('tar', wlinks) filelist = [] for w in wlinks: filelist += w.files chain['default_output'] = asmtypes.set_factory( 'tar', utils.tar_list(env.outpath, filelist, tar_name), name=tar_name, keep_name=True, tags=tags) return chain elif x[0] == 'begin': # (begin exp*) Return each intermediate inner_env = Env(outer=env) val = [] for exp in x[1:]: try: ret = eval(exp, inner_env) if ret:val.append(ret) except Exception as e: if list(e): logger.warning('Failed to eval "{}": {}'.format(to_string(exp), e)) logger.debug(traceback.format_exc()) env.errors.append(e) env.exceptions.append(traceback.format_exc()) if val: return val if len(val) > 1 else val[0] elif x[0] == 'print': for exp in x[1:]: print eval(exp, env) elif x[0] == 'prog': # same as begin, but use same env val = [] for exp in x[1:]: try: ret = eval(exp, env) if ret: val.append(ret) except Exception as e: if list(e): logger.warning('Failed to eval "{}": {}'.format(to_string(exp), e)) logger.debug(traceback.format_exc()) env.errors.append(e) env.exceptions.append(traceback.format_exc()) if val: return val if len(val) > 1 else val[0] else: # (proc exp*) exps = [eval(exp, env) for exp in x] proc = exps.pop(0) env.next_stage(x[0]) try: ## Assembly functions return proc(*exps, env=env) except TypeError as e: ## Built-in functions logger.debug(traceback.format_exc()) return proc(*exps)
def compute(self, body): self.job_list_lock.acquire() try: job_data = self.prepare_job_data(body) self.job_list.append(job_data) except: logger.error("Error in adding new job to job_list") raise finally: self.job_list_lock.release() status = '' logger.debug('job_data = {}'.format(job_data)) params = json.loads(body) job_id = params['job_id'] data_id = params['data_id'] uid = params['_id'] user = params['ARASTUSER'] token = params['oauth_token'] pipelines = params.get('pipeline') recipe = params.get('recipe') wasp_in = params.get('wasp') jobpath = os.path.join(self.datapath, user, str(data_id), str(job_id)) url = shock.verify_shock_url(self.shockurl) self.start_time = time.time() timer_thread = UpdateTimer(self.metadata, 29, time.time(), uid, self.done_flag) timer_thread.start() #### Parse pipeline to wasp exp reload(recipes) if recipe: try: wasp_exp = recipes.get(recipe[0], job_id) except AttributeError: raise Exception('"{}" recipe not found.'.format(recipe[0])) elif wasp_in: wasp_exp = wasp_in[0] elif not pipelines: wasp_exp = recipes.get('auto', job_id) elif pipelines: ## Legacy client if pipelines[0] == 'auto': wasp_exp = recipes.get('auto', job_id) ########## else: if type(pipelines[0]) is not list: # --assemblers pipelines = [pipelines] all_pipes = [] for p in pipelines: all_pipes += self.pmanager.parse_input(p) logger.debug("pipelines = {}".format(all_pipes)) wasp_exp = wasp.pipelines_to_exp(all_pipes, params['job_id']) else: raise asmtypes.ArastClientRequestError('Malformed job request.') logger.debug('Wasp Expression: {}'.format(wasp_exp)) w_engine = wasp.WaspEngine(self.pmanager, job_data, self.metadata) ###### Run Job try: w_engine.run_expression(wasp_exp, job_data) ###### Upload all result files and place them into appropriate tags uploaded_fsets = job_data.upload_results(url, token) # Format report new_report = open('{}.tmp'.format(self.out_report_name), 'w') ### Log errors if len(job_data['errors']) > 0: new_report.write('PIPELINE ERRORS\n') for i, e in enumerate(job_data['errors']): new_report.write('{}: {}\n'.format(i, e)) try: ## Get Quast output quast_report = job_data['wasp_chain'].find_module( 'quast')['data'].find_type('report')[0].files[0] with open(quast_report) as q: new_report.write(q.read()) except: new_report.write('No Summary File Generated!\n\n\n') self.out_report.close() with open(self.out_report_name) as old: new_report.write(old.read()) for log in job_data['logfiles']: new_report.write('\n{1} {0} {1}\n'.format( os.path.basename(log), '=' * 20)) with open(log) as l: new_report.write(l.read()) ### Log tracebacks if len(job_data['tracebacks']) > 0: new_report.write('EXCEPTION TRACEBACKS\n') for i, e in enumerate(job_data['tracebacks']): new_report.write('{}: {}\n'.format(i, e)) new_report.close() os.remove(self.out_report_name) shutil.move(new_report.name, self.out_report_name) res = self.upload(url, user, token, self.out_report_name) report_info = asmtypes.FileInfo(self.out_report_name, shock_url=url, shock_id=res['data']['id']) self.metadata.update_job( uid, 'report', [asmtypes.set_factory('report', [report_info])]) status = 'Complete with errors' if job_data.get( 'errors') else 'Complete' ## Make compatible with JSON dumps() del job_data['out_report'] del job_data['initial_reads'] del job_data['raw_reads'] self.metadata.update_job(uid, 'data', job_data) self.metadata.update_job(uid, 'result_data', uploaded_fsets) ###### Legacy Support ####### filesets = uploaded_fsets.append( asmtypes.set_factory('report', [report_info])) contigsets = [ fset for fset in uploaded_fsets if fset.type == 'contigs' or fset.type == 'scaffolds' ] download_ids = { fi['filename']: fi['shock_id'] for fset in uploaded_fsets for fi in fset['file_infos'] } contig_ids = { fi['filename']: fi['shock_id'] for fset in contigsets for fi in fset['file_infos'] } self.metadata.update_job(uid, 'result_data_legacy', [download_ids]) self.metadata.update_job(uid, 'contig_ids', [contig_ids]) ################### sys.stdout.flush() touch(os.path.join(jobpath, "_DONE_")) logger.info('============== JOB COMPLETE ===============') except asmtypes.ArastUserInterrupt: status = 'Terminated by user' sys.stdout.flush() touch(os.path.join(jobpath, "_CANCELLED__")) logger.info('============== JOB KILLED ===============') finally: self.remove_job_from_lists(job_data) logger.debug('Reinitialize plugin manager...' ) # Reinitialize to get live changes self.pmanager = ModuleManager(self.threads, self.kill_list, self.kill_list_lock, self.job_list, self.binpath, self.modulebin) self.metadata.update_job(uid, 'status', status)
def compute(self, body): error = False params = json.loads(body) job_id = params['job_id'] uid = params['_id'] user = params['ARASTUSER'] token = params['oauth_token'] pipelines = params['pipeline'] recipe = None wasp_in = None try: ## In case legacy recipe = params['recipe'] wasp_in = params['wasp'] except:pass #support legacy arast client if len(pipelines) > 0: if type(pipelines[0]) is not list: pipelines = [pipelines] ### Download files (if necessary) datapath, all_files = self.get_data(body) rawpath = datapath + '/raw/' jobpath = os.path.join(datapath, str(job_id)) try: os.makedirs(jobpath) except Exception as e: print e raise Exception ('Data Error') ### Create job log self.out_report_name = '{}/{}_report.txt'.format(jobpath, str(job_id)) self.out_report = open(self.out_report_name, 'w') ### Create data to pass to pipeline reads = [] reference = [] for fileset in all_files: if len(fileset['files']) != 0: if (fileset['type'] == 'single' or fileset['type'] == 'paired'): reads.append(fileset) elif fileset['type'] == 'reference': reference.append(fileset) else: raise Exception('fileset error') job_data = ArastJob({'job_id' : params['job_id'], 'uid' : params['_id'], 'user' : params['ARASTUSER'], 'reads': reads, 'logfiles': [], 'reference': reference, 'initial_reads': list(reads), 'raw_reads': copy.deepcopy(reads), 'params': [], 'exceptions': [], 'pipeline_data': {}, 'datapath': datapath, 'out_report' : self.out_report}) self.out_report.write("Arast Pipeline: Job {}\n".format(job_id)) self.job_list.append(job_data) self.start_time = time.time() timer_thread = UpdateTimer(self.metadata, 29, time.time(), uid, self.done_flag) timer_thread.start() url = "http://%s" % (self.shockurl) status = '' #### Parse pipeline to wasp exp wasp_exp = pipelines[0][0] reload(recipes) if recipe: try: wasp_exp = recipes.get(recipe[0]) except AttributeError: raise Exception('"{}" recipe not found.'.format(recipe[0])) elif wasp_in: wasp_exp = wasp_in[0] elif pipelines[0] == 'auto': wasp_exp = recipes.get('auto') else: all_pipes = [] for p in pipelines: all_pipes += self.pmanager.parse_input(p) print all_pipes wasp_exp = wasp.pipelines_to_exp(all_pipes, params['job_id']) logging.info('Wasp Expression: {}'.format(wasp_exp)) print('Wasp Expression: {}'.format(wasp_exp)) w_engine = wasp.WaspEngine(self.pmanager, job_data, self.metadata) w_engine.run_expression(wasp_exp, job_data) ###### Upload all result files and place them into appropriate tags uploaded_fsets = job_data.upload_results(url, token) for i, job in enumerate(self.job_list): if job['user'] == job_data['user'] and job['job_id'] == job_data['job_id']: self.job_list.pop(i) # Format report new_report = open('{}.tmp'.format(self.out_report_name), 'w') ### Log exceptions if len(job_data['exceptions']) > 0: new_report.write('PIPELINE ERRORS\n') for i,e in enumerate(job_data['exceptions']): new_report.write('{}: {}\n'.format(i, e)) try: ## Get Quast output quast_report = job_data['wasp_chain'].find_module('quast')['data'].find_type('report')[0].files[0] with open(quast_report) as q: new_report.write(q.read()) except: new_report.write('No Summary File Generated!\n\n\n') self.out_report.close() with open(self.out_report_name) as old: new_report.write(old.read()) for log in job_data['logfiles']: new_report.write('\n{1} {0} {1}\n'.format(os.path.basename(log), '='*20)) with open(log) as l: new_report.write(l.read()) new_report.close() os.remove(self.out_report_name) shutil.move(new_report.name, self.out_report_name) res = self.upload(url, user, token, self.out_report_name) report_info = asmtypes.FileInfo(self.out_report_name, shock_url=url, shock_id=res['data']['id']) self.metadata.update_job(uid, 'report', [asmtypes.set_factory('report', [report_info])]) status = 'Complete with errors' if job_data['exceptions'] else 'Complete' ## Make compatible with JSON dumps() del job_data['out_report'] del job_data['initial_reads'] del job_data['raw_reads'] self.metadata.update_job(uid, 'data', job_data) self.metadata.update_job(uid, 'result_data', uploaded_fsets) self.metadata.update_job(uid, 'status', status) ###### Legacy Support ####### filesets = uploaded_fsets.append(asmtypes.set_factory('report', [report_info])) contigsets = [fset for fset in uploaded_fsets if fset.type == 'contigs' or fset.type == 'scaffolds'] download_ids = {fi['filename']: fi['shock_id'] for fset in uploaded_fsets for fi in fset['file_infos']} contig_ids = {fi['filename']: fi['shock_id'] for fset in contigsets for fi in fset['file_infos']} self.metadata.update_job(uid, 'result_data_legacy', [download_ids]) self.metadata.update_job(uid, 'contig_ids', [contig_ids]) ################### print '============== JOB COMPLETE ==============='