def writeDbFromFiles(tfPath, procPath, beadPath, ionstats_alignment_json_path, ionParamsPath, status, keyPath, ionstats_basecaller_json_path, BaseCallerJsonPath, primarykeyPath, uploadStatusPath, cwd): return_message = "" afile = open(ionParamsPath, 'r') ionparams = json.load(afile) afile.close() procParams = None if os.path.exists(procPath): procParams = fileToDict(procPath) beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except: beadMetrics = None return_message += traceback.format_exc() else: beadMetrics = None return_message += 'ERROR: generating beadMetrics failed - file %s is missing\n' % beadPath tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, 'r') tfMetrics = json.load(file) file.close() except: tfMetrics = None return_message += traceback.format_exc() else: tfMetrics = None return_message += 'ERROR: generating tfMetrics failed - file %s is missing\n' % tfPath keyPeak = { 'Test Fragment': 0, 'Library': 0 } if os.path.exists(keyPath): keyPeak.update(parse_metrics(keyPath)) BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, 'r') BaseCallerMetrics = json.load(afile) afile.close() except: BaseCallerMetrics = None return_message += traceback.format_exc() else: BaseCallerMetrics = None return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing\n' % BaseCallerJsonPath ionstats_basecaller = None if os.path.exists(ionstats_basecaller_json_path): try: afile = open(ionstats_basecaller_json_path, 'r') ionstats_basecaller = json.load(afile) afile.close() except: ionstats_basecaller = None return_message += traceback.format_exc() else: ionstats_basecaller = None return_message += 'ERROR: generating ionstats_basecaller failed - file %s is missing\n' % ionstats_basecaller_json_path ionstats_alignment = None if ionparams['referenceName']: if os.path.exists(ionstats_alignment_json_path): try: afile = open(ionstats_alignment_json_path, 'r') ionstats_alignment = json.load(afile) afile.close() except: ionstats_alignment = None return_message += traceback.format_exc() else: ionstats_alignment = None return_message += 'ERROR: generating ionstats_alignment failed - file %s is missing\n' % ionstats_alignment_json_path genomeinfodict = {} try: if ionparams['referenceName']: genomeinfofilepath = '/results/referenceLibrary/%s/%s/%s.info.txt' % (ionparams['tmap_version'], ionparams['referenceName'], ionparams['referenceName']) with open(genomeinfofilepath) as genomeinfofile: for line in genomeinfofile: key, value = line.partition("\t")[::2] genomeinfodict[key.strip()] = value.strip() else: genomeinfodict['genome_version'] = 'None' genomeinfodict['index_version'] = 'None' genomeinfodict['genome_name'] = 'None' genomeinfodict['genome_length'] = 0 except: return_message += traceback.format_exc() extra_files = glob.glob(os.path.join(cwd, 'BamDuplicates*.json')) extra = {u'reads_with_adaptor': 0, u'duplicate_reads': 0, u'total_reads': 0, u'fraction_with_adaptor': 0, u'fraction_duplicates': 0} for extra_file in extra_files: if os.path.exists(extra_file): try: with open(extra_file, 'r') as afile: val = json.load(afile) for key in extra.keys(): extra[key] += val.get(key, 0) except: return_message += traceback.format_exc() else: return_message += 'INFO: generating extra metrics failed - file %s is missing\n' % extra_file else: # No data, send None/null instead of zeros extra = {u'reads_with_adaptor': None, u'duplicate_reads': None, u'total_reads': None, u'fraction_with_adaptor': None, u'fraction_duplicates': None} try: if extra[u'total_reads']: extra[u'fraction_with_adaptor'] = extra[u'reads_with_adaptor'] / float(extra[u'total_reads']) extra[u'fraction_duplicates'] = extra[u'duplicate_reads'] / float(extra[u'total_reads']) except: return_message += traceback.format_exc() # print "BamDuplicates stats: ", extra writeDbFromDict(tfMetrics, procParams, beadMetrics, ionstats_alignment, genomeinfodict, status, keyPeak, ionstats_basecaller, BaseCallerMetrics, primarykeyPath, uploadStatusPath, extra) return return_message
def writeDbFromFiles(tfPath, procPath, beadPath, ionstats_alignment_json_path, ionParamsPath, status, keyPath, ionstats_basecaller_json_path, BaseCallerJsonPath, primarykeyPath, uploadStatusPath, cwd): return_message = "" afile = open(ionParamsPath, 'r') ionparams = json.load(afile) afile.close() procParams = None if os.path.exists(procPath): procParams = fileToDict(procPath) beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except: beadMetrics = None return_message += traceback.format_exc() else: beadMetrics = None return_message += 'ERROR: generating beadMetrics failed - file %s is missing\n' % beadPath tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, 'r') tfMetrics = json.load(file) file.close() except: tfMetrics = None return_message += traceback.format_exc() else: tfMetrics = None return_message += 'ERROR: generating tfMetrics failed - file %s is missing\n' % tfPath if os.path.exists(keyPath): keyPeak = parse_metrics(keyPath) else: keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = 0 BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, 'r') BaseCallerMetrics = json.load(afile) afile.close() except: BaseCallerMetrics = None return_message += traceback.format_exc() else: BaseCallerMetrics = None return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing\n' % BaseCallerJsonPath ionstats_basecaller = None if os.path.exists(ionstats_basecaller_json_path): try: afile = open(ionstats_basecaller_json_path, 'r') ionstats_basecaller = json.load(afile) afile.close() except: ionstats_basecaller = None return_message += traceback.format_exc() else: ionstats_basecaller = None return_message += 'ERROR: generating ionstats_basecaller failed - file %s is missing\n' % ionstats_basecaller_json_path ionstats_alignment = None if ionparams['libraryName'] != 'none': if os.path.exists(ionstats_alignment_json_path): try: afile = open(ionstats_alignment_json_path, 'r') ionstats_alignment = json.load(afile) afile.close() except: ionstats_alignment = None return_message += traceback.format_exc() else: ionstats_alignment = None return_message += 'ERROR: generating ionstats_alignment failed - file %s is missing\n' % ionstats_alignment_json_path genomeinfodict = {} try: if ionparams['libraryName'] != 'none' and ionstats_alignment != None: genomeinfofilepath = '/results/referenceLibrary/%s/%s/%s.info.txt' % ( ionparams['tmap_version'], ionparams['libraryName'], ionparams['libraryName']) with open(genomeinfofilepath) as genomeinfofile: for line in genomeinfofile: key, value = line.partition("\t")[::2] genomeinfodict[key.strip()] = value.strip() else: genomeinfodict['genome_version'] = 'None' genomeinfodict['index_version'] = 'None' genomeinfodict['genome_name'] = 'None' genomeinfodict['genome_length'] = 0 except: return_message += traceback.format_exc() extra_files = glob.glob(os.path.join(cwd, 'BamDuplicates*.json')) extra = { u'reads_with_adaptor': 0, u'duplicate_reads': 0, u'total_reads': 0, u'fraction_with_adaptor': 0, u'fraction_duplicates': 0 } for extra_file in extra_files: if os.path.exists(extra_file): try: with open(extra_file, 'r') as afile: val = json.load(afile) for key in extra.keys(): extra[key] += val.get(key, 0) except: return_message += traceback.format_exc() else: return_message += 'INFO: generating extra metrics failed - file %s is missing\n' % extra_file else: # No data, send None/null instead of zeros extra = { u'reads_with_adaptor': None, u'duplicate_reads': None, u'total_reads': None, u'fraction_with_adaptor': None, u'fraction_duplicates': None } try: if extra[u'total_reads']: extra[u'fraction_with_adaptor'] = extra[ u'reads_with_adaptor'] / float(extra[u'total_reads']) extra[u'fraction_duplicates'] = extra[u'duplicate_reads'] / float( extra[u'total_reads']) except: return_message += traceback.format_exc() #print "BamDuplicates stats: ", extra writeDbFromDict(tfMetrics, procParams, beadMetrics, ionstats_alignment, genomeinfodict, status, keyPeak, ionstats_basecaller, BaseCallerMetrics, primarykeyPath, uploadStatusPath, extra) return return_message
def writeDbFromFiles(tfPath, procPath, beadPath, filterPath, libPath, status, keyPath, QualityPath, BaseCallerJsonPath, primarykeyPath, uploadStatusPath): return_message = "" tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, 'r') tfMetrics = json.load(file) file.close() except: tfMetrics = None return_message += traceback.print_exc() else: beadMetrics = None return_message += 'ERROR: generating tfMetrics failed - file %s is missing' % tfPath BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, 'r') BaseCallerMetrics = json.load(afile) afile.close() except: BaseCallerMetrics = None return_message += traceback.print_exc() else: BaseCallerMetrics = None return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing' % BaseCallerJsonPath beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except: beadMetrics = None return_message += traceback.print_exc() else: beadMetrics = None return_message += 'ERROR: generating beadMetrics failed - file %s is missing' % beadPath libMetrics = None if os.path.exists(libPath): libMetrics = parse_metrics(libPath) else: libMetrics = None QualityMetrics = None if os.path.exists(QualityPath): try: afile = open(QualityPath, 'r') QualityMetrics = json.load(afile) afile.close() except: QualityMetrics = None return_message += traceback.print_exc() else: QualityMetrics = None return_message += 'ERROR: generating QualityMetrics failed - file %s is missing' % QualityPath if os.path.exists(keyPath): keyPeak = parse_metrics(keyPath) else: keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = 0 procParams = None if os.path.exists(procPath): procParams = fileToDict(procPath) writeDbFromDict(tfMetrics, procParams, beadMetrics, None, libMetrics, status, keyPeak, QualityMetrics, BaseCallerMetrics, primarykeyPath, uploadStatusPath) return return_message
com += " -b ./forward_strand_bias.txt" com += " -c ./reverse_strand_bias.txt" printtime("DEBUG: Calling '%s'" % com) ret = subprocess.call(com,shell=True) if ret: raise BaseException("PairedEndErrorCorrection returned with error code %s" % ret) except: traceback.print_exc() set_result_status('Error') printtime("ERROR running PairedEndErrorCorrection, abort report generation") sys.exit(1) # merge raw_peak_signal try: if os.path.exists("fwd_raw_peak_signal") and os.path.exists("rev_raw_peak_signal"): fwd_keyPeak = parse_metrics("fwd_raw_peak_signal") rev_keyPeak = parse_metrics("rev_raw_peak_signal") keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = (int(fwd_keyPeak['Library']) +int(rev_keyPeak['Library']))/2 else: keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = 0 f = open('raw_peak_signal','w') f.write('Test Fragment = %s\n' % keyPeak['Test Fragment']) f.write('Library = %s\n' % keyPeak['Library']) f.close() except: printtime(traceback.format_exc())
com += " %s" % (forward_sff_path) com += " %s" % (reverse_sff_path) printtime("DEBUG: Calling '%s'" % com) ret = subprocess.call(com,shell=True) if ret: raise BaseException("PairedEndErrorCorrection returned with error code %s" % ret) except: traceback.print_exc() jobserver.updatestatus(os.path.join(os.getcwd(),'primary.key'),'Error',True) printtime("ERROR running PairedEndErrorCorrection, abort report generation") sys.exit(1) # merge raw_peak_signal try: if os.path.exists("fwd_raw_peak_signal") and os.path.exists("rev_raw_peak_signal"): fwd_keyPeak = parse_metrics("fwd_raw_peak_signal") rev_keyPeak = parse_metrics("rev_raw_peak_signal") keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = (int(fwd_keyPeak['Library']) +int(rev_keyPeak['Library']))/2 else: keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = 0 f = open('raw_peak_signal','w') f.write('Test Fragment = %s\n' % keyPeak['Test Fragment']) f.write('Library = %s\n' % keyPeak['Library']) f.close() except: printtime(traceback.format_exc())
def writeDbFromFiles( tfPath, procPath, beadPath, ionstats_alignment_json_path, ionParamsPath, status, keyPath, ionstats_basecaller_json_path, BaseCallerJsonPath, primarykeyPath, uploadStatusPath, cwd, ): return_message = "" afile = open(ionParamsPath, "r") ionparams = json.load(afile) afile.close() procParams = None if os.path.exists(procPath): procParams = fileToDict(procPath) beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except Exception: beadMetrics = None return_message += traceback.format_exc() else: beadMetrics = None return_message += ( "ERROR: generating beadMetrics failed - file %s is missing\n" % beadPath) tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, "r") tfMetrics = json.load(file) file.close() except Exception: tfMetrics = None return_message += traceback.format_exc() else: tfMetrics = None return_message += ( "ERROR: generating tfMetrics failed - file %s is missing\n" % tfPath) keyPeak = {"Test Fragment": 0, "Library": 0} if os.path.exists(keyPath): keyPeak.update(parse_metrics(keyPath)) BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, "r") BaseCallerMetrics = json.load(afile) afile.close() except Exception: BaseCallerMetrics = None return_message += traceback.format_exc() else: BaseCallerMetrics = None return_message += ( "ERROR: generating BaseCallerMetrics failed - file %s is missing\n" % BaseCallerJsonPath) ionstats_basecaller = None if os.path.exists(ionstats_basecaller_json_path): try: afile = open(ionstats_basecaller_json_path, "r") ionstats_basecaller = json.load(afile) afile.close() except Exception: ionstats_basecaller = None return_message += traceback.format_exc() else: ionstats_basecaller = None return_message += ( "ERROR: generating ionstats_basecaller failed - file %s is missing\n" % ionstats_basecaller_json_path) ionstats_alignment = None if ionparams["referenceName"]: if os.path.exists(ionstats_alignment_json_path): try: afile = open(ionstats_alignment_json_path, "r") ionstats_alignment = json.load(afile) afile.close() except Exception: ionstats_alignment = None return_message += traceback.format_exc() else: ionstats_alignment = None return_message += ( "ERROR: generating ionstats_alignment failed - file %s is missing\n" % ionstats_alignment_json_path) genomeinfodict = {} try: if ionparams["referenceName"]: genomeinfofilepath = "/results/referenceLibrary/%s/%s/%s.info.txt" % ( ionparams["tmap_version"], ionparams["referenceName"], ionparams["referenceName"], ) with open(genomeinfofilepath) as genomeinfofile: for line in genomeinfofile: key, value = line.partition("\t")[::2] genomeinfodict[key.strip()] = value.strip() else: genomeinfodict["genome_version"] = "None" genomeinfodict["index_version"] = "None" genomeinfodict["genome_name"] = "None" genomeinfodict["genome_length"] = 0 except Exception: return_message += traceback.format_exc() extra_files = glob.glob(os.path.join(cwd, "BamDuplicates*.json")) extra = { u"reads_with_adaptor": 0, u"duplicate_reads": 0, u"total_reads": 0, u"fraction_with_adaptor": 0, u"fraction_duplicates": 0, } for extra_file in extra_files: if os.path.exists(extra_file): try: with open(extra_file, "r") as afile: val = json.load(afile) for key in list(extra.keys()): extra[key] += val.get(key, 0) except Exception: return_message += traceback.format_exc() else: return_message += ( "INFO: generating extra metrics failed - file %s is missing\n" % extra_file) else: # No data, send None/null instead of zeros extra = { u"reads_with_adaptor": None, u"duplicate_reads": None, u"total_reads": None, u"fraction_with_adaptor": None, u"fraction_duplicates": None, } try: if extra[u"total_reads"]: extra[u"fraction_with_adaptor"] = extra[ u"reads_with_adaptor"] / float(extra[u"total_reads"]) extra[u"fraction_duplicates"] = extra[u"duplicate_reads"] / float( extra[u"total_reads"]) except Exception: return_message += traceback.format_exc() # print "BamDuplicates stats: ", extra writeDbFromDict( tfMetrics, procParams, beadMetrics, ionstats_alignment, genomeinfodict, status, keyPeak, ionstats_basecaller, BaseCallerMetrics, primarykeyPath, uploadStatusPath, extra, ) return return_message
def writeDbFromFiles(tfPath, procPath, beadPath, filterPath, libPath, status, keyPath, QualityPath, BaseCallerJsonPath, primarykeyPath, uploadStatusPath): tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, 'r') tfMetrics = json.load(file) file.close() except: print "ERROR: failed to create tfMetrics object from file %s" % tfPath tfMetrics = None traceback.print_exc() else: print 'ERROR: generating tfMetrics failed - file %s is missing' % tfPath beadMetrics = None BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, 'r') BaseCallerMetrics = json.load(afile) afile.close() except: print "ERROR: failed to create BaseCallerMetrics object" BaseCallerMetrics = None traceback.print_exc() else: print 'ERROR: generating BaseCallerMetrics failed - file %s is missing' % BaseCallerJsonPath BaseCallerMetrics = None beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except: print 'ERROR: generating beadMetrics failed' beadMetrics = None traceback.print_exc() else: print 'ERROR: generating beadMetrics failed - file %s is missing' % beadPath beadMetrics = None if filterPath != None: filterMetrics = None #filterMetrics = parseFilterMetrics.generateParams(filterPath) else: filterMetrics = None libMetrics = None if path.exists(libPath): libMetrics = parse_metrics(libPath) else: libMetrics = None QualityMetrics = None if path.exists(QualityPath): #if the quality metrics exist, make a dict out of them QualityMetrics = parse_metrics(QualityPath) else: QualityMetrics = None if path.exists(keyPath): keyPeak = parse_metrics(keyPath) else: keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = 0 procParams = None if os.path.exists(procPath): procParams = textToDict(procPath) writeDbFromDict(tfMetrics, procParams, beadMetrics, filterMetrics, libMetrics, status, keyPeak, QualityMetrics, BaseCallerMetrics, primarykeyPath, uploadStatusPath)