def writeDbFromFiles(tfPath, procPath, beadPath, ionstats_alignment_json_path, ionParamsPath, status, keyPath, ionstats_basecaller_json_path, BaseCallerJsonPath, primarykeyPath, uploadStatusPath, cwd): return_message = "" afile = open(ionParamsPath, 'r') ionparams = json.load(afile) afile.close() procParams = None if os.path.exists(procPath): procParams = fileToDict(procPath) beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except: beadMetrics = None return_message += traceback.format_exc() else: beadMetrics = None return_message += 'ERROR: generating beadMetrics failed - file %s is missing\n' % beadPath tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, 'r') tfMetrics = json.load(file) file.close() except: tfMetrics = None return_message += traceback.format_exc() else: tfMetrics = None return_message += 'ERROR: generating tfMetrics failed - file %s is missing\n' % tfPath if os.path.exists(keyPath): keyPeak = parse_metrics(keyPath) else: keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = 0 BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, 'r') BaseCallerMetrics = json.load(afile) afile.close() except: BaseCallerMetrics = None return_message += traceback.format_exc() else: BaseCallerMetrics = None return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing\n' % BaseCallerJsonPath ionstats_basecaller = None if os.path.exists(ionstats_basecaller_json_path): try: afile = open(ionstats_basecaller_json_path, 'r') ionstats_basecaller = json.load(afile) afile.close() except: ionstats_basecaller = None return_message += traceback.format_exc() else: ionstats_basecaller = None return_message += 'ERROR: generating ionstats_basecaller failed - file %s is missing\n' % ionstats_basecaller_json_path ionstats_alignment = None if ionparams['libraryName'] != 'none': if os.path.exists(ionstats_alignment_json_path): try: afile = open(ionstats_alignment_json_path, 'r') ionstats_alignment = json.load(afile) afile.close() except: ionstats_alignment = None return_message += traceback.format_exc() else: ionstats_alignment = None return_message += 'ERROR: generating ionstats_alignment failed - file %s is missing\n' % ionstats_alignment_json_path genomeinfodict = {} try: if ionparams['libraryName'] != 'none' and ionstats_alignment != None: genomeinfofilepath = '/results/referenceLibrary/%s/%s/%s.info.txt' % ( ionparams['tmap_version'], ionparams['libraryName'], ionparams['libraryName']) with open(genomeinfofilepath) as genomeinfofile: for line in genomeinfofile: key, value = line.partition("\t")[::2] genomeinfodict[key.strip()] = value.strip() else: genomeinfodict['genome_version'] = 'None' genomeinfodict['index_version'] = 'None' genomeinfodict['genome_name'] = 'None' genomeinfodict['genome_length'] = 0 except: return_message += traceback.format_exc() extra_files = glob.glob(os.path.join(cwd, 'BamDuplicates*.json')) extra = { u'reads_with_adaptor': 0, u'duplicate_reads': 0, u'total_reads': 0, u'fraction_with_adaptor': 0, u'fraction_duplicates': 0 } for extra_file in extra_files: if os.path.exists(extra_file): try: with open(extra_file, 'r') as afile: val = json.load(afile) for key in extra.keys(): extra[key] += val.get(key, 0) except: return_message += traceback.format_exc() else: return_message += 'INFO: generating extra metrics failed - file %s is missing\n' % extra_file else: # No data, send None/null instead of zeros extra = { u'reads_with_adaptor': None, u'duplicate_reads': None, u'total_reads': None, u'fraction_with_adaptor': None, u'fraction_duplicates': None } try: if extra[u'total_reads']: extra[u'fraction_with_adaptor'] = extra[ u'reads_with_adaptor'] / float(extra[u'total_reads']) extra[u'fraction_duplicates'] = extra[u'duplicate_reads'] / float( extra[u'total_reads']) except: return_message += traceback.format_exc() #print "BamDuplicates stats: ", extra writeDbFromDict(tfMetrics, procParams, beadMetrics, ionstats_alignment, genomeinfodict, status, keyPeak, ionstats_basecaller, BaseCallerMetrics, primarykeyPath, uploadStatusPath, extra) return return_message
def writeDbFromFiles(tfPath, procPath, beadPath, filterPath, libPath, status, keyPath, QualityPath, BaseCallerJsonPath, primarykeyPath, uploadStatusPath): return_message = "" tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, 'r') tfMetrics = json.load(file) file.close() except: tfMetrics = None return_message += traceback.print_exc() else: beadMetrics = None return_message += 'ERROR: generating tfMetrics failed - file %s is missing' % tfPath BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, 'r') BaseCallerMetrics = json.load(afile) afile.close() except: BaseCallerMetrics = None return_message += traceback.print_exc() else: BaseCallerMetrics = None return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing' % BaseCallerJsonPath beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except: beadMetrics = None return_message += traceback.print_exc() else: beadMetrics = None return_message += 'ERROR: generating beadMetrics failed - file %s is missing' % beadPath libMetrics = None if os.path.exists(libPath): libMetrics = parse_metrics(libPath) else: libMetrics = None QualityMetrics = None if os.path.exists(QualityPath): try: afile = open(QualityPath, 'r') QualityMetrics = json.load(afile) afile.close() except: QualityMetrics = None return_message += traceback.print_exc() else: QualityMetrics = None return_message += 'ERROR: generating QualityMetrics failed - file %s is missing' % QualityPath if os.path.exists(keyPath): keyPeak = parse_metrics(keyPath) else: keyPeak = {} keyPeak['Test Fragment'] = 0 keyPeak['Library'] = 0 procParams = None if os.path.exists(procPath): procParams = fileToDict(procPath) writeDbFromDict(tfMetrics, procParams, beadMetrics, None, libMetrics, status, keyPeak, QualityMetrics, BaseCallerMetrics, primarykeyPath, uploadStatusPath) return return_message
def writeDbFromFiles(tfPath, procPath, beadPath, ionstats_alignment_json_path, ionParamsPath, status, keyPath, ionstats_basecaller_json_path, BaseCallerJsonPath, primarykeyPath, uploadStatusPath, cwd): return_message = "" afile = open(ionParamsPath, 'r') ionparams = json.load(afile) afile.close() procParams = None if os.path.exists(procPath): procParams = fileToDict(procPath) beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except: beadMetrics = None return_message += traceback.format_exc() else: beadMetrics = None return_message += 'ERROR: generating beadMetrics failed - file %s is missing\n' % beadPath tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, 'r') tfMetrics = json.load(file) file.close() except: tfMetrics = None return_message += traceback.format_exc() else: tfMetrics = None return_message += 'ERROR: generating tfMetrics failed - file %s is missing\n' % tfPath keyPeak = { 'Test Fragment': 0, 'Library': 0 } if os.path.exists(keyPath): keyPeak.update(parse_metrics(keyPath)) BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, 'r') BaseCallerMetrics = json.load(afile) afile.close() except: BaseCallerMetrics = None return_message += traceback.format_exc() else: BaseCallerMetrics = None return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing\n' % BaseCallerJsonPath ionstats_basecaller = None if os.path.exists(ionstats_basecaller_json_path): try: afile = open(ionstats_basecaller_json_path, 'r') ionstats_basecaller = json.load(afile) afile.close() except: ionstats_basecaller = None return_message += traceback.format_exc() else: ionstats_basecaller = None return_message += 'ERROR: generating ionstats_basecaller failed - file %s is missing\n' % ionstats_basecaller_json_path ionstats_alignment = None if ionparams['referenceName']: if os.path.exists(ionstats_alignment_json_path): try: afile = open(ionstats_alignment_json_path, 'r') ionstats_alignment = json.load(afile) afile.close() except: ionstats_alignment = None return_message += traceback.format_exc() else: ionstats_alignment = None return_message += 'ERROR: generating ionstats_alignment failed - file %s is missing\n' % ionstats_alignment_json_path genomeinfodict = {} try: if ionparams['referenceName']: genomeinfofilepath = '/results/referenceLibrary/%s/%s/%s.info.txt' % (ionparams['tmap_version'], ionparams['referenceName'], ionparams['referenceName']) with open(genomeinfofilepath) as genomeinfofile: for line in genomeinfofile: key, value = line.partition("\t")[::2] genomeinfodict[key.strip()] = value.strip() else: genomeinfodict['genome_version'] = 'None' genomeinfodict['index_version'] = 'None' genomeinfodict['genome_name'] = 'None' genomeinfodict['genome_length'] = 0 except: return_message += traceback.format_exc() extra_files = glob.glob(os.path.join(cwd, 'BamDuplicates*.json')) extra = {u'reads_with_adaptor': 0, u'duplicate_reads': 0, u'total_reads': 0, u'fraction_with_adaptor': 0, u'fraction_duplicates': 0} for extra_file in extra_files: if os.path.exists(extra_file): try: with open(extra_file, 'r') as afile: val = json.load(afile) for key in extra.keys(): extra[key] += val.get(key, 0) except: return_message += traceback.format_exc() else: return_message += 'INFO: generating extra metrics failed - file %s is missing\n' % extra_file else: # No data, send None/null instead of zeros extra = {u'reads_with_adaptor': None, u'duplicate_reads': None, u'total_reads': None, u'fraction_with_adaptor': None, u'fraction_duplicates': None} try: if extra[u'total_reads']: extra[u'fraction_with_adaptor'] = extra[u'reads_with_adaptor'] / float(extra[u'total_reads']) extra[u'fraction_duplicates'] = extra[u'duplicate_reads'] / float(extra[u'total_reads']) except: return_message += traceback.format_exc() # print "BamDuplicates stats: ", extra writeDbFromDict(tfMetrics, procParams, beadMetrics, ionstats_alignment, genomeinfodict, status, keyPeak, ionstats_basecaller, BaseCallerMetrics, primarykeyPath, uploadStatusPath, extra) return return_message
def writeDbFromFiles( tfPath, procPath, beadPath, ionstats_alignment_json_path, ionParamsPath, status, keyPath, ionstats_basecaller_json_path, BaseCallerJsonPath, primarykeyPath, uploadStatusPath, cwd, ): return_message = "" afile = open(ionParamsPath, "r") ionparams = json.load(afile) afile.close() procParams = None if os.path.exists(procPath): procParams = fileToDict(procPath) beadMetrics = None if os.path.isfile(beadPath): try: beadMetrics = parseBeadfind.generateMetrics(beadPath) except Exception: beadMetrics = None return_message += traceback.format_exc() else: beadMetrics = None return_message += ( "ERROR: generating beadMetrics failed - file %s is missing\n" % beadPath) tfMetrics = None if os.path.exists(tfPath): try: file = open(tfPath, "r") tfMetrics = json.load(file) file.close() except Exception: tfMetrics = None return_message += traceback.format_exc() else: tfMetrics = None return_message += ( "ERROR: generating tfMetrics failed - file %s is missing\n" % tfPath) keyPeak = {"Test Fragment": 0, "Library": 0} if os.path.exists(keyPath): keyPeak.update(parse_metrics(keyPath)) BaseCallerMetrics = None if os.path.exists(BaseCallerJsonPath): try: afile = open(BaseCallerJsonPath, "r") BaseCallerMetrics = json.load(afile) afile.close() except Exception: BaseCallerMetrics = None return_message += traceback.format_exc() else: BaseCallerMetrics = None return_message += ( "ERROR: generating BaseCallerMetrics failed - file %s is missing\n" % BaseCallerJsonPath) ionstats_basecaller = None if os.path.exists(ionstats_basecaller_json_path): try: afile = open(ionstats_basecaller_json_path, "r") ionstats_basecaller = json.load(afile) afile.close() except Exception: ionstats_basecaller = None return_message += traceback.format_exc() else: ionstats_basecaller = None return_message += ( "ERROR: generating ionstats_basecaller failed - file %s is missing\n" % ionstats_basecaller_json_path) ionstats_alignment = None if ionparams["referenceName"]: if os.path.exists(ionstats_alignment_json_path): try: afile = open(ionstats_alignment_json_path, "r") ionstats_alignment = json.load(afile) afile.close() except Exception: ionstats_alignment = None return_message += traceback.format_exc() else: ionstats_alignment = None return_message += ( "ERROR: generating ionstats_alignment failed - file %s is missing\n" % ionstats_alignment_json_path) genomeinfodict = {} try: if ionparams["referenceName"]: genomeinfofilepath = "/results/referenceLibrary/%s/%s/%s.info.txt" % ( ionparams["tmap_version"], ionparams["referenceName"], ionparams["referenceName"], ) with open(genomeinfofilepath) as genomeinfofile: for line in genomeinfofile: key, value = line.partition("\t")[::2] genomeinfodict[key.strip()] = value.strip() else: genomeinfodict["genome_version"] = "None" genomeinfodict["index_version"] = "None" genomeinfodict["genome_name"] = "None" genomeinfodict["genome_length"] = 0 except Exception: return_message += traceback.format_exc() extra_files = glob.glob(os.path.join(cwd, "BamDuplicates*.json")) extra = { u"reads_with_adaptor": 0, u"duplicate_reads": 0, u"total_reads": 0, u"fraction_with_adaptor": 0, u"fraction_duplicates": 0, } for extra_file in extra_files: if os.path.exists(extra_file): try: with open(extra_file, "r") as afile: val = json.load(afile) for key in list(extra.keys()): extra[key] += val.get(key, 0) except Exception: return_message += traceback.format_exc() else: return_message += ( "INFO: generating extra metrics failed - file %s is missing\n" % extra_file) else: # No data, send None/null instead of zeros extra = { u"reads_with_adaptor": None, u"duplicate_reads": None, u"total_reads": None, u"fraction_with_adaptor": None, u"fraction_duplicates": None, } try: if extra[u"total_reads"]: extra[u"fraction_with_adaptor"] = extra[ u"reads_with_adaptor"] / float(extra[u"total_reads"]) extra[u"fraction_duplicates"] = extra[u"duplicate_reads"] / float( extra[u"total_reads"]) except Exception: return_message += traceback.format_exc() # print "BamDuplicates stats: ", extra writeDbFromDict( tfMetrics, procParams, beadMetrics, ionstats_alignment, genomeinfodict, status, keyPeak, ionstats_basecaller, BaseCallerMetrics, primarykeyPath, uploadStatusPath, extra, ) return return_message