def writeDbFromFiles(tfPath, procPath, beadPath, ionstats_alignment_json_path,
ionParamsPath, status, keyPath,
ionstats_basecaller_json_path, BaseCallerJsonPath,
primarykeyPath, uploadStatusPath, cwd):
return_message = ""
afile = open(ionParamsPath, 'r')
ionparams = json.load(afile)
afile.close()
procParams = None
if os.path.exists(procPath):
procParams = fileToDict(procPath)
beadMetrics = None
if os.path.isfile(beadPath):
try:
beadMetrics = parseBeadfind.generateMetrics(beadPath)
except:
beadMetrics = None
return_message += traceback.format_exc()
else:
beadMetrics = None
return_message += 'ERROR: generating beadMetrics failed - file %s is missing\n' % beadPath
tfMetrics = None
if os.path.exists(tfPath):
try:
file = open(tfPath, 'r')
tfMetrics = json.load(file)
file.close()
except:
tfMetrics = None
return_message += traceback.format_exc()
else:
tfMetrics = None
return_message += 'ERROR: generating tfMetrics failed - file %s is missing\n' % tfPath
if os.path.exists(keyPath):
keyPeak = parse_metrics(keyPath)
else:
keyPeak = {}
keyPeak['Test Fragment'] = 0
keyPeak['Library'] = 0
BaseCallerMetrics = None
if os.path.exists(BaseCallerJsonPath):
try:
afile = open(BaseCallerJsonPath, 'r')
BaseCallerMetrics = json.load(afile)
afile.close()
except:
BaseCallerMetrics = None
return_message += traceback.format_exc()
else:
BaseCallerMetrics = None
return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing\n' % BaseCallerJsonPath
ionstats_basecaller = None
if os.path.exists(ionstats_basecaller_json_path):
try:
afile = open(ionstats_basecaller_json_path, 'r')
ionstats_basecaller = json.load(afile)
afile.close()
except:
ionstats_basecaller = None
return_message += traceback.format_exc()
else:
ionstats_basecaller = None
return_message += 'ERROR: generating ionstats_basecaller failed - file %s is missing\n' % ionstats_basecaller_json_path
ionstats_alignment = None
if ionparams['libraryName'] != 'none':
if os.path.exists(ionstats_alignment_json_path):
try:
afile = open(ionstats_alignment_json_path, 'r')
ionstats_alignment = json.load(afile)
afile.close()
except:
ionstats_alignment = None
return_message += traceback.format_exc()
else:
ionstats_alignment = None
return_message += 'ERROR: generating ionstats_alignment failed - file %s is missing\n' % ionstats_alignment_json_path
genomeinfodict = {}
try:
if ionparams['libraryName'] != 'none' and ionstats_alignment != None:
genomeinfofilepath = '/results/referenceLibrary/%s/%s/%s.info.txt' % (
ionparams['tmap_version'], ionparams['libraryName'],
ionparams['libraryName'])
with open(genomeinfofilepath) as genomeinfofile:
for line in genomeinfofile:
key, value = line.partition("\t")[::2]
genomeinfodict[key.strip()] = value.strip()
else:
genomeinfodict['genome_version'] = 'None'
genomeinfodict['index_version'] = 'None'
genomeinfodict['genome_name'] = 'None'
genomeinfodict['genome_length'] = 0
except:
return_message += traceback.format_exc()
extra_files = glob.glob(os.path.join(cwd, 'BamDuplicates*.json'))
extra = {
u'reads_with_adaptor': 0,
u'duplicate_reads': 0,
u'total_reads': 0,
u'fraction_with_adaptor': 0,
u'fraction_duplicates': 0
}
for extra_file in extra_files:
if os.path.exists(extra_file):
try:
with open(extra_file, 'r') as afile:
val = json.load(afile)
for key in extra.keys():
extra[key] += val.get(key, 0)
except:
return_message += traceback.format_exc()
else:
return_message += 'INFO: generating extra metrics failed - file %s is missing\n' % extra_file
else:
# No data, send None/null instead of zeros
extra = {
u'reads_with_adaptor': None,
u'duplicate_reads': None,
u'total_reads': None,
u'fraction_with_adaptor': None,
u'fraction_duplicates': None
}
try:
if extra[u'total_reads']:
extra[u'fraction_with_adaptor'] = extra[
u'reads_with_adaptor'] / float(extra[u'total_reads'])
extra[u'fraction_duplicates'] = extra[u'duplicate_reads'] / float(
extra[u'total_reads'])
except:
return_message += traceback.format_exc()
#print "BamDuplicates stats: ", extra
writeDbFromDict(tfMetrics, procParams, beadMetrics, ionstats_alignment,
genomeinfodict, status, keyPeak, ionstats_basecaller,
BaseCallerMetrics, primarykeyPath, uploadStatusPath, extra)
return return_message
def writeDbFromFiles(tfPath, procPath, beadPath, filterPath, libPath, status,
keyPath, QualityPath, BaseCallerJsonPath, primarykeyPath,
uploadStatusPath):
return_message = ""
tfMetrics = None
if os.path.exists(tfPath):
try:
file = open(tfPath, 'r')
tfMetrics = json.load(file)
file.close()
except:
tfMetrics = None
return_message += traceback.print_exc()
else:
beadMetrics = None
return_message += 'ERROR: generating tfMetrics failed - file %s is missing' % tfPath
BaseCallerMetrics = None
if os.path.exists(BaseCallerJsonPath):
try:
afile = open(BaseCallerJsonPath, 'r')
BaseCallerMetrics = json.load(afile)
afile.close()
except:
BaseCallerMetrics = None
return_message += traceback.print_exc()
else:
BaseCallerMetrics = None
return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing' % BaseCallerJsonPath
beadMetrics = None
if os.path.isfile(beadPath):
try:
beadMetrics = parseBeadfind.generateMetrics(beadPath)
except:
beadMetrics = None
return_message += traceback.print_exc()
else:
beadMetrics = None
return_message += 'ERROR: generating beadMetrics failed - file %s is missing' % beadPath
libMetrics = None
if os.path.exists(libPath):
libMetrics = parse_metrics(libPath)
else:
libMetrics = None
QualityMetrics = None
if os.path.exists(QualityPath):
try:
afile = open(QualityPath, 'r')
QualityMetrics = json.load(afile)
afile.close()
except:
QualityMetrics = None
return_message += traceback.print_exc()
else:
QualityMetrics = None
return_message += 'ERROR: generating QualityMetrics failed - file %s is missing' % QualityPath
if os.path.exists(keyPath):
keyPeak = parse_metrics(keyPath)
else:
keyPeak = {}
keyPeak['Test Fragment'] = 0
keyPeak['Library'] = 0
procParams = None
if os.path.exists(procPath):
procParams = fileToDict(procPath)
writeDbFromDict(tfMetrics, procParams, beadMetrics, None, libMetrics,
status, keyPeak, QualityMetrics, BaseCallerMetrics,
primarykeyPath, uploadStatusPath)
return return_message
def writeDbFromFiles(tfPath, procPath, beadPath, ionstats_alignment_json_path, ionParamsPath, status, keyPath, ionstats_basecaller_json_path, BaseCallerJsonPath, primarykeyPath, uploadStatusPath, cwd):
return_message = ""
afile = open(ionParamsPath, 'r')
ionparams = json.load(afile)
afile.close()
procParams = None
if os.path.exists(procPath):
procParams = fileToDict(procPath)
beadMetrics = None
if os.path.isfile(beadPath):
try:
beadMetrics = parseBeadfind.generateMetrics(beadPath)
except:
beadMetrics = None
return_message += traceback.format_exc()
else:
beadMetrics = None
return_message += 'ERROR: generating beadMetrics failed - file %s is missing\n' % beadPath
tfMetrics = None
if os.path.exists(tfPath):
try:
file = open(tfPath, 'r')
tfMetrics = json.load(file)
file.close()
except:
tfMetrics = None
return_message += traceback.format_exc()
else:
tfMetrics = None
return_message += 'ERROR: generating tfMetrics failed - file %s is missing\n' % tfPath
keyPeak = {
'Test Fragment': 0,
'Library': 0
}
if os.path.exists(keyPath):
keyPeak.update(parse_metrics(keyPath))
BaseCallerMetrics = None
if os.path.exists(BaseCallerJsonPath):
try:
afile = open(BaseCallerJsonPath, 'r')
BaseCallerMetrics = json.load(afile)
afile.close()
except:
BaseCallerMetrics = None
return_message += traceback.format_exc()
else:
BaseCallerMetrics = None
return_message += 'ERROR: generating BaseCallerMetrics failed - file %s is missing\n' % BaseCallerJsonPath
ionstats_basecaller = None
if os.path.exists(ionstats_basecaller_json_path):
try:
afile = open(ionstats_basecaller_json_path, 'r')
ionstats_basecaller = json.load(afile)
afile.close()
except:
ionstats_basecaller = None
return_message += traceback.format_exc()
else:
ionstats_basecaller = None
return_message += 'ERROR: generating ionstats_basecaller failed - file %s is missing\n' % ionstats_basecaller_json_path
ionstats_alignment = None
if ionparams['referenceName']:
if os.path.exists(ionstats_alignment_json_path):
try:
afile = open(ionstats_alignment_json_path, 'r')
ionstats_alignment = json.load(afile)
afile.close()
except:
ionstats_alignment = None
return_message += traceback.format_exc()
else:
ionstats_alignment = None
return_message += 'ERROR: generating ionstats_alignment failed - file %s is missing\n' % ionstats_alignment_json_path
genomeinfodict = {}
try:
if ionparams['referenceName']:
genomeinfofilepath = '/results/referenceLibrary/%s/%s/%s.info.txt' % (ionparams['tmap_version'], ionparams['referenceName'], ionparams['referenceName'])
with open(genomeinfofilepath) as genomeinfofile:
for line in genomeinfofile:
key, value = line.partition("\t")[::2]
genomeinfodict[key.strip()] = value.strip()
else:
genomeinfodict['genome_version'] = 'None'
genomeinfodict['index_version'] = 'None'
genomeinfodict['genome_name'] = 'None'
genomeinfodict['genome_length'] = 0
except:
return_message += traceback.format_exc()
extra_files = glob.glob(os.path.join(cwd, 'BamDuplicates*.json'))
extra = {u'reads_with_adaptor': 0, u'duplicate_reads': 0, u'total_reads': 0, u'fraction_with_adaptor': 0, u'fraction_duplicates': 0}
for extra_file in extra_files:
if os.path.exists(extra_file):
try:
with open(extra_file, 'r') as afile:
val = json.load(afile)
for key in extra.keys():
extra[key] += val.get(key, 0)
except:
return_message += traceback.format_exc()
else:
return_message += 'INFO: generating extra metrics failed - file %s is missing\n' % extra_file
else:
# No data, send None/null instead of zeros
extra = {u'reads_with_adaptor': None, u'duplicate_reads': None, u'total_reads': None, u'fraction_with_adaptor': None, u'fraction_duplicates': None}
try:
if extra[u'total_reads']:
extra[u'fraction_with_adaptor'] = extra[u'reads_with_adaptor'] / float(extra[u'total_reads'])
extra[u'fraction_duplicates'] = extra[u'duplicate_reads'] / float(extra[u'total_reads'])
except:
return_message += traceback.format_exc()
# print "BamDuplicates stats: ", extra
writeDbFromDict(tfMetrics, procParams, beadMetrics, ionstats_alignment, genomeinfodict, status, keyPeak, ionstats_basecaller, BaseCallerMetrics, primarykeyPath, uploadStatusPath, extra)
return return_message
def writeDbFromFiles(
tfPath,
procPath,
beadPath,
ionstats_alignment_json_path,
ionParamsPath,
status,
keyPath,
ionstats_basecaller_json_path,
BaseCallerJsonPath,
primarykeyPath,
uploadStatusPath,
cwd,
):
return_message = ""
afile = open(ionParamsPath, "r")
ionparams = json.load(afile)
afile.close()
procParams = None
if os.path.exists(procPath):
procParams = fileToDict(procPath)
beadMetrics = None
if os.path.isfile(beadPath):
try:
beadMetrics = parseBeadfind.generateMetrics(beadPath)
except Exception:
beadMetrics = None
return_message += traceback.format_exc()
else:
beadMetrics = None
return_message += (
"ERROR: generating beadMetrics failed - file %s is missing\n" %
beadPath)
tfMetrics = None
if os.path.exists(tfPath):
try:
file = open(tfPath, "r")
tfMetrics = json.load(file)
file.close()
except Exception:
tfMetrics = None
return_message += traceback.format_exc()
else:
tfMetrics = None
return_message += (
"ERROR: generating tfMetrics failed - file %s is missing\n" %
tfPath)
keyPeak = {"Test Fragment": 0, "Library": 0}
if os.path.exists(keyPath):
keyPeak.update(parse_metrics(keyPath))
BaseCallerMetrics = None
if os.path.exists(BaseCallerJsonPath):
try:
afile = open(BaseCallerJsonPath, "r")
BaseCallerMetrics = json.load(afile)
afile.close()
except Exception:
BaseCallerMetrics = None
return_message += traceback.format_exc()
else:
BaseCallerMetrics = None
return_message += (
"ERROR: generating BaseCallerMetrics failed - file %s is missing\n"
% BaseCallerJsonPath)
ionstats_basecaller = None
if os.path.exists(ionstats_basecaller_json_path):
try:
afile = open(ionstats_basecaller_json_path, "r")
ionstats_basecaller = json.load(afile)
afile.close()
except Exception:
ionstats_basecaller = None
return_message += traceback.format_exc()
else:
ionstats_basecaller = None
return_message += (
"ERROR: generating ionstats_basecaller failed - file %s is missing\n"
% ionstats_basecaller_json_path)
ionstats_alignment = None
if ionparams["referenceName"]:
if os.path.exists(ionstats_alignment_json_path):
try:
afile = open(ionstats_alignment_json_path, "r")
ionstats_alignment = json.load(afile)
afile.close()
except Exception:
ionstats_alignment = None
return_message += traceback.format_exc()
else:
ionstats_alignment = None
return_message += (
"ERROR: generating ionstats_alignment failed - file %s is missing\n"
% ionstats_alignment_json_path)
genomeinfodict = {}
try:
if ionparams["referenceName"]:
genomeinfofilepath = "/results/referenceLibrary/%s/%s/%s.info.txt" % (
ionparams["tmap_version"],
ionparams["referenceName"],
ionparams["referenceName"],
)
with open(genomeinfofilepath) as genomeinfofile:
for line in genomeinfofile:
key, value = line.partition("\t")[::2]
genomeinfodict[key.strip()] = value.strip()
else:
genomeinfodict["genome_version"] = "None"
genomeinfodict["index_version"] = "None"
genomeinfodict["genome_name"] = "None"
genomeinfodict["genome_length"] = 0
except Exception:
return_message += traceback.format_exc()
extra_files = glob.glob(os.path.join(cwd, "BamDuplicates*.json"))
extra = {
u"reads_with_adaptor": 0,
u"duplicate_reads": 0,
u"total_reads": 0,
u"fraction_with_adaptor": 0,
u"fraction_duplicates": 0,
}
for extra_file in extra_files:
if os.path.exists(extra_file):
try:
with open(extra_file, "r") as afile:
val = json.load(afile)
for key in list(extra.keys()):
extra[key] += val.get(key, 0)
except Exception:
return_message += traceback.format_exc()
else:
return_message += (
"INFO: generating extra metrics failed - file %s is missing\n"
% extra_file)
else:
# No data, send None/null instead of zeros
extra = {
u"reads_with_adaptor": None,
u"duplicate_reads": None,
u"total_reads": None,
u"fraction_with_adaptor": None,
u"fraction_duplicates": None,
}
try:
if extra[u"total_reads"]:
extra[u"fraction_with_adaptor"] = extra[
u"reads_with_adaptor"] / float(extra[u"total_reads"])
extra[u"fraction_duplicates"] = extra[u"duplicate_reads"] / float(
extra[u"total_reads"])
except Exception:
return_message += traceback.format_exc()
# print "BamDuplicates stats: ", extra
writeDbFromDict(
tfMetrics,
procParams,
beadMetrics,
ionstats_alignment,
genomeinfodict,
status,
keyPeak,
ionstats_basecaller,
BaseCallerMetrics,
primarykeyPath,
uploadStatusPath,
extra,
)
return return_message
