def mergeSigProcResults(dirs, SIGPROC_RESULTS, plot_title):
bfmaskPath = os.path.join(SIGPROC_RESULTS, 'analysis.bfmask.bin')
bfmaskstatspath = os.path.join(SIGPROC_RESULTS, 'analysis.bfmask.stats')
########################################################
# write composite return code #
########################################################
try:
if len(dirs) == 96:
composite_return_code = 96
for subdir in dirs:
blockstatus_return_code_file = os.path.join(
subdir, "blockstatus.txt")
if os.path.exists(blockstatus_return_code_file):
with open(blockstatus_return_code_file, 'r') as f:
text = f.read()
if 'Analysis=0' in text:
composite_return_code -= 1
composite_return_code_file = os.path.join(
SIGPROC_RESULTS, "analysis_return_code.txt")
if not os.path.exists(composite_return_code_file):
printtime("DEBUG: create %s" % composite_return_code_file)
os.umask(0002)
f = open(composite_return_code_file, 'a')
f.write(str(composite_return_code))
f.close()
else:
printtime("DEBUG: skip generation of %s" %
composite_return_code_file)
except:
traceback.print_exc()
#################################################
# Merge individual block bead metrics files #
#################################################
printtime("Merging individual block bead metrics files")
try:
cmd = 'BeadmaskMerge -i analysis.bfmask.bin -o ' + bfmaskPath
for subdir in dirs:
subdir = os.path.join(SIGPROC_RESULTS, subdir)
if isbadblock(subdir,
"Merging individual block bead metrics files"):
continue
bfmaskbin = os.path.join(subdir, 'analysis.bfmask.bin')
if os.path.exists(bfmaskbin):
cmd = cmd + ' %s' % subdir
else:
printtime("ERROR: skipped %s" % bfmaskbin)
printtime("DEBUG: Calling '%s'" % cmd)
subprocess.call(cmd, shell=True)
except:
printtime("BeadmaskMerge failed")
###############################################
# Merge individual block bead stats files #
###############################################
printtime("Merging analysis.bfmask.stats files")
try:
bfmaskstatsfiles = []
for subdir in dirs:
subdir = os.path.join(SIGPROC_RESULTS, subdir)
if isbadblock(subdir, "Merging analysis.bfmask.stats files"):
continue
bfmaskstats = os.path.join(subdir, 'analysis.bfmask.stats')
if os.path.exists(bfmaskstats):
bfmaskstatsfiles.append(bfmaskstats)
else:
printtime("ERROR: Merging bfmask.stats files: skipped %s" %
bfmaskstats)
StatsMerge.main_merge(bfmaskstatsfiles, bfmaskstatspath, True)
except:
printtime("ERROR: No analysis.bfmask.stats files were found to merge")
traceback.print_exc()
########################################################
#Make Bead Density Plots #
########################################################
printtime("Make Bead Density Plots (composite report)")
printtime("DEBUG: generate composite heatmap")
if os.path.exists(bfmaskPath):
try:
# Makes Bead_density_contour.png, TODO have to read multiple blocks
beadDensityPlot.genHeatmap(bfmaskPath, bfmaskstatspath,
SIGPROC_RESULTS, plot_title)
except:
traceback.print_exc()
else:
printtime("Warning: no heatmap generated.")
printtime("Finished mergeSigProcResults")
def mergeSigProcResults(dirs, pathToRaw, skipchecksum, SIGPROC_RESULTS):
#####################################################
# Grab one of the processParameters.txt files #
#####################################################
printtime("Merging processParameters.txt")
for subdir in dirs:
subdir = os.path.join(SIGPROC_RESULTS,subdir)
ppfile = os.path.join(subdir,'processParameters.txt')
printtime(ppfile)
if os.path.isfile(ppfile):
processParametersMerge.processParametersMerge(ppfile,True)
break
########################################################
# write composite return code #
########################################################
composite_return_code=0
for subdir in dirs:
if subdir == "block_X0_Y9331":
continue
if subdir == "block_X14168_Y9331":
continue
if subdir == "block_X0_Y0":
continue
if subdir == "block_X14168_Y0":
continue
try:
f = open(os.path.join(SIGPROC_RESULTS,subdir,"analysis_return_code.txt"), 'r')
analysis_return_code = int(f.read(1))
f.close()
if analysis_return_code!=0:
printtime("DEBUG: errors in %s " % subdir)
composite_return_code=1
break
except:
traceback.print_exc()
csp = os.path.join(pathToRaw,'checksum_status.txt')
if not os.path.exists(csp) and not skipchecksum and len(dirs)==96:
printtime("DEBUG: create checksum_status.txt")
try:
os.umask(0002)
f = open(csp, 'w')
f.write(str(composite_return_code))
f.close()
except:
traceback.print_exc()
else:
printtime("DEBUG: skip generation of checksum_status.txt")
#################################################
# Merge individual block bead metrics files #
#################################################
printtime("Merging individual block bead metrics files")
try:
_tmpfile = os.path.join(SIGPROC_RESULTS,'bfmask.bin')
cmd = 'BeadmaskMerge -i bfmask.bin -o ' + _tmpfile
for subdir in dirs:
subdir = os.path.join(SIGPROC_RESULTS,subdir)
if isbadblock(subdir, "Merging individual block bead metrics files"):
continue
bfmaskbin = os.path.join(subdir,'bfmask.bin')
if os.path.exists(bfmaskbin):
cmd = cmd + ' %s' % subdir
else:
printtime("ERROR: skipped %s" % bfmaskbin)
printtime("DEBUG: Calling '%s'" % cmd)
subprocess.call(cmd,shell=True)
except:
printtime("BeadmaskMerge failed (test fragments)")
###############################################
# Merge individual block bead stats files #
###############################################
printtime("Merging bfmask.stats files")
try:
bfmaskstatsfiles = []
for subdir in dirs:
subdir = os.path.join(SIGPROC_RESULTS,subdir)
if isbadblock(subdir, "Merging bfmask.stats files"):
continue
bfmaskstats = os.path.join(subdir,'bfmask.stats')
if os.path.exists(bfmaskstats):
bfmaskstatsfiles.append(subdir)
else:
printtime("ERROR: Merging bfmask.stats files: skipped %s" % bfmaskstats)
StatsMerge.main_merge(bfmaskstatsfiles, True)
#TODO
shutil.move('bfmask.stats', SIGPROC_RESULTS)
except:
printtime("No bfmask.stats files were found to merge")
###############################################
# Merge individual block MaskBead files #
###############################################
# printtime("Merging MaskBead.mask files")
#
# try:
# bfmaskfolders = []
# for subdir in dirs:
# subdir = os.path.join(SIGPROC_RESULTS,subdir)
# printtime("DEBUG: %s:" % subdir)
#
# if isbadblock(subdir, "Merging MaskBead.mask files"):
# continue
#
# bfmaskbead = os.path.join(subdir,'MaskBead.mask')
# if not os.path.exists(bfmaskbead):
# printtime("ERROR: Merging MaskBead.mask files: skipped %s" % bfmaskbead)
# continue
#
# bfmaskfolders.append(subdir)
#
# offset_str = "use_blocks"
# MaskMerge.main_merge('MaskBead.mask', bfmaskfolders, merged_bead_mask_path, True, offset_str)
# except:
# printtime("Merging MaskBead.mask files failed")
########################################################
#Make Bead Density Plots #
########################################################
printtime("Make Bead Density Plots (composite report)")
bfmaskPath = os.path.join(SIGPROC_RESULTS,'bfmask.bin')
maskpath = os.path.join(SIGPROC_RESULTS,'MaskBead.mask')
# skip if merged MaskBead.mask exists TODO
printtime("generate MaskBead.mask")
if os.path.isfile(bfmaskPath):
com = "BeadmaskParse -m MaskBead %s" % bfmaskPath
os.system(com)
#TODO
try:
shutil.move('MaskBead.mask', maskpath)
except:
printtime("ERROR: MaskBead.mask already moved")
else:
printtime("Warning: %s doesn't exists." % bfmaskPath)
printtime("generate graph")
if os.path.exists(maskpath):
try:
# Makes Bead_density_contour.png
beadDensityPlot.genHeatmap(maskpath, SIGPROC_RESULTS) # todo, takes too much time
# os.remove(maskpath)
except:
traceback.print_exc()
else:
printtime("Warning: no MaskBead.mask file exists.")
def mergeSigProcResults(dirs, SIGPROC_RESULTS, plot_title):
bfmaskPath = os.path.join(SIGPROC_RESULTS,'analysis.bfmask.bin')
bfmaskstatspath = os.path.join(SIGPROC_RESULTS,'analysis.bfmask.stats')
########################################################
# write composite return code #
########################################################
try:
if len(dirs)==96:
composite_return_code=96
for subdir in dirs:
blockstatus_return_code_file = os.path.join(subdir,"blockstatus.txt")
if os.path.exists(blockstatus_return_code_file):
with open(blockstatus_return_code_file, 'r') as f:
text = f.read()
if 'Analysis=0' in text:
composite_return_code-=1
composite_return_code_file = os.path.join(SIGPROC_RESULTS,"analysis_return_code.txt")
if not os.path.exists(composite_return_code_file):
printtime("DEBUG: create %s" % composite_return_code_file)
os.umask(0002)
f = open(composite_return_code_file, 'a')
f.write(str(composite_return_code))
f.close()
else:
printtime("DEBUG: skip generation of %s" % composite_return_code_file)
except:
traceback.print_exc()
#################################################
# Merge individual block bead metrics files #
#################################################
printtime("Merging individual block bead metrics files")
try:
cmd = 'BeadmaskMerge -i analysis.bfmask.bin -o ' + bfmaskPath
for subdir in dirs:
subdir = os.path.join(SIGPROC_RESULTS,subdir)
if isbadblock(subdir, "Merging individual block bead metrics files"):
continue
bfmaskbin = os.path.join(subdir,'analysis.bfmask.bin')
if os.path.exists(bfmaskbin):
cmd = cmd + ' %s' % subdir
else:
printtime("ERROR: skipped %s" % bfmaskbin)
printtime("DEBUG: Calling '%s'" % cmd)
subprocess.call(cmd,shell=True)
except:
printtime("BeadmaskMerge failed")
###############################################
# Merge individual block bead stats files #
###############################################
printtime("Merging analysis.bfmask.stats files")
try:
bfmaskstatsfiles = []
for subdir in dirs:
subdir = os.path.join(SIGPROC_RESULTS,subdir)
if isbadblock(subdir, "Merging analysis.bfmask.stats files"):
continue
bfmaskstats = os.path.join(subdir,'analysis.bfmask.stats')
if os.path.exists(bfmaskstats):
bfmaskstatsfiles.append(bfmaskstats)
else:
printtime("ERROR: Merging bfmask.stats files: skipped %s" % bfmaskstats)
StatsMerge.main_merge(bfmaskstatsfiles, bfmaskstatspath, True)
except:
printtime("ERROR: No analysis.bfmask.stats files were found to merge")
traceback.print_exc()
########################################################
#Make Bead Density Plots #
########################################################
printtime("Make Bead Density Plots (composite report)")
printtime("DEBUG: generate composite heatmap")
if os.path.exists(bfmaskPath):
try:
# Makes Bead_density_contour.png, TODO have to read multiple blocks
beadDensityPlot.genHeatmap(bfmaskPath, bfmaskstatspath, SIGPROC_RESULTS, plot_title)
beadDensityPlot.genHeatmap(bfmaskPath, bfmaskstatspath, "./", plot_title)
except:
traceback.print_exc()
else:
printtime("Warning: no heatmap generated.")
printtime("Finished mergeSigProcResults")
