def _update_metrics(outdir, targetname, groupname, rowidx, z, y, aucthresh=(.5,.5)):
modeldir = outdir+"/"+targetname
# Load current predictions
with np.load(modeldir+"/predict.npz") as f:
predict_npz = { name : f[name] for name in f.keys() }
# Add prediction group
group = predict_npz["groups"][()].setdefault(groupname,{})
group["I"] = rowidx
group["Z"] = z.reshape((-1,1))
if y is not None:
group["Y"] = y.reshape((-1,1))
# Save predictions
np.savez_compressed(modeldir+"/predict.npz", **predict_npz)
deepity.call_dumpviz(modeldir+"/predict.npz")
# Load current metrics
metrics = deepity.load_metrics(modeldir+"/metrics.txt")
metrics[groupname] = deepity.calc_metrics(z, y, aucthresh=aucthresh)
deepity.save_metrics(modeldir+"/metrics.txt", metrics)
"""
def save_report(modeldir, reportdir, tfids, index_metric="auc"):
util.makepath(reportdir)
index_html = open(reportdir+"/index.html",'w')
index_html.write("<html><head><title>Training report</title></head><body>\n")
index_html.write("<table cellspacing=0 cellpadding=5 border=1>\n")
index_html.write("<tr><th>Name</th><th>train %s</th><th>test %s</th></tr>\n" % (index_metric, index_metric))
for tfid in ["TF_%d" % i for i in range(1,67)]:
if tfid not in tfids:
continue
print tfid
util.makepath(reportdir+"/"+tfid)
# Load PFMs, convert them to logo images, and dump them to the report directory
with open(reportdir+"/%s.pfms.pkl"%tfid) as f:
_ = cPickle.load(f)
pfms = _["pfms"]
ics = _["ic"]
counts = _["counts"]
pfm_order = np.argsort(-ics)
logos = []
for j in pfm_order:
pfm = pfms[j]
ic = ics[j]
if ic <= 0:
continue
pfm_rev = np.fliplr(np.flipud(pfm))
logo_fwd = deepity.tape2logo.tape2logo(pfm.T, height=50, letterwidth=10, bufferzoom=4, vmax=1.0, style="seqlogo", rna=False)
logo_rev = deepity.tape2logo.tape2logo(pfm_rev.T, height=50, letterwidth=10, bufferzoom=4, vmax=1.0, style="seqlogo", rna=False)
logo_filename = "%s/%s/pfm%02d" % (reportdir, tfid, len(logos))
scipy.misc.imsave(logo_filename+"_fwd.png", logo_fwd)
scipy.misc.imsave(logo_filename+"_rev.png", logo_rev)
logos.append((j, os.path.basename(logo_filename)+"_fwd.png", ic, counts[j]))
# Load train/test metrics so we can print them in the HTML report
metrics_file = "%s/%s/metrics.txt" % (modeldir, tfid)
metrics = deepity.load_metrics(metrics_file)
# Add row for this TF in main index.html
index_html.write("<tr>\n")
index_html.write("<td><a href=\"%s/index.html\">%s</a></td>\n" % (tfid, tfid))
if index_metric=="auc":
index_html.write("<td>%s ± %s</td>\n" % (metrics["train"]["auc.mean"], metrics["train"]["auc.std"]))
index_html.write("<td><b>%s</b> ± %s</td>\n" % (metrics["test"]["auc.mean"], metrics["test"]["auc.std"]))
elif index_metric=="pearson":
index_html.write("<td>%s (p=%s)</td>\n" % (metrics["train"]["pearson.r"], metrics["train"]["pearson.p"]))
index_html.write("<td><b>%s</b> (p=%s)</td>\n" % (metrics["test"]["pearson.r"], metrics["test"]["pearson.p"]))
index_html.write("</tr>\n")
# Build page showing filters and sequence logos for this specific TF
tf_html = open(reportdir+"/"+tfid+"/index.html", 'w')
tf_html.write("<html><head><title>Training report - %s</title></head><body>\n" % tfid)
tf_html.write("<h2>%s</h2>\n" % tfid)
# tf_html.write("<a href=\"../gmaps/%s/index.html\">gradient maps</a>)<hr/>\n"%(tfid,tfid))
tf_html.write("""
<script language="javascript">
function toggle_strand()
{
var pfms = document.getElementsByClassName('pfm')
for (var i = 0; i < pfms.length; i++)
if (pfms[i].src.search("_fwd") != -1)
pfms[i].src = pfms[i].src.replace("_fwd","_rev");
else if (pfms[i].src.search("_rev") != -1)
pfms[i].src = pfms[i].src.replace("_rev","_fwd");
}
</script></head><body>
""")
with open(metrics_file) as f:
metrics_text = f.read()
tf_html.write("<pre>%s</pre>\n"% metrics_text)
if os.path.exists(modeldir+"/%s/predict.scatter-test.png" % tfid):
tf_html.write("<table cellspacing=0 cellpadding=0 style=\"display:inline;margin-right:20px;\"><tr><td align=center>TEST predictions</td></tr><tr><td><img src=\"../../final/%s/predict.scatter-test.png\"/></td></tr></table>\n" % tfid)
if os.path.exists(modeldir+"/%s/predict.scatter-train.png" % tfid):
tf_html.write("<table cellspacing=0 cellpadding=0 style=\"display:inline\"><tr><td align=center>TRAINING predictions</td></tr><tr><td><img src=\"../../final/%s/predict.scatter-train.png\"/></td></tr></table>\n" % tfid)
# Then generate a table for the complete model
tfdir = modeldir+"/"+tfid
tf_html.write("<hr/><h3>Feature Logos</h3>\n")
tf_html.write("<input type=\"button\" value=\"toggle strand\" onclick=\"toggle_strand();\"/><br/>")
tf_html.write("<table cellspacing=0 cellpadding=4 border=0>\n")
for filter_index, logo, ic, count in logos:
tf_html.write("<tr><td>%d</td><td><img src=\"%s\" class=\"pfm\"/></td><td>%.1f bits,</td><td> %d activations</td></tr>" % (filter_index, logo, ic, count))
tf_html.write("</table><br/><br/>\n")
# Now show the actual model.
'''
shutil.copy(tfdir_final[0]+"/fold0.report/filters_.conv_seq(0).color.png", basedir+"report/%s/final_filters.png" % (tfid))
shutil.copy(tfdir_final[0]+"/fold0.report/filters_.conv_seq(0).logo.png", basedir+"report/%s/final_logos.png" % (tfid))
shutil.copy(tfdir_final[0]+"/fold0.report/filters_.conv_seq(1).png", basedir+"report/%s/final_biases.png" % (tfid))
shutil.copy(tfdir_final[0]+"/fold0.report/filters_.combiner.png", basedir+"report/%s/final_weights.png" % (tfid))
os.system("convert %sreport/%s/final_filters.png -rotate -90 %sreport/%s/final_filters_rot.png" % (basedir,tfid,basedir,tfid))
'''
tf_html.write("<hr/><h3>Actual DeepBind model</h3>\n")
if os.path.exists(tfdir + "/model.conv_seq(1).color.png"):
tf_html.write("Filters:<br/>\n")
tf_html.write("<img src=\"../../final/%s/model.conv_seq(1).color.png\"/><br/>\n" % tfid)
if os.path.exists(tfdir + "/model.combiner.png"):
tf_html.write("<br/>Combiner layer:<br/>\n")
tf_html.write("<img src=\"../../final/%s/model.combiner.png\"/><br/>\n" % tfid)
tf_html.write("</body></html>\n")
index_html.write("</table>\n")
index_html.write("</body></html>\n")
index_html.close()
def save_report(modeldir, reportdir, tfids, index_metric="auc", rna=False):
makepath(reportdir)
index_html = open(reportdir+"/index.html",'w')
index_html.write("<html><head><title>Training report</title></head><body>\n")
index_html.write("<table cellspacing=0 cellpadding=5 border=1>\n")
index_html.write("<tr><th>Name</th><th>train %s</th><th>test %s</th></tr>\n" % (index_metric, index_metric))
performance_txt = open(reportdir+"/performance.txt",'w')
performance_txt.write("\ttrain.%s\ttest.%s\n" % (index_metric, index_metric))
for tfid in tfids:
print tfid
makepath(reportdir+"/"+tfid)
# Load PFMs, convert them to logo images, and dump them to the report directory
logos = []
if os.path.exists(reportdir+"/%s.pfms.pkl"%tfid):
with open(reportdir+"/%s.pfms.pkl"%tfid) as f:
_ = cPickle.load(f)
pfms = _["pfms"]
ics = _["ic"]
counts = _["counts"]
pfm_order = np.argsort(-ics)
for j in pfm_order:
pfm = pfms[j]
ic = ics[j]
if ic <= 0:
continue
pfm_rev = np.fliplr(np.flipud(pfm))
logo_filename = "%s/%s/pfm%02d" % (reportdir, tfid, len(logos))
logos.append((j, os.path.basename(logo_filename)+"_fwd.png", ic, counts[j]))
logo_fwd = deepity.tape2logo.tape2logo(pfm.T, height=50, letterwidth=10, bufferzoom=4, vmax=1.0, style="seqlogo", rna=rna)
logo_rev = deepity.tape2logo.tape2logo(pfm_rev.T, height=50, letterwidth=10, bufferzoom=4, vmax=1.0, style="seqlogo", rna=rna)
scipy.misc.imsave(logo_filename+"_fwd.png", logo_fwd)
scipy.misc.imsave(logo_filename+"_rev.png", logo_rev)
# Load train/test metrics so we can print them in the HTML report
metrics_file = "%s/%s/metrics.txt" % (modeldir, tfid)
metrics = deepity.load_metrics(metrics_file)
# Add row for this TF in main index.html
index_html.write("<tr>\n")
index_html.write("<td><a href=\"%s/index.html\">%s</a></td>\n" % (tfid, tfid))
train_performance = ""
test_performance = ""
if index_metric=="auc":
if "train" in metrics:
index_html.write("<td>%s ± %s</td>\n" % (metrics["train"]["auc.mean"], metrics["train"]["auc.std"]))
train_performance = metrics["train"]["auc.mean"]
if "test" in metrics:
index_html.write("<td><b>%s</b> ± %s</td>\n" % (metrics["test"]["auc.mean"], metrics["test"]["auc.std"]))
test_performance = metrics["test"]["auc.mean"]
elif index_metric=="pearson":
if "train" in metrics:
index_html.write("<td>%s (p=%s)</td>\n" % (metrics["train"]["pearson.r"], metrics["train"]["pearson.p"]))
train_performance = metrics["train"]["pearson.r"]
if "test" in metrics:
index_html.write("<td><b>%s</b> (p=%s)</td>\n" % (metrics["test"]["pearson.r"], metrics["test"]["pearson.p"]))
train_performance = metrics["test"]["pearson.r"]
index_html.write("</tr>\n")
performance_txt.write("%s\t%s\t%s\n" % (tfid, train_performance, test_performance))
# Build page showing filters and sequence logos for this specific TF
tf_html = open(reportdir+"/"+tfid+"/index.html", 'w')
tf_html.write("<html><head><title>Training report - %s</title></head><body>\n" % tfid)
tf_html.write("<h2>%s</h2>\n" % tfid)
# tf_html.write("<a href=\"../gmaps/%s/index.html\">gradient maps</a>)<hr/>\n"%(tfid,tfid))
tf_html.write("""
<script language="javascript">
function toggle_strand()
{
var pfms = document.getElementsByClassName('pfm')
for (var i = 0; i < pfms.length; i++)
if (pfms[i].src.search("_fwd") != -1)
pfms[i].src = pfms[i].src.replace("_fwd","_rev");
else if (pfms[i].src.search("_rev") != -1)
pfms[i].src = pfms[i].src.replace("_rev","_fwd");
}
</script></head><body>
""")
with open(metrics_file) as f:
metrics_text = f.read()
tf_html.write("<pre>%s</pre>\n"% metrics_text)
if os.path.exists(modeldir+"/%s/predict.scatter-train.png" % tfid):
tf_html.write("<br/>Distribution of training predictions:<br/><img src=\"../../final/%s/predict.scatter-train.png\"/>\n" % tfid)
# Then generate a table for the complete model
tfdir = modeldir+"/"+tfid
if logos:
tf_html.write("<hr/><h3>Feature Logos</h3>\n")
tf_html.write("<input type=\"button\" value=\"toggle strand\" onclick=\"toggle_strand();\"/><br/>")
tf_html.write("<table cellspacing=0 cellpadding=4 border=0>\n")
for filter_index, logo, ic, count in logos:
tf_html.write("<tr><td>%d</td><td><img src=\"%s\" class=\"pfm\"/></td><td>%.1f bits,</td><td> %d activations</td></tr>" % (filter_index, logo, ic, count))
tf_html.write("</table><br/><br/>\n")
# Now show the actual model.
'''
shutil.copy(tfdir_final[0]+"/fold0.report/filters_.conv_seq(0).color.png", basedir+"report/%s/final_filters.png" % (tfid))
shutil.copy(tfdir_final[0]+"/fold0.report/filters_.conv_seq(0).logo.png", basedir+"report/%s/final_logos.png" % (tfid))
shutil.copy(tfdir_final[0]+"/fold0.report/filters_.conv_seq(1).png", basedir+"report/%s/final_biases.png" % (tfid))
shutil.copy(tfdir_final[0]+"/fold0.report/filters_.combiner.png", basedir+"report/%s/final_weights.png" % (tfid))
os.system("convert %sreport/%s/final_filters.png -rotate -90 %sreport/%s/final_filters_rot.png" % (basedir,tfid,basedir,tfid))
'''
tf_html.write("<hr/><h3>Actual DeepBind model</h3>\n")
if os.path.exists(tfdir + "/model.conv_seq(1).color.png"):
tf_html.write("Filters:<br/>\n")
tf_html.write("<img src=\"../../final/%s/model.conv_seq(1).color.png\"/><br/>\n" % tfid)
if os.path.exists(tfdir + "/model.combiner.png"):
tf_html.write("<br/>Combiner layer:<br/>\n")
tf_html.write("<img src=\"../../final/%s/model.combiner.png\"/><br/>\n" % tfid)
tf_html.write("</body></html>\n")
index_html.write("</table>\n")
index_html.write("</body></html>\n")
index_html.close()
