def form_none(mongo_results=instantiate_mongo(False).find({"": ""}), nav_caption=""):
"""
Flask callback function for all requests
path_to_url: /mnt/projects/userrig/solexa/.. -> rpd/userrig/runs/solexaProjects/..
"""
result = ""
result += ("<script>$(function(){$('.nav_caption').replaceWith('" \
+ '<span class="nav_caption">' + nav_caption + "</span>" + "');});</script>")
analysis_none = 0
analysis_started = 0
analysis_failed = 0
analysis_success = 0
for record in mongo_results:
result += "<tr class='run_row'>"
result += ("<td>" + str(record["run"]) + "</td>")
if len(str(record["timestamp"])) == 13:
result += ("<td>" + str(datetime.fromtimestamp(
record["timestamp"] / 1000).isoformat()).replace(":", "-") + "</td>")
else:
result += ("<td>" + str(record["timestamp"]) + "</td>")
result += "<td>"
if "analysis" in record:
if "Status" in record["analysis"][-1]:
if record["analysis"][-1]["Status"] == "STARTED":
analysis_started += 1
result += "<div class='hidden'>FINAL_STARTED</div>"
# elif record["analysis"][-1]["Status"].find("FAILED") != -1:
elif record["analysis"][-1]["Status"] == "FAILED":
analysis_failed += 1
result += "<div class='hidden'>FINAL_FAILED</div>"
elif record["analysis"][-1]["Status"] == "SUCCESS":
analysis_success += 1
result += "<div class='hidden'>FINAL_SUCCESS</div>"
else:
result += "<div class='hidden'>FINAL_SEQRUNFAILED</div>"
else:
result += "<div class='hidden'>FINAL_NONE</div>"
result += """
<table class='table table-bordered table-hover table-fixed table-compact analysis_table'>
<thead>
<tr>
<th>ANALYSIS_ID</th>
<th>END_TIME</th>
<th>OUT_DIR</th>
<th>STATUS</th>
<th>MUX</th>
</tr>
</thead>
<tbody class='analysis_tbody'>
"""
analysis_counter = len(record["analysis"])
for analysis in record["analysis"]:
analysis_counter -= 1
if analysis_counter == 0:
result += "<tr class='analysis_last_row'>"
else:
result += "<tr>"
result += ("<td>" + merge_cells("analysis_id", analysis) + "</td>")
result += ("<td>" + merge_cells("end_time", analysis) + "</td>")
result += ("<td><a href='" + path_to_url(
merge_cells("out_dir", analysis)).replace("//", "/").replace(":/", "://") \
+ "'>" + merge_cells("out_dir", analysis).replace("//", "/") + "</a></td>")
result += ("<td>" + merge_cells("Status", analysis) + "</td>")
result += "<td>"
if "per_mux_status" in analysis:
result += """
<table class='table table-bordered table-hover table-fixed table-compact mux_table'>
<thead>
<tr>
<th>MUX_ID</th>
<th>ARCHIVE</th>
<th>DOWNSTREAM</th>
<th>STATS</th>
<th>STATUS</th>
<th>EMAIL</th>
</tr>
</thead>
<tbody class='mux_tbody'>
"""
for mux in analysis["per_mux_status"]:
result += "<tr>"
result += ("<td>" + merge_cells("mux_id", mux) + "</td>")
result += ("<td>" + merge_cells("ArchiveSubmission", mux) + "</td>")
result += ("<td>" + merge_cells("DownstreamSubmission", mux) + "</td>")
result += ("<td>" + merge_cells("StatsSubmission", mux) + "</td>")
result += ("<td>" + merge_cells("Status", mux) + "</td>")
result += ("<td>" + merge_cells("email_sent", mux) + "</td>")
result += "</tr>"
result += "</tbody></table>"
else:
# result += "<span class='label label-pill label-default'>NONE</span>"
result += """
<table class='table table-bordered table-hover table-fixed table-compact invisible'>
<thead>
<tr>
<th>MUX_ID</th>
<th>ARCHIVE</th>
<th>DOWNSTREAM</th>
<th>STATS</th>
<th>STATUS</th>
<th>EMAIL</th>
</tr>
</thead>
<tbody>
"""
result += "</tbody></table>"
result += "</td>"
result += "</tbody></table>"
else:
analysis_none += 1
result += "<span class='label label-pill label-default'>NONE</span>"
result += "</td>"
result += "</tr>"
result += "</tr>"
result += ("<script>$(function(){$('#analysis_none').attr('data-badge', '" \
+ str(analysis_none) + "');});</script>")
result += ("<script>$(function(){$('#analysis_started').attr('data-badge', '" \
+ str(analysis_started) + "');});</script>")
result += ("<script>$(function(){$('#analysis_failed').attr('data-badge', '" \
+ str(analysis_failed) + "');});</script>")
result += ("<script>$(function(){$('#analysis_success').attr('data-badge', '" \
+ str(analysis_success) + "');});</script>")
return render_template("index.html", result=Markup(result))
def main():
"""main function
"""
stats_upload_script = os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]), "bcl_stats_upload.py"))
assert os.path.exists(stats_upload_script)
archive_upload_script = os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]), "sra_fastq_upload.py"))
assert os.path.exists(archive_upload_script)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-t',
"--testing",
action='store_true',
help="Use MongoDB test server")
default = 14
parser.add_argument(
'-w',
'--win',
type=int,
default=default,
help="Number of days to look back (default {})".format(default))
parser.add_argument('-n', "--dry-run", action='store_true', help="Dry run")
parser.add_argument('-v',
'--verbose',
action='count',
default=0,
help="Increase verbosity")
parser.add_argument('-q',
'--quiet',
action='count',
default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10 * args.quiet - 10 * args.verbose)
if not is_production_user():
logger.warning("Not a production user. Skipping sending of emails")
sys.exit(1)
connection = mongodb_conn(args.testing)
if connection is None:
sys.exit(1)
db = connection.gisds.runcomplete
epoch_present, epoch_back = generate_window(args.win)
num_emails = 0
results = db.find({
"analysis": {
"$exists": True
},
"timestamp": {
"$gt": epoch_back,
"$lt": epoch_present
}
})
logger.info("Found %s runs", results.count())
if is_devel_version() or args.testing:
mail_to = 'veeravallil' # domain added in mail function
else:
#mail_to = '*****@*****.**'
mail_to = '*****@*****.**'
for record in results:
run_number = record['run']
#print(run_number)
for (analysis_count, analysis) in enumerate(record['analysis']):
analysis_id = analysis['analysis_id']
per_mux_status = analysis.get("per_mux_status", None)
if per_mux_status is None:
continue
for (mux_count, mux_status) in enumerate(per_mux_status):
if args.dry_run:
logger.warning(
"Skipping analysis %s run %s MUX %s"
" with email_sent %s", analysis_id,
run_number, mux_status['mux_id'],
mux_status.get('email_sent', None))
continue
if mux_status.get('email_sent', None):
continue
# for all others: send email and update db
email_sent_query = "analysis.{}.per_mux_status.{}.email_sent".format(
analysis_count, mux_count)
mux_id = mux_status['mux_id']
out_dir = analysis['out_dir']
if mux_status.get('Status', None) == "FAILED":
logger.info("bcl2fastq for MUX %s from %s failed. ",
mux_status['mux_id'], run_number)
subject = 'bcl2fastq: ' + mux_id
body = "bcl2fastq for {} from {} failed.".format(
mux_id, run_number)
body += "\n\nPlease check the logs under {}".format(
out_dir + "/logs")
send_mail(subject, body, mail_to, ccaddr="rpd")
num_emails += 1
update_mongodb_email(db, run_number, analysis_id,
email_sent_query, True)
elif mux_status.get('Status', None) == "SUCCESS":
muxdir = os.path.join(out_dir, 'out',
mux_status.get('mux_dir'))
summary = path_to_url(
os.path.join(muxdir, 'html/index.html'))
body = "bcl2fastq for {} from {} successfully completed.".format(
mux_id, run_number)
body += "\n\nA summary can be found at {}".format(summary)
body += "\n\nFastQ files are located in {}".format(muxdir)
body += "\n\nData can also be downloaded from GIS-SRA (once archival is complete)"
confinfo = os.path.join(out_dir, 'conf.yaml')
#print(body)
if not os.path.exists(confinfo):
logger.fatal(
"conf info '%s' does not exist"
" under run directory.", confinfo)
continue
subject = 'bcl2fastq'
if args.testing:
subject += ' testing'
if is_devel_version():
subject += ' devel'
subject += ': ' + mux_id
send_mail(subject, body, mail_to,
ccaddr="rpd") # mail_to already set
if not args.testing and not is_devel_version():
requestor = get_requestor(mux_id, confinfo)
if requestor is not None:
#requestor = "rpd"
#subject += " (instead of requestor)"
#send_mail(subject, body, requestor, ccaddr="rpd")
send_mail(subject, body, requestor)
num_emails += 1
update_mongodb_email(db, run_number, analysis_id,
email_sent_query, True)
# close the connection to MongoDB
connection.close()
logger.info("%d emails sent", num_emails)
def form_none(
mongo_results=instantiate_mongo(False).find({"": ""}), nav_caption=""):
"""
Flask callback function for all requests
path_to_url: /mnt/projects/userrig/solexa/.. -> rpd/userrig/runs/solexaProjects/..
"""
result = ""
result += ("<script>$(function(){$('.nav_caption').replaceWith('" \
+ '<span class="nav_caption">' + nav_caption + "</span>" + "');});</script>")
analysis_none = 0
analysis_started = 0
analysis_failed = 0
analysis_success = 0
for record in mongo_results:
result += "<tr class='run_row'>"
result += ("<td>" + str(record["run"]) + "</td>")
if len(str(record["timestamp"])) == 13:
result += ("<td>" + str(
datetime.fromtimestamp(record["timestamp"] /
1000).isoformat()).replace(":", "-") +
"</td>")
else:
result += ("<td>" + str(record["timestamp"]) + "</td>")
if "raw-delete" in record:
result += "<td>" + merge_cells("Status",
record["raw-delete"]) + "</td>"
else:
result += "<td>" + "</td>"
result += "<td>"
if "analysis" in record:
if "Status" in record["analysis"][-1]:
if record["analysis"][-1]["Status"] == "STARTED":
analysis_started += 1
result += "<div class='hidden'>FINAL_STARTED</div>"
# elif record["analysis"][-1]["Status"].find("FAILED") != -1:
elif record["analysis"][-1]["Status"] == "FAILED":
analysis_failed += 1
result += "<div class='hidden'>FINAL_FAILED</div>"
elif record["analysis"][-1]["Status"] == "SUCCESS":
analysis_success += 1
result += "<div class='hidden'>FINAL_SUCCESS</div>"
else:
result += "<div class='hidden'>FINAL_SEQRUNFAILED</div>"
# else:
# result += "<div class='hidden'>FINAL_NONE</div>"
result += """
<table class='table table-bordered table-hover table-fixed table-compact analysis_table'>
<thead>
<tr>
<th>ANALYSIS_ID</th>
<th>END_TIME</th>
<th>OUT_DIR</th>
<th>STATUS</th>
<th>MUX</th>
</tr>
</thead>
<tbody class='analysis_tbody'>
"""
analysis_counter = len(record["analysis"])
for analysis in record["analysis"]:
analysis_counter -= 1
if analysis_counter == 0:
result += "<tr class='analysis_last_row'>"
else:
result += "<tr>"
result += ("<td>" + merge_cells("analysis_id", analysis) +
"</td>")
result += ("<td>" + merge_cells("end_time", analysis) +
"</td>")
result += ("<td><a href='" + path_to_url(
merge_cells("out_dir", analysis)).replace("//", "/").replace(":/", "://") \
+ "'>" + merge_cells("out_dir", analysis).replace("//", "/") + "</a></td>")
result += ("<td>" + merge_cells("Status", analysis) + "</td>")
result += "<td>"
if "per_mux_status" in analysis:
result += """
<table class='table table-bordered table-hover table-fixed table-compact mux_table'>
<thead>
<tr>
<th>MUX_ID</th>
<th>ARCHIVE</th>
<th>DOWNSTREAM</th>
<th>STATS</th>
<th>STATUS</th>
<th>EMAIL</th>
</tr>
</thead>
<tbody class='mux_tbody'>
"""
for mux in sorted(analysis["per_mux_status"],
key=itemgetter("mux_id")):
result += "<tr>"
result += ("<td>" + merge_cells("mux_id", mux) +
"</td>")
result += ("<td>" +
merge_cells("ArchiveSubmission", mux) +
"</td>")
result += ("<td>" +
merge_cells("DownstreamSubmission", mux) +
"</td>")
result += ("<td>" +
merge_cells("StatsSubmission", mux) +
"</td>")
result += ("<td>" + merge_cells("Status", mux) +
"</td>")
result += ("<td>" + merge_cells("email_sent", mux) +
"</td>")
result += "</tr>"
result += "</tbody></table>"
else:
# result += "<span class='label label-pill label-default'>NONE</span>"
result += """
<table class='table table-bordered table-hover table-fixed table-compact invisible'>
<thead>
<tr>
<th>MUX_ID</th>
<th>ARCHIVE</th>
<th>DOWNSTREAM</th>
<th>STATS</th>
<th>STATUS</th>
<th>EMAIL</th>
</tr>
</thead>
<tbody>
"""
result += "</tbody></table>"
result += "</td>"
result += "</tbody></table>"
else:
analysis_none += 1
result += "<div class='hidden'>FINAL_NONE</div>"
result += "<span class='label label-pill label-default'>NONE</span>"
# result += "<td>"
# if "raw-delete" in record:
# result += """
# <table class='table table-bordered table-hover table-fixed table-compact raw_delete_table'>
# <thead>
# <tr>
# <th>START_TIME</th>
# <th>END_TIME</th>
# <th>STATUS</th>
# </tr>
# </thead>
# <tbody class='raw_delete_tbody'>
# <tr>
# """
# result += "<td>" + record["raw-delete"]["start_time"] + "</td>"
# result += "<td>" + record["raw-delete"]["end_time"] + "</td>"
# result += "<td>" + merge_cells("Status", record["raw-delete"]) + "</td>"
# result += "</tr></tbody></table>"
# result += "</td>"
result += "</td>"
result += "</tr>"
result += "</tr>"
result += ("<script>$(function(){$('#analysis_none').attr('data-badge', '" \
+ str(analysis_none) + "');});</script>")
result += ("<script>$(function(){$('#analysis_started').attr('data-badge', '" \
+ str(analysis_started) + "');});</script>")
result += ("<script>$(function(){$('#analysis_failed').attr('data-badge', '" \
+ str(analysis_failed) + "');});</script>")
result += ("<script>$(function(){$('#analysis_success').attr('data-badge', '" \
+ str(analysis_success) + "');});</script>")
return render_template("index.html", result=Markup(result))
def main():
"""main function
"""
stats_upload_script = os.path.abspath(os.path.join(
os.path.dirname(sys.argv[0]), "bcl_stats_upload.py"))
assert os.path.exists(stats_upload_script)
archive_upload_script = os.path.abspath(os.path.join(
os.path.dirname(sys.argv[0]), "sra_fastq_upload.py"))
assert os.path.exists(archive_upload_script)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-t', "--testing", action='store_true',
help="Use MongoDB test server")
default = 14
parser.add_argument('-w', '--win', type=int, default=default,
help="Number of days to look back (default {})".format(default))
parser.add_argument('-n', "--dry-run", action='store_true',
help="Dry run")
parser.add_argument('-v', '--verbose', action='count', default=0,
help="Increase verbosity")
parser.add_argument('-q', '--quiet', action='count', default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)
user_name = getpass.getuser()
if user_name != "userrig":
logger.warning("Not a production user. Skipping sending of emails")
sys.exit(0)
connection = mongodb_conn(args.testing)
if connection is None:
sys.exit(1)
db = connection.gisds.runcomplete
epoch_present, epoch_back = generate_window(args.win)
num_emails = 0
results = db.find({"analysis" : {"$exists": True},
"timestamp": {"$gt": epoch_back, "$lt": epoch_present}})
logger.info("Found %s runs", results.count())
if is_devel_version() or args.testing:
mail_to = 'veeravallil'# domain added in mail function
else:
#mail_to = '*****@*****.**'
mail_to = '*****@*****.**'
for record in results:
run_number = record['run']
#print(run_number)
for (analysis_count, analysis) in enumerate(record['analysis']):
analysis_id = analysis['analysis_id']
per_mux_status = analysis.get("per_mux_status", None)
if per_mux_status is None:
continue
for (mux_count, mux_status) in enumerate(per_mux_status):
if args.dry_run:
logger.warning("Skipping analysis %s run %s MUX %s"
" with email_sent %s",
analysis_id, run_number, mux_status['mux_id'],
mux_status.get('email_sent', None))
continue
if mux_status.get('email_sent', None):
continue
# for all others: send email and update db
email_sent_query = "analysis.{}.per_mux_status.{}.email_sent".format(
analysis_count, mux_count)
mux_id = mux_status['mux_id']
out_dir = analysis['out_dir']
if mux_status.get('Status', None) == "FAILED":
logger.info("bcl2fastq for MUX %s from %s failed. ",
mux_status['mux_id'], run_number)
subject = 'bcl2fastq: ' + mux_id
body = "bcl2fastq for {} from {} failed.".format(mux_id, run_number)
body += "\n\nPlease check the logs under {}".format(out_dir + "/logs")
send_mail(subject, body, mail_to, ccaddr="rpd")
num_emails += 1
update_mongodb_email(db, run_number, analysis_id, email_sent_query, True)
elif mux_status.get('Status', None) == "SUCCESS":
muxdir = os.path.join(out_dir, 'out', mux_status.get('mux_dir'))
summary = path_to_url(os.path.join(muxdir, 'html/index.html'))
body = "bcl2fastq for {} from {} successfully completed.".format(
mux_id, run_number)
body += "\n\nA summary can be found at {}".format(summary)
body += "\n\nFastQ files are located in {}".format(muxdir)
body += "\n\nData can also be downloaded from GIS-SRA (once archival is complete)"
confinfo = os.path.join(out_dir, 'conf.yaml')
#print(body)
if not os.path.exists(confinfo):
logger.fatal("conf info '%s' does not exist"
" under run directory.", confinfo)
continue
subject = 'bcl2fastq'
if args.testing:
subject += ' testing'
if is_devel_version():
subject += ' devel'
subject += ': ' + mux_id
send_mail(subject, body, mail_to, ccaddr="rpd")# mail_to already set
if not args.testing and not is_devel_version():
requestor = get_requestor(mux_id, confinfo)
if requestor is not None:
#requestor = "rpd"
#subject += " (instead of requestor)"
#send_mail(subject, body, requestor, ccaddr="rpd")
send_mail(subject, body, requestor)
num_emails += 1
update_mongodb_email(db, run_number, analysis_id, email_sent_query, True)
# close the connection to MongoDB
connection.close()
logger.info("%d emails sent", num_emails)
def main():
"""main function
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-d', "--bcl2fastq-dir", required=True,
help="bcl2fastq directory (containing a conf.yaml)")
parser.add_argument('--no-mail', action='store_true',
help="Don't send email on detected failures")
parser.add_argument('-v', '--verbose', action='count', default=0,
help="Increase verbosity")
parser.add_argument('-q', '--quiet', action='count', default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)
assert os.path.isdir(args.bcl2fastq_dir)
conf_file = os.path.join(args.bcl2fastq_dir, 'conf.yaml')
if not os.path.exists(conf_file):
logger.fatal("Expected config file missing: %s", conf_file)
sys.exit(1)
with open(conf_file) as fh:
bcl2fastq_cfg = yaml.safe_load(fh)
run_num = bcl2fastq_cfg["run_num"]
machine_type = get_machine_type_from_run_num(run_num)
project_dirs = []
for _, mux_info in bcl2fastq_cfg["units"].items():
d = os.path.join(args.bcl2fastq_dir, "out", mux_info['mux_dir'])
if not os.path.exists(d):
logger.warning("Ignoring missing directory %s", d)
else:
project_dirs.append(d)
if len(project_dirs) == 0:
logger.error("Exiting because no project directories where found in %s", args.bcl2fastq_dir)
sys.exit(1)
qcfails = run_qc_checks(project_dirs, machine_type)
if qcfails:
subject = "bcl2fastq QC checks failed for {} ({}):".format(
run_num, args.bcl2fastq_dir)
body = "The following " + subject
for f in qcfails:
body += "\n- {}".format(f)
body += "\nPlease double-check here: {}\n".format(
path_to_url(args.bcl2fastq_dir))
body += "QC_FAILED"# signal for callers
print(subject + "\n" + body)
if not args.no_mail:
email_qcfails(subject, body)
else:
print("QC checks completed. No tests failed")
print("QC SUCCESS")# signal for callers
