def email_non_bcl(libraryId, runId):
"""send email for non-bcl libraries
"""
if is_devel_version():
toaddr = email_for_user()
else:
toaddr = "*****@*****.**"
subject = "bcl2fastq conversion not required for {} from run {}.".format(
libraryId, runId)
body = subject + "\n" + "Kindly start custom analysis manually. Thanks."
send_mail(subject, body, toaddr=toaddr, pass_exception=False)
def email_qcfails(subject, body):
"""email qc failures
"""
if is_devel_version():
toaddr = email_for_user()
ccaddr = None
else:
toaddr = config['email']
ccaddr = "*****@*****.**"
send_mail(subject, body, toaddr=toaddr, ccaddr=ccaddr,
pass_exception=False)
def purge(db, runid_and_flowcellid, mail_to):
"""
purging bcl data from /mnt/seq/novogene
"""
rundir = get_bcl_runfolder_for_runid(runid_and_flowcellid)
if not os.path.exists(rundir):
LOGGER.critical("Run directory '%s' does not exist.\n", rundir)
return
# Sanity checks for Sequencing run
assert os.path.exists(os.path.join(rundir, 'RunInfo.xml')), \
"No RunInfo.xml found under {}".format(rundir)
stat_info = os.stat(rundir)
#Check if uid is novogene (925)
assert stat_info.st_uid == 925, "The run {} does not belong to Novogene user".format(
rundir)
try:
start_time = generate_timestamp()
res = db.update_one({"run": runid_and_flowcellid}, \
{"$set": \
{"raw-delete": { \
"start_time" : start_time, \
"Status" : "STARTED", \
}}})
assert res.modified_count == 1, ("Modified {} documents instead of 1". \
format(res.modified_count))
#FIXME for production release
#shutil.rmtree(rundir)
end_time = generate_timestamp()
res = db.update_one({"run": runid_and_flowcellid},
{"$set": {"raw-delete.Status": "SUCCESS", \
"raw-delete.end_time": end_time}})
assert res.modified_count == 1, ("Modified {} documents instead of 1". \
format(res.modified_count))
subject = "bcl deletion: {}".format(runid_and_flowcellid)
body = "Bcl deletion completed successfully from {}".format(rundir)
send_mail(subject, body, toaddr=mail_to)
except OSError:
LOGGER.critical("Error happened while deleting '%s'", rundir)
res = db.update_one({"run": runid_and_flowcellid}, \
{"$unset": {"raw-delete": ""}})
assert res.modified_count == 1, ("Modified {} documents instead of 1". \
format(res.modified_count))
subject = "Error: bcl deletion {}".format(runid_and_flowcellid)
body = "Error happened while deleting raw data under {}".format(rundir)
send_mail(subject, body, toaddr=mail_to)
def runs_from_db(db, mail_to, ccaddr, win=34):
"""Get the runs from pipeline_run collections"""
epoch_present, epoch_back = generate_window(win)
results = db.find({"analysis" : {"$exists": False},
"timestamp": {"$gt": epoch_back, "$lt": epoch_present}})
logger.info("Found %d runs for last %s days", results.count(), win)
mail = False
subject = "Runs with missing ELM information"
body = "Dear NGSP, " + "\n"
body += subject + " for the following runs. Please include in the ELM." + "\n"
for record in results:
logger.debug("record: %s", record)
_, runid, _ = get_machine_run_flowcell_id(record.get('run'))
rest_data = get_rest_data(runid)
if not rest_data.get('runId'):
body += record.get('run')+ "\n"
mail = True
if mail:
send_mail(subject, body, toaddr=mail_to, ccaddr=ccaddr)
def main():
"""main function
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-t',
"--testing",
action='store_true',
help="Use MongoDB test server")
parser.add_argument('-n',
"--no-mail",
action='store_true',
help="Don't mail. Just print to console")
default = 14
parser.add_argument(
'-w',
'--win',
type=int,
default=default,
help="Number of days to look back (default {})".format(default))
parser.add_argument('-v',
'--verbose',
action='count',
default=0,
help="Increase verbosity")
parser.add_argument('-q',
'--quiet',
action='count',
default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10 * args.quiet - 10 * args.verbose)
connection = mongodb_conn(args.testing)
if connection is None:
sys.exit(1)
db = connection.gisds.runcomplete
epoch_present, epoch_back = generate_window(args.win)
results = db.find({"timestamp": {"$gt": epoch_back, "$lt": epoch_present}})
runs = {}
extra_text = ""
for record in results:
run_number = record['run']
timestamp = record['timestamp']
runs[timestamp] = run_number
od = collections.OrderedDict(sorted(runs.items()))
logger.info("Found %s runs", results.count())
extra_text = "Found {} runs. \n".format(results.count())
for _, v in od.items(): # v is run
results = db.find({"run": v})
for record in results:
if not 'analysis' in record:
continue
last_analysis = record['analysis'][-1]
status = last_analysis.get("Status")
if not status:
continue
if status == 'SUCCESS':
if last_analysis.get("per_mux_status"):
mux = last_analysis.get("per_mux_status")
for d in mux:
if d is None:
logger.warning("Skipping empty per_mux_status for run %s." \
"Needs fix in DB", v)
continue
if d.get(
'Status'
) == "SUCCESS": # FIXME what if key is missing?
mux_id = d['mux_id']
stats_submission = d['StatsSubmission']
if stats_submission == "FAILED":
extra_text += "StatsSubmission for mux_id {} from run {} " \
"has FAILED and out_dir is {} \n" \
.format(mux_id, v, last_analysis.get("out_dir"))
extra_text += "\n"
archive_submission = d['ArchiveSubmission']
if archive_submission == "FAILED":
extra_text += "ArchiveSubmission for mux_id {} from run {} " \
"has FAILED and out_dir is {} \n" \
.format(mux_id, v, last_analysis.get("out_dir"))
extra_text += "\n"
elif status == 'FAILED':
extra_text += "Analysis for run {} has failed. \n".format(v)
extra_text += "Analysis_id is {} and out_dir is {} \n" \
.format(last_analysis.get("analysis_id"), \
last_analysis.get("out_dir"))
extra_text += "\n"
extra_text += "---------------------------------------------------\n"
logger.info("Analysis for run %s has failed ", v)
elif status == 'STARTED':
analysis_id = last_analysis.get("analysis_id")
analysis_epoch_time = isoformat_to_epoch_time(analysis_id +
"+08:00")
run_completion_time = timestamp / 1000
rd = relative_epoch_time(run_completion_time,
analysis_epoch_time)
if rd.days > 3:
extra_text += "Analysis for run {} was started {} days ago. "\
"Please check. \n".format(v, rd.days)
extra_text += "Analysis_id is {} and out_dir is {} \n" \
.format(last_analysis.get("analysis_id"), \
last_analysis.get("out_dir"))
extra_text += "\n"
extra_text += "---------------------------------------------------\n"
extra_text += "Report generation is completed"
subject = "Report generation for bcl2fastq"
if args.testing:
subject = "Testing:" + subject
if args.no_mail:
print(
"Skipping sending of email with subject '{}' and following body:".
format(subject))
print(extra_text)
else:
send_mail(subject, extra_text)
logger.info("Report generation is completed")
def main():
"""main function
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-n',
"--dry-run",
action='store_true',
help="Don't run anything")
parser.add_argument(
'-t',
"--testing",
action='store_true',
help=
"Use MongoDB test-server here and when calling bcl2fastq wrapper (-t)")
default = 14
parser.add_argument(
'-w',
'--win',
type=int,
default=default,
help="Number of days to look back (default {})".format(default))
parser.add_argument('-v',
'--verbose',
action='count',
default=0,
help="Increase verbosity")
parser.add_argument('-q',
'--quiet',
action='count',
default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10 * args.quiet - 10 * args.verbose)
user_name = getpass.getuser()
if user_name != "userrig":
logger.warning("Not a production user. Skipping MongoDB update")
sys.exit(0)
connection = mongodb_conn(args.testing)
if connection is None:
sys.exit(1)
db = connection.gisds.runcomplete
epoch_present, epoch_back = generate_window(args.win)
results = db.find({
"analysis.per_mux_status": {
"$exists": True
},
"timestamp": {
"$gt": epoch_back,
"$lt": epoch_present
}
})
logger.info("Found %s runs", results.count())
run_list = {}
mongo_db_ref = {}
for record in results:
run_number = record['run']
mux_list = {}
for (analysis_count, analysis) in enumerate(record['analysis']):
analysis_id = analysis['analysis_id']
per_mux_status = analysis.get("per_mux_status", None)
if per_mux_status is None:
continue
for (mux_count, mux_status) in enumerate(per_mux_status):
# sanity checks against corrupted DB entries
if mux_status is None or mux_status.get('mux_id') is None:
logger.warning("mux_status is None or incomplete for run %s analysis %s."
" Requires fix in DB. Skipping entry for now.", \
run_number, analysis_id)
continue
if mux_status.get('Status', None) != "SUCCESS":
logger.info(
"MUX %s from %s is not SUCCESS. Skipping downstream analysis",
mux_status['mux_id'], run_number)
continue
mux_id = mux_status['mux_id']
out_dir = analysis['out_dir']
mux_db_id = "analysis.{}.per_mux_status.{}.DownstreamSubmission".format(
analysis_count, mux_count)
if mux_status.get('Status') == "SUCCESS" and \
mux_status.get('DownstreamSubmission') == "TODO":
mongo_list = (mux_id, mux_db_id, analysis_id)
mongo_db_ref.setdefault(run_number, []).append(mongo_list)
mux_list.setdefault(mux_id, []).append(out_dir)
for mux_id, out_dir in mux_list.items():
mux_list_success = mux_list[mux_id]
#Check if MUX has been analyzed more then 1 time successfully
if len(mux_list_success) > 1:
body = "{} has been analyzed more than 1 time successfully..".format(mux_id) \
+ "delegator is skipping the downstream analysis under {}. Please" \
"check the results.".format(mux_list_success)
subject = "Downstream delegator skipped job submission for {}".format(
mux_id)
if args.testing:
subject += " (testing)"
send_mail(subject, body, toaddr='veeravallil', ccaddr=None)
continue
mux_info = (mux_id, out_dir)
run_list.setdefault(run_number, []).append(mux_info)
for run_num_flowcell, mux_list in run_list.items():
update_status = True
pipeline_params_dict, mux_analysis_list = get_lib_details(run_num_flowcell, \
mux_list, args.testing)
if not bool(pipeline_params_dict):
logger.warning("pipeline params is empty for run num %s",
run_num_flowcell)
continue
# Insert jobs into pipeline_runs collection
for lib, lib_info in pipeline_params_dict.items():
job = {}
rd_list = {}
job['sample_cfg'] = {}
job['sample_cfg'] = {}
readunits_list = list()
rd_list['samples'] = {}
for outer_key, outer_value in lib_info.items():
if outer_key == 'readunits':
for inner_key in lib_info[outer_key]:
readunits_list.append(inner_key)
job['sample_cfg'].update({outer_key: outer_value})
if outer_key == 'references_cfg':
job['references_cfg'] = {}
job['references_cfg'] = outer_value
elif outer_key == 'cmdline':
job['cmdline'] = {}
job['cmdline'] = outer_value
elif outer_key != 'readunits':
job.update({outer_key: outer_value})
else:
rd_list['samples'][lib] = readunits_list
job['sample_cfg'].update(rd_list)
if args.dry_run:
logger.warning("Skipping job delegation for %s", \
lib)
continue
res = mongodb_insert_libjob(job, connection)
if not res:
logger.critical("Skipping rest of analysis job submission" \
"for %s from %s", lib, lib_info.run_id)
subject = "Downstream delegator failed job submission for" \
"{}".format(lib)
if args.testing:
subject += " (testing)"
body = "Downstream delegator failed to insert job submission for" \
"{}".format(lib)
send_mail(subject, body, toaddr='veeravallil', ccaddr=None)
update_status = False
logger.warning("Clean up the database for mux %s from run %s and ctime %s", \
lib_info.mux_id, lib_info.run_id, lib_info.ctime)
mongodb_remove_muxjob(lib_info.mux_id, lib_info.run_id, \
lib_info.ctime, connection)
break
# Update runcomplete collection for delegated jobs
if not args.dry_run and update_status:
value = mongo_db_ref[run_num_flowcell]
for mux_id, insert_id, analysis_id in value:
if mux_id in mux_analysis_list:
logger.info("Update mongoDb pipeline_runs for mux_id %s from run number %s" \
"and analysis_id is %s", mux_id, run_num_flowcell, analysis_id)
res = mongodb_update_runcomplete(run_num_flowcell, analysis_id, mux_id, \
insert_id, connection)
if not res:
logger.critical("Skipping rest of analysis job submission for %s" \
"from %s", mux_id, run_num_flowcell)
subject = "Downstream delegator failed job submission for {}" \
.format(mux_id)
if args.testing:
subject += " (testing)"
body = "Downstream delegator failed to insert job submission for" \
"{}".format(mux_id)
send_mail(subject,
body,
toaddr='veeravallil',
ccaddr=None)
update_status = False
break
connection.close()
def main():
"""main function
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-t', "--testing", action='store_true',
help="Use MongoDB test server")
parser.add_argument('-n', "--no-mail", action='store_true',
help="Don't mail. Just print to console")
default = 14
parser.add_argument('-w', '--win', type=int, default=default,
help="Number of days to look back (default {})".format(default))
parser.add_argument('-v', '--verbose', action='count', default=0,
help="Increase verbosity")
parser.add_argument('-q', '--quiet', action='count', default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)
connection = mongodb_conn(args.testing)
if connection is None:
sys.exit(1)
db = connection.gisds.runcomplete
epoch_present, epoch_back = generate_window(args.win)
results = db.find({"timestamp": {"$gt": epoch_back, "$lt": epoch_present}})
runs = {}
extra_text = ""
for record in results:
run_number = record['run']
timestamp = record['timestamp']
runs[timestamp] = run_number
od = collections.OrderedDict(sorted(runs.items()))
logger.info("Found %s runs", results.count())
extra_text = "Found {} runs. \n".format(results.count())
for _, v in od.items():# v is run
results = db.find({"run": v})
for record in results:
if not 'analysis' in record:
continue
last_analysis = record['analysis'][-1]
status = last_analysis.get("Status")
if not status:
continue
if status == 'SUCCESS':
if last_analysis.get("per_mux_status"):
mux = last_analysis.get("per_mux_status")
for d in mux:
if d is None:
logger.warning("Skipping empty per_mux_status for run %s." \
"Needs fix in DB", v)
continue
if d.get('Status') == "SUCCESS":# FIXME what if key is missing?
mux_id = d['mux_id']
stats_submission = d['StatsSubmission']
if stats_submission == "FAILED":
extra_text += "StatsSubmission for mux_id {} from run {} " \
"has FAILED and out_dir is {} \n" \
.format(mux_id, v, last_analysis.get("out_dir"))
extra_text += "\n"
archive_submission = d['ArchiveSubmission']
if archive_submission == "FAILED":
extra_text += "ArchiveSubmission for mux_id {} from run {} " \
"has FAILED and out_dir is {} \n" \
.format(mux_id, v, last_analysis.get("out_dir"))
extra_text += "\n"
elif status == 'FAILED':
extra_text += "Analysis for run {} has failed. \n".format(v)
extra_text += "Analysis_id is {} and out_dir is {} \n" \
.format(last_analysis.get("analysis_id"), \
last_analysis.get("out_dir"))
extra_text += "\n"
extra_text += "---------------------------------------------------\n"
logger.info("Analysis for run %s has failed ", v)
elif status == 'STARTED':
analysis_id = last_analysis.get("analysis_id")
analysis_epoch_time = isoformat_to_epoch_time(analysis_id+"+08:00")
run_completion_time = timestamp/1000
dt1 = datetime.datetime.fromtimestamp(run_completion_time)
dt2 = datetime.datetime.fromtimestamp(analysis_epoch_time)
rd = dateutil.relativedelta.relativedelta(dt1, dt2)
if rd.days > 3:
extra_text += "Analysis for run {} was started {} days ago. "\
"Please check. \n".format(v, rd.days)
extra_text += "Analysis_id is {} and out_dir is {} \n" \
.format(last_analysis.get("analysis_id"), \
last_analysis.get("out_dir"))
extra_text += "\n"
extra_text += "---------------------------------------------------\n"
extra_text += "Report generation is completed"
subject = "Report generation for bcl2fastq"
if args.testing:
subject = "Testing:" + subject
if args.no_mail:
print("Skipping sending of email with subject '{}' and following body:".format(subject))
print(extra_text)
else:
send_mail(subject, extra_text)
logger.info("Report generation is completed")
def main():
"""main function
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-n', "--dry-run", action='store_true',
help="Don't run anything")
parser.add_argument('-t', "--testing", action='store_true',
help="Use MongoDB test-server here and when calling bcl2fastq wrapper (-t)")
default = 14
parser.add_argument('-w', '--win', type=int, default=default,
help="Number of days to look back (default {})".format(default))
parser.add_argument('-v', '--verbose', action='count', default=0,
help="Increase verbosity")
parser.add_argument('-q', '--quiet', action='count', default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)
user_name = getpass.getuser()
if user_name != "userrig":
logger.warning("Not a production user. Skipping MongoDB update")
sys.exit(0)
connection = mongodb_conn(args.testing)
if connection is None:
sys.exit(1)
db = connection.gisds.runcomplete
epoch_present, epoch_back = generate_window(args.win)
results = db.find({"analysis.per_mux_status" : {"$exists": True},
"timestamp": {"$gt": epoch_back, "$lt": epoch_present}})
logger.info("Found %s runs", results.count())
run_list = {}
mongo_db_ref = {}
for record in results:
run_number = record['run']
print(run_number)
mux_list = {}
for (analysis_count, analysis) in enumerate(record['analysis']):
analysis_id = analysis['analysis_id']
per_mux_status = analysis.get("per_mux_status", None)
if per_mux_status is None:
continue
for (mux_count, mux_status) in enumerate(per_mux_status):
# sanity checks against corrupted DB entries
if mux_status is None or mux_status.get('mux_id') is None:
logger.warning("mux_status is None or incomplete for run %s analysis %s."
" Requires fix in DB. Skipping entry for now.", \
run_number, analysis_id)
continue
if mux_status.get('Status', None) != "SUCCESS":
logger.info("MUX %s from %s is not SUCCESS. Skipping downstream analysis",
mux_status['mux_id'], run_number)
continue
mux_id = mux_status['mux_id']
out_dir = analysis['out_dir']
mux_db_id = "analysis.{}.per_mux_status.{}.DownstreamSubmission".format(
analysis_count, mux_count)
if mux_status.get('Status') == "SUCCESS" and \
mux_status.get('DownstreamSubmission', None) == "TODO":
mongo_list = (mux_id, mux_db_id, analysis_id)
mongo_db_ref.setdefault(run_number, []).append(mongo_list)
mux_list.setdefault(mux_id, []).append(out_dir)
for mux_id, out_dir in mux_list.items():
mux_list_success = mux_list[mux_id]
#Check if MUX has been analyzed more then 1 time successfully
if len(mux_list_success) > 1:
body = "{} has been analyzed more than 1 time successfully..".format(mux_id) \
+ "delegator is skipping the downstream analysis under {}. Please" \
"check the results.".format(mux_list_success)
subject = "Downstream delegator skipped job submission for {}".format(mux_id)
if args.testing:
subject += " (testing)"
send_mail(subject, body, toaddr='veeravallil', ccaddr=None)
continue
mux_info = (mux_id, out_dir)
run_list.setdefault(run_number, []).append(mux_info)
for run_num_flowcell, mux_list in run_list.items():
update_status = True
pipeline_params_dict = get_lib_details(run_num_flowcell, mux_list, args.testing)
if not bool(pipeline_params_dict):
logger.warning("pipeline_paramas_dict is empty for run num %s", run_num_flowcell)
continue
for lib, lib_info in pipeline_params_dict.items():
readunits_list = list()
for outer_key in lib_info:
if outer_key == 'readunits':
for inner_key in lib_info[outer_key]:
readunits_list.append(inner_key)
lib_info['samples'] = {}
lib_info['samples'][lib] = readunits_list
if args.dry_run:
logger.warning("Skipping job delegation for %s", \
lib)
continue
res = mongodb_insert_libjob(lib_info, connection)
if not res:
logger.critical("Skipping rest of analysis job submission" \
"for %s from %s", lib, lib_info.run_id)
subject = "Downstream delegator failed job submission for" \
"{}".format(lib)
if args.testing:
subject += " (testing)"
body = "Downstream delegator failed to insert job submission for" \
"{}".format(lib)
send_mail(subject, body, toaddr='veeravallil', ccaddr=None)
update_status = False
logger.warning("Clean up the database for mux %s from run %s and ctime %s", \
lib_info.mux_id, lib_info.run_id, lib_info.ctime)
mongodb_remove_muxjob(lib_info.mux_id, lib_info.run_id, \
lib_info.ctime, connection)
break
if not args.dry_run and update_status:
value = mongo_db_ref[run_num_flowcell]
for mux_id, insert_id, analysis_id in value:
logger.info("Update mongoDb runComplete for %s and runnumber is %s" \
"and id is %s and analysis_id %s", run_num_flowcell, mux_id, \
insert_id, analysis_id)
res = mongodb_update_runcomplete(run_num_flowcell, analysis_id, mux_id, \
insert_id, connection)
if not res:
logger.critical("Skipping rest of analysis job submission for %s" \
"from %s", mux_id, run_num_flowcell)
subject = "Downstream delegator failed job submission for {}" \
.format(mux_id)
if args.testing:
subject += " (testing)"
body = "Downstream delegator failed to insert job submission for" \
"{}".format(mux_id)
send_mail(subject, body, toaddr='veeravallil', ccaddr=None)
update_status = False
break
connection.close()
def main():
"""main function
"""
stats_upload_script = os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]), "bcl_stats_upload.py"))
assert os.path.exists(stats_upload_script)
archive_upload_script = os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]), "sra_fastq_upload.py"))
assert os.path.exists(archive_upload_script)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-t',
"--testing",
action='store_true',
help="Use MongoDB test server")
default = 14
parser.add_argument(
'-w',
'--win',
type=int,
default=default,
help="Number of days to look back (default {})".format(default))
parser.add_argument('-n', "--dry-run", action='store_true', help="Dry run")
parser.add_argument('-v',
'--verbose',
action='count',
default=0,
help="Increase verbosity")
parser.add_argument('-q',
'--quiet',
action='count',
default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10 * args.quiet - 10 * args.verbose)
if not is_production_user():
logger.warning("Not a production user. Skipping sending of emails")
sys.exit(1)
connection = mongodb_conn(args.testing)
if connection is None:
sys.exit(1)
db = connection.gisds.runcomplete
epoch_present, epoch_back = generate_window(args.win)
num_emails = 0
results = db.find({
"analysis": {
"$exists": True
},
"timestamp": {
"$gt": epoch_back,
"$lt": epoch_present
}
})
logger.info("Found %s runs", results.count())
if is_devel_version() or args.testing:
mail_to = 'veeravallil' # domain added in mail function
else:
#mail_to = '*****@*****.**'
mail_to = '*****@*****.**'
for record in results:
run_number = record['run']
#print(run_number)
for (analysis_count, analysis) in enumerate(record['analysis']):
analysis_id = analysis['analysis_id']
per_mux_status = analysis.get("per_mux_status", None)
if per_mux_status is None:
continue
for (mux_count, mux_status) in enumerate(per_mux_status):
if args.dry_run:
logger.warning(
"Skipping analysis %s run %s MUX %s"
" with email_sent %s", analysis_id,
run_number, mux_status['mux_id'],
mux_status.get('email_sent', None))
continue
if mux_status.get('email_sent', None):
continue
# for all others: send email and update db
email_sent_query = "analysis.{}.per_mux_status.{}.email_sent".format(
analysis_count, mux_count)
mux_id = mux_status['mux_id']
out_dir = analysis['out_dir']
if mux_status.get('Status', None) == "FAILED":
logger.info("bcl2fastq for MUX %s from %s failed. ",
mux_status['mux_id'], run_number)
subject = 'bcl2fastq: ' + mux_id
body = "bcl2fastq for {} from {} failed.".format(
mux_id, run_number)
body += "\n\nPlease check the logs under {}".format(
out_dir + "/logs")
send_mail(subject, body, mail_to, ccaddr="rpd")
num_emails += 1
update_mongodb_email(db, run_number, analysis_id,
email_sent_query, True)
elif mux_status.get('Status', None) == "SUCCESS":
muxdir = os.path.join(out_dir, 'out',
mux_status.get('mux_dir'))
summary = path_to_url(
os.path.join(muxdir, 'html/index.html'))
body = "bcl2fastq for {} from {} successfully completed.".format(
mux_id, run_number)
body += "\n\nA summary can be found at {}".format(summary)
body += "\n\nFastQ files are located in {}".format(muxdir)
body += "\n\nData can also be downloaded from GIS-SRA (once archival is complete)"
confinfo = os.path.join(out_dir, 'conf.yaml')
#print(body)
if not os.path.exists(confinfo):
logger.fatal(
"conf info '%s' does not exist"
" under run directory.", confinfo)
continue
subject = 'bcl2fastq'
if args.testing:
subject += ' testing'
if is_devel_version():
subject += ' devel'
subject += ': ' + mux_id
send_mail(subject, body, mail_to,
ccaddr="rpd") # mail_to already set
if not args.testing and not is_devel_version():
requestor = get_requestor(mux_id, confinfo)
if requestor is not None:
#requestor = "rpd"
#subject += " (instead of requestor)"
#send_mail(subject, body, requestor, ccaddr="rpd")
send_mail(subject, body, requestor)
num_emails += 1
update_mongodb_email(db, run_number, analysis_id,
email_sent_query, True)
# close the connection to MongoDB
connection.close()
logger.info("%d emails sent", num_emails)
def main():
"""main function
"""
stats_upload_script = os.path.abspath(os.path.join(
os.path.dirname(sys.argv[0]), "bcl_stats_upload.py"))
assert os.path.exists(stats_upload_script)
archive_upload_script = os.path.abspath(os.path.join(
os.path.dirname(sys.argv[0]), "sra_fastq_upload.py"))
assert os.path.exists(archive_upload_script)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-t', "--testing", action='store_true',
help="Use MongoDB test server")
default = 14
parser.add_argument('-w', '--win', type=int, default=default,
help="Number of days to look back (default {})".format(default))
parser.add_argument('-n', "--dry-run", action='store_true',
help="Dry run")
parser.add_argument('-v', '--verbose', action='count', default=0,
help="Increase verbosity")
parser.add_argument('-q', '--quiet', action='count', default=0,
help="Decrease verbosity")
args = parser.parse_args()
# Repeateable -v and -q for setting logging level.
# See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
# and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
# script -vv -> DEBUG
# script -v -> INFO
# script -> WARNING
# script -q -> ERROR
# script -qq -> CRITICAL
# script -qqq -> no logging at all
logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)
user_name = getpass.getuser()
if user_name != "userrig":
logger.warning("Not a production user. Skipping sending of emails")
sys.exit(0)
connection = mongodb_conn(args.testing)
if connection is None:
sys.exit(1)
db = connection.gisds.runcomplete
epoch_present, epoch_back = generate_window(args.win)
num_emails = 0
results = db.find({"analysis" : {"$exists": True},
"timestamp": {"$gt": epoch_back, "$lt": epoch_present}})
logger.info("Found %s runs", results.count())
if is_devel_version() or args.testing:
mail_to = 'veeravallil'# domain added in mail function
else:
#mail_to = '*****@*****.**'
mail_to = '*****@*****.**'
for record in results:
run_number = record['run']
#print(run_number)
for (analysis_count, analysis) in enumerate(record['analysis']):
analysis_id = analysis['analysis_id']
per_mux_status = analysis.get("per_mux_status", None)
if per_mux_status is None:
continue
for (mux_count, mux_status) in enumerate(per_mux_status):
if args.dry_run:
logger.warning("Skipping analysis %s run %s MUX %s"
" with email_sent %s",
analysis_id, run_number, mux_status['mux_id'],
mux_status.get('email_sent', None))
continue
if mux_status.get('email_sent', None):
continue
# for all others: send email and update db
email_sent_query = "analysis.{}.per_mux_status.{}.email_sent".format(
analysis_count, mux_count)
mux_id = mux_status['mux_id']
out_dir = analysis['out_dir']
if mux_status.get('Status', None) == "FAILED":
logger.info("bcl2fastq for MUX %s from %s failed. ",
mux_status['mux_id'], run_number)
subject = 'bcl2fastq: ' + mux_id
body = "bcl2fastq for {} from {} failed.".format(mux_id, run_number)
body += "\n\nPlease check the logs under {}".format(out_dir + "/logs")
send_mail(subject, body, mail_to, ccaddr="rpd")
num_emails += 1
update_mongodb_email(db, run_number, analysis_id, email_sent_query, True)
elif mux_status.get('Status', None) == "SUCCESS":
muxdir = os.path.join(out_dir, 'out', mux_status.get('mux_dir'))
summary = path_to_url(os.path.join(muxdir, 'html/index.html'))
body = "bcl2fastq for {} from {} successfully completed.".format(
mux_id, run_number)
body += "\n\nA summary can be found at {}".format(summary)
body += "\n\nFastQ files are located in {}".format(muxdir)
body += "\n\nData can also be downloaded from GIS-SRA (once archival is complete)"
confinfo = os.path.join(out_dir, 'conf.yaml')
#print(body)
if not os.path.exists(confinfo):
logger.fatal("conf info '%s' does not exist"
" under run directory.", confinfo)
continue
subject = 'bcl2fastq'
if args.testing:
subject += ' testing'
if is_devel_version():
subject += ' devel'
subject += ': ' + mux_id
send_mail(subject, body, mail_to, ccaddr="rpd")# mail_to already set
if not args.testing and not is_devel_version():
requestor = get_requestor(mux_id, confinfo)
if requestor is not None:
#requestor = "rpd"
#subject += " (instead of requestor)"
#send_mail(subject, body, requestor, ccaddr="rpd")
send_mail(subject, body, requestor)
num_emails += 1
update_mongodb_email(db, run_number, analysis_id, email_sent_query, True)
# close the connection to MongoDB
connection.close()
logger.info("%d emails sent", num_emails)
def start_data_transfer(connection, mux, mux_info, site, mail_to):
""" Data transfer from source to destination
"""
run_number, downstream_id, analysis_id, bcl_path = mux_info
fastq_src = os.path.join(bcl_path, "out", "Project_"+mux)
bcl_dir = os.path.basename(bcl_path)
if is_devel_version():
fastq_dest = os.path.join(novogene_conf['FASTQ_DEST'][site]['devel'], \
mux, run_number, bcl_dir)
yaml_dest = os.path.join(novogene_conf['FASTQ_DEST'][site]['devel'], \
mux, mux +"_multisample.yaml")
else:
fastq_dest = os.path.join(novogene_conf['FASTQ_DEST'][site]['production'], \
mux, run_number, bcl_dir)
yaml_dest = os.path.join(novogene_conf['FASTQ_DEST'][site]['production'], \
mux, mux+ "_multisample.yaml")
rsync_cmd = 'rsync -va %s %s' % (fastq_src, fastq_dest)
if not os.path.exists(fastq_dest):
try:
os.makedirs(fastq_dest)
logger.info("data transfer started for %s from %s", mux, run_number)
st_time = generate_timestamp()
update_downstream_mux(connection, run_number, analysis_id, downstream_id, \
"COPYING_" + st_time)
_ = subprocess.check_output(rsync_cmd, shell=True, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
body = "The following command failed with return code {}: {}". \
format(e.returncode, rsync_cmd)
subject = "{} from {}: SG10K data transfer ({}) failed".format(mux, run_number, site)
logger.fatal(body)
logger.fatal("Output: %s", e.output.decode())
logger.fatal("Exiting")
#Send_mail
send_mail(subject, body, toaddr=mail_to, ccaddr=None)
#Delete the partial info being rsync
update_downstream_mux(connection, run_number, analysis_id, downstream_id, "ERROR")
sys.exit(1)
#Update the mongoDB for successful data transfer
sample_info = get_mux_details(run_number, mux, fastq_dest)
#Touch rsync complete file
with open(os.path.join(fastq_dest, "rsync_complete.txt"), "w") as f:
f.write("")
with open(yaml_dest, 'w') as fh:
yaml.dump(dict(sample_info), fh, default_flow_style=False)
job = {}
job['sample_cfg'] = {}
for outer_key, outer_value in sample_info.items():
ctime, _ = generate_window(1)
job['sample_cfg'].update({outer_key:outer_value})
job['site'] = site
job['pipeline_name'] = 'custom/SG10K'
job['pipeline_version'] = novogene_conf['PIPELINE_VERSION']
job['ctime'] = ctime
job['requestor'] = 'userrig'
if is_devel_version():
novogene_outdir = os.path.join(novogene_conf['NOVOGENE_OUTDIR'][site]['devel'], \
mux)
else:
novogene_outdir = os.path.join(novogene_conf['NOVOGENE_OUTDIR'][site]['production'],
mux)
job['out_dir_override'] = novogene_outdir
logger.info("Data transfer completed successfully for %s from %s", mux, run_number)
job_id = insert_muxjob(connection, mux, job)
update_downstream_mux(connection, run_number, analysis_id, downstream_id, job_id)
subject = "{} from {}: SG10K data transfer ({}) completed".format(mux, run_number, site)
body = "Data transfer successfully completed for {} from {}".format(mux, run_number)
send_mail(subject, body, toaddr=mail_to, ccaddr=None)
return True
else:
logger.critical("Mux %s from %s directory already exists under %s", mux, \
run_number, fastq_dest)
return False
def main(toaddr):
subject = "Test email from {} version {}".format(get_site(),
get_pipeline_version())
body = "Email wursts.\n\nSincerely,\nRPD"
send_mail(subject, body, toaddr=toaddr, ccaddr=None, pass_exception=False)