def main(run_name, gdocs_spreadsheet, encoded_credentials_file, run_info_yaml, analysis_dir, archive_dir, gdocs_worksheet, gdocs_projects_folder, append, split_on_project):
log.info("Processing run: %s" % run_name)
# If not supplied, assume that the configuration file is named run_info.yaml and resides in the archive dir
if not run_info_yaml:
run_info_yaml = os.path.join(archive_dir,"run_info.yaml")
log.info("No configuration file supplied, assuming it is '%s'" % run_info_yaml)
if not os.path.exists(run_info_yaml):
log.warn("Could not find required run_info.yaml configuration file at '%s'" % run_info_yaml)
return
with open(run_info_yaml) as in_handle:
run_info = {'details': yaml.load(in_handle)}
# Get the google docs crdentials
gdocs_credentials = ""
if not os.path.exists(encoded_credentials_file):
log.warn("The Google Docs credentials file could not be found. No demultiplex data was written")
return
with open(encoded_credentials_file) as fh:
gdocs_credentials = fh.read().strip()
fc_name, fc_date = get_flowcell_info(run_name)
# Get the barcode statistics
bc_metrics = get_bc_stats(fc_date,fc_name,analysis_dir,run_info)
# Write the report
write_run_report_to_gdocs(fc_date,fc_name,bc_metrics,gdocs_spreadsheet,gdocs_credentials,gdocs_worksheet,append,split_on_project)
# Write the bc project summary report
if gdocs_projects_folder:
write_project_report_to_gdocs(fc_date,fc_name,bc_metrics,gdocs_credentials,gdocs_projects_folder)
def create_report_on_gdocs(fc_date, fc_name, run_info_yaml, dirs, config):
"""Create reports on gdocs containing both demultiplexed read counts and QC data.
"""
success = True
try:
# Inject the fc_date and fc_name in the email subject
def record_processor(record):
return record.extra.__setitem__('run', "%s_%s" % (fc_date, fc_name))
# Parse the run_info.yaml file
log.debug("Loading this run_info: {}".format(run_info_yaml))
with open(run_info_yaml, "r") as fh:
run_info = yaml.load(fh)
# Get the gdocs account credentials
encoded_credentials = get_credentials(config)
if not encoded_credentials:
log.warn("Could not find Google Docs account credentials in configuration. \
No sequencing report was written")
return False
# Get the required parameters from the post_process.yaml configuration file
gdocs = config.get("gdocs_upload", None)
# Add email notification
email = gdocs.get("gdocs_email_notification", None)
smtp_host = config.get("smtp_host", "")
smtp_port = config.get("smtp_port", "")
log_handler = create_log_handler({'email': email, \
'smtp_host': smtp_host, \
'smtp_port': smtp_port}, True)
except Exception as e:
success = False
log.warn("Encountered exception when writing sequencing report to Google Docs: %s" % e)
with log_handler.applicationbound(), logbook.Processor(record_processor):
try:
log.info("Started creating sequencing report on Google docs for %s_%s on %s" \
% (fc_date, fc_name, datetime.datetime.now().isoformat()))
# Get a flowcell object
fc = Flowcell(fc_name, fc_date, run_info, dirs.get("work", None))
# Get the GDocs demultiplex result file title
gdocs_dmplx_spreadsheet = gdocs.get("gdocs_dmplx_file", None)
# Get the GDocs QC file title
gdocs_qc_spreadsheet = gdocs.get("gdocs_qc_file", None)
# FIXME: Make the bc stuff use the Flowcell module
if gdocs_dmplx_spreadsheet is not None:
# Upload the data
bc_metrics.write_run_report_to_gdocs(fc, fc_date, \
fc_name, gdocs_dmplx_spreadsheet, encoded_credentials, append=True)
else:
log.warn("Could not find Google Docs demultiplex results file \
title in configuration. No demultiplex counts were \
written to Google Docs for %s_%s" % (fc_date, fc_name))
# Parse the QC metrics
try:
qc = RTAQCMetrics(dirs.get("flowcell", None))
except:
qc = None
if gdocs_qc_spreadsheet is not None and qc is not None:
qc_metrics.write_run_report_to_gdocs(fc, qc, gdocs_qc_spreadsheet, encoded_credentials)
else:
log.warn("Could not find Google Docs QC file title in configuration. " \
"No QC data were written to Google Docs " \
"for %s_%s".format(fc_date, fc_name))
# Get the projects parent folder
projects_folder = gdocs.get("gdocs_projects_folder", None)
# Write the bc project summary report
if projects_folder is not None:
create_project_report_on_gdocs(fc, qc, \
encoded_credentials, projects_folder)
except Exception as e:
success = False
log.warn("Encountered exception when writing sequencing report " \
"to Google Docs: {}".format(e))
if success:
log.info("Sequencing report successfully created on Google " \
"docs for {}_{} on {}".format(fc_date, fc_name, datetime.datetime.now().isoformat()))
else:
log.warn("Encountered exception when writing sequencing " \
"report for %s_%s to Google docs on %s" \
% (fc_date, fc_name, datetime.datetime.now().isoformat()))
return success
def create_report_on_gdocs(fc_date, fc_name, run_info_yaml, dirs, config):
"""Create reports on gdocs containing both demultiplexed read counts and QC data.
"""
success = True
try:
# Inject the fc_date and fc_name in the email subject
def record_processor(record):
return record.extra.__setitem__('run',
"%s_%s" % (fc_date, fc_name))
# Parse the run_info.yaml file
log.debug("Loading this run_info: {}".format(run_info_yaml))
with open(run_info_yaml, "r") as fh:
run_info = yaml.load(fh)
# Get the gdocs account credentials
encoded_credentials = get_credentials(config)
if not encoded_credentials:
log.warn(
"Could not find Google Docs account credentials in configuration. \
No sequencing report was written")
return False
# Get the required parameters from the post_process.yaml configuration file
gdocs = config.get("gdocs_upload", None)
# Add email notification
email = gdocs.get("gdocs_email_notification", None)
smtp_host = config.get("smtp_host", "")
smtp_port = config.get("smtp_port", "")
log_handler = create_log_handler({'email': email, \
'smtp_host': smtp_host, \
'smtp_port': smtp_port}, True)
except Exception as e:
success = False
log.warn(
"Encountered exception when writing sequencing report to Google Docs: %s"
% e)
with log_handler.applicationbound(), logbook.Processor(record_processor):
try:
log.info("Started creating sequencing report on Google docs for %s_%s on %s" \
% (fc_date, fc_name, datetime.datetime.now().isoformat()))
# Get a flowcell object
fc = Flowcell(fc_name, fc_date, run_info, dirs.get("work", None))
# Get the GDocs demultiplex result file title
gdocs_dmplx_spreadsheet = gdocs.get("gdocs_dmplx_file", None)
# Get the GDocs QC file title
gdocs_qc_spreadsheet = gdocs.get("gdocs_qc_file", None)
# FIXME: Make the bc stuff use the Flowcell module
if gdocs_dmplx_spreadsheet is not None:
# Upload the data
bc_metrics.write_run_report_to_gdocs(fc, fc_date, \
fc_name, gdocs_dmplx_spreadsheet, encoded_credentials, append=True)
else:
log.warn("Could not find Google Docs demultiplex results file \
title in configuration. No demultiplex counts were \
written to Google Docs for %s_%s" % (fc_date, fc_name))
# Parse the QC metrics
try:
qc = RTAQCMetrics(dirs.get("flowcell", None))
except:
qc = None
if gdocs_qc_spreadsheet is not None and qc is not None:
qc_metrics.write_run_report_to_gdocs(fc, qc,
gdocs_qc_spreadsheet,
encoded_credentials)
else:
log.warn("Could not find Google Docs QC file title in configuration. " \
"No QC data were written to Google Docs " \
"for %s_%s".format(fc_date, fc_name))
# Get the projects parent folder
projects_folder = gdocs.get("gdocs_projects_folder", None)
# Write the bc project summary report
if projects_folder is not None:
create_project_report_on_gdocs(fc, qc, \
encoded_credentials, projects_folder)
except Exception as e:
success = False
log.warn("Encountered exception when writing sequencing report " \
"to Google Docs: {}".format(e))
if success:
log.info("Sequencing report successfully created on Google " \
"docs for {}_{} on {}".format(fc_date, fc_name, datetime.datetime.now().isoformat()))
else:
log.warn("Encountered exception when writing sequencing " \
"report for %s_%s to Google docs on %s" \
% (fc_date, fc_name, datetime.datetime.now().isoformat()))
return success
