def main(run_name, gdocs_spreadsheet, encoded_credentials_file, run_info_yaml, analysis_dir, archive_dir, gdocs_worksheet, gdocs_projects_folder, append, split_on_project): log.info("Processing run: %s" % run_name) # If not supplied, assume that the configuration file is named run_info.yaml and resides in the archive dir if not run_info_yaml: run_info_yaml = os.path.join(archive_dir,"run_info.yaml") log.info("No configuration file supplied, assuming it is '%s'" % run_info_yaml) if not os.path.exists(run_info_yaml): log.warn("Could not find required run_info.yaml configuration file at '%s'" % run_info_yaml) return with open(run_info_yaml) as in_handle: run_info = {'details': yaml.load(in_handle)} # Get the google docs crdentials gdocs_credentials = "" if not os.path.exists(encoded_credentials_file): log.warn("The Google Docs credentials file could not be found. No demultiplex data was written") return with open(encoded_credentials_file) as fh: gdocs_credentials = fh.read().strip() fc_name, fc_date = get_flowcell_info(run_name) # Get the barcode statistics bc_metrics = get_bc_stats(fc_date,fc_name,analysis_dir,run_info) # Write the report write_run_report_to_gdocs(fc_date,fc_name,bc_metrics,gdocs_spreadsheet,gdocs_credentials,gdocs_worksheet,append,split_on_project) # Write the bc project summary report if gdocs_projects_folder: write_project_report_to_gdocs(fc_date,fc_name,bc_metrics,gdocs_credentials,gdocs_projects_folder)
def create_report_on_gdocs(fc_date, fc_name, run_info_yaml, dirs, config): """Create reports on gdocs containing both demultiplexed read counts and QC data. """ success = True try: # Inject the fc_date and fc_name in the email subject def record_processor(record): return record.extra.__setitem__('run', "%s_%s" % (fc_date, fc_name)) # Parse the run_info.yaml file log.debug("Loading this run_info: {}".format(run_info_yaml)) with open(run_info_yaml, "r") as fh: run_info = yaml.load(fh) # Get the gdocs account credentials encoded_credentials = get_credentials(config) if not encoded_credentials: log.warn("Could not find Google Docs account credentials in configuration. \ No sequencing report was written") return False # Get the required parameters from the post_process.yaml configuration file gdocs = config.get("gdocs_upload", None) # Add email notification email = gdocs.get("gdocs_email_notification", None) smtp_host = config.get("smtp_host", "") smtp_port = config.get("smtp_port", "") log_handler = create_log_handler({'email': email, \ 'smtp_host': smtp_host, \ 'smtp_port': smtp_port}, True) except Exception as e: success = False log.warn("Encountered exception when writing sequencing report to Google Docs: %s" % e) with log_handler.applicationbound(), logbook.Processor(record_processor): try: log.info("Started creating sequencing report on Google docs for %s_%s on %s" \ % (fc_date, fc_name, datetime.datetime.now().isoformat())) # Get a flowcell object fc = Flowcell(fc_name, fc_date, run_info, dirs.get("work", None)) # Get the GDocs demultiplex result file title gdocs_dmplx_spreadsheet = gdocs.get("gdocs_dmplx_file", None) # Get the GDocs QC file title gdocs_qc_spreadsheet = gdocs.get("gdocs_qc_file", None) # FIXME: Make the bc stuff use the Flowcell module if gdocs_dmplx_spreadsheet is not None: # Upload the data bc_metrics.write_run_report_to_gdocs(fc, fc_date, \ fc_name, gdocs_dmplx_spreadsheet, encoded_credentials, append=True) else: log.warn("Could not find Google Docs demultiplex results file \ title in configuration. No demultiplex counts were \ written to Google Docs for %s_%s" % (fc_date, fc_name)) # Parse the QC metrics try: qc = RTAQCMetrics(dirs.get("flowcell", None)) except: qc = None if gdocs_qc_spreadsheet is not None and qc is not None: qc_metrics.write_run_report_to_gdocs(fc, qc, gdocs_qc_spreadsheet, encoded_credentials) else: log.warn("Could not find Google Docs QC file title in configuration. " \ "No QC data were written to Google Docs " \ "for %s_%s".format(fc_date, fc_name)) # Get the projects parent folder projects_folder = gdocs.get("gdocs_projects_folder", None) # Write the bc project summary report if projects_folder is not None: create_project_report_on_gdocs(fc, qc, \ encoded_credentials, projects_folder) except Exception as e: success = False log.warn("Encountered exception when writing sequencing report " \ "to Google Docs: {}".format(e)) if success: log.info("Sequencing report successfully created on Google " \ "docs for {}_{} on {}".format(fc_date, fc_name, datetime.datetime.now().isoformat())) else: log.warn("Encountered exception when writing sequencing " \ "report for %s_%s to Google docs on %s" \ % (fc_date, fc_name, datetime.datetime.now().isoformat())) return success
def create_report_on_gdocs(fc_date, fc_name, run_info_yaml, dirs, config): """Create reports on gdocs containing both demultiplexed read counts and QC data. """ success = True try: # Inject the fc_date and fc_name in the email subject def record_processor(record): return record.extra.__setitem__('run', "%s_%s" % (fc_date, fc_name)) # Parse the run_info.yaml file log.debug("Loading this run_info: {}".format(run_info_yaml)) with open(run_info_yaml, "r") as fh: run_info = yaml.load(fh) # Get the gdocs account credentials encoded_credentials = get_credentials(config) if not encoded_credentials: log.warn( "Could not find Google Docs account credentials in configuration. \ No sequencing report was written") return False # Get the required parameters from the post_process.yaml configuration file gdocs = config.get("gdocs_upload", None) # Add email notification email = gdocs.get("gdocs_email_notification", None) smtp_host = config.get("smtp_host", "") smtp_port = config.get("smtp_port", "") log_handler = create_log_handler({'email': email, \ 'smtp_host': smtp_host, \ 'smtp_port': smtp_port}, True) except Exception as e: success = False log.warn( "Encountered exception when writing sequencing report to Google Docs: %s" % e) with log_handler.applicationbound(), logbook.Processor(record_processor): try: log.info("Started creating sequencing report on Google docs for %s_%s on %s" \ % (fc_date, fc_name, datetime.datetime.now().isoformat())) # Get a flowcell object fc = Flowcell(fc_name, fc_date, run_info, dirs.get("work", None)) # Get the GDocs demultiplex result file title gdocs_dmplx_spreadsheet = gdocs.get("gdocs_dmplx_file", None) # Get the GDocs QC file title gdocs_qc_spreadsheet = gdocs.get("gdocs_qc_file", None) # FIXME: Make the bc stuff use the Flowcell module if gdocs_dmplx_spreadsheet is not None: # Upload the data bc_metrics.write_run_report_to_gdocs(fc, fc_date, \ fc_name, gdocs_dmplx_spreadsheet, encoded_credentials, append=True) else: log.warn("Could not find Google Docs demultiplex results file \ title in configuration. No demultiplex counts were \ written to Google Docs for %s_%s" % (fc_date, fc_name)) # Parse the QC metrics try: qc = RTAQCMetrics(dirs.get("flowcell", None)) except: qc = None if gdocs_qc_spreadsheet is not None and qc is not None: qc_metrics.write_run_report_to_gdocs(fc, qc, gdocs_qc_spreadsheet, encoded_credentials) else: log.warn("Could not find Google Docs QC file title in configuration. " \ "No QC data were written to Google Docs " \ "for %s_%s".format(fc_date, fc_name)) # Get the projects parent folder projects_folder = gdocs.get("gdocs_projects_folder", None) # Write the bc project summary report if projects_folder is not None: create_project_report_on_gdocs(fc, qc, \ encoded_credentials, projects_folder) except Exception as e: success = False log.warn("Encountered exception when writing sequencing report " \ "to Google Docs: {}".format(e)) if success: log.info("Sequencing report successfully created on Google " \ "docs for {}_{} on {}".format(fc_date, fc_name, datetime.datetime.now().isoformat())) else: log.warn("Encountered exception when writing sequencing " \ "report for %s_%s to Google docs on %s" \ % (fc_date, fc_name, datetime.datetime.now().isoformat())) return success