def test_workflows(self): config_file_path = locate_ngi_config() config = load_yaml_config(config_file_path) for workflow_name, workflow_dict in config.get("test_data", {}).get( "workflows", {}).iteritems(): # Load and rewrite config file as needed customize_config_dict = workflow_dict.get("customize_config") if customize_config_dict: config = update_dict(config, customize_config_dict) #self._install_test_files(workflow_dict) LOG.info( 'Starting test analysis pipeline for workflow "{}"'.format( workflow_name)) try: local_files = workflow_dict["local_files"] except KeyError: raise ValueError( "Required paths to input files for testing do not" "exist in config file (test_data.workflows." "{}.local_files); cannot proceed.".format(workflow_name)) try: flowcell_path = local_files["flowcell"] except KeyError: raise ValueError( "Path to flowcell is required and not specified " "in configuration file (test_data.workflows." "{}.local_files.flowcell); cannot proceed.".format( workflow_name)) try: test_project = workflow_dict["test_project"] test_proj_id = test_project["project_id"] test_proj_name = test_project["project_name"] test_proj_bpa = test_project["bpa"] except KeyError as e: raise ValueError( "Test project information is missing from config " "file (under test_data.workflows.{}.test_project " "({}); cannot proceed.".format(workflow_name, e.msg)) charon_session = CharonSession(config=config) try: charon_session.project_delete(projectid=test_proj_id) except CharonError: pass charon_session.project_create(projectid=test_proj_id, name=test_proj_name, status="OPEN", best_practice_analysis=test_proj_bpa) process_demultiplexed_flowcells([flowcell_path], fallback_libprep="A", config=config)
def test_workflows(self): config_file_path = locate_ngi_config() config = load_yaml_config(config_file_path) for workflow_name, workflow_dict in config.get("test_data", {}).get("workflows", {}).iteritems(): # Load and rewrite config file as needed customize_config_dict = workflow_dict.get("customize_config") if customize_config_dict: config = update_dict(config, customize_config_dict) #self._install_test_files(workflow_dict) LOG.info('Starting test analysis pipeline for workflow "{}"'.format(workflow_name)) try: local_files = workflow_dict["local_files"] except KeyError: raise ValueError("Required paths to input files for testing do not" "exist in config file (test_data.workflows." "{}.local_files); cannot proceed.".format(workflow_name)) try: flowcell_path = local_files["flowcell"] except KeyError: raise ValueError("Path to flowcell is required and not specified " "in configuration file (test_data.workflows." "{}.local_files.flowcell); cannot proceed.".format(workflow_name)) try: test_project = workflow_dict["test_project"] test_proj_id = test_project["project_id"] test_proj_name = test_project["project_name"] test_proj_bpa = test_project["bpa"] except KeyError as e: raise ValueError("Test project information is missing from config " "file (under test_data.workflows.{}.test_project " "({}); cannot proceed.".format(workflow_name, e.msg)) charon_session = CharonSession(config=config) try: charon_session.project_delete(projectid=test_proj_id) except CharonError: pass charon_session.project_create(projectid=test_proj_id, name=test_proj_name, status="OPEN", best_practice_analysis=test_proj_bpa) process_demultiplexed_flowcells([flowcell_path], fallback_libprep="A", config=config)
def create_charon_entries_from_project(project, best_practice_analysis="whole_genome_reseq", sequencing_facility="NGI-S", force_overwrite=False, delete_existing=False, retry_on_fail=True): """Given a project object, creates the relevant entries in Charon. This code is remarkably shoddy as I created it in a hurry and then later it became a part of the pipeline. Use at your own risk! Ha ha. :param NGIProject project: The NGIProject object :param str best_practice_analysis: The workflow to assign for this project (default "variant_calling") :param str sequencing_facility: The facility that did the sequencing :param bool force_overwrite: If this is set to true, overwrite existing entries in Charon (default false) :param bool delete_existing: Don't just update existing entries, delete them and create new ones (default false) """ charon_session = CharonSession() update_failed=False try: status = "OPEN" LOG.info('Creating project "{}" with status "{}", best practice analysis "{}", ' 'and sequencing_facility {}'.format(project, status, best_practice_analysis, sequencing_facility)) charon_session.project_create(projectid=project.project_id, name=project.name, status=status, best_practice_analysis=best_practice_analysis, sequencing_facility=sequencing_facility) LOG.info('Project "{}" created in Charon.'.format(project)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warn('Overwriting data for project "{}"'.format(project)) charon_session.project_update(projectid=project.project_id, name=project.name, status=status, best_practice_analysis=best_practice_analysis, sequencing_facility=sequencing_facility) LOG.info('Project "{}" updated in Charon.'.format(project)) else: LOG.info('Project "{}" already exists; moving to samples...'.format(project)) else: raise for sample in project: if delete_existing: LOG.warn('Deleting existing sample "{}"'.format(sample)) try: charon_session.sample_delete(projectid=project.project_id, sampleid=sample.name) except CharonError as e: update_failed=True LOG.error('Could not delete sample "{}": {}'.format(sample, e)) try: analysis_status = "TO_ANALYZE" LOG.info('Creating sample "{}" with analysis_status "{}"'.format(sample, analysis_status)) charon_session.sample_create(projectid=project.project_id, sampleid=sample.name, analysis_status=analysis_status) LOG.info('Project/sample "{}/{}" created in Charon.'.format(project, sample)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}"'.format(project, sample)) charon_session.sample_update(projectid=project.project_id, sampleid=sample.name, analysis_status=analysis_status) LOG.info('Project/sample "{}/{}" updated in Charon.'.format(project, sample)) else: LOG.info('Project "{}" / sample "{}" already exists; moving ' 'to libpreps'.format(project, sample)) else: update_failed=True LOG.error(e) continue for libprep in sample: if delete_existing: LOG.warn('Deleting existing libprep "{}"'.format(libprep)) try: charon_session.libprep_delete(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name) except CharonError as e: LOG.warn('Could not delete libprep "{}": {}'.format(libprep, e)) try: qc = "PASSED" LOG.info('Creating libprep "{}" with qc status "{}"'.format(libprep, qc)) charon_session.libprep_create(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, qc=qc) LOG.info(('Project/sample/libprep "{}/{}/{}" created in ' 'Charon').format(project, sample, libprep)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}" / libprep "{}"'.format(project, sample, libprep)) charon_session.libprep_update(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, qc=qc) LOG.info(('Project/sample/libprep "{}/{}/{}" updated in ' 'Charon').format(project, sample, libprep)) else: LOG.debug(e) LOG.info('Project "{}" / sample "{}" / libprep "{}" already ' 'exists; moving to libpreps'.format(project, sample, libprep)) else: update_failed=True LOG.error(e) continue for seqrun in libprep: if delete_existing: LOG.warn('Deleting existing seqrun "{}"'.format(seqrun)) try: charon_session.seqrun_delete(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name) except CharonError as e: update_failed=True LOG.error('Could not delete seqrun "{}": {}'.format(seqrun, e)) try: alignment_status="NOT_RUNNING" LOG.info('Creating seqrun "{}" with alignment_status "{}"'.format(seqrun, alignment_status)) charon_session.seqrun_create(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name, alignment_status=alignment_status, total_reads=0, mean_autosomal_coverage=0) LOG.info(('Project/sample/libprep/seqrun "{}/{}/{}/{}" ' 'created in Charon').format(project, sample, libprep, seqrun)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}" / libprep "{}" / ' 'seqrun "{}"'.format(project, sample, libprep, seqrun)) charon_session.seqrun_update(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name, alignment_status=alignment_status, total_reads=0, mean_autosomal_coverage=0) LOG.info(('Project/sample/libprep/seqrun "{}/{}/{}/{}" ' 'updated in Charon').format(project, sample, libprep, seqrun)) else: LOG.info('Project "{}" / sample "{}" / libprep "{}" / ' 'seqrun "{}" already exists; next...'.format(project, sample, libprep, seqrun)) else: update_failed=True LOG.error(e) continue if update_failed : if retry_on_fail: create_charon_entries_from_project(project, best_practice_analysis=best_practice_analysis, sequencing_facility=sequencing_facility, force_overwrite=force_overwrite, delete_existing=delete_existing, retry_on_fail=False) else: raise CharonError("A network error blocks Charon updating.")
def create_charon_entries_from_project(project, best_practice_analysis="whole_genome_reseq", sequencing_facility="NGI-S", force_overwrite=False, delete_existing=False): """Given a project object, creates the relevant entries in Charon. :param NGIProject project: The NGIProject object :param str best_practice_analysis: The workflow to assign for this project (default "variant_calling") :param str sequencing_facility: The facility that did the sequencing :param bool force_overwrite: If this is set to true, overwrite existing entries in Charon (default false) :param bool delete_existing: Don't just update existing entries, delete them and create new ones (default false) """ charon_session = CharonSession() try: status="OPEN" LOG.info('Creating project "{}" with status "{}", best practice analysis "{}", ' 'and sequencing_facility {}'.format(project, status, best_practice_analysis, sequencing_facility)) charon_session.project_create(projectid=project.project_id, name=project.name, status=status, best_practice_analysis=best_practice_analysis, sequencing_facility=sequencing_facility) LOG.info('Project "{}" created in Charon.'.format(project)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warn('Overwriting data for project "{}"'.format(project)) charon_session.project_update(projectid=project.project_id, name=project.name, status=status, best_practice_analysis=best_practice_analysis, sequencing_facility=sequencing_facility) LOG.info('Project "{}" updated in Charon.'.format(project)) else: LOG.info('Project "{}" already exists; moving to samples...'.format(project)) else: raise for sample in project: if delete_existing: LOG.warn('Deleting existing sample "{}"'.format(sample)) try: charon_session.sample_delete(projectid=project.project_id, sampleid=sample.name) except CharonError as e: LOG.error('Could not delete sample "{}": {}'.format(sample, e)) try: analysis_status = "TO_ANALYZE" LOG.info('Creating sample "{}" with analysis_status "{}"'.format(sample, analysis_status)) charon_session.sample_create(projectid=project.project_id, sampleid=sample.name, analysis_status=analysis_status) LOG.info('Project/sample "{}/{}" created in Charon.'.format(project, sample)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}"'.format(project, sample)) charon_session.sample_update(projectid=project.project_id, sampleid=sample.name, analysis_status=analysis_status) LOG.info('Project/sample "{}/{}" updated in Charon.'.format(project, sample)) else: LOG.info('Project "{}" / sample "{}" already exists; moving ' 'to libpreps'.format(project, sample)) else: LOG.error(e) continue for libprep in sample: if delete_existing: LOG.warn('Deleting existing libprep "{}"'.format(libprep)) try: charon_session.libprep_delete(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name) except CharonError as e: LOG.warn('Could not delete libprep "{}": {}'.format(libprep, e)) try: qc= "PASSED" LOG.info('Creating libprep "{}" with qc status "{}"'.format(libprep, qc)) charon_session.libprep_create(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, qc=qc) LOG.info(('Project/sample/libprep "{}/{}/{}" created in ' 'Charon').format(project, sample, libprep)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}" / libprep "{}"'.format(project, sample, libprep)) charon_session.libprep_update(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, qc=qc) LOG.info(('Project/sample/libprep "{}/{}/{}" updated in ' 'Charon').format(project, sample, libprep)) else: LOG.info(e) LOG.info('Project "{}" / sample "{}" / libprep "{}" already ' 'exists; moving to libpreps'.format(project, sample, libprep)) else: LOG.error(e) continue for seqrun in libprep: if delete_existing: LOG.warn('Deleting existing seqrun "{}"'.format(seqrun)) try: charon_session.seqrun_delete(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name) except CharonError as e: LOG.error('Could not delete seqrun "{}": {}'.format(seqrun, e)) try: alignment_status="NOT_RUNNING" LOG.info('Creating seqrun "{}" with alignment_status "{}"'.format(seqrun, alignment_status)) charon_session.seqrun_create(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name, alignment_status=alignment_status, total_reads=0, mean_autosomal_coverage=0) LOG.info(('Project/sample/libprep/seqrun "{}/{}/{}/{}" ' 'created in Charon').format(project, sample, libprep, seqrun)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}" / libprep "{}" / ' 'seqrun "{}"'.format(project, sample, libprep, seqrun)) charon_session.seqrun_update(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name, alignment_status=alignment_status, total_reads=0, mean_autosomal_coverage=0) LOG.info(('Project/sample/libprep/seqrun "{}/{}/{}/{}" ' 'updated in Charon').format(project, sample, libprep, seqrun)) else: LOG.info('Project "{}" / sample "{}" / libprep "{}" / ' 'seqrun "{}" already exists; next...'.format(project, sample, libprep, seqrun)) else: LOG.error(e) continue
def create_charon_entries_from_project(project, workflow="NGI", force_overwrite=False): """Given a project object, creates the relevant entries in Charon. :param NGIProject project: The NGIProject object :param str workflow: The workflow to assign for this project (default NGI) :param bool force_overwrite: If this is set to true, overwrite existing entries in Charon (default false) """ charon_session = CharonSession() try: status="SEQUENCED" LOG.info('Creating project "{}" with status "{}" and workflow "{}"'.format(project, status, workflow)) charon_session.project_create(projectid=project.project_id, name=project.name, status=status, pipeline=workflow) except CharonError: if force_overwrite: LOG.warn('Overwriting data for project "{}"'.format(project)) charon_session.project_update(projectid=project.project_id, name=project.name, status=status, pipeline=workflow) else: LOG.info('Project "{}" already exists; moving to samples...'.format(project)) for sample in project: try: LOG.info('Creating sample "{}"'.format(sample)) charon_session.sample_create(projectid=project.project_id, sampleid=sample.name, status="NEW") except CharonError: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}"'.format(project, sample)) charon_session.sample_update(projectid=project.project_id, sampleid=sample.name, status="NEW") else: LOG.info('Project "{}" / sample "{}" already exists; moving ' 'to libpreps'.format(project, sample)) for libprep in sample: try: LOG.info('Creating libprep "{}"'.format(libprep)) charon_session.libprep_create(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, status="NEW") except CharonError: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}" / libprep "{}"'.format(project, sample, libprep)) charon_session.libprep_update(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, status="NEW") else: LOG.info('Project "{}" / sample "{}" / libprep "{}" already ' 'exists; moving to libpreps'.format(project, sample, libprep)) for seqrun in libprep: try: LOG.info('Creating seqrun "{}"'.format(seqrun)) charon_session.seqrun_create(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name, total_reads=0, mean_autosomal_coverage=0, sequencing_status="DONE", alignment_status="NEW") except CharonError as e: if force_overwrite: LOG.warn('Overwriting data for project "{}" / ' 'sample "{}" / libprep "{}" / ' 'seqrun "{}"'.format(project, sample, libprep, seqrun)) charon_session.seqrun_update(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name, status="NEW") else: LOG.info('Project "{}" / sample "{}" / libprep "{}" / ' 'seqrun "{}" already exists; next...'.format(project, sample, libprep, seqrun))
def create_charon_entries_from_project( project, best_practice_analysis="whole_genome_reseq", sequencing_facility="NGI-S", force_overwrite=False, retry_on_fail=True): """Given a project object, creates the relevant entries in Charon. This code is remarkably shoddy as I created it in a hurry and then later it became a part of the pipeline. Use at your own risk! Ha ha. :param NGIProject project: The NGIProject object :param str best_practice_analysis: The workflow to assign for this project (default "variant_calling") :param str sequencing_facility: The facility that did the sequencing :param bool force_overwrite: If this is set to true, overwrite existing entries in Charon (default false) """ charon_session = CharonSession() update_failed = False try: status = "OPEN" LOG.info( 'Creating project "{}" with status "{}", best practice analysis "{}", ' 'and sequencing_facility {}'.format(project, status, best_practice_analysis, sequencing_facility)) charon_session.project_create( projectid=project.project_id, name=project.name, status=status, best_practice_analysis=best_practice_analysis, sequencing_facility=sequencing_facility) LOG.info('Project "{}" created in Charon.'.format(project)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warning( 'Overwriting data for project "{}"'.format(project)) charon_session.project_update( projectid=project.project_id, name=project.name, status=status, best_practice_analysis=best_practice_analysis, sequencing_facility=sequencing_facility) LOG.info('Project "{}" updated in Charon.'.format(project)) else: LOG.info( 'Project "{}" already exists; moving to samples...'.format( project)) else: raise for sample in project: try: analysis_status = "TO_ANALYZE" sample_data_status_value = "STALE" LOG.info('Creating sample "{}" with analysis_status "{}"'.format( sample, analysis_status)) charon_session.sample_create(projectid=project.project_id, sampleid=sample.name, analysis_status=analysis_status) LOG.info('Project/sample "{}/{}" created in Charon.'.format( project, sample)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warning('Overwriting data for project "{}" / ' 'sample "{}"'.format(project, sample)) charon_session.sample_update( projectid=project.project_id, sampleid=sample.name, analysis_status=analysis_status, status=sample_data_status_value) LOG.info( 'Project/sample "{}/{}" updated in Charon.'.format( project, sample)) else: #update the status of the sample to STALE charon_session.sample_update( projectid=project.project_id, sampleid=sample.name, status=sample_data_status_value) LOG.info( 'Project "{}" / sample "{}" already exists; moving ' 'to libpreps'.format(project, sample)) else: update_failed = True LOG.error(e) continue for libprep in sample: try: qc = "PASSED" LOG.info('Creating libprep "{}" with qc status "{}"'.format( libprep, qc)) charon_session.libprep_create(projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, qc=qc) LOG.info(('Project/sample/libprep "{}/{}/{}" created in ' 'Charon').format(project, sample, libprep)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warning('Overwriting data for project "{}" / ' 'sample "{}" / libprep "{}"'.format( project, sample, libprep)) charon_session.libprep_update( projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, qc=qc) LOG.info( ('Project/sample/libprep "{}/{}/{}" updated in ' 'Charon').format(project, sample, libprep)) else: LOG.debug(e) LOG.info( 'Project "{}" / sample "{}" / libprep "{}" already ' 'exists; moving to libpreps'.format( project, sample, libprep)) else: update_failed = True LOG.error(e) continue for seqrun in libprep: try: alignment_status = "NOT_RUNNING" LOG.info('Creating seqrun "{}" with alignment_status "{}"'. format(seqrun, alignment_status)) charon_session.seqrun_create( projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name, alignment_status=alignment_status, total_reads=0, mean_autosomal_coverage=0) LOG.info( ('Project/sample/libprep/seqrun "{}/{}/{}/{}" ' 'created in Charon').format(project, sample, libprep, seqrun)) except CharonError as e: if e.status_code == 400: if force_overwrite: LOG.warning('Overwriting data for project "{}" / ' 'sample "{}" / libprep "{}" / ' 'seqrun "{}"'.format( project, sample, libprep, seqrun)) charon_session.seqrun_update( projectid=project.project_id, sampleid=sample.name, libprepid=libprep.name, seqrunid=seqrun.name, alignment_status=alignment_status, total_reads=0, mean_autosomal_coverage=0) LOG.info( ('Project/sample/libprep/seqrun "{}/{}/{}/{}" ' 'updated in Charon').format( project, sample, libprep, seqrun)) else: LOG.info( 'Project "{}" / sample "{}" / libprep "{}" / ' 'seqrun "{}" already exists; next...'.format( project, sample, libprep, seqrun)) else: update_failed = True LOG.error(e) continue if update_failed: if retry_on_fail: create_charon_entries_from_project( project, best_practice_analysis=best_practice_analysis, sequencing_facility=sequencing_facility, force_overwrite=force_overwrite, retry_on_fail=False) else: raise CharonError("A network error blocks Charon updating.")