def test_upload_genotype( upload_context: CGConfig, case_id: str, cli_runner: CliRunner, analysis_store_trio: Store, upload_genotypes_hk_api: HousekeeperAPI, caplog, ): """Test to upload genotypes via the CLI""" caplog.set_level(logging.DEBUG) # GIVEN a context with a case that is ready for upload sequence genotypes upload_context.status_db_ = analysis_store_trio upload_context.housekeeper_api_ = upload_genotypes_hk_api case_obj = upload_context.status_db.family(case_id) assert case_obj # WHEN uploading the genotypes result = cli_runner.invoke(upload_genotypes_cmd, [case_id], obj=upload_context) # THEN check that the command exits with success assert result.exit_code == 0 # THEN assert the correct information is communicated assert "loading VCF genotypes for sample(s):" in caplog.text
def fixture_nipt_upload_api_context( cg_context: CGConfig, nipt_stats_api: StatsAPI, re_sequenced_sample_store: Store) -> CGConfig: cg_context.status_db_ = re_sequenced_sample_store cg_context.cg_stats_api_ = nipt_stats_api return cg_context
def fixture_populated_compress_context(compress_api: CompressAPI, populated_compress_store: Store, cg_config_object: CGConfig) -> CGConfig: """Return a compress context populated with a completed analysis""" # Make sure that there is a case where analysis is completed cg_config_object.meta_apis["compress_api"] = compress_api cg_config_object.status_db_ = populated_compress_store return cg_config_object
def fixture_populated_mip_context( base_context: CGConfig, analysis_store: Store, mip_dna_housekeeper: HousekeeperAPI, project_dir: Path, ) -> CGConfig: base_context.housekeeper_api_ = mip_dna_housekeeper base_context.status_db_ = analysis_store base_context.delivery_path = str(project_dir) return base_context
def fixture_demultiplex_context( demultiplexing_api: DemultiplexingAPI, stats_api: StatsAPI, real_housekeeper_api: HousekeeperAPI, cg_context: CGConfig, ) -> CGConfig: cg_context.demultiplex_api_ = demultiplexing_api cg_context.cg_stats_api_ = stats_api cg_context.housekeeper_api_ = real_housekeeper_api return cg_context
def transfer_group(context: CGConfig): """Transfer results to the status interface.""" lims_api: LimsAPI = context.lims_api status_db: Store = context.status_db hk_api: HousekeeperAPI = context.housekeeper_api stats_api: StatsAPI = context.cg_stats_api context.meta_apis["transfer_flowcell_api"] = TransferFlowcell( db=status_db, stats_api=stats_api, hk_api=hk_api ) context.meta_apis["transfer_lims_api"] = TransferLims(status=status_db, lims=lims_api)
def fixture_upload_genotypes_context( upload_genotypes_hk_api: HousekeeperAPI, genotype_api: GenotypeAPI, analysis_store_trio: Store, base_context: CGConfig, ) -> CGConfig: """Create a upload genotypes context""" base_context.genotype_api_ = genotype_api base_context.housekeeper_api_ = upload_genotypes_hk_api base_context.status_db_ = analysis_store_trio return base_context
def fixture_rna_mip_context( cg_context: CGConfig, analysis_family_single_case: dict, helpers: StoreHelpers, apptag_rna: str, case_id: str, housekeeper_api: HousekeeperAPI, ) -> CGConfig: cg_context.housekeeper_api_ = housekeeper_api analysis_family_single_case["data_analysis"] = str(Pipeline.MIP_RNA) if not cg_context.status_db.family(case_id): helpers.ensure_case_from_dict(cg_context.status_db, case_info=analysis_family_single_case, app_tag=apptag_rna) cg_context.meta_apis["analysis_api"] = MipRNAAnalysisAPI(cg_context) return cg_context
def test_cli_init(cli_runner: CliRunner, base_context: CGConfig, caplog): caplog.set_level(logging.INFO) # GIVEN you want to setup a new database using the CLI database = "./test_db.sqlite3" database_path = Path(database) database_uri = f"sqlite:///{database}" base_context.status_db_ = Store(uri=database_uri) with cli_runner.isolated_filesystem(): assert database_path.exists() is False # WHEN calling "init" result = cli_runner.invoke(init, [], obj=base_context) # THEN it should setup the database with some tables assert result.exit_code == 0 assert database_path.exists() assert len(Store(database_uri).engine.table_names()) > 0 # GIVEN the database already exists # WHEN calling the init function result = cli_runner.invoke(init, [], obj=base_context) # THEN it should print an error and give error exit code assert result.exit_code != 0 assert "Database already exists" in caplog.text # GIVEN the database already exists # WHEN calling "init" with "--reset" result = cli_runner.invoke(init, ["--reset"], input="Yes", obj=base_context) # THEN it should re-setup the tables and print new tables assert result.exit_code == 0 assert "Success!" in caplog.text
def vogue(context: CGConfig): """Load trending data into trending database""" if not context.meta_apis.get("analysis_api"): context.meta_apis["analysis_api"] = MipDNAAnalysisAPI(context) click.echo(click.style("----------------- TRENDING -----------------------"))
def test_select_command( cli_runner: CliRunner, populated_stats_api: StatsAPI, demux_results_finished_dir: Path, flowcell_object: Flowcell, demultiplex_context: CGConfig, ): demultiplex_context.cg_stats_api_ = populated_stats_api # GIVEN a stats api with some information about a flowcell flowcell_id: str = flowcell_object.flowcell_id full_flowcell_name: str = flowcell_object.flowcell_full_name assert find.get_flowcell_id(flowcell_id) demux_results = DemuxResults( demux_dir=demux_results_finished_dir / full_flowcell_name, flowcell=flowcell_object, bcl_converter="bcl2fastq", ) # GIVEN a project id project_id: str = "" for project in demux_results.projects: project_id = project.split("_")[-1] assert project_id # WHEN exporting the sample information result = cli_runner.invoke(select_project_cmd, [flowcell_id, "--project", project_id], obj=demultiplex_context) # THEN assert that the command exits with success assert result.exit_code == 0 # THEN assert that the flowcell id if in the printing assert flowcell_id in result.output
def test_create_sample_sheet_no_run_parameters( cli_runner: testing.CliRunner, flowcell_working_directory_no_run_parameters: Path, sample_sheet_context: CGConfig, caplog, mocker, ): # GIVEN a folder with a non existing sample sheet flowcell_object: Flowcell = Flowcell( flowcell_working_directory_no_run_parameters) assert flowcell_object.run_parameters_path.exists() is False mocker.patch("cg.cli.demultiplex.sample_sheet.flowcell_samples", return_value=[{ "sample": 1 }]) demux_api: DemultiplexingAPI = sample_sheet_context.demultiplex_api demux_api.run_dir = flowcell_working_directory_no_run_parameters.parent sample_sheet_context.demultiplex_api_ = demux_api # WHEN running the create sample sheet command result: testing.Result = cli_runner.invoke( create_sheet, [flowcell_object.flowcell_full_name], obj=sample_sheet_context) # THEN assert it exits with a non zero exit code assert result.exit_code != 0 # THEN assert the correct information is communicated assert "Could not find run parameters file" in caplog.text
def test_instantiate_correct_configs(base_config_dict: dict): # GIVEN a dictionary with the basic configs # WHEN instantiating a CGConfig object config_object = CGConfig(**base_config_dict) # THEN assert that it was successfully created assert isinstance(config_object, CGConfig)
def fixture_backup_context(cg_context: CGConfig) -> CGConfig: cg_context.meta_apis["backup_api"] = BackupApi( status=cg_context.status_db, pdc_api=PdcApi(), max_flowcells_on_disk=cg_context.max_flowcells or MAX_FLOWCELLS_ON_DISK, root_dir=cg_context.backup.root.dict(), ) return cg_context
def backup(context: CGConfig): """Backup utilities.""" pdc_api = PdcApi() context.meta_apis["backup_api"] = BackupApi( status=context.status_db, pdc_api=pdc_api, max_flowcells_on_disk=context.max_flowcells or MAX_FLOWCELLS_ON_DISK, root_dir=context.backup.root.dict(), )
def compress(context: CGConfig): """Compress files""" hk_api = context.housekeeper_api crunchy_api = context.crunchy_api compress_api = CompressAPI(hk_api=hk_api, crunchy_api=crunchy_api, demux_root=context.demultiplex.out_dir) context.meta_apis["compress_api"] = compress_api
def test_delete_flow_cell_housekeeper_only_sample_level( caplog, cg_context: CGConfig, demultiplexed_flowcells_working_directory: Path, flowcell_full_name: str, populated_flow_cell_store: Store, sample_level_housekeeper_api: HousekeeperAPI, tmp_fastq_paths: List[Path], ): """Test function to remove fastqs from Housekeeper when there are only files on sample level (not on flow cell name) """ caplog.set_level(logging.INFO) cg_context.housekeeper_api_ = sample_level_housekeeper_api cg_context.status_db_ = populated_flow_cell_store # GIVEN a DeleteDemuxAPI with a HousekeeperAPI with no files with flow cell name as a tag sample_level_files: List[Path] = tmp_fastq_paths wipe_demultiplex_api: DeleteDemuxAPI = DeleteDemuxAPI( config=cg_context, demultiplex_base=demultiplexed_flowcells_working_directory, dry_run=False, run_path=Path(flowcell_full_name), ) wipe_demultiplex_api._set_samples_on_flow_cell() # WHEN wiping files in Housekeeper wipe_demultiplex_api.delete_flow_cell_housekeeper() # THEN you should be notified that there are no files on flow cell names assert ( f"Housekeeper: No files found with tag: {wipe_demultiplex_api.flow_cell_name}" in caplog.text) # AND you should be notified that there were fastq files removed on sample level for file in sample_level_files: assert f"{file.as_posix()} deleted" in caplog.text
def decompress(context: CGConfig): """Decompress files""" hk_api = context.housekeeper_api crunchy_api = context.crunchy_api compress_api = CompressAPI(hk_api=hk_api, crunchy_api=crunchy_api, demux_root=context.demultiplex.out_dir) context.meta_apis["compress_api"] = compress_api LOG.info("Running decompress spring")
def store(context: CGConfig): """Command for storing files""" LOG.info("Running CG store command") housekeeper_api: HousekeeperAPI = context.housekeeper_api crunchy_api: CrunchyAPI = context.crunchy_api compress_api = CompressAPI(hk_api=housekeeper_api, crunchy_api=crunchy_api, demux_root=context.demultiplex.out_dir) context.meta_apis["compress_api"] = compress_api
def upload_context(cg_context: CGConfig) -> CGConfig: analysis_api = MipDNAAnalysisAPI(config=cg_context) cg_context.meta_apis["analysis_api"] = analysis_api cg_context.meta_apis["report_api"] = ReportAPI( store=cg_context.status_db, lims_api=cg_context.lims_api, chanjo_api=cg_context.chanjo_api, analysis_api=analysis_api, scout_api=cg_context.scout_api, ) cg_context.meta_apis["scout_upload_api"] = UploadScoutAPI( hk_api=cg_context.housekeeper_api, scout_api=cg_context.scout_api, madeline_api=cg_context.madeline_api, analysis_api=analysis_api, lims_api=cg_context.lims_api, status_db=cg_context.status_db, ) return cg_context
def test_fetching_the_status_db(base_config_dict: dict, caplog): caplog.set_level(logging.DEBUG) # GIVEN a dictionary with the basic configs # WHEN instantiating a CGConfig object config_object = CGConfig(**base_config_dict) # THEN assert that the status db exists assert isinstance(config_object.status_db, Store) # THEN assert that it was communicated that it was instantiated assert "Instantiating status db" in caplog.text
def test_delete_flow_cell_housekeeper_flowcell_name( caplog, cg_context: CGConfig, demultiplexed_flowcells_working_directory: Path, flow_cell_name_housekeeper_api: HousekeeperAPI, flowcell_full_name: str, populated_flow_cell_store: Store, tmp_fastq_paths: List[Path], tmp_sample_sheet_path: Path, ): """Test function to remove files from Housekeeper using flow cell name as a tag""" caplog.set_level(logging.INFO) cg_context.housekeeper_api_ = flow_cell_name_housekeeper_api cg_context.status_db_ = populated_flow_cell_store # GIVEN fastq_files: List[Path] = tmp_fastq_paths sample_sheet_file: Path = tmp_sample_sheet_path wipe_demultiplex_api: DeleteDemuxAPI = DeleteDemuxAPI( config=cg_context, demultiplex_base=demultiplexed_flowcells_working_directory, dry_run=False, run_path=Path(flowcell_full_name), ) wipe_demultiplex_api._set_samples_on_flow_cell() # WHEN wipe_demultiplex_api.delete_flow_cell_housekeeper() # THEN assert ( f"Housekeeper: No files found with tag: {wipe_demultiplex_api.flow_cell_name}" not in caplog.text) assert f"Deleted {sample_sheet_file.as_posix()} from housekeeper" in caplog.text for fastq_file in fastq_files: assert f"{fastq_file.as_posix()} deleted" in caplog.text
def fixture_dna_mip_context( cg_context: CGConfig, helpers: StoreHelpers, mip_case_ids: dict, real_housekeeper_api: HousekeeperAPI, tb_api, ) -> CGConfig: _store = cg_context.status_db cg_context.housekeeper_api_ = real_housekeeper_api cg_context.trailblazer_api_ = tb_api mip_analysis_api = MipDNAAnalysisAPI(config=cg_context) # Add apptag to db helpers.ensure_application_version(store=_store, application_tag="WGSA", application_type="wgs") # Add sample, cases and relationships to db for case_id in mip_case_ids: if not _store.family(case_id): case_obj = helpers.add_case( store=_store, data_analysis=Pipeline.MIP_DNA, internal_id=case_id, name=mip_case_ids[case_id]["name"], ) sample = helpers.add_sample( store=_store, customer_id="cust000", application_tag="WGSA", application_type="wgs", gender="unknown", ) helpers.add_relationship(store=_store, sample=sample, case=case_obj, status="affected") cg_context.meta_apis["analysis_api"] = mip_analysis_api return cg_context
def fixture_wipe_demultiplex_api( cg_context: CGConfig, demultiplexed_flowcells_working_directory: Path, flowcell_full_name: str, stats_api: StatsAPI, ) -> DeleteDemuxAPI: """Yield an initialized DeleteDemuxAPI""" cg_context.cg_stats_api_ = stats_api return DeleteDemuxAPI( config=cg_context, demultiplex_base=demultiplexed_flowcells_working_directory, dry_run=False, run_path=Path(flowcell_full_name), )
def test_api_instantiated_once(base_config_dict: dict, caplog): caplog.set_level(logging.DEBUG) # GIVEN a dictionary with the basic configs # GIVEN a CGConfig object config_object = CGConfig(**base_config_dict) # WHEN fetching (instantiating) the status_db # THEN assert that it was only instantiated once config_object.status_db assert "Instantiating status db" in caplog.text caplog.clear() config_object.status_db assert "Instantiating status db" not in caplog.text
def fluffy_context( cg_context: CGConfig, helpers: StoreHelpers, real_housekeeper_api: HousekeeperAPI, fluffy_samplesheet_bundle_data, fluffy_fastq_hk_bundle_data, fluffy_case_id_existing, fluffy_sample_lims_id, ) -> CGConfig: cg_context.housekeeper_api_ = real_housekeeper_api fluffy_analysis_api = FluffyAnalysisAPI(config=cg_context) helpers.ensure_hk_version(fluffy_analysis_api.housekeeper_api, bundle_data=fluffy_samplesheet_bundle_data) helpers.ensure_hk_version(fluffy_analysis_api.housekeeper_api, fluffy_fastq_hk_bundle_data) example_fluffy_case = helpers.add_case( fluffy_analysis_api.status_db, internal_id=fluffy_case_id_existing, name=fluffy_case_id_existing, data_analysis=Pipeline.FLUFFY, ) example_fluffy_sample = helpers.add_sample( fluffy_analysis_api.status_db, internal_id=fluffy_sample_lims_id, is_tumour=False, application_type="tgs", reads=100, sequenced_at=dt.datetime.now(), ) helpers.add_flowcell(fluffy_analysis_api.status_db, flowcell_id="flowcell", samples=[example_fluffy_sample]) helpers.add_relationship(fluffy_analysis_api.status_db, case=example_fluffy_case, sample=example_fluffy_sample) cg_context.meta_apis["analysis_api"] = fluffy_analysis_api return cg_context
def base( context: click.Context, config: click.File, database: Optional[str], log_level: str, verbose: bool, ): """cg - interface between tools at Clinical Genomics.""" if verbose: log_format = "%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s" else: log_format = "%(message)s" if sys.stdout.isatty() else None coloredlogs.install(level=log_level, fmt=log_format) raw_configs: dict = yaml.full_load(config) if config else {"database": database} context.obj = CGConfig(**raw_configs)
def test_running_deploy_scout(shipping_context: CGConfig, scout_config: Path, caplog): """Test to deploy scout with CG""" caplog.set_level(logging.DEBUG) # GIVEN a context with some shipping configs shipping_context.scout = CommonAppConfig(binary_path="echo", deploy_config=str(scout_config)) # GIVEN a cli runner runner = CliRunner() # WHEN running the deploy scout command in dry run mode result = runner.invoke(deploy, ["--dry-run", "scout"], obj=shipping_context) # THEN assert that the command exits without problems assert result.exit_code == 0
def test_store( cli_runner: CliRunner, balsamic_context: CGConfig, real_housekeeper_api, mock_config, mock_deliverable, mock_analysis_finish, caplog, hermes_deliverables, mocker, ): """Test to ensure all parts of store command are run successfully given ideal conditions""" caplog.set_level(logging.INFO) # GIVEN case-id for which we created a config file, deliverables file, and analysis_finish file case_id = "balsamic_case_wgs_single" # Set Housekeeper to an empty real Housekeeper store balsamic_context.housekeeper_api_ = real_housekeeper_api balsamic_context.meta_apis[ "analysis_api"].housekeeper_api = real_housekeeper_api # Make sure the bundle was not present in the store assert not balsamic_context.housekeeper_api.bundle(case_id) # Make sure analysis not already stored in ClinicalDB assert not balsamic_context.status_db.family(case_id).analyses # GIVEN that HermesAPI returns a deliverables output mocker.patch.object(HermesApi, "convert_deliverables") HermesApi.convert_deliverables.return_value = CGDeliverables( **hermes_deliverables) # WHEN running command result = cli_runner.invoke(store, [case_id, "--dry-run"], obj=balsamic_context) # THEN bundle should be successfully added to HK and STATUS assert result.exit_code == EXIT_SUCCESS assert "Analysis successfully stored in Housekeeper" in caplog.text assert "Analysis successfully stored in StatusDB" in caplog.text assert balsamic_context.status_db.family(case_id).analyses assert balsamic_context.housekeeper_api.bundle(case_id)
def test_set_dry_run_delete_demux_api( caplog, cg_context: CGConfig, demultiplexed_flowcells_working_directory: Path, flowcell_full_name: str, stats_api: StatsAPI, ): """Test to test function to set the API to run in dry run mode""" caplog.set_level(logging.DEBUG) cg_context.cg_stats_api_ = stats_api # WHEN setting the dry_run mode on a DeleteDemuxAPI wipe_demultiplex_api: DeleteDemuxAPI = DeleteDemuxAPI( config=cg_context, demultiplex_base=demultiplexed_flowcells_working_directory, dry_run=True, run_path=flowcell_full_name, ) # THEN the dry run parameter should be set to True and it should be logged assert wipe_demultiplex_api.dry_run assert f"DeleteDemuxAPI: Setting dry run mode to True" in caplog.text