def test_upload_genotype(
upload_context: CGConfig,
case_id: str,
cli_runner: CliRunner,
analysis_store_trio: Store,
upload_genotypes_hk_api: HousekeeperAPI,
caplog,
):
"""Test to upload genotypes via the CLI"""
caplog.set_level(logging.DEBUG)
# GIVEN a context with a case that is ready for upload sequence genotypes
upload_context.status_db_ = analysis_store_trio
upload_context.housekeeper_api_ = upload_genotypes_hk_api
case_obj = upload_context.status_db.family(case_id)
assert case_obj
# WHEN uploading the genotypes
result = cli_runner.invoke(upload_genotypes_cmd, [case_id],
obj=upload_context)
# THEN check that the command exits with success
assert result.exit_code == 0
# THEN assert the correct information is communicated
assert "loading VCF genotypes for sample(s):" in caplog.text
def fixture_nipt_upload_api_context(
cg_context: CGConfig, nipt_stats_api: StatsAPI,
re_sequenced_sample_store: Store) -> CGConfig:
cg_context.status_db_ = re_sequenced_sample_store
cg_context.cg_stats_api_ = nipt_stats_api
return cg_context
def fixture_populated_compress_context(compress_api: CompressAPI,
populated_compress_store: Store,
cg_config_object: CGConfig) -> CGConfig:
"""Return a compress context populated with a completed analysis"""
# Make sure that there is a case where analysis is completed
cg_config_object.meta_apis["compress_api"] = compress_api
cg_config_object.status_db_ = populated_compress_store
return cg_config_object
def fixture_populated_mip_context(
base_context: CGConfig,
analysis_store: Store,
mip_dna_housekeeper: HousekeeperAPI,
project_dir: Path,
) -> CGConfig:
base_context.housekeeper_api_ = mip_dna_housekeeper
base_context.status_db_ = analysis_store
base_context.delivery_path = str(project_dir)
return base_context
def fixture_demultiplex_context(
demultiplexing_api: DemultiplexingAPI,
stats_api: StatsAPI,
real_housekeeper_api: HousekeeperAPI,
cg_context: CGConfig,
) -> CGConfig:
cg_context.demultiplex_api_ = demultiplexing_api
cg_context.cg_stats_api_ = stats_api
cg_context.housekeeper_api_ = real_housekeeper_api
return cg_context
def transfer_group(context: CGConfig):
"""Transfer results to the status interface."""
lims_api: LimsAPI = context.lims_api
status_db: Store = context.status_db
hk_api: HousekeeperAPI = context.housekeeper_api
stats_api: StatsAPI = context.cg_stats_api
context.meta_apis["transfer_flowcell_api"] = TransferFlowcell(
db=status_db, stats_api=stats_api, hk_api=hk_api
)
context.meta_apis["transfer_lims_api"] = TransferLims(status=status_db, lims=lims_api)
def fixture_upload_genotypes_context(
upload_genotypes_hk_api: HousekeeperAPI,
genotype_api: GenotypeAPI,
analysis_store_trio: Store,
base_context: CGConfig,
) -> CGConfig:
"""Create a upload genotypes context"""
base_context.genotype_api_ = genotype_api
base_context.housekeeper_api_ = upload_genotypes_hk_api
base_context.status_db_ = analysis_store_trio
return base_context
def fixture_rna_mip_context(
cg_context: CGConfig,
analysis_family_single_case: dict,
helpers: StoreHelpers,
apptag_rna: str,
case_id: str,
housekeeper_api: HousekeeperAPI,
) -> CGConfig:
cg_context.housekeeper_api_ = housekeeper_api
analysis_family_single_case["data_analysis"] = str(Pipeline.MIP_RNA)
if not cg_context.status_db.family(case_id):
helpers.ensure_case_from_dict(cg_context.status_db,
case_info=analysis_family_single_case,
app_tag=apptag_rna)
cg_context.meta_apis["analysis_api"] = MipRNAAnalysisAPI(cg_context)
return cg_context
def test_cli_init(cli_runner: CliRunner, base_context: CGConfig, caplog):
caplog.set_level(logging.INFO)
# GIVEN you want to setup a new database using the CLI
database = "./test_db.sqlite3"
database_path = Path(database)
database_uri = f"sqlite:///{database}"
base_context.status_db_ = Store(uri=database_uri)
with cli_runner.isolated_filesystem():
assert database_path.exists() is False
# WHEN calling "init"
result = cli_runner.invoke(init, [], obj=base_context)
# THEN it should setup the database with some tables
assert result.exit_code == 0
assert database_path.exists()
assert len(Store(database_uri).engine.table_names()) > 0
# GIVEN the database already exists
# WHEN calling the init function
result = cli_runner.invoke(init, [], obj=base_context)
# THEN it should print an error and give error exit code
assert result.exit_code != 0
assert "Database already exists" in caplog.text
# GIVEN the database already exists
# WHEN calling "init" with "--reset"
result = cli_runner.invoke(init, ["--reset"],
input="Yes",
obj=base_context)
# THEN it should re-setup the tables and print new tables
assert result.exit_code == 0
assert "Success!" in caplog.text
def vogue(context: CGConfig):
"""Load trending data into trending database"""
if not context.meta_apis.get("analysis_api"):
context.meta_apis["analysis_api"] = MipDNAAnalysisAPI(context)
click.echo(click.style("----------------- TRENDING -----------------------"))
def test_select_command(
cli_runner: CliRunner,
populated_stats_api: StatsAPI,
demux_results_finished_dir: Path,
flowcell_object: Flowcell,
demultiplex_context: CGConfig,
):
demultiplex_context.cg_stats_api_ = populated_stats_api
# GIVEN a stats api with some information about a flowcell
flowcell_id: str = flowcell_object.flowcell_id
full_flowcell_name: str = flowcell_object.flowcell_full_name
assert find.get_flowcell_id(flowcell_id)
demux_results = DemuxResults(
demux_dir=demux_results_finished_dir / full_flowcell_name,
flowcell=flowcell_object,
bcl_converter="bcl2fastq",
)
# GIVEN a project id
project_id: str = ""
for project in demux_results.projects:
project_id = project.split("_")[-1]
assert project_id
# WHEN exporting the sample information
result = cli_runner.invoke(select_project_cmd,
[flowcell_id, "--project", project_id],
obj=demultiplex_context)
# THEN assert that the command exits with success
assert result.exit_code == 0
# THEN assert that the flowcell id if in the printing
assert flowcell_id in result.output
def test_create_sample_sheet_no_run_parameters(
cli_runner: testing.CliRunner,
flowcell_working_directory_no_run_parameters: Path,
sample_sheet_context: CGConfig,
caplog,
mocker,
):
# GIVEN a folder with a non existing sample sheet
flowcell_object: Flowcell = Flowcell(
flowcell_working_directory_no_run_parameters)
assert flowcell_object.run_parameters_path.exists() is False
mocker.patch("cg.cli.demultiplex.sample_sheet.flowcell_samples",
return_value=[{
"sample": 1
}])
demux_api: DemultiplexingAPI = sample_sheet_context.demultiplex_api
demux_api.run_dir = flowcell_working_directory_no_run_parameters.parent
sample_sheet_context.demultiplex_api_ = demux_api
# WHEN running the create sample sheet command
result: testing.Result = cli_runner.invoke(
create_sheet, [flowcell_object.flowcell_full_name],
obj=sample_sheet_context)
# THEN assert it exits with a non zero exit code
assert result.exit_code != 0
# THEN assert the correct information is communicated
assert "Could not find run parameters file" in caplog.text
def test_instantiate_correct_configs(base_config_dict: dict):
# GIVEN a dictionary with the basic configs
# WHEN instantiating a CGConfig object
config_object = CGConfig(**base_config_dict)
# THEN assert that it was successfully created
assert isinstance(config_object, CGConfig)
def fixture_backup_context(cg_context: CGConfig) -> CGConfig:
cg_context.meta_apis["backup_api"] = BackupApi(
status=cg_context.status_db,
pdc_api=PdcApi(),
max_flowcells_on_disk=cg_context.max_flowcells
or MAX_FLOWCELLS_ON_DISK,
root_dir=cg_context.backup.root.dict(),
)
return cg_context
def backup(context: CGConfig):
"""Backup utilities."""
pdc_api = PdcApi()
context.meta_apis["backup_api"] = BackupApi(
status=context.status_db,
pdc_api=pdc_api,
max_flowcells_on_disk=context.max_flowcells or MAX_FLOWCELLS_ON_DISK,
root_dir=context.backup.root.dict(),
)
def compress(context: CGConfig):
"""Compress files"""
hk_api = context.housekeeper_api
crunchy_api = context.crunchy_api
compress_api = CompressAPI(hk_api=hk_api,
crunchy_api=crunchy_api,
demux_root=context.demultiplex.out_dir)
context.meta_apis["compress_api"] = compress_api
def test_delete_flow_cell_housekeeper_only_sample_level(
caplog,
cg_context: CGConfig,
demultiplexed_flowcells_working_directory: Path,
flowcell_full_name: str,
populated_flow_cell_store: Store,
sample_level_housekeeper_api: HousekeeperAPI,
tmp_fastq_paths: List[Path],
):
"""Test function to remove fastqs from Housekeeper when there are only files on sample level
(not on flow cell name)
"""
caplog.set_level(logging.INFO)
cg_context.housekeeper_api_ = sample_level_housekeeper_api
cg_context.status_db_ = populated_flow_cell_store
# GIVEN a DeleteDemuxAPI with a HousekeeperAPI with no files with flow cell name as a tag
sample_level_files: List[Path] = tmp_fastq_paths
wipe_demultiplex_api: DeleteDemuxAPI = DeleteDemuxAPI(
config=cg_context,
demultiplex_base=demultiplexed_flowcells_working_directory,
dry_run=False,
run_path=Path(flowcell_full_name),
)
wipe_demultiplex_api._set_samples_on_flow_cell()
# WHEN wiping files in Housekeeper
wipe_demultiplex_api.delete_flow_cell_housekeeper()
# THEN you should be notified that there are no files on flow cell names
assert (
f"Housekeeper: No files found with tag: {wipe_demultiplex_api.flow_cell_name}"
in caplog.text)
# AND you should be notified that there were fastq files removed on sample level
for file in sample_level_files:
assert f"{file.as_posix()} deleted" in caplog.text
def decompress(context: CGConfig):
"""Decompress files"""
hk_api = context.housekeeper_api
crunchy_api = context.crunchy_api
compress_api = CompressAPI(hk_api=hk_api,
crunchy_api=crunchy_api,
demux_root=context.demultiplex.out_dir)
context.meta_apis["compress_api"] = compress_api
LOG.info("Running decompress spring")
def store(context: CGConfig):
"""Command for storing files"""
LOG.info("Running CG store command")
housekeeper_api: HousekeeperAPI = context.housekeeper_api
crunchy_api: CrunchyAPI = context.crunchy_api
compress_api = CompressAPI(hk_api=housekeeper_api,
crunchy_api=crunchy_api,
demux_root=context.demultiplex.out_dir)
context.meta_apis["compress_api"] = compress_api
def upload_context(cg_context: CGConfig) -> CGConfig:
analysis_api = MipDNAAnalysisAPI(config=cg_context)
cg_context.meta_apis["analysis_api"] = analysis_api
cg_context.meta_apis["report_api"] = ReportAPI(
store=cg_context.status_db,
lims_api=cg_context.lims_api,
chanjo_api=cg_context.chanjo_api,
analysis_api=analysis_api,
scout_api=cg_context.scout_api,
)
cg_context.meta_apis["scout_upload_api"] = UploadScoutAPI(
hk_api=cg_context.housekeeper_api,
scout_api=cg_context.scout_api,
madeline_api=cg_context.madeline_api,
analysis_api=analysis_api,
lims_api=cg_context.lims_api,
status_db=cg_context.status_db,
)
return cg_context
def test_fetching_the_status_db(base_config_dict: dict, caplog):
caplog.set_level(logging.DEBUG)
# GIVEN a dictionary with the basic configs
# WHEN instantiating a CGConfig object
config_object = CGConfig(**base_config_dict)
# THEN assert that the status db exists
assert isinstance(config_object.status_db, Store)
# THEN assert that it was communicated that it was instantiated
assert "Instantiating status db" in caplog.text
def test_delete_flow_cell_housekeeper_flowcell_name(
caplog,
cg_context: CGConfig,
demultiplexed_flowcells_working_directory: Path,
flow_cell_name_housekeeper_api: HousekeeperAPI,
flowcell_full_name: str,
populated_flow_cell_store: Store,
tmp_fastq_paths: List[Path],
tmp_sample_sheet_path: Path,
):
"""Test function to remove files from Housekeeper using flow cell name as a tag"""
caplog.set_level(logging.INFO)
cg_context.housekeeper_api_ = flow_cell_name_housekeeper_api
cg_context.status_db_ = populated_flow_cell_store
# GIVEN
fastq_files: List[Path] = tmp_fastq_paths
sample_sheet_file: Path = tmp_sample_sheet_path
wipe_demultiplex_api: DeleteDemuxAPI = DeleteDemuxAPI(
config=cg_context,
demultiplex_base=demultiplexed_flowcells_working_directory,
dry_run=False,
run_path=Path(flowcell_full_name),
)
wipe_demultiplex_api._set_samples_on_flow_cell()
# WHEN
wipe_demultiplex_api.delete_flow_cell_housekeeper()
# THEN
assert (
f"Housekeeper: No files found with tag: {wipe_demultiplex_api.flow_cell_name}"
not in caplog.text)
assert f"Deleted {sample_sheet_file.as_posix()} from housekeeper" in caplog.text
for fastq_file in fastq_files:
assert f"{fastq_file.as_posix()} deleted" in caplog.text
def fixture_dna_mip_context(
cg_context: CGConfig,
helpers: StoreHelpers,
mip_case_ids: dict,
real_housekeeper_api: HousekeeperAPI,
tb_api,
) -> CGConfig:
_store = cg_context.status_db
cg_context.housekeeper_api_ = real_housekeeper_api
cg_context.trailblazer_api_ = tb_api
mip_analysis_api = MipDNAAnalysisAPI(config=cg_context)
# Add apptag to db
helpers.ensure_application_version(store=_store,
application_tag="WGSA",
application_type="wgs")
# Add sample, cases and relationships to db
for case_id in mip_case_ids:
if not _store.family(case_id):
case_obj = helpers.add_case(
store=_store,
data_analysis=Pipeline.MIP_DNA,
internal_id=case_id,
name=mip_case_ids[case_id]["name"],
)
sample = helpers.add_sample(
store=_store,
customer_id="cust000",
application_tag="WGSA",
application_type="wgs",
gender="unknown",
)
helpers.add_relationship(store=_store,
sample=sample,
case=case_obj,
status="affected")
cg_context.meta_apis["analysis_api"] = mip_analysis_api
return cg_context
def fixture_wipe_demultiplex_api(
cg_context: CGConfig,
demultiplexed_flowcells_working_directory: Path,
flowcell_full_name: str,
stats_api: StatsAPI,
) -> DeleteDemuxAPI:
"""Yield an initialized DeleteDemuxAPI"""
cg_context.cg_stats_api_ = stats_api
return DeleteDemuxAPI(
config=cg_context,
demultiplex_base=demultiplexed_flowcells_working_directory,
dry_run=False,
run_path=Path(flowcell_full_name),
)
def test_api_instantiated_once(base_config_dict: dict, caplog):
caplog.set_level(logging.DEBUG)
# GIVEN a dictionary with the basic configs
# GIVEN a CGConfig object
config_object = CGConfig(**base_config_dict)
# WHEN fetching (instantiating) the status_db
# THEN assert that it was only instantiated once
config_object.status_db
assert "Instantiating status db" in caplog.text
caplog.clear()
config_object.status_db
assert "Instantiating status db" not in caplog.text
def fluffy_context(
cg_context: CGConfig,
helpers: StoreHelpers,
real_housekeeper_api: HousekeeperAPI,
fluffy_samplesheet_bundle_data,
fluffy_fastq_hk_bundle_data,
fluffy_case_id_existing,
fluffy_sample_lims_id,
) -> CGConfig:
cg_context.housekeeper_api_ = real_housekeeper_api
fluffy_analysis_api = FluffyAnalysisAPI(config=cg_context)
helpers.ensure_hk_version(fluffy_analysis_api.housekeeper_api,
bundle_data=fluffy_samplesheet_bundle_data)
helpers.ensure_hk_version(fluffy_analysis_api.housekeeper_api,
fluffy_fastq_hk_bundle_data)
example_fluffy_case = helpers.add_case(
fluffy_analysis_api.status_db,
internal_id=fluffy_case_id_existing,
name=fluffy_case_id_existing,
data_analysis=Pipeline.FLUFFY,
)
example_fluffy_sample = helpers.add_sample(
fluffy_analysis_api.status_db,
internal_id=fluffy_sample_lims_id,
is_tumour=False,
application_type="tgs",
reads=100,
sequenced_at=dt.datetime.now(),
)
helpers.add_flowcell(fluffy_analysis_api.status_db,
flowcell_id="flowcell",
samples=[example_fluffy_sample])
helpers.add_relationship(fluffy_analysis_api.status_db,
case=example_fluffy_case,
sample=example_fluffy_sample)
cg_context.meta_apis["analysis_api"] = fluffy_analysis_api
return cg_context
def base(
context: click.Context,
config: click.File,
database: Optional[str],
log_level: str,
verbose: bool,
):
"""cg - interface between tools at Clinical Genomics."""
if verbose:
log_format = "%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s"
else:
log_format = "%(message)s" if sys.stdout.isatty() else None
coloredlogs.install(level=log_level, fmt=log_format)
raw_configs: dict = yaml.full_load(config) if config else {"database": database}
context.obj = CGConfig(**raw_configs)
def test_running_deploy_scout(shipping_context: CGConfig, scout_config: Path,
caplog):
"""Test to deploy scout with CG"""
caplog.set_level(logging.DEBUG)
# GIVEN a context with some shipping configs
shipping_context.scout = CommonAppConfig(binary_path="echo",
deploy_config=str(scout_config))
# GIVEN a cli runner
runner = CliRunner()
# WHEN running the deploy scout command in dry run mode
result = runner.invoke(deploy, ["--dry-run", "scout"],
obj=shipping_context)
# THEN assert that the command exits without problems
assert result.exit_code == 0
def test_store(
cli_runner: CliRunner,
balsamic_context: CGConfig,
real_housekeeper_api,
mock_config,
mock_deliverable,
mock_analysis_finish,
caplog,
hermes_deliverables,
mocker,
):
"""Test to ensure all parts of store command are run successfully given ideal conditions"""
caplog.set_level(logging.INFO)
# GIVEN case-id for which we created a config file, deliverables file, and analysis_finish file
case_id = "balsamic_case_wgs_single"
# Set Housekeeper to an empty real Housekeeper store
balsamic_context.housekeeper_api_ = real_housekeeper_api
balsamic_context.meta_apis[
"analysis_api"].housekeeper_api = real_housekeeper_api
# Make sure the bundle was not present in the store
assert not balsamic_context.housekeeper_api.bundle(case_id)
# Make sure analysis not already stored in ClinicalDB
assert not balsamic_context.status_db.family(case_id).analyses
# GIVEN that HermesAPI returns a deliverables output
mocker.patch.object(HermesApi, "convert_deliverables")
HermesApi.convert_deliverables.return_value = CGDeliverables(
**hermes_deliverables)
# WHEN running command
result = cli_runner.invoke(store, [case_id, "--dry-run"],
obj=balsamic_context)
# THEN bundle should be successfully added to HK and STATUS
assert result.exit_code == EXIT_SUCCESS
assert "Analysis successfully stored in Housekeeper" in caplog.text
assert "Analysis successfully stored in StatusDB" in caplog.text
assert balsamic_context.status_db.family(case_id).analyses
assert balsamic_context.housekeeper_api.bundle(case_id)
def test_set_dry_run_delete_demux_api(
caplog,
cg_context: CGConfig,
demultiplexed_flowcells_working_directory: Path,
flowcell_full_name: str,
stats_api: StatsAPI,
):
"""Test to test function to set the API to run in dry run mode"""
caplog.set_level(logging.DEBUG)
cg_context.cg_stats_api_ = stats_api
# WHEN setting the dry_run mode on a DeleteDemuxAPI
wipe_demultiplex_api: DeleteDemuxAPI = DeleteDemuxAPI(
config=cg_context,
demultiplex_base=demultiplexed_flowcells_working_directory,
dry_run=True,
run_path=flowcell_full_name,
)
# THEN the dry run parameter should be set to True and it should be logged
assert wipe_demultiplex_api.dry_run
assert f"DeleteDemuxAPI: Setting dry run mode to True" in caplog.text
