def test_upload_rna_coverage_bigwig_to_scout(
caplog: Generator[LogCaptureFixture, None, None],
dna_sample_son_id: str,
mip_rna_analysis_hk_api: HousekeeperAPI,
rna_case_id: str,
rna_sample_son_id: str,
rna_store: Store,
upload_scout_api: UploadScoutAPI,
):
"""Test that A RNA case's bigWig file for all samples can be loaded via a cg CLI
command into an already existing DNA case"""
# GIVEN an existing RNA case with related sample
# GIVEN an existing DNA case with related sample
# GIVEN a sample in the RNA case is connected to a sample in the DNA case via subject_id (i.e. same subject_id)
upload_scout_api.status_db = rna_store
# GIVEN the connected RNA sample has a bigWig in Housekeeper
# WHEN running the method to upload RNA files to Scout
caplog.set_level(logging.INFO)
upload_scout_api.upload_rna_coverage_bigwig_to_scout(case_id=rna_case_id,
dry_run=True)
# THEN the bigWig file should have been uploaded to the connected sample on the dna case in scout
assert "Upload RNA coverage bigwig file finished!" in caplog.text
# THEN the customers dna samples name should have been mentioned in the logging (and used in the upload)
dna_customer_sample_name: str = rna_store.sample(
internal_id=dna_sample_son_id).name
assert dna_customer_sample_name in caplog.text
def test_upload_rna_fusion_report_to_scout_no_subject_id(
caplog: Generator[LogCaptureFixture, None, None],
dna_case_id: str,
mip_rna_analysis_hk_api: HousekeeperAPI,
rna_case_id: str,
rna_store: Store,
upload_scout_api: UploadScoutAPI,
):
"""Test that A RNA case's gene fusion report"""
# GIVEN a sample in the RNA case is NOT connected to a sample in the DNA case via subject_id (i.e. same subject_id)
for link in rna_store.family(rna_case_id).links:
link.sample.subject_id = ""
for link in rna_store.family(dna_case_id).links:
link.sample.subject_id = ""
rna_store.commit()
upload_scout_api.status_db = rna_store
# GIVEN the connected RNA case has a research fusion report in Housekeeper
# WHEN running the method to upload RNA files to Scout
caplog.set_level(logging.INFO)
# THEN an exception should be raised on unconnected data
with pytest.raises(CgDataError):
upload_scout_api.upload_fusion_report_to_scout(case_id=rna_case_id,
dry_run=True)
def test_upload_splice_junctions_bed_to_scout_no_subject_id(
caplog: Generator[LogCaptureFixture, None, None],
dna_case_id: str,
mip_rna_analysis_hk_api: HousekeeperAPI,
rna_case_id: str,
rna_store: Store,
upload_scout_api: UploadScoutAPI,
):
"""Test that A RNA case's junction splice files for all samples can be loaded via a cg CLI
command into an already existing DNA case"""
# GIVEN a sample in the RNA case is NOT connected to a sample in the DNA case via subject_id (i.e. same subject_id)
for link in rna_store.family(rna_case_id).links:
link.sample.subject_id = ""
for link in rna_store.family(dna_case_id).links:
link.sample.subject_id = ""
rna_store.commit()
upload_scout_api.status_db = rna_store
# GIVEN the connected RNA sample has a junction bed in Housekeeper
# WHEN running the method to upload RNA files to Scout
caplog.set_level(logging.INFO)
# THEN an exception should be raised on unconnected data
with pytest.raises(CgDataError):
upload_scout_api.upload_splice_junctions_bed_to_scout(
case_id=rna_case_id, dry_run=True)
def test_upload_rna_junctions_to_scout(
caplog: Generator[LogCaptureFixture, None, None],
mip_rna_analysis_hk_api: HousekeeperAPI,
rna_case_id: str,
rna_store: Store,
upload_scout_api: UploadScoutAPI,
):
"""Test that A RNA case's gene fusion report and junction splice files for all samples can be loaded via a cg CLI
command into an already existing DNA case"""
# GIVEN an existing RNA case with related sample
# GIVEN an existing DNA case with related sample
# GIVEN a sample in the RNA case is connected to a sample in the DNA case via subject_id (i.e. same subject_id)
upload_scout_api.status_db = rna_store
# GIVEN the connected RNA sample has a junction bed in Housekeeper
# GIVEN the connected RNA sample has a bigWig in Housekeeper
# WHEN running the method to upload RNA files to Scout
caplog.set_level(logging.INFO)
upload_scout_api.upload_rna_junctions_to_scout(case_id=rna_case_id,
dry_run=True)
# THEN the 2 files should have been uploaded to the connected sample on the dna case in scout
assert "Upload splice junctions bed file finished!" in caplog.text
assert "Upload RNA coverage bigwig file finished!" in caplog.text
def test_save_config_creates_file(upload_scout_api: UploadScoutAPI,
mip_load_config: ScoutLoadConfig, tmp_file):
""" "Tests that save config creates a file"""
# GIVEN a scout_config object and a path to save it on
# WHEN calling method to save config
upload_scout_api.save_config_file(upload_config=mip_load_config,
file_path=tmp_file)
# THEN the config should exist on disk
assert _file_exists_on_disk(tmp_file)
def test_save_config_creates_yaml(upload_scout_api: UploadScoutAPI,
mip_load_config: ScoutLoadConfig, tmp_file):
"""Tests that the file created by save_config_file create a yaml"""
# GIVEN a scout_config dict and a path to save it on
# WHEN calling method to save config
upload_scout_api.save_config_file(upload_config=mip_load_config,
file_path=tmp_file)
# THEN the should be of yaml type
assert _file_is_yaml(tmp_file)
def test_add_scout_config_to_hk_existing_files(
upload_scout_api: UploadScoutAPI, tmp_file):
"""Test that scout config is not updated in housekeeper if it already exists"""
# GIVEN a hk_mock with an scout upload file and a file path to scout load config
housekeeper_api = upload_scout_api.housekeeper
# GIVEN that there are files in the hk mock
housekeeper_api.new_file(tmp_file.name)
assert housekeeper_api.files
# WHEN adding the file path to hk_api
with pytest.raises(FileExistsError):
# THEN assert File exists exception is raised
upload_scout_api.add_scout_config_to_hk(config_file_path=tmp_file,
case_id="dummy")
def test_add_scout_config_to_hk(upload_scout_api: UploadScoutAPI, tmp_file):
"""Test that scout load config is added to housekeeper"""
# GIVEN a hk_mock and a file path to scout load config
tag_name = UploadScoutAPI.get_load_config_tag()
housekeeper_api = upload_scout_api.housekeeper
# GIVEN a hk mock that does not return a config file
housekeeper_api.add_missing_tag(tag_name)
# WHEN adding the file path to hk_api
upload_scout_api.add_scout_config_to_hk(config_file_path=tmp_file,
case_id="dummy")
# THEN assert that the file path is added to hk
# file added to hk-db
assert housekeeper_api.is_file_added() is True
assert housekeeper_api.is_file_included() is True
def test_generate_config_adds_case_paths(
sample_id: str,
mip_dna_analysis_obj: Store.Analysis,
upload_mip_analysis_scout_api: UploadScoutAPI,
):
"""Test that generate config adds case file paths"""
# GIVEN a status db and hk with an analysis
# WHEN generating the scout config for the analysis
result_data: ScoutLoadConfig = upload_mip_analysis_scout_api.generate_config(
mip_dna_analysis_obj)
# THEN the config should contain the multiqc file path
assert result_data.multiqc
def test_generate_balsamic_load_config(
balsamic_analysis_obj: models.Analysis,
upload_balsamic_analysis_scout_api: UploadScoutAPI):
# GIVEN a analysis object that have been run with balsamic
assert balsamic_analysis_obj.pipeline == Pipeline.BALSAMIC
# GIVEN a upload scout api with some balsamic information
# WHEN generating a load config
config = upload_balsamic_analysis_scout_api.generate_config(
analysis_obj=balsamic_analysis_obj)
# THEN assert that the config is a balsamic config
assert isinstance(config, BalsamicLoadConfig)
def test_generate_config_adds_meta_result_key(
result_key: str,
mip_dna_analysis_obj: models.Analysis,
upload_mip_analysis_scout_api: UploadScoutAPI,
):
"""Test that generate config adds the expected result keys"""
# GIVEN a status db and hk with an analysis
assert mip_dna_analysis_obj
# WHEN generating the scout config for the analysis
result_data: ScoutLoadConfig = upload_mip_analysis_scout_api.generate_config(
analysis_obj=mip_dna_analysis_obj)
# THEN the config should contain the rank model version used
assert result_data.dict()[result_key]
def test_generate_config_adds_sample_paths(
sample_id: str,
mip_dna_analysis_obj: models.Analysis,
upload_mip_analysis_scout_api: UploadScoutAPI,
):
"""Test that generate config adds vcf2cytosure file"""
# GIVEN a status db and hk with an analysis
# WHEN generating the scout config for the analysis
result_data: ScoutLoadConfig = upload_mip_analysis_scout_api.generate_config(
mip_dna_analysis_obj)
# THEN the config should contain the sample file path for each sample
sample: ScoutMipIndividual
for sample in result_data.samples:
if sample.sample_id == sample_id:
assert sample.vcf2cytosure
def fixture_scout_hk_bundle_data(case_id: str, scout_load_config: Path,
timestamp: datetime):
"""Get some bundle data for housekeeper"""
tag_name = UploadScoutAPI.get_load_config_tag()
return {
"name":
case_id,
"created":
timestamp,
"expires":
timestamp,
"files": [{
"path": str(scout_load_config),
"archive": False,
"tags": [tag_name],
}],
}
def fixture_upload_scout_api(
scout_api: MockScoutAPI,
madeline_api: MockMadelineAPI,
lims_samples: List[dict],
housekeeper_api: MockHousekeeperAPI,
store: Store,
) -> UploadScoutAPI:
"""Fixture for upload_scout_api"""
analysis_mock = MockMipAnalysis()
lims_api = MockLimsAPI(samples=lims_samples)
return UploadScoutAPI(
hk_api=housekeeper_api,
scout_api=scout_api,
madeline_api=madeline_api,
analysis_api=analysis_mock,
lims_api=lims_api,
status_db=store,
)
def upload_context(cg_context: CGConfig) -> CGConfig:
analysis_api = MipDNAAnalysisAPI(config=cg_context)
cg_context.meta_apis["analysis_api"] = analysis_api
cg_context.meta_apis["report_api"] = ReportAPI(
store=cg_context.status_db,
lims_api=cg_context.lims_api,
chanjo_api=cg_context.chanjo_api,
analysis_api=analysis_api,
scout_api=cg_context.scout_api,
)
cg_context.meta_apis["scout_upload_api"] = UploadScoutAPI(
hk_api=cg_context.housekeeper_api,
scout_api=cg_context.scout_api,
madeline_api=cg_context.madeline_api,
analysis_api=analysis_api,
lims_api=cg_context.lims_api,
status_db=cg_context.status_db,
)
return cg_context
def upload_case_to_scout(context: CGConfig, re_upload: bool, dry_run: bool, case_id: str):
"""Upload variants and case from analysis to Scout."""
LOG.info("----------------- UPLOAD -----------------------")
housekeeper_api: HousekeeperAPI = context.housekeeper_api
scout_api: ScoutAPI = context.scout_api
tag_name = UploadScoutAPI.get_load_config_tag()
version_obj = housekeeper_api.last_version(case_id)
scout_config_file: Optional[hk_models.File] = housekeeper_api.fetch_file_from_version(
version_obj=version_obj, tags={tag_name}
)
if scout_config_file is None:
raise FileNotFoundError(f"No scout load config was found in housekeeper for {case_id}")
LOG.info("uploading case %s to scout", case_id)
if not dry_run:
scout_api.upload(scout_load_config=scout_config_file.full_path, force=re_upload)
LOG.info("uploaded to scout using load config %s", scout_config_file.full_path)
LOG.info("Case loaded successfully to Scout")
def upload(context: click.Context, family_id: Optional[str],
force_restart: bool):
"""Upload results from analyses."""
config_object: CGConfig = context.obj
if not config_object.meta_apis.get("analysis_api"):
config_object.meta_apis["analysis_api"] = MipDNAAnalysisAPI(
context.obj)
analysis_api: AnalysisAPI = config_object.meta_apis["analysis_api"]
status_db: Store = config_object.status_db
click.echo(click.style("----------------- UPLOAD ----------------------"))
if family_id:
try:
analysis_api.verify_case_id_in_statusdb(case_id=family_id)
except CgError:
raise click.Abort
case_obj: models.Family = status_db.family(family_id)
if not case_obj.analyses:
message = f"no analysis exists for family: {family_id}"
click.echo(click.style(message, fg="red"))
raise click.Abort
analysis_obj: models.Analysis = case_obj.analyses[0]
if analysis_obj.uploaded_at is not None:
message = f"analysis already uploaded: {analysis_obj.uploaded_at.date()}"
click.echo(click.style(message, fg="red"))
raise click.Abort
if not force_restart and analysis_obj.upload_started_at is not None:
if dt.datetime.now(
) - analysis_obj.upload_started_at > dt.timedelta(hours=24):
raise AnalysisUploadError(
f"The upload started at {analysis_obj.upload_started_at} "
f"something went wrong, restart it with the --restart flag"
)
message = f"analysis upload already started: {analysis_obj.upload_started_at.date()}"
click.echo(click.style(message, fg="yellow"))
return
context.obj.meta_apis["report_api"] = ReportAPI(
store=status_db,
lims_api=config_object.lims_api,
chanjo_api=config_object.chanjo_api,
analysis_api=analysis_api,
scout_api=config_object.scout_api,
)
context.obj.meta_apis["scout_upload_api"] = UploadScoutAPI(
hk_api=config_object.housekeeper_api,
scout_api=config_object.scout_api,
madeline_api=config_object.madeline_api,
analysis_api=analysis_api,
lims_api=config_object.lims_api,
status_db=status_db,
)
if context.invoked_subcommand is not None:
return
if not family_id:
suggest_cases_to_upload(status_db=status_db)
raise click.Abort
case_obj: models.Family = status_db.family(family_id)
analysis_obj: models.Analysis = case_obj.analyses[0]
if analysis_obj.uploaded_at is not None:
message = f"analysis already uploaded: {analysis_obj.uploaded_at.date()}"
click.echo(click.style(message, fg="yellow"))
else:
analysis_obj.upload_started_at = dt.datetime.now()
status_db.commit()
context.invoke(coverage, re_upload=True, family_id=family_id)
context.invoke(validate, family_id=family_id)
context.invoke(genotypes, re_upload=False, family_id=family_id)
context.invoke(observations, case_id=family_id)
context.invoke(scout, case_id=family_id)
analysis_obj.uploaded_at = dt.datetime.now()
status_db.commit()
click.echo(click.style(f"{family_id}: analysis uploaded!", fg="green"))
