def test_cli_init(cli_runner: CliRunner, base_context: CGConfig, caplog):
caplog.set_level(logging.INFO)
# GIVEN you want to setup a new database using the CLI
database = "./test_db.sqlite3"
database_path = Path(database)
database_uri = f"sqlite:///{database}"
base_context.status_db_ = Store(uri=database_uri)
with cli_runner.isolated_filesystem():
assert database_path.exists() is False
# WHEN calling "init"
result = cli_runner.invoke(init, [], obj=base_context)
# THEN it should setup the database with some tables
assert result.exit_code == 0
assert database_path.exists()
assert len(Store(database_uri).engine.table_names()) > 0
# GIVEN the database already exists
# WHEN calling the init function
result = cli_runner.invoke(init, [], obj=base_context)
# THEN it should print an error and give error exit code
assert result.exit_code != 0
assert "Database already exists" in caplog.text
# GIVEN the database already exists
# WHEN calling "init" with "--reset"
result = cli_runner.invoke(init, ["--reset"],
input="Yes",
obj=base_context)
# THEN it should re-setup the tables and print new tables
assert result.exit_code == 0
assert "Success!" in caplog.text
def fetch_flowcell(context: click.Context, dry_run: bool, flowcell: str):
"""Fetch the first flowcell in the requested queue from backup."""
status_api = Store(context.obj["database"])
max_flowcells_on_disk = context.obj.get("max_flowcells",
MAX_FLOWCELLS_ON_DISK)
pdc_api = PdcApi()
backup_api = BackupApi(status=status_api,
pdc_api=pdc_api,
max_flowcells_on_disk=max_flowcells_on_disk)
if flowcell:
flowcell_obj = status_api.flowcell(flowcell)
if flowcell_obj is None:
LOG.error(f"{flowcell}: not found in database")
context.abort()
else:
flowcell_obj = None
retrieval_time = backup_api.fetch_flowcell(flowcell_obj=flowcell_obj,
dry_run=dry_run)
if retrieval_time:
hours = retrieval_time / 60 / 60
LOG.info(f"Retrieval time: {hours:.1}h")
return
if not flowcell:
return
if not dry_run:
LOG.info(f"{flowcell}: updating flowcell status to requested")
flowcell_obj.status = "requested"
status_api.commit()
def test_upload_splice_junctions_bed_to_scout_no_subject_id(
caplog: Generator[LogCaptureFixture, None, None],
dna_case_id: str,
mip_rna_analysis_hk_api: HousekeeperAPI,
rna_case_id: str,
rna_store: Store,
upload_scout_api: UploadScoutAPI,
):
"""Test that A RNA case's junction splice files for all samples can be loaded via a cg CLI
command into an already existing DNA case"""
# GIVEN a sample in the RNA case is NOT connected to a sample in the DNA case via subject_id (i.e. same subject_id)
for link in rna_store.family(rna_case_id).links:
link.sample.subject_id = ""
for link in rna_store.family(dna_case_id).links:
link.sample.subject_id = ""
rna_store.commit()
upload_scout_api.status_db = rna_store
# GIVEN the connected RNA sample has a junction bed in Housekeeper
# WHEN running the method to upload RNA files to Scout
caplog.set_level(logging.INFO)
# THEN an exception should be raised on unconnected data
with pytest.raises(CgDataError):
upload_scout_api.upload_splice_junctions_bed_to_scout(
case_id=rna_case_id, dry_run=True)
def add_application(
store: Store,
application_tag: str = "dummy_tag",
application_type: str = "wgs",
description: str = None,
is_archived: bool = False,
is_accredited: bool = False,
is_external: bool = False,
min_sequencing_depth: int = 30,
**kwargs,
) -> models.Application:
"""Utility function to add a application to a store"""
application = store.application(tag=application_tag)
if application:
return application
if not description:
description = "dummy_description"
application = store.add_application(
tag=application_tag,
category=application_type,
description=description,
is_archived=is_archived,
percent_kth=80,
percent_reads_guaranteed=75,
is_accredited=is_accredited,
limitations="A limitation",
is_external=is_external,
min_sequencing_depth=min_sequencing_depth,
**kwargs,
)
store.add_commit(application)
return application
def fixture_analysis_obj(
analysis_store_trio: Store, case_id: str, timestamp: datetime, helpers
) -> models.Analysis:
"""Return a analysis object with a trio"""
case_obj = analysis_store_trio.family(case_id)
helpers.add_analysis(store=analysis_store_trio, case=case_obj, started_at=timestamp)
return analysis_store_trio.family(case_id).analyses[0]
def test_upload_rna_fusion_report_to_scout_no_subject_id(
caplog: Generator[LogCaptureFixture, None, None],
dna_case_id: str,
mip_rna_analysis_hk_api: HousekeeperAPI,
rna_case_id: str,
rna_store: Store,
upload_scout_api: UploadScoutAPI,
):
"""Test that A RNA case's gene fusion report"""
# GIVEN a sample in the RNA case is NOT connected to a sample in the DNA case via subject_id (i.e. same subject_id)
for link in rna_store.family(rna_case_id).links:
link.sample.subject_id = ""
for link in rna_store.family(dna_case_id).links:
link.sample.subject_id = ""
rna_store.commit()
upload_scout_api.status_db = rna_store
# GIVEN the connected RNA case has a research fusion report in Housekeeper
# WHEN running the method to upload RNA files to Scout
caplog.set_level(logging.INFO)
# THEN an exception should be raised on unconnected data
with pytest.raises(CgDataError):
upload_scout_api.upload_fusion_report_to_scout(case_id=rna_case_id,
dry_run=True)
def add_application(
store: Store,
application_tag: str = "dummy_tag",
application_type: str = "wgs",
description: str = None,
is_accredited: bool = False,
is_external: bool = False,
**kwargs,
) -> models.Application:
"""Utility function to add a application to a store"""
application = store.application(tag=application_tag)
if application:
return application
if not description:
description = "dummy_description"
application = store.add_application(
tag=application_tag,
category=application_type,
description=description,
percent_kth=80,
is_accredited=is_accredited,
limitations="A limitation",
is_external=is_external,
)
store.add_commit(application)
return application
def add_family(
self,
store: Store,
family_id: str = "family_test",
customer_id: str = "cust000",
panels: List = None,
family_obj: models.Family = None,
) -> models.Family:
"""Utility function to add a family to use in tests,
If no family object is used a autogenerated family id will be used
"""
customer = self.ensure_customer(store, customer_id)
if family_obj:
panels = family_obj.panels
if not panels:
panels = ["panel_test"]
for panel_name in panels:
self.ensure_panel(store,
panel_id=panel_name,
customer_id=customer_id)
if not family_obj:
family_obj = store.add_family(name=family_id, panels=panels)
print("Adding family with name %s (%s)" %
(family_obj.name, family_obj.internal_id))
family_obj.customer = customer
store.add_commit(family_obj)
return family_obj
def ensure_application_version(
store: Store,
application_tag: str = "dummy_tag",
application_type: str = "wgs",
is_external: bool = False,
is_rna: bool = False,
description: str = None,
sequencing_depth: int = None,
is_accredited: bool = False,
) -> models.ApplicationVersion:
"""Utility function to return existing or create application version for tests"""
if is_rna:
application_tag = "rna_tag"
application_type = "wts"
application = store.application(tag=application_tag)
if not application:
application = StoreHelpers.add_application(
store,
application_tag,
application_type,
is_external=is_external,
description=description,
is_accredited=is_accredited,
sequencing_depth=sequencing_depth,
)
prices = {"standard": 10, "priority": 20, "express": 30, "research": 5}
version = store.application_version(application, 1)
if not version:
version = store.add_version(application, 1, valid_from=datetime.now(), prices=prices)
store.add_commit(version)
return version
def test_store_items_in_status_control_has_stored_value(
sarscov2_order_to_submit: dict, base_store: Store):
# GIVEN sarscov2 order with three samples with control value
order: OrderIn = OrderIn.parse_obj(sarscov2_order_to_submit,
OrderType.SARS_COV_2)
control_value = ControlEnum.positive
sample: SarsCov2Sample
for sample in order.samples:
sample.control: ControlEnum = control_value
submitter: SarsCov2Submitter = SarsCov2Submitter(status=base_store,
lims=None)
status_data = submitter.order_to_status(order=order)
# WHEN storing the order
submitter.store_items_in_status(
comment="",
customer=order.customer,
data_analysis=Pipeline.SARS_COV_2,
data_delivery=DataDelivery.FASTQ,
order="",
ordered=dt.datetime.now(),
ticket=123456,
items=status_data.get("samples"),
)
# THEN control should exist on the sample in the store
customer = base_store.customer(order.customer)
sample: SarsCov2Sample
for sample in order.samples:
stored_sample: models.Sample = base_store.find_samples(
customer=customer, name=sample.name).first()
assert stored_sample.control == control_value
def add_sample(
self,
store: Store,
sample_id: str = "sample_test",
gender: str = "female",
is_tumour: bool = False,
is_rna: bool = False,
is_external: bool = False,
data_analysis: str = "balsamic",
application_tag: str = "dummy_tag",
application_type: str = "tgs",
customer_name: str = None,
reads: int = None,
**kwargs,
) -> models.Sample:
"""Utility function to add a sample to use in tests"""
customer_name = customer_name or "cust000"
customer = self.ensure_customer(store, customer_name)
application_version = self.ensure_application_version(
store,
application_tag=application_tag,
application_type=application_type,
is_external=is_external,
is_rna=is_rna,
)
print(repr(application_version))
application_version_id = application_version.id
sample = store.add_sample(
name=sample_id,
sex=gender,
tumour=is_tumour,
sequenced_at=datetime.now(),
data_analysis=data_analysis,
reads=reads,
)
sample.application_version_id = application_version_id
sample.customer = customer
print("Set is external to %s", is_external)
sample.is_external = is_external
if kwargs.get("delivered_at"):
print("Adding delivered")
sample.delivered_at = kwargs["delivered_at"]
if kwargs.get("received_at"):
print("Adding received_at")
sample.received_at = kwargs["received_at"]
if kwargs.get("prepared_at"):
print("Adding prepared")
sample.received_at = kwargs["prepared_at"]
if kwargs.get("flowcell"):
print("Adding flowcell")
sample.flowcells.append(kwargs["flowcell"])
store.add_commit(sample)
return sample
def test_get_case_files_from_version(
analysis_store: Store,
case_id: str,
real_housekeeper_api: HousekeeperAPI,
case_hk_bundle_no_files: dict,
bed_file: str,
vcf_file: Path,
project_dir: Path,
helpers=StoreHelpers,
):
# GIVEN a store with a case
case_obj = analysis_store.family(case_id)
assert case_obj.internal_id == case_id
# GIVEN a delivery api
deliver_api = DeliverAPI(
store=analysis_store,
hk_api=real_housekeeper_api,
case_tags=[{"case-tag"}],
sample_tags=[{"sample-tag"}],
project_base_path=project_dir,
delivery_type="balsamic",
)
# GIVEN a housekeeper db populated with a bundle including a case specific file and a sample specific file
case_hk_bundle_no_files["files"] = [
{
"path": bed_file,
"archive": False,
"tags": ["case-tag"]
},
{
"path": str(vcf_file),
"archive": False,
"tags": ["sample-tag", "ADM1"]
},
]
helpers.ensure_hk_bundle(real_housekeeper_api,
bundle_data=case_hk_bundle_no_files)
# GIVEN a version object where two file exists
version_obj: hk_models.Version = real_housekeeper_api.last_version(case_id)
assert len(version_obj.files) == 2
# GIVEN the sample ids of the samples
link_objs: List[FamilySample] = analysis_store.family_samples(case_id)
samples: List[Sample] = [link.sample for link in link_objs]
sample_ids: Set[str] = set([sample.internal_id for sample in samples])
# WHEN fetching the case files
case_files = deliver_api.get_case_files_from_version(
version_obj=version_obj, sample_ids=sample_ids)
# THEN we should only get the case specific files back
nr_files: int = 0
case_file: Path
for nr_files, case_file in enumerate(case_files, 1):
assert case_file.name == Path(bed_file).name
# THEN assert that only the case-tag file was returned
assert nr_files == 1
def update_delivery_report_date(
status_api: Store, case_id: str, analysis_date: datetime
) -> None:
"""Update date on analysis when delivery report was created"""
case_obj = status_api.family(case_id)
analysis_obj = status_api.analysis(case_obj, analysis_date)
analysis_obj.delivery_report_created_at = datetime.now()
status_api.commit()
def get_links(store: Store,
case_id: str = None,
sample_id: str = None) -> List[models.FamilySample]:
"""Get link objects for a SAMPLE_ID
Args:
case_id(str): petname
sample_id(str): ACC6395A2
Returns:
link_objs(list): [models.FamilySample]
"""
link_objs = []
if case_id and sample_id:
LOG.info("Link only one sample in a case")
link_obj = store.link(family_id=case_id, sample_id=sample_id)
if not link_obj:
LOG.error("Could not find link for case %s and sample %s", case_id,
sample_id)
raise click.Abort
link_objs = [link_obj]
elif case_id:
LOG.info("Link all samples in a case")
case_obj = store.family(case_id)
if not case_obj:
LOG.error("Could not find case %s", case_id)
raise click.Abort
if not case_obj.links:
LOG.error("Could not find links for case %s", case_id)
raise click.Abort
link_objs = case_obj.links
elif sample_id:
LOG.info("Link sample %s in all its families", sample_id)
sample_obj = store.sample(sample_id)
if not sample_obj:
LOG.error(
"Could not find sample %s. Did you intend %s as a case-id?",
sample_id,
sample_id,
)
raise click.Abort
if not sample_obj.links:
LOG.error("Could not find links for sample %s", sample_id)
raise click.Abort
link_objs = sample_obj.links
else:
LOG.error("Please provide case and/or sample")
raise click.Abort
return link_objs
def test_add_customer_group(store: Store):
# GIVEN an empty database
assert store.CustomerGroup.query.first() is None
internal_id, name = "cust_group", "Test customer group"
# WHEN adding a new customer group
new_customer_group = store.add_customer_group(internal_id=internal_id, name=name)
store.add_commit(new_customer_group)
# THEN it should be stored in the database
assert store.CustomerGroup.query.first() == new_customer_group
def add_subject_id_to_sample(
store: Store, sample_id: str, subject_id: str = "a subject_id"
) -> Optional[models.Sample]:
"""Function for adding a subject_id to a sample in the database"""
sample_obj: models.Sample = store.sample(internal_id=sample_id)
if not sample_obj:
LOG.warning("Could not find sample")
return None
sample_obj.subject_id = subject_id
store.commit()
return sample_obj
def add_phenotype_terms_to_sample(
store: Store, sample_id: str, phenotype_terms: [str] = ["a phenotype term"]
) -> Optional[models.Sample]:
"""Function for adding a phenotype term to a sample in the database"""
sample_obj: models.Sample = store.sample(internal_id=sample_id)
if not sample_obj:
LOG.warning("Could not find sample")
return None
sample_obj.phenotype_terms = phenotype_terms
store.commit()
return sample_obj
def add_synopsis_to_case(
store: Store, case_id: str, synopsis: str = "a synopsis"
) -> Optional[models.Family]:
"""Function for adding a synopsis to a case in the database"""
case_obj: models.Family = store.family(internal_id=case_id)
if not case_obj:
LOG.warning("Could not find case")
return None
case_obj.synopsis = synopsis
store.commit()
return case_obj
def ensure_organism(
store: Store,
organism_id: str = "organism_test",
name: str = "organism_name",
reference_genome: str = "reference_genome_test",
) -> models.Organism:
"""Utility function to add an organism to use in tests"""
organism = StoreHelpers.add_organism(
store, internal_id=organism_id, name=name, reference_genome=reference_genome
)
store.add_commit(organism)
return organism
def store_samples_with_names_from_order(store: Store, helpers: StoreHelpers,
order_data: OrderIn):
customer_obj = store.customer(order_data.customer)
for sample in order_data.samples:
sample_name = sample.name
if not store.find_samples(customer=customer_obj,
name=sample_name).first():
sample_obj = helpers.add_sample(
store=store,
name=sample_name,
customer_id=customer_obj.internal_id)
store.add_commit(sample_obj)
def test_reset_observation(store: Store):
# GIVEN a store with a case with loqus-links
family = add_family(store)
sample = add_sample(store, loqusdb_links=True)
store.relate_sample(family=family, sample=sample, status="unknown")
assert sample.loqusdb_id is not None
# WHEN calling reset observations
store.reset_observations(case_id=family.internal_id)
# THEN the links to observations in loqusdb should have been reset
assert sample.loqusdb_id is None
def test_new_external_case_not_in_result(base_store: Store, helpers):
"""Test that a case with one sample that has specified data_analysis does show up"""
# GIVEN a database with a case with one sequenced samples for MIP analysis
pipeline = Pipeline.BALSAMIC
test_sample = helpers.add_sample(base_store, sequenced_at=None, is_external=True)
test_case = helpers.add_case(base_store, data_analysis=pipeline)
base_store.relate_sample(test_case, test_sample, "unknown")
# WHEN getting cases to analyse
cases = base_store.cases_to_analyze(pipeline=pipeline)
# THEN cases should contain the test case
assert test_case not in cases
def test_exclude_other_pipeline_analysis_from_result(base_store: Store, helpers):
"""Test that a case with specified analysis and with one sample does not show up among
others"""
# GIVEN a database with a case with one sequenced samples for specified analysis
test_sample = helpers.add_sample(base_store, sequenced_at=datetime.now())
test_case = helpers.add_case(base_store, data_analysis=Pipeline.BALSAMIC)
base_store.relate_sample(test_case, test_sample, "unknown")
# WHEN getting cases to analyse
cases = base_store.cases_to_analyze(pipeline=Pipeline.MIP_DNA)
# THEN cases should not contain the test case
assert not cases
def test_mip_analysis_in_result(base_store: Store):
"""Test that a family with one sample that has MIP data_analysis does show up"""
# GIVEN a database with a family with one sequenced samples for MIP analysis
test_sample = add_sample(base_store, sequenced=True, data_analysis="MIP")
test_family = add_family(base_store)
base_store.relate_sample(test_family, test_sample, "unknown")
# WHEN getting families to analyse
families = base_store.cases_to_mip_analyze()
# THEN families should contain the test family
assert families
assert test_family in families
def add_sample(
store: Store,
application_tag: str = "dummy_tag",
application_type: str = "tgs",
control: str = "",
customer_id: str = None,
gender: str = "female",
is_external: bool = False,
is_rna: bool = False,
is_tumour: bool = False,
reads: int = None,
name: str = "sample_test",
ticket: int = None,
**kwargs,
) -> models.Sample:
"""Utility function to add a sample to use in tests"""
customer_id = customer_id or "cust000"
customer = StoreHelpers.ensure_customer(store, customer_id=customer_id)
application_version = StoreHelpers.ensure_application_version(
store=store,
application_tag=application_tag,
application_type=application_type,
is_external=is_external,
is_rna=is_rna,
)
application_version_id = application_version.id
sample = store.add_sample(
control=control,
name=name,
reads=reads,
sex=gender,
ticket=ticket,
tumour=is_tumour,
)
sample.application_version_id = application_version_id
sample.customer = customer
sample.ordered_at = datetime.now()
for key, value in kwargs.items():
if key == "flowcell":
sample.flowcells.append(kwargs["flowcell"])
elif hasattr(sample, key):
setattr(sample, key, value)
else:
raise AttributeError(f"Unknown sample attribute/feature: {key}, {value}")
store.add_commit(sample)
return sample
def test_exclude_balsamic_only_analysis_from_result(base_store: Store):
"""Test that a family with one sample that is a Balsamic only does not show up"""
# GIVEN a database with a family with one sequenced samples for Balsamic analysis
test_sample = add_sample(base_store,
sequenced=True,
data_analysis="Balsamic")
test_family = add_family(base_store)
base_store.relate_sample(test_family, test_sample, "unknown")
# WHEN getting families to analyse
families = base_store.cases_to_mip_analyze()
# THEN families should not contain the test family
assert not families
def test_family_to_re_analyse(base_store: Store):
"""Test that a family marked for re-analyse with one sample that has been sequenced and
with completed analysis do show up among the families to analyse"""
# GIVEN a database with a family with one of one sequenced samples and completed analysis
test_sample = add_sample(base_store, sequenced=True)
test_analysis = add_analysis(base_store, completed=True, reanalyse=True)
base_store.relate_sample(test_analysis.family, test_sample, "unknown")
# WHEN getting families to analyse
families = base_store.cases_to_mip_analyze()
# THEN families should contain the test family
assert families
assert test_analysis.family in families
def test_all_samples_and_analysis_completed(base_store: Store):
"""Test that a family with one sample that has been sequenced and with completed
analysis don't show up among the families to analyse"""
# GIVEN a database with a family with one of one sequenced samples and completed analysis
test_sample = add_sample(base_store, sequenced=True)
test_analysis = add_analysis(base_store, completed=True)
base_store.relate_sample(test_analysis.family, test_sample, "unknown")
# WHEN getting families to analyse
families = base_store.cases_to_mip_analyze()
# THEN families should not contain the test family
assert not families
def test_versions_are_not_same(applications_store: Store, application_versions_file: str):
# GIVEN a database with some applications loaded
# GIVEN an excel price row
# NOT same price row committed to the database
excel_versions: List[ApplicationVersionSchema] = list(
parse_application_versions(excel_path=application_versions_file)
)
version: ApplicationVersionSchema = excel_versions[0]
application_obj: models.Application = applications_store.application(version.app_tag)
sign = "DummySign"
db_version: models.ApplicationVersion = add_application_version(
application_obj=application_obj,
latest_version=None,
version=version,
sign=sign,
store=applications_store,
)
another_version: ApplicationVersionSchema = excel_versions[1]
# WHEN calling versions are same
should_not_be_same: bool = versions_are_same(
version_obj=db_version, application_version=another_version
)
# THEN versions are not considered same
assert should_not_be_same is False
def test_missing(analysis_store: Store, helpers):
"""Tests that analyses that are completed but lacks delivery report are returned"""
# GIVEN an analysis that is delivered but has no delivery report
timestamp = datetime.now()
analysis = helpers.add_analysis(analysis_store, started_at=timestamp, uploaded_at=timestamp)
sample = helpers.add_sample(analysis_store, delivered_at=timestamp)
analysis_store.relate_sample(family=analysis.family, sample=sample, status="unknown")
assert sample.delivered_at is not None
assert analysis.delivery_report_created_at is None
# WHEN calling the analyses_to_delivery_report
analyses = analysis_store.analyses_to_delivery_report().all()
# THEN this analyse should be returned
assert analysis in analyses
