def test_cli_init(cli_runner: CliRunner, base_context: CGConfig, caplog): caplog.set_level(logging.INFO) # GIVEN you want to setup a new database using the CLI database = "./test_db.sqlite3" database_path = Path(database) database_uri = f"sqlite:///{database}" base_context.status_db_ = Store(uri=database_uri) with cli_runner.isolated_filesystem(): assert database_path.exists() is False # WHEN calling "init" result = cli_runner.invoke(init, [], obj=base_context) # THEN it should setup the database with some tables assert result.exit_code == 0 assert database_path.exists() assert len(Store(database_uri).engine.table_names()) > 0 # GIVEN the database already exists # WHEN calling the init function result = cli_runner.invoke(init, [], obj=base_context) # THEN it should print an error and give error exit code assert result.exit_code != 0 assert "Database already exists" in caplog.text # GIVEN the database already exists # WHEN calling "init" with "--reset" result = cli_runner.invoke(init, ["--reset"], input="Yes", obj=base_context) # THEN it should re-setup the tables and print new tables assert result.exit_code == 0 assert "Success!" in caplog.text
def fetch_flowcell(context: click.Context, dry_run: bool, flowcell: str): """Fetch the first flowcell in the requested queue from backup.""" status_api = Store(context.obj["database"]) max_flowcells_on_disk = context.obj.get("max_flowcells", MAX_FLOWCELLS_ON_DISK) pdc_api = PdcApi() backup_api = BackupApi(status=status_api, pdc_api=pdc_api, max_flowcells_on_disk=max_flowcells_on_disk) if flowcell: flowcell_obj = status_api.flowcell(flowcell) if flowcell_obj is None: LOG.error(f"{flowcell}: not found in database") context.abort() else: flowcell_obj = None retrieval_time = backup_api.fetch_flowcell(flowcell_obj=flowcell_obj, dry_run=dry_run) if retrieval_time: hours = retrieval_time / 60 / 60 LOG.info(f"Retrieval time: {hours:.1}h") return if not flowcell: return if not dry_run: LOG.info(f"{flowcell}: updating flowcell status to requested") flowcell_obj.status = "requested" status_api.commit()
def test_upload_splice_junctions_bed_to_scout_no_subject_id( caplog: Generator[LogCaptureFixture, None, None], dna_case_id: str, mip_rna_analysis_hk_api: HousekeeperAPI, rna_case_id: str, rna_store: Store, upload_scout_api: UploadScoutAPI, ): """Test that A RNA case's junction splice files for all samples can be loaded via a cg CLI command into an already existing DNA case""" # GIVEN a sample in the RNA case is NOT connected to a sample in the DNA case via subject_id (i.e. same subject_id) for link in rna_store.family(rna_case_id).links: link.sample.subject_id = "" for link in rna_store.family(dna_case_id).links: link.sample.subject_id = "" rna_store.commit() upload_scout_api.status_db = rna_store # GIVEN the connected RNA sample has a junction bed in Housekeeper # WHEN running the method to upload RNA files to Scout caplog.set_level(logging.INFO) # THEN an exception should be raised on unconnected data with pytest.raises(CgDataError): upload_scout_api.upload_splice_junctions_bed_to_scout( case_id=rna_case_id, dry_run=True)
def add_application( store: Store, application_tag: str = "dummy_tag", application_type: str = "wgs", description: str = None, is_archived: bool = False, is_accredited: bool = False, is_external: bool = False, min_sequencing_depth: int = 30, **kwargs, ) -> models.Application: """Utility function to add a application to a store""" application = store.application(tag=application_tag) if application: return application if not description: description = "dummy_description" application = store.add_application( tag=application_tag, category=application_type, description=description, is_archived=is_archived, percent_kth=80, percent_reads_guaranteed=75, is_accredited=is_accredited, limitations="A limitation", is_external=is_external, min_sequencing_depth=min_sequencing_depth, **kwargs, ) store.add_commit(application) return application
def fixture_analysis_obj( analysis_store_trio: Store, case_id: str, timestamp: datetime, helpers ) -> models.Analysis: """Return a analysis object with a trio""" case_obj = analysis_store_trio.family(case_id) helpers.add_analysis(store=analysis_store_trio, case=case_obj, started_at=timestamp) return analysis_store_trio.family(case_id).analyses[0]
def test_upload_rna_fusion_report_to_scout_no_subject_id( caplog: Generator[LogCaptureFixture, None, None], dna_case_id: str, mip_rna_analysis_hk_api: HousekeeperAPI, rna_case_id: str, rna_store: Store, upload_scout_api: UploadScoutAPI, ): """Test that A RNA case's gene fusion report""" # GIVEN a sample in the RNA case is NOT connected to a sample in the DNA case via subject_id (i.e. same subject_id) for link in rna_store.family(rna_case_id).links: link.sample.subject_id = "" for link in rna_store.family(dna_case_id).links: link.sample.subject_id = "" rna_store.commit() upload_scout_api.status_db = rna_store # GIVEN the connected RNA case has a research fusion report in Housekeeper # WHEN running the method to upload RNA files to Scout caplog.set_level(logging.INFO) # THEN an exception should be raised on unconnected data with pytest.raises(CgDataError): upload_scout_api.upload_fusion_report_to_scout(case_id=rna_case_id, dry_run=True)
def add_application( store: Store, application_tag: str = "dummy_tag", application_type: str = "wgs", description: str = None, is_accredited: bool = False, is_external: bool = False, **kwargs, ) -> models.Application: """Utility function to add a application to a store""" application = store.application(tag=application_tag) if application: return application if not description: description = "dummy_description" application = store.add_application( tag=application_tag, category=application_type, description=description, percent_kth=80, is_accredited=is_accredited, limitations="A limitation", is_external=is_external, ) store.add_commit(application) return application
def add_family( self, store: Store, family_id: str = "family_test", customer_id: str = "cust000", panels: List = None, family_obj: models.Family = None, ) -> models.Family: """Utility function to add a family to use in tests, If no family object is used a autogenerated family id will be used """ customer = self.ensure_customer(store, customer_id) if family_obj: panels = family_obj.panels if not panels: panels = ["panel_test"] for panel_name in panels: self.ensure_panel(store, panel_id=panel_name, customer_id=customer_id) if not family_obj: family_obj = store.add_family(name=family_id, panels=panels) print("Adding family with name %s (%s)" % (family_obj.name, family_obj.internal_id)) family_obj.customer = customer store.add_commit(family_obj) return family_obj
def ensure_application_version( store: Store, application_tag: str = "dummy_tag", application_type: str = "wgs", is_external: bool = False, is_rna: bool = False, description: str = None, sequencing_depth: int = None, is_accredited: bool = False, ) -> models.ApplicationVersion: """Utility function to return existing or create application version for tests""" if is_rna: application_tag = "rna_tag" application_type = "wts" application = store.application(tag=application_tag) if not application: application = StoreHelpers.add_application( store, application_tag, application_type, is_external=is_external, description=description, is_accredited=is_accredited, sequencing_depth=sequencing_depth, ) prices = {"standard": 10, "priority": 20, "express": 30, "research": 5} version = store.application_version(application, 1) if not version: version = store.add_version(application, 1, valid_from=datetime.now(), prices=prices) store.add_commit(version) return version
def test_store_items_in_status_control_has_stored_value( sarscov2_order_to_submit: dict, base_store: Store): # GIVEN sarscov2 order with three samples with control value order: OrderIn = OrderIn.parse_obj(sarscov2_order_to_submit, OrderType.SARS_COV_2) control_value = ControlEnum.positive sample: SarsCov2Sample for sample in order.samples: sample.control: ControlEnum = control_value submitter: SarsCov2Submitter = SarsCov2Submitter(status=base_store, lims=None) status_data = submitter.order_to_status(order=order) # WHEN storing the order submitter.store_items_in_status( comment="", customer=order.customer, data_analysis=Pipeline.SARS_COV_2, data_delivery=DataDelivery.FASTQ, order="", ordered=dt.datetime.now(), ticket=123456, items=status_data.get("samples"), ) # THEN control should exist on the sample in the store customer = base_store.customer(order.customer) sample: SarsCov2Sample for sample in order.samples: stored_sample: models.Sample = base_store.find_samples( customer=customer, name=sample.name).first() assert stored_sample.control == control_value
def add_sample( self, store: Store, sample_id: str = "sample_test", gender: str = "female", is_tumour: bool = False, is_rna: bool = False, is_external: bool = False, data_analysis: str = "balsamic", application_tag: str = "dummy_tag", application_type: str = "tgs", customer_name: str = None, reads: int = None, **kwargs, ) -> models.Sample: """Utility function to add a sample to use in tests""" customer_name = customer_name or "cust000" customer = self.ensure_customer(store, customer_name) application_version = self.ensure_application_version( store, application_tag=application_tag, application_type=application_type, is_external=is_external, is_rna=is_rna, ) print(repr(application_version)) application_version_id = application_version.id sample = store.add_sample( name=sample_id, sex=gender, tumour=is_tumour, sequenced_at=datetime.now(), data_analysis=data_analysis, reads=reads, ) sample.application_version_id = application_version_id sample.customer = customer print("Set is external to %s", is_external) sample.is_external = is_external if kwargs.get("delivered_at"): print("Adding delivered") sample.delivered_at = kwargs["delivered_at"] if kwargs.get("received_at"): print("Adding received_at") sample.received_at = kwargs["received_at"] if kwargs.get("prepared_at"): print("Adding prepared") sample.received_at = kwargs["prepared_at"] if kwargs.get("flowcell"): print("Adding flowcell") sample.flowcells.append(kwargs["flowcell"]) store.add_commit(sample) return sample
def test_get_case_files_from_version( analysis_store: Store, case_id: str, real_housekeeper_api: HousekeeperAPI, case_hk_bundle_no_files: dict, bed_file: str, vcf_file: Path, project_dir: Path, helpers=StoreHelpers, ): # GIVEN a store with a case case_obj = analysis_store.family(case_id) assert case_obj.internal_id == case_id # GIVEN a delivery api deliver_api = DeliverAPI( store=analysis_store, hk_api=real_housekeeper_api, case_tags=[{"case-tag"}], sample_tags=[{"sample-tag"}], project_base_path=project_dir, delivery_type="balsamic", ) # GIVEN a housekeeper db populated with a bundle including a case specific file and a sample specific file case_hk_bundle_no_files["files"] = [ { "path": bed_file, "archive": False, "tags": ["case-tag"] }, { "path": str(vcf_file), "archive": False, "tags": ["sample-tag", "ADM1"] }, ] helpers.ensure_hk_bundle(real_housekeeper_api, bundle_data=case_hk_bundle_no_files) # GIVEN a version object where two file exists version_obj: hk_models.Version = real_housekeeper_api.last_version(case_id) assert len(version_obj.files) == 2 # GIVEN the sample ids of the samples link_objs: List[FamilySample] = analysis_store.family_samples(case_id) samples: List[Sample] = [link.sample for link in link_objs] sample_ids: Set[str] = set([sample.internal_id for sample in samples]) # WHEN fetching the case files case_files = deliver_api.get_case_files_from_version( version_obj=version_obj, sample_ids=sample_ids) # THEN we should only get the case specific files back nr_files: int = 0 case_file: Path for nr_files, case_file in enumerate(case_files, 1): assert case_file.name == Path(bed_file).name # THEN assert that only the case-tag file was returned assert nr_files == 1
def update_delivery_report_date( status_api: Store, case_id: str, analysis_date: datetime ) -> None: """Update date on analysis when delivery report was created""" case_obj = status_api.family(case_id) analysis_obj = status_api.analysis(case_obj, analysis_date) analysis_obj.delivery_report_created_at = datetime.now() status_api.commit()
def get_links(store: Store, case_id: str = None, sample_id: str = None) -> List[models.FamilySample]: """Get link objects for a SAMPLE_ID Args: case_id(str): petname sample_id(str): ACC6395A2 Returns: link_objs(list): [models.FamilySample] """ link_objs = [] if case_id and sample_id: LOG.info("Link only one sample in a case") link_obj = store.link(family_id=case_id, sample_id=sample_id) if not link_obj: LOG.error("Could not find link for case %s and sample %s", case_id, sample_id) raise click.Abort link_objs = [link_obj] elif case_id: LOG.info("Link all samples in a case") case_obj = store.family(case_id) if not case_obj: LOG.error("Could not find case %s", case_id) raise click.Abort if not case_obj.links: LOG.error("Could not find links for case %s", case_id) raise click.Abort link_objs = case_obj.links elif sample_id: LOG.info("Link sample %s in all its families", sample_id) sample_obj = store.sample(sample_id) if not sample_obj: LOG.error( "Could not find sample %s. Did you intend %s as a case-id?", sample_id, sample_id, ) raise click.Abort if not sample_obj.links: LOG.error("Could not find links for sample %s", sample_id) raise click.Abort link_objs = sample_obj.links else: LOG.error("Please provide case and/or sample") raise click.Abort return link_objs
def test_add_customer_group(store: Store): # GIVEN an empty database assert store.CustomerGroup.query.first() is None internal_id, name = "cust_group", "Test customer group" # WHEN adding a new customer group new_customer_group = store.add_customer_group(internal_id=internal_id, name=name) store.add_commit(new_customer_group) # THEN it should be stored in the database assert store.CustomerGroup.query.first() == new_customer_group
def add_subject_id_to_sample( store: Store, sample_id: str, subject_id: str = "a subject_id" ) -> Optional[models.Sample]: """Function for adding a subject_id to a sample in the database""" sample_obj: models.Sample = store.sample(internal_id=sample_id) if not sample_obj: LOG.warning("Could not find sample") return None sample_obj.subject_id = subject_id store.commit() return sample_obj
def add_phenotype_terms_to_sample( store: Store, sample_id: str, phenotype_terms: [str] = ["a phenotype term"] ) -> Optional[models.Sample]: """Function for adding a phenotype term to a sample in the database""" sample_obj: models.Sample = store.sample(internal_id=sample_id) if not sample_obj: LOG.warning("Could not find sample") return None sample_obj.phenotype_terms = phenotype_terms store.commit() return sample_obj
def add_synopsis_to_case( store: Store, case_id: str, synopsis: str = "a synopsis" ) -> Optional[models.Family]: """Function for adding a synopsis to a case in the database""" case_obj: models.Family = store.family(internal_id=case_id) if not case_obj: LOG.warning("Could not find case") return None case_obj.synopsis = synopsis store.commit() return case_obj
def ensure_organism( store: Store, organism_id: str = "organism_test", name: str = "organism_name", reference_genome: str = "reference_genome_test", ) -> models.Organism: """Utility function to add an organism to use in tests""" organism = StoreHelpers.add_organism( store, internal_id=organism_id, name=name, reference_genome=reference_genome ) store.add_commit(organism) return organism
def store_samples_with_names_from_order(store: Store, helpers: StoreHelpers, order_data: OrderIn): customer_obj = store.customer(order_data.customer) for sample in order_data.samples: sample_name = sample.name if not store.find_samples(customer=customer_obj, name=sample_name).first(): sample_obj = helpers.add_sample( store=store, name=sample_name, customer_id=customer_obj.internal_id) store.add_commit(sample_obj)
def test_reset_observation(store: Store): # GIVEN a store with a case with loqus-links family = add_family(store) sample = add_sample(store, loqusdb_links=True) store.relate_sample(family=family, sample=sample, status="unknown") assert sample.loqusdb_id is not None # WHEN calling reset observations store.reset_observations(case_id=family.internal_id) # THEN the links to observations in loqusdb should have been reset assert sample.loqusdb_id is None
def test_new_external_case_not_in_result(base_store: Store, helpers): """Test that a case with one sample that has specified data_analysis does show up""" # GIVEN a database with a case with one sequenced samples for MIP analysis pipeline = Pipeline.BALSAMIC test_sample = helpers.add_sample(base_store, sequenced_at=None, is_external=True) test_case = helpers.add_case(base_store, data_analysis=pipeline) base_store.relate_sample(test_case, test_sample, "unknown") # WHEN getting cases to analyse cases = base_store.cases_to_analyze(pipeline=pipeline) # THEN cases should contain the test case assert test_case not in cases
def test_exclude_other_pipeline_analysis_from_result(base_store: Store, helpers): """Test that a case with specified analysis and with one sample does not show up among others""" # GIVEN a database with a case with one sequenced samples for specified analysis test_sample = helpers.add_sample(base_store, sequenced_at=datetime.now()) test_case = helpers.add_case(base_store, data_analysis=Pipeline.BALSAMIC) base_store.relate_sample(test_case, test_sample, "unknown") # WHEN getting cases to analyse cases = base_store.cases_to_analyze(pipeline=Pipeline.MIP_DNA) # THEN cases should not contain the test case assert not cases
def test_mip_analysis_in_result(base_store: Store): """Test that a family with one sample that has MIP data_analysis does show up""" # GIVEN a database with a family with one sequenced samples for MIP analysis test_sample = add_sample(base_store, sequenced=True, data_analysis="MIP") test_family = add_family(base_store) base_store.relate_sample(test_family, test_sample, "unknown") # WHEN getting families to analyse families = base_store.cases_to_mip_analyze() # THEN families should contain the test family assert families assert test_family in families
def add_sample( store: Store, application_tag: str = "dummy_tag", application_type: str = "tgs", control: str = "", customer_id: str = None, gender: str = "female", is_external: bool = False, is_rna: bool = False, is_tumour: bool = False, reads: int = None, name: str = "sample_test", ticket: int = None, **kwargs, ) -> models.Sample: """Utility function to add a sample to use in tests""" customer_id = customer_id or "cust000" customer = StoreHelpers.ensure_customer(store, customer_id=customer_id) application_version = StoreHelpers.ensure_application_version( store=store, application_tag=application_tag, application_type=application_type, is_external=is_external, is_rna=is_rna, ) application_version_id = application_version.id sample = store.add_sample( control=control, name=name, reads=reads, sex=gender, ticket=ticket, tumour=is_tumour, ) sample.application_version_id = application_version_id sample.customer = customer sample.ordered_at = datetime.now() for key, value in kwargs.items(): if key == "flowcell": sample.flowcells.append(kwargs["flowcell"]) elif hasattr(sample, key): setattr(sample, key, value) else: raise AttributeError(f"Unknown sample attribute/feature: {key}, {value}") store.add_commit(sample) return sample
def test_exclude_balsamic_only_analysis_from_result(base_store: Store): """Test that a family with one sample that is a Balsamic only does not show up""" # GIVEN a database with a family with one sequenced samples for Balsamic analysis test_sample = add_sample(base_store, sequenced=True, data_analysis="Balsamic") test_family = add_family(base_store) base_store.relate_sample(test_family, test_sample, "unknown") # WHEN getting families to analyse families = base_store.cases_to_mip_analyze() # THEN families should not contain the test family assert not families
def test_family_to_re_analyse(base_store: Store): """Test that a family marked for re-analyse with one sample that has been sequenced and with completed analysis do show up among the families to analyse""" # GIVEN a database with a family with one of one sequenced samples and completed analysis test_sample = add_sample(base_store, sequenced=True) test_analysis = add_analysis(base_store, completed=True, reanalyse=True) base_store.relate_sample(test_analysis.family, test_sample, "unknown") # WHEN getting families to analyse families = base_store.cases_to_mip_analyze() # THEN families should contain the test family assert families assert test_analysis.family in families
def test_all_samples_and_analysis_completed(base_store: Store): """Test that a family with one sample that has been sequenced and with completed analysis don't show up among the families to analyse""" # GIVEN a database with a family with one of one sequenced samples and completed analysis test_sample = add_sample(base_store, sequenced=True) test_analysis = add_analysis(base_store, completed=True) base_store.relate_sample(test_analysis.family, test_sample, "unknown") # WHEN getting families to analyse families = base_store.cases_to_mip_analyze() # THEN families should not contain the test family assert not families
def test_versions_are_not_same(applications_store: Store, application_versions_file: str): # GIVEN a database with some applications loaded # GIVEN an excel price row # NOT same price row committed to the database excel_versions: List[ApplicationVersionSchema] = list( parse_application_versions(excel_path=application_versions_file) ) version: ApplicationVersionSchema = excel_versions[0] application_obj: models.Application = applications_store.application(version.app_tag) sign = "DummySign" db_version: models.ApplicationVersion = add_application_version( application_obj=application_obj, latest_version=None, version=version, sign=sign, store=applications_store, ) another_version: ApplicationVersionSchema = excel_versions[1] # WHEN calling versions are same should_not_be_same: bool = versions_are_same( version_obj=db_version, application_version=another_version ) # THEN versions are not considered same assert should_not_be_same is False
def test_missing(analysis_store: Store, helpers): """Tests that analyses that are completed but lacks delivery report are returned""" # GIVEN an analysis that is delivered but has no delivery report timestamp = datetime.now() analysis = helpers.add_analysis(analysis_store, started_at=timestamp, uploaded_at=timestamp) sample = helpers.add_sample(analysis_store, delivered_at=timestamp) analysis_store.relate_sample(family=analysis.family, sample=sample, status="unknown") assert sample.delivered_at is not None assert analysis.delivery_report_created_at is None # WHEN calling the analyses_to_delivery_report analyses = analysis_store.analyses_to_delivery_report().all() # THEN this analyse should be returned assert analysis in analyses