def test_non_initialised_db(hk_config): """Test to use a database that is not initialised""" # GIVEN a housekeeper api and some hk configs api = HousekeeperAPI(hk_config) # GIVEN a api without the database with pytest.raises(OperationalError): # THEN it should raise a operational error api.add_tag("a-tag")
def test_init_db(hk_config): """Test to setup the database""" # GIVEN a housekeeper api and some hk configs api = HousekeeperAPI(hk_config) # WHEN initiating the database api.initialise_db() # THEN the api should work assert api.add_tag("a-tag")
def test_get_versions_one_bundle(housekeeper_api: HousekeeperAPI, spring_bundle: dict): """Test to get latest versions of bundles when no bundles exists""" # GIVEN a populated housekeeper_api housekeeper_api.add_bundle(spring_bundle) # WHEN fetching versions versions = helpers.get_versions(housekeeper_api) # THEN assert no versions was returned assert sum(1 for version in versions) == 1
def fixture_upload_genotypes_hk_api( real_housekeeper_api: HousekeeperAPI, upload_genotypes_hk_bundle: dict, analysis_obj: models.Analysis, helpers, ) -> HousekeeperAPI: """Add and include files from upload genotypes hk bundle""" helpers.ensure_hk_bundle(real_housekeeper_api, upload_genotypes_hk_bundle) hk_version = real_housekeeper_api.last_version( analysis_obj.family.internal_id) real_housekeeper_api.include(hk_version) return real_housekeeper_api
def get_delivery_report_from_hk(hk_api: HousekeeperAPI, case_id: str) -> str: delivery_report_tag_name = "delivery-report" version_obj = hk_api.last_version(case_id) uploaded_delivery_report_files = hk_api.get_files( bundle=case_id, tags=[delivery_report_tag_name], version=version_obj.id, ) if uploaded_delivery_report_files.count() == 0: raise FileNotFoundError(f"No delivery report was found in housekeeper for {case_id}") return uploaded_delivery_report_files[0].full_path
def test_delete_file(populated_housekeeper_api: HousekeeperAPI): """Test to fetch a file from the database""" # GIVEN a housekeeper api with a file file_obj = populated_housekeeper_api.files().first() assert file_obj # GIVEN the id of a file that exists in HK file_id = file_obj.id # WHEN deleting the file populated_housekeeper_api.delete_file(file_id) # THEN assert a file was removed assert populated_housekeeper_api.get_file(file_id) is None
def test_get_file(populated_housekeeper_api: HousekeeperAPI): """Test to fetch a file from the database""" # GIVEN a housekeeper api with a file file_obj = populated_housekeeper_api.files().first() assert file_obj # GIVEN the id of a file that exists in HK file_id = file_obj.id # WHEN fetching the file with get_file hk_file = populated_housekeeper_api.get_file(file_id) # THEN assert a file was returned assert hk_file is not None
def test_get_tag_names_from_file(populated_housekeeper_api: HousekeeperAPI): """Test get tag names on a file""" # GIVEN a housekeeper api with a file file_obj = populated_housekeeper_api.files().first() assert file_obj.tags # WHEN fetching tags of a file tag_names = populated_housekeeper_api.get_tag_names_from_file(file_obj) # THEN a list of tag names is returned assert tag_names is not None # THEN the return type is a list of strings assert isinstance(tag_names, list) assert all(isinstance(elem, str) for elem in tag_names)
def _get_multiqc_latest_file(hk_api: HousekeeperAPI, case_name: str) -> str: """Get latest multiqc_data.json path for a case_name Args: case_name(str): onemite Returns: multiqc_data_path(str): /path/to/multiqc.json """ version_obj = hk_api.last_version(case_name) multiqc_json_file = hk_api.get_files( bundle=case_name, tags=["multiqc-json"], version=version_obj.id ) if len(list(multiqc_json_file)) == 0: raise FileNotFoundError(f"No multiqc.json was found in housekeeper for {case_name}") return multiqc_json_file[0].full_path
def housekeeper_api(self) -> HousekeeperAPI: housekeeper_api = self.__dict__.get("housekeeper_api_") if housekeeper_api is None: LOG.debug("Instantiating housekeeper api") housekeeper_api = HousekeeperAPI(config=self.dict()) self.housekeeper_api_ = housekeeper_api return housekeeper_api
def test_get_case_files_from_version( analysis_store: Store, case_id: str, real_housekeeper_api: HousekeeperAPI, case_hk_bundle_no_files: dict, bed_file: str, vcf_file: Path, project_dir: Path, helpers=StoreHelpers, ): # GIVEN a store with a case case_obj = analysis_store.family(case_id) assert case_obj.internal_id == case_id # GIVEN a delivery api deliver_api = DeliverAPI( store=analysis_store, hk_api=real_housekeeper_api, case_tags=[{"case-tag"}], sample_tags=[{"sample-tag"}], project_base_path=project_dir, delivery_type="balsamic", ) # GIVEN a housekeeper db populated with a bundle including a case specific file and a sample specific file case_hk_bundle_no_files["files"] = [ { "path": bed_file, "archive": False, "tags": ["case-tag"] }, { "path": str(vcf_file), "archive": False, "tags": ["sample-tag", "ADM1"] }, ] helpers.ensure_hk_bundle(real_housekeeper_api, bundle_data=case_hk_bundle_no_files) # GIVEN a version object where two file exists version_obj: hk_models.Version = real_housekeeper_api.last_version(case_id) assert len(version_obj.files) == 2 # GIVEN the sample ids of the samples link_objs: List[FamilySample] = analysis_store.family_samples(case_id) samples: List[Sample] = [link.sample for link in link_objs] sample_ids: Set[str] = set([sample.internal_id for sample in samples]) # WHEN fetching the case files case_files = deliver_api.get_case_files_from_version( version_obj=version_obj, sample_ids=sample_ids) # THEN we should only get the case specific files back nr_files: int = 0 case_file: Path for nr_files, case_file in enumerate(case_files, 1): assert case_file.name == Path(bed_file).name # THEN assert that only the case-tag file was returned assert nr_files == 1
def test_correct_spring_paths( housekeeper_api: HousekeeperAPI, spring_bundle_symlink_problem: dict, symlinked_fastqs: dict, new_dir: Path, ): """docstring for test_correct_spring_paths""" # GIVEN a populated housekeeper_api housekeeper_api.add_bundle(spring_bundle_symlink_problem) # GIVEN that the spring files exists in the wrong location versions = helpers.get_versions(housekeeper_api) version = next(versions) for file_path in version.files: assert not Path(file_path.full_path).exists() # WHEN updating the spring paths helpers.correct_spring_paths(housekeeper_api) # THEN assert that the spring paths has been moved for file_path in version.files: assert Path(file_path.full_path).exists()
def add_delivery_report_to_hk( self, delivery_report_file: Path, hk_api: HousekeeperAPI, case_id: str, analysis_date: datetime, ) -> Optional[housekeeper.store.models.File]: """ Add a delivery report to a analysis bundle for a case in Housekeeper If there is already a delivery report for the bundle doesn't add to it If successful the method returns a pointer to the new file in Housekeeper """ delivery_report_tag_name = "delivery-report" version_obj = hk_api.version(case_id, analysis_date) is_bundle_missing_delivery_report = False try: self.get_delivery_report_from_hk(hk_api=hk_api, case_id=case_id) except FileNotFoundError: is_bundle_missing_delivery_report = True if is_bundle_missing_delivery_report: file_obj = hk_api.add_file( delivery_report_file.name, version_obj, delivery_report_tag_name ) hk_api.include_file(file_obj, version_obj) hk_api.add_commit(file_obj) return file_obj return None
def get_versions(hk_api: HousekeeperAPI, bundle_name: str = None) -> Iterator[hk_models.Version]: """Generates versions from hk bundles If no bundle name is given generate latest version for every bundle """ if bundle_name: bundle = hk_api.bundle(bundle_name) if not bundle: LOG.info("Could not find bundle %s", bundle_name) return bundles = [bundle] else: bundles = hk_api.bundles() for bundle in bundles: LOG.debug("Check for versions in %s", bundle.name) last_version = hk_api.last_version(bundle.name) if not last_version: LOG.warning("No bundle found for %s in housekeeper", bundle.name) return yield last_version
def fetch_file_from_hk(self, hk_tags: Set[str]) -> Optional[str]: """Fetch a file from housekeeper and return the path as a string. If file does not exist return None """ LOG.info("Fetch file with tags %s", hk_tags) if not hk_tags: LOG.debug("No tags provided, skipping") return None hk_file: Optional[ hk_models.File] = HousekeeperAPI.fetch_file_from_version( version_obj=self.hk_version_obj, tags=hk_tags) if hk_file is None: return hk_file return hk_file.full_path
def has_protected_tags(file: hk_models.File, protected_tags_lists: List[List[str]]) -> bool: """Check if a file has any protected tags""" LOG.info(f"File {file.full_path} has the tags {file.tags}") file_tags: List[str] = HousekeeperAPI.get_tag_names_from_file(file) _has_protected_tags: bool = False for protected_tags in protected_tags_lists: if set(protected_tags).issubset(set(file_tags)): LOG.debug( "File %s has the protected tag(s) %s, skipping.", file.full_path, protected_tags, ) _has_protected_tags = True break if not _has_protected_tags: LOG.info("File %s has no protected tags.", file.full_path) return _has_protected_tags
def fixture_mip_dna_housekeeper( real_housekeeper_api: HousekeeperAPI, mip_delivery_bundle: dict, fastq_delivery_bundle: dict, helpers: StoreHelpers, ) -> HousekeeperAPI: helpers.ensure_hk_bundle(real_housekeeper_api, bundle_data=mip_delivery_bundle) helpers.ensure_hk_bundle(real_housekeeper_api, bundle_data=fastq_delivery_bundle) # assert that the files exists version_obj_mip: hk_models.Version = real_housekeeper_api.last_version( mip_delivery_bundle["name"]) version_obj_fastq: hk_models.Version = real_housekeeper_api.last_version( fastq_delivery_bundle["name"]) real_housekeeper_api.include(version_obj=version_obj_mip) real_housekeeper_api.include(version_obj=version_obj_fastq) return real_housekeeper_api
def ensure_hk_bundle( store: HousekeeperAPI, bundle_data: dict, include: bool = False ) -> hk_models.Bundle: """Utility function to add a bundle of information to a housekeeper api""" bundle_exists = False for bundle in store.bundles(): if bundle.name != bundle_data["name"]: continue bundle_exists = True _bundle = bundle break if not bundle_exists: _bundle, _version = store.add_bundle(bundle_data) store.add_commit(_bundle, _version) if include: store.include(_version) return _bundle
def ensure_hk_version(store: HousekeeperAPI, bundle_data: dict) -> hk_models.Version: """Utility function to return existing or create an version for tests""" _bundle = StoreHelpers.ensure_hk_bundle(store, bundle_data) return store.last_version(_bundle.name)
def fixture_real_housekeeper_api(hk_config_dict: dict) -> HousekeeperAPI: """Setup a real Housekeeper store.""" _api = HousekeeperAPI(hk_config_dict) _api.initialise_db() yield _api
def get_tag_names_from_file(file) -> [str]: """Fetch a tag""" return HousekeeperAPI.get_tag_names_from_file(file=file)
def fixture_housekeeper_api(hk_config): """Setup Housekeeper store.""" _api = HousekeeperAPI(hk_config) _api.initialise_db() yield _api _api.destroy_db()