def test_non_initialised_db(hk_config):
"""Test to use a database that is not initialised"""
# GIVEN a housekeeper api and some hk configs
api = HousekeeperAPI(hk_config)
# GIVEN a api without the database
with pytest.raises(OperationalError):
# THEN it should raise a operational error
api.add_tag("a-tag")
def test_init_db(hk_config):
"""Test to setup the database"""
# GIVEN a housekeeper api and some hk configs
api = HousekeeperAPI(hk_config)
# WHEN initiating the database
api.initialise_db()
# THEN the api should work
assert api.add_tag("a-tag")
def test_get_versions_one_bundle(housekeeper_api: HousekeeperAPI,
spring_bundle: dict):
"""Test to get latest versions of bundles when no bundles exists"""
# GIVEN a populated housekeeper_api
housekeeper_api.add_bundle(spring_bundle)
# WHEN fetching versions
versions = helpers.get_versions(housekeeper_api)
# THEN assert no versions was returned
assert sum(1 for version in versions) == 1
def fixture_upload_genotypes_hk_api(
real_housekeeper_api: HousekeeperAPI,
upload_genotypes_hk_bundle: dict,
analysis_obj: models.Analysis,
helpers,
) -> HousekeeperAPI:
"""Add and include files from upload genotypes hk bundle"""
helpers.ensure_hk_bundle(real_housekeeper_api, upload_genotypes_hk_bundle)
hk_version = real_housekeeper_api.last_version(
analysis_obj.family.internal_id)
real_housekeeper_api.include(hk_version)
return real_housekeeper_api
def get_delivery_report_from_hk(hk_api: HousekeeperAPI, case_id: str) -> str:
delivery_report_tag_name = "delivery-report"
version_obj = hk_api.last_version(case_id)
uploaded_delivery_report_files = hk_api.get_files(
bundle=case_id,
tags=[delivery_report_tag_name],
version=version_obj.id,
)
if uploaded_delivery_report_files.count() == 0:
raise FileNotFoundError(f"No delivery report was found in housekeeper for {case_id}")
return uploaded_delivery_report_files[0].full_path
def test_delete_file(populated_housekeeper_api: HousekeeperAPI):
"""Test to fetch a file from the database"""
# GIVEN a housekeeper api with a file
file_obj = populated_housekeeper_api.files().first()
assert file_obj
# GIVEN the id of a file that exists in HK
file_id = file_obj.id
# WHEN deleting the file
populated_housekeeper_api.delete_file(file_id)
# THEN assert a file was removed
assert populated_housekeeper_api.get_file(file_id) is None
def test_get_file(populated_housekeeper_api: HousekeeperAPI):
"""Test to fetch a file from the database"""
# GIVEN a housekeeper api with a file
file_obj = populated_housekeeper_api.files().first()
assert file_obj
# GIVEN the id of a file that exists in HK
file_id = file_obj.id
# WHEN fetching the file with get_file
hk_file = populated_housekeeper_api.get_file(file_id)
# THEN assert a file was returned
assert hk_file is not None
def test_get_tag_names_from_file(populated_housekeeper_api: HousekeeperAPI):
"""Test get tag names on a file"""
# GIVEN a housekeeper api with a file
file_obj = populated_housekeeper_api.files().first()
assert file_obj.tags
# WHEN fetching tags of a file
tag_names = populated_housekeeper_api.get_tag_names_from_file(file_obj)
# THEN a list of tag names is returned
assert tag_names is not None
# THEN the return type is a list of strings
assert isinstance(tag_names, list)
assert all(isinstance(elem, str) for elem in tag_names)
def _get_multiqc_latest_file(hk_api: HousekeeperAPI, case_name: str) -> str:
"""Get latest multiqc_data.json path for a case_name
Args:
case_name(str): onemite
Returns:
multiqc_data_path(str): /path/to/multiqc.json
"""
version_obj = hk_api.last_version(case_name)
multiqc_json_file = hk_api.get_files(
bundle=case_name, tags=["multiqc-json"], version=version_obj.id
)
if len(list(multiqc_json_file)) == 0:
raise FileNotFoundError(f"No multiqc.json was found in housekeeper for {case_name}")
return multiqc_json_file[0].full_path
def housekeeper_api(self) -> HousekeeperAPI:
housekeeper_api = self.__dict__.get("housekeeper_api_")
if housekeeper_api is None:
LOG.debug("Instantiating housekeeper api")
housekeeper_api = HousekeeperAPI(config=self.dict())
self.housekeeper_api_ = housekeeper_api
return housekeeper_api
def test_get_case_files_from_version(
analysis_store: Store,
case_id: str,
real_housekeeper_api: HousekeeperAPI,
case_hk_bundle_no_files: dict,
bed_file: str,
vcf_file: Path,
project_dir: Path,
helpers=StoreHelpers,
):
# GIVEN a store with a case
case_obj = analysis_store.family(case_id)
assert case_obj.internal_id == case_id
# GIVEN a delivery api
deliver_api = DeliverAPI(
store=analysis_store,
hk_api=real_housekeeper_api,
case_tags=[{"case-tag"}],
sample_tags=[{"sample-tag"}],
project_base_path=project_dir,
delivery_type="balsamic",
)
# GIVEN a housekeeper db populated with a bundle including a case specific file and a sample specific file
case_hk_bundle_no_files["files"] = [
{
"path": bed_file,
"archive": False,
"tags": ["case-tag"]
},
{
"path": str(vcf_file),
"archive": False,
"tags": ["sample-tag", "ADM1"]
},
]
helpers.ensure_hk_bundle(real_housekeeper_api,
bundle_data=case_hk_bundle_no_files)
# GIVEN a version object where two file exists
version_obj: hk_models.Version = real_housekeeper_api.last_version(case_id)
assert len(version_obj.files) == 2
# GIVEN the sample ids of the samples
link_objs: List[FamilySample] = analysis_store.family_samples(case_id)
samples: List[Sample] = [link.sample for link in link_objs]
sample_ids: Set[str] = set([sample.internal_id for sample in samples])
# WHEN fetching the case files
case_files = deliver_api.get_case_files_from_version(
version_obj=version_obj, sample_ids=sample_ids)
# THEN we should only get the case specific files back
nr_files: int = 0
case_file: Path
for nr_files, case_file in enumerate(case_files, 1):
assert case_file.name == Path(bed_file).name
# THEN assert that only the case-tag file was returned
assert nr_files == 1
def test_correct_spring_paths(
housekeeper_api: HousekeeperAPI,
spring_bundle_symlink_problem: dict,
symlinked_fastqs: dict,
new_dir: Path,
):
"""docstring for test_correct_spring_paths"""
# GIVEN a populated housekeeper_api
housekeeper_api.add_bundle(spring_bundle_symlink_problem)
# GIVEN that the spring files exists in the wrong location
versions = helpers.get_versions(housekeeper_api)
version = next(versions)
for file_path in version.files:
assert not Path(file_path.full_path).exists()
# WHEN updating the spring paths
helpers.correct_spring_paths(housekeeper_api)
# THEN assert that the spring paths has been moved
for file_path in version.files:
assert Path(file_path.full_path).exists()
def add_delivery_report_to_hk(
self,
delivery_report_file: Path,
hk_api: HousekeeperAPI,
case_id: str,
analysis_date: datetime,
) -> Optional[housekeeper.store.models.File]:
"""
Add a delivery report to a analysis bundle for a case in Housekeeper
If there is already a delivery report for the bundle doesn't add to it
If successful the method returns a pointer to the new file in Housekeeper
"""
delivery_report_tag_name = "delivery-report"
version_obj = hk_api.version(case_id, analysis_date)
is_bundle_missing_delivery_report = False
try:
self.get_delivery_report_from_hk(hk_api=hk_api, case_id=case_id)
except FileNotFoundError:
is_bundle_missing_delivery_report = True
if is_bundle_missing_delivery_report:
file_obj = hk_api.add_file(
delivery_report_file.name, version_obj, delivery_report_tag_name
)
hk_api.include_file(file_obj, version_obj)
hk_api.add_commit(file_obj)
return file_obj
return None
def get_versions(hk_api: HousekeeperAPI,
bundle_name: str = None) -> Iterator[hk_models.Version]:
"""Generates versions from hk bundles
If no bundle name is given generate latest version for every bundle
"""
if bundle_name:
bundle = hk_api.bundle(bundle_name)
if not bundle:
LOG.info("Could not find bundle %s", bundle_name)
return
bundles = [bundle]
else:
bundles = hk_api.bundles()
for bundle in bundles:
LOG.debug("Check for versions in %s", bundle.name)
last_version = hk_api.last_version(bundle.name)
if not last_version:
LOG.warning("No bundle found for %s in housekeeper", bundle.name)
return
yield last_version
def fetch_file_from_hk(self, hk_tags: Set[str]) -> Optional[str]:
"""Fetch a file from housekeeper and return the path as a string.
If file does not exist return None
"""
LOG.info("Fetch file with tags %s", hk_tags)
if not hk_tags:
LOG.debug("No tags provided, skipping")
return None
hk_file: Optional[
hk_models.File] = HousekeeperAPI.fetch_file_from_version(
version_obj=self.hk_version_obj, tags=hk_tags)
if hk_file is None:
return hk_file
return hk_file.full_path
def has_protected_tags(file: hk_models.File,
protected_tags_lists: List[List[str]]) -> bool:
"""Check if a file has any protected tags"""
LOG.info(f"File {file.full_path} has the tags {file.tags}")
file_tags: List[str] = HousekeeperAPI.get_tag_names_from_file(file)
_has_protected_tags: bool = False
for protected_tags in protected_tags_lists:
if set(protected_tags).issubset(set(file_tags)):
LOG.debug(
"File %s has the protected tag(s) %s, skipping.",
file.full_path,
protected_tags,
)
_has_protected_tags = True
break
if not _has_protected_tags:
LOG.info("File %s has no protected tags.", file.full_path)
return _has_protected_tags
def fixture_mip_dna_housekeeper(
real_housekeeper_api: HousekeeperAPI,
mip_delivery_bundle: dict,
fastq_delivery_bundle: dict,
helpers: StoreHelpers,
) -> HousekeeperAPI:
helpers.ensure_hk_bundle(real_housekeeper_api,
bundle_data=mip_delivery_bundle)
helpers.ensure_hk_bundle(real_housekeeper_api,
bundle_data=fastq_delivery_bundle)
# assert that the files exists
version_obj_mip: hk_models.Version = real_housekeeper_api.last_version(
mip_delivery_bundle["name"])
version_obj_fastq: hk_models.Version = real_housekeeper_api.last_version(
fastq_delivery_bundle["name"])
real_housekeeper_api.include(version_obj=version_obj_mip)
real_housekeeper_api.include(version_obj=version_obj_fastq)
return real_housekeeper_api
def ensure_hk_bundle(
store: HousekeeperAPI, bundle_data: dict, include: bool = False
) -> hk_models.Bundle:
"""Utility function to add a bundle of information to a housekeeper api"""
bundle_exists = False
for bundle in store.bundles():
if bundle.name != bundle_data["name"]:
continue
bundle_exists = True
_bundle = bundle
break
if not bundle_exists:
_bundle, _version = store.add_bundle(bundle_data)
store.add_commit(_bundle, _version)
if include:
store.include(_version)
return _bundle
def ensure_hk_version(store: HousekeeperAPI, bundle_data: dict) -> hk_models.Version:
"""Utility function to return existing or create an version for tests"""
_bundle = StoreHelpers.ensure_hk_bundle(store, bundle_data)
return store.last_version(_bundle.name)
def fixture_real_housekeeper_api(hk_config_dict: dict) -> HousekeeperAPI:
"""Setup a real Housekeeper store."""
_api = HousekeeperAPI(hk_config_dict)
_api.initialise_db()
yield _api
def get_tag_names_from_file(file) -> [str]:
"""Fetch a tag"""
return HousekeeperAPI.get_tag_names_from_file(file=file)
def fixture_housekeeper_api(hk_config):
"""Setup Housekeeper store."""
_api = HousekeeperAPI(hk_config)
_api.initialise_db()
yield _api
_api.destroy_db()