def test_is_spring_decompression_possible_no_fastq(
crunchy_config_dict: dict, compression_object: CompressionData,
caplog):
"""Test if decompression is possible when there are no FASTQ files
The function should return true since there are no FASTQ files
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN a existing SPRING file
compression_object.spring_path.touch()
assert compression_object.spring_path.exists()
# GIVEN that there are no fastq files
compression_object.fastq_first.unlink()
compression_object.fastq_second.unlink()
assert not compression_object.fastq_first.exists()
assert not compression_object.fastq_second.exists()
# WHEN checking if SPRING compression is done
result = crunchy_api.is_spring_decompression_possible(compression_object)
# THEN result should be True since there are no fastq files
assert result is True
# THEN assert the correct information is communicated
assert "Decompression is possible" in caplog.text
def test_spring_to_fastq(
compression_object: CompressionData,
spring_metadata_file: Path,
crunchy_config_dict: dict,
mocker,
):
"""Test SPRING to FASTQ method
Test to decompress SPRING to FASTQ. This test will make sure that the correct sbatch content
was submitted to the Process api
"""
# GIVEN a crunchy-api given an existing SPRING metadata file
assert spring_metadata_file.exists()
mocker_submit_sbatch = mocker.patch.object(SlurmAPI, "submit_sbatch")
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN that the pending path does not exist
assert compression_object.pending_exists() is False
# WHEN calling bam_to_cram method on bam-path
crunchy_api.spring_to_fastq(compression_obj=compression_object)
# THEN _submit_sbatch method is called
mocker_submit_sbatch.assert_called()
# THEN assert that the pending path was created
assert compression_object.pending_exists() is True
def test_get_index_path(crunchy_config_dict, crunchy_test_dir):
"""Test get_index_path"""
# GIVEM a bam_path
crunchy_api = CrunchyAPI(crunchy_config_dict)
bam_path = crunchy_test_dir / "file.bam"
# WHEN creating bam path
bai_paths = crunchy_api.get_index_path(file_path=bam_path)
# THEN this should replace current suffix with .bam.bai for double suffix
# type and .bai for single suffix type
assert bai_paths["double_suffix"].suffixes == [".bam", ".bai"]
assert bai_paths["single_suffix"].suffixes == [".bai"]
# GIVEN a bam_path
cram_path = crunchy_test_dir / "file.cram"
# WHEN creating flag path
crai_paths = crunchy_api.get_index_path(file_path=cram_path)
# THEN this should replace current suffix with .cram.crai for double suffix
# type, and .crai for single suffix type
assert crai_paths["double_suffix"].suffixes == [".cram", ".crai"]
assert crai_paths["single_suffix"].suffixes == [".crai"]
def test_is_compression_done_spring_new_files(
crunchy_config_dict: dict,
compression_object: CompressionData,
spring_metadata_file: Path,
caplog,
):
"""Test if compression is done when FASTQ files have been unpacked
This test should fail since the FASTQ files are new
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api, and FASTQ paths
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN a existing SPRING file
compression_object.spring_path.touch()
assert compression_object.spring_path.exists()
# GIVEN a existing flag file
metadata_file = compression_object.spring_metadata_path
assert spring_metadata_file == compression_object.spring_metadata_path
assert metadata_file.exists()
# GIVEN that the files where updated less than three weeks ago
update_metadata_date(metadata_file)
with open(metadata_file, "r") as infile:
content: List[Dict[str, str]] = json.load(infile)
for file_info in content:
assert "updated" in file_info
# WHEN checking if SPRING compression is done
result = crunchy_api.is_fastq_compression_done(compression_object)
# THEN result should be False since the updated date < 3 weeks
assert result is False
# THEN assert that correct information is logged
assert "FASTQ files are not old enough" in caplog.text
def test_is_spring_decompression_done_all_files_exist_not_updated(
crunchy_config_dict, compression_object, spring_metadata_file, caplog):
"""Test if SPRING decompression is done when FASTQ files are not unarchived
The function should return False since the files has not been updated in the metadata file
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN that the FASTQ paths exists
compression_object.fastq_first.touch()
compression_object.fastq_second.touch()
# GIVEN a existing SPRING file
compression_object.spring_path.touch()
# GIVEN a existing flag file
# GIVEN that the files are missing the updated tag
with open(spring_metadata_file, "r") as infile:
content = json.load(infile)
# GIVEN that the files where updated more than three weeks ago
for file_info in content:
assert "updated" not in file_info
# WHEN checking if SPRING decompression is done
result = crunchy_api.is_spring_decompression_done(compression_object)
# THEN result should be False since all files exists but miss the updated tag
assert result is False
# THEN assert that it was communicated that files have not been updated
assert "Files have not been unarchived" in caplog.text
def test_is_spring_decompression_possible(crunchy_config_dict,
compression_object,
spring_metadata_file, caplog):
"""Test if SPRING decompression is possible
The function should return True since there is a SPRING file and no FASTQ files
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN a existing SPRING file
compression_object.spring_path.touch()
# GIVEN that the FASTQ files does not exist
compression_object.fastq_first.unlink()
compression_object.fastq_second.unlink()
assert not compression_object.fastq_first.exists()
assert not compression_object.fastq_second.exists()
# WHEN checking if SPRING decompression is done
result = crunchy_api.is_spring_decompression_possible(compression_object)
# THEN result should be True since decompression is possible
assert result is True
# THEN assert that it was communicated that compression is pending
assert "Decompression is possible" in caplog.text
def test_is_spring_decompression_done_missing_fastq_files(
crunchy_config_dict, compression_object, spring_metadata_file, caplog):
"""Test if SPRING decompression is done when FASTQ files are missing
The function should return False since FASTQ files are missing
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN that the FASTQ paths does not exist
compression_object.fastq_first.unlink()
compression_object.fastq_second.unlink()
assert not compression_object.fastq_first.exists()
assert not compression_object.fastq_second.exists()
# GIVEN a existing SPRING file
compression_object.spring_path.touch()
# WHEN checking if SPRING decompression is done
result = crunchy_api.is_spring_decompression_done(compression_object)
# THEN result should be False since FASTQ files are missing
assert result is False
# THEN assert that it was communicated that decompression is done
assert "does not exist" in caplog.text
def test_fastq_to_spring_sbatch(
crunchy_config_dict: dict,
compression_object: CompressionData,
sbatch_process: Process,
sbatch_job_number: int,
caplog,
):
"""Test fastq_to_spring method"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api, and FASTQ paths
crunchy_api = CrunchyAPI(crunchy_config_dict)
crunchy_api.slurm_api.process = sbatch_process
spring_path: Path = compression_object.spring_path
log_path: Path = get_log_dir(spring_path)
run_name: str = compression_object.run_name
sbatch_path: Path = get_fastq_to_spring_sbatch_path(log_dir=log_path, run_name=run_name)
# GIVEN that the sbatch file does not exist
assert not sbatch_path.is_file()
# GIVEN that the pending path does not exist
assert compression_object.pending_exists() is False
# WHEN calling fastq_to_spring on FASTQ files
job_number: int = crunchy_api.fastq_to_spring(compression_obj=compression_object)
# THEN assert that the sbatch file was created
assert sbatch_path.is_file()
# THEN assert that correct job number was returned
assert job_number == sbatch_job_number
# THEN assert that the pending path was created
assert compression_object.pending_exists() is True
def test_is_spring_decompression_possible(crunchy_config_dict,
compression_object,
spring_metadata_file):
"""Test if SPRING decompression is possible when decompression is already done
The function should return False since decompression is already done
"""
# GIVEN a crunchy-api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN that the FASTQ paths exists
compression_object.fastq_first.touch()
compression_object.fastq_second.touch()
# GIVEN a existing SPRING file
compression_object.spring_path.touch()
# GIVEN a existing flag file
# GIVEN that the files have an updated tag
old_date = "2019-01-01"
with open(spring_metadata_file, "r") as infile:
content = json.load(infile)
# GIVEN that the files where updated more than three weeks ago
for file_info in content:
file_info["updated"] = old_date
with open(spring_metadata_file, "w") as outfile:
outfile.write(json.dumps(content))
# WHEN checking if SPRING decompression is done
result = crunchy_api.is_spring_decompression_possible(compression_object)
# THEN result should be False since decompression is already done
assert result is False
def test_is_fastq_compression_possible_compression_pending(
crunchy_config_dict: dict, compression_object: CompressionData,
caplog):
"""Test if FASTQ compression is possible when FASTQ compression is pending
This means that there should exist a FASTQ compression flag
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api, and existing FASTQ paths
crunchy_api = CrunchyAPI(crunchy_config_dict)
compression_object.fastq_first.touch()
compression_object.fastq_second.touch()
# GIVEN that the pending path exists
compression_object.pending_path.touch()
# GIVEN no SPRING file exists
spring_file = compression_object.spring_path
assert not spring_file.exists()
# WHEN checking if SPRING compression is done
result = crunchy_api.is_fastq_compression_possible(compression_object)
# THEN result should be False since the compression flag exists
assert result is False
# THEN assert that the correct message was communicated
assert "Compression/decompression is pending for" in caplog.text
def test_is_cram_compression_done_no_cram(crunchy_config_dict, bam_path):
"""test cram_compression_done without created CRAM file"""
# GIVEN a crunchy-api, and a bam_path
crunchy_api = CrunchyAPI(crunchy_config_dict)
# WHEN checking if cram compression is done
result = crunchy_api.is_cram_compression_done(bam_path=bam_path)
# THEN result should be false
assert not result
def test_is_bam_compression_possible_no_bam(crunchy_config_dict, bam_path):
"""Test bam_compression_possible for non-existing BAM"""
# GIVEN a bam path to non existing file and a crunchy api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# WHEN calling test_bam_compression_possible
result = crunchy_api.is_bam_compression_possible(bam_path=bam_path)
# THEN this should return False
assert not result
def test_get_cram_path_from_bam(crunchy_config_dict, crunchy_test_dir):
"""Test change_suffic_bam_to_cram"""
# GIVEN a bam_path
crunchy_api = CrunchyAPI(crunchy_config_dict)
bam_path = crunchy_test_dir / "file.bam"
# WHEN changing suffix to cram
cram_path = crunchy_api.get_cram_path_from_bam(bam_path=bam_path)
# THEN suffix should be .cram
assert cram_path.suffix == CRAM_SUFFIX
def test_is_cram_compression_done(crunchy_config_dict, compressed_bam):
"""Test cram_compression_done with created CRAM, CRAI, and flag files"""
# GIVEN a crunchy-api, and a bam_path, cram_path, crai_path, and flag_path
crunchy_api = CrunchyAPI(crunchy_config_dict)
bam_path = compressed_bam
# WHEN checking if cram compression is done
result = crunchy_api.is_cram_compression_done(bam_path=bam_path)
# THEN result should be True
assert result
def test_set_dry_run(crunchy_config_dict: dict):
"""Test to set the dry run of the api"""
# GIVEN a crunchy API where dry run is False
crunchy_api = CrunchyAPI(crunchy_config_dict)
assert crunchy_api.dry_run is False
# WHEN updating the dry run
crunchy_api.set_dry_run(True)
# THEN assert that the api has true dry run
assert crunchy_api.dry_run is True
def test_get_flag_path(crunchy_config_dict, crunchy_test_dir):
"""Test get_flag_path"""
# GIVEM a bam_path
crunchy_api = CrunchyAPI(crunchy_config_dict)
bam_path = crunchy_test_dir / "file.bam"
# WHEN creating flag path
flag_path = crunchy_api.get_flag_path(file_path=bam_path)
# THEN this should replace current suffix with FLAG_PATH_SUFFIX
assert flag_path.suffixes == [".crunchy", ".txt"]
def test_is_bam_compression_possible(crunchy_config_dict, crunchy_test_dir):
"""Test bam_compression_possible compressable BAM"""
# GIVEN a bam path to existing file and a crunchy api
crunchy_api = CrunchyAPI(crunchy_config_dict)
bam_path = crunchy_test_dir / "file.bam"
bam_path.touch()
# WHEN calling test_bam_compression_possible
result = crunchy_api.is_bam_compression_possible(bam_path=bam_path)
# THEN this will return True
assert result
def test_is_cram_compression_done_no_flag(crunchy_config_dict,
compressed_bam_without_flag):
"""test cram_compression_done without created flag file"""
# GIVEN a crunchy-api, and a bam_path, cram_path, crai_path
crunchy_api = CrunchyAPI(crunchy_config_dict)
bam_path = compressed_bam_without_flag
# WHEN checking if cram compression is done
result = crunchy_api.is_cram_compression_done(bam_path=bam_path)
# THEN result should be False
assert not result
def test_is_not_pending_when_no_flag_file(crunchy_config_dict: dict,
compression_object: CompressionData):
"""Test if SPRING compression is pending when no flag file"""
# GIVEN a crunchy-api, and a FASTQ file
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN a non existing pending flag
assert not compression_object.pending_path.exists()
# WHEN checking if SPRING compression is ongoing
result = crunchy_api.is_compression_pending(compression_object)
# THEN result should be False since the pending flag is not there
assert result is False
def test_bam_to_cram(crunchy_config_dict, sbatch_content, bam_path, mocker):
"""Test bam_to_cram method"""
# GIVEN a crunchy-api, and a bam_path
mocker_submit_sbatch = mocker.patch.object(CrunchyAPI, "_submit_sbatch")
crunchy_api = CrunchyAPI(crunchy_config_dict)
# WHEN calling bam_to_cram method on bam-path
crunchy_api.bam_to_cram(bam_path=bam_path, dry_run=False, ntasks=1, mem=2)
# THEN _submit_sbatch method is called with expected sbatch-content
mocker_submit_sbatch.assert_called_with(sbatch_content=sbatch_content,
dry_run=False)
def test_is_pending(crunchy_config_dict: dict,
compression_object: CompressionData):
"""Test if SPRING compression is pending when pending file exists"""
# GIVEN a crunchy-api, and FASTQ files
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN a existing pending flag
compression_object.pending_path.touch()
assert compression_object.pending_path.exists()
# WHEN checking if SPRING compression is pending
result = crunchy_api.is_compression_pending(compression_object)
# THEN result should be True since the pending_path exists
assert result is True
def crunchy_api(self) -> CrunchyAPI:
api = self.__dict__.get("crunchy_api_")
if api is None:
LOG.debug("Instantiating crunchy api")
api = CrunchyAPI(config=self.dict())
self.crunchy_api_ = api
return api
def test_is_bam_compression_possible_cram_done(crunchy_config_dict,
crunchy_test_dir,
mock_bam_to_cram, mocker):
"""Test bam_compression_possible for existing compression"""
# GIVEN a bam path to a existing file and a crunchy api
mocker.patch.object(CrunchyAPI,
"bam_to_cram",
side_effect=mock_bam_to_cram)
crunchy_api = CrunchyAPI(crunchy_config_dict)
bam_path = crunchy_test_dir / "file.bam"
bam_path.touch()
# WHEN calling test_bam_compression_possible when cram_compression is done
crunchy_api.bam_to_cram(bam_path=bam_path, dry_run=False, ntasks=1, mem=2)
result = crunchy_api.is_bam_compression_possible(bam_path=bam_path)
# THEN this should return False
assert not result
def test_is_spring_decompression_done_empty_metadata_file(
crunchy_config_dict, compression_object, caplog):
"""Test if SPRING decompression is done when SPRING metadata file has no content
The function should return False since the metadata file has no content
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN that the SPRING metadata file has no content
compression_object.spring_metadata_path.touch()
# WHEN checking if SPRING decompression is done
with pytest.raises(SyntaxError):
# THEN assert that an exception should be raised since the file is malformed
crunchy_api.is_spring_decompression_done(compression_object)
# THEN assert that it was communicated that the content is malformed
assert "Malformed metadata content" in caplog.text
def test_is_spring_decompression_possible_no_spring(
crunchy_config_dict: dict, compression_object: CompressionData,
caplog):
"""Test if decompression is possible when there are no SPRING archive
The function should return False since there is no SPRING archive
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api, and FASTQ paths
crunchy_api = CrunchyAPI(crunchy_config_dict)
# WHEN checking if SPRING compression is done
result = crunchy_api.is_spring_decompression_possible(compression_object)
# THEN result should be False since there is no SPRING archive
assert result is False
# THEN assert the correct information is communicated
assert "No SPRING file found" in caplog.text
def test_is_compression_done_no_spring(crunchy_config_dict: dict,
compression_object: CompressionData,
caplog):
"""Test if compression is done when no SPRING archive"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api, and FASTQ paths
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN no SPRING file exists
spring_file = compression_object.spring_path
assert not spring_file.exists()
# WHEN checking if SPRING compression is done
result = crunchy_api.is_fastq_compression_done(compression_object)
# THEN result should be false
assert not result
# THEN assert that the correct message was communicated
assert f"No SPRING file for {compression_object.run_name}" in caplog.text
def test_is_spring_decompression_possible_decompression_pending(
crunchy_config_dict, compression_object, spring_metadata_file, caplog):
"""Test if SPRING decompression is possible when decompression is pending
The function should return False since decompression is pending
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN a existing pending file
compression_object.pending_path.touch()
# WHEN checking if SPRING decompression is done
result = crunchy_api.is_spring_decompression_possible(compression_object)
# THEN result should be False since decompression is pending
assert result is False
# THEN assert that it was communicated that compression is pending
assert "decompression is pending" in caplog.text
def test_is_spring_decompression_done_missing_metadata_file(
crunchy_config_dict, compression_object, caplog):
"""Test if SPRING decompression is done when SPRING metadata file is missing
The function should return False since no metadata file is found
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN that the SPRING metadata file is missing
assert not compression_object.spring_metadata_path.exists()
# WHEN checking if SPRING decompression is done
result = crunchy_api.is_spring_decompression_done(compression_object)
# THEN result should be False since the metadata file is missing
assert result is False
# THEN assert that it was communicated that files have not been updated
assert "No SPRING metadata file found" in caplog.text
def test_is_compression_done_no_flag_spring(
crunchy_config_dict: dict, compression_object: CompressionData,
caplog):
"""Test if SPRING compression is done when no metadata file"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api, and FASTQ paths
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN a existing SPRING file
compression_object.spring_path.touch()
assert compression_object.spring_path.exists()
# GIVEN a non existing flag file
assert not compression_object.spring_metadata_path.exists()
# WHEN checking if SPRING compression is done
result = crunchy_api.is_fastq_compression_done(compression_object)
# THEN result should be false
assert not result
# THEN assert that the correct message was communicated
assert "No metadata file found" in caplog.text
def test_is_fastq_compression_possible_spring_exists(
crunchy_config_dict: dict, compression_object: CompressionData,
caplog):
"""Test if FASTQ compression is possible when FASTQ compression is done
This means that the SPRING file exists
"""
caplog.set_level(logging.DEBUG)
# GIVEN a crunchy-api, and existing FASTQ paths
crunchy_api = CrunchyAPI(crunchy_config_dict)
# GIVEN that the SPRING path exists
compression_object.spring_path.touch()
spring_file = compression_object.spring_path
assert spring_file.exists()
# WHEN checking if SPRING compression is done
result = crunchy_api.is_fastq_compression_possible(compression_object)
# THEN result should be False since the compression flag exists
assert result is False
# THEN assert that the correct message was communicated
assert "SPRING file found" in caplog.text