def test_pull_singularity_image(self):
tmp_dir = tempfile.mkdtemp()
download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_dir)
download_obj.pull_singularity_image("a-container")
# Clean up
shutil.rmtree(tmp_dir)
def test_fetch_workflow_details_for_github_ressource_take_master(
self, mock_workflows):
download_obj = DownloadWorkflow(pipeline="myorg/dummy")
mock_workflows.remote_workflows = []
download_obj.fetch_workflow_details(mock_workflows)
assert download_obj.release == "master"
def test_download_shub_image_on_sucess(self, mock_request, mock_json,
mock_content, mock_progressbar,
mock_md5):
tmp_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(tmp_dir, 'singularity-images'))
download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_dir)
# simulauates the first response querying the API for the container
# information
resp_shub = requests.Response()
resp_shub.status_code = 200
mock_json.side_effect = [{'image': 'my-container', 'version': 'h4sh'}]
# simulates the second response querying the
# container download stream
resp_download = requests.Response()
resp_download.status_code = 200
resp_download.headers = {'content-length': 1024}
mock_content.side_effect = b"Awesome"
# assign the response order
mock_request.side_effect = [resp_shub, resp_download]
# test
download_obj.download_shub_image("awesome-container")
# Clean up
shutil.rmtree(tmp_dir)
def test_download_shub_image_on_sucess(self, mock_request, mock_json,
mock_content, mock_progressbar,
mock_md5):
# we need this for the fake download
tmp_dir = "/tmp/singularity-images"
if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir)
download_obj = DownloadWorkflow(pipeline="dummy", outdir="/tmp")
# simulauates the first response querying the API for the container
# information
resp_shub = requests.Response()
resp_shub.status_code = 200
mock_json.side_effect = [{'image': 'my-container', 'version': 'h4sh'}]
# simulates the second response querying the
# container download stream
resp_download = requests.Response()
resp_download.status_code = 200
resp_download.headers = {'content-length': 1024}
mock_content.side_effect = b"Awesome"
# assign the response order
mock_request.side_effect = [resp_shub, resp_download]
# test
download_obj.download_shub_image("awesome-container")
def test_download_configs(self, outdir):
download_obj = DownloadWorkflow(pipeline="nf-core/methylseq",
revision="1.6")
download_obj.outdir = outdir
download_obj.download_configs()
assert os.path.exists(
os.path.join(outdir, "configs", "nfcore_custom.config"))
def test_download_image_shub_wrong_url(self, mock_request, mock_json):
# we need this for the fake download
tmp_dir = "/tmp/singularity-images"
if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir)
download_obj = DownloadWorkflow(pipeline="dummy",
outdir=tempfile.mkdtemp())
# simulauates the first response querying the API for the container
# information
resp_shub = requests.Response()
resp_shub.status_code = 200
mock_json.side_effect = [{'image': 'my-container', 'version': 'h4sh'}]
# simulates the second response querying the
# container download stream
resp_download = requests.Response()
resp_download.status_code = 404
# assign the response order
mock_request.side_effect = [resp_shub, resp_download]
# test
download_obj.download_shub_image("awesome-container")
# Clean up
shutil.rmtree(tmp_dir)
def test_singularity_pull_image(self, mock_rich_progress):
tmp_dir = tempfile.mkdtemp()
download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_dir)
download_obj.singularity_pull_image("a-container", tmp_dir, None,
mock_rich_progress)
# Clean up
shutil.rmtree(tmp_dir)
def test_fetch_workflow_details_for_dev_version(self, mock_workflows,
mock_workflow):
download_obj = DownloadWorkflow(pipeline="dummy")
mock_workflow.name = "dummy"
mock_workflow.releases = []
mock_workflows.remote_workflows = [mock_workflow]
download_obj.fetch_workflow_details(mock_workflows)
def test_download_configs(self):
outdir = tempfile.mkdtemp()
download_obj = DownloadWorkflow(pipeline="nf-core/methylseq",
release="1.6")
download_obj.outdir = outdir
download_obj.download_configs()
assert os.path.exists(
os.path.join(outdir, "configs", "nfcore_custom.config"))
def test_find_container_images(self, tmp_path, mock_fetch_wf_config):
download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path)
mock_fetch_wf_config.return_value = {
"process.mapping.container": "cutting-edge-container",
"process.nocontainer": "not-so-cutting-edge",
}
download_obj.find_container_images()
assert len(download_obj.containers) == 1
assert download_obj.containers[0] == "cutting-edge-container"
def test_find_singularity_images(self, mock_fetch_wf_config):
download_obj = DownloadWorkflow(pipeline="dummy", outdir="/tmp")
mock_fetch_wf_config.return_value = {
'process.mapping.container': 'cutting-edge-container',
'process.nocontainer': 'not-so-cutting-edge'
}
download_obj.find_singularity_images()
assert len(download_obj.containers) == 1
assert download_obj.containers[0] == 'cutting-edge-container'
def test_mismatching_md5sums(self):
download_obj = DownloadWorkflow(pipeline="dummy")
test_hash = hashlib.md5()
test_hash.update(b"other value")
val_hash = test_hash.hexdigest()
with open("/tmp/test", "w") as f:
f.write("test")
download_obj.validate_md5("/tmp/test", val_hash)
def test_download_workflow_with_success(self, mock_download_shub,
mock_download_image):
download_obj = DownloadWorkflow(pipeline="nf-core/methylseq",
outdir="/tmp/testdir",
singularity=True)
mock_download_shub.side_effect = RuntimeWarning()
download_obj.download_workflow()
def test_matching_md5sums(self, tmpfile):
download_obj = DownloadWorkflow(pipeline="dummy")
test_hash = hashlib.md5()
test_hash.update(b"test")
val_hash = test_hash.hexdigest()
with open(tmpfile.name, "w") as f:
f.write("test")
download_obj.validate_md5(tmpfile.name, val_hash)
def test_fetch_workflow_details_for_unknown_release(
self, mock_workflows, mock_workflow):
download_obj = DownloadWorkflow(pipeline="dummy", release="1.2.0")
mock_workflow.name = "dummy"
mock_workflow.releases = [{
"tag_name": "1.0.0",
"tag_sha": "n3v3rl4nd"
}]
mock_workflows.remote_workflows = [mock_workflow]
download_obj.fetch_workflow_details(mock_workflows)
def test_download_workflow_with_success(self, mock_download_image):
tmp_dir = tempfile.mkdtemp()
download_obj = DownloadWorkflow(pipeline="nf-core/methylseq",
outdir=os.path.join(tmp_dir, "new"),
singularity=True)
download_obj.download_workflow()
# Clean up
shutil.rmtree(tmp_dir)
def test_fetch_workflow_details_and_autoset_release(
self, mock_workflows, mock_workflow):
download_obj = DownloadWorkflow(pipeline="dummy")
mock_workflow.name = "dummy"
mock_workflow.releases = [{
"tag_name": "1.0.0",
"tag_sha": "n3v3rl4nd"
}]
mock_workflows.remote_workflows = [mock_workflow]
download_obj.fetch_workflow_details(mock_workflows)
assert download_obj.release == "1.0.0"
def test_get_release_hash_non_existent_release(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
pipeline = "methylseq"
download_obj = DownloadWorkflow(pipeline=pipeline,
release="thisisfake")
(
download_obj.pipeline,
download_obj.wf_releases,
download_obj.wf_branches,
) = nf_core.utils.get_repo_releases_branches(pipeline, wfs)
download_obj.get_release_hash()
def test_download_workflow_with_success(self, tmp_dir, mock_download_image,
mock_singularity_installed):
os.environ["NXF_SINGULARITY_CACHEDIR"] = "foo"
download_obj = DownloadWorkflow(
pipeline="nf-core/methylseq",
outdir=os.path.join(tmp_dir, "new"),
container="singularity",
revision="1.6",
compress_type="none",
)
download_obj.download_workflow()
def test_mismatching_md5sums(self):
download_obj = DownloadWorkflow(pipeline="dummy")
test_hash = hashlib.md5()
test_hash.update(b"other value")
val_hash = test_hash.hexdigest()
tmpfilehandle, tmpfile = tempfile.mkstemp()
with open(tmpfile, "w") as f:
f.write("test")
download_obj.validate_md5(tmpfile[1], val_hash)
# Clean up
os.remove(tmpfile)
def test_download_workflow_with_success(self, mock_download_shub,
mock_download_image):
tmp_dir = os.path.join(tempfile.mkdtemp(), 'new')
download_obj = DownloadWorkflow(pipeline="nf-core/methylseq",
outdir=tmp_dir,
singularity=True)
mock_download_shub.side_effect = RuntimeWarning()
download_obj.download_workflow()
# Clean up
shutil.rmtree(tmp_dir)
def test_get_release_hash_release(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
pipeline = "methylseq"
download_obj = DownloadWorkflow(pipeline=pipeline, release="1.6")
(
download_obj.pipeline,
download_obj.wf_releases,
download_obj.wf_branches,
) = nf_core.utils.get_repo_releases_branches(pipeline, wfs)
download_obj.get_release_hash()
assert download_obj.wf_sha == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395"
assert download_obj.outdir == "nf-core-methylseq-1.6"
assert (
download_obj.wf_download_url ==
"https://github.com/nf-core/methylseq/archive/b3e5e3b95aaf01d98391a62a10a3990c0a4de395.zip"
)
def test_download_image_shub_without_hit(self, mock_request):
# we need this for the fake download
tmp_dir = "/tmp/singularity-images"
if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir)
download_obj = DownloadWorkflow(pipeline="dummy", outdir="/tmp")
# simulauates the first response querying the API for the container
# information
resp_shub = requests.Response()
resp_shub.status_code = 404
# assign the response order
mock_request.side_effect = [resp_shub]
# test
download_obj.download_shub_image("awesome-container")
def test_get_release_hash_branch(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
# Exoseq pipeline is archived, so `dev` branch should be stable
pipeline = "exoseq"
download_obj = DownloadWorkflow(pipeline=pipeline, release="dev")
(
download_obj.pipeline,
download_obj.wf_releases,
download_obj.wf_branches,
) = nf_core.utils.get_repo_releases_branches(pipeline, wfs)
download_obj.get_release_hash()
assert download_obj.wf_sha == "819cbac792b76cf66c840b567ed0ee9a2f620db7"
assert download_obj.outdir == "nf-core-exoseq-dev"
assert (
download_obj.wf_download_url ==
"https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip"
)
def test_download_workflow_with_success(self, mock_download_image,
mock_singularity_installed):
tmp_dir = tempfile.mkdtemp()
os.environ["NXF_SINGULARITY_CACHEDIR"] = "foo"
download_obj = DownloadWorkflow(
pipeline="nf-core/methylseq",
outdir=os.path.join(tmp_dir, "new"),
container="singularity",
release="1.6",
compress_type="none",
)
download_obj.download_workflow()
# Clean up
shutil.rmtree(tmp_dir)
def test_download_wf_files(self):
download_obj = DownloadWorkflow(pipeline="dummy",
release="1.2.0",
outdir=tempfile.mkdtemp())
download_obj.wf_name = "nf-core/methylseq"
download_obj.wf_sha = "1.0"
download_obj.wf_download_url = "https://github.com/nf-core/methylseq/archive/1.0.zip"
download_obj.download_wf_files()
def test_download_wf_files(self, outdir):
download_obj = DownloadWorkflow(pipeline="nf-core/methylseq",
revision="1.6")
download_obj.outdir = outdir
download_obj.wf_sha = "b3e5e3b95aaf01d98391a62a10a3990c0a4de395"
download_obj.wf_download_url = (
"https://github.com/nf-core/methylseq/archive/b3e5e3b95aaf01d98391a62a10a3990c0a4de395.zip"
)
download_obj.download_wf_files()
assert os.path.exists(os.path.join(outdir, "workflow", "main.nf"))
def test_wf_use_local_configs(self):
# Get a workflow and configs
test_outdir = tempfile.mkdtemp()
download_obj = DownloadWorkflow(pipeline="dummy",
release="1.2.0",
outdir=test_outdir)
shutil.copytree(PATH_WORKING_EXAMPLE,
os.path.join(test_outdir, 'workflow'))
download_obj.download_configs()
# Test the function
download_obj.wf_use_local_configs()
wf_config = nf_core.utils.fetch_wf_config(
os.path.join(test_outdir, 'workflow'))
assert wf_config['params.custom_config_base'] == "'../configs/'"
def test_wf_use_local_configs(self):
# Get a workflow and configs
test_pipeline_dir = os.path.join(tempfile.mkdtemp(),
"nf-core-testpipeline")
create_obj = nf_core.create.PipelineCreate("testpipeline",
"This is a test pipeline",
"Test McTestFace",
outdir=test_pipeline_dir)
create_obj.init_pipeline()
test_outdir = tempfile.mkdtemp()
download_obj = DownloadWorkflow(pipeline="dummy",
release="1.2.0",
outdir=test_outdir)
shutil.copytree(test_pipeline_dir,
os.path.join(test_outdir, "workflow"))
download_obj.download_configs()
# Test the function
download_obj.wf_use_local_configs()
wf_config = nf_core.utils.fetch_wf_config(
os.path.join(test_outdir, "workflow"))
assert wf_config["params.custom_config_base"] == "'../configs/'"
def test_wf_use_local_configs(self, tmp_path):
# Get a workflow and configs
test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline")
create_obj = nf_core.create.PipelineCreate("testpipeline",
"This is a test pipeline",
"Test McTestFace",
outdir=test_pipeline_dir)
create_obj.init_pipeline()
with tempfile.TemporaryDirectory() as test_outdir:
download_obj = DownloadWorkflow(pipeline="dummy",
revision="1.2.0",
outdir=test_outdir)
shutil.copytree(test_pipeline_dir,
os.path.join(test_outdir, "workflow"))
download_obj.download_configs()
# Test the function
download_obj.wf_use_local_configs()
wf_config = nf_core.utils.fetch_wf_config(os.path.join(
test_outdir, "workflow"),
cache_config=False)
assert wf_config[
"params.custom_config_base"] == f"'{test_outdir}/workflow/../configs/'"