def stage_downstream(job):
"""Transfer all files for a job, back from the HPC machine.
A method for staging files for each job to from target HPC host. The
underlying utility behind this transfer is rsync, thus it is possible
to supply rsync file masks to blacklist unwanted large files. By default
rsync is configured to transfer blockwise and only transfer the
newest/changed blocks, this saves a lot of time during persistant staging.
Required arguments are:
job (dictionary) - A single job dictionary, this is often simply passed in
as a subset of the main jobs dictionary.
"""
LOG.info("For job '%s' staging files downstream.", job["jobname"])
# Download the whole directory with rsync.
try:
shellwrappers.download(job)
except exceptions.RsyncError:
raise exceptions.StagingError(
"Could not download a file from '{0}' to '{1}'".format(
job["destdir"], job["localworkdir"]))
LOG.info("Staging complete.")
def test_download_srcpath():
"""
Test that the absolutepatherror exception is raised for non absolute
source path.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"destdir": "source/directory/path"
}
with pytest.raises(exceptions.AbsolutepathError):
download(job)
def test_download_pathslash(mock_sendtorsync):
"""
Check that the source path has a slash appended to it if there isn't one
at the end.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"destdir": "~/destination/directory/path",
"localworkdir": "/destination/directory/path",
"download-include": "",
"download-exclude": ""
}
download(job)
callargs = mock_sendtorsync.call_args[0][1]
assert callargs.endswith("/")
def test_download_exceptiontest(mock_sendtorsync):
"""
Check that if the rsync method raises the rsync exception that it
percolates up.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"destdir": "~/destination/directory/path",
"localworkdir": "/destination/directory/path",
"download-include": "",
"download-exclude": ""
}
mock_sendtorsync.side_effect = exceptions.RsyncError("RsyncError", "Error")
with pytest.raises(exceptions.RsyncError):
download(job)
def test_download_pathformat(mock_sendtorsync):
"""
Check that the remote path is constructed properly.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"destdir": "~/destination/directory/path",
"localworkdir": "/destination/directory/path",
"download-include": "",
"download-exclude": ""
}
download(job)
callargs = mock_sendtorsync.call_args[0][1]
testargs = job["user"] + "@" + job["host"] + ":" + job["destdir"]
assert callargs == testargs