def stage_upstream(jobs):
"""Transfer files for all jobs, to a remote HPC machine.
A method for staging files for each job to the target HPC host. The
underlying utility behind this transfer is rsync, thus it is possible
to supply rsync file masks to blacklist unwanted large files. By default
rsync is configured to transfer blockwise and only transfer the
newest/changed blocks, this saves a lot of time during persistant staging.
Required arguments are:
jobs (dictionary) - The Longbow jobs data structure, see configuration.py
for more information about the format of this
structure.
"""
LOG.info("Staging files for job/s.")
for item in [a for a in jobs if "lbowconf" not in a]:
job = jobs[item]
destdir = job["destdir"]
LOG.info("Transfering files for job '%s' to host '%s'", item,
job["resource"])
try:
shellwrappers.sendtossh(job, ["mkdir -p " + destdir + "\n"])
LOG.info("Creation of directory '%s' - successful.", destdir)
except exceptions.SSHError:
LOG.error(
"Creation of directory '%s' - failed. Make sure that you "
"have write permissions at the top level of the path given.",
destdir)
raise
# Transfer files upstream.
try:
shellwrappers.upload(job)
except exceptions.RsyncError:
raise exceptions.StagingError(
"Could not stage '{0}' upstream, make sure that you have "
"supplied the correct remote working directory and that you "
"have chosen a path that you can write to.".format(
job["localworkdir"]))
LOG.info("Staging files upstream - complete.")
def test_upload_srcpath():
"""
Test that the absolutepatherror exception is raised for non absolute
source path.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"localworkdir": "source/directory/path"
}
with pytest.raises(exceptions.AbsolutepathError):
upload(job)
def test_upload_pathslash(mock_sendtorsync):
"""
Check that the source path has a slash appended to it if there isn't one
at the end.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"destdir": "~/destination/directory/path",
"localworkdir": "/destination/directory/path",
"upload-include": "",
"upload-exclude": ""
}
upload(job)
callargs = mock_sendtorsync.call_args[0][1]
assert callargs.endswith("/")
def test_upload_exceptiontest(mock_sendtorsync):
"""
Check that if the rsync method raises the rsync exception that it
percolates up.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"destdir": "~/destination/directory/path",
"localworkdir": "/destination/directory/path",
"upload-include": "",
"upload-exclude": ""
}
mock_sendtorsync.side_effect = exceptions.RsyncError("RsyncError", "Error")
with pytest.raises(exceptions.RsyncError):
upload(job)
def test_upload_pathformat(mock_sendtorsync):
"""
Check that the remote path is constructed properly.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"destdir": "~/destination/directory/path",
"localworkdir": "/destination/directory/path",
"upload-include": "",
"upload-exclude": ""
}
upload(job)
callargs = mock_sendtorsync.call_args[0][2]
testargs = job["user"] + "@" + job["host"] + ":" + job["destdir"]
assert callargs == testargs