def test_cleanup_multiple(mock_delete, mock_list):
"""
Test that the correct number of function calls are made.
"""
jobs = {
"lbowconf": {
"recoveryfile": "rec.file"
},
"jobone": {
"destdir": "/path/to/jobone12484",
"remoteworkdir": "/path/to/local/dir",
},
"jobtwo": {
"destdir": "/path/to/jobtwo12484",
"remoteworkdir": "/path/to/local/dir",
},
"jobthree": {
"destdir": "/path/to/jobthree12484",
"remoteworkdir": "/path/to/local/dir",
}
}
cleanup(jobs)
assert mock_delete.call_count == 3, \
"There is only one job, this should only be called once"
assert mock_list.call_count == 3, \
"There is only one job, this should only be called once"
def recovery(jobs, recoveryfile):
"""Recover a Longbow session.
This method is for attempting to recover a failed Longbow session or to
reconnect to an intentionally disconnected session. It will try to take the
recovery file, written shortly after submission to recover the whole
session. Once the data has been loaded from the recovery file and a new job
data structure populated, this method will then re-enter the monitoring
function to continue where it left off. Any jobs that finished in the
meantime will be marked accordingly and then file staging will continue.
Required inputs are:
recoveryfile (string): A path to the recovery file.
"""
jobfile = os.path.join(os.path.expanduser('~/.longbow'), recoveryfile)
LOG.info("Attempting to find the recovery file '{0}'".format(jobfile))
# Load the jobs recovery file.
if os.path.isfile(jobfile):
LOG.info("Recovery file found.")
_, _, jobparams = configuration.loadconfigs(jobfile)
# Copy to jobs so when exceptions are raised the structure is
# available.
for param in jobparams:
jobs[param] = jobparams[param]
else:
raise exceptions.RequiredinputError(
"Recovery file could not be found, make sure you haven't deleted "
"the recovery file and that you are not providing the full path, "
"just the file name is needed.")
# Rejoin at the monitoring stage. This will assume that all jobs that
# are no longer in the queue have completed.
scheduling.monitor(jobs)
# Cleanup the remote working directory.
staging.cleanup(jobs)
def test_cleanup_excepttest4(mock_delete, mock_list):
"""
Test that the KeyError exception is entercepted and does not percolate
up from here.
"""
jobs = {
"lbowconf": {
"recoveryfile": "rec.file"
},
"jobone": {
"destdir": "/path/to/jobone12484",
"remoteworkdir": "/path/to/local/dir",
}
}
mock_delete.return_value = None
mock_list.side_effect = NameError("blah")
cleanup(jobs)
def update(jobs, updatefile):
"""Trigger update of a disconnected Longbow session.
This method will start the update process on an existing but disconnected
Longbow session. All job statuses will be checked and updated in the
recovery file and all output files will be synced before disconnecting."""
jobfile = os.path.join(os.path.expanduser('~/.longbow'), updatefile)
LOG.info("Attempting to find the recovery file '{0}'".format(jobfile))
# Load the jobs recovery file.
if os.path.isfile(jobfile):
LOG.info("Recovery file found.")
_, _, jobparams = configuration.loadconfigs(jobfile)
# Copy to jobs so when exceptions are raised the structure is
# available.
for param in jobparams:
jobs[param] = jobparams[param]
else:
raise exceptions.RequiredinputError(
"Recovery file could not be found, make sure you haven't deleted "
"the recovery file and that you are not providing the full path, "
"just the file name is needed.")
# Add the updater key
jobs["lbowconf"]["update"] = True
# Enter monitoring loop
scheduling.monitor(jobs)
# Cleanup the remote working directory.
staging.cleanup(jobs)
def test_cleanup_nodelete(mock_delete, mock_list):
"""
Test that the following exception is handled correctly.
"""
jobs = {
"lbowconf": {
"recoveryfile": "rec.file"
},
"jobone": {
"destdir": "/path/to/jobone12484",
"remoteworkdir": "/path/to/jobone12484",
}
}
mock_list.return_value = None
mock_delete.return_value = None
cleanup(jobs)
assert mock_delete.call_count == 0, "Should not be called in this case."
def test_cleanup_recoveryfilerm3(m_delete, m_list, m_isfile, m_remove):
"""
Test that the recoveryfile would be removed.
"""
jobs = {
"lbowconf": {
"recoveryfile": "rec.file"
},
"jobone": {
"destdir": "/path/to/jobone12484",
"remoteworkdir": "/path/to/local/dir",
}
}
m_isfile.return_value = False
m_delete.return_value = None
m_list.return_value = None
cleanup(jobs)
assert m_remove.call_count == 0
def test_cleanup_params(mock_delete, mock_list):
"""
Test the correct arguments make it to the method calls.
"""
jobs = {
"lbowconf": {
"recoveryfile": "rec.file"
},
"jobone": {
"destdir": "/path/to/jobone12484",
"remoteworkdir": "/path/to/local/dir",
}
}
cleanup(jobs)
listarg1 = mock_list.call_args[0][0]
deletearg1 = mock_delete.call_args[0][0]
assert isinstance(listarg1, dict)
assert isinstance(deletearg1, dict)
def launcher():
"""Entry point for Longbow when used as an application.
This method is the main entry point for Longbow launched as an application.
Library users should not use this method when linking Longbow at a high
level. Developers doing high level linking should be calling Longbow()
directly with the parameters dictionary already setup.
This method takes the information from sys.argv and processes this into a
dictionary format ready to fire longbow().
"""
# -------------------------------------------------------------------------
# Some defaults and parameter initialisation
# Fetch command line arguments as list and remove longbow exec
commandlineargs = sys.argv
commandlineargs.pop(0)
# Initialise parameters that could alternatively be provided in
# configuration files
parameters = {
"debug": False,
"disconnect": False,
"executable": "",
"executableargs": "",
"hosts": "",
"job": "",
"jobname": "",
"log": "",
"maxtime": "",
"nochecks": False,
"recover": "",
"resource": "",
"replicates": "",
"update": "",
"verbose": False
}
# Specify all recognised longbow arguments
alllongbowargs = [
"--about", "--debug", "--disconnect", "--examples", "-h", "--help",
"--hosts", "--job", "--jobname", "--log", "--maxtime", "--nochecks",
"--recover", "--resource", "--replicates", "--update", "-V",
"--verbose", "--version"
]
# -------------------------------------------------------------------------
# Detection of commandline flags and sub functionality.
# Detect Longbow arguments, the executable and the executable arguments
# from the command-line.
longbowargs = _commandlineproc(alllongbowargs, commandlineargs, parameters)
# Check for information flags such as help or about
_messageflags(longbowargs)
# Check if user is wanting to download examples
_downloadexamples(longbowargs)
# Grab the Longbow command-line arguments and their values.
_parsecommandlineswitches(parameters, longbowargs)
# Logging should be started here, such that only users of the application
# have logging rules and filters setup. Library users will want/need to
# set up their own handlers.
_setuplogger(parameters)
# -------------------------------------------------------------------------
# Setup the top level exception handler, this handler should give the user
# nicely formatted and understandable error messages (unless run in debug
# mode).
# The top level exception handler, this level is simply for the graceful
# exit and final reporting of errors only. All actions should have been
# taken by this stage.
try:
# Log the start up message, if the user got this far then we are ok to
# properly start Longbow.
LOG.info("Welcome to Longbow!")
LOG.info("This software was developed as part of the EPSRC-funded "
"HECBioSim project (http://www.hecbiosim.ac.uk/)")
LOG.info("HECBioSim facilitates high-end biomolecular simulation "
"on resources such as ARCHER")
LOG.info("Longbow is Copyright (C) of Science and Technology "
"Facilities Council and The University of Nottingham.")
LOG.info("Longbow was created by Dr James T. Gebbie-Rayet, Dr Gareth "
"B. Shannon and Prof Charles A. Laughton.")
LOG.info("Please cite our paper: Gebbie-Rayet, J, Shannon, G, "
"Loeffler, H H and Laughton, C A 2016 Longbow: A "
"Lightweight Remote Job Submission Tool. Journal of "
"Open Research Software, 4: e1, "
"DOI: http://dx.doi.org/10.5334/jors.95")
LOG.info("Python version: %s", PYTHONVERSION)
LOG.info("Longbow version: %s", LONGBOWVERSION)
LOG.info("Longbow Commandline: %s", (" ").join(sys.argv))
_hostfileproc(parameters)
_jobfileproc(parameters)
LOG.info("hosts file is: '%s'", parameters["hosts"])
# If no executable and jobfile has been given then fail.
if (parameters["executable"] == "" and parameters["job"] == ""
and parameters["recover"] == ""
and parameters["update"] == ""):
raise exceptions.RequiredinputError(
"There was no executable or job file given on the "
"command-line, you need to supply one or the other otherwise "
"Longbow cannot decipher what you would like to do.")
# ---------------------------------------------------------------------
# Call one of the main methods at the top level of the library.
jobs = {}
# If recovery or update mode is not active then this is a new run.
if parameters["recover"] == "" and parameters["update"] == "":
LOG.info("Initialisation complete.")
longbow(jobs, parameters)
# If recovery mode is set then start the recovery process.
elif parameters["recover"] != "" and parameters["update"] == "":
LOG.info("Starting recovery mode to reconnect monitoring of jobs.")
recovery(jobs, parameters["recover"])
# If update mode is set then start the update process.
elif parameters["recover"] == "" and parameters["update"] != "":
LOG.info("Starting update mode to refresh progress of jobs.")
update(jobs, parameters["update"])
# If too many arguments are set, we have a problem
else:
raise exceptions.CommandlineargsError(
"You have both the --recover and --update command-line flags "
"set, these cannot be used together as they enable "
"conflicting functionality. Either reconnect with persistent "
"monitoring (--recover) or reconnect to refresh the status of "
"jobs and sync current files before disconnecting again "
"(--update).")
# If the user interrupts Longbow then they are aborting the jobs, so kill
# off any running jobs and then remove the job directories. Otherwise just
# raise all other errors to the top level where in future we can attempt to
# recover.
except KeyboardInterrupt:
LOG.info("User interrupt detected.")
if len([a for a in jobs if "lbowconf" not in a]) >= 1:
LOG.info("Kill any queued or running jobs and clean up.")
# If we are exiting at this stage then we need to kill off
for item in [a for a in jobs if "lbowconf" not in a]:
job = jobs[item]
if "laststatus" in job:
# If job is not finished delete and stage.
if (job["laststatus"] != "Complete"
and job["laststatus"] != "Finished"
and job["laststatus"] != "Submit Error"):
# Kill it.
scheduling.delete(job)
# Transfer the directories as they are.
staging.stage_downstream(job)
# Job is finished then just stage.
elif job["laststatus"] != "Submit Error":
# Transfer the directories as they are.
staging.stage_downstream(job)
staging.cleanup(jobs)
# If disconnect mode is enabled then the disconnect exception is raised,
# allow to disconnect gracefully.
except exceptions.DisconnectException:
LOG.info("User specified --disconnect flag on command-line, so "
"Longbow will exit.")
LOG.info("You can reconnect this session for persistent monitoring by "
"using the recovery file:")
LOG.info("longbow --recover {0} --verbose".format(
jobs["lbowconf"]["recoveryfile"]))
LOG.info("Or an update of current progress followed by disconnecting "
"can be done using:")
LOG.info("longbow --update {0} --verbose".format(
jobs["lbowconf"]["recoveryfile"]))
# If disconnect mode is enabled then the disconnect exception is raised,
# allow to disconnect gracefully.
except exceptions.UpdateExit:
LOG.info("Update of current job progress has completed, exiting.")
LOG.info("You can reconnect this session for persistent monitoring by "
"using the recovery file:")
LOG.info("longbow --recover {0} --verbose".format(
jobs["lbowconf"]["recoveryfile"]))
LOG.info("Or an update of current progress followed by disconnecting "
"can be done using:")
LOG.info("longbow --update {0} --verbose".format(
jobs["lbowconf"]["recoveryfile"]))
# If a problem happens assign the correct level of debug logging.
except Exception as err:
if parameters["debug"] is True:
LOG.exception(err)
else:
LOG.error(err)
exit(1)
# Show nice exit message.
finally:
LOG.info("Good bye from Longbow!")
LOG.info("Check out http://www.hecbiosim.ac.uk/ for other "
"powerful biomolecular simulation software tools.")
def longbow(jobs, parameters):
"""Entry point at the top level of the Longbow library.
Being the top level method that makes calls on the Longbow library.
This is a good place to link against Longbow if a developer does not want
to link against the executable, or if low level linking is not needed or is
over-kill.
Required inputs are:
parameters (dictionary): A dictionary containing the parameters and
overrides from the command-line.
"""
# A failure at this level will result in jobs being killed off before
# escalating the exception to trigger graceful exit.
# Load configurations and initialise Longbow data structures.
jobparams = configuration.processconfigs(parameters)
# Copy to jobs so when exceptions are raised the structure is available.
for param in jobparams:
jobs[param] = jobparams[param]
# Test all connection/s specified in the job configurations
shellwrappers.checkconnections(jobs)
# Test the hosts listed in the jobs configuration file have their
# scheduler environments listed, if not then test and save them.
scheduling.checkenv(jobs, parameters["hosts"])
# Test that for the applications listed in the job configuration
# file are available and that the executable is present.
if parameters["nochecks"] is False:
applications.checkapp(jobs)
# Process the jobs command line arguments and find files for
# staging.
applications.processjobs(jobs)
# Create jobfile and add it to the list of files that needs
# uploading.
scheduling.prepare(jobs)
# Stage all of the job files along with the scheduling script.
staging.stage_upstream(jobs)
# Submit all jobs.
scheduling.submit(jobs)
# Process the disconnect function.
if parameters["disconnect"] is True:
raise exceptions.DisconnectException
# Monitor all jobs.
scheduling.monitor(jobs)
# Clean up all jobs
staging.cleanup(jobs)
