def clean():
"""
Clean jobs that satisfy parameters in current request context.
Parameters are given in request URL, they are:
'id': a list of job IDs
'name': a substring that has to be present in job names
'state': state that jobs have to be in
Returns:
status 200: A string with number of cleaned jobs.
status 401: A string with error message.
"""
try:
proxyid = getProxyId()
except errors.NoSuchProxyError:
return 'Wrong or no client certificate', 401
try:
jobids = getIDs()
except Exception:
return 'Invalid id parameter', 400
name_filter = request.args.get('name', default='')
state_filter = request.args.get('state', default='')
jmgr = jobmgr.JobManager()
numDeleted = jmgr.cleanJobs(proxyid, jobids, state_filter, name_filter)
return json.dumps(numDeleted)
def getResults():
"""
Return a .zip archive of job results folder.
Request potentionaly accepts a list of IDs but only the first one is
processed because it's easier to respond for one job.
Function creates a .zip archive of job results.
.zip archive could just be sent in response but since response is the last
thing that function does, there is no simple way of deleting archive
afterwards and cleaning the job. The solution is therefore to create an
archive, read it's binary presentation (read it as a binary file) to
memory, remove the archive and send byte stream as a file to the client.
Returns:
status 200: A .zip archive of job results.
status 4** or 500: A string with error message.
"""
try:
proxyid = getProxyId()
except errors.NoSuchProxyError:
return 'Wrong or no client certificate', 401
try:
jobids = getIDs()
except Exception:
return 'Invalid id parameter', 400
if not jobids:
return 'No job ID given', 400
jobid = [jobids[0]] # only take first job
# get job results
jmgr = jobmgr.JobManager()
results = jmgr.getJobs(proxyid, jobid)
if not results.jobdicts:
return 'Results for job not found', 404
resultDir = results.jobdicts[0]['dir']
# create archive and read bytes
# TODO: exception handling is not optimal, there should be finally
# part that closes file but that would require more understanding of
# the exceptions raised for different functions which is impossible
# to find in python documentation.
# TODO: should probably split this for more systematic error handling
try:
archivePath = shutil.make_archive(resultDir, 'zip', resultDir)
archive = open(archivePath, 'rb')
byteStream = io.BytesIO(archive.read())
archive.close()
os.remove(archivePath)
except Exception as e:
return 'Server error: {}'.format(str(e)), 500
return send_file(byteStream,
mimetype='application/zip',
as_attachment=True,
attachment_filename=archivePath.split('/')[-1])
def patch():
"""
Set jobs' state based on request parameters.
Parameter that defines operation is passed in body of request in
JSON format. It is a JSON object with a single property "arcstate",
that has to be one of possible settable states
(for instance {"arcstate": "tofetch"}).
Other parameters are passed in URL, they are:
'id': a list of job IDs
'name': a substring that has to be present in job names
'state': state that jobs have to be in
Returns:
status 200: A string with a number of affected jobs.
status 4**: A string with error message.
"""
try:
proxyid = getProxyId()
except errors.NoSuchProxyError:
return 'Wrong or no client certificate', 401
try:
jobids = getIDs()
except Exception:
return 'Invalid id parameter', 400
name_filter = request.args.get('name', default='')
state_filter = request.args.get('state', default='')
jmgr = jobmgr.JobManager()
# force ignores incomptable mimetype, silent returns None instead of
# calling on_json_loading_failed() of request object
patch = request.get_json(force=True, silent=True)
if not patch:
return 'Request data is not valid JSON', 400
try:
arcstate = patch['arcstate']
except KeyError:
return "Request data has no 'arcstate' property", 400
if arcstate == 'tofetch':
num = jmgr.fetchJobs(proxyid, jobids, name_filter)
elif arcstate == 'tocancel':
num = jmgr.killJobs(proxyid, jobids, state_filter, name_filter)
elif arcstate == 'toresubmit':
num = jmgr.resubmitJobs(proxyid, jobids, name_filter)
else:
return "'arcstate' should be either 'tofetch' or 'tocancel' or 'toresubmit'", 400
return json.dumps(num)
def stat():
"""
Return status info for jobs in JSON format.
There are several parameters that can be given in URL. Possible
filtering parameters are:
'id': a list of job IDs
'name': a substring that has to be present in job names
'state': state that jobs have to be in
There are also two parameters that define which attributes should be
returned:
'client': a list of column names from client table
'arc': a list of column names from arc table
Returns:
status 200: A JSON list of JSON objects with jobs' status info.
status 4**: A string with error message.
"""
try:
proxyid = getProxyId()
except errors.NoSuchProxyError:
return 'Wrong or no client certificate', 401
try:
jobids = getIDs()
except Exception:
return 'Invalid id parameter', 400
name_filter = request.args.get('name', default='')
state_filter = request.args.get('state', default='')
clicols = request.args.get('client', default=[])
if clicols:
clicols = clicols.split(',')
arccols = request.args.get('arc', default=[])
if arccols:
arccols = arccols.split(',')
jmgr = jobmgr.JobManager()
try:
jobdicts = jmgr.getJobStats(proxyid, jobids, state_filter, name_filter,
clicols, arccols)
except Exception as e:
# TODO: could also be server error
return str(e), 400
else:
return json.dumps(jobdicts)
def submit():
"""
Submit job from current request context.
Submission of jobs is done with multipart/form-data POST request.
Job is submitted as a form containing xRSL file (name="xrsl")
and site name (name="site").
Returns:
status 200: A string with id of submitted job.
status 4** or 500: A string with error message.
"""
try:
proxyid = getProxyId()
except errors.NoSuchProxyError:
return 'Wrong or no client certificate', 401
jmgr = jobmgr.JobManager()
site = request.form.get('site', None)
if not site:
return 'No site given', 400
xrsl_file = request.files.get('xrsl', None)
if not xrsl_file:
return 'No job description file given', 400
jobdesc = xrsl_file.read()
try:
jobmgr.checkJobDesc(jobdesc)
jobmgr.checkSite(site)
except errors.InvalidJobDescriptionError as e:
return 'Invalid job description', 400
except errors.NoSuchSiteError as e:
return 'Invalid site', 400
else:
try:
jobid = jmgr.clidb.insertJobAndDescription(jobdesc, proxyid, site)
except Exception as e:
return 'Server error: {}'.format(str(e)), 500
else:
return str(jobid)
def deleteProxies():
"""
Delete proxies from database.
Parameter has to be given in url: 'id' which is a list of proxy IDs that
should be deleted.
Function first fetches all proxies that match the DN of a certificate
from request. Then it deletes those whose IDs are in 'id' parameter.
This is done so that user cannot delete any proxies but his own.
Returns:
status 200: A string with a number of deleted proxies.
status 401: A string with error message.
"""
dn = getCertDN()
pmgr = proxymgr.ProxyManager()
jmgr = jobmgr.JobManager()
proxies = pmgr.getProxiesWithDN(dn, columns=['id'])
try:
proxyids = getIDs()
except Exception:
return 'Invalid id parameter', 400
if not proxyids:
return 'Wrong or no client certificate', 401
numDeleted = 0
for proxy in proxies:
if proxy['id'] in proxyids:
# do not remove a proxy on which jobs depend
if not jmgr.getJobStats(proxy['id'], [], '', '', clicols=['id']):
pmgr.arcdb.deleteProxy(proxy['id'])
proxyids.remove(proxy['id']) # optimize a little bit
numDeleted += 1
return json.dumps(numDeleted)
def main():
# parse arguments
parser = argparse.ArgumentParser(description='Get jobs from aCT')
parser.add_argument('-a',
'--all',
action='store_true',
help='all jobs that match other criteria')
parser.add_argument('-j',
'--jobs',
default='',
help='comma separated list of job IDs or ranges')
parser.add_argument(
'-f',
'--find',
default='',
help='get only jobs with matching (sub)string in their name')
parser.add_argument('-s',
'--state',
default='',
help='get only jobs with certain state')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='show more information')
parser.add_argument('-p',
'--proxy',
default=None,
help='custom path to proxy certificate')
parser.add_argument('-n',
'--no-clean',
action='store_true',
help='do not clean jobs')
clicommon.showHelpOnCommandOnly(parser)
args = parser.parse_args()
# logging
logFormat = "[%(asctime)s] [%(filename)s:%(lineno)d] [%(levelname)s] - %(message)s"
if args.verbose:
logging.basicConfig(format=logFormat,
level=logging.DEBUG,
stream=sys.stdout)
else:
logging.basicConfig(format=logFormat,
level=logging.DEBUG,
filename=os.devnull)
# create a list of jobs to work on
if args.all:
jobs = [] # empty means all jobs
elif args.jobs:
try:
jobs = jobmgr.getIDsFromList(args.jobs)
except InvalidJobRangeError as e:
print("error: range '{}' is not a valid range".format(e.jobRange))
sys.exit(2)
except InvalidJobIDError as e:
print("error: ID '{}' is not a valid ID".format(e.jobid))
sys.exit(3)
else:
print("error: no jobs specified (use -a or -j)")
sys.exit(10)
# get proxy ID given proxy
proxyid = clicommon.getProxyIdFromProxy(args.proxy)
# get job info
manager = jobmgr.JobManager()
try:
results = manager.getJobs(proxyid, jobs, args.state, args.find)
except TmpConfigurationError:
print('error: tmp directory not configured')
sys.exit(5)
if not results.jobdicts:
print('no jobs to get')
sys.exit(0)
# copy job results
dontRemove = []
for result in results.jobdicts:
try:
if result['dir']: # if there are job results in tmp
dst_dirname = os.path.basename(os.path.normpath(
result['name']))
dstdir = getLocalDir(dst_dirname)
shutil.copytree(result['dir'], dstdir)
print('Results stored at: {}'.format(dstdir))
else:
raise NoJobDirectoryError(result['dir'])
except NoJobDirectoryError as e:
print('error: tmp results directory {} does not exist'.format(
e.jobdir))
except TargetDirExistsError as e:
print('error: job destination {} already exists'.format(e.dstdir))
# don't clean job that could not be removed
dontRemove.append(result['id'])
# delete jobs that should not be removed from results
for jobid in dontRemove:
for result in results.jobdicts:
if result['id'] == jobid:
jobix = results.clientIDs.index(result['id'])
del results.clientIDs[jobix]
del results.arcIDs[jobix]
del results.jobdicts[jobix]
# clean jobs
if not args.no_clean:
manager.forceCleanJobs(results)
def main():
# parse arguments
parser = argparse.ArgumentParser(description='Kill jobs')
parser.add_argument('-a',
'--all',
action='store_true',
help='all jobs that match other criteria')
parser.add_argument('-j',
'--jobs',
default='',
help='comma separated list of job IDs or ranges')
parser.add_argument(
'-f',
'--find',
default='',
help='get only jobs with matching (sub)string in their name')
parser.add_argument('-s',
'--state',
default='',
help='get only jobs with certain state')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='show more information')
parser.add_argument('-p',
'--proxy',
default=None,
help='custom path to proxy certificate')
clicommon.showHelpOnCommandOnly(parser)
args = parser.parse_args()
# logging
logFormat = "[%(asctime)s] [%(filename)s:%(lineno)d] [%(levelname)s] - %(message)s"
if args.verbose:
logging.basicConfig(format=logFormat,
level=logging.DEBUG,
stream=sys.stdout)
else:
logging.basicConfig(format=logFormat,
level=logging.DEBUG,
filename=os.devnull)
# create a list of jobs to work on
if args.all:
jobs = [] # empty means all jobs
elif args.jobs:
try:
jobs = jobmgr.getIDsFromList(args.jobs)
except InvalidJobRangeError as e:
print("error: range '{}' is not a valid range".format(e.jobRange))
sys.exit(2)
except InvalidJobIDError as e:
print("error: ID '{}' is not a valid ID".format(e.jobid))
sys.exit(3)
else:
print("error: no jobs specified (use -a or -j)")
sys.exit(10)
# get proxy ID given proxy
proxyid = clicommon.getProxyIdFromProxy(args.proxy)
# kill jobs
manager = jobmgr.JobManager()
numKilled = manager.killJobs(proxyid, jobs, args.state, args.find)
print('Jobs killed: {}'.format(numKilled))
def main():
# parse arguments
parser = argparse.ArgumentParser(description='Get job info from aCT')
parser.add_argument('-a',
'--all',
action='store_true',
help='all jobs that match other criteria')
parser.add_argument(
'-j',
'--jobs',
default='',
help='ID/range(id1-id2;id1<id2)/comma separated list of IDs/ranges')
parser.add_argument(
'-f',
'--find',
default='',
help='get only jobs with matching (sub)string in their name')
parser.add_argument('-s',
'--state',
default='',
help='get only jobs with certain state')
parser.add_argument('-n',
'--name',
default='',
help='get only jobs with given name')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='show more information')
parser.add_argument('-p',
'--proxy',
default=None,
help='custom path to proxy certificate')
# arguments passed directly to arccat
parser.add_argument('-o',
'--stdout',
action='store_true',
help='show the stdout of the job (default)',
default=True)
parser.add_argument('-e',
'--stderr',
action='store_true',
help='show the stderr of the job')
#parser.add_argument('-l', '--joblog', action='store_true',
# help='show A-REX\'s error log of the job')
#parser.add_argument('-P', '--listplugins', action='store_true',
# help='list the available plugins')
#parser.add_argument('-t', '--timeout', type=int, nargs=1,
# help='timeout in seconds (default 20)', default=20)
clicommon.showHelpOnCommandOnly(parser)
args = parser.parse_args()
# logging
logFormat = "[%(asctime)s] [%(filename)s:%(lineno)d] [%(levelname)s] - %(message)s"
if args.verbose:
logging.basicConfig(format=logFormat,
level=logging.DEBUG,
stream=sys.stdout)
else:
logging.basicConfig(format=logFormat,
level=logging.DEBUG,
filename=os.devnull)
# get column names from database
manager = jobmgr.JobManager()
# create a list of jobs to work on
if args.all:
jobs = [] # empty means all jobs
elif args.jobs:
try:
jobs = jobmgr.getIDsFromList(args.jobs)
except InvalidJobRangeError as e:
print("error: range '{}' is not a valid range".format(e.jobRange))
sys.exit(2)
except InvalidJobIDError as e:
print("error: ID '{}' is not a valid ID".format(e.jobid))
sys.exit(3)
else:
print("error: no jobs specified (use -a or -j)")
sys.exit(10)
proxyid = clicommon.getProxyIdFromProxy(args.proxy)
# get ARC job IDs of jobs that match filters
try:
jobdicts = manager.getJobStats(proxyid,
jobs,
args.state,
args.find,
clicols=[],
arccols=["JobID", "StdOut", "StdErr"],
jobname=args.name)
except Exception as e:
print('error: {}'.format(str(e)))
sys.exit(9)
if not jobdicts: # no jobs so just exit
print('no jobs found that fit given filters')
sys.exit(0)
for job in jobdicts:
url = job["a_JobID"] + "/"
if args.stderr:
url += job["a_StdErr"]
elif args.stdout:
url += job["a_StdOut"]
subprocess.run(["arccp", url, "-"])
def main():
# parse arguments
parser = argparse.ArgumentParser(description='Get job info from aCT')
parser.add_argument('-a',
'--all',
action='store_true',
help='all jobs that match other criteria')
parser.add_argument(
'-j',
'--jobs',
default='',
help='ID/range(id1-id2;id1<id2)/comma separated list of IDs/ranges')
parser.add_argument(
'-f',
'--find',
default='',
help='get only jobs with matching (sub)string in their name')
parser.add_argument('-s',
'--state',
default='',
help='get only jobs with certain state')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='show more information')
parser.add_argument('-p',
'--proxy',
default=None,
help='custom path to proxy certificate')
parser.add_argument('--arc-cols',
default='JobID,State,arcstate',
help='columns from ARC table that should be fetched')
parser.add_argument(
'--client-cols',
default='id,jobname',
help='columns from client table that should be fetched')
parser.add_argument('--get-cols',
action='store_true',
help='print all available column names')
clicommon.showHelpOnCommandOnly(parser)
args = parser.parse_args()
# logging
logFormat = "[%(asctime)s] [%(filename)s:%(lineno)d] [%(levelname)s] - %(message)s"
if args.verbose:
logging.basicConfig(format=logFormat,
level=logging.DEBUG,
stream=sys.stdout)
else:
logging.basicConfig(format=logFormat,
level=logging.DEBUG,
filename=os.devnull)
# get column names from database
manager = jobmgr.JobManager()
if args.get_cols:
clientCols = manager.getClientColumns()
arcCols = manager.getArcColumns()
print('client cols:', end=' ')
for col in clientCols:
print(col, end=' ')
print()
print('arc cols:', end=' ')
for col in arcCols:
print(col, end=' ')
print()
sys.exit(0)
# create a list of jobs to work on
if args.all:
jobs = [] # empty means all jobs
elif args.jobs:
try:
jobs = jobmgr.getIDsFromList(args.jobs)
except InvalidJobRangeError as e:
print("error: range '{}' is not a valid range".format(e.jobRange))
sys.exit(2)
except InvalidJobIDError as e:
print("error: ID '{}' is not a valid ID".format(e.jobid))
sys.exit(3)
else:
print("error: no jobs specified (use -a or -j)")
sys.exit(10)
# create column lists
if not args.client_cols:
clicols = []
else:
clicols = args.client_cols.split(',')
if not args.arc_cols:
arccols = []
else:
arccols = args.arc_cols.split(',')
# get proxy ID given proxy
proxyid = clicommon.getProxyIdFromProxy(args.proxy)
# get information
try:
jobdicts = manager.getJobStats(proxyid,
jobs,
args.state,
args.find,
clicols=clicols,
arccols=arccols)
except Exception as e:
print('error: {}'.format(str(e)))
sys.exit(9)
if not jobdicts: # no jobs so just exit
sys.exit(0)
# For each column, determine biggest sized value so that output can
# be nicely formatted.
colsizes = {}
for job in jobdicts:
for key, value in job.items():
# All keys have a letter and underscore prepended, which is not
# used when printing
colsize = max(len(str(key[2:])), len(str(value)))
try:
if colsize > colsizes[key]:
colsizes[key] = colsize
except KeyError:
colsizes[key] = colsize
# Print table header
for col in clicols:
print('{:<{width}}'.format(col, width=colsizes['c_' + col]), end=' ')
for col in arccols:
print('{:<{width}}'.format(col, width=colsizes['a_' + col]), end=' ')
print()
line = ''
for value in colsizes.values():
line += '-' * value
line += '-' * (len(colsizes) - 1)
print(line)
# Print jobs
for job in jobdicts:
for col in clicols:
fullKey = 'c_' + col
txt = job.get(fullKey)
# just in case the value is a bunch of whitespace
# TODO: This (str(txt)) might not be a general fix; it is a direct
# fix for the problem encountered with
# datetime.datetime object for 'created' field that
# has to be converted to a string.
# The same fix is used for arccols below.
# TODO: This fix assumes that all job fields are properly
# convertible to string. Is that really so?
if not txt or str(txt).strip() == '': # short circuit important!
txt = "''"
print('{:<{width}}'.format(txt, width=colsizes[fullKey]), end=' ')
for col in arccols:
fullKey = 'a_' + col
txt = job.get(fullKey)
# just in case the value is a bunch of whitespace
if not txt or str(txt).strip() == '': # short circuit important!
txt = "''"
print('{:<{width}}'.format(str(txt), width=colsizes[fullKey]),
end=' ')
print()
