def getoptions():
""" process comandline options """
retdata = {
"log": logging.INFO,
"resource": None,
"config": None,
"mindate": datetime.now() - timedelta(days=3)
}
opts, _ = getopt(sys.argv[1:], "r:c:m:adqh", ["resource=", "config=", "mindate=", "all", "debug", "quiet", "help"])
for opt in opts:
if opt[0] in ("-r", "--resource"):
retdata['resource'] = opt[1]
elif opt[0] in ("-d", "--debug"):
retdata['log'] = logging.DEBUG
elif opt[0] in ("-q", "--quiet"):
retdata['log'] = logging.ERROR
elif opt[0] in ("-c", "--config"):
retdata['config'] = opt[1]
elif opt[0] in ("-m", "--mindate"):
retdata['mindate'] = parsetime(opt[1])
elif opt[0] in ("-a", "--all"):
retdata['mindate'] = None
elif opt[0] in ("-h", "--help"):
usage()
sys.exit(0)
return retdata
def getoptions():
""" process comandline options """
retdata = {
"log": logging.INFO,
"resource": None,
"config": None,
"debugfile": None,
"mindate": datetime.now() - timedelta(days=DAY_DELTA),
"maxdate": datetime.now() - timedelta(minutes=10),
"keep_csv": False,
"num_threads": 1
}
opts, _ = getopt(
sys.argv[1:],
"r:c:m:M:D:adqht:k",
["resource=", "config=", "mindate=", "maxdate=", "debugfile", "all", "debug", "quiet", "help", "threads=", "keep-csv"]
)
for opt in opts:
if opt[0] in ("-r", "--resource"):
retdata['resource'] = opt[1]
elif opt[0] in ("-d", "--debug"):
retdata['log'] = logging.DEBUG
elif opt[0] in ("-q", "--quiet"):
retdata['log'] = logging.ERROR
elif opt[0] in ("-c", "--config"):
retdata['config'] = opt[1]
elif opt[0] in ("-m", "--mindate"):
retdata['mindate'] = parsetime(opt[1])
elif opt[0] in ("-M", "--maxdate"):
retdata['maxdate'] = parsetime(opt[1])
elif opt[0] in ("-D", "--debugfile"):
retdata["debugfile"] = opt[1]
elif opt[0] in ("-a", "--all"):
retdata['mindate'] = None
retdata['maxdate'] = None
elif opt[0] in ("-t", "--threads"):
retdata["num_threads"] = int(opt[1])
elif opt[0] in ("-k", "--keep-csv"):
retdata["keep_csv"] = True
elif opt[0] in ("-h", "--help"):
usage()
sys.exit(0)
return retdata
def getoptions(has_mpi):
""" process comandline options """
localjobid = None
resource = None
starttime = None
endtime = None
joboutdir = None
retdata = {
"log": logging.INFO,
"threads": 1,
"dodelete": True,
"extractonly": False,
"libextract": False,
"process_all": False,
"process_bad": False,
"process_old": False,
"process_notdone": False,
"process_current": False,
"process_big": False,
"process_error": 0,
"max_nodes": 0,
"max_nodetime": None,
"min_duration": None,
"min_parallel_duration": None,
"max_duration": 0,
"job_output_dir": None,
"tag": None,
"dump_proclist": False,
"force_timeout": 2 * 24 * 3600,
"resource": None,
"dry_run": False,
"fail_fast": False
}
opts, _ = getopt(sys.argv[1:], "ABONCbP:M:j:r:t:dqs:e:LT:t:D:Eo:hn", [
"localjobid=", "resource=", "threads=", "debug", "quiet", "start=",
"end=", "process-all", "process-bad", "process-old", "process-notdone",
"process-current", "process-big", "process-error=", "max-nodes=",
"max-nodetime=", "min-duration=", "min-parallel-duration=",
"max-duration=", "timeout=", "dump-proclist", "tag=", "delete=",
"extract-only", "use-lib-extract", "output=", "help", "dry-run",
"fail-fast"
])
for opt in opts:
if opt[0] in ("-j", "--localjobid"):
localjobid = opt[1]
if opt[0] in ("-r", "--resource"):
resource = opt[1]
if opt[0] in ("-d", "--debug"):
retdata['log'] = logging.DEBUG
if opt[0] in ("-q", "--quiet"):
retdata['log'] = logging.ERROR
if opt[0] in ("-t", "--threads"):
retdata['threads'] = int(opt[1])
if opt[0] in ("-s", "--start"):
starttime = parsetime(opt[1])
if opt[0] in ("-e", "--end"):
endtime = parsetime(opt[1])
if opt[0] in ("-A", "--process-all"):
retdata['process_all'] = True
if opt[0] in ("-B", "--process-bad"):
retdata['process_bad'] = True
if opt[0] in ("-O", "--process-old"):
retdata['process_old'] = True
if opt[0] in ("-N", "--process-notdone"):
retdata['process_notdone'] = True
if opt[0] in ("-C", "--process-current"):
retdata['process_current'] = True
if opt[0] in ("-b", "--process-big"):
retdata['process_big'] = True
if opt[0] in ("-P", "--process-error"):
retdata['process_error'] = int(opt[1])
if opt[0] in ("-L", "--use-lib-extract"):
retdata['libextract'] = True
if opt[0] in ("-M", "--max-nodes"):
retdata['max_nodes'] = int(opt[1])
if opt[0] == "--max-nodetime":
retdata['max_nodetime'] = int(opt[1])
if opt[0] == "--min-duration":
retdata['min_duration'] = int(opt[1])
if opt[0] == "--min-parallel-duration":
retdata['min_parallel_duration'] = int(opt[1])
if opt[0] == "--max-duration":
retdata['max_duration'] = int(opt[1])
if opt[0] in ("-T", "--timeout"):
retdata['force_timeout'] = int(opt[1])
if opt[0] == "--tag":
retdata['tag'] = str(opt[1])
if opt[0] == '--dump-proclist':
retdata['dump_proclist'] = True
if opt[0] in ("-D", "--delete"):
retdata['dodelete'] = True if opt[1].upper().startswith(
"T") else False
if opt[0] in ("-E", "--extract-only"):
retdata['extractonly'] = True
if opt[0] in ("-o", "--output"):
joboutdir = opt[1]
if opt[0] in ("-n", "--dry-run"):
retdata["dry_run"] = True
if opt[0] == "--fail-fast":
retdata["fail_fast"] = True
if opt[0] in ("-h", "--help"):
usage(has_mpi)
sys.exit(0)
if retdata['extractonly']:
# extract-only supresses archive delete
retdata['dodelete'] = False
# If all options selected, treat as all to optimize the job selection query
if retdata['process_bad'] and retdata['process_old'] and retdata[
'process_notdone'] and retdata['process_current']:
retdata['process_all'] = True
if not (starttime == None and endtime == None):
if starttime == None or endtime == None:
usage(has_mpi)
sys.exit(1)
retdata.update({
"mode": "timerange",
"start": starttime,
"end": endtime,
"resource": resource
})
# Preserve the existing mode where just specifying a timerange does all jobs
if not retdata['process_bad'] and not retdata[
'process_old'] and not retdata[
'process_notdone'] and not retdata[
'process_current'] and not retdata[
'process_big'] and retdata['process_error'] == 0:
retdata['process_all'] = True
return retdata
else:
if not retdata['process_bad'] and not retdata[
'process_old'] and not retdata[
'process_notdone'] and not retdata[
'process_current'] and not retdata[
'process_big'] and retdata['process_error'] == 0:
# Preserve the existing mode where unprocessed jobs are selected when no time range given
retdata['process_bad'] = True
retdata['process_old'] = True
retdata['process_notdone'] = True
if (retdata['process_bad'] and retdata['process_old']
and retdata['process_notdone']
and retdata['process_current']) or retdata['process_all']:
# Sanity checking to not do every job in the DB
logging.error("Cannot process all jobs without a time range")
sys.exit(1)
if localjobid == None and resource == None:
retdata.update({"mode": "all"})
return retdata
if localjobid != None and resource != None:
retdata.update({
"mode": "single",
"local_job_id": localjobid,
"resource": resource,
"job_output_dir": joboutdir
})
return retdata
if resource != None:
retdata.update({"mode": "resource", "resource": resource})
return retdata
usage(has_mpi)
sys.exit(1)
def getoptions():
""" process comandline options """
localjobid = None
resource = None
starttime = None
endtime = None
joboutdir = None
retdata = {
"log": logging.INFO,
"dodelete": True,
"extractonly": False,
"job_output_dir": None,
"force_timeout": 2 * 24 * 3600,
"resource": None
}
opts, _ = getopt(sys.argv[1:], "j:r:dqs:e:T:D:Eo:h",
["localjobid=",
"resource=",
"debug",
"quiet",
"start=",
"end=",
"timeout=",
"delete=",
"extract-only",
"output=",
"help"])
for opt in opts:
if opt[0] in ("-j", "--jobid"):
localjobid = opt[1]
if opt[0] in ("-r", "--resource"):
resource = opt[1]
if opt[0] in ("-d", "--debug"):
retdata['log'] = logging.DEBUG
if opt[0] in ("-q", "--quiet"):
retdata['log'] = logging.ERROR
if opt[0] in ("-s", "--start"):
starttime = parsetime(opt[1])
if opt[0] in ("-e", "--end"):
endtime = parsetime(opt[1])
if opt[0] in ("-T", "--timeout"):
retdata['force_timeout'] = int(opt[1])
if opt[0] in ("-D", "--delete"):
retdata['dodelete'] = True if opt[1].upper().startswith("T") else False
if opt[0] in ("-E", "--extract-only"):
retdata['extractonly'] = True
if opt[0] in ("-o", "--output"):
joboutdir = opt[1]
if opt[0] in ("-h", "--help"):
usage()
sys.exit(0)
if retdata['extractonly']:
# extract-only supresses archive delete
retdata['dodelete'] = False
if not (starttime == None and endtime == None):
if starttime == None or endtime == None:
usage()
sys.exit(1)
retdata.update({"mode": "timerange", "start": starttime, "end": endtime, "resource": resource})
return retdata
if localjobid == None and resource == None:
retdata.update({"mode": "all"})
return retdata
if localjobid != None and resource != None:
retdata.update({"mode": "single", "local_job_id": localjobid, "resource": resource, "job_output_dir": joboutdir})
return retdata
if resource != None:
retdata.update({"mode": "resource", "resource": resource})
return retdata
usage()
sys.exit(1)
def getoptions():
""" process comandline options """
localjobid = None
resource = None
starttime = None
endtime = None
joboutdir = None
retdata = {
"log": logging.INFO,
"threads": 1,
"dodelete": True,
"extractonly": False,
"job_output_dir": None,
"force_timeout": 2 * 24 * 3600,
"resource": None
}
opts, _ = getopt(sys.argv[1:], "j:r:t:dqs:e:T:D:Eo:h",
["localjobid=",
"resource=",
"threads=",
"debug",
"quiet",
"start=",
"end=",
"timeout=",
"delete=",
"extract-only",
"output=",
"help"])
for opt in opts:
if opt[0] in ("-j", "--jobid"):
localjobid = opt[1]
if opt[0] in ("-r", "--resource"):
resource = opt[1]
if opt[0] in ("-d", "--debug"):
retdata['log'] = logging.DEBUG
if opt[0] in ("-q", "--quiet"):
retdata['log'] = logging.ERROR
if opt[0] in ("-t", "--threads"):
retdata['threads'] = int(opt[1])
if opt[0] in ("-s", "--start"):
starttime = parsetime(opt[1])
if opt[0] in ("-e", "--end"):
endtime = parsetime(opt[1])
if opt[0] in ("-T", "--timeout"):
retdata['force_timeout'] = int(opt[1])
if opt[0] in ("-D", "--delete"):
retdata['dodelete'] = True if opt[1].upper().startswith("T") else False
if opt[0] in ("-E", "--extract-only"):
retdata['extractonly'] = True
if opt[0] in ("-o", "--output"):
joboutdir = opt[1]
if opt[0] in ("-h", "--help"):
usage()
sys.exit(0)
if retdata['extractonly']:
# extract-only supresses archive delete
retdata['dodelete'] = False
if not (starttime == None and endtime == None):
if starttime == None or endtime == None:
usage()
sys.exit(1)
retdata.update({"mode": "timerange", "start": starttime, "end": endtime, "resource": resource})
return retdata
if localjobid == None and resource == None:
retdata.update({"mode": "all"})
return retdata
if localjobid != None and resource != None:
retdata.update({"mode": "single", "local_job_id": localjobid, "resource": resource, "job_output_dir": joboutdir})
return retdata
if resource != None:
retdata.update({"mode": "resource", "resource": resource})
return retdata
usage()
sys.exit(1)