def __init__(self, name, display_prefix):
"""
Constructor for a Registry Slice object
Args:
name (str): Slightly descriptive name of the slice
display_prefix (str): Prefix for slice to help modify display and other string reps of the object
"""
super(RegistrySlice, self).__init__()
self.objects = oDict()
self.name = name
self._display_prefix = display_prefix
self._display_columns = config[self._display_prefix + '_columns']
self._display_columns_show_empty = config[self._display_prefix +
"_columns_show_empty"]
self._display_columns_width = config[self._display_prefix +
"_columns_width"]
self._display_columns_functions = {}
try:
col_funcs = config[self._display_prefix + '_columns_functions']
for this_col_func in col_funcs:
self._display_columns_functions[this_col_func] = eval(
col_funcs[this_col_func])
except Exception as x:
logger.error(
"Error on evaluating display column functions from config file: %s: %s"
% (getName(x), x))
from Ganga.Utility.ColourText import Effects
self._colour_normal = Effects().normal
self._proxyClass = None
def __init__(self, name):
super(TaskRegistrySlice, self).__init__(name, display_prefix="tasks")
from Ganga.Utility.ColourText import Foreground, Background, Effects
fg = Foreground()
fx = Effects()
bg = Background()
self.status_colours = {
'new': fx.normal,
'submitted': fg.orange,
'running': fg.green,
'completed': fg.blue,
'failed': fg.red
}
self.fx = fx
self._proxyClass = TaskRegistrySliceProxy
def __init__(self, name):
super(JobRegistrySlice, self).__init__(name, display_prefix="jobs")
from Ganga.Utility.ColourText import Foreground, Background, Effects
fg = Foreground()
fx = Effects()
bg = Background()
try:
status_colours = config['jobs_status_colours']
self.status_colours = dict([(k, eval(v)) for k, v in status_colours.iteritems()])
except Exception as x:
logger.warning('configuration problem with colour specification: "%s"', str(x))
status_colours = config.options['jobs_status_colours'].default_value
self.status_colours = dict([(k, eval(v)) for k, v in status_colours.iteritems()])
self.fx = fx
self._proxyClass = JobRegistrySliceProxy
def __init__(self, name):
super(TransientRegistrySlice, self).__init__(
name, display_prefix="box")
from Ganga.Utility.ColourText import Foreground, Background, Effects
fg = Foreground()
fx = Effects()
bg = Background()
self.fx = fx
self.name = 'box' # needed to ensure that select works properly
self.status_colours = {'default': fx.normal,
'JobTemplate': fg.orange,
'Task': fg.green,
'Job': fg.blue}
self._display_columns_functions["id"] = lambda obj: obj.id
self._display_columns_functions["type"] = lambda obj: obj._name
self._display_columns_functions["name"] = lambda obj: obj.name
self._proxyClass = TransientRegistrySliceProxy
def __init__(self, name):
super(BoxRegistrySlice, self).__init__(name, display_prefix="box")
self._display_columns_functions["id"] = lambda obj: obj._getRegistry(
).find(obj)
self._display_columns_functions["type"] = lambda obj: obj._name
self._display_columns_functions["name"] = lambda obj: obj._getRegistry(
)._getName(obj)
from Ganga.Utility.ColourText import Foreground, Background, Effects
fg = Foreground()
fx = Effects()
bg = Background()
self.fx = fx
self.status_colours = {
'default': fx.normal,
'backends': fg.orange,
'applications': fg.green,
'jobs': fg.blue
}
self._proxyClass = BoxRegistrySliceProxy
def __init__(self, name, display_prefix):
super(RegistrySlice, self).__init__()
self.objects = oDict()
self.name = name
self._display_prefix = display_prefix
self._display_columns = config[self._display_prefix + '_columns']
self._display_columns_show_empty = config[self._display_prefix +
"_columns_show_empty"]
self._display_columns_width = config[self._display_prefix +
"_columns_width"]
self._display_columns_functions = {}
try:
col_funcs = config[self._display_prefix + '_columns_functions']
for this_col_func in col_funcs:
self._display_columns_functions[this_col_func] = eval(
col_funcs[this_col_func])
except Exception as x:
logger.error(
"Error on evaluating display column functions from config file: %s: %s"
% (getName(x), x))
from Ganga.Utility.ColourText import Effects
self._colour_normal = Effects().normal
self._proxyClass = None
from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import DQ2Dataset
from Ganga.GPIDev.Lib.Tasks.common import *
from Ganga.GPIDev.Lib.Tasks import Task
from MultiTransform import MultiTransform
from Ganga.Utility.ColourText import overview_colours, ANSIMarkup, fgcol, Effects
markup = ANSIMarkup()
fx = Effects()
from Ganga.Core.exceptions import ApplicationConfigurationError
from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import dq2_lock, dq2
from GangaAtlas.Lib.Credentials.ProxyHelper import getNickname
from dq2.clientapi.DQ2 import DQ2, DQUnknownDatasetException, DQDatasetExistsException, DQFileExistsInDatasetException, DQInvalidRequestException
from dq2.container.exceptions import DQContainerAlreadyHasDataset, DQContainerDoesNotHaveDataset
from dq2.common.DQException import DQException
from Ganga.GPIDev.Schema import *
import copy
o = [""]
def c(s):
return markup(s, fgcol("blue"))
o.append(markup(" *** Task for Multiple Athena Analyses ***", fgcol("blue")))
#o.append(" Analysis Application : "+c("t.analysis.application"))
#o.append(" Set Dataset : "+c("t.setDataset('my.dataset')"))
#o.append(" Input Dataset Object : "+c("t.analysis.inputdata"))
def _display(self, interactive=0):
"""Prints content of the shareref metadata in a well formatted way.
"""
if len(self.__getName()) > 0:
from Ganga.GPIDev.Lib.File import getSharedPath
fstring = " %48s | %20s | %15s"
disp_string = fstring % ("Shared directory", "Date created",
"Reference count\n")
# print fstring % (" ", " "," ")
disp_string += fstring % (
"------------------------------------------------",
"--------------------", "---------------\n")
zero_ref = False
unsorted = []
all_elements = copy.deepcopy(self.__getName())
for element in all_elements:
full_shareddir_path = os.path.join(getSharedPath(),
os.path.basename(element))
if os.path.isdir(full_shareddir_path):
unsorted.append(
shareref_data(
os.path.basename(element),
int(os.path.getctime(full_shareddir_path)),
self.__getName()[element]))
else:
unsorted.append(
shareref_data(os.path.basename(element),
"Directory not found",
self.__getName()[element]))
decorated = sorted(
(name.date, i, name) for i, name in enumerate(unsorted))
sorted_refs = [name for date, i, name in decorated]
for line in sorted_refs:
if isinstance(line.date, int):
tmp_string = fstring % (
os.path.basename(line.name),
time.strftime("%d %b %Y %H:%M:%S",
time.localtime(line.date)), line.counter)
else:
tmp_string = fstring % (os.path.basename(
line.name), line.date, line.counter)
if (line.counter == 0) or (isinstance(line.date, str)):
from Ganga.Utility.ColourText import ANSIMarkup, NoMarkup, Foreground, Background, Effects
fg = Foreground()
fg = Background()
fx = Effects()
if interactive:
m = ANSIMarkup()
else:
m = NoMarkup()
disp_string += fg.red + tmp_string + fx.normal + '\n'
#disp_string += m(tmp_string,code=fg.red)
if (line.counter == 0):
zero_ref = True
else:
disp_string += tmp_string + '\n'
disp_string += "\nThe shared directory repository is rooted at " + \
getSharedPath() + "\n"
if zero_ref:
disp_string += "\nShared directories with a zero reference count will be removed when Ganga exits.\n"
else:
disp_string = "No objects stored in the shared directory."
return disp_string
def updateMonitoringInformation(jobs):
jobDict = {}
for job in jobs:
if job.backend.id:
jobDict[job.backend.id] = job
idList = jobDict.keys()
if not idList:
return
queryCommand = " ".join\
([
"condor_q -global" if getConfig(
"Condor")["query_global_queues"] else "condor_q",
"-format \"%s \" GlobalJobId",
"-format \"%s \" RemoteHost",
"-format \"%d \" JobStatus",
"-format \"%f\\n\" RemoteUserCpu"
])
status, output = commands.getstatusoutput(queryCommand)
if 0 != status:
logger.error("Problem retrieving status for Condor jobs")
return
if ("All queues are empty" == output):
infoList = []
else:
infoList = output.split("\n")
allDict = {}
for infoString in infoList:
tmpList = infoString.split()
id, host, status, cputime = ("", "", "", "")
if 3 == len(tmpList):
id, status, cputime = tmpList
if 4 == len(tmpList):
id, host, status, cputime = tmpList
if id:
allDict[id] = {}
allDict[id]["status"] = Condor.statusDict[status]
allDict[id]["cputime"] = cputime
allDict[id]["host"] = host
fg = Foreground()
fx = Effects()
status_colours = {'submitted': fg.orange,
'running': fg.green,
'completed': fg.blue}
for id in idList:
printStatus = False
if jobDict[id].status == "killed":
continue
localId = id.split("#")[-1]
globalId = id
if globalId == localId:
queryCommand = " ".join\
([
"condor_q -global" if getConfig(
"Condor")["query_global_queues"] else "condor_q",
"-format \"%s\" GlobalJobId",
id
])
status, output = commands.getstatusoutput(queryCommand)
if 0 == status:
globalId = output
if globalId in allDict.keys():
status = allDict[globalId]["status"]
host = allDict[globalId]["host"]
cputime = allDict[globalId]["cputime"]
if status != jobDict[id].backend.status:
printStatus = True
stripProxy(jobDict[id])._getWriteAccess()
jobDict[id].backend.status = status
if jobDict[id].backend.status == "Running":
jobDict[id].updateStatus("running")
if host:
if jobDict[id].backend.actualCE != host:
jobDict[id].backend.actualCE = host
jobDict[id].backend.cputime = cputime
else:
jobDict[id].backend.status = ""
outDir = jobDict[id].getOutputWorkspace().getPath()
condorLogPath = "".join([outDir, "condorLog"])
checkExit = True
if os.path.isfile(condorLogPath):
checkExit = False
for line in open(condorLogPath):
if -1 != line.find("terminated"):
checkExit = True
break
if -1 != line.find("aborted"):
checkExit = True
break
if checkExit:
printStatus = True
stdoutPath = "".join([outDir, "stdout"])
jobStatus = "failed"
if os.path.isfile(stdoutPath):
with open(stdoutPath) as stdout:
lineList = stdout.readlines()
try:
exitLine = lineList[-1]
exitCode = exitLine.strip().split()[-1]
except IndexError:
exitCode = -1
if exitCode.isdigit():
jobStatus = "completed"
else:
logger.error("Problem extracting exit code from job %s. Line found was '%s'." % (
jobDict[id].fqid, exitLine))
jobDict[id].updateStatus(jobStatus)
if printStatus:
if jobDict[id].backend.actualCE:
hostInfo = jobDict[id].backend.actualCE
else:
hostInfo = "Condor"
status = jobDict[id].status
if status in status_colours:
colour = status_colours[status]
else:
colour = fg.magenta
if "submitted" == status:
preposition = "to"
else:
preposition = "on"
if jobDict[id].backend.status:
backendStatus = "".join\
([" (", jobDict[id].backend.status, ") "])
else:
backendStatus = ""
logger.info(colour + 'Job %s %s%s %s %s - %s' + fx.normal,
jobDict[
id].fqid, status, backendStatus, preposition, hostInfo,
time.strftime('%c'))
return None
def list_cached_files(self, loop=True, opts=''):
"""
Lists the uploaded files.
if loop = True, it prints also the uploaded files associated with subjobs.
"""
fc = 0
ds = ''
doColoring = True
fg = Foreground()
fx = Effects()
status_colors = {'inuse': fg.orange, 'free': fg.blue, 'gone': fg.red}
status_mapping = {
'new': 'inuse',
'submitted': 'inuse',
'submitting': 'inuse',
'running': 'inuse',
'completed': 'free',
'completing': 'free',
'failed': 'free',
'killed': 'free'
}
if doColoring:
markup = ANSIMarkup()
else:
markup = NoMarkup()
def __markup_by_status__(fileIndex, counter, status):
fmtStr = '\n%4d\t%-30s\t%-12s\t%s' % (counter, fileIndex.name,
status, fileIndex.id)
try:
return markup(fmtStr, status_colors[status])
except KeyError:
return markup(fmtStr, fx.normal)
j = self.getJobObject()
for f in self.get_cached_files(opts=opts):
my_status = 'unknown'
if j:
try:
my_status = status_mapping[j.status]
except KeyError:
pass
ds += __markup_by_status__(f, fc, my_status)
fc += 1
if j and loop:
for sj in j.subjobs:
for f in sj.backend.sandboxcache.get_cached_files(opts=opts):
my_status = 'unknown'
try:
my_status = status_mapping[sj.status]
except KeyError:
pass
ds += __markup_by_status__(f, fc, my_status)
fc += 1
return ds