def _display(self, colour):
from Ganga.Utility.ColourText import ANSIMarkup, NoMarkup, Foreground, Effects
if colour:
markup = ANSIMarkup()
else:
markup = NoMarkup()
fg = Foreground()
from cStringIO import StringIO
sio = StringIO()
sio.write("Ganga Configuration" + '\n')
sections = sorted(stripProxy(self).keys())
maxcol = 0
for p in sections:
if len(p) > maxcol:
maxcol = len(p)
if maxcol > 50:
maxcol = 50
for p in sections:
sio.write('%-*s : %s' % (maxcol, p,
markup(
stripProxy(self)[p].docstring.split(
'\n')[0], fg.boldgrey)) + '\n')
return sio.getvalue()
def __init__(self, name):
super(TaskRegistrySlice, self).__init__(name, display_prefix="tasks")
from Ganga.Utility.ColourText import Foreground, Background, Effects
fg = Foreground()
fx = Effects()
bg = Background()
self.status_colours = {
'new': fx.normal,
'submitted': fg.orange,
'running': fg.green,
'completed': fg.blue,
'failed': fg.red
}
self.fx = fx
self._proxyClass = TaskRegistrySliceProxy
def __init__(self, name):
super(JobRegistrySlice, self).__init__(name, display_prefix="jobs")
from Ganga.Utility.ColourText import Foreground, Background, Effects
fg = Foreground()
fx = Effects()
bg = Background()
try:
status_colours = config['jobs_status_colours']
self.status_colours = dict([(k, eval(v)) for k, v in status_colours.iteritems()])
except Exception as x:
logger.warning('configuration problem with colour specification: "%s"', str(x))
status_colours = config.options['jobs_status_colours'].default_value
self.status_colours = dict([(k, eval(v)) for k, v in status_colours.iteritems()])
self.fx = fx
self._proxyClass = JobRegistrySliceProxy
def __init__(self, name):
super(TransientRegistrySlice, self).__init__(
name, display_prefix="box")
from Ganga.Utility.ColourText import Foreground, Background, Effects
fg = Foreground()
fx = Effects()
bg = Background()
self.fx = fx
self.name = 'box' # needed to ensure that select works properly
self.status_colours = {'default': fx.normal,
'JobTemplate': fg.orange,
'Task': fg.green,
'Job': fg.blue}
self._display_columns_functions["id"] = lambda obj: obj.id
self._display_columns_functions["type"] = lambda obj: obj._name
self._display_columns_functions["name"] = lambda obj: obj.name
self._proxyClass = TransientRegistrySliceProxy
def __init__(self, name):
super(BoxRegistrySlice, self).__init__(name, display_prefix="box")
self._display_columns_functions["id"] = lambda obj: obj._getRegistry(
).find(obj)
self._display_columns_functions["type"] = lambda obj: obj._name
self._display_columns_functions["name"] = lambda obj: obj._getRegistry(
)._getName(obj)
from Ganga.Utility.ColourText import Foreground, Background, Effects
fg = Foreground()
fx = Effects()
bg = Background()
self.fx = fx
self.status_colours = {
'default': fx.normal,
'backends': fg.orange,
'applications': fg.green,
'jobs': fg.blue
}
self._proxyClass = BoxRegistrySliceProxy
def _display(self, interactive=0):
"""Prints content of the shareref metadata in a well formatted way.
"""
if len(self.__getName()) > 0:
from Ganga.GPIDev.Lib.File import getSharedPath
fstring = " %48s | %20s | %15s"
disp_string = fstring % ("Shared directory", "Date created",
"Reference count\n")
# print fstring % (" ", " "," ")
disp_string += fstring % (
"------------------------------------------------",
"--------------------", "---------------\n")
zero_ref = False
unsorted = []
all_elements = copy.deepcopy(self.__getName())
for element in all_elements:
full_shareddir_path = os.path.join(getSharedPath(),
os.path.basename(element))
if os.path.isdir(full_shareddir_path):
unsorted.append(
shareref_data(
os.path.basename(element),
int(os.path.getctime(full_shareddir_path)),
self.__getName()[element]))
else:
unsorted.append(
shareref_data(os.path.basename(element),
"Directory not found",
self.__getName()[element]))
decorated = sorted(
(name.date, i, name) for i, name in enumerate(unsorted))
sorted_refs = [name for date, i, name in decorated]
for line in sorted_refs:
if isinstance(line.date, int):
tmp_string = fstring % (
os.path.basename(line.name),
time.strftime("%d %b %Y %H:%M:%S",
time.localtime(line.date)), line.counter)
else:
tmp_string = fstring % (os.path.basename(
line.name), line.date, line.counter)
if (line.counter == 0) or (isinstance(line.date, str)):
from Ganga.Utility.ColourText import ANSIMarkup, NoMarkup, Foreground, Background, Effects
fg = Foreground()
fg = Background()
fx = Effects()
if interactive:
m = ANSIMarkup()
else:
m = NoMarkup()
disp_string += fg.red + tmp_string + fx.normal + '\n'
#disp_string += m(tmp_string,code=fg.red)
if (line.counter == 0):
zero_ref = True
else:
disp_string += tmp_string + '\n'
disp_string += "\nThe shared directory repository is rooted at " + \
getSharedPath() + "\n"
if zero_ref:
disp_string += "\nShared directories with a zero reference count will be removed when Ganga exits.\n"
else:
disp_string = "No objects stored in the shared directory."
return disp_string
def updateMonitoringInformation(jobs):
jobDict = {}
for job in jobs:
if job.backend.id:
jobDict[job.backend.id] = job
idList = jobDict.keys()
if not idList:
return
queryCommand = " ".join\
([
"condor_q -global" if getConfig(
"Condor")["query_global_queues"] else "condor_q",
"-format \"%s \" GlobalJobId",
"-format \"%s \" RemoteHost",
"-format \"%d \" JobStatus",
"-format \"%f\\n\" RemoteUserCpu"
])
status, output = commands.getstatusoutput(queryCommand)
if 0 != status:
logger.error("Problem retrieving status for Condor jobs")
return
if ("All queues are empty" == output):
infoList = []
else:
infoList = output.split("\n")
allDict = {}
for infoString in infoList:
tmpList = infoString.split()
id, host, status, cputime = ("", "", "", "")
if 3 == len(tmpList):
id, status, cputime = tmpList
if 4 == len(tmpList):
id, host, status, cputime = tmpList
if id:
allDict[id] = {}
allDict[id]["status"] = Condor.statusDict[status]
allDict[id]["cputime"] = cputime
allDict[id]["host"] = host
fg = Foreground()
fx = Effects()
status_colours = {'submitted': fg.orange,
'running': fg.green,
'completed': fg.blue}
for id in idList:
printStatus = False
if jobDict[id].status == "killed":
continue
localId = id.split("#")[-1]
globalId = id
if globalId == localId:
queryCommand = " ".join\
([
"condor_q -global" if getConfig(
"Condor")["query_global_queues"] else "condor_q",
"-format \"%s\" GlobalJobId",
id
])
status, output = commands.getstatusoutput(queryCommand)
if 0 == status:
globalId = output
if globalId in allDict.keys():
status = allDict[globalId]["status"]
host = allDict[globalId]["host"]
cputime = allDict[globalId]["cputime"]
if status != jobDict[id].backend.status:
printStatus = True
stripProxy(jobDict[id])._getWriteAccess()
jobDict[id].backend.status = status
if jobDict[id].backend.status == "Running":
jobDict[id].updateStatus("running")
if host:
if jobDict[id].backend.actualCE != host:
jobDict[id].backend.actualCE = host
jobDict[id].backend.cputime = cputime
else:
jobDict[id].backend.status = ""
outDir = jobDict[id].getOutputWorkspace().getPath()
condorLogPath = "".join([outDir, "condorLog"])
checkExit = True
if os.path.isfile(condorLogPath):
checkExit = False
for line in open(condorLogPath):
if -1 != line.find("terminated"):
checkExit = True
break
if -1 != line.find("aborted"):
checkExit = True
break
if checkExit:
printStatus = True
stdoutPath = "".join([outDir, "stdout"])
jobStatus = "failed"
if os.path.isfile(stdoutPath):
with open(stdoutPath) as stdout:
lineList = stdout.readlines()
try:
exitLine = lineList[-1]
exitCode = exitLine.strip().split()[-1]
except IndexError:
exitCode = -1
if exitCode.isdigit():
jobStatus = "completed"
else:
logger.error("Problem extracting exit code from job %s. Line found was '%s'." % (
jobDict[id].fqid, exitLine))
jobDict[id].updateStatus(jobStatus)
if printStatus:
if jobDict[id].backend.actualCE:
hostInfo = jobDict[id].backend.actualCE
else:
hostInfo = "Condor"
status = jobDict[id].status
if status in status_colours:
colour = status_colours[status]
else:
colour = fg.magenta
if "submitted" == status:
preposition = "to"
else:
preposition = "on"
if jobDict[id].backend.status:
backendStatus = "".join\
([" (", jobDict[id].backend.status, ") "])
else:
backendStatus = ""
logger.info(colour + 'Job %s %s%s %s %s - %s' + fx.normal,
jobDict[
id].fqid, status, backendStatus, preposition, hostInfo,
time.strftime('%c'))
return None
def _display(self, colour):
from Ganga.Utility.ColourText import ANSIMarkup, NoMarkup, getColour, Foreground, Effects
if colour:
markup = ANSIMarkup()
else:
markup = NoMarkup()
fg = Foreground()
display_config = getConfig('Display')
name_colour = getColour(display_config['config_name_colour'])
docstring_colour = getColour(
display_config['config_docstring_colour']) # fg.boldgrey
value_colour = getColour(
display_config['config_value_colour']) # fx.normal
levels = ['**', '* ', ' ']
levels = map(lambda x: markup(x, fg.red), levels)
from cStringIO import StringIO
sio = StringIO()
sio.write('%s' % markup(stripProxy(self).name, name_colour) + ' : ' +
markup(stripProxy(self).docstring, docstring_colour) + '\n')
opts = sorted(stripProxy(self).options.keys())
INDENT = ' ' * 2
p = re.compile('[\.\w]*\.')
for o in opts:
sio.write(
levels[stripProxy(self).getEffectiveLevel(o)] + ' ' +
markup(o, name_colour) + ' = ' +
markup(p.sub('', repr(stripProxy(self)[o])), value_colour) +
'\n')
sio.write(
textwrap.fill(markup(
stripProxy(self).options[o].docstring.strip(),
docstring_colour),
width=80,
initial_indent=INDENT,
subsequent_indent=INDENT) + '\n')
typelist = stripProxy(self).options[o].typelist
if not typelist:
typedesc = 'Type: ' + \
p.sub('',str(type(stripProxy(self).options[o].default_value)))
else:
typedesc = 'Allowed types: ' + \
str([p.sub('',str(t)) for t in typelist])
sio.write(markup(INDENT + typedesc, docstring_colour) + '\n')
filter = stripProxy(self).options[o].filter
if filter:
filter_doc = filter.__doc__
if not filter_doc:
filter_doc = "undocumented"
sio.write(
markup(INDENT + "Filter: " +
filter_doc, docstring_colour) + '\n')
examples = stripProxy(self).options[o].examples
if examples:
sio.write(
markup(INDENT + "Examples:", docstring_colour) + '\n')
for e in examples.splitlines():
sio.write(
markup(INDENT + e.strip(), docstring_colour) + '\n')
return sio.getvalue()
def list_cached_files(self, loop=True, opts=''):
"""
Lists the uploaded files.
if loop = True, it prints also the uploaded files associated with subjobs.
"""
fc = 0
ds = ''
doColoring = True
fg = Foreground()
fx = Effects()
status_colors = {'inuse': fg.orange, 'free': fg.blue, 'gone': fg.red}
status_mapping = {
'new': 'inuse',
'submitted': 'inuse',
'submitting': 'inuse',
'running': 'inuse',
'completed': 'free',
'completing': 'free',
'failed': 'free',
'killed': 'free'
}
if doColoring:
markup = ANSIMarkup()
else:
markup = NoMarkup()
def __markup_by_status__(fileIndex, counter, status):
fmtStr = '\n%4d\t%-30s\t%-12s\t%s' % (counter, fileIndex.name,
status, fileIndex.id)
try:
return markup(fmtStr, status_colors[status])
except KeyError:
return markup(fmtStr, fx.normal)
j = self.getJobObject()
for f in self.get_cached_files(opts=opts):
my_status = 'unknown'
if j:
try:
my_status = status_mapping[j.status]
except KeyError:
pass
ds += __markup_by_status__(f, fc, my_status)
fc += 1
if j and loop:
for sj in j.subjobs:
for f in sj.backend.sandboxcache.get_cached_files(opts=opts):
my_status = 'unknown'
try:
my_status = status_mapping[sj.status]
except KeyError:
pass
ds += __markup_by_status__(f, fc, my_status)
fc += 1
return ds
