def onJobSubmit(self, wms, jobObj, jobNum):
token = wms.getAccessToken(jobObj.wmsId)
taskId = self._task.substVars(self._taskname,
jobNum,
addDict={
'DATASETNICK': ''
}).strip('_')
self._tp.start_thread(
"Notifying dashboard about job submission %d" % jobNum,
self._publish, jobObj, jobNum, taskId,
[{
'user': os.environ['LOGNAME'],
'GridName': '/CN=%s' % token.getUsername(),
'CMSUser': token.getUsername(),
'tool': 'grid-control',
'JSToolVersion': getVersion(),
'SubmissionType': 'direct',
'tool_ui': os.environ.get('HOSTNAME', ''),
'application': self._app,
'exe': 'shellscript',
'taskType': self._tasktype,
'scheduler': wms.wmsName,
'vo': token.getGroup()
},
self._task.getSubmitInfo(jobNum)])
def getTaskConfig(self):
taskConfig = {
# Storage element
'SE_MINFILESIZE':
self.seMinSize,
# Sandbox
'SB_OUTPUT_FILES':
str.join(' ', self.getSBOutFiles()),
'SB_INPUT_FILES':
str.join(' ', imap(lambda x: x.pathRel, self.getSBInFiles())),
# Runtime
'GC_JOBTIMEOUT':
self.nodeTimeout,
'GC_RUNTIME':
self.getCommand(),
# Seeds and substitutions
'SUBST_FILES':
str.join(' ', imap(os.path.basename, self.getSubstFiles())),
'GC_SUBST_OLD_STYLE':
str('__' in self._varCheck.markers).lower(),
# Task infos
'GC_TASK_CONF':
self.taskConfigName,
'GC_TASK_DATE':
self.taskDate,
'GC_TASK_ID':
self.taskID,
'GC_VERSION':
utils.getVersion(),
}
return utils.mergeDicts([taskConfig, self.taskVariables])
def run(args=None, intro=True):
# display the 'grid-control' logo and version
if intro and not os.environ.get('GC_DISABLE_INTRO'):
sys.stdout.write(SafeFile(utils.pathShare('logo.txt'), 'r').read())
sys.stdout.write('Revision: %s\n' % utils.getVersion())
pyver = (sys.version_info[0], sys.version_info[1])
if pyver < (2, 3):
utils.deprecated(
'This python version (%d.%d) is not supported anymore!' % pyver)
Activity.root = Activity('Running grid-control',
name='root') # top level activity instance
# main try... except block to catch exceptions and show error message
try:
config = gc_create_config(args or sys.argv[1:], useDefaultFiles=True)
workflow = gc_create_workflow(config)
try:
sys.exit(workflow.run())
finally:
sys.stdout.write('\n')
except SystemExit: # avoid getting caught for Python < 2.5
raise
except Exception: # coverage overrides sys.excepthook
gc_excepthook(*sys.exc_info())
sys.exit(os.EX_SOFTWARE)
def onJobSubmit(self, wms, jobObj, jobNum):
token = wms.getAccessToken(jobObj.wmsId)
self._start_publish(jobObj, jobNum, 'submission', [{
'user': os.environ['LOGNAME'], 'GridName': '/CN=%s' % token.getUsername(), 'CMSUser': token.getUsername(),
'tool': 'grid-control', 'JSToolVersion': getVersion(),
'SubmissionType':'direct', 'tool_ui': os.environ.get('HOSTNAME', ''),
'application': self._app, 'exe': 'shellscript', 'taskType': self._tasktype,
'scheduler': wms.wmsName, 'vo': token.getGroup()}, self._task.getSubmitInfo(jobNum)])
def onJobSubmit(self, wms, jobObj, jobNum):
token = wms.getAccessToken(jobObj.wmsId)
taskId = self.task.substVars(self.taskname, jobNum, addDict = {'DATASETNICK': ''}).strip('_')
utils.gcStartThread("Notifying dashboard about job submission %d" % jobNum,
self.publish, jobObj, jobNum, taskId, [{
'user': os.environ['LOGNAME'], 'GridName': '/CN=%s' % token.getUsername(), 'CMSUser': token.getUsername(),
'tool': 'grid-control', 'JSToolVersion': utils.getVersion(),
'SubmissionType':'direct', 'tool_ui': os.environ.get('HOSTNAME', ''),
'application': self.app, 'exe': 'shellscript', 'taskType': self.tasktype,
'scheduler': wms.wmsName, 'vo': token.getGroup()}, self.task.getSubmitInfo(jobNum)])
def onJobSubmit(self, wms, jobObj, jobNum):
token = wms.getAccessToken(jobObj.gcID)
jobInfo = self._task.getJobConfig(jobNum)
self._start_publish(jobObj, jobNum, 'submission', [{
'user': os.environ['LOGNAME'], 'GridName': '/CN=%s' % token.getUsername(), 'CMSUser': token.getUsername(),
'tool': 'grid-control', 'JSToolVersion': getVersion(),
'SubmissionType':'direct', 'tool_ui': os.environ.get('HOSTNAME', ''),
'application': jobInfo.get('SCRAM_PROJECTVERSION', self._app),
'exe': jobInfo.get('CMSSW_EXEC', 'shellscript'), 'taskType': self._tasktype,
'scheduler': wms.getObjectName(), 'vo': token.getGroup(),
'nevtJob': jobInfo.get('MAX_EVENTS', 0),
'datasetFull': jobInfo.get('DATASETPATH', 'none')}])
def getTaskConfig(self):
taskConfig = {
# Storage element
'SE_MINFILESIZE': self.seMinSize,
# Sandbox
'SB_OUTPUT_FILES': str.join(' ', self.getSBOutFiles()),
'SB_INPUT_FILES': str.join(' ', map(lambda x: x.pathRel, self.getSBInFiles())),
# Runtime
'GC_JOBTIMEOUT': self.nodeTimeout,
'GC_RUNTIME': self.getCommand(),
# Seeds and substitutions
'SUBST_FILES': str.join(' ', map(os.path.basename, self.getSubstFiles())),
# Task infos
'GC_TASK_CONF': self.taskConfigName,
'GC_TASK_DATE': self.taskDate,
'GC_TASK_ID': self.taskID,
'GC_VERSION': utils.getVersion(),
}
return utils.mergeDicts([taskConfig, self.taskVariables])
def _getBaseJDLData(self, task, queryArguments):
"""Create a sequence of default attributes for a submission JDL"""
jdlData = [
'+submitTool = "GridControl (version %s)"' %
utils.getVersion(),
'should_transfer_files = YES',
'when_to_transfer_output = ON_EXIT',
'periodic_remove = (JobStatus == 5 && HoldReasonCode != 16)',
'environment = CONDOR_WMS_DASHID=https://%s:/$(Cluster).$(Process)'
% self.parentPool.wmsName,
'Universe = %s' %
self.parentPool._jobSettings["Universe"], # TODO: Unhack me
'+GcID = "%s"' % self.parentPool._createGcId(
HTCJobID(gcJobNum='$(GcJobNum)',
gcTaskID=task.taskID,
clusterID='$(Cluster)',
procID='$(Process)',
scheddURI=self.getURI(),
typed=False)),
'+GcJobNumToWmsID = "$(GcJobNum)@$(Cluster).$(Process)"',
'+GcJobNumToGcID = "$(GcJobNum)@$(GcID)"',
'Log = %s' %
os.path.join(self.getStagingDir(), 'gcJobs.log'),
'job_ad_information_attrs = %s' % ','.join(
[arg for arg in queryArguments if arg not in ['JobStatus']]),
]
for key in queryArguments:
try:
# is this a match string? '+JOB_GLIDEIN_Entry_Name = "$$(GLIDEIN_Entry_Name:Unknown)"' -> MATCH_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" && MATCH_EXP_JOB_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2"
matchKey = re.match("(?:MATCH_EXP_JOB_|MATCH_|JOB_)(.*)",
key).group(1)
jdlData['Head']['+JOB_%s' %
matchKey] = "$$(%s:Unknown)" % matchKey
except AttributeError:
clear_current_exception()
for line in self.parentPool._jobSettings["ClassAd"]:
jdlData.append('+' + line)
for line in self.parentPool._jobSettings["JDL"]:
jdlData.append(line)
return jdlData
return jdlData
def onJobSubmit(self, wms, jobObj, jobNum):
token = wms.getAccessToken(jobObj.gcID)
jobInfo = self._task.getJobConfig(jobNum)
self._start_publish(
jobObj, jobNum, 'submission',
[{
'user': os.environ['LOGNAME'],
'GridName': '/CN=%s' % token.getUsername(),
'CMSUser': token.getUsername(),
'tool': 'grid-control',
'JSToolVersion': getVersion(),
'SubmissionType': 'direct',
'tool_ui': os.environ.get('HOSTNAME', ''),
'application': jobInfo.get('SCRAM_PROJECTVERSION', self._app),
'exe': jobInfo.get('CMSSW_EXEC', 'shellscript'),
'taskType': self._tasktype,
'scheduler': wms.getObjectName(),
'vo': token.getGroup(),
'nevtJob': jobInfo.get('MAX_EVENTS', 0),
'datasetFull': jobInfo.get('DATASETPATH', 'none')
}])
def run():
# set up signal handler for interrupts and debug session requests
signal.signal(signal.SIGINT, handle_abort_interrupt)
signal.signal(signal.SIGURG, handle_debug_interrupt)
# display the 'grid-control' logo and version
sys.stdout.write(open(utils.pathShare('logo.txt'), 'r').read())
sys.stdout.write('Revision: %s\n' % utils.getVersion())
pyver = (sys.version_info[0], sys.version_info[1])
if pyver < (2, 3):
utils.deprecated('This python version (%d.%d) is not supported anymore!' % pyver)
# main try... except block to catch exceptions and show error message
try:
config = gc_create_config(sys.argv[1:])
workflow = gc_create_workflow(config)
sys.exit(workflow.run())
except SystemExit:
pass
except Exception: # coverage overrides sys.excepthook
gc_excepthook(*sys.exc_info())
def run():
# set up signal handler for interrupts and debug session requests
signal.signal(signal.SIGINT, handle_abort_interrupt)
signal.signal(signal.SIGURG, handle_debug_interrupt)
# display the 'grid-control' logo and version
sys.stdout.write(open(utils.pathShare('logo.txt'), 'r').read())
sys.stdout.write('Revision: %s\n' % utils.getVersion())
pyver = (sys.version_info[0], sys.version_info[1])
if pyver < (2, 3):
utils.deprecated(
'This python version (%d.%d) is not supported anymore!' % pyver)
# main try... except block to catch exceptions and show error message
try:
config = gc_create_config(sys.argv[1:])
workflow = gc_create_workflow(config)
sys.exit(workflow.run())
except SystemExit:
pass
except Exception: # coverage overrides sys.excepthook
gc_excepthook(*sys.exc_info())
def run(args = None, intro = True):
# display the 'grid-control' logo and version
if intro and not os.environ.get('GC_DISABLE_INTRO'):
sys.stdout.write(SafeFile(utils.pathShare('logo.txt'), 'r').read())
sys.stdout.write('Revision: %s\n' % utils.getVersion())
pyver = (sys.version_info[0], sys.version_info[1])
if pyver < (2, 3):
utils.deprecated('This python version (%d.%d) is not supported anymore!' % pyver)
Activity.root = Activity('Running grid-control', name = 'root') # top level activity instance
# main try... except block to catch exceptions and show error message
try:
config = gc_create_config(args or sys.argv[1:], useDefaultFiles = True)
workflow = gc_create_workflow(config)
try:
sys.exit(workflow.run())
finally:
sys.stdout.write('\n')
except SystemExit: # avoid getting caught for Python < 2.5
raise
except Exception: # coverage overrides sys.excepthook
gc_excepthook(*sys.exc_info())
sys.exit(os.EX_SOFTWARE)
def _getBaseJDLData(self, task, queryArguments):
"""Create a sequence of default attributes for a submission JDL"""
jdlData = [
'+submitTool = "GridControl (version %s)"' % utils.getVersion(),
'should_transfer_files = YES',
'when_to_transfer_output = ON_EXIT',
'periodic_remove = (JobStatus == 5 && HoldReasonCode != 16)',
'environment = CONDOR_WMS_DASHID=https://%s:/$(Cluster).$(Process)' % self.parentPool.wmsName,
'Universe = %s' % self.parentPool._jobSettings["Universe"], # TODO: Unhack me
'+GcID = "%s"' % self.parentPool._createGcId(
HTCJobID(
gcJobNum = '$(GcJobNum)',
gcTaskID = task.taskID,
clusterID = '$(Cluster)',
procID = '$(Process)',
scheddURI = self.getURI(),
typed = False)
),
'+GcJobNumToWmsID = "$(GcJobNum)@$(Cluster).$(Process)"',
'+GcJobNumToGcID = "$(GcJobNum)@$(GcID)"',
'Log = %s' % os.path.join(self.getStagingDir(), 'gcJobs.log'),
'job_ad_information_attrs = %s' %','.join([ arg for arg in queryArguments if arg not in ['JobStatus']]),
]
for key in queryArguments:
try:
# is this a match string? '+JOB_GLIDEIN_Entry_Name = "$$(GLIDEIN_Entry_Name:Unknown)"' -> MATCH_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" && MATCH_EXP_JOB_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2"
matchKey=re.match("(?:MATCH_EXP_JOB_|MATCH_|JOB_)(.*)",key).group(1)
jdlData['Head']['+JOB_%s'%matchKey] = "$$(%s:Unknown)" % matchKey
except AttributeError:
pass
for line in self.parentPool._jobSettings["ClassAd"]:
jdlData.append( '+' + line )
for line in self.parentPool._jobSettings["JDL"]:
jdlData.append( line )
return jdlData
return jdlData
log = None
# set up signal handler for interrupts
def interrupt(sig, frame):
global log, handler
utils.abort(True)
log = utils.ActivityLog('Quitting grid-control! (This can take a few seconds...)')
signal.signal(signal.SIGINT, handler)
handler = signal.signal(signal.SIGINT, interrupt)
# set up signal handler for debug session requests
signal.signal(signal.SIGURG, debugInterruptHandler)
# display the 'grid-control' logo and version
utils.vprint(open(utils.pathShare('logo.txt'), 'r').read(), -1)
utils.vprint('Revision: %s' % utils.getVersion(), -1)
pyver = sys.version_info[0] + sys.version_info[1] / 10.0
if pyver < 2.3:
utils.deprecated('This python version (%.1f) is not supported anymore!' % pyver)
usage = 'Syntax: %s [OPTIONS] <config file>\n' % sys.argv[0]
parser = optparse.OptionParser(add_help_option=False)
parser.add_option('-h', '--help', dest='help', default=False, action='store_true')
parser.add_option('', '--help-conf', dest='help_cfg', default=False, action='store_true')
parser.add_option('', '--help-confmin', dest='help_scfg', default=False, action='store_true')
parser.add_option('-i', '--init', dest='init', default=False, action='store_true')
parser.add_option('-q', '--resync', dest='resync', default=False, action='store_true')
parser.add_option('', '--debug', dest='debug', default=False, action='store_true')
parser.add_option('-s', '--no-submission', dest='submission', default=True, action='store_false')
parser.add_option('-c', '--continuous', dest='continuous', default=None, action='store_true')
parser.add_option('-o', '--override', dest='override', default=[], action='append')