def _get_const_job_env(self):
return {
# Runtime
'GC_JOBTIMEOUT':
self._job_timeout,
'GC_RUNTIME':
self.get_command(),
# Seeds and substitutions
'SUBST_FILES':
str.join(' ', imap(os.path.basename, self._get_subst_fn_list())),
'GC_SUBST_OLD_STYLE':
str('__' in self._var_checker.markers).lower(),
# Task infos
'GC_TASK_CONF':
self._task_config_name,
'GC_TASK_DATE':
self._task_date,
'GC_TASK_ID':
self._task_id,
'GC_TASK_NAME':
self._task_name,
'GC_TASK_TIME':
self._task_time,
'GC_VERSION':
get_version(),
}
def on_job_submit(self, wms, job_obj, jobnum):
# Called on job submission
token = wms.get_access_token(job_obj.gc_id)
job_config_dict = self._task.get_job_dict(jobnum)
self._start_publish(job_obj, jobnum, 'submission', [{
'user':
get_local_username(),
'GridName':
'/CN=%s' % token.get_user_name(),
'CMSUser':
token.get_user_name(),
'tool':
'grid-control',
'JSToolVersion':
get_version(),
'SubmissionType':
'direct',
'tool_ui':
os.environ.get('HOSTNAME', ''),
'application':
job_config_dict.get('SCRAM_PROJECTVERSION', self._app),
'exe':
job_config_dict.get('CMSSW_EXEC', 'shellscript'),
'taskType':
self._tasktype,
'scheduler':
wms.get_object_name(),
'vo':
token.get_group(),
'nevtJob':
job_config_dict.get('MAX_EVENTS', 0),
'datasetFull':
job_config_dict.get('DATASETPATH', 'none')
}])
def get_task_dict(
self): # OLD API: Get environment variables for gc_config.sh
task_base_dict = {
# Storage element
'SE_MINFILESIZE':
self._se_min_size,
# Sandbox
'SB_OUTPUT_FILES':
str.join(' ', self.get_sb_out_fn_list()),
'SB_INPUT_FILES':
str.join(' ', imap(lambda x: x.path_rel,
self.get_sb_in_fpi_list())),
# Runtime
'GC_JOBTIMEOUT':
self._job_timeout,
'GC_RUNTIME':
self.get_command(),
# Seeds and substitutions
'SUBST_FILES':
str.join(' ', imap(os.path.basename, self._get_subst_fn_list())),
'GC_SUBST_OLD_STYLE':
str('__' in self._var_checker.markers).lower(),
# Task infos
'GC_TASK_CONF':
self._task_config_name,
'GC_TASK_DATE':
self._task_date,
'GC_TASK_TIME':
self._task_time,
'GC_TASK_ID':
self._task_id,
'GC_VERSION':
get_version(),
}
return dict_union(task_base_dict, self._task_var_dict)
def on_job_submit(self, wms, job_obj, jobnum):
# Called on job submission
token = wms.get_access_token(job_obj.gc_id)
job_config_dict = self._task.get_job_dict(jobnum)
self._start_publish(job_obj, jobnum, 'submission', [{'user': get_local_username(),
'GridName': '/CN=%s' % token.get_user_name(), 'CMSUser': token.get_user_name(),
'tool': 'grid-control', 'JSToolVersion': get_version(),
'SubmissionType': 'direct', 'tool_ui': os.environ.get('HOSTNAME', ''),
'application': job_config_dict.get('SCRAM_PROJECTVERSION', self._app),
'exe': job_config_dict.get('CMSSW_EXEC', 'shellscript'), 'taskType': self._tasktype,
'scheduler': wms.get_object_name(), 'vo': token.get_group(),
'nevtJob': job_config_dict.get('MAX_EVENTS', 0),
'datasetFull': job_config_dict.get('DATASETPATH', 'none')}])
def _get_const_job_env(self):
return {
# Runtime
'GC_JOBTIMEOUT': self._job_timeout,
'GC_RUNTIME': self.get_command(),
# Seeds and substitutions
'SUBST_FILES': str.join(' ', imap(os.path.basename, self._get_subst_fn_list())),
'GC_SUBST_OLD_STYLE': str('__' in self._var_checker.markers).lower(),
# Task infos
'GC_TASK_CONF': self._task_config_name,
'GC_TASK_DATE': self._task_date,
'GC_TASK_ID': self._task_id,
'GC_TASK_NAME': self._task_name,
'GC_TASK_TIME': self._task_time,
'GC_VERSION': get_version(),
}
def gc_run(args=None, intro=True):
# display the 'grid-control' logo and version
if intro and not os.environ.get('GC_DISABLE_INTRO'):
sys.stdout.write(SafeFile(get_path_share('logo.txt'), 'r').read_close())
sys.stdout.write('Revision: %s\n' % get_version())
pyver = (sys.version_info[0], sys.version_info[1])
if pyver < (2, 3):
deprecated('This python version (%d.%d) is not supported anymore!' % pyver)
atexit.register(lambda: sys.stdout.write('\n'))
# main try... except block to catch exceptions and show error message
try:
return _gc_run(args)
except SystemExit: # avoid getting caught for Python < 2.5
abort(True)
raise
except Exception: # coverage overrides sys.excepthook
abort(True)
gc_excepthook(*sys.exc_info())
sys.exit(os.EX_SOFTWARE)
def _getBaseJDLData(self, task, queryArguments):
"""Create a sequence of default attributes for a submission JDL"""
jdlData = [
'+submitTool = "GridControl (version %s)"' %
get_version(),
'should_transfer_files = YES',
'when_to_transfer_output = ON_EXIT',
'periodic_remove = (JobStatus == 5 && HoldReasonCode != 16)',
'environment = CONDOR_WMS_DASHID=https://%s:/$(Cluster).$(Process)'
% self.parentPool.wms_name,
'Universe = %s' %
self.parentPool._jobSettings["Universe"], # TODO: Unhack me
'+GcID = "%s"' % self.parentPool._createGcId(
HTCJobID(gcJobNum='$(GcJobNum)',
gcTaskID=task.task_id,
clusterID='$(Cluster)',
procID='$(Process)',
scheddURI=self.getURI(),
typed=False)),
'+GcJobNumToWmsID = "$(GcJobNum)@$(Cluster).$(Process)"',
'+GcJobNumToGcID = "$(GcJobNum)@$(GcID)"',
'Log = %s' %
os.path.join(self.getStagingDir(), 'gcJobs.log'),
'job_ad_information_attrs = %s' % ','.join(
[arg for arg in queryArguments if arg not in ['JobStatus']]),
]
for key in queryArguments:
try:
# is this a match string? '+JOB_GLIDEIN_Entry_Name = "$$(GLIDEIN_Entry_Name:Unknown)"' -> MATCH_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" && MATCH_EXP_JOB_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2"
matchKey = re.match("(?:MATCH_EXP_JOB_|MATCH_|JOB_)(.*)",
key).group(1)
jdlData['Head']['+JOB_%s' %
matchKey] = "$$(%s:Unknown)" % matchKey
except AttributeError:
clear_current_exception()
for line in self.parentPool._jobSettings["ClassAd"]:
jdlData.append('+' + line)
for line in self.parentPool._jobSettings["JDL"]:
jdlData.append(line)
return jdlData
return jdlData
def get_task_dict(self): # OLD API: Get environment variables for gc_config.sh
task_base_dict = {
# Storage element
'SE_MINFILESIZE': self._se_min_size,
# Sandbox
'SB_OUTPUT_FILES': str.join(' ', self.get_sb_out_fn_list()),
'SB_INPUT_FILES': str.join(' ', imap(lambda x: x.path_rel, self.get_sb_in_fpi_list())),
# Runtime
'GC_JOBTIMEOUT': self._job_timeout,
'GC_RUNTIME': self.get_command(),
# Seeds and substitutions
'SUBST_FILES': str.join(' ', imap(os.path.basename, self._get_subst_fn_list())),
'GC_SUBST_OLD_STYLE': str('__' in self._var_checker.markers).lower(),
# Task infos
'GC_TASK_CONF': self._task_config_name,
'GC_TASK_DATE': self._task_date,
'GC_TASK_TIME': self._task_time,
'GC_TASK_ID': self._task_id,
'GC_VERSION': get_version(),
}
return dict_union(task_base_dict, self._task_var_dict)
def gc_run(args=None, intro=True):
# display the 'grid-control' logo and version
if intro and not os.environ.get('GC_DISABLE_INTRO'):
sys.stdout.write(
SafeFile(get_path_share('logo.txt'), 'r').read_close())
sys.stdout.write('Revision: %s\n' % get_version())
pyver = (sys.version_info[0], sys.version_info[1])
if pyver < (2, 3):
deprecated('This python version (%d.%d) is not supported anymore!' %
pyver)
atexit.register(lambda: sys.stdout.write('\n'))
# main try... except block to catch exceptions and show error message
try:
return _gc_run(args)
except SystemExit: # avoid getting caught for Python < 2.5
abort(True)
raise
except Exception: # coverage overrides sys.excepthook
abort(True)
gc_excepthook(*sys.exc_info())
sys.exit(os.EX_SOFTWARE)
def _getBaseJDLData(self, task, queryArguments):
"""Create a sequence of default attributes for a submission JDL"""
jdlData = [
'+submitTool = "GridControl (version %s)"' % get_version(),
'should_transfer_files = YES',
'when_to_transfer_output = ON_EXIT',
'periodic_remove = (JobStatus == 5 && HoldReasonCode != 16)',
'environment = CONDOR_WMS_DASHID=https://%s:/$(Cluster).$(Process)' % self.parentPool.wms_name,
'Universe = %s' % self.parentPool._jobSettings["Universe"], # TODO: Unhack me
'+GcID = "%s"' % self.parentPool._createGcId(
HTCJobID(
gcJobNum = '$(GcJobNum)',
gcTaskID = task.task_id,
clusterID = '$(Cluster)',
procID = '$(Process)',
scheddURI = self.getURI(),
typed = False)
),
'+GcJobNumToWmsID = "$(GcJobNum)@$(Cluster).$(Process)"',
'+GcJobNumToGcID = "$(GcJobNum)@$(GcID)"',
'Log = %s' % os.path.join(self.getStagingDir(), 'gcJobs.log'),
'job_ad_information_attrs = %s' %','.join([ arg for arg in queryArguments if arg not in ['JobStatus']]),
]
for key in queryArguments:
try:
# is this a match string? '+JOB_GLIDEIN_Entry_Name = "$$(GLIDEIN_Entry_Name:Unknown)"' -> MATCH_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" && MATCH_EXP_JOB_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2"
matchKey=re.match("(?:MATCH_EXP_JOB_|MATCH_|JOB_)(.*)",key).group(1)
jdlData['Head']['+JOB_%s'%matchKey] = "$$(%s:Unknown)" % matchKey
except AttributeError:
clear_current_exception()
for line in self.parentPool._jobSettings["ClassAd"]:
jdlData.append( '+' + line )
for line in self.parentPool._jobSettings["JDL"]:
jdlData.append( line )
return jdlData
return jdlData
