def _get_const_job_env(self): return { # Runtime 'GC_JOBTIMEOUT': self._job_timeout, 'GC_RUNTIME': self.get_command(), # Seeds and substitutions 'SUBST_FILES': str.join(' ', imap(os.path.basename, self._get_subst_fn_list())), 'GC_SUBST_OLD_STYLE': str('__' in self._var_checker.markers).lower(), # Task infos 'GC_TASK_CONF': self._task_config_name, 'GC_TASK_DATE': self._task_date, 'GC_TASK_ID': self._task_id, 'GC_TASK_NAME': self._task_name, 'GC_TASK_TIME': self._task_time, 'GC_VERSION': get_version(), }
def on_job_submit(self, wms, job_obj, jobnum): # Called on job submission token = wms.get_access_token(job_obj.gc_id) job_config_dict = self._task.get_job_dict(jobnum) self._start_publish(job_obj, jobnum, 'submission', [{ 'user': get_local_username(), 'GridName': '/CN=%s' % token.get_user_name(), 'CMSUser': token.get_user_name(), 'tool': 'grid-control', 'JSToolVersion': get_version(), 'SubmissionType': 'direct', 'tool_ui': os.environ.get('HOSTNAME', ''), 'application': job_config_dict.get('SCRAM_PROJECTVERSION', self._app), 'exe': job_config_dict.get('CMSSW_EXEC', 'shellscript'), 'taskType': self._tasktype, 'scheduler': wms.get_object_name(), 'vo': token.get_group(), 'nevtJob': job_config_dict.get('MAX_EVENTS', 0), 'datasetFull': job_config_dict.get('DATASETPATH', 'none') }])
def get_task_dict( self): # OLD API: Get environment variables for gc_config.sh task_base_dict = { # Storage element 'SE_MINFILESIZE': self._se_min_size, # Sandbox 'SB_OUTPUT_FILES': str.join(' ', self.get_sb_out_fn_list()), 'SB_INPUT_FILES': str.join(' ', imap(lambda x: x.path_rel, self.get_sb_in_fpi_list())), # Runtime 'GC_JOBTIMEOUT': self._job_timeout, 'GC_RUNTIME': self.get_command(), # Seeds and substitutions 'SUBST_FILES': str.join(' ', imap(os.path.basename, self._get_subst_fn_list())), 'GC_SUBST_OLD_STYLE': str('__' in self._var_checker.markers).lower(), # Task infos 'GC_TASK_CONF': self._task_config_name, 'GC_TASK_DATE': self._task_date, 'GC_TASK_TIME': self._task_time, 'GC_TASK_ID': self._task_id, 'GC_VERSION': get_version(), } return dict_union(task_base_dict, self._task_var_dict)
def on_job_submit(self, wms, job_obj, jobnum): # Called on job submission token = wms.get_access_token(job_obj.gc_id) job_config_dict = self._task.get_job_dict(jobnum) self._start_publish(job_obj, jobnum, 'submission', [{'user': get_local_username(), 'GridName': '/CN=%s' % token.get_user_name(), 'CMSUser': token.get_user_name(), 'tool': 'grid-control', 'JSToolVersion': get_version(), 'SubmissionType': 'direct', 'tool_ui': os.environ.get('HOSTNAME', ''), 'application': job_config_dict.get('SCRAM_PROJECTVERSION', self._app), 'exe': job_config_dict.get('CMSSW_EXEC', 'shellscript'), 'taskType': self._tasktype, 'scheduler': wms.get_object_name(), 'vo': token.get_group(), 'nevtJob': job_config_dict.get('MAX_EVENTS', 0), 'datasetFull': job_config_dict.get('DATASETPATH', 'none')}])
def gc_run(args=None, intro=True): # display the 'grid-control' logo and version if intro and not os.environ.get('GC_DISABLE_INTRO'): sys.stdout.write(SafeFile(get_path_share('logo.txt'), 'r').read_close()) sys.stdout.write('Revision: %s\n' % get_version()) pyver = (sys.version_info[0], sys.version_info[1]) if pyver < (2, 3): deprecated('This python version (%d.%d) is not supported anymore!' % pyver) atexit.register(lambda: sys.stdout.write('\n')) # main try... except block to catch exceptions and show error message try: return _gc_run(args) except SystemExit: # avoid getting caught for Python < 2.5 abort(True) raise except Exception: # coverage overrides sys.excepthook abort(True) gc_excepthook(*sys.exc_info()) sys.exit(os.EX_SOFTWARE)
def _getBaseJDLData(self, task, queryArguments): """Create a sequence of default attributes for a submission JDL""" jdlData = [ '+submitTool = "GridControl (version %s)"' % get_version(), 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT', 'periodic_remove = (JobStatus == 5 && HoldReasonCode != 16)', 'environment = CONDOR_WMS_DASHID=https://%s:/$(Cluster).$(Process)' % self.parentPool.wms_name, 'Universe = %s' % self.parentPool._jobSettings["Universe"], # TODO: Unhack me '+GcID = "%s"' % self.parentPool._createGcId( HTCJobID(gcJobNum='$(GcJobNum)', gcTaskID=task.task_id, clusterID='$(Cluster)', procID='$(Process)', scheddURI=self.getURI(), typed=False)), '+GcJobNumToWmsID = "$(GcJobNum)@$(Cluster).$(Process)"', '+GcJobNumToGcID = "$(GcJobNum)@$(GcID)"', 'Log = %s' % os.path.join(self.getStagingDir(), 'gcJobs.log'), 'job_ad_information_attrs = %s' % ','.join( [arg for arg in queryArguments if arg not in ['JobStatus']]), ] for key in queryArguments: try: # is this a match string? '+JOB_GLIDEIN_Entry_Name = "$$(GLIDEIN_Entry_Name:Unknown)"' -> MATCH_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" && MATCH_EXP_JOB_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" matchKey = re.match("(?:MATCH_EXP_JOB_|MATCH_|JOB_)(.*)", key).group(1) jdlData['Head']['+JOB_%s' % matchKey] = "$$(%s:Unknown)" % matchKey except AttributeError: clear_current_exception() for line in self.parentPool._jobSettings["ClassAd"]: jdlData.append('+' + line) for line in self.parentPool._jobSettings["JDL"]: jdlData.append(line) return jdlData return jdlData
def get_task_dict(self): # OLD API: Get environment variables for gc_config.sh task_base_dict = { # Storage element 'SE_MINFILESIZE': self._se_min_size, # Sandbox 'SB_OUTPUT_FILES': str.join(' ', self.get_sb_out_fn_list()), 'SB_INPUT_FILES': str.join(' ', imap(lambda x: x.path_rel, self.get_sb_in_fpi_list())), # Runtime 'GC_JOBTIMEOUT': self._job_timeout, 'GC_RUNTIME': self.get_command(), # Seeds and substitutions 'SUBST_FILES': str.join(' ', imap(os.path.basename, self._get_subst_fn_list())), 'GC_SUBST_OLD_STYLE': str('__' in self._var_checker.markers).lower(), # Task infos 'GC_TASK_CONF': self._task_config_name, 'GC_TASK_DATE': self._task_date, 'GC_TASK_TIME': self._task_time, 'GC_TASK_ID': self._task_id, 'GC_VERSION': get_version(), } return dict_union(task_base_dict, self._task_var_dict)
def gc_run(args=None, intro=True): # display the 'grid-control' logo and version if intro and not os.environ.get('GC_DISABLE_INTRO'): sys.stdout.write( SafeFile(get_path_share('logo.txt'), 'r').read_close()) sys.stdout.write('Revision: %s\n' % get_version()) pyver = (sys.version_info[0], sys.version_info[1]) if pyver < (2, 3): deprecated('This python version (%d.%d) is not supported anymore!' % pyver) atexit.register(lambda: sys.stdout.write('\n')) # main try... except block to catch exceptions and show error message try: return _gc_run(args) except SystemExit: # avoid getting caught for Python < 2.5 abort(True) raise except Exception: # coverage overrides sys.excepthook abort(True) gc_excepthook(*sys.exc_info()) sys.exit(os.EX_SOFTWARE)
def _getBaseJDLData(self, task, queryArguments): """Create a sequence of default attributes for a submission JDL""" jdlData = [ '+submitTool = "GridControl (version %s)"' % get_version(), 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT', 'periodic_remove = (JobStatus == 5 && HoldReasonCode != 16)', 'environment = CONDOR_WMS_DASHID=https://%s:/$(Cluster).$(Process)' % self.parentPool.wms_name, 'Universe = %s' % self.parentPool._jobSettings["Universe"], # TODO: Unhack me '+GcID = "%s"' % self.parentPool._createGcId( HTCJobID( gcJobNum = '$(GcJobNum)', gcTaskID = task.task_id, clusterID = '$(Cluster)', procID = '$(Process)', scheddURI = self.getURI(), typed = False) ), '+GcJobNumToWmsID = "$(GcJobNum)@$(Cluster).$(Process)"', '+GcJobNumToGcID = "$(GcJobNum)@$(GcID)"', 'Log = %s' % os.path.join(self.getStagingDir(), 'gcJobs.log'), 'job_ad_information_attrs = %s' %','.join([ arg for arg in queryArguments if arg not in ['JobStatus']]), ] for key in queryArguments: try: # is this a match string? '+JOB_GLIDEIN_Entry_Name = "$$(GLIDEIN_Entry_Name:Unknown)"' -> MATCH_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" && MATCH_EXP_JOB_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" matchKey=re.match("(?:MATCH_EXP_JOB_|MATCH_|JOB_)(.*)",key).group(1) jdlData['Head']['+JOB_%s'%matchKey] = "$$(%s:Unknown)" % matchKey except AttributeError: clear_current_exception() for line in self.parentPool._jobSettings["ClassAd"]: jdlData.append( '+' + line ) for line in self.parentPool._jobSettings["JDL"]: jdlData.append( line ) return jdlData return jdlData