class ScriptMonitoring(Monitoring): alias = ['scripts'] configSections = EventHandler.configSections + ['scripts'] def __init__(self, config, name, task): Monitoring.__init__(self, config, name, task) self._silent = config.getBool('silent', True, onChange = None) self._evtSubmit = config.getCommand('on submit', '', onChange = None) self._evtStatus = config.getCommand('on status', '', onChange = None) self._evtOutput = config.getCommand('on output', '', onChange = None) self._evtFinish = config.getCommand('on finish', '', onChange = None) self._runningMax = config.getTime('script runtime', 5, onChange = None) self._workPath = config.getWorkPath() self._tp = GCThreadPool() # Get both task and job config / state dicts def _scriptThread(self, script, jobNum = None, jobObj = None, allDict = None): try: tmp = {} if jobNum is not None: tmp.update(self._task.getSubmitInfo(jobNum)) if jobObj is not None: tmp.update(jobObj.getAll()) tmp['WORKDIR'] = self._workPath tmp.update(self._task.getTaskConfig()) if jobNum is not None: tmp.update(self._task.getJobConfig(jobNum)) tmp.update(self._task.getSubmitInfo(jobNum)) tmp.update(allDict or {}) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key os.environ[key] = str(value) script = self._task.substVars(script, jobNum, tmp) if not self._silent: proc = LocalProcess(script) self._log.info(proc.get_output(timeout = self._runningMax)) else: os.system(script) except Exception: self._log.exception('Error while running user script!') def _runInBackground(self, script, jobNum = None, jobObj = None, addDict = None): if script != '': self._tp.start_thread('Running monitoring script %s' % script, self._scriptThread, script, jobNum, jobObj, addDict) # Called on job submission def onJobSubmit(self, wms, jobObj, jobNum): self._runInBackground(self._evtSubmit, jobNum, jobObj) # Called on job status update def onJobUpdate(self, wms, jobObj, jobNum, data): self._runInBackground(self._evtStatus, jobNum, jobObj, {'STATUS': Job.enum2str(jobObj.state)}) # Called on job status update def onJobOutput(self, wms, jobObj, jobNum, retCode): self._runInBackground(self._evtOutput, jobNum, jobObj, {'RETCODE': retCode}) # Called at the end of the task def onTaskFinish(self, nJobs): self._runInBackground(self._evtFinish, addDict = {'NJOBS': nJobs}) self._tp.wait_and_drop(self._runningMax)
class ScriptMonitoring(Monitoring): alias = ['scripts'] configSections = EventHandler.configSections + ['scripts'] def __init__(self, config, name, task): Monitoring.__init__(self, config, name, task) self._silent = config.getBool('silent', True, onChange=None) self._evtSubmit = config.getCommand('on submit', '', onChange=None) self._evtStatus = config.getCommand('on status', '', onChange=None) self._evtOutput = config.getCommand('on output', '', onChange=None) self._evtFinish = config.getCommand('on finish', '', onChange=None) self._runningMax = config.getTime('script runtime', 5, onChange=None) self._workPath = config.getWorkPath() self._tp = GCThreadPool() # Get both task and job config / state dicts def _scriptThread(self, script, jobNum=None, jobObj=None, allDict=None): try: tmp = {} if jobNum is not None: tmp.update(self._task.getSubmitInfo(jobNum)) if jobObj is not None: tmp.update(jobObj.getAll()) tmp['WORKDIR'] = self._workPath tmp.update(self._task.getTaskConfig()) if jobNum is not None: tmp.update(self._task.getJobConfig(jobNum)) tmp.update(self._task.getSubmitInfo(jobNum)) tmp.update(allDict or {}) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key os.environ[key] = str(value) script = self._task.substVars(script, jobNum, tmp) if not self._silent: proc = LocalProcess(script) self._log.info(proc.get_output(timeout=self._runningMax)) else: os.system(script) except Exception: self._log.exception('Error while running user script!') def _runInBackground(self, script, jobNum=None, jobObj=None, addDict=None): if script != '': self._tp.start_thread('Running monitoring script %s' % script, self._scriptThread, script, jobNum, jobObj, addDict) # Called on job submission def onJobSubmit(self, wms, jobObj, jobNum): self._runInBackground(self._evtSubmit, jobNum, jobObj) # Called on job status update def onJobUpdate(self, wms, jobObj, jobNum, data): self._runInBackground(self._evtStatus, jobNum, jobObj, {'STATUS': Job.enum2str(jobObj.state)}) # Called on job status update def onJobOutput(self, wms, jobObj, jobNum, retCode): self._runInBackground(self._evtOutput, jobNum, jobObj, {'RETCODE': retCode}) # Called at the end of the task def onTaskFinish(self, nJobs): self._runInBackground(self._evtFinish, addDict={'NJOBS': nJobs}) self._tp.wait_and_drop(self._runningMax)
class DashBoard(Monitoring): configSections = Monitoring.configSections + ['dashboard'] def __init__(self, config, name, task): Monitoring.__init__(self, config, name, task) jobDesc = task.getDescription( None) # TODO: use the other variables for monitoring self._app = config.get('application', 'shellscript', onChange=None) self._runningMax = config.getTime('dashboard timeout', 5, onChange=None) self._tasktype = config.get('task', jobDesc.jobType or 'analysis', onChange=None) self._taskname = config.get('task name', '@GC_TASK_ID@_@DATASETNICK@', onChange=None) self._statusMap = { Job.DONE: 'DONE', Job.FAILED: 'DONE', Job.SUCCESS: 'DONE', Job.RUNNING: 'RUNNING', Job.ABORTED: 'ABORTED', Job.CANCELLED: 'CANCELLED' } self._tp = GCThreadPool() def getScript(self): yield pathShare('mon.dashboard.sh', pkg='grid_control_cms') def getTaskConfig(self): result = { 'TASK_NAME': self._taskname, 'DB_EXEC': self._app, 'DATASETNICK': '' } result.update(Monitoring.getTaskConfig(self)) return result def getFiles(self): yield pathShare('mon.dashboard.sh', pkg='grid_control_cms') for fn in ('DashboardAPI.py', 'Logger.py', 'apmon.py', 'report.py'): yield pathShare('..', 'DashboardAPI', fn, pkg='grid_control_cms') def _publish(self, jobObj, jobNum, taskId, usermsg): (_, backend, rawId) = jobObj.gcID.split('.', 2) dashId = '%s_%s' % (jobNum, rawId) if 'http' not in jobObj.gcID: dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId) msg = mergeDicts([{ 'taskId': taskId, 'jobId': dashId, 'sid': rawId }] + usermsg) DashboardAPI( taskId, dashId).publish(**filterDict(msg, vF=lambda v: v is not None)) def _start_publish(self, jobObj, jobNum, desc, message): taskId = self._task.substVars('dashboard task id', self._taskname, jobNum, addDict={ 'DATASETNICK': '' }).strip('_') self._tp.start_thread( 'Notifying dashboard about %s of job %d' % (desc, jobNum), self._publish, jobObj, jobNum, taskId, message) # Called on job submission def onJobSubmit(self, wms, jobObj, jobNum): token = wms.getAccessToken(jobObj.gcID) jobInfo = self._task.getJobConfig(jobNum) self._start_publish( jobObj, jobNum, 'submission', [{ 'user': os.environ['LOGNAME'], 'GridName': '/CN=%s' % token.getUsername(), 'CMSUser': token.getUsername(), 'tool': 'grid-control', 'JSToolVersion': getVersion(), 'SubmissionType': 'direct', 'tool_ui': os.environ.get('HOSTNAME', ''), 'application': jobInfo.get('SCRAM_PROJECTVERSION', self._app), 'exe': jobInfo.get('CMSSW_EXEC', 'shellscript'), 'taskType': self._tasktype, 'scheduler': wms.getObjectName(), 'vo': token.getGroup(), 'nevtJob': jobInfo.get('MAX_EVENTS', 0), 'datasetFull': jobInfo.get('DATASETPATH', 'none') }]) # Called on job status update and output def _updateDashboard(self, wms, jobObj, jobNum, data, addMsg): # Translate status into dashboard status message statusDashboard = self._statusMap.get(jobObj.state, 'PENDING') self._start_publish(jobObj, jobNum, 'status', [{ 'StatusValue': statusDashboard, 'StatusValueReason': data.get('reason', statusDashboard).upper(), 'StatusEnterTime': data.get('timestamp', time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime())), 'StatusDestination': data.get('dest', '') }, addMsg]) def onJobUpdate(self, wms, jobObj, jobNum, data): self._updateDashboard(wms, jobObj, jobNum, jobObj, {}) def onJobOutput(self, wms, jobObj, jobNum, retCode): self._updateDashboard(wms, jobObj, jobNum, jobObj, {'ExeExitCode': retCode}) def onFinish(self): self._tp.wait_and_drop(self._runningMax)
class DashBoard(Monitoring): configSections = Monitoring.configSections + ['dashboard'] def __init__(self, config, name, task): Monitoring.__init__(self, config, name, task) jobDesc = task.getDescription(None) # TODO: use the other variables for monitoring self._app = config.get('application', 'shellscript', onChange = None) self._runningMax = config.getTime('dashboard timeout', 5, onChange = None) self._tasktype = config.get('task', jobDesc.jobType or 'analysis', onChange = None) self._taskname = config.get('task name', '@GC_TASK_ID@_@DATASETNICK@', onChange = None) self._statusMap = {Job.DONE: 'DONE', Job.FAILED: 'DONE', Job.SUCCESS: 'DONE', Job.RUNNING: 'RUNNING', Job.ABORTED: 'ABORTED', Job.CANCELLED: 'CANCELLED'} self._tp = GCThreadPool() def getScript(self): yield pathShare('mon.dashboard.sh', pkg = 'grid_control_cms') def getTaskConfig(self): result = {'TASK_NAME': self._taskname, 'DB_EXEC': self._app, 'DATASETNICK': ''} result.update(Monitoring.getTaskConfig(self)) return result def getFiles(self): yield pathShare('mon.dashboard.sh', pkg = 'grid_control_cms') for fn in ('DashboardAPI.py', 'Logger.py', 'apmon.py', 'report.py'): yield pathShare('..', 'DashboardAPI', fn, pkg = 'grid_control_cms') def _publish(self, jobObj, jobNum, taskId, usermsg): (_, backend, rawId) = jobObj.gcID.split('.', 2) dashId = '%s_%s' % (jobNum, rawId) if 'http' not in jobObj.gcID: dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId) msg = mergeDicts([{'taskId': taskId, 'jobId': dashId, 'sid': rawId}] + usermsg) DashboardAPI(taskId, dashId).publish(**filterDict(msg, vF = lambda v: v is not None)) def _start_publish(self, jobObj, jobNum, desc, message): taskId = self._task.substVars('dashboard task id', self._taskname, jobNum, addDict = {'DATASETNICK': ''}).strip('_') self._tp.start_thread('Notifying dashboard about %s of job %d' % (desc, jobNum), self._publish, jobObj, jobNum, taskId, message) # Called on job submission def onJobSubmit(self, wms, jobObj, jobNum): token = wms.getAccessToken(jobObj.gcID) jobInfo = self._task.getJobConfig(jobNum) self._start_publish(jobObj, jobNum, 'submission', [{ 'user': os.environ['LOGNAME'], 'GridName': '/CN=%s' % token.getUsername(), 'CMSUser': token.getUsername(), 'tool': 'grid-control', 'JSToolVersion': getVersion(), 'SubmissionType':'direct', 'tool_ui': os.environ.get('HOSTNAME', ''), 'application': jobInfo.get('SCRAM_PROJECTVERSION', self._app), 'exe': jobInfo.get('CMSSW_EXEC', 'shellscript'), 'taskType': self._tasktype, 'scheduler': wms.getObjectName(), 'vo': token.getGroup(), 'nevtJob': jobInfo.get('MAX_EVENTS', 0), 'datasetFull': jobInfo.get('DATASETPATH', 'none')}]) # Called on job status update and output def _updateDashboard(self, wms, jobObj, jobNum, data, addMsg): # Translate status into dashboard status message statusDashboard = self._statusMap.get(jobObj.state, 'PENDING') self._start_publish(jobObj, jobNum, 'status', [{'StatusValue': statusDashboard, 'StatusValueReason': data.get('reason', statusDashboard).upper(), 'StatusEnterTime': data.get('timestamp', time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime())), 'StatusDestination': data.get('dest', '') }, addMsg]) def onJobUpdate(self, wms, jobObj, jobNum, data): self._updateDashboard(wms, jobObj, jobNum, jobObj, {}) def onJobOutput(self, wms, jobObj, jobNum, retCode): self._updateDashboard(wms, jobObj, jobNum, jobObj, {'ExeExitCode': retCode}) def onFinish(self): self._tp.wait_and_drop(self._runningMax)