def update_dashboard(self, crab_retry): """ Need a doc string here. """ if not self.task_ad: return params = { 'tool': 'crab3', 'SubmissionType': 'crab3', 'JSToolVersion': '3.3.0', 'tool_ui': os.environ.get('HOSTNAME', ''), 'scheduler': 'GLIDEIN', 'GridName': self.task_ad['CRAB_UserDN'], 'ApplicationVersion': self.task_ad['CRAB_JobSW'], 'taskType': self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'), 'vo': 'cms', 'CMSUser': self.task_ad['CRAB_UserHN'], 'user': self.task_ad['CRAB_UserHN'], 'taskId': self.task_ad['CRAB_ReqName'], 'datasetFull': self.task_ad['DESIRED_CMSDataset'], 'resubmitter': self.task_ad['CRAB_UserHN'], 'exe': 'cmsRun', 'broker': self.backend, 'bossId': str(self.job_id), 'localId': '', 'SyncGridJobId': 'https://glidein.cern.ch/%s/%s' % (self.job_id, self.task_ad['CRAB_ReqName'].replace("_", ":")), } if not self.userWebDirPrx: storage_rules = htcondor.param['CRAB_StorageRules'] self.userWebDirPrx = getWebdirForDb( str(self.task_ad.get('CRAB_ReqName')), storage_rules) self.logger.info("User web dir: %s", self.userWebDirPrx) insertJobIdSid(params, self.job_id, self.task_ad['CRAB_ReqName'], crab_retry)
def update_dashboard(self, crab_retry): """ Need a doc string here. """ if not self.task_ad: return params = {'tool': 'crab3', 'SubmissionType': 'crab3', 'JSToolVersion': '3.3.0', 'tool_ui': os.environ.get('HOSTNAME', ''), 'scheduler': 'GLIDEIN', 'GridName': self.task_ad['CRAB_UserDN'], 'ApplicationVersion': self.task_ad['CRAB_JobSW'], 'taskType': self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'), 'vo': 'cms', 'CMSUser': self.task_ad['CRAB_UserHN'], 'user': self.task_ad['CRAB_UserHN'], 'taskId': self.task_ad['CRAB_ReqName'], 'datasetFull': self.task_ad['DESIRED_CMSDataset'], 'resubmitter': self.task_ad['CRAB_UserHN'], 'exe': 'cmsRun', 'broker': self.backend, 'bossId': str(self.job_id), 'localId': '', 'SyncGridJobId': 'https://glidein.cern.ch/%s/%s' % (self.job_id, self.task_ad['CRAB_ReqName'].replace("_", ":")), } storage_rules = htcondor.param['CRAB_StorageRules'] userWebDir = getWebdirForDb(str(self.task_ad.get('CRAB_ReqName')), storage_rules) userWebDirPrx = "" try: with open('proxied_webdir') as fd: proxied_webdir = fd.read() userWebDirPrx = proxied_webdir except IOError as e: self.logger.error(("'I/O error(%s): %s', when looking for the proxied_webdir file. Might be normal" " if the schedd does not have a proxiedurl in the REST external config." % (e.errno, e.strerror))) self.logger.info("User web dir proxy: " + userWebDirPrx) self.logger.info("web dir: " + userWebDir) if userWebDirPrx: setDashboardLogs(params, userWebDirPrx, self.job_id, crab_retry) elif userWebDir: setDashboardLogs(params, userWebDir, self.job_id, crab_retry) else: print("Not setting dashboard logfiles as I cannot find CRAB_UserWebDir nor CRAB_UserWebDirPrx.") insertJobIdSid(params, self.job_id, self.task_ad['CRAB_ReqName'], crab_retry) apmon = ApmonIf() self.logger.debug("Dashboard task info: %s" % str(params)) apmon.sendToML(params) apmon.free()
def alter_submit(self, crab_retry): """ Copy the content of the generic file Job.submit into a job-specific file Job.<job_id>.submit and add attributes that are job-specific (e.g. CRAB_Retry). Add also parameters that can be overwritten at each manual job resubmission (e.g. MaxWallTimeMins, RequestMemory, RequestCpus, JobPrio, DESIRED_SITES). """ ## Start the Job.<job_id>.submit content with the CRAB_Retry. new_submit_text = '+CRAB_Retry = %d\n' % (crab_retry) msg = "Setting CRAB_Retry = %s" % (crab_retry) self.logger.info(msg) ## For the parameters that can be overwritten at each manual job resubmission, ## read them from the task ad, unless there is resubmission information there ## and this job is not one that has to be resubmitted, in which case we should ## use the same parameters (site black- and whitelists, requested memory, etc) ## as used by the previous job retry (which are saved in self.resubmit_info). CRAB_ResubmitList_in_taskad = ('CRAB_ResubmitList' in self.task_ad) use_resubmit_info = False resubmit_jobids = [] if 'CRAB_ResubmitList' in self.task_ad: resubmit_jobids = self.task_ad['CRAB_ResubmitList'] try: resubmit_jobids = set(resubmit_jobids) if resubmit_jobids and self.job_id not in resubmit_jobids: use_resubmit_info = True except TypeError: resubmit_jobids = True ## If there is no resubmit_info, we can of course not use it. if not self.resubmit_info: use_resubmit_info = False ## Get the resubmission parameters. maxjobruntime = None maxmemory = None numcores = None priority = None if not use_resubmit_info: #if 'MaxWallTimeMins_RAW' in self.task_ad: # if self.task_ad['MaxWallTimeMins_RAW'] != 1315: # maxjobruntime = self.task_ad.lookup('MaxWallTimeMins_RAW') # self.resubmit_info['maxjobruntime'] = maxjobruntime if 'MaxWallTimeMins' in self.task_ad: maxjobruntime = int(str(self.task_ad.lookup('MaxWallTimeMins'))) if 'RequestMemory' in self.task_ad: maxmemory = int(str(self.task_ad.lookup('RequestMemory'))) if 'RequestCpus' in self.task_ad: numcores = int(str(self.task_ad.lookup('RequestCpus'))) if 'JobPrio' in self.task_ad: priority = int(str(self.task_ad['JobPrio'])) else: inkey = str(crab_retry) if crab_retry == 0 else str(crab_retry - 1) while inkey not in self.resubmit_info and int(inkey) > 0: inkey = str(int(inkey) - 1) maxjobruntime = self.resubmit_info[inkey].get('maxjobruntime') maxmemory = self.resubmit_info[inkey].get('maxmemory') numcores = self.resubmit_info[inkey].get('numcores') priority = self.resubmit_info[inkey].get('priority') ## Save the (new) values of the resubmission parameters in self.resubmit_info ## for the current job retry number. outkey = str(crab_retry) if outkey not in self.resubmit_info: self.resubmit_info[outkey] = {} self.resubmit_info[outkey]['maxjobruntime'] = maxjobruntime self.resubmit_info[outkey]['maxmemory'] = maxmemory self.resubmit_info[outkey]['numcores'] = numcores self.resubmit_info[outkey]['priority'] = priority self.resubmit_info[outkey]['use_resubmit_info'] = use_resubmit_info self.resubmit_info[outkey]['CRAB_ResubmitList_in_taskad'] = CRAB_ResubmitList_in_taskad ## Add the resubmission parameters to the Job.<job_id>.submit content. if maxjobruntime is not None: new_submit_text += '+MaxWallTimeMins = %s\n' % (str(maxjobruntime)) if maxmemory is not None: new_submit_text += '+RequestMemory = %s\n' % (str(maxmemory)) if numcores is not None: new_submit_text += '+RequestCpus = %s\n' % (str(numcores)) if priority is not None: new_submit_text += '+JobPrio = %s\n' % (str(priority)) ## Within the schedd, order the first few jobs in the task before all other tasks of the same priority. pre_job_prio = 1 if self.job_id <= 5: pre_job_prio = 0 new_submit_text += '+PreJobPrio1 = %d\n' % pre_job_prio ## The schedd will use PostJobPrio1 as a secondary job-priority sorting key: it ## will first run jobs by JobPrio; then, for jobs with the same JobPrio, it will ## run the job with the higher PostJobPrio1. new_submit_text += '+PostJobPrio1 = -%s\n' % str(self.task_ad.lookup('QDate')) ## Order retries before all other jobs in this task new_submit_text += '+PostJobPrio2 = %d\n' % crab_retry ## This is used to send to dashbord the location of the logfiles try: storage_rules = htcondor.param['CRAB_StorageRules'] except: storage_rules = "^/home/remoteGlidein,http://submit-5.t2.ucsd.edu/CSstoragePath" new_submit_text += '+CRAB_UserWebDir = "%s"\n' % getWebdirForDb(str(self.task_ad.get('CRAB_ReqName')), storage_rules) try: with open('proxied_webdir') as fd: proxied_webdir = fd.read() new_submit_text += '+CRAB_UserWebDirPrx = "%s"\n' % proxied_webdir except IOError as e: self.logger.error(("'I/O error(%s): %s', when looking for the proxied_webdir file. Might be normal" " if the schedd does not have a proxiedurl in the REST external config." % (e.errno, e.strerror))) ## Add the site black- and whitelists and the DESIRED_SITES to the ## Job.<job_id>.submit content. new_submit_text = self.redo_sites(new_submit_text, crab_retry, use_resubmit_info) ## Add group information: username = self.task_ad.get('CRAB_UserHN') if 'CMSGroups' in self.task_ad: new_submit_text += '+CMSGroups = %s\n' % classad.quote(self.task_ad['CMSGroups']) elif username: groups = CMSGroupMapper.map_user_to_groups(username) if groups: new_submit_text += '+CMSGroups = %s\n' % classad.quote(groups) ## Finally add (copy) all the content of the generic Job.submit file. with open("Job.submit", 'r') as fd: new_submit_text += fd.read() ## Write the Job.<job_id>.submit file. with open("Job.%d.submit" % (self.job_id), 'w') as fd: fd.write(new_submit_text)