def update_dashboard(self, crab_retry):
"""
Need a doc string here.
"""
if not self.task_ad:
return
params = {
'tool':
'crab3',
'SubmissionType':
'crab3',
'JSToolVersion':
'3.3.0',
'tool_ui':
os.environ.get('HOSTNAME', ''),
'scheduler':
'GLIDEIN',
'GridName':
self.task_ad['CRAB_UserDN'],
'ApplicationVersion':
self.task_ad['CRAB_JobSW'],
'taskType':
self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'),
'vo':
'cms',
'CMSUser':
self.task_ad['CRAB_UserHN'],
'user':
self.task_ad['CRAB_UserHN'],
'taskId':
self.task_ad['CRAB_ReqName'],
'datasetFull':
self.task_ad['DESIRED_CMSDataset'],
'resubmitter':
self.task_ad['CRAB_UserHN'],
'exe':
'cmsRun',
'broker':
self.backend,
'bossId':
str(self.job_id),
'localId':
'',
'SyncGridJobId':
'https://glidein.cern.ch/%s/%s' %
(self.job_id, self.task_ad['CRAB_ReqName'].replace("_", ":")),
}
if not self.userWebDirPrx:
storage_rules = htcondor.param['CRAB_StorageRules']
self.userWebDirPrx = getWebdirForDb(
str(self.task_ad.get('CRAB_ReqName')), storage_rules)
self.logger.info("User web dir: %s", self.userWebDirPrx)
insertJobIdSid(params, self.job_id, self.task_ad['CRAB_ReqName'],
crab_retry)
def update_dashboard(self, crab_retry):
"""
Need a doc string here.
"""
if not self.task_ad:
return
params = {'tool': 'crab3',
'SubmissionType': 'crab3',
'JSToolVersion': '3.3.0',
'tool_ui': os.environ.get('HOSTNAME', ''),
'scheduler': 'GLIDEIN',
'GridName': self.task_ad['CRAB_UserDN'],
'ApplicationVersion': self.task_ad['CRAB_JobSW'],
'taskType': self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'),
'vo': 'cms',
'CMSUser': self.task_ad['CRAB_UserHN'],
'user': self.task_ad['CRAB_UserHN'],
'taskId': self.task_ad['CRAB_ReqName'],
'datasetFull': self.task_ad['DESIRED_CMSDataset'],
'resubmitter': self.task_ad['CRAB_UserHN'],
'exe': 'cmsRun',
'broker': self.backend,
'bossId': str(self.job_id),
'localId': '',
'SyncGridJobId': 'https://glidein.cern.ch/%s/%s' % (self.job_id, self.task_ad['CRAB_ReqName'].replace("_", ":")),
}
storage_rules = htcondor.param['CRAB_StorageRules']
userWebDir = getWebdirForDb(str(self.task_ad.get('CRAB_ReqName')), storage_rules)
userWebDirPrx = ""
try:
with open('proxied_webdir') as fd:
proxied_webdir = fd.read()
userWebDirPrx = proxied_webdir
except IOError as e:
self.logger.error(("'I/O error(%s): %s', when looking for the proxied_webdir file. Might be normal"
" if the schedd does not have a proxiedurl in the REST external config." % (e.errno, e.strerror)))
self.logger.info("User web dir proxy: " + userWebDirPrx)
self.logger.info("web dir: " + userWebDir)
if userWebDirPrx:
setDashboardLogs(params, userWebDirPrx, self.job_id, crab_retry)
elif userWebDir:
setDashboardLogs(params, userWebDir, self.job_id, crab_retry)
else:
print("Not setting dashboard logfiles as I cannot find CRAB_UserWebDir nor CRAB_UserWebDirPrx.")
insertJobIdSid(params, self.job_id, self.task_ad['CRAB_ReqName'], crab_retry)
apmon = ApmonIf()
self.logger.debug("Dashboard task info: %s" % str(params))
apmon.sendToML(params)
apmon.free()
def update_dashboard(self, crab_retry):
"""
Need a doc string here.
"""
if not self.task_ad:
return
params = {'tool': 'crab3',
'SubmissionType': 'crab3',
'JSToolVersion': '3.3.0',
'tool_ui': os.environ.get('HOSTNAME', ''),
'scheduler': 'GLIDEIN',
'GridName': self.task_ad['CRAB_UserDN'],
'ApplicationVersion': self.task_ad['CRAB_JobSW'],
'taskType': self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'),
'vo': 'cms',
'CMSUser': self.task_ad['CRAB_UserHN'],
'user': self.task_ad['CRAB_UserHN'],
'taskId': self.task_ad['CRAB_ReqName'],
'datasetFull': self.task_ad['DESIRED_CMSDataset'],
'resubmitter': self.task_ad['CRAB_UserHN'],
'exe': 'cmsRun',
'broker': self.backend,
'bossId': str(self.job_id),
'localId': '',
'SyncGridJobId': 'https://glidein.cern.ch/%s/%s' % (self.job_id, self.task_ad['CRAB_ReqName'].replace("_", ":")),
}
storage_rules = htcondor.param['CRAB_StorageRules']
userWebDir = getWebdirForDb(str(self.task_ad.get('CRAB_ReqName')), storage_rules)
userWebDirPrx = ""
try:
with open('proxied_webdir') as fd:
proxied_webdir = fd.read()
userWebDirPrx = proxied_webdir
except IOError as e:
self.logger.error(("'I/O error(%s): %s', when looking for the proxied_webdir file. Might be normal"
" if the schedd does not have a proxiedurl in the REST external config." % (e.errno, e.strerror)))
self.logger.info("User web dir proxy: " + userWebDirPrx)
self.logger.info("web dir: " + userWebDir)
if userWebDirPrx:
setDashboardLogs(params, userWebDirPrx, self.job_id, crab_retry)
elif userWebDir:
setDashboardLogs(params, userWebDir, self.job_id, crab_retry)
else:
print("Not setting dashboard logfiles as I cannot find CRAB_UserWebDir nor CRAB_UserWebDirPrx.")
insertJobIdSid(params, self.job_id, self.task_ad['CRAB_ReqName'], crab_retry)
apmon = ApmonIf()
self.logger.debug("Dashboard task info: %s" % str(params))
apmon.sendToML(params)
apmon.free()
def alter_submit(self, crab_retry):
"""
Copy the content of the generic file Job.submit into a job-specific file
Job.<job_id>.submit and add attributes that are job-specific (e.g. CRAB_Retry).
Add also parameters that can be overwritten at each manual job resubmission
(e.g. MaxWallTimeMins, RequestMemory, RequestCpus, JobPrio, DESIRED_SITES).
"""
## Start the Job.<job_id>.submit content with the CRAB_Retry.
new_submit_text = '+CRAB_Retry = %d\n' % (crab_retry)
msg = "Setting CRAB_Retry = %s" % (crab_retry)
self.logger.info(msg)
## For the parameters that can be overwritten at each manual job resubmission,
## read them from the task ad, unless there is resubmission information there
## and this job is not one that has to be resubmitted, in which case we should
## use the same parameters (site black- and whitelists, requested memory, etc)
## as used by the previous job retry (which are saved in self.resubmit_info).
CRAB_ResubmitList_in_taskad = ('CRAB_ResubmitList' in self.task_ad)
use_resubmit_info = False
resubmit_jobids = []
if 'CRAB_ResubmitList' in self.task_ad:
resubmit_jobids = self.task_ad['CRAB_ResubmitList']
try:
resubmit_jobids = set(resubmit_jobids)
if resubmit_jobids and self.job_id not in resubmit_jobids:
use_resubmit_info = True
except TypeError:
resubmit_jobids = True
## If there is no resubmit_info, we can of course not use it.
if not self.resubmit_info:
use_resubmit_info = False
## Get the resubmission parameters.
maxjobruntime = None
maxmemory = None
numcores = None
priority = None
if not use_resubmit_info:
#if 'MaxWallTimeMins_RAW' in self.task_ad:
# if self.task_ad['MaxWallTimeMins_RAW'] != 1315:
# maxjobruntime = self.task_ad.lookup('MaxWallTimeMins_RAW')
# self.resubmit_info['maxjobruntime'] = maxjobruntime
if 'MaxWallTimeMins' in self.task_ad:
maxjobruntime = int(str(self.task_ad.lookup('MaxWallTimeMins')))
if 'RequestMemory' in self.task_ad:
maxmemory = int(str(self.task_ad.lookup('RequestMemory')))
if 'RequestCpus' in self.task_ad:
numcores = int(str(self.task_ad.lookup('RequestCpus')))
if 'JobPrio' in self.task_ad:
priority = int(str(self.task_ad['JobPrio']))
else:
inkey = str(crab_retry) if crab_retry == 0 else str(crab_retry - 1)
while inkey not in self.resubmit_info and int(inkey) > 0:
inkey = str(int(inkey) - 1)
maxjobruntime = self.resubmit_info[inkey].get('maxjobruntime')
maxmemory = self.resubmit_info[inkey].get('maxmemory')
numcores = self.resubmit_info[inkey].get('numcores')
priority = self.resubmit_info[inkey].get('priority')
## Save the (new) values of the resubmission parameters in self.resubmit_info
## for the current job retry number.
outkey = str(crab_retry)
if outkey not in self.resubmit_info:
self.resubmit_info[outkey] = {}
self.resubmit_info[outkey]['maxjobruntime'] = maxjobruntime
self.resubmit_info[outkey]['maxmemory'] = maxmemory
self.resubmit_info[outkey]['numcores'] = numcores
self.resubmit_info[outkey]['priority'] = priority
self.resubmit_info[outkey]['use_resubmit_info'] = use_resubmit_info
self.resubmit_info[outkey]['CRAB_ResubmitList_in_taskad'] = CRAB_ResubmitList_in_taskad
## Add the resubmission parameters to the Job.<job_id>.submit content.
if maxjobruntime is not None:
new_submit_text += '+MaxWallTimeMins = %s\n' % (str(maxjobruntime))
if maxmemory is not None:
new_submit_text += '+RequestMemory = %s\n' % (str(maxmemory))
if numcores is not None:
new_submit_text += '+RequestCpus = %s\n' % (str(numcores))
if priority is not None:
new_submit_text += '+JobPrio = %s\n' % (str(priority))
## Within the schedd, order the first few jobs in the task before all other tasks of the same priority.
pre_job_prio = 1
if self.job_id <= 5:
pre_job_prio = 0
new_submit_text += '+PreJobPrio1 = %d\n' % pre_job_prio
## The schedd will use PostJobPrio1 as a secondary job-priority sorting key: it
## will first run jobs by JobPrio; then, for jobs with the same JobPrio, it will
## run the job with the higher PostJobPrio1.
new_submit_text += '+PostJobPrio1 = -%s\n' % str(self.task_ad.lookup('QDate'))
## Order retries before all other jobs in this task
new_submit_text += '+PostJobPrio2 = %d\n' % crab_retry
## This is used to send to dashbord the location of the logfiles
try:
storage_rules = htcondor.param['CRAB_StorageRules']
except:
storage_rules = "^/home/remoteGlidein,http://submit-5.t2.ucsd.edu/CSstoragePath"
new_submit_text += '+CRAB_UserWebDir = "%s"\n' % getWebdirForDb(str(self.task_ad.get('CRAB_ReqName')), storage_rules)
try:
with open('proxied_webdir') as fd:
proxied_webdir = fd.read()
new_submit_text += '+CRAB_UserWebDirPrx = "%s"\n' % proxied_webdir
except IOError as e:
self.logger.error(("'I/O error(%s): %s', when looking for the proxied_webdir file. Might be normal"
" if the schedd does not have a proxiedurl in the REST external config." % (e.errno, e.strerror)))
## Add the site black- and whitelists and the DESIRED_SITES to the
## Job.<job_id>.submit content.
new_submit_text = self.redo_sites(new_submit_text, crab_retry, use_resubmit_info)
## Add group information:
username = self.task_ad.get('CRAB_UserHN')
if 'CMSGroups' in self.task_ad:
new_submit_text += '+CMSGroups = %s\n' % classad.quote(self.task_ad['CMSGroups'])
elif username:
groups = CMSGroupMapper.map_user_to_groups(username)
if groups:
new_submit_text += '+CMSGroups = %s\n' % classad.quote(groups)
## Finally add (copy) all the content of the generic Job.submit file.
with open("Job.submit", 'r') as fd:
new_submit_text += fd.read()
## Write the Job.<job_id>.submit file.
with open("Job.%d.submit" % (self.job_id), 'w') as fd:
fd.write(new_submit_text)
