def sendDashboardJobs(self, params, info):
apmon = ApmonIf()
for job in info:
job.update(params)
self.logger.debug("Dashboard job info: %s" % str(job))
apmon.sendToML(job)
apmon.free()
def update_dashboard(self, retry, id, reqname, backend):
if not self.task_ad:
return
params = {'tool': 'crab3',
'SubmissionType':'crab3',
'JSToolVersion': '3.3.0',
'tool_ui': os.environ.get('HOSTNAME',''),
'scheduler': 'GLIDEIN',
'GridName': self.task_ad['CRAB_UserDN'],
'ApplicationVersion': self.task_ad['CRAB_JobSW'],
'taskType': self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'),
'vo': 'cms',
'CMSUser': self.task_ad['CRAB_UserHN'],
'user': self.task_ad['CRAB_UserHN'],
'taskId': self.task_ad['CRAB_ReqName'],
'datasetFull': self.task_ad['CRAB_InputData'],
'resubmitter': self.task_ad['CRAB_UserHN'],
'exe': 'cmsRun',
'jobId': ("%d_https://glidein.cern.ch/%d/%s_%d" % (id, id, self.task_ad['CRAB_ReqName'].replace("_", ":"), retry)),
'sid': "https://glidein.cern.ch/%d/%s" % (id, self.task_ad['CRAB_ReqName'].replace("_", ":")),
'broker': backend,
'bossId': str(id),
'localId' : '',
}
apmon = ApmonIf()
print("Dashboard task info: %s" % str(params))
apmon.sendToML(params)
apmon.free()
def sendDashboardJobs(self, params, info):
"""Send dashboard information about jobs of task"""
apmon = ApmonIf()
for job in info:
job.update(params)
apmon.sendToML(job)
self.logger.debug("Dashboard job info submitted. Last job for example is: %s", job)
apmon.free()
def sendDashboardTask(self):
apmon = ApmonIf()
params = self.buildDashboardInfo()
params_copy = dict(params)
params_copy['jobId'] = 'TaskMeta'
self.logger.debug("Dashboard task info: %s" % str(params_copy))
apmon.sendToML(params_copy)
apmon.free()
return params
def update_dashboard(self, crab_retry):
"""
Need a doc string here.
"""
if not self.task_ad:
return
params = {'tool': 'crab3',
'SubmissionType': 'crab3',
'JSToolVersion': '3.3.0',
'tool_ui': os.environ.get('HOSTNAME', ''),
'scheduler': 'GLIDEIN',
'GridName': self.task_ad['CRAB_UserDN'],
'ApplicationVersion': self.task_ad['CRAB_JobSW'],
'taskType': self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'),
'vo': 'cms',
'CMSUser': self.task_ad['CRAB_UserHN'],
'user': self.task_ad['CRAB_UserHN'],
'taskId': self.task_ad['CRAB_ReqName'],
'datasetFull': self.task_ad['DESIRED_CMSDataset'],
'resubmitter': self.task_ad['CRAB_UserHN'],
'exe': 'cmsRun',
'broker': self.backend,
'bossId': str(self.job_id),
'localId': '',
'SyncGridJobId': 'https://glidein.cern.ch/%s/%s' % (self.job_id, self.task_ad['CRAB_ReqName'].replace("_", ":")),
}
storage_rules = htcondor.param['CRAB_StorageRules']
userWebDir = getWebdirForDb(str(self.task_ad.get('CRAB_ReqName')), storage_rules)
userWebDirPrx = ""
try:
with open('proxied_webdir') as fd:
proxied_webdir = fd.read()
userWebDirPrx = proxied_webdir
except IOError as e:
self.logger.error(("'I/O error(%s): %s', when looking for the proxied_webdir file. Might be normal"
" if the schedd does not have a proxiedurl in the REST external config." % (e.errno, e.strerror)))
self.logger.info("User web dir proxy: " + userWebDirPrx)
self.logger.info("web dir: " + userWebDir)
if userWebDirPrx:
setDashboardLogs(params, userWebDirPrx, self.job_id, crab_retry)
elif userWebDir:
setDashboardLogs(params, userWebDir, self.job_id, crab_retry)
else:
print("Not setting dashboard logfiles as I cannot find CRAB_UserWebDir nor CRAB_UserWebDirPrx.")
insertJobIdSid(params, self.job_id, self.task_ad['CRAB_ReqName'], crab_retry)
apmon = ApmonIf()
self.logger.debug("Dashboard task info: %s" % str(params))
apmon.sendToML(params)
apmon.free()
def sendDashboardJobs(self, params, info):
apmon = ApmonIf()
for job in info:
job.update(params)
self.logger.debug("Dashboard job info: %s" % str(job))
apmon.sendToML(job)
apmon.free()
def sendDashboardJobs(self, params, info):
"""Send dashboard information about jobs of task"""
apmon = ApmonIf()
for job in info:
job.update(params)
apmon.sendToML(job)
self.logger.debug("Dashboard job info submitted. Last job for example is: %s", job)
apmon.free()
def sendDashboardTask(self):
apmon = ApmonIf()
params = self.buildDashboardInfo()
params_copy = dict(params)
params_copy['jobId'] = 'TaskMeta'
self.logger.debug("Dashboard task info: %s" % str(params_copy))
apmon.sendToML(params_copy)
apmon.free()
return params
def update_dashboard(self, crab_retry):
"""
Need a doc string here.
"""
if not self.task_ad:
return
params = {'tool': 'crab3',
'SubmissionType': 'crab3',
'JSToolVersion': '3.3.0',
'tool_ui': os.environ.get('HOSTNAME', ''),
'scheduler': 'GLIDEIN',
'GridName': self.task_ad['CRAB_UserDN'],
'ApplicationVersion': self.task_ad['CRAB_JobSW'],
'taskType': self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'),
'vo': 'cms',
'CMSUser': self.task_ad['CRAB_UserHN'],
'user': self.task_ad['CRAB_UserHN'],
'taskId': self.task_ad['CRAB_ReqName'],
'datasetFull': self.task_ad['DESIRED_CMSDataset'],
'resubmitter': self.task_ad['CRAB_UserHN'],
'exe': 'cmsRun',
'broker': self.backend,
'bossId': str(self.job_id),
'localId': '',
'SyncGridJobId': 'https://glidein.cern.ch/%s/%s' % (self.job_id, self.task_ad['CRAB_ReqName'].replace("_", ":")),
}
storage_rules = htcondor.param['CRAB_StorageRules']
userWebDir = getWebdirForDb(str(self.task_ad.get('CRAB_ReqName')), storage_rules)
userWebDirPrx = ""
try:
with open('proxied_webdir') as fd:
proxied_webdir = fd.read()
userWebDirPrx = proxied_webdir
except IOError as e:
self.logger.error(("'I/O error(%s): %s', when looking for the proxied_webdir file. Might be normal"
" if the schedd does not have a proxiedurl in the REST external config." % (e.errno, e.strerror)))
self.logger.info("User web dir proxy: " + userWebDirPrx)
self.logger.info("web dir: " + userWebDir)
if userWebDirPrx:
setDashboardLogs(params, userWebDirPrx, self.job_id, crab_retry)
elif userWebDir:
setDashboardLogs(params, userWebDir, self.job_id, crab_retry)
else:
print("Not setting dashboard logfiles as I cannot find CRAB_UserWebDir nor CRAB_UserWebDirPrx.")
insertJobIdSid(params, self.job_id, self.task_ad['CRAB_ReqName'], crab_retry)
apmon = ApmonIf()
self.logger.debug("Dashboard task info: %s" % str(params))
apmon.sendToML(params)
apmon.free()
def update_dashboard(self, retry, id, reqname, backend):
if not self.task_ad:
return
params = {
'tool':
'crab3',
'SubmissionType':
'crab3',
'JSToolVersion':
'3.3.0',
'tool_ui':
os.environ.get('HOSTNAME', ''),
'scheduler':
'GLIDEIN',
'GridName':
self.task_ad['CRAB_UserDN'],
'ApplicationVersion':
self.task_ad['CRAB_JobSW'],
'taskType':
self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'),
'vo':
'cms',
'CMSUser':
self.task_ad['CRAB_UserHN'],
'user':
self.task_ad['CRAB_UserHN'],
'taskId':
self.task_ad['CRAB_ReqName'],
'datasetFull':
self.task_ad['CRAB_InputData'],
'resubmitter':
self.task_ad['CRAB_UserHN'],
'exe':
'cmsRun',
'jobId':
("%d_https://glidein.cern.ch/%d/%s_%d" %
(id, id, self.task_ad['CRAB_ReqName'].replace("_", ":"), retry)),
'sid':
"https://glidein.cern.ch/%d/%s" %
(id, self.task_ad['CRAB_ReqName'].replace("_", ":")),
'broker':
backend,
'bossId':
str(id),
'localId':
'',
}
apmon = ApmonIf()
print("Dashboard task info: %s" % str(params))
apmon.sendToML(params)
apmon.free()
def update_dashboard(self, crab_retry):
"""
Need a doc string here.
"""
if not self.task_ad:
return
params = {
'tool':
'crab3',
'SubmissionType':
'crab3',
'JSToolVersion':
'3.3.0',
'tool_ui':
os.environ.get('HOSTNAME', ''),
'scheduler':
'GLIDEIN',
'GridName':
self.task_ad['CRAB_UserDN'],
'ApplicationVersion':
self.task_ad['CRAB_JobSW'],
'taskType':
self.task_ad.get("CRAB_DashboardTaskType", 'analysistest'),
'vo':
'cms',
'CMSUser':
self.task_ad['CRAB_UserHN'],
'user':
self.task_ad['CRAB_UserHN'],
'taskId':
self.task_ad['CRAB_ReqName'],
'datasetFull':
self.task_ad['DESIRED_CMSDataset'],
'resubmitter':
self.task_ad['CRAB_UserHN'],
'exe':
'cmsRun',
'broker':
self.backend,
'bossId':
str(self.job_id),
'localId':
'',
'SyncGridJobId':
'https://glidein.cern.ch/%s/%s' %
(self.job_id, self.task_ad['CRAB_ReqName'].replace("_", ":")),
}
if not self.userWebDirPrx:
storage_rules = htcondor.param['CRAB_StorageRules']
self.userWebDirPrx = getWebdirForDb(
str(self.task_ad.get('CRAB_ReqName')), storage_rules)
self.logger.info("User web dir: %s", self.userWebDirPrx)
if self.userWebDirPrx:
setDashboardLogs(params, self.userWebDirPrx, self.job_id,
crab_retry)
else:
print("Not setting dashboard logfiles as I cannot find a web dir.")
insertJobIdSid(params, self.job_id, self.task_ad['CRAB_ReqName'],
crab_retry)
apmon = ApmonIf()
self.logger.debug("Dashboard task info: %s", str(params))
apmon.sendToML(params)
apmon.free()
def initialize_(self, opts):
# Process the '-continue' option first because
# in the case of continuation the CRAB configuration
# parameters are loaded from already existing Working Space.
self.processContinueOption_(opts)
# Process ini-file first, then command line options
# because they override ini-file settings.
self.processIniFile_(opts)
if self.flag_continue: opts = self.loadConfiguration_(opts)
self.processOptions_(opts)
srvName = 'default'
self.UseServer = False
if opts.has_key('-use_server'):
self.UseServer = int(opts['-use_server'])
if self.UseServer==1:
opts['-server_name']='default'
if opts.has_key('-server_name'):
srvName = opts.get('-server_name', None)
if srvName == 'None':
srvName = None
self.UseServer = int( srvName is not None ) # cast bool to int
common._db = DBinterface(self.cfg_params)
isCreating = False
if not self.flag_continue:
self.createWorkingSpace_()
common._db.configureDB()
optsToBeSaved={}
optsToBeSavedDB={}
isCreating = True
for it in opts.keys():
if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
pass
else:
optsToBeSavedDB[it[1:]]=opts[it]
optsToBeSaved[it]=opts[it]
if self.UseServer==0: optsToBeSavedDB['server_name']= srvName
parsToBeSaved={}
for it in self.cfg_params.keys():
sec,act = string.split(it,".")
if sec == string.upper(common.prog_name):
tmp = "-"+act
if (tmp in self.main_actions) or (tmp in self.aux_actions) or (it == '-debug'):
pass
else:
parsToBeSaved[it]=self.cfg_params[it]
else:
parsToBeSaved[it]=self.cfg_params[it]
common.work_space.saveConfiguration(optsToBeSaved, parsToBeSaved)
else:
common._db.loadDB()
# At this point all configuration options have been read.
args = string.join(sys.argv,' ')
self.updateHistory_(args)
from crab_util import Color
Color=Color(False)
common.logger = CrabLogger(args)
common.logger.info(Color.yellow+Color.bold+self.headerString_()+Color.end)
## disallow old client releases
if checkCRABVersion(self.version()) is False:
msg = 'ERROR: you are using an old client release\n'
msg += '\tPlease upgrade to a more recent version:\n'
msg += '\thttps://twiki.cern.ch/twiki/bin/viewauth/CMS/SWGuideCrab#How_to_get_CRAB'
raise CrabException(msg)
## wmbs
self.automation=self.cfg_params.get('WMBS.automation','0')
if int(self.automation)==1 and self.UseServer!=1:
msg ='ERROR: if you want run automated WorkFlow you must use the server\n'
msg +='\tPlease check your configuration.'
raise CrabException(msg)
common.apmon = ApmonIf()
self.createScheduler_()
if not self.flag_continue:
common._db.createTask_(optsToBeSavedDB)
cmdOut = runCommand('uname -a')
msg = ('%s \nUsed properties:\n'%cmdOut)
if isCreating and common.debugLevel < 2 :
msg = ('%s \nUsed properties:\n'%cmdOut)
msg += self.UserCfgProperties(msg)
msg += 'End of used properties.\n'
common.logger.debug( msg)
else:
msg += self.UserCfgProperties(msg)
msg += 'End of used properties.\n'
common.logger.log(10-1, msg)
self.initializeActions_(opts)
return
