def validateConfig(self):
"""
__validateConfig__
Checking if needed input parameters are there
"""
if getattr(self.configuration, 'General', None) is None:
return False, "Crab configuration problem: general section is missing. "
if getattr(self.configuration, 'User', None) is None:
return False, "Crab configuration problem: User section is missing ."
if getattr(self.configuration, 'Data', None) is None:
return False, "Crab configuration problem: Data section is missing. "
else:
if hasattr(self.configuration.Data, 'unitsPerJob'):
#check that it is a valid number
try:
float(self.configuration.Data.unitsPerJob)
except ValueError:
return False, "Crab configuration problem: unitsPerJob must be a valid number, not %s" % self.configuration.Data.unitsPerJob
if getattr(self.configuration.Data, 'publication', None) and not (
hasattr(self.configuration.Data, 'publishDataName')
and hasattr(self.configuration.Data, 'publishDbsUrl')):
return False, "Crab configuration problem: if publication is selected publishDataName and publishDbsUrl are both required"
if getattr(self.configuration, 'Site', None) is None:
return False, "Crab configuration problem: Site section is missing. "
elif getattr(self.configuration.Site, "storageSite", None) is None:
return False, "Crab configuration problem: Site.storageSite parameter is missing. "
if getattr(self.configuration, 'JobType', None) is None:
return False, "Crab configuration problem: JobType section is missing. "
else:
if getattr(self.configuration.JobType, 'pluginName', None) is None and\
getattr(self.configuration.JobType, 'externalPluginFile', None) is None:
return False, "Crab configuration problem: one of JobType.pluginName or JobType.externalPlugin parameters is required. "
if getattr(self.configuration.JobType, 'pluginName', None) is not None and\
getattr(self.configuration.JobType, 'externalPluginFile', None) is not None:
return False, "Crab configuration problem: only one between JobType.pluginName or JobType.externalPlugin parameters is required. "
externalPlugin = getattr(self.configuration.JobType,
'externalPluginFile', None)
if externalPlugin is not None:
addPlugin(externalPlugin)
elif upper(self.configuration.JobType.pluginName
) not in getJobTypes():
msg = "JobType %s not found or not supported." % self.configuration.JobType.pluginName
return False, msg
return True, "Valid configuration"
def validateConfig(self):
"""
__validateConfig__
Checking if needed input parameters are there
"""
if getattr(self.configuration, 'General', None) is None:
return False, "Crab configuration problem: general section is missing. "
if getattr(self.configuration, 'User', None) is None:
return False, "Crab configuration problem: User section is missing ."
if getattr(self.configuration, 'Data', None) is None:
return False, "Crab configuration problem: Data section is missing. "
else:
if hasattr(self.configuration.Data, 'unitsPerJob'):
#check that it is a valid number
try:
float(self.configuration.Data.unitsPerJob)
except ValueError:
return False, "Crab configuration problem: unitsPerJob must be a valid number, not %s" % self.configuration.Data.unitsPerJob
if getattr(self.configuration.Data, 'publication', None) and not (hasattr(self.configuration.Data, 'publishDataName') and hasattr(self.configuration.Data, 'publishDbsUrl')):
return False, "Crab configuration problem: if publication is selected publishDataName and publishDbsUrl are both required"
if getattr(self.configuration, 'Site', None) is None:
return False, "Crab configuration problem: Site section is missing. "
elif getattr(self.configuration.Site, "storageSite", None) is None:
return False, "Crab configuration problem: Site.storageSite parameter is missing. "
if getattr(self.configuration, 'JobType', None) is None:
return False, "Crab configuration problem: JobType section is missing. "
else:
if getattr(self.configuration.JobType, 'pluginName', None) is None and\
getattr(self.configuration.JobType, 'externalPluginFile', None) is None:
return False, "Crab configuration problem: one of JobType.pluginName or JobType.externalPlugin parameters is required. "
if getattr(self.configuration.JobType, 'pluginName', None) is not None and\
getattr(self.configuration.JobType, 'externalPluginFile', None) is not None:
return False, "Crab configuration problem: only one between JobType.pluginName or JobType.externalPlugin parameters is required. "
externalPlugin = getattr(self.configuration.JobType, 'externalPluginFile', None)
if externalPlugin is not None:
addPlugin(externalPlugin)
elif upper(self.configuration.JobType.pluginName) not in getJobTypes():
msg = "JobType %s not found or not supported." % self.configuration.JobType.pluginName
return False, msg
return True, "Valid configuration"
def __call__(self):
valid = False
configmsg = 'Default'
self.logger.debug("Started submission")
serverFactory = CRABClient.Emulator.getEmulator('rest')
# Get some debug parameters
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
all_config_params = [x for x in parameters_mapping['other-config-params']]
for _, val in parameters_mapping['on-server'].iteritems():
if val['config']:
all_config_params.extend(val['config'])
SpellChecker.DICTIONARY = SpellChecker.train(all_config_params)
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.\nPlease refer to <https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial#CRAB_configuration_parameters> for list of valid parameter.\nSee the ./crab.log file for more details"' % par
msg += '' if SpellChecker.correct(par) == par else '\nOr maybe did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
uniquerequestname = None
self.logger.debug("Working on %s" % str(self.requestarea))
configreq = {}
for param in parameters_mapping['on-server']:
mustbetype = getattr(types, parameters_mapping['on-server'][param]['type'])
default = parameters_mapping['on-server'][param]['default']
config_params = parameters_mapping['on-server'][param]['config']
for config_param in config_params:
attrs = config_param.split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp is not None:
configreq[param] = temp
break
elif default is not None:
configreq[param] = default
temp = default
else:
## Parameter not strictly required.
pass
## Check that the requestname is of the right type.
## This is not checked in SubCommand.validateConfig().
if param == 'workflow':
if mustbetype == type(self.requestname):
configreq['workflow'] = self.requestname
## Translate boolean flags into integers.
elif param in ['savelogsflag', 'publication', 'nonprodsw', 'useparent', 'ignorelocality', 'saveoutput', 'oneEventMode']:
configreq[param] = 1 if temp else 0
## Translate DBS URL aliases into DBS URLs.
elif param in ['dbsurl', 'publishdbsurl']:
if param == 'dbsurl':
dbstype = 'reader'
elif param == 'publishdbsurl':
dbstype = 'writer'
allowed_dbsurls = DBSURLS[dbstype].values()
allowed_dbsurls_aliases = DBSURLS[dbstype].keys()
if configreq[param] in allowed_dbsurls_aliases:
configreq[param] = DBSURLS[dbstype][configreq[param]]
elif configreq[param].rstrip('/') in allowed_dbsurls:
configreq[param] = configreq[param].rstrip('/')
elif param == 'scriptexe' and 'scriptexe' in configreq:
configreq[param] = os.path.basename(configreq[param])
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = serverFactory.getCACertPath()
#get the backend URLs from the server external configuration
serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, getUrl(self.instance, resource='info'))
#if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored)
self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None
pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ]
crab_job_types = getJobTypes()
if upper(configreq['jobtype']) in crab_job_types:
plugjobtype = crab_job_types[upper(configreq['jobtype'])](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = configreq['jobtype']
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
if configreq['publication']:
non_edm_files = jobconfig['tfileoutfiles'] + jobconfig['addoutputfiles']
if non_edm_files:
msg = "%sWARNING%s: The following output files will not be published, as they are not EDM files: %s" % (colors.RED, colors.NORMAL, non_edm_files)
self.logger.warning(msg)
if not configreq['publishname']:
configreq['publishname'] = isbchecksum
else:
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str(configreq))
## TODO: this shouldn't be hard-coded.
listParams = ['adduserfiles', 'addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist', \
'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles', 'scriptargs', 'extrajdl']
configreq_encoded = self._encodeRequest(configreq, listParams)
self.logger.debug('Encoded submit request: %s' % (configreq_encoded))
dictresult, status, reason = server.put( self.uri, data = configreq_encoded)
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname,
voRole=self.voRole, voGroup=self.voGroup, instance=self.instance,
originalConfig = self.configuration)
self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL))
if not self.options.wait:
self.logger.info("Task name: %s" % uniquerequestname)
self.logger.info("Please use 'crab status' to check how the submission process proceed")
if self.options.wait:
self.checkStatusLoop(server,uniquerequestname)
self.logger.debug("About to return")
return {'requestname' : self.requestname , 'uniquerequestname' : uniquerequestname }
def validateConfig(self):
"""
__validateConfig__
Checking if needed input parameters are there
"""
valid, msg = SubCommand.validateConfig(self)
if not valid:
return False, msg
## Check that the configuration object has the sections we expect it to have.
## (WMCore already checks that attributes added to the configuration object are of type ConfigSection.)
## Even if not all configuration sections need to be there, we anyway request
## the user to add all the sections in the configuration file.
if not hasattr(self.configuration, 'General'):
msg = "CRAB configuration problem: Section 'General' is missing"
return False, msg
if not hasattr(self.configuration, 'JobType'):
msg = "CRAB configuration problem: Section 'JobType' is missing"
return False, msg
if not hasattr(self.configuration, 'Data'):
msg = "CRAB configuration problem: Section 'Data' is missing"
return False, msg
if not hasattr(self.configuration, 'Site'):
msg = "CRAB configuration problem: Section 'Site' is missing"
return False, msg
## Some parameters may have been renamed. Check here if the configuration file has an old
## parameter defined, and in that case tell the user what is the new parameter name.
for old_param, new_param in renamed_params.iteritems():
if len(old_param.split('.')) != 2 or len(new_param.split('.')) != 2:
continue
old_param_section, old_param_name = old_param.split('.')
if hasattr(self.configuration, old_param_section) and hasattr(getattr(self.configuration, old_param_section), old_param_name):
msg = "CRAB configuration problem: Parameter %s has been renamed to %s; please change your configuration file accordingly" % (old_param, new_param)
return False, msg
## Check that Data.unitsPerjob is specified.
if hasattr(self.configuration.Data, 'unitsPerJob'):
try:
float(self.configuration.Data.unitsPerJob)
except ValueError:
msg = "CRAB configuration problem: Parameter Data.unitsPerJob must be a valid number, not %s" % self.configuration.Data.unitsPerJob
return False, msg
## Check that JobType.pluginName and JobType.externalPluginFile are not both specified.
if hasattr(self.configuration.JobType, 'pluginName') and hasattr(self.configuration.JobType, 'externalPluginFile'):
msg = "CRAB configuration problem: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified"
pluginName_default = getParamDefaultValue('JobType.pluginName')
if pluginName_default:
msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified, the default JobType.pluginName = '%s' would be used" \
% pluginName_default
return False, msg
## Load the external plugin or check that the crab plugin is valid.
external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None)
crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None)
crab_job_types = {'ANALYSIS': None, 'PRIVATEMC': None} #getJobTypes()
if external_plugin_name:
addPlugin(external_plugin_name) # Do we need to do this here?
if crab_plugin_name and upper(crab_plugin_name) not in crab_job_types:
msg = "CRAB configuration problem: Parameter JobType.pluginName has an invalid value '%s'" % crab_plugin_name
msg += "\nAllowed values are: %s" % ", ".join(['%s' % job_type for job_type in crab_job_types.keys()])
return False, msg
## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified.
if hasattr(self.configuration.Data, 'publication') and hasattr(self.configuration.General, 'transferOutputs'):
if self.configuration.Data.publication and not self.configuration.General.transferOutputs:
msg = "CRAB configuration problem: Data.publication is on, but General.transferOutputs is off"
msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage"
return False, msg
## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True.
if not hasattr(self.configuration.Site, 'storageSite'):
if (hasattr(self.configuration.General, 'transferOutputs') and self.configuration.General.transferOutputs) or \
(hasattr(self.configuration.General, 'transferLogs') and self.configuration.General.transferLogs):
msg = "CRAB configuration problem: Parameter Site.storageSite is missing"
return False, msg
## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one.
## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER.
if hasattr(self.configuration.Data, 'inputDBS'):
if hasattr(self.configuration.Data, 'inputDataset'):
msg = None
dbs_urls_aliases = DBSURLS['reader'].keys()
dbs_urls = DBSURLS['reader'].values()
if (self.configuration.Data.inputDBS not in dbs_urls_aliases) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls):
msg = "CRAB configuration problem: Parameter Data.inputDBS has an invalid value '%s'" % self.configuration.Data.inputDBS
msg += "\nAllowed values are: "
msg += "\n ".join(["'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems()])
local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03']
local_dbs_urls = [DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader']]
if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases:
inputDataset_parts = self.configuration.Data.inputDataset.split('/')
inputDataset_parts.pop(0)
inputDataset_tier = inputDataset_parts[-1] if len(inputDataset_parts) == 3 else None
user_data_tiers = ['USER']
if inputDataset_tier not in user_data_tiers:
msg = "CRAB configuration problem: A local DBS instance '%s' was specified for reading an input dataset of tier %s" \
% (self.configuration.Data.inputDBS, inputDataset_tier)
msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'" \
% (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0])
if msg:
inputDBS_default = getParamDefaultValue('Data.inputDBS')
if inputDBS_default:
inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias(inputDBS_default, 'reader')
if inputDBS_default and inputDBS_default_alias:
msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used" % (inputDBS_default_alias, inputDBS_default)
return False, msg
## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one.
if hasattr(self.configuration.Data, 'publishDBS'):
publication_default = getParamDefaultValue('Data.publication')
if getattr(self.configuration.Data, 'publication', publication_default):
dbs_urls = DBSURLS['writer'].values()
dbs_urls_aliases = DBSURLS['writer'].keys()
if (self.configuration.Data.publishDBS not in dbs_urls_aliases) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls):
msg = "CRAB configuration problem: Parameter Data.publishDBS has an invalid value '%s'" % self.configuration.Data.publishDBS
msg += "\nAllowed values are: "
msg += "\n ".join(["'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems()])
publishDBS_default = getParamDefaultValue('Data.publishDBS')
if publishDBS_default:
publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias(publishDBS_default, 'writer')
if publishDBS_default and publishDBS_default_alias:
msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used" \
% (publishDBS_default_alias, publishDBS_default)
return False, msg
if hasattr(self.configuration.JobType, 'scriptExe'):
if not os.path.isfile(self.configuration.JobType.scriptExe):
msg = "Cannot find the file %s specified in the scriptExe configuration parameter" % self.configuration.JobType.scriptExe
return False, msg
return True, "Valid configuration"
def __call__(self):
valid = False
configmsg = 'Default'
if not os.path.isfile(self.options.config):
raise MissingOptionException("Configuration file '%s' not found" % self.options.config)
#store the configuration file in self.configuration
self.loadConfig( self.options.config, self.args )
requestarea, requestname, self.logfile = createWorkArea( self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None)
)
self.logger.debug("Started submission")
#determine the serverurl
if self.options.server:
self.serverurl = self.options.server
elif getattr( self.configuration.General, 'serverUrl', None ) is not None:
self.serverurl = self.configuration.General.serverUrl
#TODO: For sure the server url should not be handled here. Find an intelligent way for this
else:
self.serverurl = 'http://cmsweb.cern.ch'
if not hasattr( self.configuration.General, 'ufccacheUrl' ):
self.configuration.General.ufccacheUrl = self.serverurl
if not hasattr( self.configuration.General, 'configcacheUrl' ):
#https is required because configcache does not use ServerInteractions
self.configuration.General.configcacheUrl = 'https://' + self.serverurl + '/couchdb'
if not hasattr( self.configuration.General, 'configcacheName' ):
self.configuration.General.configcacheName = 'analysis_reqmgr_config_cache'
self.createCache( self.serverurl )
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
#delegating the proxy (creation done in SubCommand)
self.voRole = getattr(self.configuration.User, "voRole", "")
self.voGroup = getattr(self.configuration.User, "voGroup", "")
self.handleProxy()
uniquerequestname = None
self.logger.debug("Working on %s" % str(requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException(1, "Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException(1, "Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(requestname):
configreq["workflow"] = requestname
elif param == "savelogsflag":
configreq["savelogsflag"] = 1 if temp else 0
elif param == "publication":
configreq["publication"] = 1 if temp else 0
elif param == "blacklistT1":
blacklistT1 = self.voRole != 't1access'
#if the user choose to remove the automatic T1 blacklisting and has not the t1acces role
if getattr (self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1:
self.logger.info("WARNING: You disabled the T1 automatic blacklisting without having the t1access role")
blacklistT1 = False
configreq["blacklistT1"] = 1 if blacklistT1 else 0
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
pluginParams = [ self.configuration, self.logger, os.path.join(requestarea, 'inputs') ]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str( configreq ) )
dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) )
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache( requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole = self.voRole, voGroup = self.voGroup )
self.logger.info("Submission completed")
self.logger.debug("Request ID: %s " % uniquerequestname)
self.logger.debug("Ended submission")
return uniquerequestname
def __call__(self):
valid = False
configmsg = 'Default'
self.logger.debug("Started submission")
# Get some debug parameters
oneEventMode = hasattr(self.configuration, 'Debug') and \
getattr(self.configuration.Debug, 'oneEventMode')
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
uniquerequestname = None
self.logger.debug("Working on %s" % str(self.requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp is not None:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
temp = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(self.requestname):
configreq["workflow"] = self.requestname
elif param in ['savelogsflag','publication','nonprodsw','ignorelocality','saveoutput']:#TODO use clientmappig to do this
configreq[param] = 1 if temp else 0
elif param in ['dbsurl','publishdbsurl']:
if param == 'dbsurl':
dbstype = 'reader'
elif param == 'publishdbsurl':
dbstype = 'writer'
alloweddbsurls = DBSURLS[dbstype].values()
alloweddbsurlsaliases = DBSURLS[dbstype].keys()
if configreq[param] in alloweddbsurlsaliases:
configreq[param] = DBSURLS[dbstype][configreq[param]]
else:
if configreq[param].rstrip('/') in alloweddbsurls:
configreq[param] = configreq[param].rstrip('/')
else:
raise ConfigurationException("Invalid argument " + configreq[param] + " for parameter " + self.requestmapper[param]['config'] + " in the configuration.")
if (configreq['saveoutput'] or configreq['savelogsflag']) and 'asyncdest' not in configreq:
raise ConfigurationException("Missing parameter " + self.requestmapper['asyncdest']['config'] + " from the configuration.")
# Add debug parameters to the configreq dict
configreq['oneEventMode'] = int(oneEventMode)
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
#get the backend URLs from the server external configuration
serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info'))
#if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored)
self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None
pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
if not configreq['publishname']:
configreq['publishname'] = isbchecksum
else:
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str( configreq ) )
dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) )
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname,
voRole=self.voRole, voGroup=self.voGroup, instance=self.instance,
originalConfig = self.configuration)
self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL))
if not self.options.wait:
self.logger.info("Please use 'crab status' to check how the submission process proceed")
self.logger.debug("Request ID: %s " % uniquerequestname)
if self.options.wait:
self.checkStatusLoop(server,uniquerequestname)
return uniquerequestname
def __call__(self):
valid = False
configmsg = 'Default'
if not os.path.isfile(self.options.config):
raise MissingOptionException("Configuration file '%s' not found" %
self.options.config)
#store the configuration file in self.configuration
self.loadConfig(self.options.config, self.args)
requestarea, requestname, self.logfile = createWorkArea(
self.logger, getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
self.logger.debug("Started submission")
#determine the serverurl
if self.options.server:
self.serverurl = self.options.server
elif getattr(self.configuration.General, 'serverUrl',
None) is not None:
self.serverurl = self.configuration.General.serverUrl
#TODO: For sure the server url should not be handled here. Find an intelligent way for this
else:
self.serverurl = 'http://cmsweb.cern.ch'
if not hasattr(self.configuration.General, 'ufccacheUrl'):
self.configuration.General.ufccacheUrl = self.serverurl
if not hasattr(self.configuration.General, 'configcacheUrl'):
#https is required because configcache does not use ServerInteractions
self.configuration.General.configcacheUrl = 'https://' + self.serverurl + '/couchdb'
if not hasattr(self.configuration.General, 'configcacheName'):
self.configuration.General.configcacheName = 'analysis_reqmgr_config_cache'
self.createCache(self.serverurl)
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct(par):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(
par
) == par else ' Did you mean %s?' % SpellChecker.correct(
par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
#delegating the proxy (creation done in SubCommand)
self.voRole = getattr(self.configuration.User, "voRole", "")
self.voGroup = getattr(self.configuration.User, "voGroup", "")
self.handleProxy()
uniquerequestname = None
self.logger.debug("Working on %s" % str(requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException(1, "Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException(
1, "Missing parameter " +
self.requestmapper[param]['config'] +
" from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(requestname):
configreq["workflow"] = requestname
elif param == "savelogsflag":
configreq["savelogsflag"] = 1 if temp else 0
elif param == "publication":
configreq["publication"] = 1 if temp else 0
elif param == "blacklistT1":
blacklistT1 = self.voRole != 't1access'
#if the user choose to remove the automatic T1 blacklisting and has not the t1acces role
if getattr(self.configuration.Site, 'removeT1Blacklisting',
False) and blacklistT1:
self.logger.info(
"WARNING: You disabled the T1 automatic blacklisting without having the t1access role"
)
blacklistT1 = False
configreq["blacklistT1"] = 1 if blacklistT1 else 0
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
pluginParams = [
self.configuration, self.logger,
os.path.join(requestarea, 'inputs')
]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(
self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
configreq['publishname'] = "%s-%s" % (configreq['publishname'],
isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str(configreq))
dictresult, status, reason = server.put(
self.uri, data=self._encodeRequest(configreq))
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (
str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(requestarea,
tmpsplit[0],
tmpsplit[1] if len(tmpsplit) > 1 else '',
uniquerequestname,
voRole=self.voRole,
voGroup=self.voGroup)
self.logger.info("Submission completed")
self.logger.debug("Request ID: %s " % uniquerequestname)
self.logger.debug("Ended submission")
return uniquerequestname
def __call__(self):
valid = False
configmsg = 'Default'
self.logger.debug("Started submission")
serverFactory = CRABClient.Emulator.getEmulator('rest')
# Get some debug parameters
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
all_config_params = [
x for x in parameters_mapping['other-config-params']
]
for _, val in parameters_mapping['on-server'].iteritems():
if val['config']:
all_config_params.extend(val['config'])
SpellChecker.DICTIONARY = SpellChecker.train(all_config_params)
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct(par):
msg = 'The parameter %s is not known.\nPlease refer to <https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial#CRAB_configuration_parameters> for list of valid parameter.\nSee the ./crab.log file for more details"' % par
msg += '' if SpellChecker.correct(
par
) == par else '\nOr maybe did you mean %s?' % SpellChecker.correct(
par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
uniquerequestname = None
self.logger.debug("Working on %s" % str(self.requestarea))
configreq = {}
for param in parameters_mapping['on-server']:
mustbetype = getattr(
types, parameters_mapping['on-server'][param]['type'])
default = parameters_mapping['on-server'][param]['default']
config_params = parameters_mapping['on-server'][param]['config']
for config_param in config_params:
attrs = config_param.split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp is not None:
configreq[param] = temp
break
elif default is not None:
configreq[param] = default
temp = default
else:
## Parameter not strictly required.
pass
## Check that the requestname is of the right type.
## This is not checked in SubCommand.validateConfig().
if param == 'workflow':
if mustbetype == type(self.requestname):
configreq['workflow'] = self.requestname
## Translate boolean flags into integers.
elif param in [
'savelogsflag', 'publication', 'nonprodsw', 'useparent',
'ignorelocality', 'saveoutput', 'oneEventMode'
]:
configreq[param] = 1 if temp else 0
## Translate DBS URL aliases into DBS URLs.
elif param in ['dbsurl', 'publishdbsurl']:
if param == 'dbsurl':
dbstype = 'reader'
elif param == 'publishdbsurl':
dbstype = 'writer'
allowed_dbsurls = DBSURLS[dbstype].values()
allowed_dbsurls_aliases = DBSURLS[dbstype].keys()
if configreq[param] in allowed_dbsurls_aliases:
configreq[param] = DBSURLS[dbstype][configreq[param]]
elif configreq[param].rstrip('/') in allowed_dbsurls:
configreq[param] = configreq[param].rstrip('/')
elif param == 'scriptexe' and 'scriptexe' in configreq:
configreq[param] = os.path.basename(configreq[param])
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = serverFactory.getCACertPath()
#get the backend URLs from the server external configuration
serverBackendURLs = server_info('backendurls', self.serverurl,
self.proxyfilename,
getUrl(self.instance, resource='info'))
#if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored)
self.configuration.JobType.filecacheurl = serverBackendURLs[
'cacheSSL'] if 'cacheSSL' in serverBackendURLs else None
pluginParams = [
self.configuration, self.logger,
os.path.join(self.requestarea, 'inputs')
]
crab_job_types = getJobTypes()
if upper(configreq['jobtype']) in crab_job_types:
plugjobtype = crab_job_types[upper(
configreq['jobtype'])](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = configreq['jobtype']
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
if configreq['publication']:
non_edm_files = jobconfig['tfileoutfiles'] + jobconfig[
'addoutputfiles']
if non_edm_files:
msg = "%sWARNING%s: The following output files will not be published, as they are not EDM files: %s" % (
colors.RED, colors.NORMAL, non_edm_files)
self.logger.warning(msg)
if not configreq['publishname']:
configreq['publishname'] = isbchecksum
else:
configreq['publishname'] = "%s-%s" % (configreq['publishname'],
isbchecksum)
configreq.update(jobconfig)
server = serverFactory(self.serverurl,
self.proxyfilename,
self.proxyfilename,
version=__version__)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str(configreq))
## TODO: this shouldn't be hard-coded.
listParams = ['adduserfiles', 'addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist', \
'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles', 'scriptargs', 'extrajdl']
configreq_encoded = self._encodeRequest(configreq, listParams)
self.logger.debug('Encoded submit request: %s' % (configreq_encoded))
dictresult, status, reason = server.put(self.uri,
data=configreq_encoded)
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (
str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(self.requestarea,
tmpsplit[0],
tmpsplit[1] if len(tmpsplit) > 1 else '',
uniquerequestname,
voRole=self.voRole,
voGroup=self.voGroup,
instance=self.instance,
originalConfig=self.configuration)
self.logger.info(
"%sSuccess%s: Your task has been delivered to the CRAB3 server." %
(colors.GREEN, colors.NORMAL))
if not self.options.wait:
self.logger.info("Task name: %s" % uniquerequestname)
self.logger.info(
"Please use 'crab status' to check how the submission process proceed"
)
if self.options.wait:
self.checkStatusLoop(server, uniquerequestname)
self.logger.debug("About to return")
return {
'requestname': self.requestname,
'uniquerequestname': uniquerequestname
}
def validateConfig(self):
"""
__validateConfig__
Checking if needed input parameters are there
"""
valid, msg = SubCommand.validateConfig(self)
if not valid:
return False, msg
## Check that the configuration object has the sections we expect it to have.
## (WMCore already checks that attributes added to the configuration object are of type ConfigSection.)
## Even if not all configuration sections need to be there, we anyway request
## the user to add all the sections in the configuration file.
if not hasattr(self.configuration, 'General'):
msg = "CRAB configuration problem: Section 'General' is missing"
return False, msg
if not hasattr(self.configuration, 'JobType'):
msg = "CRAB configuration problem: Section 'JobType' is missing"
return False, msg
if not hasattr(self.configuration, 'Data'):
msg = "CRAB configuration problem: Section 'Data' is missing"
return False, msg
if not hasattr(self.configuration, 'Site'):
msg = "CRAB configuration problem: Section 'Site' is missing"
return False, msg
## Some parameters may have been renamed. Check here if the configuration file has an old
## parameter defined, and in that case tell the user what is the new parameter name.
for old_param, new_param in renamed_params.iteritems():
if len(old_param.split('.')) != 2 or len(
new_param.split('.')) != 2:
continue
old_param_section, old_param_name = old_param.split('.')
if hasattr(self.configuration, old_param_section) and hasattr(
getattr(self.configuration, old_param_section),
old_param_name):
msg = "CRAB configuration problem: Parameter %s has been renamed to %s; please change your configuration file accordingly" % (
old_param, new_param)
return False, msg
## Check that Data.unitsPerjob is specified.
if hasattr(self.configuration.Data, 'unitsPerJob'):
try:
float(self.configuration.Data.unitsPerJob)
except ValueError:
msg = "CRAB configuration problem: Parameter Data.unitsPerJob must be a valid number, not %s" % self.configuration.Data.unitsPerJob
return False, msg
## Check that JobType.pluginName and JobType.externalPluginFile are not both specified.
if hasattr(self.configuration.JobType, 'pluginName') and hasattr(
self.configuration.JobType, 'externalPluginFile'):
msg = "CRAB configuration problem: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified"
pluginName_default = getParamDefaultValue('JobType.pluginName')
if pluginName_default:
msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified, the default JobType.pluginName = '%s' would be used" \
% pluginName_default
return False, msg
## Load the external plugin or check that the crab plugin is valid.
external_plugin_name = getattr(self.configuration.JobType,
'externalPluginFile', None)
crab_plugin_name = getattr(self.configuration.JobType, 'pluginName',
None)
crab_job_types = {'ANALYSIS': None, 'PRIVATEMC': None} #getJobTypes()
if external_plugin_name:
addPlugin(external_plugin_name) # Do we need to do this here?
if crab_plugin_name and upper(crab_plugin_name) not in crab_job_types:
msg = "CRAB configuration problem: Parameter JobType.pluginName has an invalid value '%s'" % crab_plugin_name
msg += "\nAllowed values are: %s" % ", ".join(
['%s' % job_type for job_type in crab_job_types.keys()])
return False, msg
## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified.
if hasattr(self.configuration.Data, 'publication') and hasattr(
self.configuration.General, 'transferOutputs'):
if self.configuration.Data.publication and not self.configuration.General.transferOutputs:
msg = "CRAB configuration problem: Data.publication is on, but General.transferOutputs is off"
msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage"
return False, msg
## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True.
if not hasattr(self.configuration.Site, 'storageSite'):
if (hasattr(self.configuration.General, 'transferOutputs') and self.configuration.General.transferOutputs) or \
(hasattr(self.configuration.General, 'transferLogs') and self.configuration.General.transferLogs):
msg = "CRAB configuration problem: Parameter Site.storageSite is missing"
return False, msg
## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one.
## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER.
if hasattr(self.configuration.Data, 'inputDBS'):
if hasattr(self.configuration.Data, 'inputDataset'):
msg = None
dbs_urls_aliases = DBSURLS['reader'].keys()
dbs_urls = DBSURLS['reader'].values()
if (self.configuration.Data.inputDBS not in dbs_urls_aliases
) and (self.configuration.Data.inputDBS.rstrip('/')
not in dbs_urls):
msg = "CRAB configuration problem: Parameter Data.inputDBS has an invalid value '%s'" % self.configuration.Data.inputDBS
msg += "\nAllowed values are: "
msg += "\n ".join([
"'%s' ('%s')" % (alias, url)
for alias, url in DBSURLS['reader'].iteritems()
])
local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03']
local_dbs_urls = [
DBSURLS['reader'][alias]
for alias in local_dbs_urls_aliases
if alias in DBSURLS['reader']
]
if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases:
inputDataset_parts = self.configuration.Data.inputDataset.split(
'/')
inputDataset_parts.pop(0)
inputDataset_tier = inputDataset_parts[-1] if len(
inputDataset_parts) == 3 else None
user_data_tiers = ['USER']
if inputDataset_tier not in user_data_tiers:
msg = "CRAB configuration problem: A local DBS instance '%s' was specified for reading an input dataset of tier %s" \
% (self.configuration.Data.inputDBS, inputDataset_tier)
msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'" \
% (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0])
if msg:
inputDBS_default = getParamDefaultValue('Data.inputDBS')
if inputDBS_default:
inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias(
inputDBS_default, 'reader')
if inputDBS_default and inputDBS_default_alias:
msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used" % (
inputDBS_default_alias, inputDBS_default)
return False, msg
## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one.
if hasattr(self.configuration.Data, 'publishDBS'):
publication_default = getParamDefaultValue('Data.publication')
if getattr(self.configuration.Data, 'publication',
publication_default):
dbs_urls = DBSURLS['writer'].values()
dbs_urls_aliases = DBSURLS['writer'].keys()
if (self.configuration.Data.publishDBS not in dbs_urls_aliases
) and (self.configuration.Data.publishDBS.rstrip('/')
not in dbs_urls):
msg = "CRAB configuration problem: Parameter Data.publishDBS has an invalid value '%s'" % self.configuration.Data.publishDBS
msg += "\nAllowed values are: "
msg += "\n ".join([
"'%s' ('%s')" % (alias, url)
for alias, url in DBSURLS['writer'].iteritems()
])
publishDBS_default = getParamDefaultValue(
'Data.publishDBS')
if publishDBS_default:
publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias(
publishDBS_default, 'writer')
if publishDBS_default and publishDBS_default_alias:
msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used" \
% (publishDBS_default_alias, publishDBS_default)
return False, msg
if hasattr(self.configuration.JobType, 'scriptExe'):
if not os.path.isfile(self.configuration.JobType.scriptExe):
msg = "Cannot find the file %s specified in the scriptExe configuration parameter" % self.configuration.JobType.scriptExe
return False, msg
return True, "Valid configuration"
def __call__(self):
valid = False
configmsg = 'Default'
if not os.path.isfile(self.options.config):
raise MissingOptionException("Configuration file '%s' not found" % self.options.config)
self.logger.debug("Started submission")
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
uniquerequestname = None
self.logger.debug("Working on %s" % str(self.requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(self.requestname):
configreq["workflow"] = self.requestname
elif param == "savelogsflag":
configreq["savelogsflag"] = 1 if temp else 0
elif param == "publication":
configreq["publication"] = 1 if temp else 0
elif param == "blacklistT1":
blacklistT1 = self.voRole != 't1access'
#if the user choose to remove the automatic T1 blacklisting and has not the t1acces role
if getattr (self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1:
self.logger.info("WARNING: You disabled the T1 automatic blacklisting without having the t1access role")
blacklistT1 = False
configreq["blacklistT1"] = 1 if blacklistT1 else 0
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
#get the backend URLs from the server external configuration
serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info'))
#if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored)
self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None
#otherwise we will contact the baseurl to get the cache hostname
self.configuration.JobType.baseurl = serverBackendURLs['baseURL']
pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
if not configreq['publishname']:
configreq['publishname'] = isbchecksum
else:
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str( configreq ) )
dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) )
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname,
voRole=self.voRole, voGroup=self.voGroup, instance=self.instance)
self.logger.info("Submission completed")
self.logger.debug("Request ID: %s " % uniquerequestname)
self.logger.debug("Ended submission")
return uniquerequestname
