def __init__(self, logger, cmdargs = []):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
self.usage = "usage: %prog " + self.name + " [options] [args]"
self.logger = logger
self.logfile = ''
self.logger.debug("Executing command: '%s'" % str(self.name))
self.crab3dic=self.getConfiDict()
##Get the mapping
self.loadMapping()
self.parser = OptionParser(description = self.__doc__, usage = self.usage, add_help_option = True)
## TODO: check on self.name should be removed (creating another abstraction in between or refactoring this)
if self.name == 'submit':
self.parser.disable_interspersed_args()
self.setSuperOptions()
##Parse the command line parameters
(self.options, self.args) = self.parser.parse_args( cmdargs )
##Validate the command line parameters before initing the proxy
self.validateOptions()
##if we get an input configuration we load it
if hasattr(self.options, 'config') and self.options.config is not None:
self.loadConfig( self.options.config, self.args )
self.requestarea, self.requestname, self.logfile = createWorkArea(self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
self.voRole = self.options.voRole if self.options.voRole else getattr(self.configuration.User, "voRole", "")
self.voGroup = self.options.voGroup if self.options.voGroup else getattr(self.configuration.User, "voGroup", "")
##if we get an input task we load the cache and set the url from it
if hasattr(self.options, 'task') and self.options.task:
self.loadLocalCache()
## if the server url isn't already set we check the args and then the config
if not hasattr(self, 'serverurl'):
self.instance, self.serverurl = self.serverInstance()
self.updateCrab3()
self.handleProxy(self.getUrl(self.instance, resource='info'))
self.checkversion(self.getUrl(self.instance, resource='info'))
self.uri = self.getUrl(self.instance)
self.logger.debug("Instance is %s" %(self.instance))
self.logger.debug("Server base url is %s" %(self.serverurl))
self.logger.debug("Command url %s" %(self.uri))
def __call__(self):
valid = False
configmsg = 'Default'
if not os.path.isfile(self.options.config):
raise MissingOptionException("Configuration file '%s' not found" % self.options.config)
#store the configuration file in self.configuration
self.loadConfig( self.options.config, self.args )
requestarea, requestname, self.logfile = createWorkArea( self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None)
)
self.logger.debug("Started submission")
#determine the serverurl
if self.options.server:
self.serverurl = self.options.server
elif getattr( self.configuration.General, 'serverUrl', None ) is not None:
self.serverurl = self.configuration.General.serverUrl
#TODO: For sure the server url should not be handled here. Find an intelligent way for this
else:
self.serverurl = 'http://cmsweb.cern.ch'
if not hasattr( self.configuration.General, 'ufccacheUrl' ):
self.configuration.General.ufccacheUrl = self.serverurl
if not hasattr( self.configuration.General, 'configcacheUrl' ):
#https is required because configcache does not use ServerInteractions
self.configuration.General.configcacheUrl = 'https://' + self.serverurl + '/couchdb'
if not hasattr( self.configuration.General, 'configcacheName' ):
self.configuration.General.configcacheName = 'analysis_reqmgr_config_cache'
self.createCache( self.serverurl )
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
#delegating the proxy (creation done in SubCommand)
self.voRole = getattr(self.configuration.User, "voRole", "")
self.voGroup = getattr(self.configuration.User, "voGroup", "")
self.handleProxy()
uniquerequestname = None
self.logger.debug("Working on %s" % str(requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException(1, "Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException(1, "Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(requestname):
configreq["workflow"] = requestname
elif param == "savelogsflag":
configreq["savelogsflag"] = 1 if temp else 0
elif param == "publication":
configreq["publication"] = 1 if temp else 0
elif param == "blacklistT1":
blacklistT1 = self.voRole != 't1access'
#if the user choose to remove the automatic T1 blacklisting and has not the t1acces role
if getattr (self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1:
self.logger.info("WARNING: You disabled the T1 automatic blacklisting without having the t1access role")
blacklistT1 = False
configreq["blacklistT1"] = 1 if blacklistT1 else 0
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
pluginParams = [ self.configuration, self.logger, os.path.join(requestarea, 'inputs') ]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str( configreq ) )
dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) )
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache( requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole = self.voRole, voGroup = self.voGroup )
self.logger.info("Submission completed")
self.logger.debug("Request ID: %s " % uniquerequestname)
self.logger.debug("Ended submission")
return uniquerequestname
def __init__(self, logger, cmdargs = None, disable_interspersed_args = False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
self.usage = "usage: %prog " + self.name + " [options] [args]"
self.logger = logger
self.logfile = self.logger.logfile
self.logger.debug("Executing command: '%s'" % str(self.name))
self.proxy = None
self.restClass = CRABClient.Emulator.getEmulator('rest')
self.cmdconf = commands_configuration.get(self.name)
self.crab3dic = self.getConfiDict()
self.parser = OptionParser(description = self.__doc__, usage = self.usage, add_help_option = True)
## TODO: check on self.name should be removed (creating another abstraction in between or refactoring this)
if disable_interspersed_args:
self.parser.disable_interspersed_args()
self.setSuperOptions()
## Parse the command line parameters.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
## Validate the command line parameters before initializing the proxy.
self.validateOptions()
## Retrieve VO role/group from the command line parameters.
self.voRole = self.options.voRole if self.options.voRole is not None else ''
self.voGroup = self.options.voGroup if self.options.voGroup is not None else ''
## Create the object that will do the proxy operations. We ignore the VO role/group and
## server URL in the initialization, because they are not used until we do the proxy
## delegation to myproxy server. And the delegation happends in handleProxy(), which is
## called after we load the configuration file and retrieve the final values for those
## parameters. handleProxy() takes care of passing those parameters to self.proxy.
self.proxy = CredentialInteractions('', '', self.voRole, self.voGroup, self.logger, '')
## Create a new proxy (if there isn't a valid one already) with current VO role/group.
## The VO role/group are not relevant for the proxy. We only want to check that the
## user doesn't expect the proxy to have different VO role/group than those specified
## in the configuration file, but this check is done later in handleProxy() after we
## load the configuration file.
self.proxy_created = False
if not self.options.proxy and self.cmdconf['initializeProxy']:
self.proxy_created = self.proxy.createNewVomsProxySimple(time_left_threshold = 720)
## If we get an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
## Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
## Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
## If VO role/group were not given in the command options, get them from the configuration file.
## If they are specified in both places, print a message saying that the command options will be used.
if self.options.voRole is None and hasattr(self.configuration, 'User'):
self.voRole = getattr(self.configuration.User, 'voRole', '')
if self.options.voGroup is None and hasattr(self.configuration, 'User'):
self.voGroup = getattr(self.configuration.User, 'voGroup', '')
if (self.options.voRole is not None) and (hasattr(self.configuration, 'User') and hasattr(self.configuration.User, 'voRole' )):
msg = "Ignoring the VO role specified in the configuration file. Using VO role \"%s\" "
if self.voRole == '': msg += "(i.e. no VO role) "
msg += "as specified in the command line."
self.logger.info(msg % self.voRole)
if (self.options.voGroup is not None) and (hasattr(self.configuration, 'User') and hasattr(self.configuration.User, 'voGroup')):
msg = "Ignoring the VO group specified in the configuration file. Using VO group \"%s\" "
if self.voGroup == '': msg += "(i.e. no VO group) "
msg += "as specified in the command line."
self.logger.info(msg % self.voGroup)
## If we get an input task, we load the cache and set the server URL and VO role/group from it.
if hasattr(self.options, 'task') and self.options.task:
self.loadLocalCache()
## If the server url isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
## Update (or create) the .crab3 cache file.
self.updateCrab3()
## At this point there should be a valid proxy, because we have already checked that and
## eventually created a new one. If the proxy was not created by CRAB, we check that the
## VO role/group in the proxy are the same as specified by the user in the configuration
## file (or in the command line options). If it is not, we ask the user if he wants to
## overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
## and ask him to provide the VO role/group as in the existing proxy.
## Finally, delegate the proxy to myproxy server.
self.handleProxy()
## Logging user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion(self.getUrl(self.instance, resource='info'))
self.uri = self.getUrl(self.instance)
self.logger.debug("Instance is %s" %(self.instance))
self.logger.debug("Server base url is %s" %(self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command url %s" %(self.uri))
def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
self.usage = "usage: %prog " + self.name + " [options] [args]"
self.logger = logger
self.logfile = self.logger.logfile
self.logger.debug("Executing command: '%s'" % str(self.name))
self.proxy = None
self.restClass = CRABClient.Emulator.getEmulator('rest')
self.cmdconf = commands_configuration.get(self.name)
self.crab3dic = self.getConfiDict()
self.parser = OptionParser(description=self.__doc__,
usage=self.usage,
add_help_option=True)
## TODO: check on self.name should be removed (creating another abstraction in between or refactoring this)
if disable_interspersed_args:
self.parser.disable_interspersed_args()
self.setSuperOptions()
## Parse the command line parameters.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
## Validate the command line parameters before initializing the proxy.
self.validateOptions()
## Retrieve VO role/group from the command line parameters.
self.voRole = self.options.voRole if self.options.voRole is not None else ''
self.voGroup = self.options.voGroup if self.options.voGroup is not None else ''
## Create the object that will do the proxy operations. We ignore the VO role/group and
## server URL in the initialization, because they are not used until we do the proxy
## delegation to myproxy server. And the delegation happends in handleProxy(), which is
## called after we load the configuration file and retrieve the final values for those
## parameters. handleProxy() takes care of passing those parameters to self.proxy.
self.proxy = CredentialInteractions('', '', self.voRole, self.voGroup,
self.logger, '')
## Create a new proxy (if there isn't a valid one already) with current VO role/group.
## The VO role/group are not relevant for the proxy. We only want to check that the
## user doesn't expect the proxy to have different VO role/group than those specified
## in the configuration file, but this check is done later in handleProxy() after we
## load the configuration file.
self.proxy_created = False
if not self.options.proxy and self.cmdconf['initializeProxy']:
self.proxy_created = self.proxy.createNewVomsProxySimple(
time_left_threshold=720)
## If we get an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
## Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
## Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(
self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
## If VO role/group were not given in the command options, get them from the configuration file.
## If they are specified in both places, print a message saying that the command options will be used.
if self.options.voRole is None and hasattr(self.configuration,
'User'):
self.voRole = getattr(self.configuration.User, 'voRole', '')
if self.options.voGroup is None and hasattr(
self.configuration, 'User'):
self.voGroup = getattr(self.configuration.User, 'voGroup', '')
if (self.options.voRole is not None) and (
hasattr(self.configuration, 'User')
and hasattr(self.configuration.User, 'voRole')):
msg = "Ignoring the VO role specified in the configuration file. Using VO role \"%s\" "
if self.voRole == '': msg += "(i.e. no VO role) "
msg += "as specified in the command line."
self.logger.info(msg % self.voRole)
if (self.options.voGroup is not None) and (
hasattr(self.configuration, 'User')
and hasattr(self.configuration.User, 'voGroup')):
msg = "Ignoring the VO group specified in the configuration file. Using VO group \"%s\" "
if self.voGroup == '': msg += "(i.e. no VO group) "
msg += "as specified in the command line."
self.logger.info(msg % self.voGroup)
## If we get an input task, we load the cache and set the server URL and VO role/group from it.
if hasattr(self.options, 'task') and self.options.task:
self.loadLocalCache()
## If the server url isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
## Update (or create) the .crab3 cache file.
self.updateCrab3()
## At this point there should be a valid proxy, because we have already checked that and
## eventually created a new one. If the proxy was not created by CRAB, we check that the
## VO role/group in the proxy are the same as specified by the user in the configuration
## file (or in the command line options). If it is not, we ask the user if he wants to
## overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
## and ask him to provide the VO role/group as in the existing proxy.
## Finally, delegate the proxy to myproxy server.
self.handleProxy()
## Logging user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion(self.getUrl(self.instance, resource='info'))
self.uri = self.getUrl(self.instance)
self.logger.debug("Instance is %s" % (self.instance))
self.logger.debug("Server base url is %s" % (self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command url %s" % (self.uri))
def _prepareWorkArea(self):
self.reqarea, self.reqname, _ = createWorkArea(
self.logger, requestName=self.TEST_REQ_NAME)
server = {}
createCache(self.reqarea, "127.0.0.1", 8518, self.reqname)
def _prepareWorkArea(self):
self.reqarea, self.reqname, _ = createWorkArea(self.logger, requestName = self.TEST_REQ_NAME)
server = {}
createCache(self.reqarea, "127.0.0.1", 8518, self.reqname)
def __call__(self):
valid = False
configmsg = 'Default'
if not os.path.isfile(self.options.config):
raise MissingOptionException("Configuration file '%s' not found" %
self.options.config)
#store the configuration file in self.configuration
self.loadConfig(self.options.config, self.args)
requestarea, requestname, self.logfile = createWorkArea(
self.logger, getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
self.logger.debug("Started submission")
#determine the serverurl
if self.options.server:
self.serverurl = self.options.server
elif getattr(self.configuration.General, 'serverUrl',
None) is not None:
self.serverurl = self.configuration.General.serverUrl
#TODO: For sure the server url should not be handled here. Find an intelligent way for this
else:
self.serverurl = 'http://cmsweb.cern.ch'
if not hasattr(self.configuration.General, 'ufccacheUrl'):
self.configuration.General.ufccacheUrl = self.serverurl
if not hasattr(self.configuration.General, 'configcacheUrl'):
#https is required because configcache does not use ServerInteractions
self.configuration.General.configcacheUrl = 'https://' + self.serverurl + '/couchdb'
if not hasattr(self.configuration.General, 'configcacheName'):
self.configuration.General.configcacheName = 'analysis_reqmgr_config_cache'
self.createCache(self.serverurl)
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct(par):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(
par
) == par else ' Did you mean %s?' % SpellChecker.correct(
par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
#delegating the proxy (creation done in SubCommand)
self.voRole = getattr(self.configuration.User, "voRole", "")
self.voGroup = getattr(self.configuration.User, "voGroup", "")
self.handleProxy()
uniquerequestname = None
self.logger.debug("Working on %s" % str(requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException(1, "Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException(
1, "Missing parameter " +
self.requestmapper[param]['config'] +
" from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(requestname):
configreq["workflow"] = requestname
elif param == "savelogsflag":
configreq["savelogsflag"] = 1 if temp else 0
elif param == "publication":
configreq["publication"] = 1 if temp else 0
elif param == "blacklistT1":
blacklistT1 = self.voRole != 't1access'
#if the user choose to remove the automatic T1 blacklisting and has not the t1acces role
if getattr(self.configuration.Site, 'removeT1Blacklisting',
False) and blacklistT1:
self.logger.info(
"WARNING: You disabled the T1 automatic blacklisting without having the t1access role"
)
blacklistT1 = False
configreq["blacklistT1"] = 1 if blacklistT1 else 0
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
pluginParams = [
self.configuration, self.logger,
os.path.join(requestarea, 'inputs')
]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(
self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
configreq['publishname'] = "%s-%s" % (configreq['publishname'],
isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str(configreq))
dictresult, status, reason = server.put(
self.uri, data=self._encodeRequest(configreq))
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (
str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(requestarea,
tmpsplit[0],
tmpsplit[1] if len(tmpsplit) > 1 else '',
uniquerequestname,
voRole=self.voRole,
voGroup=self.voGroup)
self.logger.info("Submission completed")
self.logger.debug("Request ID: %s " % uniquerequestname)
self.logger.debug("Ended submission")
return uniquerequestname
def __init__(self, logger, cmdargs = [], disable_interspersed_args = False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
self.usage = "usage: %prog " + self.name + " [options] [args]"
self.logger = logger
self.logfile = ''
self.logger.debug("Executing command: '%s'" % str(self.name))
self.proxy = None
##Get the mapping
self.loadMapping()
self.crab3dic = self.getConfiDict()
self.parser = OptionParser(description = self.__doc__, usage = self.usage, add_help_option = True)
## TODO: check on self.name should be removed (creating another abstraction in between or refactoring this)
if disable_interspersed_args:
self.parser.disable_interspersed_args()
self.setSuperOptions()
##Parse the command line parameters
(self.options, self.args) = self.parser.parse_args( cmdargs )
##Validate the command line parameters before initing the proxy
self.validateOptions()
self.voRole = self.options.voRole if self.options.voRole is not None else ''
self.voGroup = self.options.voGroup if self.options.voGroup is not None else ''
##if we get an input configuration we load it
if hasattr(self.options, 'config') and self.options.config is not None:
self.loadConfig( self.options.config, self.args )
self.requestarea, self.requestname, self.logfile = createWorkArea(self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
if self.options.voRole is None and hasattr(self.configuration, 'User'):
self.voRole = getattr(self.configuration.User, 'voRole', '')
if self.options.voGroup is None and hasattr(self.configuration, 'User'):
self.voGroup = getattr(self.configuration.User, 'voGroup', '')
if (self.options.voRole is not None) and (hasattr(self.configuration, 'User') and hasattr(self.configuration.User, 'voRole' )):
msg = "Ignoring the VO role specified in the configuration file. Using VO role \"%s\" "
if self.voRole == '': msg += "(i.e. no VO role) "
msg += "as specified in the command line."
self.logger.info(msg % self.voRole)
if (self.options.voGroup is not None) and (hasattr(self.configuration, 'User') and hasattr(self.configuration.User, 'voGroup')):
msg = "Ignoring the VO group specified in the configuration file. Using VO group \"%s\" "
if self.voGroup == '': msg += "(i.e. no VO group) "
msg += "as specified in the command line."
self.logger.info(msg % self.voGroup)
if getattr(self.configuration.Data, 'publication', None) and \
not getattr(self.configuration.General, 'transferOutput', self.requestmapper['saveoutput']['default']):
msg = "Crab configuration problem: Data.publication is ON, but General.transferOutput is OFF.\n"
msg += "Publication can not be performed if the output files are not transferred to a permanent storage."
self.logger.info(msg)
raise ConfigurationException("Incompatible configuration options Data.publication and General.transferOutput")
##if we get an input task we load the cache and set the url from it
if hasattr(self.options, 'task') and self.options.task:
self.loadLocalCache()
## if the server url isn't already set we check the args and then the config
if not hasattr(self, 'serverurl') and self.requiresREST:
self.instance, self.serverurl = self.serverInstance()
elif not self.requiresREST:
self.instance, self.serverurl = None, None
self.updateCrab3()
self.handleProxy()
#logging user command and optin use for debuging purpose
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.requiresREST:
self.checkversion(self.getUrl(self.instance, resource='info'))
self.uri = self.getUrl(self.instance)
self.logger.debug("Instance is %s" %(self.instance))
self.logger.debug("Server base url is %s" %(self.serverurl))
if self.requiresREST:
self.logger.debug("Command url %s" %(self.uri))
