def __call__(self):
credentialHandler = CredentialInteractions(self.logger)
days = self.options.days
credentialHandler.setMyProxyValidity(int(days) * 24 * 60) # minutes
# give a bit of slack to the threshold, avoid that repeating the c
timeLeftThreshold = int(days - 1) * 24 * 60 * 60 # seconds
self.logger.info("Checking credentials")
# need an X509 proxy in order to talk with CRABServer to get list of myproxy authorized retrievers
credentialHandler.createNewVomsProxy(timeLeftThreshold=720)
alldns = server_info(crabserver=self.crabserver,
subresource='delegatedn')
for authorizedDNs in alldns['services']:
credentialHandler.setRetrievers(authorizedDNs)
self.logger.info("Registering user credentials in myproxy")
(credentialName,
myproxyTimeleft) = credentialHandler.createNewMyProxy(
timeleftthreshold=timeLeftThreshold)
self.logger.info(
"Credential exists on myproxy: username: %s - validity: %s",
credentialName, str(timedelta(seconds=myproxyTimeleft)))
return
def __call__(self):
proxy = CredentialInteractions('', '', '', '', self.logger)
days = self.options.days
proxy.setMyProxyValidity(int(days) * 24 * 60) # minutes
timeLeftThreshold = int(days) * 24 * 60 * 60 # seconds
self.logger.info("Checking credentials")
# need an X509 proxy in order to talk with CRABServer to get list of myproxy authorized retrievers
proxy.proxyInfo = proxy.createNewVomsProxy(timeLeftThreshold=720)
proxyfilename = proxy.proxyInfo['filename']
serverurl = 'cmsweb.cern.ch'
baseurl = '/crabserver/prod/info'
alldns = server_info(subresource='delegatedn', serverurl=serverurl, proxyfilename=proxyfilename, baseurl=baseurl)
for serverdn in alldns['services']:
proxy.defaultDelegation['serverDN'] = serverdn
proxy.defaultDelegation['myProxySvr'] = 'myproxy.cern.ch'
self.logger.info("Registering user credentials for server %s" % serverdn)
(credentialName, myproxyTimeleft) = proxy.createNewMyProxy(timeleftthreshold=timeLeftThreshold, nokey=True)
self.logger.info("Credential exists on myproxy: username: %s - validity: %s", credentialName,
str(timedelta(seconds=myproxyTimeleft)))
(credentialName, myproxyTimeleft) = proxy.createNewMyProxy2(timeleftthreshold=timeLeftThreshold, nokey=True)
self.logger.info("Credential exists on myproxy: username: %s - validity: %s", credentialName,
str(timedelta(seconds=myproxyTimeleft)))
return
def handleProxy(server, baseurl, voRole, voGroup, days):
"""
"""
proxy = CredentialInteractions('', '', voRole, voGroup, logging)
#proxy.myproxyDesiredValidity = days
proxy.setMyProxyValidity(int(days) * 24 * 60)
logging.info("Checking credentials")
proxy.proxyInfo = proxy.createNewVomsProxy(timeLeftThreshold=720)
proxyfilename = proxy.proxyInfo['filename']
#get the dn of the agents from the server
alldns = server_info('delegatedn', server, proxyfilename, baseurl)
for serverdn in alldns['services']:
proxy.defaultDelegation['serverDN'] = serverdn
proxy.defaultDelegation['myProxySvr'] = 'myproxy.cern.ch'
logging.info("Registering user credentials for server %s" % serverdn)
proxy.createNewMyProxy(timeleftthreshold=60 * 60 * 24 *
RENEW_MYPROXY_THRESHOLD,
nokey=True)
proxy.createNewMyProxy2(timeleftthreshold=60 * 60 * 24 *
RENEW_MYPROXY_THRESHOLD,
nokey=True)
def handleProxy(self ):
""" Init the user proxy, and delegate it if necessary.
"""
if not self.options.skipProxy and self.initializeProxy:
proxy = CredentialInteractions('', '', self.voRole, self.voGroup, self.logger, myproxyAccount=self.serverurl)
self.proxy = proxy
self.logger.debug("Checking credentials")
_, self.proxyfilename = proxy.createNewVomsProxy( timeleftthreshold = 720 )
if self.requiresREST: #if the command does not contact the REST we can't delegate the proxy
proxy.myproxyAccount = self.serverurl
baseurl = self.getUrl(self.instance, resource='info')
#get the dn of the task workers from the server
alldns = server_info('delegatedn', self.serverurl, self.proxyfilename, baseurl)
for serverdn in alldns['services']:
proxy.defaultDelegation['serverDN'] = serverdn
proxy.defaultDelegation['myProxySvr'] = 'myproxy.cern.ch'
self.logger.debug("Registering user credentials for server %s" % serverdn)
proxy.createNewMyProxy( timeleftthreshold = 60 * 60 * 24 * RENEW_MYPROXY_THRESHOLD, nokey=True)
else:
self.proxyfilename = self.options.skipProxy
os.environ['X509_USER_PROXY'] = self.options.skipProxy
self.logger.debug('Skipping proxy creation')
def initProxy(voRole, voGroup, logger):
proxy = CredentialInteractions('', '', voRole, voGroup, logger)
logger.debug("Checking credentials")
userdn, proxyfilename = proxy.createNewVomsProxy(timeleftthreshold=600)
#return also the proxy because successive proxy delegations needs to use the
#same proxy instsance
return userdn, proxyfilename, proxy
def initProxy(voRole, voGroup, logger):
proxy = CredentialInteractions(
'',
'',
voRole,
voGroup,
logger
)
logger.debug("Checking credentials")
userdn, proxyfilename = proxy.createNewVomsProxy( timeleftthreshold = 600 )
#return also the proxy because successive proxy delegations needs to use the
#same proxy instsance
return userdn, proxyfilename, proxy
class SubCommand(ConfigCommand):
####### These options can be overrhidden if needed ########
## setting visible = False doesn't allow the sub-command to be called from CLI
visible = True
proxyfilename = None
shortnames = []
usage = "usage: %prog [command-options] [args]"
#Default command name is the name of the command class, but it is also possible to set the name attribute in the subclass
#if the command name does not correspond to the class name (getlog => get-log)
def getUrl(self, instance='prod', resource='workflow'):
"""
Retrieve the url depending on the resource we are accessing and the instance.
"""
if instance in SERVICE_INSTANCES.keys():
return BASEURL + instance + '/' + resource
elif instance == 'private':
return BASEURL + 'dev' + '/' + resource
raise ConfigurationException('Error: only %s instances can be used.' % str(SERVICE_INSTANCES.keys()))
def __init__(self, logger, cmdargs = None, disable_interspersed_args = False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
## The command logger.
self.logger = logger
self.logfile = self.logger.logfile
self.logger.debug("Executing command: '%s'" % str(self.name))
self.proxy = None
self.restClass = CRABClient.Emulator.getEmulator('rest')
## Get the command configuration.
self.cmdconf = commandsConfiguration.get(self.name)
## Get the CRAB cache file.
self.crab3dic = self.getConfiDict()
## The options parser.
self.parser = CRABCmdOptParser(self.name, self.__doc__, disable_interspersed_args)
## Define the command options.
self.setSuperOptions()
## Parse the command options/arguments.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
## Validate the command options.
self.validateOptions()
## Get the VO group/role from the command options (if the command requires these
## options).
proxyOptsSetPlace = {'set_in': {'group': "default", 'role': "default"}, 'for_set_use': ""}
msgadd = []
self.voGroup, self.voRole = "", "NULL"
if self.cmdconf['requiresProxyVOOptions']:
proxyOptsSetPlace['for_set_use'] = "cmdopts"
if self.options.voGroup is not None:
self.voGroup = self.options.voGroup
proxyOptsSetPlace['set_in']['group'] = "cmdopts"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.options.voRole is not None:
self.voRole = self.options.voRole if self.options.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "cmdopts"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the crab command options." % (" and ".join(msgadd))
self.logger.debug(msg)
## Create the object that will do the proxy operations. We don't really care
## what VO role and group and server URL we pass to the constructor, because
## these are not used until we do the proxy delegation to the myproxy server.
## And this happens in handleProxy(), which is called after we load the
## configuration file and retrieve the final values for those parameters.
## handleProxy() takes care of passing those parameters to self.proxy.
self.proxy = CredentialInteractions('', '', self.voRole, self.voGroup, self.logger, '')
## If the user didn't use the --proxy command line option, and if there isn't a
## valid proxy already, we create a new one with the current VO role and group
## (as commented above, we don't really care what are the VO role and group so
## far).
self.proxyCreated = False
if not self.options.proxy and self.cmdconf['initializeProxy']:
self.proxyCreated = self.proxy.createNewVomsProxySimple(timeLeftThreshold = 720)
## If there is an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
proxyOptsSetPlace['for_set_use'] = "config"
## Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
## Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(self.logger, \
getattr(self.configuration.General, 'workArea', None), \
getattr(self.configuration.General, 'requestName', None))
## Get the VO group/role from the configuration file.
msgadd = []
if hasattr(self.configuration, 'User') and hasattr(self.configuration.User, 'voGroup'):
self.voGroup = self.configuration.User.voGroup
proxyOptsSetPlace['set_in']['group'] = "config"
msgadd.append("VO group '%s'" % (self.voGroup))
if hasattr(self.configuration, 'User') and hasattr(self.configuration.User, 'voRole'):
self.voRole = self.configuration.User.voRole if self.configuration.User.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "config"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the CRAB configuration file." % (" and ".join(msgadd))
self.logger.debug(msg)
## If an input project directory was given, load the request cache and take the
## server URL from it. If the VO group/role was not given in the command options,
## take it also from the request cache.
if self.cmdconf['requiresDirOption']:
self.loadLocalCache(proxyOptsSetPlace)
## If the server URL isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
## Update (or create) the CRAB cache file.
self.updateCRABCacheFile()
## At this point there should be a valid proxy, because we have already checked that and
## eventually created a new one. If the proxy was not created by CRAB, we check that the
## VO role/group in the proxy are the same as specified by the user in the configuration
## file (or in the command line options). If it is not, we ask the user if he wants to
## overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
## and ask him to provide the VO role/group as in the existing proxy.
## Finally, delegate the proxy to myproxy server.
self.handleProxy(proxyOptsSetPlace)
## Logging user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion(self.getUrl(self.instance, resource='info'))
self.uri = self.getUrl(self.instance)
self.logger.debug("Instance is %s" %(self.instance))
self.logger.debug("Server base url is %s" %(self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command url %s" %(self.uri))
def serverInstance(self):
"""
Deriving the correct instance to use and the server url. Client is allowed to propagate the instance name and corresponding url
via crabconfig.py or crab option --instance. The variable passed via crab option will always be used over the variable
in crabconfig.py. Instance name other than specify in the SERVICE_INSTANCE will be treated as a private instance.
"""
if hasattr(self.options, 'instance') and self.options.instance is not None:
if hasattr(self, 'configuration') and hasattr(self.configuration.General, 'instance') and self.configuration.General.instance is not None:
msg = "%sWarning%s: CRAB configuration parameter General.instance is overwritten by the command option --instance;" % (colors.RED, colors.NORMAL)
msg += " %s intance will be used." % (self.options.instance)
self.logger.info(msg)
if self.options.instance in SERVICE_INSTANCES.keys():
instance = self.options.instance
serverurl = SERVICE_INSTANCES[instance]
else:
instance = 'private'
serverurl = self.options.instance
elif hasattr(self, 'configuration') and hasattr(self.configuration.General, 'instance') and self.configuration.General.instance is not None:
if self.configuration.General.instance in SERVICE_INSTANCES.keys():
instance = self.configuration.General.instance
serverurl = SERVICE_INSTANCES[instance]
else:
instance = 'private'
serverurl = self.configuration.General.instance
else:
instance = getParamDefaultValue('General.instance')
serverurl = SERVICE_INSTANCES[instance]
return instance, serverurl
def checkversion(self, baseurl = None):
compatibleVersions = server_info('version', self.serverurl, self.proxyfilename, baseurl)
for item in compatibleVersions:
if re.match(item, __version__):
self.logger.debug("CRABClient version: %s" % (__version__))
break
else:
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " Incompatible CRABClient version %s" % (__version__ )
msg += "\nServer is saying that compatible versions are: %s" % [v.replace("\\", "") for v in compatibleVersions]
self.logger.info(msg)
def handleProxy(self, proxyOptsSetPlace):
"""
Init the user proxy, and delegate it if necessary.
"""
if not self.options.proxy:
if self.cmdconf['initializeProxy']:
self.proxy.setVOGroupVORole(self.voGroup, self.voRole)
self.proxy.setMyProxyAccount(self.serverurl)
self.proxyfilename = self.proxy.createNewVomsProxy(timeLeftThreshold = 720, \
doProxyGroupRoleCheck = self.cmdconf['doProxyGroupRoleCheck'], \
proxyCreatedByCRAB = self.proxyCreated, \
proxyOptsSetPlace = proxyOptsSetPlace)
if self.cmdconf['requiresREST']: ## If the command doesn't contact the REST, we can't delegate the proxy.
self.proxy.myproxyAccount = self.serverurl
baseurl = self.getUrl(self.instance, resource = 'info')
## Get the DN of the task workers from the server.
all_task_workers_dns = server_info('delegatedn', self.serverurl, self.proxyfilename, baseurl)
for serverdn in all_task_workers_dns['services']:
self.proxy.setServerDN(serverdn)
self.proxy.setMyProxyServer('myproxy.cern.ch')
self.logger.debug("Registering user credentials for server %s" % serverdn)
self.proxy.createNewMyProxy(timeleftthreshold = 60 * 60 * 24 * RENEW_MYPROXY_THRESHOLD, nokey = True)
else:
self.proxyfilename = self.options.proxy
os.environ['X509_USER_PROXY'] = self.options.proxy
self.logger.debug('Skipping proxy creation')
def loadLocalCache(self, proxyOptsSetPlace):
"""
Loads the client cache and set up the server url
"""
self.requestarea, self.requestname = getWorkArea(self.options.projdir)
self.cachedinfo, self.logfile = loadCache(self.requestarea, self.logger)
port = ':' + self.cachedinfo['Port'] if self.cachedinfo['Port'] else ''
self.instance = self.cachedinfo['instance']
self.serverurl = self.cachedinfo['Server'] + port
msgadd = []
if self.cmdconf['requiresProxyVOOptions'] and self.options.voGroup is None:
self.voGroup = self.cachedinfo['voGroup']
proxyOptsSetPlace['set_in']['group'] = "cache"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.cmdconf['requiresProxyVOOptions'] and self.options.voRole is None:
self.voRole = self.cachedinfo['voRole']
proxyOptsSetPlace['set_in']['role'] = "cache"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as written in the request cache file for this task." % (" and ".join(msgadd))
self.logger.debug(msg)
def getConfiDict(self):
"""
Load the CRAB cache file (~/.crab3). If it doesn't exist, create one.
"""
crabCacheFileName = self.crabcachepath()
if not os.path.isfile(crabCacheFileName):
msg = "Could not find CRAB cache file %s; creating a new one." % (crabCacheFileName)
self.logger.debug(msg)
configdict = {'crab_project_directory': ''}
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(configdict, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
return configdict
try:
msg = "Found CRAB cache file %s" % (crabCacheFileName)
self.logger.debug(msg)
with open(crabCacheFileName, 'r') as fd:
configdict = json.load(fd)
except ValueError:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Error loading CRAB cache file."
msg += " Try to do 'rm -rf %s' and run the crab command again." % (crabCacheFileName)
raise ConfigurationException(msg)
if 'crab_project_directory' not in configdict:
configdict['crab_project_directory'] = configdict.get('taskname', '')
if 'taskname' in configdict:
del configdict['taskname']
return configdict
def crabcachepath(self):
if 'CRAB3_CACHE_FILE' in os.environ:
if os.path.isabs(os.environ['CRAB3_CACHE_FILE']):
return os.environ['CRAB3_CACHE_FILE']
else:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Invalid path in environment variable CRAB3_CACHE_FILE: %s" % (os.environ['CRAB3_CACHE_FILE'])
msg += " Please export a valid full path."
raise EnvironmentException(msg)
else:
return str(os.path.expanduser('~')) + '/.crab3'
def updateCRABCacheFile(self):
"""
Update the CRAB cache file.
So far this file contains only the path of the last used CRAB project directory.
"""
if self.cmdconf['requiresDirOption'] or getattr(self, 'requestarea', None):
self.crab3dic['crab_project_directory'] = self.requestarea
crabCacheFileName = self.crabcachepath()
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(self.crab3dic, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
def __call__(self):
self.logger.info("This is a 'nothing to do' command")
raise NotImplementedError
def terminate(self, exitcode):
#We do not want to print logfile for each command...
if exitcode < 2000:
if getattr(self.options, 'dump', False) or getattr(self.options, 'xroot', False):
self.logger.debug("Log file is %s" % os.path.abspath(self.logfile))
else:
self.logger.info("Log file is %s" % os.path.abspath(self.logfile))
def setOptions(self):
raise NotImplementedError
def setSuperOptions(self):
try:
#add command related options
self.setOptions()
except NotImplementedError:
pass
self.parser.addCommonOptions(self.cmdconf)
def validateOptions(self):
"""
__validateOptions__
Validate the command line options of the command.
Raise a ConfigurationException in case of error; don't do anything if ok.
"""
if self.cmdconf['requiresDirOption']:
if self.options.projdir is None:
if len(self.args) > 1:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command accepts at most 1 argument (a path to a CRAB project directory), %d given." % (self.name, len(self.args))
raise ConfigurationException(msg)
elif len(self.args) == 1 and self.args[0]:
self.options.projdir = self.args.pop(0)
elif self.cmdconf['useCache'] and self.crab3dic.get('crab_project_directory'):
self.options.projdir = str(self.crab3dic['crab_project_directory'])
if self.options.projdir is None:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Please indicate the CRAB project directory with --dir=<project-directory>."
ex = MissingOptionException(msg)
ex.missingOption = "task"
raise ex
if not os.path.isdir(self.options.projdir):
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " %s is not a valid CRAB project directory." % (self.options.projdir)
raise ConfigurationException(msg)
## If the command does not take any arguments, but some arguments were passed,
## clear the arguments list and give a warning message saying that the given
## arguments will be ignored.
if not self.cmdconf['acceptsArguments'] and len(self.args):
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command takes no arguments, %d given." % (self.name, len(self.args))
msg += " Ignoring arguments %s." % (self.args)
self.logger.warning(msg)
self.args = []
class SubCommand(ConfigCommand):
####### These options can be overrhidden if needed ########
## setting visible = False doesn't allow the sub-command to be called from CLI
visible = True
proxyfilename = None
shortnames = []
usage = "usage: %prog [command-options] [args]"
#Default command name is the name of the command class, but it is also possible to set the name attribute in the subclass
#if the command name does not correspond to the class name (getlog => get-log)
def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
ConfigCommand.__init__(self)
## The command logger.
self.logger = logger
self.logfile = self.logger.logfile
localSystem = subprocess.check_output(['uname', '-a']).strip('\n')
try:
localOS = subprocess.check_output(
['grep', 'PRETTY_NAME', '/etc/os-release'],
stderr=subprocess.STDOUT).strip('\n')
localOS = localOS.split('=')[1].strip('"')
except:
try:
localOS = subprocess.check_output(['lsb_release',
'-d']).strip('\n')
localOS = localOS.split(':')[1].strip()
except:
localOS = "Unknown Operating System"
self.logger.debug("Running on: " + localSystem + " - " + localOS)
opensslInfo = subprocess.check_output(["openssl",
"version"]).strip('\n')
self.logger.debug("OpenSSl version: %s", opensslInfo)
opensslVersion = opensslInfo.split()[1]
nDots = opensslVersion.count(".")
if float(opensslVersion.rsplit(".", nDots - 1)[0]) > 1:
raise EnvironmentException(
"Your OpenSSl version (%s) is not supported. Supported versions are < 1.1"
% opensslVersion)
self.logger.debug("Executing command: '%s'" % str(self.name))
self.proxy = None
self.restClass = CRABClient.Emulator.getEmulator('rest')
## Get the command configuration.
self.cmdconf = commandsConfiguration.get(self.name)
if not self.cmdconf:
raise RuntimeError(
"Canot find command %s in commandsConfiguration inside ClientMapping. Are you a developer"
"trying to add a command without it's correspondant configuration?"
% self.name)
## Get the CRAB cache file.
self.cachedinfo = None
self.crab3dic = self.getConfiDict()
## The options parser.
self.parser = CRABCmdOptParser(self.name, self.__doc__,
disable_interspersed_args)
## Define the command options.
self.setSuperOptions()
## Parse the command options/arguments.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
self.transferringIds = None
self.dest = None
self.validateLogpathOption()
## Validate first the SubCommand options
SubCommand.validateOptions(self)
## then the config option for the submit command
self.validateConfigOption()
## Get the VO group/role from the command options (if the command requires these
## options).
proxyOptsSetPlace = {
'set_in': {
'group': "default",
'role': "default"
},
'for_set_use': ""
}
msgadd = []
self.voGroup, self.voRole = "", "NULL"
if self.cmdconf['requiresProxyVOOptions']:
proxyOptsSetPlace['for_set_use'] = "cmdopts"
if self.options.voGroup is not None:
self.voGroup = self.options.voGroup
proxyOptsSetPlace['set_in']['group'] = "cmdopts"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.options.voRole is not None:
self.voRole = self.options.voRole if self.options.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "cmdopts"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the crab command options." % (
" and ".join(msgadd))
self.logger.debug(msg)
## Create the object that will do the proxy operations. We don't really care
## what VO role and group and server URL we pass to the constructor, because
## these are not used until we do the proxy delegation to the myproxy server.
## And this happens in handleProxy(), which is called after we load the
## configuration file and retrieve the final values for those parameters.
## handleProxy() takes care of passing those parameters to self.proxy.
self.proxy = CredentialInteractions('', '', self.voRole, self.voGroup,
self.logger, '')
## If the user didn't use the --proxy command line option, and if there isn't a
## valid proxy already, we create a new one with the current VO role and group
## (as commented above, we don't really care what are the VO role and group so
## far).
self.proxyCreated = False
if not self.options.proxy and self.cmdconf['initializeProxy']:
self.proxyCreated = self.proxy.createNewVomsProxySimple(
timeLeftThreshold=720)
## If there is an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
proxyOptsSetPlace['for_set_use'] = "config"
## Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
## Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(self.logger, \
getattr(self.configuration.General, 'workArea', None), \
getattr(self.configuration.General, 'requestName', None))
## Get the VO group/role from the configuration file.
msgadd = []
if hasattr(self.configuration, 'User') and hasattr(
self.configuration.User, 'voGroup'):
self.voGroup = self.configuration.User.voGroup
proxyOptsSetPlace['set_in']['group'] = "config"
msgadd.append("VO group '%s'" % (self.voGroup))
if hasattr(self.configuration, 'User') and hasattr(
self.configuration.User, 'voRole'):
self.voRole = self.configuration.User.voRole if self.configuration.User.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "config"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the CRAB configuration file." % (
" and ".join(msgadd))
self.logger.debug(msg)
## If the VO group/role was not given in the command options, take it from the request cache.
if self.cmdconf['requiresDirOption']:
self.setCachedProxy(proxyOptsSetPlace)
## If the server URL isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
## Update (or create) the CRAB cache file.
self.updateCRABCacheFile()
## At this point there should be a valid proxy, because we have already checked that and
## eventually created a new one. If the proxy was not created by CRAB, we check that the
## VO role/group in the proxy are the same as specified by the user in the configuration
## file (or in the command line options). If it is not, we ask the user if he wants to
## overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
## and ask him to provide the VO role/group as in the existing proxy.
## Finally, delegate the proxy to myproxy server.
self.handleProxy(proxyOptsSetPlace)
## Validate the command options
self.validateOptions()
## Logging user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion(getUrl(self.instance, resource='info'))
self.uri = getUrl(self.instance)
self.logger.debug("Instance is %s" % (self.instance))
self.logger.debug("Server base url is %s" % (self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command url %s" % (self.uri))
def serverInstance(self):
"""
Deriving the correct instance to use and the server url. Client is allowed to propagate the instance name and corresponding url
via crabconfig.py or crab option --instance. The variable passed via crab option will always be used over the variable
in crabconfig.py. Instance name other than specify in the SERVICE_INSTANCE will be treated as a private instance.
"""
if hasattr(self.options,
'instance') and self.options.instance is not None:
if hasattr(self, 'configuration') and hasattr(
self.configuration, 'General') and hasattr(
self.configuration.General, 'instance'
) and self.configuration.General.instance is not None:
msg = "%sWarning%s: CRAB configuration parameter General.instance is overwritten by the command option --instance;" % (
colors.RED, colors.NORMAL)
msg += " %s intance will be used." % (self.options.instance)
self.logger.info(msg)
if self.options.instance in SERVICE_INSTANCES.keys():
instance = self.options.instance
serverurl = SERVICE_INSTANCES[instance]
else:
instance = 'private'
serverurl = self.options.instance
elif hasattr(self, 'configuration') and hasattr(
self.configuration, 'General') and hasattr(
self.configuration.General, 'instance'
) and self.configuration.General.instance is not None:
if self.configuration.General.instance in SERVICE_INSTANCES.keys():
instance = self.configuration.General.instance
serverurl = SERVICE_INSTANCES[instance]
else:
instance = 'private'
serverurl = self.configuration.General.instance
else:
instance = getParamDefaultValue('General.instance')
serverurl = SERVICE_INSTANCES[instance]
return instance, serverurl
def checkversion(self, baseurl=None):
compatibleVersions = server_info('version', self.serverurl,
self.proxyfilename, baseurl)
for item in compatibleVersions:
if re.match(item, __version__):
self.logger.debug("CRABClient version: %s" % (__version__))
break
else:
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " Incompatible CRABClient version %s" % (__version__)
msg += "\nServer is saying that compatible versions are: %s" % [
v.replace("\\", "") for v in compatibleVersions
]
self.logger.info(msg)
def handleProxy(self, proxyOptsSetPlace):
"""
Init the user proxy, and delegate it if necessary.
"""
if not self.options.proxy:
if self.cmdconf['initializeProxy']:
self.proxy.setVOGroupVORole(self.voGroup, self.voRole)
self.proxy.setMyProxyAccount(self.serverurl)
self.proxyfilename = self.proxy.createNewVomsProxy(timeLeftThreshold = 720, \
doProxyGroupRoleCheck = self.cmdconf['doProxyGroupRoleCheck'], \
proxyCreatedByCRAB = self.proxyCreated, \
proxyOptsSetPlace = proxyOptsSetPlace)
if self.cmdconf[
'requiresREST']: ## If the command doesn't contact the REST, we can't delegate the proxy.
self.proxy.myproxyAccount = self.serverurl
baseurl = getUrl(self.instance, resource='info')
## Get the DN of the task workers from the server.
all_task_workers_dns = server_info('delegatedn',
self.serverurl,
self.proxyfilename,
baseurl)
for serverdn in all_task_workers_dns['services']:
self.proxy.setServerDN(serverdn)
self.proxy.setMyProxyServer('myproxy.cern.ch')
self.logger.debug(
"Registering user credentials for server %s" %
serverdn)
self.proxy.createNewMyProxy(timeleftthreshold=60 * 60 *
24 *
RENEW_MYPROXY_THRESHOLD,
nokey=True)
else:
self.proxyfilename = self.options.proxy
os.environ['X509_USER_PROXY'] = self.options.proxy
self.logger.debug('Skipping proxy creation')
def loadLocalCache(self):
"""
Loads the client cache and set up the server url
"""
self.requestarea, self.requestname = getWorkArea(self.options.projdir)
try:
self.cachedinfo, self.logfile = loadCache(self.requestarea,
self.logger)
port = ':' + self.cachedinfo['Port'] if self.cachedinfo[
'Port'] else ''
self.instance = self.cachedinfo['instance']
self.serverurl = self.cachedinfo['Server'] + port
except CachefileNotFoundException as ex:
if self.cmdconf['requiresLocalCache']:
raise ex
def setCachedProxy(self, proxyOptsSetPlace):
"""
Set the proxy parameters from the cache if not specified otherwise
"""
msgadd = []
if self.cmdconf[
'requiresProxyVOOptions'] and self.options.voGroup is None:
self.voGroup = self.cachedinfo['voGroup']
proxyOptsSetPlace['set_in']['group'] = "cache"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.cmdconf[
'requiresProxyVOOptions'] and self.options.voRole is None:
self.voRole = self.cachedinfo['voRole']
proxyOptsSetPlace['set_in']['role'] = "cache"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as written in the request cache file for this task." % (
" and ".join(msgadd))
self.logger.debug(msg)
def getConfiDict(self):
"""
Load the CRAB cache file (~/.crab3). If it doesn't exist, create one.
"""
crabCacheFileName = self.crabcachepath()
if not os.path.isfile(crabCacheFileName):
msg = "Could not find CRAB cache file %s; creating a new one." % (
crabCacheFileName)
self.logger.debug(msg)
configdict = {'crab_project_directory': ''}
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(configdict, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
return configdict
try:
msg = "Found CRAB cache file %s" % (crabCacheFileName)
self.logger.debug(msg)
with open(crabCacheFileName, 'r') as fd:
configdict = json.load(fd)
except ValueError:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Error loading CRAB cache file."
msg += " Try to do 'rm -rf %s' and run the crab command again." % (
crabCacheFileName)
raise ConfigurationException(msg)
if 'crab_project_directory' not in configdict:
configdict['crab_project_directory'] = configdict.get(
'taskname', '')
if 'taskname' in configdict:
del configdict['taskname']
return configdict
def crabcachepath(self):
if 'CRAB3_CACHE_FILE' in os.environ:
if os.path.isabs(os.environ['CRAB3_CACHE_FILE']):
return os.environ['CRAB3_CACHE_FILE']
else:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Invalid path in environment variable CRAB3_CACHE_FILE: %s" % (
os.environ['CRAB3_CACHE_FILE'])
msg += " Please export a valid full path."
raise EnvironmentException(msg)
else:
return str(os.path.expanduser('~')) + '/.crab3'
def updateCRABCacheFile(self):
"""
Update the CRAB cache file.
So far this file contains only the path of the last used CRAB project directory.
"""
if self.cmdconf['requiresDirOption'] or getattr(
self, 'requestarea', None):
self.crab3dic['crab_project_directory'] = self.requestarea
crabCacheFileName = self.crabcachepath()
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(self.crab3dic, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
def __call__(self):
self.logger.info("This is a 'nothing to do' command")
raise NotImplementedError
def terminate(self, exitcode):
#We do not want to print logfile for each command...
if exitcode < 2000:
if getattr(self.options, 'dump', False) or getattr(
self.options, 'xroot', False):
self.logger.debug("Log file is %s" %
os.path.abspath(self.logfile))
else:
self.logger.info("Log file is %s" %
os.path.abspath(self.logfile))
def setOptions(self):
raise NotImplementedError
def setSuperOptions(self):
try:
#add command related options
self.setOptions()
except NotImplementedError:
pass
self.parser.addCommonOptions(self.cmdconf)
def validateOptions(self):
"""
__validateOptions__
Validate the command line options of the command.
Raise a ConfigurationException in case of error; don't do anything if ok.
"""
if self.cmdconf['requiresDirOption']:
if self.options.projdir is None:
if len(self.args) > 1:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command accepts at most 1 argument (a path to a CRAB project directory), %d given." % (
self.name, len(self.args))
raise ConfigurationException(msg)
elif len(self.args) == 1 and self.args[0]:
self.options.projdir = self.args.pop(0)
elif self.cmdconf['useCache'] and self.crab3dic.get(
'crab_project_directory'):
self.options.projdir = str(
self.crab3dic['crab_project_directory'])
if self.options.projdir is None:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Please indicate the CRAB project directory with --dir=<project-directory>."
ex = MissingOptionException(msg)
ex.missingOption = "task"
raise ex
if not os.path.isdir(self.options.projdir):
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " %s is not a valid CRAB project directory." % (
self.options.projdir)
raise ConfigurationException(msg)
## If an input project directory was given, load the request cache and take the
## server URL from it.
self.loadLocalCache()
## If the command does not take any arguments, but some arguments were passed,
## clear the arguments list and give a warning message saying that the given
## arguments will be ignored.
if not self.cmdconf['acceptsArguments'] and len(self.args):
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command takes no arguments, %d given." % (
self.name, len(self.args))
msg += " Ignoring arguments %s." % (self.args)
self.logger.warning(msg)
self.args = []
def validateLogpathOption(self):
pass
def validateConfigOption(self):
pass
class SubCommand(ConfigCommand):
#### These options can be overrhidden if needed ####
# setting visible = False doesn't allow the sub-command to be called from CLI
visible = True
proxyfilename = None
shortnames = []
usage = "usage: %prog [command-options] [args]"
#Default command name is the name of the command class, but it is also possible to set the name attribute in the subclass
#if the command name does not correspond to the class name (getlog => get-log)
def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
ConfigCommand.__init__(self)
# The command logger.
self.logger = logger
self.logfile = self.logger.logfile
stdout, _, _ = execute_command(command='uname -a')
localSystem = stdout.strip()
try:
localOS, _, _ = execute_command('grep PRETTY_NAME /etc/os-release')
localOS = localOS.strip().split('=')[1].strip('"')
except Exception as ex: # pylint: disable=unused-variable
try:
localOS, _, _ = execute_command(command='lsb_release -d')
localOS = localOS.strip().split(':')[1].strip()
except Exception as ex: # pylint: disable=unused-variable
localOS = "Unknown Operating System"
self.logger.debug("CRAB Client version: %s", __version__)
self.logger.debug("Running on: " + localSystem + " - " + localOS)
self.logger.debug("Executing command: '%s'" % str(self.name))
# Get the command configuration.
self.cmdconf = commandsConfiguration.get(self.name)
if not self.cmdconf:
raise RuntimeError(
"Canot find command %s in commandsConfiguration inside ClientMapping. Are you a developer"
"trying to add a command without it's correspondant configuration?"
% self.name)
# Get the CRAB cache file.
self.cachedinfo = None
self.crab3dic = self.getConfiDict()
# The options parser.
self.parser = CRABCmdOptParser(self.name, self.__doc__,
disable_interspersed_args)
# Define the command options.
self.setSuperOptions()
# Parse the command options/arguments.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
self.transferringIds = None
self.dest = None
self.validateLogpathOption()
# Validate first the SubCommand options
SubCommand.validateOptions(self)
# then the config option for the submit command
self.validateConfigOption()
# Get the VO group/role from the command options (if the command requires these
# options).
proxyOptsSetPlace = {
'set_in': {
'group': "default",
'role': "default"
},
'for_set_use': ""
}
msgadd = []
self.voGroup, self.voRole = "", "NULL"
if self.cmdconf['requiresProxyVOOptions']:
proxyOptsSetPlace['for_set_use'] = "cmdopts"
if self.options.voGroup is not None:
self.voGroup = self.options.voGroup
proxyOptsSetPlace['set_in']['group'] = "cmdopts"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.options.voRole is not None:
self.voRole = self.options.voRole if self.options.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "cmdopts"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the crab command options." % (
" and ".join(msgadd))
self.logger.debug(msg)
# Create the object that will do the proxy operations. We don't really care
# what VO role and group and server URL we pass to the constructor, because
# these are not used until we do the proxy delegation to the myproxy server.
# And this happens in handleProxy(), which is called after we load the
# configuration file and retrieve the final values for those parameters.
# handleProxy() takes care of passing those parameters to self.credentialHandler
self.credentialHandler = CredentialInteractions(self.logger)
# If the user didn't use the --proxy command line option, and if there isn't a
# valid proxy already, we will create a new one with the current VO role and group
# (as commented above, we don't really care what are the VO role and group so
# far).
self.proxyCreated = False
# If there is an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
proxyOptsSetPlace['for_set_use'] = "config"
# Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
# Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(
self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
# Get the VO group/role from the configuration file.
msgadd = []
if hasattr(self.configuration, 'User') and hasattr(
self.configuration.User, 'voGroup'):
self.voGroup = self.configuration.User.voGroup
proxyOptsSetPlace['set_in']['group'] = "config"
msgadd.append("VO group '%s'" % (self.voGroup))
if hasattr(self.configuration, 'User') and hasattr(
self.configuration.User, 'voRole'):
self.voRole = self.configuration.User.voRole if self.configuration.User.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "config"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the CRAB configuration file." % (
" and ".join(msgadd))
self.logger.debug(msg)
# If the VO group/role was not given in the command options, take it from the request cache.
if self.cmdconf['requiresDirOption']:
self.setCachedProxy(proxyOptsSetPlace)
# If the server URL isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
# Update (or create) the CRAB cache file.
self.updateCRABCacheFile()
# At this point we check if there is a valid proxy, and
# eventually create a new one. If the proxy was not created by CRAB, we check that the
# VO role/group in the proxy are the same as specified by the user in the configuration
# file (or in the command line options). If it is not, we ask the user if he wants to
# overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
# and ask him to provide the VO role/group as in the existing proxy.
# Finally, delegate the proxy to myproxy server.
self.handleVomsProxy(proxyOptsSetPlace)
# only if this command talks to the REST we create a CRABRest object to communicate with CRABServer
# and check/upate credentials on myproxy
# this is usually the first time that a call to the server is made, so where Emulator('rest') is initialized
# arguments to Emulator('rest') call must match those for HTTPRequest.__init__ in RESTInteractions.py
#server = CRABClient.Emulator.getEmulator('rest')(url=serverurl, localcert=proxyfilename, localkey=proxyfilename,
# version=__version__, retry=2, logger=logger)
if self.cmdconf['requiresREST']:
crabRest = CRABClient.Emulator.getEmulator('rest')
self.crabserver = crabRest(hostname=self.serverurl,
localcert=self.proxyfilename,
localkey=self.proxyfilename,
retry=2,
logger=self.logger,
verbose=False,
version=__version__,
userAgent='CRABClient')
self.crabserver.setDbInstance(self.instance)
# prepare also a test crabserver instance which will send tarballs to S3
self.s3tester = crabRest(hostname='cmsweb-testbed.cern.ch',
localcert=self.proxyfilename,
localkey=self.proxyfilename,
retry=0,
logger=self.logger,
verbose=False,
version=__version__,
userAgent='CRABClient')
self.s3tester.setDbInstance('preprod')
self.handleMyProxy()
# Validate the command options
self.validateOptions()
self.validateOptions()
# Log user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion()
self.defaultApi = 'workflow'
self.logger.debug("Instance is %s" % (self.instance))
self.logger.debug("Server base url is %s" % (self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command api %s" % (self.defaultApi))
def serverInstance(self):
"""
Deriving the correct instance to use and the server url. Client is allowed to propagate the instance name and corresponding url
via crabconfig.py or crab option --instance. The variable passed via crab option will always be used over the variable
in crabconfig.py.
"""
if hasattr(self.options,
'instance') and self.options.instance is not None:
if hasattr(self, 'configuration') and hasattr(
self.configuration, 'General') and hasattr(
self.configuration.General, 'instance'
) and self.configuration.General.instance is not None:
msg = "%sWarning%s: CRAB configuration parameter General.instance is overwritten by the command option --instance;" % (
colors.RED, colors.NORMAL)
msg += " %s intance will be used." % (self.options.instance)
self.logger.info(msg)
instance = self.options.instance
elif hasattr(self, 'configuration') and hasattr(
self.configuration, 'General') and hasattr(
self.configuration.General, 'instance'
) and self.configuration.General.instance is not None:
instance = self.configuration.General.instance
else:
instance = getParamDefaultValue('General.instance')
# validate instance
if not instance in SERVICE_INSTANCES:
msg = 'Invalid value "%s" for configuration.General.instance\n' % instance
msg += 'valid values are %s ' % SERVICE_INSTANCES.keys()
raise ConfigurationException(msg)
if instance != 'other':
self.restHost = SERVICE_INSTANCES[instance]['restHost']
self.dbInstance = SERVICE_INSTANCES[instance]['dbInstance']
else:
self.restHost = self.configuration.General.restHost
self.dbInstance = self.configuration.General.dbInstance
# attempt at backward cmpatibility
#self.serverurl = self.serverHost
#self.instance = self.dbInstance
return self.dbInstance, self.restHost
def checkversion(self):
compatibleVersions = server_info(crabserver=self.crabserver,
subresource='version')
for item in compatibleVersions:
if re.match(item, __version__):
self.logger.debug("CRABClient version: %s" % (__version__))
break
else:
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " Incompatible CRABClient version %s" % (__version__)
msg += "\nServer is saying that compatible versions are: %s" % [
v.replace("\\", "") for v in compatibleVersions
]
self.logger.info(msg)
def handleVomsProxy(self, proxyOptsSetPlace):
"""
Make sure that there is a valid VOMS proxy
:param proxyOptsSetPlace: a complicated dictionary to keep track of VOMS group/roles in options/proxyfile/requestcache
used by CredentialInteractions/createNewVomsProxy()
:return: nothing
a dictionary with proxy info is added to self as self.proxyInfo and
the file name of a valid proxy is addded as self.proxyfilename
"""
if self.options.proxy: # user passed a proxy as option
self.proxyfilename = self.options.proxy
os.environ['X509_USER_PROXY'] = self.options.proxy
self.logger.debug('Skipping proxy creation')
return
if self.cmdconf[
'initializeProxy']: # actually atm all commands require a proxy, see ClientMapping.py
self.credentialHandler.setVOGroupVORole(self.voGroup, self.voRole)
proxyInfo = self.credentialHandler.createNewVomsProxy(timeLeftThreshold=720, \
proxyCreatedByCRAB=self.proxyCreated, \
proxyOptsSetPlace=proxyOptsSetPlace)
self.proxyfilename = proxyInfo['filename']
return
def handleMyProxy(self):
"""
check myproxy credential and delegate again it if necessary.
takes no input and returns no output, bur raises exception if delegation failed
"""
if not self.cmdconf[
'requiresREST']: # If the command doesn't contact the REST, we can't delegate the proxy.
return
if self.options.proxy: # if user passed a proxy as option we don't contact myproxy
return
if not self.options.proxy:
# Get the DN of the task workers from the server.
all_task_workers_dns = server_info(self.crabserver,
subresource='delegatedn')
for authorizedDNs in all_task_workers_dns['services']:
self.credentialHandler.setRetrievers(authorizedDNs)
self.logger.debug(
"Registering user credentials on myproxy for %s" %
authorizedDNs)
try:
(credentialName, myproxyTimeleft) = \
self.credentialHandler.createNewMyProxy(timeleftthreshold=60 * 60 * 24 * RENEW_MYPROXY_THRESHOLD)
p1 = True
msg1 = "Credential exists on myproxy: username: %s - validity: %s" %\
(credentialName, str(timedelta(seconds=myproxyTimeleft)))
except Exception as ex:
p1 = False
msg1 = "Error trying to create credential:\n %s" % str(ex)
if (not p1):
from CRABClient.ClientExceptions import ProxyCreationException
raise ProxyCreationException(
"Problems delegating My-proxy.\n%s" % msg1)
self.logger.debug("Result of myproxy credential check:\n %s",
msg1)
def loadLocalCache(self):
"""
Loads the client cache and set up the server url
"""
self.requestarea, self.requestname = getWorkArea(self.options.projdir)
try:
self.cachedinfo, self.logfile = loadCache(self.requestarea,
self.logger)
port = ':' + self.cachedinfo['Port'] if self.cachedinfo[
'Port'] else ''
self.instance = self.cachedinfo['instance']
self.serverurl = self.cachedinfo['Server'] + port
except CachefileNotFoundException as ex:
if self.cmdconf['requiresLocalCache']:
raise ex
def setCachedProxy(self, proxyOptsSetPlace):
"""
Set the proxy parameters from the cache if not specified otherwise
"""
msgadd = []
if self.cmdconf[
'requiresProxyVOOptions'] and self.options.voGroup is None:
self.voGroup = self.cachedinfo['voGroup']
proxyOptsSetPlace['set_in']['group'] = "cache"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.cmdconf[
'requiresProxyVOOptions'] and self.options.voRole is None:
self.voRole = self.cachedinfo['voRole']
proxyOptsSetPlace['set_in']['role'] = "cache"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as written in the request cache file for this task." % (
" and ".join(msgadd))
self.logger.debug(msg)
def getConfiDict(self):
"""
Load the CRAB cache file (~/.crab3). If it doesn't exist, create one.
"""
crabCacheFileName = self.crabcachepath()
if not os.path.isfile(crabCacheFileName):
msg = "Could not find CRAB cache file %s; creating a new one." % (
crabCacheFileName)
self.logger.debug(msg)
configdict = {'crab_project_directory': ''}
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(configdict, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
return configdict
try:
msg = "Found CRAB cache file %s" % (crabCacheFileName)
self.logger.debug(msg)
with open(crabCacheFileName, 'r') as fd:
configdict = json.load(fd)
except ValueError:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Error loading CRAB cache file."
msg += " Try to do 'rm -rf %s' and run the crab command again." % (
crabCacheFileName)
raise ConfigurationException(msg)
if 'crab_project_directory' not in configdict:
configdict['crab_project_directory'] = configdict.get(
'taskname', '')
if 'taskname' in configdict:
del configdict['taskname']
return configdict
def crabcachepath(self):
if 'CRAB3_CACHE_FILE' in os.environ:
if os.path.isabs(os.environ['CRAB3_CACHE_FILE']):
return os.environ['CRAB3_CACHE_FILE']
else:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Invalid path in environment variable CRAB3_CACHE_FILE: %s" % (
os.environ['CRAB3_CACHE_FILE'])
msg += " Please export a valid full path."
raise EnvironmentException(msg)
else:
return str(os.path.expanduser('~')) + '/.crab3'
def updateCRABCacheFile(self):
"""
Update the CRAB cache file.
So far this file contains only the path of the last used CRAB project directory.
"""
if self.cmdconf['requiresDirOption'] or getattr(
self, 'requestarea', None):
self.crab3dic['crab_project_directory'] = self.requestarea
crabCacheFileName = self.crabcachepath()
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(self.crab3dic, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
def __call__(self):
self.logger.info("This is a 'nothing to do' command")
raise NotImplementedError
def terminate(self, exitcode):
#We do not want to print logfile for each command...
if exitcode < 2000:
if getattr(self.options, 'dump', False) or getattr(
self.options, 'xroot', False):
self.logger.debug("Log file is %s" %
os.path.abspath(self.logfile))
else:
self.logger.info("Log file is %s" %
os.path.abspath(self.logfile))
def setOptions(self):
raise NotImplementedError
def setSuperOptions(self):
try:
#add command related options
self.setOptions()
except NotImplementedError:
pass
self.parser.addCommonOptions(self.cmdconf)
def validateOptions(self):
"""
__validateOptions__
Validate the command line options of the command.
Raise a ConfigurationException in case of error; don't do anything if ok.
"""
if self.cmdconf['requiresDirOption']:
if self.options.projdir is None:
if len(self.args) > 1:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command accepts at most 1 argument (a path to a CRAB project directory), %d given." % (
self.name, len(self.args))
raise ConfigurationException(msg)
elif len(self.args) == 1 and self.args[0]:
self.options.projdir = self.args.pop(0)
elif self.cmdconf['useCache'] and self.crab3dic.get(
'crab_project_directory'):
self.options.projdir = str(
self.crab3dic['crab_project_directory'])
if self.options.projdir is None:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Please indicate the CRAB project directory with --dir=<project-directory>."
ex = MissingOptionException(msg)
ex.missingOption = "task"
raise ex
if not os.path.isdir(self.options.projdir):
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " %s is not a valid CRAB project directory." % (
self.options.projdir)
raise ConfigurationException(msg)
# If an input project directory was given, load the request cache and take the
# server URL from it.
self.loadLocalCache()
# If the command does not take any arguments, but some arguments were passed,
# clear the arguments list and give a warning message saying that the given
# arguments will be ignored.
if not self.cmdconf['acceptsArguments'] and len(self.args):
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command takes no arguments, %d given." % (
self.name, len(self.args))
msg += " Ignoring arguments %s." % (self.args)
self.logger.warning(msg)
self.args = []
def validateLogpathOption(self):
pass
def validateConfigOption(self):
pass
class SubCommand(ConfigCommand):
####### These options can be overrhidden if needed ########
## setting visible = False doesn't allow the sub-command to be called from CLI
visible = True
proxyfilename = None
shortnames = []
usage = "usage: %prog [command-options] [args]"
#Default command name is the name of the command class, but it is also possible to set the name attribute in the subclass
#if the command name does not correspond to the class name (getlog => get-log)
def getUrl(self, instance='prod', resource='workflow'):
"""
Retrieve the url depending on the resource we are accessing and the instance.
"""
if instance in SERVICE_INSTANCES.keys():
return BASEURL + instance + '/' + resource
elif instance == 'private':
return BASEURL + 'dev' + '/' + resource
raise ConfigurationException('Error: only %s instances can be used.' %str(SERVICE_INSTANCES.keys()))
def __init__(self, logger, cmdargs = None, disable_interspersed_args = False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
self.usage = "usage: %prog " + self.name + " [options] [args]"
self.logger = logger
self.logfile = self.logger.logfile
self.logger.debug("Executing command: '%s'" % str(self.name))
self.proxy = None
self.restClass = CRABClient.Emulator.getEmulator('rest')
self.cmdconf = commands_configuration.get(self.name)
self.crab3dic = self.getConfiDict()
self.parser = OptionParser(description = self.__doc__, usage = self.usage, add_help_option = True)
## TODO: check on self.name should be removed (creating another abstraction in between or refactoring this)
if disable_interspersed_args:
self.parser.disable_interspersed_args()
self.setSuperOptions()
## Parse the command line parameters.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
## Validate the command line parameters before initializing the proxy.
self.validateOptions()
## Retrieve VO role/group from the command line parameters.
self.voRole = self.options.voRole if self.options.voRole is not None else ''
self.voGroup = self.options.voGroup if self.options.voGroup is not None else ''
## Create the object that will do the proxy operations. We ignore the VO role/group and
## server URL in the initialization, because they are not used until we do the proxy
## delegation to myproxy server. And the delegation happends in handleProxy(), which is
## called after we load the configuration file and retrieve the final values for those
## parameters. handleProxy() takes care of passing those parameters to self.proxy.
self.proxy = CredentialInteractions('', '', self.voRole, self.voGroup, self.logger, '')
## Create a new proxy (if there isn't a valid one already) with current VO role/group.
## The VO role/group are not relevant for the proxy. We only want to check that the
## user doesn't expect the proxy to have different VO role/group than those specified
## in the configuration file, but this check is done later in handleProxy() after we
## load the configuration file.
self.proxy_created = False
if not self.options.proxy and self.cmdconf['initializeProxy']:
self.proxy_created = self.proxy.createNewVomsProxySimple(time_left_threshold = 720)
## If we get an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
## Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
## Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
## If VO role/group were not given in the command options, get them from the configuration file.
## If they are specified in both places, print a message saying that the command options will be used.
if self.options.voRole is None and hasattr(self.configuration, 'User'):
self.voRole = getattr(self.configuration.User, 'voRole', '')
if self.options.voGroup is None and hasattr(self.configuration, 'User'):
self.voGroup = getattr(self.configuration.User, 'voGroup', '')
if (self.options.voRole is not None) and (hasattr(self.configuration, 'User') and hasattr(self.configuration.User, 'voRole' )):
msg = "Ignoring the VO role specified in the configuration file. Using VO role \"%s\" "
if self.voRole == '': msg += "(i.e. no VO role) "
msg += "as specified in the command line."
self.logger.info(msg % self.voRole)
if (self.options.voGroup is not None) and (hasattr(self.configuration, 'User') and hasattr(self.configuration.User, 'voGroup')):
msg = "Ignoring the VO group specified in the configuration file. Using VO group \"%s\" "
if self.voGroup == '': msg += "(i.e. no VO group) "
msg += "as specified in the command line."
self.logger.info(msg % self.voGroup)
## If we get an input task, we load the cache and set the server URL and VO role/group from it.
if hasattr(self.options, 'task') and self.options.task:
self.loadLocalCache()
## If the server url isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
## Update (or create) the .crab3 cache file.
self.updateCrab3()
## At this point there should be a valid proxy, because we have already checked that and
## eventually created a new one. If the proxy was not created by CRAB, we check that the
## VO role/group in the proxy are the same as specified by the user in the configuration
## file (or in the command line options). If it is not, we ask the user if he wants to
## overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
## and ask him to provide the VO role/group as in the existing proxy.
## Finally, delegate the proxy to myproxy server.
self.handleProxy()
## Logging user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion(self.getUrl(self.instance, resource='info'))
self.uri = self.getUrl(self.instance)
self.logger.debug("Instance is %s" %(self.instance))
self.logger.debug("Server base url is %s" %(self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command url %s" %(self.uri))
def serverInstance(self):
"""
Deriving the correct instance to use and the server url. Client is allowed to propagate the instance name and corresponding url
via crabconfig.py or crab option --instance. The variable passed via crab option will always be used over the variable
in crabconfig.py. Instance name other than specify in the SERVICE_INSTANCE will be treated as a private instance.
"""
serverurl = None
#Will be use to print available instances
available_instances = ', '.join(SERVICE_INSTANCES)
if hasattr(self.options, 'instance') and not self.options.instance is None:
if hasattr(self, 'configuration') and hasattr(self.configuration.General, 'instance') and not self.configuration.General.instance is None:
self.logger.info('%sWarning%s: Instance value in configuration file is overwritten by the option command, %s intance will be use ' % (colors.RED, colors.NORMAL, self.options.instance))
if self.options.instance in SERVICE_INSTANCES.keys():
instance = self.options.instance
serverurl = SERVICE_INSTANCES[instance]
else:
instance = 'private'
serverurl = self.options.instance
elif hasattr(self, 'configuration') and hasattr(self.configuration.General, 'instance') and not self.configuration.General.instance is None:
if self.configuration.General.instance in SERVICE_INSTANCES.keys():
instance = self.configuration.General.instance
serverurl = SERVICE_INSTANCES[instance]
else:
instance = 'private'
serverurl = self.configuration.General.instance
else:
instance = 'prod'
serverurl = SERVICE_INSTANCES[instance]
return instance, serverurl
def checkversion(self, baseurl = None):
compatibleversion = server_info('version', self.serverurl, self.proxyfilename, baseurl)
if __version__ in compatibleversion:
self.logger.debug("CRABClient version: %s Compatible" % __version__)
else:
self.logger.info("%sWARNING%s: Incompatible CRABClient version \"%s\" " % (colors.RED, colors.NORMAL , __version__ ))
self.logger.info("Server is saying that compatible versions are: %s" % compatibleversion)
def handleProxy(self):
"""
Init the user proxy, and delegate it if necessary.
"""
if not self.options.proxy:
if self.cmdconf['initializeProxy']:
self.proxy.setVOGroupVORole(self.voGroup, self.voRole)
self.proxy.setMyProxyAccount(self.serverurl)
_, self.proxyfilename = self.proxy.createNewVomsProxy(time_left_threshold = 720, proxy_created_by_crab = self.proxy_created)
if self.cmdconf['requiresREST']: ## If the command doesn't contact the REST, we can't delegate the proxy.
self.proxy.myproxyAccount = self.serverurl
baseurl = self.getUrl(self.instance, resource = 'info')
## Get the DN of the task workers from the server.
all_task_workers_dns = server_info('delegatedn', self.serverurl, self.proxyfilename, baseurl)
for serverdn in all_task_workers_dns['services']:
self.proxy.setServerDN(serverdn)
self.proxy.setMyProxyServer('myproxy.cern.ch')
self.logger.debug("Registering user credentials for server %s" % serverdn)
self.proxy.createNewMyProxy(timeleftthreshold = 60 * 60 * 24 * RENEW_MYPROXY_THRESHOLD, nokey = True)
else:
self.proxyfilename = self.options.proxy
os.environ['X509_USER_PROXY'] = self.options.proxy
self.logger.debug('Skipping proxy creation')
def loadLocalCache(self, serverurl = None):
"""
Loads the client cache and set up the server url
"""
self.requestarea, self.requestname = getWorkArea( self.options.task )
self.cachedinfo, self.logfile = loadCache(self.requestarea, self.logger)
port = ':' + self.cachedinfo['Port'] if self.cachedinfo['Port'] else ''
self.instance = self.cachedinfo['instance']
self.serverurl = self.cachedinfo['Server'] + port
self.voRole = self.cachedinfo['voRole'] #if not self.options.voRole else self.options.voRole
self.voGroup = self.cachedinfo['voGroup'] #if not self.options.voGroup else self.options.voGroup
def getConfiDict(self):
crab3fdir = self.crabcachepath()
if not os.path.isfile(crab3fdir):
self.logger.debug("Could not find %s file; creating a new one" % crab3fdir)
crab3f = open(crab3fdir,'w')
#creating a user dict, do add for future use
configdict = { "taskname" : None }
json.dump(configdict,crab3f)
crab3f.close()
return configdict
else:
try :
self.logger.debug("Found %s file" % crab3fdir)
crab3f = open(crab3fdir,'r')
configdict = json.load(crab3f)
crab3f.close()
except ValueError:
self.logger.info('%sError%s: Error in reading json file\nTry to do "rm -rf ~/.crab3", and run the crab command again'% (colors.RED, colors.NORMAL))
raise ConfigurationException
return configdict
def crabcachepath(self):
if 'CRAB3_CACHE_FILE' in os.environ and os.path.isabs(os.environ['CRAB3_CACHE_FILE']):
return os.environ['CRAB3_CACHE_FILE']
elif 'CRAB3_CACHE_FILE' in os.environ and not os.path.isabs(os.environ['CRAB3_CACHE_FILE']):
msg = '%sError%s: An invalid path is use for CRAB3_CACHE_FILE, please export a valid full path' % (colors.RED, colors.NORMAL)
raise EnvironmentException(msg)
else:
return str(os.path.expanduser('~')) + '/.crab3'
def updateCrab3(self):
if self.cmdconf['requiresTaskOption'] or hasattr(self,'requestname') and self.requestname != None:
crab3fdir=self.crabcachepath()
crab3f = open(crab3fdir, 'w')
self.crab3dic['taskname'] = self.requestarea
json.dump(self.crab3dic, crab3f)
crab3f.close()
def __call__(self):
self.logger.info("This is a 'nothing to do' command")
raise NotImplementedError
def terminate(self, exitcode):
#We do not want to print logfile for each command...
if exitcode < 2000:
if getattr(self.options, 'dump', False) or getattr(self.options, 'xroot', False):
self.logger.debug("Log file is %s" % os.path.abspath(self.logfile))
else:
self.logger.info("Log file is %s" % os.path.abspath(self.logfile))
def setOptions(self):
raise NotImplementedError
def setSuperOptions(self):
try:
self.setOptions()
except NotImplementedError:
pass
self.parser.add_option("--proxy",
dest = "proxy",
default = False,
help = "Use the given proxy. Skip Grid proxy creation and myproxy delegation.")
if self.cmdconf['requiresTaskOption']:
self.parser.add_option("-d", "--dir",
dest = "task",
default = None,
help = "Path to the crab project directory for which the crab command should be executed.")
self.parser.add_option("-t", "--task",
dest = "oldtask",
default = None,
help = "Deprecated option renamed to -d/--dir in CRAB v3.3.12.")
self.parser.add_option("--voRole",
dest = "voRole",
default = None)
self.parser.add_option("--voGroup",
dest = "voGroup",
default = None)
if self.cmdconf['requiresREST']:
self.parser.add_option("--instance",
dest = "instance",
type = "string",
help = "Running instance of CRAB service. Valid values are %s." %str(SERVICE_INSTANCES.keys()))
def validateOptions(self):
"""
__validateOptions__
Validate the command line options of the command.
Raise a ConfigurationException in case of error; don't do anything if ok.
"""
if self.cmdconf['requiresTaskOption'] and self.options.oldtask is not None:
msg = "CRAB command line option error: the option -t/--task has been renamed to -d/--dir."
raise UnknownOptionException(msg)
if self.cmdconf['requiresTaskOption'] and self.options.task is None:
if len(self.args) == 1 and self.args[0]:
self.options.task = self.args[0]
elif self.cmdconf['useCache'] and self.crab3dic['taskname'] != None:
self.options.task = self.crab3dic['taskname']
else:
raise MissingOptionException('ERROR: Task option is required.')
class SubCommand(ConfigCommand):
####### These options can be overrhidden if needed ########
## setting visible = False doesn't allow the sub-command to be called from CLI
visible = True
proxyfilename = None
shortnames = []
usage = "usage: %prog [command-options] [args]"
#Default command name is the name of the command class, but it is also possible to set the name attribute in the subclass
#if the command name does not correspond to the class name (getlog => get-log)
def getUrl(self, instance='prod', resource='workflow'):
"""
Retrieve the url depending on the resource we are accessing and the instance.
"""
if instance in SERVICE_INSTANCES.keys():
return BASEURL + instance + '/' + resource
elif instance == 'private':
return BASEURL + 'dev' + '/' + resource
raise ConfigurationException('Error: only %s instances can be used.' %
str(SERVICE_INSTANCES.keys()))
def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
self.usage = "usage: %prog " + self.name + " [options] [args]"
self.logger = logger
self.logfile = self.logger.logfile
self.logger.debug("Executing command: '%s'" % str(self.name))
self.proxy = None
self.restClass = CRABClient.Emulator.getEmulator('rest')
self.cmdconf = commands_configuration.get(self.name)
self.crab3dic = self.getConfiDict()
self.parser = OptionParser(description=self.__doc__,
usage=self.usage,
add_help_option=True)
## TODO: check on self.name should be removed (creating another abstraction in between or refactoring this)
if disable_interspersed_args:
self.parser.disable_interspersed_args()
self.setSuperOptions()
## Parse the command line parameters.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
## Validate the command line parameters before initializing the proxy.
self.validateOptions()
## Retrieve VO role/group from the command line parameters.
self.voRole = self.options.voRole if self.options.voRole is not None else ''
self.voGroup = self.options.voGroup if self.options.voGroup is not None else ''
## Create the object that will do the proxy operations. We ignore the VO role/group and
## server URL in the initialization, because they are not used until we do the proxy
## delegation to myproxy server. And the delegation happends in handleProxy(), which is
## called after we load the configuration file and retrieve the final values for those
## parameters. handleProxy() takes care of passing those parameters to self.proxy.
self.proxy = CredentialInteractions('', '', self.voRole, self.voGroup,
self.logger, '')
## Create a new proxy (if there isn't a valid one already) with current VO role/group.
## The VO role/group are not relevant for the proxy. We only want to check that the
## user doesn't expect the proxy to have different VO role/group than those specified
## in the configuration file, but this check is done later in handleProxy() after we
## load the configuration file.
self.proxy_created = False
if not self.options.proxy and self.cmdconf['initializeProxy']:
self.proxy_created = self.proxy.createNewVomsProxySimple(
time_left_threshold=720)
## If we get an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
## Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
## Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(
self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
## If VO role/group were not given in the command options, get them from the configuration file.
## If they are specified in both places, print a message saying that the command options will be used.
if self.options.voRole is None and hasattr(self.configuration,
'User'):
self.voRole = getattr(self.configuration.User, 'voRole', '')
if self.options.voGroup is None and hasattr(
self.configuration, 'User'):
self.voGroup = getattr(self.configuration.User, 'voGroup', '')
if (self.options.voRole is not None) and (
hasattr(self.configuration, 'User')
and hasattr(self.configuration.User, 'voRole')):
msg = "Ignoring the VO role specified in the configuration file. Using VO role \"%s\" "
if self.voRole == '': msg += "(i.e. no VO role) "
msg += "as specified in the command line."
self.logger.info(msg % self.voRole)
if (self.options.voGroup is not None) and (
hasattr(self.configuration, 'User')
and hasattr(self.configuration.User, 'voGroup')):
msg = "Ignoring the VO group specified in the configuration file. Using VO group \"%s\" "
if self.voGroup == '': msg += "(i.e. no VO group) "
msg += "as specified in the command line."
self.logger.info(msg % self.voGroup)
## If we get an input task, we load the cache and set the server URL and VO role/group from it.
if hasattr(self.options, 'task') and self.options.task:
self.loadLocalCache()
## If the server url isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
## Update (or create) the .crab3 cache file.
self.updateCrab3()
## At this point there should be a valid proxy, because we have already checked that and
## eventually created a new one. If the proxy was not created by CRAB, we check that the
## VO role/group in the proxy are the same as specified by the user in the configuration
## file (or in the command line options). If it is not, we ask the user if he wants to
## overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
## and ask him to provide the VO role/group as in the existing proxy.
## Finally, delegate the proxy to myproxy server.
self.handleProxy()
## Logging user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion(self.getUrl(self.instance, resource='info'))
self.uri = self.getUrl(self.instance)
self.logger.debug("Instance is %s" % (self.instance))
self.logger.debug("Server base url is %s" % (self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command url %s" % (self.uri))
def serverInstance(self):
"""
Deriving the correct instance to use and the server url. Client is allowed to propagate the instance name and corresponding url
via crabconfig.py or crab option --instance. The variable passed via crab option will always be used over the variable
in crabconfig.py. Instance name other than specify in the SERVICE_INSTANCE will be treated as a private instance.
"""
serverurl = None
#Will be use to print available instances
available_instances = ', '.join(SERVICE_INSTANCES)
if hasattr(self.options,
'instance') and not self.options.instance is None:
if hasattr(self, 'configuration') and hasattr(
self.configuration.General, 'instance'
) and not self.configuration.General.instance is None:
self.logger.info(
'%sWarning%s: Instance value in configuration file is overwritten by the option command, %s intance will be use '
% (colors.RED, colors.NORMAL, self.options.instance))
if self.options.instance in SERVICE_INSTANCES.keys():
instance = self.options.instance
serverurl = SERVICE_INSTANCES[instance]
else:
instance = 'private'
serverurl = self.options.instance
elif hasattr(self, 'configuration') and hasattr(
self.configuration.General, 'instance'
) and not self.configuration.General.instance is None:
if self.configuration.General.instance in SERVICE_INSTANCES.keys():
instance = self.configuration.General.instance
serverurl = SERVICE_INSTANCES[instance]
else:
instance = 'private'
serverurl = self.configuration.General.instance
else:
instance = 'prod'
serverurl = SERVICE_INSTANCES[instance]
return instance, serverurl
def checkversion(self, baseurl=None):
compatibleversion = server_info('version', self.serverurl,
self.proxyfilename, baseurl)
if __version__ in compatibleversion:
self.logger.debug("CRABClient version: %s Compatible" %
__version__)
else:
self.logger.info(
"%sWARNING%s: Incompatible CRABClient version \"%s\" " %
(colors.RED, colors.NORMAL, __version__))
self.logger.info(
"Server is saying that compatible versions are: %s" %
compatibleversion)
def handleProxy(self):
"""
Init the user proxy, and delegate it if necessary.
"""
if not self.options.proxy:
if self.cmdconf['initializeProxy']:
self.proxy.setVOGroupVORole(self.voGroup, self.voRole)
self.proxy.setMyProxyAccount(self.serverurl)
_, self.proxyfilename = self.proxy.createNewVomsProxy(
time_left_threshold=720,
proxy_created_by_crab=self.proxy_created)
if self.cmdconf[
'requiresREST']: ## If the command doesn't contact the REST, we can't delegate the proxy.
self.proxy.myproxyAccount = self.serverurl
baseurl = self.getUrl(self.instance, resource='info')
## Get the DN of the task workers from the server.
all_task_workers_dns = server_info('delegatedn',
self.serverurl,
self.proxyfilename,
baseurl)
for serverdn in all_task_workers_dns['services']:
self.proxy.setServerDN(serverdn)
self.proxy.setMyProxyServer('myproxy.cern.ch')
self.logger.debug(
"Registering user credentials for server %s" %
serverdn)
self.proxy.createNewMyProxy(timeleftthreshold=60 * 60 *
24 *
RENEW_MYPROXY_THRESHOLD,
nokey=True)
else:
self.proxyfilename = self.options.proxy
os.environ['X509_USER_PROXY'] = self.options.proxy
self.logger.debug('Skipping proxy creation')
def loadLocalCache(self, serverurl=None):
"""
Loads the client cache and set up the server url
"""
self.requestarea, self.requestname = getWorkArea(self.options.task)
self.cachedinfo, self.logfile = loadCache(self.requestarea,
self.logger)
port = ':' + self.cachedinfo['Port'] if self.cachedinfo['Port'] else ''
self.instance = self.cachedinfo['instance']
self.serverurl = self.cachedinfo['Server'] + port
self.voRole = self.cachedinfo[
'voRole'] #if not self.options.voRole else self.options.voRole
self.voGroup = self.cachedinfo[
'voGroup'] #if not self.options.voGroup else self.options.voGroup
def getConfiDict(self):
crab3fdir = self.crabcachepath()
if not os.path.isfile(crab3fdir):
self.logger.debug("Could not find %s file; creating a new one" %
crab3fdir)
crab3f = open(crab3fdir, 'w')
#creating a user dict, do add for future use
configdict = {"taskname": None}
json.dump(configdict, crab3f)
crab3f.close()
return configdict
else:
try:
self.logger.debug("Found %s file" % crab3fdir)
crab3f = open(crab3fdir, 'r')
configdict = json.load(crab3f)
crab3f.close()
except ValueError:
self.logger.info(
'%sError%s: Error in reading json file\nTry to do "rm -rf ~/.crab3", and run the crab command again'
% (colors.RED, colors.NORMAL))
raise ConfigurationException
return configdict
def crabcachepath(self):
if 'CRAB3_CACHE_FILE' in os.environ and os.path.isabs(
os.environ['CRAB3_CACHE_FILE']):
return os.environ['CRAB3_CACHE_FILE']
elif 'CRAB3_CACHE_FILE' in os.environ and not os.path.isabs(
os.environ['CRAB3_CACHE_FILE']):
msg = '%sError%s: An invalid path is use for CRAB3_CACHE_FILE, please export a valid full path' % (
colors.RED, colors.NORMAL)
raise EnvironmentException(msg)
else:
return str(os.path.expanduser('~')) + '/.crab3'
def updateCrab3(self):
if self.cmdconf['requiresTaskOption'] or hasattr(
self, 'requestname') and self.requestname != None:
crab3fdir = self.crabcachepath()
crab3f = open(crab3fdir, 'w')
self.crab3dic['taskname'] = self.requestarea
json.dump(self.crab3dic, crab3f)
crab3f.close()
def __call__(self):
self.logger.info("This is a 'nothing to do' command")
raise NotImplementedError
def terminate(self, exitcode):
#We do not want to print logfile for each command...
if exitcode < 2000:
if getattr(self.options, 'dump', False) or getattr(
self.options, 'xroot', False):
self.logger.debug("Log file is %s" %
os.path.abspath(self.logfile))
else:
self.logger.info("Log file is %s" %
os.path.abspath(self.logfile))
def setOptions(self):
raise NotImplementedError
def setSuperOptions(self):
try:
self.setOptions()
except NotImplementedError:
pass
self.parser.add_option(
"--proxy",
dest="proxy",
default=False,
help=
"Use the given proxy. Skip Grid proxy creation and myproxy delegation."
)
if self.cmdconf['requiresTaskOption']:
self.parser.add_option(
"-d",
"--dir",
dest="task",
default=None,
help=
"Path to the crab project directory for which the crab command should be executed."
)
self.parser.add_option(
"-t",
"--task",
dest="oldtask",
default=None,
help="Deprecated option renamed to -d/--dir in CRAB v3.3.12.")
self.parser.add_option("--voRole", dest="voRole", default=None)
self.parser.add_option("--voGroup", dest="voGroup", default=None)
if self.cmdconf['requiresREST']:
self.parser.add_option(
"--instance",
dest="instance",
type="string",
help="Running instance of CRAB service. Valid values are %s." %
str(SERVICE_INSTANCES.keys()))
def validateOptions(self):
"""
__validateOptions__
Validate the command line options of the command.
Raise a ConfigurationException in case of error; don't do anything if ok.
"""
if self.cmdconf[
'requiresTaskOption'] and self.options.oldtask is not None:
msg = "CRAB command line option error: the option -t/--task has been renamed to -d/--dir."
raise UnknownOptionException(msg)
if self.cmdconf['requiresTaskOption'] and self.options.task is None:
if len(self.args) == 1 and self.args[0]:
self.options.task = self.args[0]
elif self.cmdconf['useCache'] and self.crab3dic['taskname'] != None:
self.options.task = self.crab3dic['taskname']
else:
raise MissingOptionException('ERROR: Task option is required.')