def handleProxy(server, baseurl, voRole, voGroup, days):
"""
"""
proxy = CredentialInteractions('', '', voRole, voGroup, logging)
#proxy.myproxyDesiredValidity = days
proxy.setMyProxyValidity(int(days) * 24 * 60)
logging.info("Checking credentials")
proxy.proxyInfo = proxy.createNewVomsProxy(timeLeftThreshold=720)
proxyfilename = proxy.proxyInfo['filename']
#get the dn of the agents from the server
alldns = server_info('delegatedn', server, proxyfilename, baseurl)
for serverdn in alldns['services']:
proxy.defaultDelegation['serverDN'] = serverdn
proxy.defaultDelegation['myProxySvr'] = 'myproxy.cern.ch'
logging.info("Registering user credentials for server %s" % serverdn)
proxy.createNewMyProxy(timeleftthreshold=60 * 60 * 24 *
RENEW_MYPROXY_THRESHOLD,
nokey=True)
proxy.createNewMyProxy2(timeleftthreshold=60 * 60 * 24 *
RENEW_MYPROXY_THRESHOLD,
nokey=True)
def __call__(self):
proxy = CredentialInteractions('', '', '', '', self.logger)
days = self.options.days
proxy.setMyProxyValidity(int(days) * 24 * 60) # minutes
timeLeftThreshold = int(days) * 24 * 60 * 60 # seconds
self.logger.info("Checking credentials")
# need an X509 proxy in order to talk with CRABServer to get list of myproxy authorized retrievers
proxy.proxyInfo = proxy.createNewVomsProxy(timeLeftThreshold=720)
proxyfilename = proxy.proxyInfo['filename']
serverurl = 'cmsweb.cern.ch'
baseurl = '/crabserver/prod/info'
alldns = server_info(subresource='delegatedn', serverurl=serverurl, proxyfilename=proxyfilename, baseurl=baseurl)
for serverdn in alldns['services']:
proxy.defaultDelegation['serverDN'] = serverdn
proxy.defaultDelegation['myProxySvr'] = 'myproxy.cern.ch'
self.logger.info("Registering user credentials for server %s" % serverdn)
(credentialName, myproxyTimeleft) = proxy.createNewMyProxy(timeleftthreshold=timeLeftThreshold, nokey=True)
self.logger.info("Credential exists on myproxy: username: %s - validity: %s", credentialName,
str(timedelta(seconds=myproxyTimeleft)))
(credentialName, myproxyTimeleft) = proxy.createNewMyProxy2(timeleftthreshold=timeLeftThreshold, nokey=True)
self.logger.info("Credential exists on myproxy: username: %s - validity: %s", credentialName,
str(timedelta(seconds=myproxyTimeleft)))
return
class SubCommand(ConfigCommand):
####### These options can be overrhidden if needed ########
## setting visible = False doesn't allow the sub-command to be called from CLI
visible = True
proxyfilename = None
shortnames = []
usage = "usage: %prog [command-options] [args]"
#Default command name is the name of the command class, but it is also possible to set the name attribute in the subclass
#if the command name does not correspond to the class name (getlog => get-log)
def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
ConfigCommand.__init__(self)
## The command logger.
self.logger = logger
self.logfile = self.logger.logfile
localSystem = subprocess.check_output(['uname', '-a']).strip('\n')
try:
localOS = subprocess.check_output(
['grep', 'PRETTY_NAME', '/etc/os-release'],
stderr=subprocess.STDOUT).strip('\n')
localOS = localOS.split('=')[1].strip('"')
except Exception as ex: # pylint: disable=unused-variable
try:
localOS = subprocess.check_output(['lsb_release',
'-d']).strip('\n')
localOS = localOS.split(':')[1].strip()
except Exception as ex: # pylint: disable=unused-variable
localOS = "Unknown Operating System"
self.logger.debug("CRAB Client version: %s", __version__)
self.logger.debug("Running on: " + localSystem + " - " + localOS)
self.logger.debug("Executing command: '%s'" % str(self.name))
self.proxy = None
self.restClass = CRABClient.Emulator.getEmulator('rest')
## Get the command configuration.
self.cmdconf = commandsConfiguration.get(self.name)
if not self.cmdconf:
raise RuntimeError(
"Canot find command %s in commandsConfiguration inside ClientMapping. Are you a developer"
"trying to add a command without it's correspondant configuration?"
% self.name)
## Get the CRAB cache file.
self.cachedinfo = None
self.crab3dic = self.getConfiDict()
## The options parser.
self.parser = CRABCmdOptParser(self.name, self.__doc__,
disable_interspersed_args)
## Define the command options.
self.setSuperOptions()
## Parse the command options/arguments.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
self.transferringIds = None
self.dest = None
self.validateLogpathOption()
## Validate first the SubCommand options
SubCommand.validateOptions(self)
## then the config option for the submit command
self.validateConfigOption()
## Get the VO group/role from the command options (if the command requires these
## options).
proxyOptsSetPlace = {
'set_in': {
'group': "default",
'role': "default"
},
'for_set_use': ""
}
msgadd = []
self.voGroup, self.voRole = "", "NULL"
if self.cmdconf['requiresProxyVOOptions']:
proxyOptsSetPlace['for_set_use'] = "cmdopts"
if self.options.voGroup is not None:
self.voGroup = self.options.voGroup
proxyOptsSetPlace['set_in']['group'] = "cmdopts"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.options.voRole is not None:
self.voRole = self.options.voRole if self.options.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "cmdopts"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the crab command options." % (
" and ".join(msgadd))
self.logger.debug(msg)
## Create the object that will do the proxy operations. We don't really care
## what VO role and group and server URL we pass to the constructor, because
## these are not used until we do the proxy delegation to the myproxy server.
## And this happens in handleProxy(), which is called after we load the
## configuration file and retrieve the final values for those parameters.
## handleProxy() takes care of passing those parameters to self.proxy.
self.proxy = CredentialInteractions('', '', self.voRole, self.voGroup,
self.logger)
## If the user didn't use the --proxy command line option, and if there isn't a
## valid proxy already, we will create a new one with the current VO role and group
## (as commented above, we don't really care what are the VO role and group so
## far).
self.proxyCreated = False
## If there is an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
proxyOptsSetPlace['for_set_use'] = "config"
## Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
## Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(self.logger, \
getattr(self.configuration.General, 'workArea', None), \
getattr(self.configuration.General, 'requestName', None))
## Get the VO group/role from the configuration file.
msgadd = []
if hasattr(self.configuration, 'User') and hasattr(
self.configuration.User, 'voGroup'):
self.voGroup = self.configuration.User.voGroup
proxyOptsSetPlace['set_in']['group'] = "config"
msgadd.append("VO group '%s'" % (self.voGroup))
if hasattr(self.configuration, 'User') and hasattr(
self.configuration.User, 'voRole'):
self.voRole = self.configuration.User.voRole if self.configuration.User.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "config"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the CRAB configuration file." % (
" and ".join(msgadd))
self.logger.debug(msg)
## If the VO group/role was not given in the command options, take it from the request cache.
if self.cmdconf['requiresDirOption']:
self.setCachedProxy(proxyOptsSetPlace)
## following commented out code should be replaced with something which
## looks for configuration.General.instance, compares with known nicknames,
## extraxts restHost and dbInstance if OK, otherwise gets them from
## configuration.Genarl.restHost and configuration.General.dbInstance
## and sets them in self for all code to use.
## need to review all uses of self.instance and self.serverurl and change or
## check if they can simply be set to dbInstance and restHost repsecitvely
## If the server URL isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
## Update (or create) the CRAB cache file.
self.updateCRABCacheFile()
## At this point we check if there is a valid proxy, and
## eventually create a new one. If the proxy was not created by CRAB, we check that the
## VO role/group in the proxy are the same as specified by the user in the configuration
## file (or in the command line options). If it is not, we ask the user if he wants to
## overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
## and ask him to provide the VO role/group as in the existing proxy.
## Finally, delegate the proxy to myproxy server.
self.handleProxy(proxyOptsSetPlace)
## Validate the command options
self.validateOptions()
## Logging user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion(getUrl(self.instance, resource='info'))
self.uri = getUrl(self.instance)
self.logger.debug("Instance is %s" % (self.instance))
self.logger.debug("Server base url is %s" % (self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command url %s" % (self.uri))
def serverInstance(self):
"""
Deriving the correct instance to use and the server url. Client is allowed to propagate the instance name and corresponding url
via crabconfig.py or crab option --instance. The variable passed via crab option will always be used over the variable
in crabconfig.py.
"""
if hasattr(self.options,
'instance') and self.options.instance is not None:
if hasattr(self, 'configuration') and hasattr(
self.configuration, 'General') and hasattr(
self.configuration.General, 'instance'
) and self.configuration.General.instance is not None:
msg = "%sWarning%s: CRAB configuration parameter General.instance is overwritten by the command option --instance;" % (
colors.RED, colors.NORMAL)
msg += " %s intance will be used." % (self.options.instance)
self.logger.info(msg)
instance = self.options.instance
elif hasattr(self, 'configuration') and hasattr(
self.configuration, 'General') and hasattr(
self.configuration.General, 'instance'
) and self.configuration.General.instance is not None:
instance = self.configuration.General.instance
else:
instance = getParamDefaultValue('General.instance')
# validate instance
if not instance in SERVICE_INSTANCES:
msg = 'Invalid value "%s" for configuration.General.instance\n' % instance
msg += 'valid values are %s ' % SERVICE_INSTANCES.keys()
raise ConfigurationException(msg)
if instance is not 'other':
self.restHost = SERVICE_INSTANCES[instance]['restHost']
self.dbInstance = SERVICE_INSTANCES[instance]['dbInstance']
else:
self.restHost = self.configuration.General.restHost
self.dbInstance = self.configuration.General.dbInstance
# attempt at backward cmpatibility
#self.serverurl = self.serverHost
#self.instance = self.dbInstance
return self.dbInstance, self.restHost
def checkversion(self, baseurl=None):
compatibleVersions = server_info(subresource='version',
serverurl=self.serverurl,
proxyfilename=self.proxyfilename,
baseurl=baseurl,
logger=self.logger)
for item in compatibleVersions:
if re.match(item, __version__):
self.logger.debug("CRABClient version: %s" % (__version__))
break
else:
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " Incompatible CRABClient version %s" % (__version__)
msg += "\nServer is saying that compatible versions are: %s" % [
v.replace("\\", "") for v in compatibleVersions
]
self.logger.info(msg)
def handleProxy(self, proxyOptsSetPlace):
"""
Init the user proxy, and delegate it if necessary.
"""
if not self.options.proxy:
if self.cmdconf['initializeProxy']:
self.proxy.setVOGroupVORole(self.voGroup, self.voRole)
self.proxy.proxyInfo = self.proxy.createNewVomsProxy(timeLeftThreshold=720, \
doProxyGroupRoleCheck=self.cmdconf['doProxyGroupRoleCheck'], \
proxyCreatedByCRAB=self.proxyCreated, \
proxyOptsSetPlace=proxyOptsSetPlace)
self.proxyfilename = self.proxy.proxyInfo['filename']
if self.cmdconf[
'requiresREST']: ## If the command doesn't contact the REST, we can't delegate the proxy.
baseurl = getUrl(self.instance, resource='info')
## Get the DN of the task workers from the server.
all_task_workers_dns = server_info(
subresource='delegatedn',
serverurl=self.serverurl,
proxyfilename=self.proxyfilename,
baseurl=baseurl,
logger=self.logger)
for serverdn in all_task_workers_dns['services']:
self.proxy.setServerDN(serverdn)
self.proxy.setMyProxyServer('myproxy.cern.ch')
self.logger.debug(
"Registering user credentials for server %s" %
serverdn)
try:
(credentialName,
myproxyTimeleft) = self.proxy.createNewMyProxy(
timeleftthreshold=60 * 60 * 24 *
RENEW_MYPROXY_THRESHOLD,
nokey=True)
p1 = True
msg1 = "Credential exists on myproxy: username: %s - validity: %s" %\
(credentialName, str(timedelta(seconds=myproxyTimeleft)))
except Exception as ex:
p1 = False
msg1 = "Error trying to create credential:\n %s" % str(
ex)
try:
(credentialName,
myproxyTimeleft) = self.proxy.createNewMyProxy2(
timeleftthreshold=60 * 60 * 24 *
RENEW_MYPROXY_THRESHOLD,
nokey=True)
p2 = True
msg2 = "Credential exists on myproxy: username: %s - validity: %s" %\
(credentialName, str(timedelta(seconds=myproxyTimeleft)))
except Exception as ex:
p2 = False
msg2 = "Error trying to create credential:\n %s" % str(
ex)
if (not p1) and (not p2):
from CRABClient.ClientExceptions import ProxyCreationException
raise ProxyCreationException(
"Problems delegating My-proxy.\n%s\n%s" %
(msg1, msg2))
self.logger.debug(
"Result of myproxy credential check:\n %s\n %s",
msg1, msg2)
else:
self.proxyfilename = self.options.proxy
os.environ['X509_USER_PROXY'] = self.options.proxy
self.logger.debug('Skipping proxy creation')
def loadLocalCache(self):
"""
Loads the client cache and set up the server url
"""
self.requestarea, self.requestname = getWorkArea(self.options.projdir)
try:
self.cachedinfo, self.logfile = loadCache(self.requestarea,
self.logger)
port = ':' + self.cachedinfo['Port'] if self.cachedinfo[
'Port'] else ''
self.instance = self.cachedinfo['instance']
self.serverurl = self.cachedinfo['Server'] + port
except CachefileNotFoundException as ex:
if self.cmdconf['requiresLocalCache']:
raise ex
def setCachedProxy(self, proxyOptsSetPlace):
"""
Set the proxy parameters from the cache if not specified otherwise
"""
msgadd = []
if self.cmdconf[
'requiresProxyVOOptions'] and self.options.voGroup is None:
self.voGroup = self.cachedinfo['voGroup']
proxyOptsSetPlace['set_in']['group'] = "cache"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.cmdconf[
'requiresProxyVOOptions'] and self.options.voRole is None:
self.voRole = self.cachedinfo['voRole']
proxyOptsSetPlace['set_in']['role'] = "cache"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as written in the request cache file for this task." % (
" and ".join(msgadd))
self.logger.debug(msg)
def getConfiDict(self):
"""
Load the CRAB cache file (~/.crab3). If it doesn't exist, create one.
"""
crabCacheFileName = self.crabcachepath()
if not os.path.isfile(crabCacheFileName):
msg = "Could not find CRAB cache file %s; creating a new one." % (
crabCacheFileName)
self.logger.debug(msg)
configdict = {'crab_project_directory': ''}
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(configdict, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
return configdict
try:
msg = "Found CRAB cache file %s" % (crabCacheFileName)
self.logger.debug(msg)
with open(crabCacheFileName, 'r') as fd:
configdict = json.load(fd)
except ValueError:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Error loading CRAB cache file."
msg += " Try to do 'rm -rf %s' and run the crab command again." % (
crabCacheFileName)
raise ConfigurationException(msg)
if 'crab_project_directory' not in configdict:
configdict['crab_project_directory'] = configdict.get(
'taskname', '')
if 'taskname' in configdict:
del configdict['taskname']
return configdict
def crabcachepath(self):
if 'CRAB3_CACHE_FILE' in os.environ:
if os.path.isabs(os.environ['CRAB3_CACHE_FILE']):
return os.environ['CRAB3_CACHE_FILE']
else:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Invalid path in environment variable CRAB3_CACHE_FILE: %s" % (
os.environ['CRAB3_CACHE_FILE'])
msg += " Please export a valid full path."
raise EnvironmentException(msg)
else:
return str(os.path.expanduser('~')) + '/.crab3'
def updateCRABCacheFile(self):
"""
Update the CRAB cache file.
So far this file contains only the path of the last used CRAB project directory.
"""
if self.cmdconf['requiresDirOption'] or getattr(
self, 'requestarea', None):
self.crab3dic['crab_project_directory'] = self.requestarea
crabCacheFileName = self.crabcachepath()
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(self.crab3dic, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
def __call__(self):
self.logger.info("This is a 'nothing to do' command")
raise NotImplementedError
def terminate(self, exitcode):
#We do not want to print logfile for each command...
if exitcode < 2000:
if getattr(self.options, 'dump', False) or getattr(
self.options, 'xroot', False):
self.logger.debug("Log file is %s" %
os.path.abspath(self.logfile))
else:
self.logger.info("Log file is %s" %
os.path.abspath(self.logfile))
def setOptions(self):
raise NotImplementedError
def setSuperOptions(self):
try:
#add command related options
self.setOptions()
except NotImplementedError:
pass
self.parser.addCommonOptions(self.cmdconf)
def validateOptions(self):
"""
__validateOptions__
Validate the command line options of the command.
Raise a ConfigurationException in case of error; don't do anything if ok.
"""
if self.cmdconf['requiresDirOption']:
if self.options.projdir is None:
if len(self.args) > 1:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command accepts at most 1 argument (a path to a CRAB project directory), %d given." % (
self.name, len(self.args))
raise ConfigurationException(msg)
elif len(self.args) == 1 and self.args[0]:
self.options.projdir = self.args.pop(0)
elif self.cmdconf['useCache'] and self.crab3dic.get(
'crab_project_directory'):
self.options.projdir = str(
self.crab3dic['crab_project_directory'])
if self.options.projdir is None:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Please indicate the CRAB project directory with --dir=<project-directory>."
ex = MissingOptionException(msg)
ex.missingOption = "task"
raise ex
if not os.path.isdir(self.options.projdir):
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " %s is not a valid CRAB project directory." % (
self.options.projdir)
raise ConfigurationException(msg)
## If an input project directory was given, load the request cache and take the
## server URL from it.
self.loadLocalCache()
## If the command does not take any arguments, but some arguments were passed,
## clear the arguments list and give a warning message saying that the given
## arguments will be ignored.
if not self.cmdconf['acceptsArguments'] and len(self.args):
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command takes no arguments, %d given." % (
self.name, len(self.args))
msg += " Ignoring arguments %s." % (self.args)
self.logger.warning(msg)
self.args = []
def validateLogpathOption(self):
pass
def validateConfigOption(self):
pass