def handleProxy(server, baseurl, voRole, voGroup, days): """ """ proxy = CredentialInteractions('', '', voRole, voGroup, logging) #proxy.myproxyDesiredValidity = days proxy.setMyProxyValidity(int(days) * 24 * 60) logging.info("Checking credentials") proxy.proxyInfo = proxy.createNewVomsProxy(timeLeftThreshold=720) proxyfilename = proxy.proxyInfo['filename'] #get the dn of the agents from the server alldns = server_info('delegatedn', server, proxyfilename, baseurl) for serverdn in alldns['services']: proxy.defaultDelegation['serverDN'] = serverdn proxy.defaultDelegation['myProxySvr'] = 'myproxy.cern.ch' logging.info("Registering user credentials for server %s" % serverdn) proxy.createNewMyProxy(timeleftthreshold=60 * 60 * 24 * RENEW_MYPROXY_THRESHOLD, nokey=True) proxy.createNewMyProxy2(timeleftthreshold=60 * 60 * 24 * RENEW_MYPROXY_THRESHOLD, nokey=True)
def __call__(self): proxy = CredentialInteractions('', '', '', '', self.logger) days = self.options.days proxy.setMyProxyValidity(int(days) * 24 * 60) # minutes timeLeftThreshold = int(days) * 24 * 60 * 60 # seconds self.logger.info("Checking credentials") # need an X509 proxy in order to talk with CRABServer to get list of myproxy authorized retrievers proxy.proxyInfo = proxy.createNewVomsProxy(timeLeftThreshold=720) proxyfilename = proxy.proxyInfo['filename'] serverurl = 'cmsweb.cern.ch' baseurl = '/crabserver/prod/info' alldns = server_info(subresource='delegatedn', serverurl=serverurl, proxyfilename=proxyfilename, baseurl=baseurl) for serverdn in alldns['services']: proxy.defaultDelegation['serverDN'] = serverdn proxy.defaultDelegation['myProxySvr'] = 'myproxy.cern.ch' self.logger.info("Registering user credentials for server %s" % serverdn) (credentialName, myproxyTimeleft) = proxy.createNewMyProxy(timeleftthreshold=timeLeftThreshold, nokey=True) self.logger.info("Credential exists on myproxy: username: %s - validity: %s", credentialName, str(timedelta(seconds=myproxyTimeleft))) (credentialName, myproxyTimeleft) = proxy.createNewMyProxy2(timeleftthreshold=timeLeftThreshold, nokey=True) self.logger.info("Credential exists on myproxy: username: %s - validity: %s", credentialName, str(timedelta(seconds=myproxyTimeleft))) return
class SubCommand(ConfigCommand): ####### These options can be overrhidden if needed ######## ## setting visible = False doesn't allow the sub-command to be called from CLI visible = True proxyfilename = None shortnames = [] usage = "usage: %prog [command-options] [args]" #Default command name is the name of the command class, but it is also possible to set the name attribute in the subclass #if the command name does not correspond to the class name (getlog => get-log) def __init__(self, logger, cmdargs=None, disable_interspersed_args=False): """ Initialize common client parameters """ if not hasattr(self, 'name'): self.name = self.__class__.__name__ ConfigCommand.__init__(self) ## The command logger. self.logger = logger self.logfile = self.logger.logfile localSystem = subprocess.check_output(['uname', '-a']).strip('\n') try: localOS = subprocess.check_output( ['grep', 'PRETTY_NAME', '/etc/os-release'], stderr=subprocess.STDOUT).strip('\n') localOS = localOS.split('=')[1].strip('"') except Exception as ex: # pylint: disable=unused-variable try: localOS = subprocess.check_output(['lsb_release', '-d']).strip('\n') localOS = localOS.split(':')[1].strip() except Exception as ex: # pylint: disable=unused-variable localOS = "Unknown Operating System" self.logger.debug("CRAB Client version: %s", __version__) self.logger.debug("Running on: " + localSystem + " - " + localOS) self.logger.debug("Executing command: '%s'" % str(self.name)) self.proxy = None self.restClass = CRABClient.Emulator.getEmulator('rest') ## Get the command configuration. self.cmdconf = commandsConfiguration.get(self.name) if not self.cmdconf: raise RuntimeError( "Canot find command %s in commandsConfiguration inside ClientMapping. Are you a developer" "trying to add a command without it's correspondant configuration?" % self.name) ## Get the CRAB cache file. self.cachedinfo = None self.crab3dic = self.getConfiDict() ## The options parser. self.parser = CRABCmdOptParser(self.name, self.__doc__, disable_interspersed_args) ## Define the command options. self.setSuperOptions() ## Parse the command options/arguments. cmdargs = cmdargs or [] (self.options, self.args) = self.parser.parse_args(cmdargs) self.transferringIds = None self.dest = None self.validateLogpathOption() ## Validate first the SubCommand options SubCommand.validateOptions(self) ## then the config option for the submit command self.validateConfigOption() ## Get the VO group/role from the command options (if the command requires these ## options). proxyOptsSetPlace = { 'set_in': { 'group': "default", 'role': "default" }, 'for_set_use': "" } msgadd = [] self.voGroup, self.voRole = "", "NULL" if self.cmdconf['requiresProxyVOOptions']: proxyOptsSetPlace['for_set_use'] = "cmdopts" if self.options.voGroup is not None: self.voGroup = self.options.voGroup proxyOptsSetPlace['set_in']['group'] = "cmdopts" msgadd.append("VO group '%s'" % (self.voGroup)) if self.options.voRole is not None: self.voRole = self.options.voRole if self.options.voRole != "" else "NULL" proxyOptsSetPlace['set_in']['role'] = "cmdopts" msgadd.append("VO role '%s'" % (self.voRole)) if msgadd: msg = "Using %s as specified in the crab command options." % ( " and ".join(msgadd)) self.logger.debug(msg) ## Create the object that will do the proxy operations. We don't really care ## what VO role and group and server URL we pass to the constructor, because ## these are not used until we do the proxy delegation to the myproxy server. ## And this happens in handleProxy(), which is called after we load the ## configuration file and retrieve the final values for those parameters. ## handleProxy() takes care of passing those parameters to self.proxy. self.proxy = CredentialInteractions('', '', self.voRole, self.voGroup, self.logger) ## If the user didn't use the --proxy command line option, and if there isn't a ## valid proxy already, we will create a new one with the current VO role and group ## (as commented above, we don't really care what are the VO role and group so ## far). self.proxyCreated = False ## If there is an input configuration file: if hasattr(self.options, 'config') and self.options.config is not None: proxyOptsSetPlace['for_set_use'] = "config" ## Load the configuration file and validate it. self.loadConfig(self.options.config, self.args) ## Create the CRAB project directory. self.requestarea, self.requestname, self.logfile = createWorkArea(self.logger, \ getattr(self.configuration.General, 'workArea', None), \ getattr(self.configuration.General, 'requestName', None)) ## Get the VO group/role from the configuration file. msgadd = [] if hasattr(self.configuration, 'User') and hasattr( self.configuration.User, 'voGroup'): self.voGroup = self.configuration.User.voGroup proxyOptsSetPlace['set_in']['group'] = "config" msgadd.append("VO group '%s'" % (self.voGroup)) if hasattr(self.configuration, 'User') and hasattr( self.configuration.User, 'voRole'): self.voRole = self.configuration.User.voRole if self.configuration.User.voRole != "" else "NULL" proxyOptsSetPlace['set_in']['role'] = "config" msgadd.append("VO role '%s'" % (self.voRole)) if msgadd: msg = "Using %s as specified in the CRAB configuration file." % ( " and ".join(msgadd)) self.logger.debug(msg) ## If the VO group/role was not given in the command options, take it from the request cache. if self.cmdconf['requiresDirOption']: self.setCachedProxy(proxyOptsSetPlace) ## following commented out code should be replaced with something which ## looks for configuration.General.instance, compares with known nicknames, ## extraxts restHost and dbInstance if OK, otherwise gets them from ## configuration.Genarl.restHost and configuration.General.dbInstance ## and sets them in self for all code to use. ## need to review all uses of self.instance and self.serverurl and change or ## check if they can simply be set to dbInstance and restHost repsecitvely ## If the server URL isn't already set, we check the args and then the config. if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']: self.instance, self.serverurl = self.serverInstance() elif not self.cmdconf['requiresREST']: self.instance, self.serverurl = None, None ## Update (or create) the CRAB cache file. self.updateCRABCacheFile() ## At this point we check if there is a valid proxy, and ## eventually create a new one. If the proxy was not created by CRAB, we check that the ## VO role/group in the proxy are the same as specified by the user in the configuration ## file (or in the command line options). If it is not, we ask the user if he wants to ## overwrite the current proxy. If he doesn't want to overwrite it, we don't continue ## and ask him to provide the VO role/group as in the existing proxy. ## Finally, delegate the proxy to myproxy server. self.handleProxy(proxyOptsSetPlace) ## Validate the command options self.validateOptions() ## Logging user command and options used for debuging purpose. self.logger.debug('Command use: %s' % self.name) self.logger.debug('Options use: %s' % cmdargs) if self.cmdconf['requiresREST']: self.checkversion(getUrl(self.instance, resource='info')) self.uri = getUrl(self.instance) self.logger.debug("Instance is %s" % (self.instance)) self.logger.debug("Server base url is %s" % (self.serverurl)) if self.cmdconf['requiresREST']: self.logger.debug("Command url %s" % (self.uri)) def serverInstance(self): """ Deriving the correct instance to use and the server url. Client is allowed to propagate the instance name and corresponding url via crabconfig.py or crab option --instance. The variable passed via crab option will always be used over the variable in crabconfig.py. """ if hasattr(self.options, 'instance') and self.options.instance is not None: if hasattr(self, 'configuration') and hasattr( self.configuration, 'General') and hasattr( self.configuration.General, 'instance' ) and self.configuration.General.instance is not None: msg = "%sWarning%s: CRAB configuration parameter General.instance is overwritten by the command option --instance;" % ( colors.RED, colors.NORMAL) msg += " %s intance will be used." % (self.options.instance) self.logger.info(msg) instance = self.options.instance elif hasattr(self, 'configuration') and hasattr( self.configuration, 'General') and hasattr( self.configuration.General, 'instance' ) and self.configuration.General.instance is not None: instance = self.configuration.General.instance else: instance = getParamDefaultValue('General.instance') # validate instance if not instance in SERVICE_INSTANCES: msg = 'Invalid value "%s" for configuration.General.instance\n' % instance msg += 'valid values are %s ' % SERVICE_INSTANCES.keys() raise ConfigurationException(msg) if instance is not 'other': self.restHost = SERVICE_INSTANCES[instance]['restHost'] self.dbInstance = SERVICE_INSTANCES[instance]['dbInstance'] else: self.restHost = self.configuration.General.restHost self.dbInstance = self.configuration.General.dbInstance # attempt at backward cmpatibility #self.serverurl = self.serverHost #self.instance = self.dbInstance return self.dbInstance, self.restHost def checkversion(self, baseurl=None): compatibleVersions = server_info(subresource='version', serverurl=self.serverurl, proxyfilename=self.proxyfilename, baseurl=baseurl, logger=self.logger) for item in compatibleVersions: if re.match(item, __version__): self.logger.debug("CRABClient version: %s" % (__version__)) break else: msg = "%sWarning%s:" % (colors.RED, colors.NORMAL) msg += " Incompatible CRABClient version %s" % (__version__) msg += "\nServer is saying that compatible versions are: %s" % [ v.replace("\\", "") for v in compatibleVersions ] self.logger.info(msg) def handleProxy(self, proxyOptsSetPlace): """ Init the user proxy, and delegate it if necessary. """ if not self.options.proxy: if self.cmdconf['initializeProxy']: self.proxy.setVOGroupVORole(self.voGroup, self.voRole) self.proxy.proxyInfo = self.proxy.createNewVomsProxy(timeLeftThreshold=720, \ doProxyGroupRoleCheck=self.cmdconf['doProxyGroupRoleCheck'], \ proxyCreatedByCRAB=self.proxyCreated, \ proxyOptsSetPlace=proxyOptsSetPlace) self.proxyfilename = self.proxy.proxyInfo['filename'] if self.cmdconf[ 'requiresREST']: ## If the command doesn't contact the REST, we can't delegate the proxy. baseurl = getUrl(self.instance, resource='info') ## Get the DN of the task workers from the server. all_task_workers_dns = server_info( subresource='delegatedn', serverurl=self.serverurl, proxyfilename=self.proxyfilename, baseurl=baseurl, logger=self.logger) for serverdn in all_task_workers_dns['services']: self.proxy.setServerDN(serverdn) self.proxy.setMyProxyServer('myproxy.cern.ch') self.logger.debug( "Registering user credentials for server %s" % serverdn) try: (credentialName, myproxyTimeleft) = self.proxy.createNewMyProxy( timeleftthreshold=60 * 60 * 24 * RENEW_MYPROXY_THRESHOLD, nokey=True) p1 = True msg1 = "Credential exists on myproxy: username: %s - validity: %s" %\ (credentialName, str(timedelta(seconds=myproxyTimeleft))) except Exception as ex: p1 = False msg1 = "Error trying to create credential:\n %s" % str( ex) try: (credentialName, myproxyTimeleft) = self.proxy.createNewMyProxy2( timeleftthreshold=60 * 60 * 24 * RENEW_MYPROXY_THRESHOLD, nokey=True) p2 = True msg2 = "Credential exists on myproxy: username: %s - validity: %s" %\ (credentialName, str(timedelta(seconds=myproxyTimeleft))) except Exception as ex: p2 = False msg2 = "Error trying to create credential:\n %s" % str( ex) if (not p1) and (not p2): from CRABClient.ClientExceptions import ProxyCreationException raise ProxyCreationException( "Problems delegating My-proxy.\n%s\n%s" % (msg1, msg2)) self.logger.debug( "Result of myproxy credential check:\n %s\n %s", msg1, msg2) else: self.proxyfilename = self.options.proxy os.environ['X509_USER_PROXY'] = self.options.proxy self.logger.debug('Skipping proxy creation') def loadLocalCache(self): """ Loads the client cache and set up the server url """ self.requestarea, self.requestname = getWorkArea(self.options.projdir) try: self.cachedinfo, self.logfile = loadCache(self.requestarea, self.logger) port = ':' + self.cachedinfo['Port'] if self.cachedinfo[ 'Port'] else '' self.instance = self.cachedinfo['instance'] self.serverurl = self.cachedinfo['Server'] + port except CachefileNotFoundException as ex: if self.cmdconf['requiresLocalCache']: raise ex def setCachedProxy(self, proxyOptsSetPlace): """ Set the proxy parameters from the cache if not specified otherwise """ msgadd = [] if self.cmdconf[ 'requiresProxyVOOptions'] and self.options.voGroup is None: self.voGroup = self.cachedinfo['voGroup'] proxyOptsSetPlace['set_in']['group'] = "cache" msgadd.append("VO group '%s'" % (self.voGroup)) if self.cmdconf[ 'requiresProxyVOOptions'] and self.options.voRole is None: self.voRole = self.cachedinfo['voRole'] proxyOptsSetPlace['set_in']['role'] = "cache" msgadd.append("VO role '%s'" % (self.voRole)) if msgadd: msg = "Using %s as written in the request cache file for this task." % ( " and ".join(msgadd)) self.logger.debug(msg) def getConfiDict(self): """ Load the CRAB cache file (~/.crab3). If it doesn't exist, create one. """ crabCacheFileName = self.crabcachepath() if not os.path.isfile(crabCacheFileName): msg = "Could not find CRAB cache file %s; creating a new one." % ( crabCacheFileName) self.logger.debug(msg) configdict = {'crab_project_directory': ''} crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid()) with open(crabCacheFileName_tmp, 'w') as fd: json.dump(configdict, fd) os.rename(crabCacheFileName_tmp, crabCacheFileName) return configdict try: msg = "Found CRAB cache file %s" % (crabCacheFileName) self.logger.debug(msg) with open(crabCacheFileName, 'r') as fd: configdict = json.load(fd) except ValueError: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Error loading CRAB cache file." msg += " Try to do 'rm -rf %s' and run the crab command again." % ( crabCacheFileName) raise ConfigurationException(msg) if 'crab_project_directory' not in configdict: configdict['crab_project_directory'] = configdict.get( 'taskname', '') if 'taskname' in configdict: del configdict['taskname'] return configdict def crabcachepath(self): if 'CRAB3_CACHE_FILE' in os.environ: if os.path.isabs(os.environ['CRAB3_CACHE_FILE']): return os.environ['CRAB3_CACHE_FILE'] else: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Invalid path in environment variable CRAB3_CACHE_FILE: %s" % ( os.environ['CRAB3_CACHE_FILE']) msg += " Please export a valid full path." raise EnvironmentException(msg) else: return str(os.path.expanduser('~')) + '/.crab3' def updateCRABCacheFile(self): """ Update the CRAB cache file. So far this file contains only the path of the last used CRAB project directory. """ if self.cmdconf['requiresDirOption'] or getattr( self, 'requestarea', None): self.crab3dic['crab_project_directory'] = self.requestarea crabCacheFileName = self.crabcachepath() crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid()) with open(crabCacheFileName_tmp, 'w') as fd: json.dump(self.crab3dic, fd) os.rename(crabCacheFileName_tmp, crabCacheFileName) def __call__(self): self.logger.info("This is a 'nothing to do' command") raise NotImplementedError def terminate(self, exitcode): #We do not want to print logfile for each command... if exitcode < 2000: if getattr(self.options, 'dump', False) or getattr( self.options, 'xroot', False): self.logger.debug("Log file is %s" % os.path.abspath(self.logfile)) else: self.logger.info("Log file is %s" % os.path.abspath(self.logfile)) def setOptions(self): raise NotImplementedError def setSuperOptions(self): try: #add command related options self.setOptions() except NotImplementedError: pass self.parser.addCommonOptions(self.cmdconf) def validateOptions(self): """ __validateOptions__ Validate the command line options of the command. Raise a ConfigurationException in case of error; don't do anything if ok. """ if self.cmdconf['requiresDirOption']: if self.options.projdir is None: if len(self.args) > 1: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " 'crab %s' command accepts at most 1 argument (a path to a CRAB project directory), %d given." % ( self.name, len(self.args)) raise ConfigurationException(msg) elif len(self.args) == 1 and self.args[0]: self.options.projdir = self.args.pop(0) elif self.cmdconf['useCache'] and self.crab3dic.get( 'crab_project_directory'): self.options.projdir = str( self.crab3dic['crab_project_directory']) if self.options.projdir is None: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Please indicate the CRAB project directory with --dir=<project-directory>." ex = MissingOptionException(msg) ex.missingOption = "task" raise ex if not os.path.isdir(self.options.projdir): msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " %s is not a valid CRAB project directory." % ( self.options.projdir) raise ConfigurationException(msg) ## If an input project directory was given, load the request cache and take the ## server URL from it. self.loadLocalCache() ## If the command does not take any arguments, but some arguments were passed, ## clear the arguments list and give a warning message saying that the given ## arguments will be ignored. if not self.cmdconf['acceptsArguments'] and len(self.args): msg = "%sWarning%s:" % (colors.RED, colors.NORMAL) msg += " 'crab %s' command takes no arguments, %d given." % ( self.name, len(self.args)) msg += " Ignoring arguments %s." % (self.args) self.logger.warning(msg) self.args = [] def validateLogpathOption(self): pass def validateConfigOption(self): pass