def __init__(self, requirements, check_file=False, create=False): """ Args: requirements (ICredentialRequirement): An object specifying the requirements check_file (bool): Raise an exception if the file does not exist create (bool): Create the credential file """ self.shell = Shell() super(AfsTokenInfo, self).__init__(requirements, check_file, create)
def guess_version(appname): """Guess the default Gaudi application version""" s = Shell() tmp = tempfile.NamedTemporaryFile(suffix='.log') command = 'SetupProject.sh --ask %s' % appname rc,output,m=s.cmd1("echo 'q\n' | %s >& %s; echo" % (command,tmp.name)) output = tmp.read() tmp.close() version = output[output.rfind('[')+1:output.rfind(']')] return version
def available_versions(appname): """Provide a list of the available Gaudi application versions""" s = Shell() tmp = tempfile.NamedTemporaryFile(suffix='.log') command = 'SetupProject.sh --ask %s' % appname rc,output,m=s.cmd1("echo 'q\n' | %s >& %s; echo" % (command,tmp.name)) output = tmp.read() tmp.close() versions = output[output.rfind('(')+1:output.rfind('q[uit]')].split() return versions
def shell(self): """ Construct and store a shell which has the appropriate DIRAC env saved in it """ if self._shell is None: self._shell = Shell() self._shell.env.update(getDiracEnv()) return self._shell
def constructShell(): """ Construct a grid shell based upon either the GLITE_SETUP or GLITE_LOCATION as possibly defined by the user """ values = {} for key in ['X509_CERT_DIR', 'X509_VOMS_DIR']: try: values[key] = os.environ[key] except KeyError: pass config = getConfig('LCG') # 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value # 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup) # 3. else take the default GLITE_SETUP as session value if config.getEffectiveLevel( 'GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ: s = Shell() else: if os.path.exists(config['GLITE_SETUP']): s = Shell(config['GLITE_SETUP']) else: logger.error("Configuration of GLITE for LCG: ") logger.error("File not found: %s" % config['GLITE_SETUP']) return None for key, val in values.items(): s.env[key] = val # check and set env. variables for default LFC setup if 'LFC_HOST' not in s.env: try: s.env['LFC_HOST'] = config['DefaultLFC'] except ConfigError: pass if 'LFC_CONNTIMEOUT' not in s.env: s.env['LFC_CONNTIMEOUT'] = '20' if 'LFC_CONRETRY' not in s.env: s.env['LFC_CONRETRY'] = '0' if 'LFC_CONRETRYINT' not in s.env: s.env['LFC_CONRETRYINT'] = '1' return s
def _getShell(self): fd = tempfile.NamedTemporaryFile() script = '#!/bin/bash\n' gaudirunShell = os.environ["GAUDIRUNENV"] #cmd = '%s' % (gaudirunShell) cmd = 'source %s' % (gaudirunShell) script += '%s \n' % cmd fd.write(script) fd.flush() logger.debug("zhangxm log: run boss env script:\n%s" % script) shell = Shell(setup=fd.name) return shell
def __init__(self): logger.info("crabbackend init") super(CRABBackend, self).__init__() config = Config.getConfig('CMSSW') shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh')) #shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'), # [config['CMSSW_VERSION'], config['CRAB_VERSION']]) self.crab_env = shell.env config = Config.getConfig('CRAB_CFG') self.server_name = config['server_name'] self.apiresource = config['apiresource'] self.userproxy = config['userproxy'] self.asyncdest = config['asyncdest'] logger.info("asyncdest %s" % self.asyncdest)
def constructShell(): """ Construct a grid shell based upon either the GLITE_SETUP or GLITE_LOCATION as possibly defined by the user """ values = {} for key in ['X509_CERT_DIR', 'X509_VOMS_DIR']: try: values[key] = os.environ[key] except KeyError: pass config = getConfig('LCG') # 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value # 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup) # 3. else take the default GLITE_SETUP as session value if config.getEffectiveLevel('GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ: s = Shell() else: if os.path.exists(config['GLITE_SETUP']): s = Shell(config['GLITE_SETUP']) else: logger.error("Configuration of GLITE for LCG: ") logger.error("File not found: %s" % config['GLITE_SETUP']) return None for key, val in values.items(): s.env[key] = val # check and set env. variables for default LFC setup if 'LFC_HOST' not in s.env: try: s.env['LFC_HOST'] = config['DefaultLFC'] except ConfigError: pass if 'LFC_CONNTIMEOUT' not in s.env: s.env['LFC_CONNTIMEOUT'] = '20' if 'LFC_CONRETRY' not in s.env: s.env['LFC_CONRETRY'] = '0' if 'LFC_CONRETRYINT' not in s.env: s.env['LFC_CONRETRYINT'] = '1' return s
class AfsTokenInfo(ICredentialInfo): """ A wrapper around an AFS token For now it is very CERN-specific (or at least only follows the CERN use-case) """ should_warn = False info_pattern = re.compile( r"^User's \(AFS ID \d*\) tokens for (?P<id>\w*@\S*) \[Expires (?P<expires>.*)\]$", re.MULTILINE) __slots__ = ('shell', 'cache', 'initial_requirements') def __init__(self, requirements, check_file=False, create=False): """ Args: requirements (ICredentialRequirement): An object specifying the requirements check_file (bool): Raise an exception if the file does not exist create (bool): Create the credential file """ self.shell = Shell() super(AfsTokenInfo, self).__init__(requirements, check_file, create) @retry_command def create(self): """ Creates a new AFS token Raises: CredentialRenewalError: If the renewal process returns a non-zero value """ command = 'kinit' process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdoutdata, stderrdata = process.communicate( getpass('Kerberos password: '******'AFS token %s created. Valid for %s', self.location, self.time_left()) else: raise CredentialRenewalError(stderrdata) def renew(self): """ Renews the AFS token Raises: CredentialRenewalError: If the renewal process returns a non-zero value """ status, output, message = self.shell.cmd1('kinit -R') if status != 0: logger.debug('kinit -R failed, creating as new') self.create() def destroy(self): """ This removes the kerberos token from disk """ self.shell.cmd1('unlog') if self.location: os.remove(self.location) @property @cache def info(self): """ This returns a summary of the token infor on disk """ status, output, message = self.shell.cmd1('tokens') return output @cache def expiry_time(self): """ This calculates the number of seconds left for the kerberos token on disk """ info = self.info matches = re.finditer(AfsTokenInfo.info_pattern, info) if not matches: return datetime.timedelta() all_tokens = [match.group('expires') for match in matches] if len(all_tokens) > 1: if AfsTokenInfo.should_warn: logger.warning( "Found multiple AFS tokens, taking soonest expiring one for safety" ) logger.warning("Tokens found for: %s".format(" ".join( [match.group('id') for match in matches]))) AfsTokenInfo.should_warn = False soonest = None for expires in all_tokens: expires = datetime.datetime.strptime(expires, '%b %d %H:%M') now = datetime.datetime.now() expires = expires.replace(year=now.year) # If the expiration date is in the past then assume it should be in the future if expires < now: expires = expires.replace(year=now.year + 1) if not soonest or expires < soonest: soonest = expires return soonest def default_location(self): """ This returns the default location of a kerberos token on disk as determined from the uid """ krb_env_var = os.getenv('KRB5CCNAME', '') if krb_env_var.startswith('FILE:'): krb_env_var = krb_env_var[5:] # If file already exists if os.path.exists(krb_env_var): return krb_env_var # Lets try to find it if we can't get it from the env default_name_prefix = '/tmp/krb5cc_{uid}'.format(uid=os.getuid()) matches = glob(default_name_prefix + '*') # Check for partial matches on disk if len(matches) == 1: # If one then use it filename_guess = matches[0] else: # Otherwise use the default filename_guess = default_name_prefix return filename_guess
def getShell(force=False): """ Utility function for getting Grid Shell. Caller should take responsibility of credential checking if proxy is needed. Arguments: force (bool): False: if the shell already exists in local cache return the previous created instance ; True: recreate the shell and if not None update the cache """ global shell_cache logger = getLogger() if shell_cache is not None and not force: return shell_cache values = {} for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']: try: values[key] = os.environ[key] except KeyError: pass try: config = getConfig('LCG') except: logger.warning( '[LCG] configuration section not found. Cannot set up a proper grid shell.' ) return None s = None # 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value # 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup) # 3. else take the default GLITE_SETUP as session value if config.getEffectiveLevel( 'GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ: s = Shell() else: if os.path.exists(config['GLITE_SETUP']): s = Shell(config['GLITE_SETUP']) else: if config['GLITE_ENABLE']: logger.warning("Configuration of GLITE for LCG: ") logger.warning("File not found: %s" % config['GLITE_SETUP']) if s: for key, val in values.items(): s.env[key] = val # check and set env. variables for default LFC setup if 'LFC_HOST' not in s.env: try: s.env['LFC_HOST'] = config['DefaultLFC'] except ConfigError: pass if 'LFC_CONNTIMEOUT' not in s.env: s.env['LFC_CONNTIMEOUT'] = '20' if 'LFC_CONRETRY' not in s.env: s.env['LFC_CONRETRY'] = '0' if 'LFC_CONRETRYINT' not in s.env: s.env['LFC_CONRETRYINT'] = '1' shell_cache = s return s
class AfsTokenInfo(ICredentialInfo): """ A wrapper around an AFS token For now it is very CERN-specific (or at least only follows the CERN use-case) """ should_warn = False info_pattern = re.compile(r"^User's \(AFS ID \d*\) tokens for (?P<id>\w*@\S*) \[Expires (?P<expires>.*)\]$", re.MULTILINE) __slots__=('shell', 'cache', 'initial_requirements') def __init__(self, requirements, check_file=False, create=False): """ Args: requirements (ICredentialRequirement): An object specifying the requirements check_file (bool): Raise an exception if the file does not exist create (bool): Create the credential file """ self.shell = Shell() super(AfsTokenInfo, self).__init__(requirements, check_file, create) @retry_command def create(self): """ Creates a new AFS token Raises: CredentialRenewalError: If the renewal process returns a non-zero value """ command = 'kinit' process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdoutdata, stderrdata = process.communicate(getpass('Kerberos password: '******'AFS token %s created. Valid for %s', self.location, self.time_left()) else: raise CredentialRenewalError(stderrdata) def renew(self): """ Renews the AFS token Raises: CredentialRenewalError: If the renewal process returns a non-zero value """ status, output, message = self.shell.cmd1('kinit -R') if status != 0: logger.debug('kinit -R failed, creating as new') self.create() def destroy(self): """ This removes the kerberos token from disk """ self.shell.cmd1('unlog') if self.location: os.remove(self.location) @property @cache def info(self): """ This returns a summary of the token infor on disk """ status, output, message = self.shell.cmd1('tokens') return output @cache def expiry_time(self): """ This calculates the number of seconds left for the kerberos token on disk """ info = self.info matches = re.finditer(AfsTokenInfo.info_pattern, info) if not matches: return datetime.timedelta() all_tokens = [match.group('expires') for match in matches] if len(all_tokens) > 1: if AfsTokenInfo.should_warn: logger.warning("Found multiple AFS tokens, taking soonest expiring one for safety") logger.warning("Tokens found for: %s".format(" ".join([match.group('id') for match in matches]))) AfsTokenInfo.should_warn = False soonest = None for expires in all_tokens: expires = datetime.datetime.strptime(expires, '%b %d %H:%M') now = datetime.datetime.now() expires = expires.replace(year=now.year) # If the expiration date is in the past then assume it should be in the future if expires < now: expires = expires.replace(year=now.year+1) if not soonest or expires < soonest: soonest = expires return soonest def default_location(self): """ This returns the default location of a kerberos token on disk as determined from the uid """ krb_env_var = os.getenv('KRB5CCNAME', '') if krb_env_var.startswith('FILE:'): krb_env_var = krb_env_var[5:] # If file already exists if os.path.exists(krb_env_var): return krb_env_var # Lets try to find it if we can't get it from the env default_name_prefix = '/tmp/krb5cc_{uid}'.format(uid=os.getuid()) matches = glob(default_name_prefix+'*') # Check for partial matches on disk if len(matches) == 1: # If one then use it filename_guess = matches[0] else: # Otherwise use the default filename_guess = default_name_prefix return filename_guess
def getShell(middleware='EDG', force=False): """ Utility function for getting Grid Shell. Caller should take responsiblity of credential checking if proxy is needed. Argumennts: middleware - grid m/w used force - False : if the shell already exists in local cache return the previous created instance True : recreate the shell and if not None update the cache """ logger = getLogger() if not middleware: logger.debug('No middleware specified, assuming default EDG') middleware = 'EDG' if middleware in _allShells.keys() and not force: return _allShells[middleware] values = {} for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']: try: values[key] = os.environ[key] except KeyError: pass configname = "" if middleware == 'EDG' or middleware == 'GLITE': configname = 'LCG' else: configname = middleware config = None try: config = getConfig(configname) except: logger.warning( '[%s] configuration section not found. Cannot set up a proper grid shell.' % configname) return None s = None key = '%s_SETUP' % middleware # 1. check if the *_SETUP is changed by user -> take the user's value as session value # 2. else check if *_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup) # 3. else take the default *_SETUP as session value MIDDLEWARE_LOCATION = '%s_LOCATION' % middleware if config.getEffectiveLevel(key) == 2 and MIDDLEWARE_LOCATION in os.environ: s = Shell() else: if os.path.exists(config[key]): # FIXME: Hardcoded rule for ARC middleware setup (pass explicitly # the $ARC_LOCATION as argument), this is hardcoded to maintain # backwards compatibility (and avoid any side effects) for EDG and # GLITE setup scripts which did not take any arguments if key.startswith('ARC') and MIDDLEWARE_LOCATION in os.environ: s = Shell( config[key], setup_args=[os.environ[MIDDLEWARE_LOCATION]]) else: s = Shell(config[key]) else: logger.warning("Configuration of %s for %s: " % (middleware, configname)) logger.warning("File not found: %s" % config[key]) if s: for key, val in values.items(): s.env[key] = val # check and set env. variables for default LFC setup if 'LFC_HOST' not in s.env: try: s.env['LFC_HOST'] = config['DefaultLFC'] except ConfigError: pass if 'LFC_CONNTIMEOUT' not in s.env: s.env['LFC_CONNTIMEOUT'] = '20' if 'LFC_CONRETRY' not in s.env: s.env['LFC_CONRETRY'] = '0' if 'LFC_CONRETRYINT' not in s.env: s.env['LFC_CONRETRYINT'] = '1' _allShells[middleware] = s return s
############################################################################### # A simple ATLAS dataset # # ATLAS/ARDA import os, re from Ganga.GPIDev.Lib.Dataset import Dataset from Ganga.GPIDev.Schema import * from Ganga.Utility.logging import getLogger from Ganga.Utility.Shell import Shell logger = getLogger() shell = Shell() class ATLASCastorDataset(Dataset): '''ATLAS data as list of files on CASTOR''' _schema = Schema(Version(1,0), { 'location' : SimpleItem(defvalue = '', doc = 'A directory on castor that contains the datasets in directories'), 'pattern' : SimpleItem(defvalue = '', doc = 'A regexp filter to select the correct files'), 'dataset' : SimpleItem(defvalue = '', doc = 'The name of the selected dataset'), 'names' : SimpleItem(defvalue = [], typelist=['str'], sequence = 1, doc = 'The selected file names') })
class ICredential( GangaObject ): """ Interface class for working with credentials """ _schema = Schema( Version( 1, 0 ), { "maxTry" : SimpleItem( defvalue = 1, doc = "Number of password attempts allowed when creating credential" ), "minValidity" : SimpleItem( defvalue = "00:15", doc = "Default minimum validity" ), "validityAtCreation" : SimpleItem( defvalue = "24:00", doc = "Default credential validity at creation" ), "command" : ComponentItem( category = "credential_commands", defvalue = "ICommandSet", doc = "Set of commands to be used for credential-related operations" ) } ) _category = "credentials" _name = "ICredential" _hidden = 1 _exportmethods = [ "create", "destroy", "isAvailable", "isValid",\ "location", "renew", "timeleft" ] def __init__( self ): super( ICredential, self ).__init__() self.shell = Shell() self.inputPW_Widget = None return def create( self, validity = "", maxTry = 0, minValidity = "", \ check = False ): """ Create credential. Arguments other than self: validity - Validity with which credential should be created, specified as string of format "hh:mm" [ Defaults to value of self.validityAtCreation ] maxTry - Number of password attempts allowed [ Defaults to value of self.maxTry ] minValidity - Minimum validity in case checking of pre-existing credential is performed, specified as strong of format "hh:mm" [ Defaults to value of self.minValidity ] check - Flag to request checking of pre-existing credential; if flag is set to true, then new credential is created only if the validity of any pre-existing credential is less than the value of minValidity [ Default: False ] Note: create is the same as renew, except for the default value of check Return value: True if credential is created successfully, and False otherwise. """ global logTimeStamp dummy = False if not self.command.init: dummy = True if self.command.init_parameters.has_key( "valid" ): if not self.command.init_parameters[ "valid" ]: dummy = True if dummy: logger.warning( "Dummy CommandSet used - no credential created" ) return False if not maxTry: maxTry = self.maxTry if not minValidity: minValidity = self.minValidity if not validity: validity = self.validityAtCreation validityInSeconds = self.timeInSeconds( validity ) if not validityInSeconds: logger.warning( "Problems with requested validity: %s" \ % str( validity ) ) return False if check and self.isValid( minValidity ): return True ntry = 0 while ntry < maxTry: ntry = ntry + 1 # Test if GUI widget is to be used. if self.inputPW_Widget: # Since self.inputPW_Widget is called, current arguments are # ignored since renew() and create() in GUI mode will not be # called with any arguments. if self.inputPW_Widget.ask( self._proxyObject ): logger.debug( "Proceeding to retrieve password from inputPW_Widget." ) __pw = self.inputPW_Widget.getPassword( self._proxyObject ) if not __pw: logger.warning( "Password/passphrase expected!" ) return False try: tFile = tempfile.NamedTemporaryFile() tFile.write( __pw ) tFile.flush() except: del __pw logger.warning( "Could not create secure temporary file for password!" ) return False del __pw else: # Current credential modification denied for various reasons. # see GangaGUI.customDialogs.ask() method for more details. return False # self.inputPW_Widget.ask() may have modified parameters. # Calling buildOpts() to take them into account. self.buildOpts( self.command.init, False ) # Create initialisation list with the 'pipe' parameter initList = [ self.command.init, self.command.init_parameters[ "pipe" ] ] # Append option value pairs for optName, optVal in self.command.currentOpts.iteritems(): initList.append( "%s %s" % ( optName, optVal ) ) status = self.shell.system( "cat %s|%s" % ( tFile.name, " ".join( initList ) ) ) tFile.close() # self.inputPW_Widget dialog postprocessing. # E.g. disable autorenew mechanism if status != 0. self.inputPW_Widget.renewalStatus( self._proxyObject, status ) if status == 0: logger.info( "%s creation/renewal successful." % self._name ) return True else: logger.warning( "%s creation/renewal failed [%s]." % ( self._name, status ) ) return False else: # Non-GUI credential renewal/creation # Check if renewal is from main process (i.e. by bootstrap or user) if threading.currentThread().getName() == 'MainThread': if self.command.init_parameters.has_key( "valid" ): self.command.currentOpts\ [ self.command.init_parameters[ 'valid' ] ] = validity initList = [ self.command.init ] # Append option value pairs for optName, optVal in self.command.currentOpts.iteritems(): initList.append( "%s %s" % ( optName, optVal ) ) status = self.shell.system( " ".join( initList ) ) if status == 0: logger.info( "%s creation/renewal successful." % self._name ) return True else: logger.warning( "%s creation/renewal failed [%s]." % ( self._name, status ) ) else: # create initiated from worker thread from monitoring component. currTime = time.time() if currTime - logTimeStamp >= logRepeatDuration: logTimeStamp = currTime # Check validity but print logging messages this time self.isValid( "", True ) _credentialObject = self._name[ 0 ].lower() + self._name[ 1: ] logger.warning( \ "Renew by typing '%s.renew()' at the prompt." % \ ( _credentialObject ) ) #notify the Core that the credential is not valid _validity = self.timeInSeconds(self.timeleft()) _minValidity = self.timeInSeconds(minValidity)/2 if _validity <= max(120,_minValidity): Coordinator.notifyInvalidCredential(self) return True logger.warning( "%s creation/renewal attempts exceeded %s tries!" % ( self._name, maxTry ) ) return False def destroy( self, allowed_exit = [ 0 ] ): """ Destroy credential Argument other than self: allowed_exit - List of exit codes accepted without error when issuing system command for destroying credential Return value: False if command for destroying credential is undefined, or True otherwise """ if not self.command.destroy: logger.warning( "Dummy CommandSet used - no credential created" ) return False destroyList = [ self.command.destroy ] for optName, optVal in self.command.destroyOpts.iteritems(): destroyList.append( "%s %s" % ( optName, optVal ) ) Coordinator.notifyInvalidCredential(self) status, output, message = \ self.shell.cmd1( " ".join( destroyList ), allowed_exit ) proxyPath = self.location() if proxyPath: os.remove( proxyPath ) return True def isAvailable( self ): """ Check whether credential is available with system/configuration used No arguments other than self Return value: True if credential is available, false otherwise """ logger.warning( "Dummy method used - this always returns True" ) return True def isValid( self, validity = "", log = False, force_check = False ): """ Check validity Arguments other than self: validity - Minimum time for which credential should be valid, specified as string of format "hh:mm" [ Defaults to valud of self.minValidity ] log - Print logger messages if credential not valid force_check - Force credential check, rather than relying on cache Return value: True if credential is valid for required time, False otherwise. """ valid = True if not validity: validity = self.minValidity validityInSeconds = self.timeInSeconds( validity ) timeleft = self.timeleft( force_check = force_check ) if not timeleft: valid = False else: timeleftInSeconds = self.timeInSeconds( timeleft ) if timeleftInSeconds <= validityInSeconds: valid = False if not valid and log: _tl = self.timeleft( force_check = force_check ) if _tl == "-1" or _tl == "0:00:00": _expiryStatement = "has expired!" else: _expiryStatement = "will expire in %s!" % _tl itemList = [] text = self._name[ 0 ] for i in range( len( self._name ) - 1 ): character = self._name[ i + 1 ] if character.isupper(): itemList.append( text ) text = character.lower() else: text = "".join( [ text, character ] ) itemList.append( text ) _credentialName = " ".join( itemList ) logger.warning( "%s %s" % \ ( _credentialName, _expiryStatement ) ) return valid def location( self ): """ Determine credential location No arguments other than self Return value: Path to credential if found, or empty string otherwise """ return "" def renew( self, validity = "", maxTry = 0, minValidity = "", \ check = True ): """ Renew credential. Arguments other than self: validity - Validity with which credential should be created, specified as string of format "hh:mm" [ Defaults to value of self.validityAtCreation ] maxTry - Number of password attempts allowed [ Defaults to value of self.maxTry ] minValidity - Minimum validity in case checking of pre-existing credential is performed, specified as strong of format "hh:mm" [ Defaults to value of self.minValidity ] check - Flag to request checking of pre-existing credential; if flag is set to true, then new credential is created only if the validity of any pre-existing credential is less than the value of minValidity [ Default: True ] Note: renew is the same as create, except for the default value of check Return value: True if new credential is created successfully, and False otherwise. """ status = self.create( validity, maxTry, minValidity, check ) return status def timeInSeconds( self, timeString = "" ): """ Convert time string to time in seconds Arguments other than self: timeString - Time specified as string of format "hh:mm:ss" Return value: Time in seconds (integer) """ totalTime = 0 timeList = timeString.split( ":" ) if len( timeList ) >= 1: totalTime = totalTime + int( timeList[ 0 ] ) * 60 * 60 if len( timeList ) >= 2: totalTime = totalTime + int( timeList[ 1 ] ) * 60 if len( timeList ) >= 3: totalTime = totalTime + int (timeList[ 2 ] ) return totalTime def timeleft( self, units = "hh:mm:ss", force_check = False ): """ Check time for which credential is valid. Arguments other than self: units - String specifying units in which time is returned force_check - Force credential check, rather than relying on cache Allowed values for units are: "hours" - time returned as in hours "minutes" - time returned in minutes "seconds" - time returned in seconds "hh:mm:ss" [default] - time returned as hours, minutes seconds Return value: Credential validity as string giving time in requested units, or empty string if command for querying credential validity is unavailable """ timeRemaining = self.timeleftInHMS( force_check = force_check ) if timeRemaining not in [ "", "-1" ]: if units in [ "hours", "minutes", "seconds" ]: timeleftInSeconds = self.timeInSeconds( timeRemaining ) if "seconds" == units: timeRemaining = "%.2f" % ( timeleftInSeconds ) elif "minutes" == units: timeRemaining = "%.2f" % ( timeleftInSeconds / 60. ) elif "hours" == units: timeRemaining = "%.2f" % ( timeleftInSeconds / ( 60. * 60. ) ) return timeRemaining def timeleftInHMS( self, force_check = False ): """ Determine remaining validity of credential in hours, minutes and seconds Argument other than self: force_check - Force credential check, rather than relying on cache Return value: String giving credential validity, or empty string if command for querying credential validity is unavailable """ logger.warning( "Dummy method used - no information returned" ) return ""
def getShell(force=False): """ Utility function for getting Grid Shell. Caller should take responsibility of credential checking if proxy is needed. Arguments: force (bool): False: if the shell already exists in local cache return the previous created instance ; True: recreate the shell and if not None update the cache """ global shell_cache logger = getLogger() if shell_cache is not None and not force: return shell_cache values = {} for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']: try: values[key] = os.environ[key] except KeyError: pass try: config = getConfig('LCG') except: logger.warning('[LCG] configuration section not found. Cannot set up a proper grid shell.') return None s = None # 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value # 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup) # 3. else take the default GLITE_SETUP as session value if config.getEffectiveLevel('GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ: s = Shell() else: if os.path.exists(config['GLITE_SETUP']): s = Shell(config['GLITE_SETUP']) else: if config['GLITE_ENABLE']: logger.warning("Configuration of GLITE for LCG: ") logger.warning("File not found: %s" % config['GLITE_SETUP']) if s: for key, val in values.items(): s.env[key] = val # check and set env. variables for default LFC setup if 'LFC_HOST' not in s.env: try: s.env['LFC_HOST'] = config['DefaultLFC'] except ConfigError: pass if 'LFC_CONNTIMEOUT' not in s.env: s.env['LFC_CONNTIMEOUT'] = '20' if 'LFC_CONRETRY' not in s.env: s.env['LFC_CONRETRY'] = '0' if 'LFC_CONRETRYINT' not in s.env: s.env['LFC_CONRETRYINT'] = '1' shell_cache = s return s
def __init__(self): super(ICredential, self).__init__() self.shell = Shell() self.inputPW_Widget = None return
class ICredential(GangaObject): """ Interface class for working with credentials """ _schema = Schema( Version(1, 0), { "maxTry": SimpleItem( defvalue=1, typelist=[int], doc= "Number of password attempts allowed when creating credential" ), "minValidity": SimpleItem(defvalue="00:15", typelist=[str], doc="Default minimum validity"), "validityAtCreation": SimpleItem(defvalue="24:00", typelist=[str], doc="Default credential validity at creation"), "command": ComponentItem( category="credential_commands", defvalue="ICommandSet", doc= "Set of commands to be used for credential-related operations") }) _category = "credentials" _name = "ICredential" _hidden = 1 _exportmethods = [ "create", "destroy", "isAvailable", "isValid", "location", "renew", "timeleft" ] def __init__(self): super(ICredential, self).__init__() self.shell = Shell() self.inputPW_Widget = None return def create(self, validity="", maxTry=0, minValidity="", check=False): """ Create credential. Arguments other than self: validity - Validity with which credential should be created, specified as string of format "hh:mm" [ Defaults to value of self.validityAtCreation ] maxTry - Number of password attempts allowed [ Defaults to value of self.maxTry ] minValidity - Minimum validity in case checking of pre-existing credential is performed, specified as strong of format "hh:mm" [ Defaults to value of self.minValidity ] check - Flag to request checking of pre-existing credential; if flag is set to true, then new credential is created only if the validity of any pre-existing credential is less than the value of minValidity [ Default: False ] Note: create is the same as renew, except for the default value of check Return value: True if credential is created successfully, and False otherwise. """ global logTimeStamp dummy = False if not self.command.init: dummy = True if "valid" in self.command.init_parameters: if not self.command.init_parameters["valid"]: dummy = True if dummy: logger.warning("Dummy CommandSet used - no credential created") return False if not maxTry: maxTry = self.maxTry if not minValidity: minValidity = self.minValidity if not validity: validity = self.validityAtCreation validityInSeconds = self.timeInSeconds(validity) if not validityInSeconds: logger.warning("Problems with requested validity: %s" % str(validity)) return False if check and self.isValid(minValidity): return True ntry = 0 while ntry < maxTry: ntry = ntry + 1 # Test if GUI widget is to be used. if self.inputPW_Widget: # Since self.inputPW_Widget is called, current arguments are # ignored since renew() and create() in GUI mode will not be # called with any arguments. #proxy_obj = self._proxyObject ## This is removed to get rid of ref to _proxyObject proxy_obj = self if self.inputPW_Widget.ask(proxy_obj): logger.dg( "Proceeding to retrieve password from inputPW_Widget.") __pw = self.inputPW_Widget.getPassword(proxy_obj) if not __pw: logger.warning("Password/passphrase expected!") return False try: tFile = tempfile.NamedTemporaryFile() tFile.write(__pw) tFile.flush() except: del __pw logger.warning( "Could not create secure temporary file for password!" ) return False del __pw else: # Current credential modification denied for various reasons. # see GangaGUI.customDialogs.ask() method for more details. return False # self.inputPW_Widget.ask() may have modified parameters. # Calling buildOpts() to take them into account. self.buildOpts(self.command.init, False) # Create initialisation list with the 'pipe' parameter initList = [ self.command.init, self.command.init_parameters["pipe"] ] # Append option value pairs for optName, optVal in self.command.currentOpts.iteritems(): initList.append("%s %s" % (optName, optVal)) status = self.shell.system("cat %s|%s" % (tFile.name, " ".join(initList))) tFile.close() # self.inputPW_Widget dialog postprocessing. # E.g. disable autorenew mechanism if status != 0. self.inputPW_Widget.renewalStatus(proxy_obj, status) if status == 0: logger.info("%s creation/renewal successful." % self._name) return True else: logger.warning("%s creation/renewal failed [%s]." % (self._name, status)) return False else: # Non-GUI credential renewal/creation # Check if renewal is from main process (i.e. by bootstrap or # user) if threading.currentThread().getName() == 'MainThread' or\ threading.currentThread().getName().startswith('GANGA_Update_Thread_Ganga_Worker_'): if "valid" in self.command.init_parameters: self.command.currentOpts[ self.command.init_parameters['valid']] = validity initList = [self.command.init] # Append option value pairs for optName, optVal in self.command.currentOpts.iteritems( ): initList.append("%s %s" % (optName, optVal)) status = self.shell.system(" ".join(initList)) if status == 0: logger.info("%s creation/renewal successful." % self._name) return True else: logger.warning("%s creation/renewal failed [%s]." % (self._name, status)) # create initiated from worker thread from monitoring # component. else: currTime = time.time() if currTime - logTimeStamp >= logRepeatDuration: logTimeStamp = currTime # Check validity but print logging messages this time self.isValid("", True) _credentialObject = self._name[0].lower( ) + self._name[1:] logger.warning( "Renew by typing '%s.renew()' at the prompt." % (_credentialObject)) # notify the Core that the credential is not valid _validity = self.timeInSeconds(self.timeleft()) _minValidity = self.timeInSeconds(minValidity) / 2. if _validity <= max(120, _minValidity): Coordinator.notifyInvalidCredential(self) return True logger.warning("%s creation/renewal attempts exceeded %s tries!" % (self._name, maxTry)) return False def destroy(self, allowed_exit=[0]): """ Destroy credential Argument other than self: allowed_exit - List of exit codes accepted without error when issuing system command for destroying credential Return value: False if command for destroying credential is undefined, or True otherwise """ if not self.command.destroy: logger.warning("Dummy CommandSet used - no credential created") return False destroyList = [self.command.destroy] for optName, optVal in self.command.destroyOpts.iteritems(): destroyList.append("%s %s" % (optName, optVal)) Coordinator.notifyInvalidCredential(self) status, output, message = \ self.shell.cmd1(" ".join(destroyList), allowed_exit) proxyPath = self.location() if proxyPath: os.remove(proxyPath) return True def isAvailable(self): """ Check whether credential is available with system/configuration used No arguments other than self Return value: True if credential is available, false otherwise """ logger.warning("Dummy method used - this always returns True") return True def isValid(self, validity="", log=False, force_check=False): """ Check validity Arguments other than self: validity - Minimum time for which credential should be valid, specified as string of format "hh:mm" [ Defaults to valud of self.minValidity ] log - Print logger messages if credential not valid force_check - Force credential check, rather than relying on cache Return value: True if credential is valid for required time, False otherwise. """ valid = True if not validity or validity is None: validity = self.minValidity validityInSeconds = self.timeInSeconds(validity) timeleft = self.timeleft(force_check=force_check) if not timeleft: valid = False else: timeleftInSeconds = self.timeInSeconds(timeleft) if timeleftInSeconds <= validityInSeconds: valid = False if not valid and log: _tl = self.timeleft(force_check=force_check) if _tl == "-1" or _tl == "0:00:00": _expiryStatement = "has expired!" else: _expiryStatement = "will expire in %s!" % _tl itemList = [] text = self._name[0] for i in range(len(self._name) - 1): character = self._name[i + 1] if character.isupper(): itemList.append(text) text = character.lower() else: text = "".join([text, character]) itemList.append(text) _credentialName = " ".join(itemList) logger.warning("%s %s" % (_credentialName, _expiryStatement)) return valid def location(self): """ Determine credential location No arguments other than self Return value: Path to credential if found, or empty string otherwise """ return "" def renew(self, validity="", maxTry=0, minValidity="", check=True): """ Renew credential. Arguments other than self: validity - Validity with which credential should be created, specified as string of format "hh:mm" [ Defaults to value of self.validityAtCreation ] maxTry - Number of password attempts allowed [ Defaults to value of self.maxTry ] minValidity - Minimum validity in case checking of pre-existing credential is performed, specified as strong of format "hh:mm" [ Defaults to value of self.minValidity ] check - Flag to request checking of pre-existing credential; if flag is set to true, then new credential is created only if the validity of any pre-existing credential is less than the value of minValidity [ Default: True ] Note: renew is the same as create, except for the default value of check Return value: True if new credential is created successfully, and False otherwise. """ status = self.create(validity, maxTry, minValidity, check) return status def timeInSeconds(self, timeString=""): """ Convert time string to time in seconds Arguments other than self: timeString - Time specified as string of format "hh:mm:ss" Return value: Time in seconds (integer) """ totalTime = 0 timeList = timeString.split(":") if len(timeList) >= 1: totalTime = totalTime + int(timeList[0]) * 60 * 60 if len(timeList) >= 2: totalTime = totalTime + int(timeList[1]) * 60 if len(timeList) >= 3: totalTime = totalTime + int(timeList[2]) return totalTime def timeleft(self, units="hh:mm:ss", force_check=False): """ Check time for which credential is valid. Arguments other than self: units - String specifying units in which time is returned force_check - Force credential check, rather than relying on cache Allowed values for units are: "hours" - time returned as in hours "minutes" - time returned in minutes "seconds" - time returned in seconds "hh:mm:ss" [default] - time returned as hours, minutes seconds Return value: Credential validity as string giving time in requested units, or empty string if command for querying credential validity is unavailable """ timeRemaining = self.timeleftInHMS(force_check=force_check) if timeRemaining not in ["", "-1"]: if units in ["hours", "minutes", "seconds"]: timeleftInSeconds = self.timeInSeconds(timeRemaining) if "seconds" == units: timeRemaining = "%.2f" % (timeleftInSeconds) elif "minutes" == units: timeRemaining = "%.2f" % (timeleftInSeconds / 60.) elif "hours" == units: timeRemaining = "%.2f" % (timeleftInSeconds / (60. * 60.)) return timeRemaining def timeleftInHMS(self, force_check=False): """ Determine remaining validity of credential in hours, minutes and seconds Argument other than self: force_check - Force credential check, rather than relying on cache Return value: String giving credential validity, or empty string if command for querying credential validity is unavailable """ logger.warning("Dummy method used - no information returned") return ""
def __init__(self): super(CRABBackend, self).__init__() config = Config.getConfig('CMSSW') shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'), [config['CMSSW_VERSION'], config['CRAB_VERSION']]) self.crab_env = shell.env
def __init__( self ): super( ICredential, self ).__init__() self.shell = Shell() self.inputPW_Widget = None return
#script+= '%s %s %s %s\n' % (use,setupProjectOptions,appname,app.version) #script+= '%s %s\n' % (use,setupProjectOptions) command=command.replace('###CMT###',cmtcmd + ' ' + cmtoption) logger.debug('zhangxm log: Will execute the command: '+command) script += command + '\n' logger.debug('zhangxm log: The full script for execution:\n'+script) # write file try: tmppath = tempfile.mkdtemp() fn = os.path.join(tmppath, 'cmtcommand_script') file1 = open(fn, 'w') except Exception, e: logger.error("Can not create temporary file %s", fn) return else: try: file1.write(script) finally: file1.close() # make file executable os.chmod(fn, 0777) shell = Shell() rc=shell.system(fn) if os.path.exists(tmppath): shutil.rmtree(tmppath) return True #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
def CMTscript(app, command=''): """Function to execute a cmt command for a specific job. Returns the unix exit code. Arguments: app - The Gaudi application object to take information from command - String [default ''] The cmt command to execute. """ cmtcmd = 'cmt' warnings.filterwarnings('ignore', 'tempnam', RuntimeWarning) use = '' if app.masterpackage: (pack, alg, ver) = parse_master_package(app.masterpackage) use = '--use "%s %s %s"' % (alg, ver, pack) ura = app.user_release_area if not ura: expanded = os.path.expandvars("$User_release_area") if expanded == "$User_release_area": ura = "" else: ura = expanded.split(os.pathsep)[0] cmtoption = '' # generate shell script script = '#!/bin/sh\n' script += 'unalias -a\n' script += '. `which LbLogin.sh` -c ' + str(app.platform) + '\n' script += 'export User_release_area=' + str(ura) + '\n' script += 'unset CMTPROJECTPATH\n' script += '. setenvProject.sh ' setupProjectOptions = '' if app.setupProjectOptions: setupProjectOptions = app.setupProjectOptions script += '%s %s %s %s\n' % (use, setupProjectOptions, app.appname, app.version) command = command.replace('###CMT###', cmtcmd + ' ' + cmtoption) logger.debug('Will execute the command: ' + command) script += command + '\n' logger.debug('The full script for execution:\n' + script) # write file try: tmppath = tempfile.mkdtemp() fn = os.path.join(tmppath, 'cmtcommand_script') file1 = open(fn, 'w') except Exception as e: logger.error("Can not create temporary file %s", fn) return else: try: file1.write(script) finally: file1.close() # make file executable os.chmod(fn, 0o777) shell = Shell() rc = shell.system(fn) if os.path.exists(tmppath): shutil.rmtree(tmppath) return rc