def __init__(self, requirements, check_file=False, create=False):
"""
Args:
requirements (ICredentialRequirement): An object specifying the requirements
check_file (bool): Raise an exception if the file does not exist
create (bool): Create the credential file
"""
self.shell = Shell()
super(AfsTokenInfo, self).__init__(requirements, check_file, create)
def guess_version(appname):
"""Guess the default Gaudi application version"""
s = Shell()
tmp = tempfile.NamedTemporaryFile(suffix='.log')
command = 'SetupProject.sh --ask %s' % appname
rc,output,m=s.cmd1("echo 'q\n' | %s >& %s; echo" % (command,tmp.name))
output = tmp.read()
tmp.close()
version = output[output.rfind('[')+1:output.rfind(']')]
return version
def available_versions(appname):
"""Provide a list of the available Gaudi application versions"""
s = Shell()
tmp = tempfile.NamedTemporaryFile(suffix='.log')
command = 'SetupProject.sh --ask %s' % appname
rc,output,m=s.cmd1("echo 'q\n' | %s >& %s; echo" % (command,tmp.name))
output = tmp.read()
tmp.close()
versions = output[output.rfind('(')+1:output.rfind('q[uit]')].split()
return versions
def shell(self):
"""
Construct and store a shell which has the appropriate DIRAC env saved in it
"""
if self._shell is None:
self._shell = Shell()
self._shell.env.update(getDiracEnv())
return self._shell
def constructShell():
"""
Construct a grid shell based upon either the GLITE_SETUP or GLITE_LOCATION as possibly defined by the user
"""
values = {}
for key in ['X509_CERT_DIR', 'X509_VOMS_DIR']:
try:
values[key] = os.environ[key]
except KeyError:
pass
config = getConfig('LCG')
# 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value
# 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup)
# 3. else take the default GLITE_SETUP as session value
if config.getEffectiveLevel(
'GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ:
s = Shell()
else:
if os.path.exists(config['GLITE_SETUP']):
s = Shell(config['GLITE_SETUP'])
else:
logger.error("Configuration of GLITE for LCG: ")
logger.error("File not found: %s" % config['GLITE_SETUP'])
return None
for key, val in values.items():
s.env[key] = val
# check and set env. variables for default LFC setup
if 'LFC_HOST' not in s.env:
try:
s.env['LFC_HOST'] = config['DefaultLFC']
except ConfigError:
pass
if 'LFC_CONNTIMEOUT' not in s.env:
s.env['LFC_CONNTIMEOUT'] = '20'
if 'LFC_CONRETRY' not in s.env:
s.env['LFC_CONRETRY'] = '0'
if 'LFC_CONRETRYINT' not in s.env:
s.env['LFC_CONRETRYINT'] = '1'
return s
def __init__(self, requirements, check_file=False, create=False):
"""
Args:
requirements (ICredentialRequirement): An object specifying the requirements
check_file (bool): Raise an exception if the file does not exist
create (bool): Create the credential file
"""
self.shell = Shell()
super(AfsTokenInfo, self).__init__(requirements, check_file, create)
def _getShell(self):
fd = tempfile.NamedTemporaryFile()
script = '#!/bin/bash\n'
gaudirunShell = os.environ["GAUDIRUNENV"]
#cmd = '%s' % (gaudirunShell)
cmd = 'source %s' % (gaudirunShell)
script += '%s \n' % cmd
fd.write(script)
fd.flush()
logger.debug("zhangxm log: run boss env script:\n%s" % script)
shell = Shell(setup=fd.name)
return shell
def __init__(self):
logger.info("crabbackend init")
super(CRABBackend, self).__init__()
config = Config.getConfig('CMSSW')
shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'))
#shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
# [config['CMSSW_VERSION'], config['CRAB_VERSION']])
self.crab_env = shell.env
config = Config.getConfig('CRAB_CFG')
self.server_name = config['server_name']
self.apiresource = config['apiresource']
self.userproxy = config['userproxy']
self.asyncdest = config['asyncdest']
logger.info("asyncdest %s" % self.asyncdest)
def constructShell():
"""
Construct a grid shell based upon either the GLITE_SETUP or GLITE_LOCATION as possibly defined by the user
"""
values = {}
for key in ['X509_CERT_DIR', 'X509_VOMS_DIR']:
try:
values[key] = os.environ[key]
except KeyError:
pass
config = getConfig('LCG')
# 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value
# 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup)
# 3. else take the default GLITE_SETUP as session value
if config.getEffectiveLevel('GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ:
s = Shell()
else:
if os.path.exists(config['GLITE_SETUP']):
s = Shell(config['GLITE_SETUP'])
else:
logger.error("Configuration of GLITE for LCG: ")
logger.error("File not found: %s" % config['GLITE_SETUP'])
return None
for key, val in values.items():
s.env[key] = val
# check and set env. variables for default LFC setup
if 'LFC_HOST' not in s.env:
try:
s.env['LFC_HOST'] = config['DefaultLFC']
except ConfigError:
pass
if 'LFC_CONNTIMEOUT' not in s.env:
s.env['LFC_CONNTIMEOUT'] = '20'
if 'LFC_CONRETRY' not in s.env:
s.env['LFC_CONRETRY'] = '0'
if 'LFC_CONRETRYINT' not in s.env:
s.env['LFC_CONRETRYINT'] = '1'
return s
class AfsTokenInfo(ICredentialInfo):
"""
A wrapper around an AFS token
For now it is very CERN-specific (or at least only follows the CERN use-case)
"""
should_warn = False
info_pattern = re.compile(
r"^User's \(AFS ID \d*\) tokens for (?P<id>\w*@\S*) \[Expires (?P<expires>.*)\]$",
re.MULTILINE)
__slots__ = ('shell', 'cache', 'initial_requirements')
def __init__(self, requirements, check_file=False, create=False):
"""
Args:
requirements (ICredentialRequirement): An object specifying the requirements
check_file (bool): Raise an exception if the file does not exist
create (bool): Create the credential file
"""
self.shell = Shell()
super(AfsTokenInfo, self).__init__(requirements, check_file, create)
@retry_command
def create(self):
"""
Creates a new AFS token
Raises:
CredentialRenewalError: If the renewal process returns a non-zero value
"""
command = 'kinit'
process = subprocess.Popen(command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdoutdata, stderrdata = process.communicate(
getpass('Kerberos password: '******'AFS token %s created. Valid for %s', self.location,
self.time_left())
else:
raise CredentialRenewalError(stderrdata)
def renew(self):
"""
Renews the AFS token
Raises:
CredentialRenewalError: If the renewal process returns a non-zero value
"""
status, output, message = self.shell.cmd1('kinit -R')
if status != 0:
logger.debug('kinit -R failed, creating as new')
self.create()
def destroy(self):
"""
This removes the kerberos token from disk
"""
self.shell.cmd1('unlog')
if self.location:
os.remove(self.location)
@property
@cache
def info(self):
"""
This returns a summary of the token infor on disk
"""
status, output, message = self.shell.cmd1('tokens')
return output
@cache
def expiry_time(self):
"""
This calculates the number of seconds left for the kerberos token on disk
"""
info = self.info
matches = re.finditer(AfsTokenInfo.info_pattern, info)
if not matches:
return datetime.timedelta()
all_tokens = [match.group('expires') for match in matches]
if len(all_tokens) > 1:
if AfsTokenInfo.should_warn:
logger.warning(
"Found multiple AFS tokens, taking soonest expiring one for safety"
)
logger.warning("Tokens found for: %s".format(" ".join(
[match.group('id') for match in matches])))
AfsTokenInfo.should_warn = False
soonest = None
for expires in all_tokens:
expires = datetime.datetime.strptime(expires, '%b %d %H:%M')
now = datetime.datetime.now()
expires = expires.replace(year=now.year)
# If the expiration date is in the past then assume it should be in the future
if expires < now:
expires = expires.replace(year=now.year + 1)
if not soonest or expires < soonest:
soonest = expires
return soonest
def default_location(self):
"""
This returns the default location of a kerberos token on disk as determined from the uid
"""
krb_env_var = os.getenv('KRB5CCNAME', '')
if krb_env_var.startswith('FILE:'):
krb_env_var = krb_env_var[5:]
# If file already exists
if os.path.exists(krb_env_var):
return krb_env_var
# Lets try to find it if we can't get it from the env
default_name_prefix = '/tmp/krb5cc_{uid}'.format(uid=os.getuid())
matches = glob(default_name_prefix +
'*') # Check for partial matches on disk
if len(matches) == 1: # If one then use it
filename_guess = matches[0]
else: # Otherwise use the default
filename_guess = default_name_prefix
return filename_guess
def getShell(force=False):
"""
Utility function for getting Grid Shell.
Caller should take responsibility of credential checking if proxy is needed.
Arguments:
force (bool): False: if the shell already exists in local cache return the previous created instance ; True: recreate the shell and if not None update the cache
"""
global shell_cache
logger = getLogger()
if shell_cache is not None and not force:
return shell_cache
values = {}
for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']:
try:
values[key] = os.environ[key]
except KeyError:
pass
try:
config = getConfig('LCG')
except:
logger.warning(
'[LCG] configuration section not found. Cannot set up a proper grid shell.'
)
return None
s = None
# 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value
# 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup)
# 3. else take the default GLITE_SETUP as session value
if config.getEffectiveLevel(
'GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ:
s = Shell()
else:
if os.path.exists(config['GLITE_SETUP']):
s = Shell(config['GLITE_SETUP'])
else:
if config['GLITE_ENABLE']:
logger.warning("Configuration of GLITE for LCG: ")
logger.warning("File not found: %s" % config['GLITE_SETUP'])
if s:
for key, val in values.items():
s.env[key] = val
# check and set env. variables for default LFC setup
if 'LFC_HOST' not in s.env:
try:
s.env['LFC_HOST'] = config['DefaultLFC']
except ConfigError:
pass
if 'LFC_CONNTIMEOUT' not in s.env:
s.env['LFC_CONNTIMEOUT'] = '20'
if 'LFC_CONRETRY' not in s.env:
s.env['LFC_CONRETRY'] = '0'
if 'LFC_CONRETRYINT' not in s.env:
s.env['LFC_CONRETRYINT'] = '1'
shell_cache = s
return s
class AfsTokenInfo(ICredentialInfo):
"""
A wrapper around an AFS token
For now it is very CERN-specific (or at least only follows the CERN use-case)
"""
should_warn = False
info_pattern = re.compile(r"^User's \(AFS ID \d*\) tokens for (?P<id>\w*@\S*) \[Expires (?P<expires>.*)\]$", re.MULTILINE)
__slots__=('shell', 'cache', 'initial_requirements')
def __init__(self, requirements, check_file=False, create=False):
"""
Args:
requirements (ICredentialRequirement): An object specifying the requirements
check_file (bool): Raise an exception if the file does not exist
create (bool): Create the credential file
"""
self.shell = Shell()
super(AfsTokenInfo, self).__init__(requirements, check_file, create)
@retry_command
def create(self):
"""
Creates a new AFS token
Raises:
CredentialRenewalError: If the renewal process returns a non-zero value
"""
command = 'kinit'
process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdoutdata, stderrdata = process.communicate(getpass('Kerberos password: '******'AFS token %s created. Valid for %s', self.location, self.time_left())
else:
raise CredentialRenewalError(stderrdata)
def renew(self):
"""
Renews the AFS token
Raises:
CredentialRenewalError: If the renewal process returns a non-zero value
"""
status, output, message = self.shell.cmd1('kinit -R')
if status != 0:
logger.debug('kinit -R failed, creating as new')
self.create()
def destroy(self):
"""
This removes the kerberos token from disk
"""
self.shell.cmd1('unlog')
if self.location:
os.remove(self.location)
@property
@cache
def info(self):
"""
This returns a summary of the token infor on disk
"""
status, output, message = self.shell.cmd1('tokens')
return output
@cache
def expiry_time(self):
"""
This calculates the number of seconds left for the kerberos token on disk
"""
info = self.info
matches = re.finditer(AfsTokenInfo.info_pattern, info)
if not matches:
return datetime.timedelta()
all_tokens = [match.group('expires') for match in matches]
if len(all_tokens) > 1:
if AfsTokenInfo.should_warn:
logger.warning("Found multiple AFS tokens, taking soonest expiring one for safety")
logger.warning("Tokens found for: %s".format(" ".join([match.group('id') for match in matches])))
AfsTokenInfo.should_warn = False
soonest = None
for expires in all_tokens:
expires = datetime.datetime.strptime(expires, '%b %d %H:%M')
now = datetime.datetime.now()
expires = expires.replace(year=now.year)
# If the expiration date is in the past then assume it should be in the future
if expires < now:
expires = expires.replace(year=now.year+1)
if not soonest or expires < soonest:
soonest = expires
return soonest
def default_location(self):
"""
This returns the default location of a kerberos token on disk as determined from the uid
"""
krb_env_var = os.getenv('KRB5CCNAME', '')
if krb_env_var.startswith('FILE:'):
krb_env_var = krb_env_var[5:]
# If file already exists
if os.path.exists(krb_env_var):
return krb_env_var
# Lets try to find it if we can't get it from the env
default_name_prefix = '/tmp/krb5cc_{uid}'.format(uid=os.getuid())
matches = glob(default_name_prefix+'*') # Check for partial matches on disk
if len(matches) == 1: # If one then use it
filename_guess = matches[0]
else: # Otherwise use the default
filename_guess = default_name_prefix
return filename_guess
def getShell(middleware='EDG', force=False):
"""
Utility function for getting Grid Shell.
Caller should take responsiblity of credential checking if proxy is needed.
Argumennts:
middleware - grid m/w used
force - False : if the shell already exists in local cache return the previous created instance
True : recreate the shell and if not None update the cache
"""
logger = getLogger()
if not middleware:
logger.debug('No middleware specified, assuming default EDG')
middleware = 'EDG'
if middleware in _allShells.keys() and not force:
return _allShells[middleware]
values = {}
for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']:
try:
values[key] = os.environ[key]
except KeyError:
pass
configname = ""
if middleware == 'EDG' or middleware == 'GLITE':
configname = 'LCG'
else:
configname = middleware
config = None
try:
config = getConfig(configname)
except:
logger.warning(
'[%s] configuration section not found. Cannot set up a proper grid shell.' % configname)
return None
s = None
key = '%s_SETUP' % middleware
# 1. check if the *_SETUP is changed by user -> take the user's value as session value
# 2. else check if *_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup)
# 3. else take the default *_SETUP as session value
MIDDLEWARE_LOCATION = '%s_LOCATION' % middleware
if config.getEffectiveLevel(key) == 2 and MIDDLEWARE_LOCATION in os.environ:
s = Shell()
else:
if os.path.exists(config[key]):
# FIXME: Hardcoded rule for ARC middleware setup (pass explicitly
# the $ARC_LOCATION as argument), this is hardcoded to maintain
# backwards compatibility (and avoid any side effects) for EDG and
# GLITE setup scripts which did not take any arguments
if key.startswith('ARC') and MIDDLEWARE_LOCATION in os.environ:
s = Shell(
config[key], setup_args=[os.environ[MIDDLEWARE_LOCATION]])
else:
s = Shell(config[key])
else:
logger.warning("Configuration of %s for %s: " %
(middleware, configname))
logger.warning("File not found: %s" % config[key])
if s:
for key, val in values.items():
s.env[key] = val
# check and set env. variables for default LFC setup
if 'LFC_HOST' not in s.env:
try:
s.env['LFC_HOST'] = config['DefaultLFC']
except ConfigError:
pass
if 'LFC_CONNTIMEOUT' not in s.env:
s.env['LFC_CONNTIMEOUT'] = '20'
if 'LFC_CONRETRY' not in s.env:
s.env['LFC_CONRETRY'] = '0'
if 'LFC_CONRETRYINT' not in s.env:
s.env['LFC_CONRETRYINT'] = '1'
_allShells[middleware] = s
return s
###############################################################################
# A simple ATLAS dataset
#
# ATLAS/ARDA
import os, re
from Ganga.GPIDev.Lib.Dataset import Dataset
from Ganga.GPIDev.Schema import *
from Ganga.Utility.logging import getLogger
from Ganga.Utility.Shell import Shell
logger = getLogger()
shell = Shell()
class ATLASCastorDataset(Dataset):
'''ATLAS data as list of files on CASTOR'''
_schema = Schema(Version(1,0), {
'location' : SimpleItem(defvalue = '',
doc = 'A directory on castor that contains the datasets in directories'),
'pattern' : SimpleItem(defvalue = '',
doc = 'A regexp filter to select the correct files'),
'dataset' : SimpleItem(defvalue = '',
doc = 'The name of the selected dataset'),
'names' : SimpleItem(defvalue = [], typelist=['str'], sequence = 1,
doc = 'The selected file names')
})
class ICredential( GangaObject ):
"""
Interface class for working with credentials
"""
_schema = Schema( Version( 1, 0 ), {
"maxTry" : SimpleItem( defvalue = 1,
doc = "Number of password attempts allowed when creating credential" ),
"minValidity" : SimpleItem( defvalue = "00:15",
doc = "Default minimum validity" ),
"validityAtCreation" : SimpleItem( defvalue = "24:00",
doc = "Default credential validity at creation" ),
"command" : ComponentItem( category = "credential_commands",
defvalue = "ICommandSet",
doc = "Set of commands to be used for credential-related operations" )
} )
_category = "credentials"
_name = "ICredential"
_hidden = 1
_exportmethods = [ "create", "destroy", "isAvailable", "isValid",\
"location", "renew", "timeleft" ]
def __init__( self ):
super( ICredential, self ).__init__()
self.shell = Shell()
self.inputPW_Widget = None
return
def create( self, validity = "", maxTry = 0, minValidity = "", \
check = False ):
"""
Create credential.
Arguments other than self:
validity - Validity with which credential should be created,
specified as string of format "hh:mm"
[ Defaults to value of self.validityAtCreation ]
maxTry - Number of password attempts allowed
[ Defaults to value of self.maxTry ]
minValidity - Minimum validity in case checking of
pre-existing credential is performed,
specified as strong of format "hh:mm"
[ Defaults to value of self.minValidity ]
check - Flag to request checking of pre-existing
credential; if flag is set to true, then new
credential is created only if the validity of
any pre-existing credential is less than the
value of minValidity
[ Default: False ]
Note: create is the same as renew, except for the default value of check
Return value: True if credential is created successfully, and False
otherwise.
"""
global logTimeStamp
dummy = False
if not self.command.init:
dummy = True
if self.command.init_parameters.has_key( "valid" ):
if not self.command.init_parameters[ "valid" ]:
dummy = True
if dummy:
logger.warning( "Dummy CommandSet used - no credential created" )
return False
if not maxTry:
maxTry = self.maxTry
if not minValidity:
minValidity = self.minValidity
if not validity:
validity = self.validityAtCreation
validityInSeconds = self.timeInSeconds( validity )
if not validityInSeconds:
logger.warning( "Problems with requested validity: %s" \
% str( validity ) )
return False
if check and self.isValid( minValidity ):
return True
ntry = 0
while ntry < maxTry:
ntry = ntry + 1
# Test if GUI widget is to be used.
if self.inputPW_Widget:
# Since self.inputPW_Widget is called, current arguments are
# ignored since renew() and create() in GUI mode will not be
# called with any arguments.
if self.inputPW_Widget.ask( self._proxyObject ):
logger.debug( "Proceeding to retrieve password from inputPW_Widget." )
__pw = self.inputPW_Widget.getPassword( self._proxyObject )
if not __pw:
logger.warning( "Password/passphrase expected!" )
return False
try:
tFile = tempfile.NamedTemporaryFile()
tFile.write( __pw )
tFile.flush()
except:
del __pw
logger.warning( "Could not create secure temporary file for password!" )
return False
del __pw
else:
# Current credential modification denied for various reasons.
# see GangaGUI.customDialogs.ask() method for more details.
return False
# self.inputPW_Widget.ask() may have modified parameters.
# Calling buildOpts() to take them into account.
self.buildOpts( self.command.init, False )
# Create initialisation list with the 'pipe' parameter
initList = [ self.command.init, self.command.init_parameters[ "pipe" ] ]
# Append option value pairs
for optName, optVal in self.command.currentOpts.iteritems():
initList.append( "%s %s" % ( optName, optVal ) )
status = self.shell.system( "cat %s|%s" % ( tFile.name, " ".join( initList ) ) )
tFile.close()
# self.inputPW_Widget dialog postprocessing.
# E.g. disable autorenew mechanism if status != 0.
self.inputPW_Widget.renewalStatus( self._proxyObject, status )
if status == 0:
logger.info( "%s creation/renewal successful." % self._name )
return True
else:
logger.warning( "%s creation/renewal failed [%s]." % ( self._name, status ) )
return False
else: # Non-GUI credential renewal/creation
# Check if renewal is from main process (i.e. by bootstrap or user)
if threading.currentThread().getName() == 'MainThread':
if self.command.init_parameters.has_key( "valid" ):
self.command.currentOpts\
[ self.command.init_parameters[ 'valid' ] ] = validity
initList = [ self.command.init ]
# Append option value pairs
for optName, optVal in self.command.currentOpts.iteritems():
initList.append( "%s %s" % ( optName, optVal ) )
status = self.shell.system( " ".join( initList ) )
if status == 0:
logger.info( "%s creation/renewal successful." % self._name )
return True
else:
logger.warning( "%s creation/renewal failed [%s]." % ( self._name, status ) )
else: # create initiated from worker thread from monitoring component.
currTime = time.time()
if currTime - logTimeStamp >= logRepeatDuration:
logTimeStamp = currTime
# Check validity but print logging messages this time
self.isValid( "", True )
_credentialObject = self._name[ 0 ].lower() + self._name[ 1: ]
logger.warning( \
"Renew by typing '%s.renew()' at the prompt." % \
( _credentialObject ) )
#notify the Core that the credential is not valid
_validity = self.timeInSeconds(self.timeleft())
_minValidity = self.timeInSeconds(minValidity)/2
if _validity <= max(120,_minValidity):
Coordinator.notifyInvalidCredential(self)
return True
logger.warning( "%s creation/renewal attempts exceeded %s tries!" % ( self._name, maxTry ) )
return False
def destroy( self, allowed_exit = [ 0 ] ):
"""
Destroy credential
Argument other than self:
allowed_exit - List of exit codes accepted without error
when issuing system command for destroying credential
Return value: False if command for destroying credential is undefined,
or True otherwise
"""
if not self.command.destroy:
logger.warning( "Dummy CommandSet used - no credential created" )
return False
destroyList = [ self.command.destroy ]
for optName, optVal in self.command.destroyOpts.iteritems():
destroyList.append( "%s %s" % ( optName, optVal ) )
Coordinator.notifyInvalidCredential(self)
status, output, message = \
self.shell.cmd1( " ".join( destroyList ), allowed_exit )
proxyPath = self.location()
if proxyPath:
os.remove( proxyPath )
return True
def isAvailable( self ):
"""
Check whether credential is available with system/configuration used
No arguments other than self
Return value: True if credential is available, false otherwise
"""
logger.warning( "Dummy method used - this always returns True" )
return True
def isValid( self, validity = "", log = False, force_check = False ):
"""
Check validity
Arguments other than self:
validity - Minimum time for which credential should be valid,
specified as string of format "hh:mm"
[ Defaults to valud of self.minValidity ]
log - Print logger messages if credential not valid
force_check - Force credential check, rather than relying on cache
Return value: True if credential is valid for required time, False
otherwise.
"""
valid = True
if not validity:
validity = self.minValidity
validityInSeconds = self.timeInSeconds( validity )
timeleft = self.timeleft( force_check = force_check )
if not timeleft:
valid = False
else:
timeleftInSeconds = self.timeInSeconds( timeleft )
if timeleftInSeconds <= validityInSeconds:
valid = False
if not valid and log:
_tl = self.timeleft( force_check = force_check )
if _tl == "-1" or _tl == "0:00:00":
_expiryStatement = "has expired!"
else:
_expiryStatement = "will expire in %s!" % _tl
itemList = []
text = self._name[ 0 ]
for i in range( len( self._name ) - 1 ):
character = self._name[ i + 1 ]
if character.isupper():
itemList.append( text )
text = character.lower()
else:
text = "".join( [ text, character ] )
itemList.append( text )
_credentialName = " ".join( itemList )
logger.warning( "%s %s" % \
( _credentialName, _expiryStatement ) )
return valid
def location( self ):
"""
Determine credential location
No arguments other than self
Return value: Path to credential if found, or empty string otherwise
"""
return ""
def renew( self, validity = "", maxTry = 0, minValidity = "", \
check = True ):
"""
Renew credential.
Arguments other than self:
validity - Validity with which credential should be created,
specified as string of format "hh:mm"
[ Defaults to value of self.validityAtCreation ]
maxTry - Number of password attempts allowed
[ Defaults to value of self.maxTry ]
minValidity - Minimum validity in case checking of
pre-existing credential is performed,
specified as strong of format "hh:mm"
[ Defaults to value of self.minValidity ]
check - Flag to request checking of pre-existing
credential; if flag is set to true, then new
credential is created only if the validity of
any pre-existing credential is less than the
value of minValidity
[ Default: True ]
Note: renew is the same as create, except for the default value of check
Return value: True if new credential is created successfully, and False
otherwise.
"""
status = self.create( validity, maxTry, minValidity, check )
return status
def timeInSeconds( self, timeString = "" ):
"""
Convert time string to time in seconds
Arguments other than self:
timeString - Time specified as string of format "hh:mm:ss"
Return value: Time in seconds (integer)
"""
totalTime = 0
timeList = timeString.split( ":" )
if len( timeList ) >= 1:
totalTime = totalTime + int( timeList[ 0 ] ) * 60 * 60
if len( timeList ) >= 2:
totalTime = totalTime + int( timeList[ 1 ] ) * 60
if len( timeList ) >= 3:
totalTime = totalTime + int (timeList[ 2 ] )
return totalTime
def timeleft( self, units = "hh:mm:ss", force_check = False ):
"""
Check time for which credential is valid.
Arguments other than self:
units - String specifying units in which time is returned
force_check - Force credential check, rather than relying on cache
Allowed values for units are:
"hours" - time returned as in hours
"minutes" - time returned in minutes
"seconds" - time returned in seconds
"hh:mm:ss" [default] - time returned as hours, minutes seconds
Return value: Credential validity as string giving time in requested
units, or empty string if command for querying credential validity
is unavailable
"""
timeRemaining = self.timeleftInHMS( force_check = force_check )
if timeRemaining not in [ "", "-1" ]:
if units in [ "hours", "minutes", "seconds" ]:
timeleftInSeconds = self.timeInSeconds( timeRemaining )
if "seconds" == units:
timeRemaining = "%.2f" % ( timeleftInSeconds )
elif "minutes" == units:
timeRemaining = "%.2f" % ( timeleftInSeconds / 60. )
elif "hours" == units:
timeRemaining = "%.2f" % ( timeleftInSeconds / ( 60. * 60. ) )
return timeRemaining
def timeleftInHMS( self, force_check = False ):
"""
Determine remaining validity of credential in hours, minutes and seconds
Argument other than self:
force_check - Force credential check, rather than relying on cache
Return value: String giving credential validity, or empty string
if command for querying credential validity is unavailable
"""
logger.warning( "Dummy method used - no information returned" )
return ""
def getShell(force=False):
"""
Utility function for getting Grid Shell.
Caller should take responsibility of credential checking if proxy is needed.
Arguments:
force (bool): False: if the shell already exists in local cache return the previous created instance ; True: recreate the shell and if not None update the cache
"""
global shell_cache
logger = getLogger()
if shell_cache is not None and not force:
return shell_cache
values = {}
for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']:
try:
values[key] = os.environ[key]
except KeyError:
pass
try:
config = getConfig('LCG')
except:
logger.warning('[LCG] configuration section not found. Cannot set up a proper grid shell.')
return None
s = None
# 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value
# 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup)
# 3. else take the default GLITE_SETUP as session value
if config.getEffectiveLevel('GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ:
s = Shell()
else:
if os.path.exists(config['GLITE_SETUP']):
s = Shell(config['GLITE_SETUP'])
else:
if config['GLITE_ENABLE']:
logger.warning("Configuration of GLITE for LCG: ")
logger.warning("File not found: %s" % config['GLITE_SETUP'])
if s:
for key, val in values.items():
s.env[key] = val
# check and set env. variables for default LFC setup
if 'LFC_HOST' not in s.env:
try:
s.env['LFC_HOST'] = config['DefaultLFC']
except ConfigError:
pass
if 'LFC_CONNTIMEOUT' not in s.env:
s.env['LFC_CONNTIMEOUT'] = '20'
if 'LFC_CONRETRY' not in s.env:
s.env['LFC_CONRETRY'] = '0'
if 'LFC_CONRETRYINT' not in s.env:
s.env['LFC_CONRETRYINT'] = '1'
shell_cache = s
return s
def __init__(self):
super(ICredential, self).__init__()
self.shell = Shell()
self.inputPW_Widget = None
return
class ICredential(GangaObject):
"""
Interface class for working with credentials
"""
_schema = Schema(
Version(1, 0), {
"maxTry":
SimpleItem(
defvalue=1,
typelist=[int],
doc=
"Number of password attempts allowed when creating credential"
),
"minValidity":
SimpleItem(defvalue="00:15",
typelist=[str],
doc="Default minimum validity"),
"validityAtCreation":
SimpleItem(defvalue="24:00",
typelist=[str],
doc="Default credential validity at creation"),
"command":
ComponentItem(
category="credential_commands",
defvalue="ICommandSet",
doc=
"Set of commands to be used for credential-related operations")
})
_category = "credentials"
_name = "ICredential"
_hidden = 1
_exportmethods = [
"create", "destroy", "isAvailable", "isValid", "location", "renew",
"timeleft"
]
def __init__(self):
super(ICredential, self).__init__()
self.shell = Shell()
self.inputPW_Widget = None
return
def create(self, validity="", maxTry=0, minValidity="", check=False):
"""
Create credential.
Arguments other than self:
validity - Validity with which credential should be created,
specified as string of format "hh:mm"
[ Defaults to value of self.validityAtCreation ]
maxTry - Number of password attempts allowed
[ Defaults to value of self.maxTry ]
minValidity - Minimum validity in case checking of
pre-existing credential is performed,
specified as strong of format "hh:mm"
[ Defaults to value of self.minValidity ]
check - Flag to request checking of pre-existing
credential; if flag is set to true, then new
credential is created only if the validity of
any pre-existing credential is less than the
value of minValidity
[ Default: False ]
Note: create is the same as renew, except for the default value of check
Return value: True if credential is created successfully, and False
otherwise.
"""
global logTimeStamp
dummy = False
if not self.command.init:
dummy = True
if "valid" in self.command.init_parameters:
if not self.command.init_parameters["valid"]:
dummy = True
if dummy:
logger.warning("Dummy CommandSet used - no credential created")
return False
if not maxTry:
maxTry = self.maxTry
if not minValidity:
minValidity = self.minValidity
if not validity:
validity = self.validityAtCreation
validityInSeconds = self.timeInSeconds(validity)
if not validityInSeconds:
logger.warning("Problems with requested validity: %s" %
str(validity))
return False
if check and self.isValid(minValidity):
return True
ntry = 0
while ntry < maxTry:
ntry = ntry + 1
# Test if GUI widget is to be used.
if self.inputPW_Widget:
# Since self.inputPW_Widget is called, current arguments are
# ignored since renew() and create() in GUI mode will not be
# called with any arguments.
#proxy_obj = self._proxyObject ## This is removed to get rid of ref to _proxyObject
proxy_obj = self
if self.inputPW_Widget.ask(proxy_obj):
logger.dg(
"Proceeding to retrieve password from inputPW_Widget.")
__pw = self.inputPW_Widget.getPassword(proxy_obj)
if not __pw:
logger.warning("Password/passphrase expected!")
return False
try:
tFile = tempfile.NamedTemporaryFile()
tFile.write(__pw)
tFile.flush()
except:
del __pw
logger.warning(
"Could not create secure temporary file for password!"
)
return False
del __pw
else:
# Current credential modification denied for various reasons.
# see GangaGUI.customDialogs.ask() method for more details.
return False
# self.inputPW_Widget.ask() may have modified parameters.
# Calling buildOpts() to take them into account.
self.buildOpts(self.command.init, False)
# Create initialisation list with the 'pipe' parameter
initList = [
self.command.init, self.command.init_parameters["pipe"]
]
# Append option value pairs
for optName, optVal in self.command.currentOpts.iteritems():
initList.append("%s %s" % (optName, optVal))
status = self.shell.system("cat %s|%s" %
(tFile.name, " ".join(initList)))
tFile.close()
# self.inputPW_Widget dialog postprocessing.
# E.g. disable autorenew mechanism if status != 0.
self.inputPW_Widget.renewalStatus(proxy_obj, status)
if status == 0:
logger.info("%s creation/renewal successful." % self._name)
return True
else:
logger.warning("%s creation/renewal failed [%s]." %
(self._name, status))
return False
else: # Non-GUI credential renewal/creation
# Check if renewal is from main process (i.e. by bootstrap or
# user)
if threading.currentThread().getName() == 'MainThread' or\
threading.currentThread().getName().startswith('GANGA_Update_Thread_Ganga_Worker_'):
if "valid" in self.command.init_parameters:
self.command.currentOpts[
self.command.init_parameters['valid']] = validity
initList = [self.command.init]
# Append option value pairs
for optName, optVal in self.command.currentOpts.iteritems(
):
initList.append("%s %s" % (optName, optVal))
status = self.shell.system(" ".join(initList))
if status == 0:
logger.info("%s creation/renewal successful." %
self._name)
return True
else:
logger.warning("%s creation/renewal failed [%s]." %
(self._name, status))
# create initiated from worker thread from monitoring
# component.
else:
currTime = time.time()
if currTime - logTimeStamp >= logRepeatDuration:
logTimeStamp = currTime
# Check validity but print logging messages this time
self.isValid("", True)
_credentialObject = self._name[0].lower(
) + self._name[1:]
logger.warning(
"Renew by typing '%s.renew()' at the prompt." %
(_credentialObject))
# notify the Core that the credential is not valid
_validity = self.timeInSeconds(self.timeleft())
_minValidity = self.timeInSeconds(minValidity) / 2.
if _validity <= max(120, _minValidity):
Coordinator.notifyInvalidCredential(self)
return True
logger.warning("%s creation/renewal attempts exceeded %s tries!" %
(self._name, maxTry))
return False
def destroy(self, allowed_exit=[0]):
"""
Destroy credential
Argument other than self:
allowed_exit - List of exit codes accepted without error
when issuing system command for destroying credential
Return value: False if command for destroying credential is undefined,
or True otherwise
"""
if not self.command.destroy:
logger.warning("Dummy CommandSet used - no credential created")
return False
destroyList = [self.command.destroy]
for optName, optVal in self.command.destroyOpts.iteritems():
destroyList.append("%s %s" % (optName, optVal))
Coordinator.notifyInvalidCredential(self)
status, output, message = \
self.shell.cmd1(" ".join(destroyList), allowed_exit)
proxyPath = self.location()
if proxyPath:
os.remove(proxyPath)
return True
def isAvailable(self):
"""
Check whether credential is available with system/configuration used
No arguments other than self
Return value: True if credential is available, false otherwise
"""
logger.warning("Dummy method used - this always returns True")
return True
def isValid(self, validity="", log=False, force_check=False):
"""
Check validity
Arguments other than self:
validity - Minimum time for which credential should be valid,
specified as string of format "hh:mm"
[ Defaults to valud of self.minValidity ]
log - Print logger messages if credential not valid
force_check - Force credential check, rather than relying on cache
Return value: True if credential is valid for required time, False
otherwise.
"""
valid = True
if not validity or validity is None:
validity = self.minValidity
validityInSeconds = self.timeInSeconds(validity)
timeleft = self.timeleft(force_check=force_check)
if not timeleft:
valid = False
else:
timeleftInSeconds = self.timeInSeconds(timeleft)
if timeleftInSeconds <= validityInSeconds:
valid = False
if not valid and log:
_tl = self.timeleft(force_check=force_check)
if _tl == "-1" or _tl == "0:00:00":
_expiryStatement = "has expired!"
else:
_expiryStatement = "will expire in %s!" % _tl
itemList = []
text = self._name[0]
for i in range(len(self._name) - 1):
character = self._name[i + 1]
if character.isupper():
itemList.append(text)
text = character.lower()
else:
text = "".join([text, character])
itemList.append(text)
_credentialName = " ".join(itemList)
logger.warning("%s %s" % (_credentialName, _expiryStatement))
return valid
def location(self):
"""
Determine credential location
No arguments other than self
Return value: Path to credential if found, or empty string otherwise
"""
return ""
def renew(self, validity="", maxTry=0, minValidity="", check=True):
"""
Renew credential.
Arguments other than self:
validity - Validity with which credential should be created,
specified as string of format "hh:mm"
[ Defaults to value of self.validityAtCreation ]
maxTry - Number of password attempts allowed
[ Defaults to value of self.maxTry ]
minValidity - Minimum validity in case checking of
pre-existing credential is performed,
specified as strong of format "hh:mm"
[ Defaults to value of self.minValidity ]
check - Flag to request checking of pre-existing
credential; if flag is set to true, then new
credential is created only if the validity of
any pre-existing credential is less than the
value of minValidity
[ Default: True ]
Note: renew is the same as create, except for the default value of check
Return value: True if new credential is created successfully, and False
otherwise.
"""
status = self.create(validity, maxTry, minValidity, check)
return status
def timeInSeconds(self, timeString=""):
"""
Convert time string to time in seconds
Arguments other than self:
timeString - Time specified as string of format "hh:mm:ss"
Return value: Time in seconds (integer)
"""
totalTime = 0
timeList = timeString.split(":")
if len(timeList) >= 1:
totalTime = totalTime + int(timeList[0]) * 60 * 60
if len(timeList) >= 2:
totalTime = totalTime + int(timeList[1]) * 60
if len(timeList) >= 3:
totalTime = totalTime + int(timeList[2])
return totalTime
def timeleft(self, units="hh:mm:ss", force_check=False):
"""
Check time for which credential is valid.
Arguments other than self:
units - String specifying units in which time is returned
force_check - Force credential check, rather than relying on cache
Allowed values for units are:
"hours" - time returned as in hours
"minutes" - time returned in minutes
"seconds" - time returned in seconds
"hh:mm:ss" [default] - time returned as hours, minutes seconds
Return value: Credential validity as string giving time in requested
units, or empty string if command for querying credential validity
is unavailable
"""
timeRemaining = self.timeleftInHMS(force_check=force_check)
if timeRemaining not in ["", "-1"]:
if units in ["hours", "minutes", "seconds"]:
timeleftInSeconds = self.timeInSeconds(timeRemaining)
if "seconds" == units:
timeRemaining = "%.2f" % (timeleftInSeconds)
elif "minutes" == units:
timeRemaining = "%.2f" % (timeleftInSeconds / 60.)
elif "hours" == units:
timeRemaining = "%.2f" % (timeleftInSeconds / (60. * 60.))
return timeRemaining
def timeleftInHMS(self, force_check=False):
"""
Determine remaining validity of credential in hours, minutes and seconds
Argument other than self:
force_check - Force credential check, rather than relying on cache
Return value: String giving credential validity, or empty string
if command for querying credential validity is unavailable
"""
logger.warning("Dummy method used - no information returned")
return ""
def __init__(self):
super(CRABBackend, self).__init__()
config = Config.getConfig('CMSSW')
shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
[config['CMSSW_VERSION'], config['CRAB_VERSION']])
self.crab_env = shell.env
def __init__( self ):
super( ICredential, self ).__init__()
self.shell = Shell()
self.inputPW_Widget = None
return
#script+= '%s %s %s %s\n' % (use,setupProjectOptions,appname,app.version)
#script+= '%s %s\n' % (use,setupProjectOptions)
command=command.replace('###CMT###',cmtcmd + ' ' + cmtoption)
logger.debug('zhangxm log: Will execute the command: '+command)
script += command + '\n'
logger.debug('zhangxm log: The full script for execution:\n'+script)
# write file
try:
tmppath = tempfile.mkdtemp()
fn = os.path.join(tmppath, 'cmtcommand_script')
file1 = open(fn, 'w')
except Exception, e:
logger.error("Can not create temporary file %s", fn)
return
else:
try:
file1.write(script)
finally:
file1.close()
# make file executable
os.chmod(fn, 0777)
shell = Shell()
rc=shell.system(fn)
if os.path.exists(tmppath): shutil.rmtree(tmppath)
return True
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
def CMTscript(app, command=''):
"""Function to execute a cmt command for a specific job. Returns the unix
exit code.
Arguments:
app - The Gaudi application object to take information from
command - String [default ''] The cmt command to execute.
"""
cmtcmd = 'cmt'
warnings.filterwarnings('ignore', 'tempnam', RuntimeWarning)
use = ''
if app.masterpackage:
(pack, alg, ver) = parse_master_package(app.masterpackage)
use = '--use "%s %s %s"' % (alg, ver, pack)
ura = app.user_release_area
if not ura:
expanded = os.path.expandvars("$User_release_area")
if expanded == "$User_release_area":
ura = ""
else:
ura = expanded.split(os.pathsep)[0]
cmtoption = ''
# generate shell script
script = '#!/bin/sh\n'
script += 'unalias -a\n'
script += '. `which LbLogin.sh` -c ' + str(app.platform) + '\n'
script += 'export User_release_area=' + str(ura) + '\n'
script += 'unset CMTPROJECTPATH\n'
script += '. setenvProject.sh '
setupProjectOptions = ''
if app.setupProjectOptions:
setupProjectOptions = app.setupProjectOptions
script += '%s %s %s %s\n' % (use,
setupProjectOptions, app.appname, app.version)
command = command.replace('###CMT###', cmtcmd + ' ' + cmtoption)
logger.debug('Will execute the command: ' + command)
script += command + '\n'
logger.debug('The full script for execution:\n' + script)
# write file
try:
tmppath = tempfile.mkdtemp()
fn = os.path.join(tmppath, 'cmtcommand_script')
file1 = open(fn, 'w')
except Exception as e:
logger.error("Can not create temporary file %s", fn)
return
else:
try:
file1.write(script)
finally:
file1.close()
# make file executable
os.chmod(fn, 0o777)
shell = Shell()
rc = shell.system(fn)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
return rc
def CMTscript(app, command=''):
"""Function to execute a cmt command for a specific job. Returns the unix
exit code.
Arguments:
app - The Gaudi application object to take information from
command - String [default ''] The cmt command to execute.
"""
cmtcmd = 'cmt'
warnings.filterwarnings('ignore', 'tempnam', RuntimeWarning)
use = ''
if app.masterpackage:
(pack, alg, ver) = parse_master_package(app.masterpackage)
use = '--use "%s %s %s"' % (alg, ver, pack)
ura = app.user_release_area
if not ura:
expanded = os.path.expandvars("$User_release_area")
if expanded == "$User_release_area":
ura = ""
else:
ura = expanded.split(os.pathsep)[0]
cmtoption = ''
# generate shell script
script = '#!/bin/sh\n'
script += 'unalias -a\n'
script += '. `which LbLogin.sh` -c ' + str(app.platform) + '\n'
script += 'export User_release_area=' + str(ura) + '\n'
script += 'unset CMTPROJECTPATH\n'
script += '. setenvProject.sh '
setupProjectOptions = ''
if app.setupProjectOptions:
setupProjectOptions = app.setupProjectOptions
script += '%s %s %s %s\n' % (use, setupProjectOptions, app.appname,
app.version)
command = command.replace('###CMT###', cmtcmd + ' ' + cmtoption)
logger.debug('Will execute the command: ' + command)
script += command + '\n'
logger.debug('The full script for execution:\n' + script)
# write file
try:
tmppath = tempfile.mkdtemp()
fn = os.path.join(tmppath, 'cmtcommand_script')
file1 = open(fn, 'w')
except Exception as e:
logger.error("Can not create temporary file %s", fn)
return
else:
try:
file1.write(script)
finally:
file1.close()
# make file executable
os.chmod(fn, 0o777)
shell = Shell()
rc = shell.system(fn)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
return rc
