def constructShell():
"""
Construct a grid shell based upon either the GLITE_SETUP or GLITE_LOCATION as possibly defined by the user
"""
values = {}
for key in ['X509_CERT_DIR', 'X509_VOMS_DIR']:
try:
values[key] = os.environ[key]
except KeyError:
pass
config = getConfig('LCG')
# 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value
# 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup)
# 3. else take the default GLITE_SETUP as session value
if config.getEffectiveLevel(
'GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ:
s = Shell()
else:
if os.path.exists(config['GLITE_SETUP']):
s = Shell(config['GLITE_SETUP'])
else:
logger.error("Configuration of GLITE for LCG: ")
logger.error("File not found: %s" % config['GLITE_SETUP'])
return None
for key, val in values.items():
s.env[key] = val
# check and set env. variables for default LFC setup
if 'LFC_HOST' not in s.env:
try:
s.env['LFC_HOST'] = config['DefaultLFC']
except ConfigError:
pass
if 'LFC_CONNTIMEOUT' not in s.env:
s.env['LFC_CONNTIMEOUT'] = '20'
if 'LFC_CONRETRY' not in s.env:
s.env['LFC_CONRETRY'] = '0'
if 'LFC_CONRETRYINT' not in s.env:
s.env['LFC_CONRETRYINT'] = '1'
return s
def shell(self):
"""
Construct and store a shell which has the appropriate DIRAC env saved in it
"""
if self._shell is None:
self._shell = Shell()
self._shell.env.update(getDiracEnv())
return self._shell
def __init__(self, requirements, check_file=False, create=False):
"""
Args:
requirements (ICredentialRequirement): An object specifying the requirements
check_file (bool): Raise an exception if the file does not exist
create (bool): Create the credential file
"""
self.shell = Shell()
super(AfsTokenInfo, self).__init__(requirements, check_file, create)
def guess_version(appname):
"""Guess the default Gaudi application version"""
s = Shell()
tmp = tempfile.NamedTemporaryFile(suffix='.log')
command = 'SetupProject.sh --ask %s' % appname
rc,output,m=s.cmd1("echo 'q\n' | %s >& %s; echo" % (command,tmp.name))
output = tmp.read()
tmp.close()
version = output[output.rfind('[')+1:output.rfind(']')]
return version
def available_versions(appname):
"""Provide a list of the available Gaudi application versions"""
s = Shell()
tmp = tempfile.NamedTemporaryFile(suffix='.log')
command = 'SetupProject.sh --ask %s' % appname
rc,output,m=s.cmd1("echo 'q\n' | %s >& %s; echo" % (command,tmp.name))
output = tmp.read()
tmp.close()
versions = output[output.rfind('(')+1:output.rfind('q[uit]')].split()
return versions
def _getShell(self):
fd = tempfile.NamedTemporaryFile()
script = '#!/bin/bash\n'
gaudirunShell = os.environ["GAUDIRUNENV"]
#cmd = '%s' % (gaudirunShell)
cmd = 'source %s' % (gaudirunShell)
script += '%s \n' % cmd
fd.write(script)
fd.flush()
logger.debug("zhangxm log: run boss env script:\n%s" % script)
shell = Shell(setup=fd.name)
return shell
def __init__(self):
logger.info("crabbackend init")
super(CRABBackend, self).__init__()
config = Config.getConfig('CMSSW')
shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'))
#shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
# [config['CMSSW_VERSION'], config['CRAB_VERSION']])
self.crab_env = shell.env
config = Config.getConfig('CRAB_CFG')
self.server_name = config['server_name']
self.apiresource = config['apiresource']
self.userproxy = config['userproxy']
self.asyncdest = config['asyncdest']
logger.info("asyncdest %s" % self.asyncdest)
def getShell(middleware='EDG', force=False):
"""
Utility function for getting Grid Shell.
Caller should take responsiblity of credential checking if proxy is needed.
Argumennts:
middleware - grid m/w used
force - False : if the shell already exists in local cache return the previous created instance
True : recreate the shell and if not None update the cache
"""
logger = getLogger()
if not middleware:
logger.debug('No middleware specified, assuming default EDG')
middleware = 'EDG'
if middleware in _allShells.keys() and not force:
return _allShells[middleware]
values = {}
for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']:
try:
values[key] = os.environ[key]
except KeyError:
pass
configname = ""
if middleware == 'EDG' or middleware == 'GLITE':
configname = 'LCG'
else:
configname = middleware
config = None
try:
config = getConfig(configname)
except:
logger.warning(
'[%s] configuration section not found. Cannot set up a proper grid shell.' % configname)
return None
s = None
key = '%s_SETUP' % middleware
# 1. check if the *_SETUP is changed by user -> take the user's value as session value
# 2. else check if *_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup)
# 3. else take the default *_SETUP as session value
MIDDLEWARE_LOCATION = '%s_LOCATION' % middleware
if config.getEffectiveLevel(key) == 2 and MIDDLEWARE_LOCATION in os.environ:
s = Shell()
else:
if os.path.exists(config[key]):
# FIXME: Hardcoded rule for ARC middleware setup (pass explicitly
# the $ARC_LOCATION as argument), this is hardcoded to maintain
# backwards compatibility (and avoid any side effects) for EDG and
# GLITE setup scripts which did not take any arguments
if key.startswith('ARC') and MIDDLEWARE_LOCATION in os.environ:
s = Shell(
config[key], setup_args=[os.environ[MIDDLEWARE_LOCATION]])
else:
s = Shell(config[key])
else:
logger.warning("Configuration of %s for %s: " %
(middleware, configname))
logger.warning("File not found: %s" % config[key])
if s:
for key, val in values.items():
s.env[key] = val
# check and set env. variables for default LFC setup
if 'LFC_HOST' not in s.env:
try:
s.env['LFC_HOST'] = config['DefaultLFC']
except ConfigError:
pass
if 'LFC_CONNTIMEOUT' not in s.env:
s.env['LFC_CONNTIMEOUT'] = '20'
if 'LFC_CONRETRY' not in s.env:
s.env['LFC_CONRETRY'] = '0'
if 'LFC_CONRETRYINT' not in s.env:
s.env['LFC_CONRETRYINT'] = '1'
_allShells[middleware] = s
return s
#script+= '%s %s %s %s\n' % (use,setupProjectOptions,appname,app.version)
#script+= '%s %s\n' % (use,setupProjectOptions)
command=command.replace('###CMT###',cmtcmd + ' ' + cmtoption)
logger.debug('zhangxm log: Will execute the command: '+command)
script += command + '\n'
logger.debug('zhangxm log: The full script for execution:\n'+script)
# write file
try:
tmppath = tempfile.mkdtemp()
fn = os.path.join(tmppath, 'cmtcommand_script')
file1 = open(fn, 'w')
except Exception, e:
logger.error("Can not create temporary file %s", fn)
return
else:
try:
file1.write(script)
finally:
file1.close()
# make file executable
os.chmod(fn, 0777)
shell = Shell()
rc=shell.system(fn)
if os.path.exists(tmppath): shutil.rmtree(tmppath)
return True
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
def __init__(self):
super(ICredential, self).__init__()
self.shell = Shell()
self.inputPW_Widget = None
return
def __init__(self):
super(CRABBackend, self).__init__()
config = Config.getConfig('CMSSW')
shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
[config['CMSSW_VERSION'], config['CRAB_VERSION']])
self.crab_env = shell.env
def getShell(force=False):
"""
Utility function for getting Grid Shell.
Caller should take responsibility of credential checking if proxy is needed.
Arguments:
force (bool): False: if the shell already exists in local cache return the previous created instance ; True: recreate the shell and if not None update the cache
"""
global shell_cache
logger = getLogger()
if shell_cache is not None and not force:
return shell_cache
values = {}
for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']:
try:
values[key] = os.environ[key]
except KeyError:
pass
try:
config = getConfig('LCG')
except:
logger.warning(
'[LCG] configuration section not found. Cannot set up a proper grid shell.'
)
return None
s = None
# 1. check if the GLITE_SETUP is changed by user -> take the user's value as session value
# 2. else check if GLITE_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup)
# 3. else take the default GLITE_SETUP as session value
if config.getEffectiveLevel(
'GLITE_SETUP') == 2 and 'GLITE_LOCATION' in os.environ:
s = Shell()
else:
if os.path.exists(config['GLITE_SETUP']):
s = Shell(config['GLITE_SETUP'])
else:
if config['GLITE_ENABLE']:
logger.warning("Configuration of GLITE for LCG: ")
logger.warning("File not found: %s" % config['GLITE_SETUP'])
if s:
for key, val in values.items():
s.env[key] = val
# check and set env. variables for default LFC setup
if 'LFC_HOST' not in s.env:
try:
s.env['LFC_HOST'] = config['DefaultLFC']
except ConfigError:
pass
if 'LFC_CONNTIMEOUT' not in s.env:
s.env['LFC_CONNTIMEOUT'] = '20'
if 'LFC_CONRETRY' not in s.env:
s.env['LFC_CONRETRY'] = '0'
if 'LFC_CONRETRYINT' not in s.env:
s.env['LFC_CONRETRYINT'] = '1'
shell_cache = s
return s
def CMTscript(app, command=''):
"""Function to execute a cmt command for a specific job. Returns the unix
exit code.
Arguments:
app - The Gaudi application object to take information from
command - String [default ''] The cmt command to execute.
"""
cmtcmd = 'cmt'
warnings.filterwarnings('ignore', 'tempnam', RuntimeWarning)
use = ''
if app.masterpackage:
(pack, alg, ver) = parse_master_package(app.masterpackage)
use = '--use "%s %s %s"' % (alg, ver, pack)
ura = app.user_release_area
if not ura:
expanded = os.path.expandvars("$User_release_area")
if expanded == "$User_release_area":
ura = ""
else:
ura = expanded.split(os.pathsep)[0]
cmtoption = ''
# generate shell script
script = '#!/bin/sh\n'
script += 'unalias -a\n'
script += '. `which LbLogin.sh` -c ' + str(app.platform) + '\n'
script += 'export User_release_area=' + str(ura) + '\n'
script += 'unset CMTPROJECTPATH\n'
script += '. setenvProject.sh '
setupProjectOptions = ''
if app.setupProjectOptions:
setupProjectOptions = app.setupProjectOptions
script += '%s %s %s %s\n' % (use, setupProjectOptions, app.appname,
app.version)
command = command.replace('###CMT###', cmtcmd + ' ' + cmtoption)
logger.debug('Will execute the command: ' + command)
script += command + '\n'
logger.debug('The full script for execution:\n' + script)
# write file
try:
tmppath = tempfile.mkdtemp()
fn = os.path.join(tmppath, 'cmtcommand_script')
file1 = open(fn, 'w')
except Exception as e:
logger.error("Can not create temporary file %s", fn)
return
else:
try:
file1.write(script)
finally:
file1.close()
# make file executable
os.chmod(fn, 0o777)
shell = Shell()
rc = shell.system(fn)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
return rc
