def test_execute_env():
''' This tests that the scripts can update an env based upon input and handle an appropriate python_setup sent in execute args '''
# Test correctly picking up env
env = {
'ALEX': '/hello/world',
'PATH': os.environ.get('PATH', ''),
'PYTHONPATH': os.environ.get('PYTHONPATH', '')
}
# env.update(os.environ)
assert execute('echo $ALEX', shell=True, env=env).strip() == '/hello/world'
assert execute('import os\noutput(os.environ.get("ALEX","BROKEN"))',
env=env,
python_setup='#',
shell=False).strip() == '/hello/world'
# Test env not updated by default
execute('export NEWTEST=/new/test', shell=True, env=env)
assert 'NEWTEST' not in env
execute('import os\nos.environ["NEWTEST"]="/new/test"',
env=env,
python_setup='#')
assert 'NEWTEST' not in env
# Test updating of env
execute('export NEWTEST=/new/test', shell=True, env=env, update_env=True)
assert 'NEWTEST' in env
del env['NEWTEST']
assert 'NEWTEST' not in env
execute('import os\nos.environ["NEWTEST"]="/new/test"',
env=env,
python_setup='#',
update_env=True,
shell=False)
assert 'NEWTEST' in env
def test_execute_output():
''' This tests the various levels when an import can occur and that complex objects can be returned via stdout '''
# Test printout of pickle dump interpreted correctly
d = execute(
'import pickle, datetime\nprint(pickle.dumps(datetime.datetime(2013,12,12)))',
shell=False)
assert hasattr(d, 'month')
assert d.month == 12
# Test straight printout doesn't work without includes, stdout as str
# returned by default
d1 = execute('import datetime\nprint(datetime.datetime(2013,12,12))',
shell=False)
d2 = execute('import datetime\nrepr(datetime.datetime(2013,12,12))',
shell=False)
assert not hasattr(d1, 'month')
assert isinstance(d1, str)
assert not hasattr(d2, 'month')
assert isinstance(d2, str)
# Test printout works with the right includes
d1 = execute('print(datetime.datetime(2013,12,12))',
python_setup='import datetime',
eval_includes='import datetime',
shell=False)
d2 = execute('print(repr(datetime.datetime(2013,12,12)))',
python_setup='import datetime',
eval_includes='import datetime',
shell=False)
assert not hasattr(d1, 'month')
assert isinstance(d1, str)
assert hasattr(d2, 'month')
assert d2.month == 12
def get_env(env_source):
"""
Given a source command, return the DIRAC environment that the
command created.
Args:
env_source: a command which can be sourced, providing the desired environment
Returns:
dict: the environment
"""
logger.debug('Running DIRAC source command %s', env_source)
env = dict(os.environ)
gexecute.execute('source {0}'.format(env_source), shell=True, env=env, update_env=True)
if not any(key.startswith('DIRAC') for key in env):
fake_dict = {}
with open(env_source) as _env:
for _line in _env.readlines():
split_val = _line.split('=')
if len(split_val) == 2:
key = split_val[0]
value = split_val[1]
fake_dict[key] = value
if not any(key.startswith('DIRAC') for key in fake_dict):
logger.error("Env: %s" % str(env))
logger.error("Fake: %s" % str(fake_dict))
raise RuntimeError("'DIRAC*' not found in environment")
else:
return fake_dict
return env
def cmt(self, command):
"""Execute a cmt command in the cmt user area pointed to by the
application. Will execute the command "cmt <command>" after the
proper configuration. Do not include the word "cmt" yourself."""
execute('cmt %s' % command,
shell=True,
timeout=None,
env=self.getenv(False),
cwd=self.user_release_area)
def cmt(self, command):
"""Execute a cmt command in the cmt user area pointed to by the
application. Will execute the command "cmt <command>" after the
proper configuration. Do not include the word "cmt" yourself."""
execute('cmt %s' % command,
shell=True,
timeout=None,
env=self.getenv(False),
cwd=self.user_release_area)
def make(self, argument=''):
"""Build the code in the release area the application object points
to. The actual command executed is "cmt broadcast make <argument>"
after the proper configuration has taken place."""
from Ganga.Utility.Config import getConfig
config = getConfig('GAUDI')
execute('cmt broadcast %s %s' % (config['make_cmd'], argument),
shell=True,
timeout=None,
env=self.getenv(False),
cwd=self.user_release_area)
def make(self, argument=''):
"""Build the code in the release area the application object points
to. The actual command executed is "cmt broadcast make <argument>"
after the proper configuration has taken place."""
from Ganga.Utility.Config import getConfig
config = getConfig('GAUDI')
execute('cmt broadcast %s %s' % (config['make_cmd'], argument),
shell=True,
timeout=None,
env=self.getenv(False),
cwd=self.user_release_area)
def __init__(self, setup=None, setup_args=[]):
"""
THIS EXPECTS THE BASH SHELL TO AT LEAST BE AVAILABLE TO RUN THESE COMMANDS!
The setup script is sourced (with possible arguments) and the
environment is captured. The environment variables are expanded
automatically (this is a fix for bug #44259: GangaLHCb tests fail due to
gridProxy check).
Example of variable expansion:
os.environ['BAR'] = 'rabarbar'
os.environ['FOO'] = '$BAR'
s = Shell() # with or without the setup script
assert s.env['FOO'] == 'rabarbar' # NOT literal string '$BAR'
NOTE: the behaviour is not 100% bash compatible: undefined variables in
bash are empty strings, Shell() leaves the literals unchanged,so:
os.environ['FOO'] = '$NO_BAR'
s = Shell()
if 'NO_BAR' not in os.environ:
assert s.env['FOO'] == '$NO_BAR'
will store an env from:
source setup setup_args[0] setup_args[1]
e.g.
source . && myCmd.sh someoption
source = '.'
source_args = ['&&', 'myCmd.sh', 'someoption']
Args:
setup (str): typically a file or '.' being sourced in bash
setup_args (list): list of strings which are executed directly with a ' ' character spacing
"""
if setup is not None:
self.env = dict(os.environ)
execute('source {0} {1}'.format(setup, " ".join(setup_args)),
shell=True,
env=self.env,
update_env=True)
else:
# bug #44334: Ganga/Utility/Shell.py does not save environ
env = dict(os.environ)
self.env = expand_vars(env)
self.dirname = None
def test_execute_timeouts():
''' This tests that the timeouts on scripts cause them to stop as expected '''
# Test timeouts
assert execute('while true; do sleep 1; done', shell=True, timeout=1) == 'Command timed out!'
assert execute('while True: pass', timeout=1, shell=False) == 'Command timed out!'
# Test timeout doesn't hinder a normal command
assert execute('cd "{0}"; pwd'.format(os.getcwd()), shell=True, timeout=10, cwd=os.getcwd()).strip() == os.getcwd()
assert timeit.timeit(
'''
from Ganga.Utility.execute import execute
execute('cd "{0}"; pwd', shell=True, timeout=10)
'''.format(os.getcwd()), number=1) < 11
def __init__(self, setup=None, setup_args=[]):
"""
THIS EXPECTS THE BASH SHELL TO AT LEAST BE AVAILABLE TO RUN THESE COMMANDS!
The setup script is sourced (with possible arguments) and the
environment is captured. The environment variables are expanded
automatically (this is a fix for bug #44259: GangaLHCb tests fail due to
gridProxy check).
Example of variable expansion:
os.environ['BAR'] = 'rabarbar'
os.environ['FOO'] = '$BAR'
s = Shell() # with or without the setup script
assert s.env['FOO'] == 'rabarbar' # NOT literal string '$BAR'
NOTE: the behaviour is not 100% bash compatible: undefined variables in
bash are empty strings, Shell() leaves the literals unchanged,so:
os.environ['FOO'] = '$NO_BAR'
s = Shell()
if 'NO_BAR' not in os.environ:
assert s.env['FOO'] == '$NO_BAR'
will store an env from:
source setup setup_args[0] setup_args[1]
e.g.
source . && myCmd.sh someoption
source = '.'
source_args = ['&&', 'myCmd.sh', 'someoption']
Args:
setup (str): typically a file or '.' being sourced in bash
setup_args (list): list of strings which are executed directly with a ' ' character spacing
"""
if setup is not None:
self.env = dict(os.environ)
execute('source {0} {1}'.format(setup," ".join(setup_args)), shell=True, env=self.env, update_env=True)
else:
# bug #44334: Ganga/Utility/Shell.py does not save environ
env = dict(os.environ)
self.env = expand_vars(env)
self.dirname = None
def _store_dirac_environment():
from GangaDirac.Lib.Utilities.DiracUtilities import write_env_cache
platform = os.environ['CMTOPT']
diracversion = Ganga.Utility.Config.getConfig('LHCb')['LHCbDiracVersion']
fdir = os.path.join(os.path.expanduser("~/.cache/Ganga/GangaLHCb"), platform)
fname = os.path.join(fdir, diracversion)
if not os.path.exists(fname) or not os.path.getsize(fname):
cmd = 'lb-run LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format(version=diracversion)
env = execute(cmd)
if isinstance(env, str):
try:
env_temp = eval(env)
env = env_temp
except SyntaxError:
logger.error("LHCbDirac version {version} does not exist".format(version=diracversion))
raise OptionValueError("LHCbDirac version {version} does not exist".format(version=diracversion))
try:
write_env_cache(env, fname)
logger.info("Storing new LHCbDirac environment (%s:%s)" % (str(diracversion), str(platform)))
except (OSError, IOError, TypeError):
logger.error("Unable to store LHCbDirac environment")
raise
logger.info("Using LHCbDirac version %s", diracversion)
os.environ['GANGADIRACENVIRONMENT'] = fname
def test_execute_timeouts():
''' This tests the timeouts cause the scripts to fail as expected '''
# Test timeouts
assert execute('while true; do sleep 1; done', shell=True, timeout=1) == 'Command timed out!'
assert execute('while True: pass', timeout=1, shell=False) == 'Command timed out!'
# Test timeout doesn't hinder a normal command
assert execute('import os\noutput(os.getcwd())', timeout=10, cwd=os.getcwd(), shell=False).strip() == os.getcwd()
# Test timeout doesn't delay normal command
assert timeit.timeit(
"""
import os
from Ganga.Utility.execute import execute
execute('import os\\noutput(os.getcwd())',timeout=10, cwd=os.getcwd(), shell=False)
""", number=1) < 11
def test_execute_output():
''' This tests the abilty to send complex objects back through the output stream to Ganga '''
# Test pkl pipe output is interpreted properly
d = execute('import datetime\noutput(datetime.datetime(2013,12,12))', shell=False)
assert hasattr(d, 'month')
assert d.month == 12
def store_dirac_environment():
"""Store the LHCbDIRAC environment in a cache file."""
# platform_env_var = 'CMTCONFIG'
# try:
# platform = os.environ[platform_env_var]
# except KeyError:
# logger.error("Environment variable %s is missing. Can't cache LHCbDIRAC environment.", platform_env_var)
# raise PluginError
#While LHCbDirac is only available for gcc49 we shall unfortunately hard-code the platform.
platform = 'x86_64-slc6-gcc49-opt'
wildcard = Ganga.Utility.Config.getConfig('LHCb')['LHCbDiracVersion']
diracversion = select_dirac_version(wildcard)
fdir = join(expanduser("~/.cache/Ganga/GangaLHCb"), platform)
fname = join(fdir, diracversion)
if not exists(fname) or not getsize(fname):
cmd = 'lb-run -c x86_64-slc6-gcc49-opt LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format(version=diracversion)
env = execute(cmd)
if isinstance(env, str):
try:
env_temp = eval(env)
env = env_temp
except SyntaxError:
logger.error("LHCbDirac version %s does not exist", diracversion)
raise OptionValueError("LHCbDirac version {version} does not exist".format(version=diracversion))
try:
write_env_cache(env, fname)
logger.info("Storing new LHCbDirac environment (%s:%s)", str(diracversion), str(platform))
except (OSError, IOError, TypeError):
logger.error("Unable to store LHCbDirac environment")
raise PluginError
logger.info("Using LHCbDirac version %s", diracversion)
os.environ['GANGADIRACENVIRONMENT'] = fname
def _store_dirac_environment():
from GangaDirac.Lib.Utilities.DiracUtilities import write_env_cache
platform = os.environ['CMTOPT']
diracversion = Ganga.Utility.Config.getConfig('LHCb')['LHCbDiracVersion']
fdir = os.path.join(os.path.expanduser("~/.cache/Ganga/GangaLHCb"),
platform)
fname = os.path.join(fdir, diracversion)
if not os.path.exists(fname) or not os.path.getsize(fname):
cmd = 'lb-run LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format(
version=diracversion)
env = execute(cmd)
if isinstance(env, str):
try:
env_temp = eval(env)
env = env_temp
except SyntaxError:
logger.error(
"LHCbDirac version {version} does not exist".format(
version=diracversion))
raise OptionValueError(
"LHCbDirac version {version} does not exist".format(
version=diracversion))
try:
write_env_cache(env, fname)
logger.info("Storing new LHCbDirac environment (%s:%s)" %
(str(diracversion), str(platform)))
except (OSError, IOError, TypeError):
logger.error("Unable to store LHCbDirac environment")
raise
logger.info("Using LHCbDirac version %s", diracversion)
os.environ['GANGADIRACENVIRONMENT'] = fname
def test_execute_output():
''' This tests the abilty to send complex objects back through the output stream to Ganga '''
# Test pkl pipe output is interpreted properly
d = execute('import datetime\noutput(datetime.datetime(2013,12,12))',
shell=False)
assert hasattr(d, 'month')
assert d.month == 12
def get_env(env_source):
"""
Given a source command, return the DIRAC environment that the
command created.
Args:
env_source: a command which can be sourced, providing the desired environment
Returns:
dict: the environment
"""
logger.debug('Running DIRAC source command %s', env_source)
env = dict(os.environ)
execute('source {0}'.format(env_source), shell=True, env=env, update_env=True)
if not any(key.startswith('DIRAC') for key in env):
raise RuntimeError("'DIRAC*' not found in environment")
return env
def test_execute_timeouts():
''' This tests that the timeouts on scripts cause them to stop as expected '''
# Test timeouts
assert execute('while true; do sleep 1; done', shell=True,
timeout=1) == 'Command timed out!'
assert execute('while True: pass', timeout=1,
shell=False) == 'Command timed out!'
# Test timeout doesn't hinder a normal command
assert execute('cd "{0}"; pwd'.format(os.getcwd()),
shell=True,
timeout=10,
cwd=os.getcwd()).strip() == os.getcwd()
assert timeit.timeit('''
from Ganga.Utility.execute import execute
execute('cd "{0}"; pwd', shell=True, timeout=10)
'''.format(os.getcwd()),
number=1) < 11
def get_env(env_source):
"""
Given a source command, return the DIRAC environment that the
command created.
Args:
env_source: a command which can be sourced, providing the desired environment
Returns:
dict: the environment
"""
logger.debug('Running DIRAC source command %s', env_source)
env = dict(os.environ)
execute('source {0}'.format(env_source),
shell=True,
env=env,
update_env=True)
if not any(key.startswith('DIRAC') for key in env):
raise RuntimeError("'DIRAC*' not found in environment")
return env
def test_execute_timeouts():
''' This tests the timeouts cause the scripts to fail as expected '''
# Test timeouts
assert execute('while true; do sleep 1; done', shell=True,
timeout=1) == 'Command timed out!'
assert execute('while True: pass', timeout=1,
shell=False) == 'Command timed out!'
# Test timeout doesn't hinder a normal command
assert execute('import os\noutput(os.getcwd())',
timeout=10,
cwd=os.getcwd(),
shell=False).strip() == os.getcwd()
# Test timeout doesn't delay normal command
assert timeit.timeit("""
import os
from Ganga.Utility.execute import execute
execute('import os\\noutput(os.getcwd())',timeout=10, cwd=os.getcwd(), shell=False)
""",
number=1) < 11
def _store_dirac_environment():
from GangaDirac.Lib.Utilities.DiracUtilities import write_env_cache, get_env
diracversion = _guess_version('LHCBDIRAC')
platform = os.environ['CMTOPT']
fdir = os.path.join(os.path.expanduser("~/.cache/Ganga/GangaLHCb"), platform)
fname = os.path.join(fdir, diracversion)
if not os.path.exists(fname) or not os.path.getsize(fname):
logger.info("Storing new LHCbDirac environment (%s:%s)" % (str(diracversion), str(platform)))
cmd = 'lb-run LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format(version=diracversion)
env = execute(cmd) # grab the stdout text
env = eval(env) # env is a string so convert it to a dict
write_env_cache(env, fname)
os.environ['GANGADIRACENVIRONMENT'] = fname
def test_execute_env():
''' This tests that the scripts can update an env based upon input and handle an appropriate python_setup sent in execute args '''
# Test correctly picking up env
env = {'ALEX': '/hello/world',
'PATH': os.environ.get('PATH', ''),
'PYTHONPATH': os.environ.get('PYTHONPATH', '')}
# env.update(os.environ)
assert execute('echo $ALEX', shell=True, env=env).strip() == '/hello/world'
assert execute('import os\noutput(os.environ.get("ALEX","BROKEN"))', env=env, python_setup='#', shell=False).strip() == '/hello/world'
# Test env not updated by default
execute('export NEWTEST=/new/test', shell=True, env=env)
assert 'NEWTEST' not in env
execute('import os\nos.environ["NEWTEST"]="/new/test"', env=env, python_setup='#')
assert 'NEWTEST' not in env
# Test updating of env
execute('export NEWTEST=/new/test', shell=True, env=env, update_env=True)
assert 'NEWTEST' in env
del env['NEWTEST']
assert 'NEWTEST' not in env
execute('import os\nos.environ["NEWTEST"]="/new/test"', env=env, python_setup='#', update_env=True, shell=False)
assert 'NEWTEST' in env
def test_execute_output():
''' This tests the various levels when an import can occur and that complex objects can be returned via stdout '''
# Test printout of pickle dump interpreted correctly
d = execute('import pickle, datetime\nprint(pickle.dumps(datetime.datetime(2013,12,12)))', shell=False)
assert hasattr(d, 'month')
assert d.month == 12
# Test straight printout doesn't work without includes, stdout as str
# returned by default
d1 = execute('import datetime\nprint(datetime.datetime(2013,12,12))', shell=False)
d2 = execute('import datetime\nrepr(datetime.datetime(2013,12,12))', shell=False)
assert not hasattr(d1, 'month')
assert isinstance(d1, str)
assert not hasattr(d2, 'month')
assert isinstance(d2, str)
# Test printout works with the right includes
d1 = execute('print(datetime.datetime(2013,12,12))', python_setup='import datetime', eval_includes='import datetime', shell=False)
d2 = execute('print(repr(datetime.datetime(2013,12,12)))', python_setup='import datetime', eval_includes='import datetime', shell=False)
assert not hasattr(d1, 'month')
assert isinstance(d1, str)
assert hasattr(d2, 'month')
assert d2.month == 12
def execute(command,
timeout=getConfig('DIRAC')['Timeout'],
env=None,
cwd=None,
shell=False,
python_setup='',
eval_includes=None,
update_env=False):
"""
Execute a command on the local DIRAC server.
This function blocks until the server returns.
"""
if env is None:
env = getDiracEnv()
if python_setup == '':
python_setup = getDiracCommandIncludes()
_checkProxy()
#logger.debug("Executing command:\n'%s'" % str(command))
#logger.debug("python_setup:\n'%s'" % str(python_setup))
#logger.debug("eval_includes:\n'%s'" % str(eval_includes))
returnable = gexecute.execute(command,
timeout=timeout,
env=env,
cwd=cwd,
shell=shell,
python_setup=python_setup,
eval_includes=eval_includes,
update_env=update_env)
# rcurrie I've seen problems with just returning this raw object,
# expanding it to be sure that an instance remains in memory
myObject = {}
if hasattr(returnable, 'keys'):
# Expand object(s) in dictionaries
myObject = _expand_object(returnable)
elif type(returnable) == type([]):
# Expand object(s) in lists
myObject = _expand_list(returnable)
else:
# Copy object(s) so thet they definately are in memory
myObject = copy.deepcopy(returnable)
return myObject
def execute(command,
timeout=getConfig('DIRAC')['Timeout'],
env=None,
cwd=None,
shell=False,
python_setup='',
eval_includes=None,
update_env=False):
"""
Execute a command on the local DIRAC server.
This function blocks until the server returns.
"""
if env is None:
env = getDiracEnv()
if python_setup == '':
python_setup = getDiracCommandIncludes()
_checkProxy()
#logger.debug("Executing command:\n'%s'" % str(command))
#logger.debug("python_setup:\n'%s'" % str(python_setup))
#logger.debug("eval_includes:\n'%s'" % str(eval_includes))
returnable = gexecute.execute(command,
timeout=timeout,
env=env,
cwd=cwd,
shell=shell,
python_setup=python_setup,
eval_includes=eval_includes,
update_env=update_env)
# rcurrie I've seen problems with just returning this raw object,
# expanding it to be sure that an instance remains in memory
myObject = {}
if hasattr(returnable, 'keys'):
# Expand object(s) in dictionaries
myObject = _expand_object(returnable)
elif type(returnable) in [list, tuple]:
# Expand object(s) in lists
myObject = _expand_list(returnable)
else:
# Copy object(s) so thet they definately are in memory
myObject = copy.deepcopy(returnable)
return myObject
def execute(command,
timeout=getConfig('DIRAC')['Timeout'],
env=None,
cwd=None,
shell=False,
python_setup='',
eval_includes=None,
update_env=False):
"""
Execute a command on the local DIRAC server.
This function blocks until the server returns.
Args:
command (str): This is the command we're running within our DIRAC session
timeout (bool): This is the length of time that a DIRAC call has before it's decided some interaction has timed out
env (dict): an optional environment to execute the DIRAC code in
cwd (str): an optional string to a valid path where this code should be executed
shell (bool): Should this code be executed in a new shell environment
python_setup (str): Optional extra code to pass to python when executing
eval_incldes (???): TODO document me
update_env (bool): Should this modify the given env object with the env after the command has executed
"""
if env is None:
env = getDiracEnv()
if python_setup == '':
python_setup = getDiracCommandIncludes()
_checkProxy()
#logger.info("Executing command:\n'%s'" % str(command))
#logger.debug("python_setup:\n'%s'" % str(python_setup))
#logger.debug("eval_includes:\n'%s'" % str(eval_includes))
returnable = gexecute.execute(command,
timeout=timeout,
env=env,
cwd=cwd,
shell=shell,
python_setup=python_setup,
eval_includes=eval_includes,
update_env=update_env)
return deepcopy(returnable)
def select_dirac_version(wildcard):
"""
Find the LHCbDIRAC version that should be used based on the confuguration
system. Wildcards can be used and soflinks are dereferenced.
"""
cmd = 'lb-run -c x86_64-slc6-gcc49-opt -l LHCbDIRAC'
out = execute(cmd)
if out == '':
raise PluginError("Can't find any LHCbDirac versions from '%s'" % cmd)
versions = [s.split() for s in out.splitlines() if fnmatch(s.split()[0], wildcard)]
if len(versions) == 0:
raise PluginError("Can't find LHCbDIRAC version matching %s.", wildcard)
versions.sort(key=lambda v: v[0])
version = versions[-1]
dereferenced_version = basename(realpath(version[2]))[10:]
return dereferenced_version
def select_dirac_version(wildcard):
"""
Find the LHCbDIRAC version that should be used based on the confuguration
system. Wildcards can be used and soflinks are dereferenced.
"""
cmd = 'lb-run -c x86_64-slc6-gcc49-opt -l LHCbDIRAC'
out = execute(cmd)
if out == '':
raise PluginError("Can't find any LHCbDirac versions from '%s'" % cmd)
versions = [
s.split() for s in out.splitlines() if fnmatch(s.split()[0], wildcard)
]
if len(versions) == 0:
raise PluginError("Can't find LHCbDIRAC version matching %s.",
wildcard)
versions.sort(key=lambda v: v[0])
version = versions[-1]
dereferenced_version = basename(realpath(version[2]))[10:]
return dereferenced_version
def store_dirac_environment():
"""Store the LHCbDIRAC environment in a cache file."""
# platform_env_var = 'CMTCONFIG'
# try:
# platform = os.environ[platform_env_var]
# except KeyError:
# logger.error("Environment variable %s is missing. Can't cache LHCbDIRAC environment.", platform_env_var)
# raise PluginError
#While LHCbDirac is only available for gcc49 we shall unfortunately hard-code the platform.
platform = 'x86_64-slc6-gcc49-opt'
wildcard = Ganga.Utility.Config.getConfig('LHCb')['LHCbDiracVersion']
diracversion = select_dirac_version(wildcard)
fdir = join(expanduser("~/.cache/Ganga/GangaLHCb"), platform)
fname = join(fdir, diracversion)
if not exists(fname) or not getsize(fname):
cmd = 'lb-run -c x86_64-slc6-gcc49-opt LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format(
version=diracversion)
env = execute(cmd)
if isinstance(env, str):
try:
env_temp = eval(env)
env = env_temp
except SyntaxError:
logger.error("LHCbDirac version %s does not exist",
diracversion)
raise OptionValueError(
"LHCbDirac version {version} does not exist".format(
version=diracversion))
try:
write_env_cache(env, fname)
logger.info("Storing new LHCbDirac environment (%s:%s)",
str(diracversion), str(platform))
except (OSError, IOError, TypeError):
logger.error("Unable to store LHCbDirac environment")
raise PluginError
logger.info("Using LHCbDirac version %s", diracversion)
os.environ['GANGADIRACENVIRONMENT'] = fname
def test_output():
"""Test that ``output`` works as expected"""
assert execute('output("foo")', shell=False).strip() == 'foo'
def test_execute():
''' This tests that the script correctly changes the dir as requested via the script '''
assert execute('cd "{0}"; pwd'.format(os.getcwd()),
shell=True,
cwd=os.getcwd()).strip() == os.getcwd()
def test_execute_cwd():
''' This tests that the cwd of the script is changed as per the arguments to execute '''
# Test changing dir
assert execute('import os\noutput(os.getcwd())', cwd='/', shell=False).strip() == '/'
def test_print():
"""Test that ``print`` works as expected"""
assert execute('print("foo")', shell=False).strip() == 'foo'
def test_execute():
''' This tests that the cwd returned from the output is what is requested in the script '''
# Test shell vs python mode
assert execute('import os\noutput(os.getcwd())', cwd=os.getcwd(), shell=False).strip() == os.getcwd()
def test_output():
"""Test that ``output`` works as expected"""
assert execute('output("foo")', shell=False).strip() == 'foo'
def execute(command,
timeout=getConfig('DIRAC')['Timeout'],
env=None,
cwd=None,
shell=False,
python_setup='',
eval_includes=None,
update_env=False,
):
"""
Execute a command on the local DIRAC server.
This function blocks until the server returns.
Args:
command (str): This is the command we're running within our DIRAC session
timeout (int): This is the length of time that a DIRAC call has before it's decided some interaction has timed out
env (dict): an optional environment to execute the DIRAC code in
cwd (str): an optional string to a valid path where this code should be executed
shell (bool): Should this code be executed in a new shell environment
python_setup (str): Optional extra code to pass to python when executing
eval_includes (???): TODO document me
update_env (bool): Should this modify the given env object with the env after the command has executed
"""
if env is None:
env = getDiracEnv()
if python_setup == '':
python_setup = getDiracCommandIncludes()
# We're about to perform an expensive operation so being safe before we run it shouldn't cost too much
_checkProxy(force = True)
#logger.debug("Executing command:\n'%s'" % str(command))
#logger.debug("python_setup:\n'%s'" % str(python_setup))
#logger.debug("eval_includes:\n'%s'" % str(eval_includes))
if cwd is None:
# We can in all likelyhood be in a temp folder on a shared (SLOW) filesystem
# If we are we do NOT want to execute commands which will involve any I/O on the system that isn't needed
cwd_ = tempfile.mkdtemp()
else:
# We know were whe want to run, lets just run there
cwd_ = cwd
global last_modified_valid
if not last_modified_valid:
return None
returnable = gexecute.execute(command,
timeout=timeout,
env=env,
cwd=cwd_,
shell=shell,
python_setup=python_setup,
eval_includes=eval_includes,
update_env=update_env)
# TODO we would like some way of working out if the code has been executed correctly
# Most commands will be OK now that we've added the check for the valid proxy before executing commands here
if cwd is None:
shutil.rmtree(cwd_, ignore_errors=True)
return returnable
def test_execute_cwd():
''' This tests that the cwd of the script is changed as per the arguments to execute '''
# Test changing dir
assert execute('import os\noutput(os.getcwd())', cwd='/',
shell=False).strip() == '/'
def execute(command,
timeout=getConfig('DIRAC')['Timeout'],
env=None,
cwd=None,
shell=False,
python_setup='',
eval_includes=None,
update_env=False,
return_raw_dict=False,
cred_req=None,
):
"""
Execute a command on the local DIRAC server.
This function blocks until the server returns.
Args:
command (str): This is the command we're running within our DIRAC session
timeout (int): This is the length of time that a DIRAC call has before it's decided some interaction has timed out
env (dict): an optional environment to execute the DIRAC code in
cwd (str): an optional string to a valid path where this code should be executed
shell (bool): Should this code be executed in a new shell environment
python_setup (str): Optional extra code to pass to python when executing
eval_includes (???): TODO document me
update_env (bool): Should this modify the given env object with the env after the command has executed
return_raw_dict(bool): Should we return the raw dict from the DIRAC interface or parse it here
cred_req (ICredentialRequirement): What credentials does this call need
"""
if env is None:
if cred_req is None:
env = getDiracEnv()
else:
env = getDiracEnv(cred_req.dirac_env)
if python_setup == '':
python_setup = getDiracCommandIncludes()
if cred_req is not None:
env['X509_USER_PROXY'] = credential_store[cred_req].location
if cwd is None:
# We can in all likelyhood be in a temp folder on a shared (SLOW) filesystem
# If we are we do NOT want to execute commands which will involve any I/O on the system that isn't needed
cwd_ = tempfile.mkdtemp()
else:
# We know were whe want to run, lets just run there
cwd_ = cwd
returnable = gexecute.execute(command,
timeout=timeout,
env=env,
cwd=cwd_,
shell=shell,
python_setup=python_setup,
eval_includes=eval_includes,
update_env=update_env)
# If the time
if returnable == 'Command timed out!':
raise GangaDiracError("DIRAC command timed out")
# TODO we would like some way of working out if the code has been executed correctly
# Most commands will be OK now that we've added the check for the valid proxy before executing commands here
if cwd is None:
shutil.rmtree(cwd_, ignore_errors=True)
if isinstance(returnable, dict) and not return_raw_dict:
# If the output is a dictionary allow for automatic error detection
if returnable['OK']:
return returnable['Value']
else:
raise GangaDiracError(returnable['Message'])
elif isinstance(returnable, dict):
# If the output is a dictionary return and it has been requested, then return it
return returnable
else:
# Else raise an exception as it should be a dictionary
raise GangaDiracError(returnable)
def test_execute():
''' This tests that the cwd returned from the output is what is requested in the script '''
# Test shell vs python mode
assert execute('import os\noutput(os.getcwd())',
cwd=os.getcwd(),
shell=False).strip() == os.getcwd()
def test_execute_cwd():
''' This tests that teh cwd is set correctly as requeted in the arguments '''
assert execute('pwd', shell=True, cwd='/').strip() == '/'
def test_execute():
''' This tests that the script correctly changes the dir as requested via the script '''
assert execute('cd "{0}"; pwd'.format(os.getcwd()), shell=True, cwd=os.getcwd()).strip() == os.getcwd()
def execute(
command,
timeout=getConfig('DIRAC')['Timeout'],
env=None,
cwd=None,
shell=False,
python_setup='',
eval_includes=None,
update_env=False,
):
"""
Execute a command on the local DIRAC server.
This function blocks until the server returns.
Args:
command (str): This is the command we're running within our DIRAC session
timeout (int): This is the length of time that a DIRAC call has before it's decided some interaction has timed out
env (dict): an optional environment to execute the DIRAC code in
cwd (str): an optional string to a valid path where this code should be executed
shell (bool): Should this code be executed in a new shell environment
python_setup (str): Optional extra code to pass to python when executing
eval_includes (???): TODO document me
update_env (bool): Should this modify the given env object with the env after the command has executed
"""
if env is None:
env = getDiracEnv()
if python_setup == '':
python_setup = getDiracCommandIncludes()
# We're about to perform an expensive operation so being safe before we run it shouldn't cost too much
_checkProxy(force=True)
#logger.debug("Executing command:\n'%s'" % str(command))
#logger.debug("python_setup:\n'%s'" % str(python_setup))
#logger.debug("eval_includes:\n'%s'" % str(eval_includes))
if cwd is None:
# We can in all likelyhood be in a temp folder on a shared (SLOW) filesystem
# If we are we do NOT want to execute commands which will involve any I/O on the system that isn't needed
cwd_ = tempfile.mkdtemp()
else:
# We know were whe want to run, lets just run there
cwd_ = cwd
global last_modified_valid
if not last_modified_valid:
return None
returnable = gexecute.execute(command,
timeout=timeout,
env=env,
cwd=cwd_,
shell=shell,
python_setup=python_setup,
eval_includes=eval_includes,
update_env=update_env)
# TODO we would like some way of working out if the code has been executed correctly
# Most commands will be OK now that we've added the check for the valid proxy before executing commands here
if cwd is None:
shutil.rmtree(cwd_, ignore_errors=True)
return returnable
def test_execute_cwd():
''' This tests that teh cwd is set correctly as requeted in the arguments '''
assert execute('pwd', shell=True, cwd='/').strip() == '/'
def __worker_thread(self, thread):
"""
Code run by worker threads to allow parallelism in Ganga.
Can be used for executing non-blocking calls to local DIRAC server
"""
# Occasionally when shutting down the Queue import at top has been garbage collected
# and line "except Queue.Empty: continue" will throw
# <type 'exceptions.AttributeError'>: 'NoneType' object has no attribute 'Empty'
# im hoping that importing within the thread will avoid this.
import Queue
oldname = thread.gangaName
# Note can use threading.current_thread to get the thread rather than passing it as an arg
# easier to unit test this way though with a dummy thread.
while not thread.should_stop():
try:
item = self.__queue.get(True, 0.05)
except Queue.Empty:
# wait 0.05 sec then loop again to give shutdown a chance
continue
# regster as a working thread
if isinstance(item, QueueElement):
oldname = thread.gangaName
thread.gangaName = item.name
thread.register()
if not isinstance(item, QueueElement):
logger.error("Unrecognised queue element: '%s'" % repr(item))
logger.error(" expected: 'QueueElement'")
self.__queue.task_done()
thread.unregister()
continue
if isinstance(item.command_input, FunctionInput):
thread._command = item.command_input.function.__name__
elif isinstance(item.command_input, CommandInput):
thread._command = item.command_input.command
thread._timeout = item.command_input.timeout
else:
logger.error("Unrecognised input command type: '%s'" % repr(item.command_input))
logger.error(" expected: ('FunctionInput' or 'CommandInput')")
self.__queue.task_done()
thread.unregister()
continue
try:
if isinstance(item.command_input, FunctionInput):
these_args = item.command_input.args
if isinstance(these_args, str):
these_args = (these_args, )
result = item.command_input.function(*these_args, **item.command_input.kwargs)
else:
result = execute(*item.command_input)
except Exception as e:
logger.error("Exception raised executing '%s' in Thread '%s':\n%s" % (thread._command, thread.gangaName, traceback.format_exc()))
if item.fallback_func.function is not None:
if isinstance(item.fallback_func, FunctionInput):
thread._command = item.fallback_func.function.__name__
thread._timeout = 'N/A'
try:
item.fallback_func.function(e, *item.fallback_func.args, **item.fallback_func.kwargs)
except Exception as x:
logger.error("Exception raised in fallback function '%s' of Thread '%s':\n%s" % (thread._command, thread.gangaName, traceback.format_exc()))
else:
logger.error("Unrecognised fallback_func type: '%s'" % repr(item.fallback_func))
logger.error(" expected: 'FunctionInput'")
else:
if item.callback_func.function is not None:
if isinstance(item.callback_func, FunctionInput):
thread._command = item.callback_func.function.__name__
thread._timeout = 'N/A'
try:
item.callback_func.function(
result, *item.callback_func.args, **item.callback_func.kwargs)
except Exception as e:
logger.error("Exception raised in callback_func '%s' of Thread '%s': %s" % (
thread._command, thread.gangaName, traceback.format_exc()))
else:
logger.error("Unrecognised callback_func type: '%s'" % repr(item.callback_func))
logger.error(" expected: 'FunctionInput'")
finally:
# unregister as a working thread bcoz free
thread._command = 'idle'
thread._timeout = 'N/A'
self.__queue.task_done()
thread.unregister()
thread.gangaName = oldname
def test_print():
"""Test that ``print`` works as expected"""
assert execute('print("foo")', shell=False).strip() == 'foo'
def __worker_thread(self, thread):
"""
Code run by worker threads to allow parallelism in Ganga.
Can be used for executing non-blocking calls to local DIRAC server
"""
# Occasionally when shutting down the Queue import at top has been garbage collected
# and line "except Queue.Empty: continue" will throw
# <type 'exceptions.AttributeError'>: 'NoneType' object has no attribute 'Empty'
# im hoping that importing within the thread will avoid this.
import Queue
oldname = thread.gangaName
# Note can use threading.current_thread to get the thread rather than passing it as an arg
# easier to unit test this way though with a dummy thread.
while not thread.should_stop():
try:
item = self.__queue.get(True, 0.05)
except Queue.Empty:
# wait 0.05 sec then loop again to give shutdown a chance
continue
# regster as a working thread
if isinstance(item, QueueElement):
oldname = thread.gangaName
thread.gangaName = item.name
thread.register()
if not isinstance(item, QueueElement):
logger.error("Unrecognised queue element: '%s'" % repr(item))
logger.error(" expected: 'QueueElement'")
self.__queue.task_done()
thread.unregister()
continue
if isinstance(item.command_input, FunctionInput):
thread._command = getName(item.command_input.function)
elif isinstance(item.command_input, CommandInput):
thread._command = item.command_input.command
thread._timeout = item.command_input.timeout
else:
logger.error("Unrecognised input command type: '%s'" %
repr(item.command_input))
logger.error(
" expected: ('FunctionInput' or 'CommandInput')"
)
self.__queue.task_done()
thread.unregister()
continue
try:
if isinstance(item.command_input, FunctionInput):
these_args = item.command_input.args
if isinstance(these_args, str):
these_args = (these_args, )
result = item.command_input.function(
*these_args, **item.command_input.kwargs)
else:
result = execute(*item.command_input)
except Exception as e:
logger.error(
"Exception raised executing '%s' in Thread '%s':\n%s" %
(thread._command, thread.gangaName,
traceback.format_exc()))
if item.fallback_func.function is not None:
if isinstance(item.fallback_func, FunctionInput):
thread._command = getName(item.fallback_func.function)
thread._timeout = 'N/A'
try:
item.fallback_func.function(
e, *item.fallback_func.args,
**item.fallback_func.kwargs)
except Exception as x:
logger.error(
"Exception raised in fallback function '%s' of Thread '%s':\n%s"
% (thread._command, thread.gangaName,
traceback.format_exc()))
else:
logger.error("Unrecognised fallback_func type: '%s'" %
repr(item.fallback_func))
logger.error(
" expected: 'FunctionInput'")
else:
if item.callback_func.function is not None:
if isinstance(item.callback_func, FunctionInput):
thread._command = getName(item.callback_func.function)
thread._timeout = 'N/A'
try:
item.callback_func.function(
result, *item.callback_func.args,
**item.callback_func.kwargs)
except Exception as e:
logger.error(
"Exception raised in callback_func '%s' of Thread '%s': %s"
% (thread._command, thread.gangaName,
traceback.format_exc()))
else:
logger.error("Unrecognised callback_func type: '%s'" %
repr(item.callback_func))
logger.error(
" expected: 'FunctionInput'")
finally:
# unregister as a working thread bcoz free
thread._command = 'idle'
thread._timeout = 'N/A'
self.__queue.task_done()
thread.unregister()
thread.gangaName = oldname
