def test_execute_env(): ''' This tests that the scripts can update an env based upon input and handle an appropriate python_setup sent in execute args ''' # Test correctly picking up env env = { 'ALEX': '/hello/world', 'PATH': os.environ.get('PATH', ''), 'PYTHONPATH': os.environ.get('PYTHONPATH', '') } # env.update(os.environ) assert execute('echo $ALEX', shell=True, env=env).strip() == '/hello/world' assert execute('import os\noutput(os.environ.get("ALEX","BROKEN"))', env=env, python_setup='#', shell=False).strip() == '/hello/world' # Test env not updated by default execute('export NEWTEST=/new/test', shell=True, env=env) assert 'NEWTEST' not in env execute('import os\nos.environ["NEWTEST"]="/new/test"', env=env, python_setup='#') assert 'NEWTEST' not in env # Test updating of env execute('export NEWTEST=/new/test', shell=True, env=env, update_env=True) assert 'NEWTEST' in env del env['NEWTEST'] assert 'NEWTEST' not in env execute('import os\nos.environ["NEWTEST"]="/new/test"', env=env, python_setup='#', update_env=True, shell=False) assert 'NEWTEST' in env
def test_execute_output(): ''' This tests the various levels when an import can occur and that complex objects can be returned via stdout ''' # Test printout of pickle dump interpreted correctly d = execute( 'import pickle, datetime\nprint(pickle.dumps(datetime.datetime(2013,12,12)))', shell=False) assert hasattr(d, 'month') assert d.month == 12 # Test straight printout doesn't work without includes, stdout as str # returned by default d1 = execute('import datetime\nprint(datetime.datetime(2013,12,12))', shell=False) d2 = execute('import datetime\nrepr(datetime.datetime(2013,12,12))', shell=False) assert not hasattr(d1, 'month') assert isinstance(d1, str) assert not hasattr(d2, 'month') assert isinstance(d2, str) # Test printout works with the right includes d1 = execute('print(datetime.datetime(2013,12,12))', python_setup='import datetime', eval_includes='import datetime', shell=False) d2 = execute('print(repr(datetime.datetime(2013,12,12)))', python_setup='import datetime', eval_includes='import datetime', shell=False) assert not hasattr(d1, 'month') assert isinstance(d1, str) assert hasattr(d2, 'month') assert d2.month == 12
def get_env(env_source): """ Given a source command, return the DIRAC environment that the command created. Args: env_source: a command which can be sourced, providing the desired environment Returns: dict: the environment """ logger.debug('Running DIRAC source command %s', env_source) env = dict(os.environ) gexecute.execute('source {0}'.format(env_source), shell=True, env=env, update_env=True) if not any(key.startswith('DIRAC') for key in env): fake_dict = {} with open(env_source) as _env: for _line in _env.readlines(): split_val = _line.split('=') if len(split_val) == 2: key = split_val[0] value = split_val[1] fake_dict[key] = value if not any(key.startswith('DIRAC') for key in fake_dict): logger.error("Env: %s" % str(env)) logger.error("Fake: %s" % str(fake_dict)) raise RuntimeError("'DIRAC*' not found in environment") else: return fake_dict return env
def cmt(self, command): """Execute a cmt command in the cmt user area pointed to by the application. Will execute the command "cmt <command>" after the proper configuration. Do not include the word "cmt" yourself.""" execute('cmt %s' % command, shell=True, timeout=None, env=self.getenv(False), cwd=self.user_release_area)
def make(self, argument=''): """Build the code in the release area the application object points to. The actual command executed is "cmt broadcast make <argument>" after the proper configuration has taken place.""" from Ganga.Utility.Config import getConfig config = getConfig('GAUDI') execute('cmt broadcast %s %s' % (config['make_cmd'], argument), shell=True, timeout=None, env=self.getenv(False), cwd=self.user_release_area)
def __init__(self, setup=None, setup_args=[]): """ THIS EXPECTS THE BASH SHELL TO AT LEAST BE AVAILABLE TO RUN THESE COMMANDS! The setup script is sourced (with possible arguments) and the environment is captured. The environment variables are expanded automatically (this is a fix for bug #44259: GangaLHCb tests fail due to gridProxy check). Example of variable expansion: os.environ['BAR'] = 'rabarbar' os.environ['FOO'] = '$BAR' s = Shell() # with or without the setup script assert s.env['FOO'] == 'rabarbar' # NOT literal string '$BAR' NOTE: the behaviour is not 100% bash compatible: undefined variables in bash are empty strings, Shell() leaves the literals unchanged,so: os.environ['FOO'] = '$NO_BAR' s = Shell() if 'NO_BAR' not in os.environ: assert s.env['FOO'] == '$NO_BAR' will store an env from: source setup setup_args[0] setup_args[1] e.g. source . && myCmd.sh someoption source = '.' source_args = ['&&', 'myCmd.sh', 'someoption'] Args: setup (str): typically a file or '.' being sourced in bash setup_args (list): list of strings which are executed directly with a ' ' character spacing """ if setup is not None: self.env = dict(os.environ) execute('source {0} {1}'.format(setup, " ".join(setup_args)), shell=True, env=self.env, update_env=True) else: # bug #44334: Ganga/Utility/Shell.py does not save environ env = dict(os.environ) self.env = expand_vars(env) self.dirname = None
def test_execute_timeouts(): ''' This tests that the timeouts on scripts cause them to stop as expected ''' # Test timeouts assert execute('while true; do sleep 1; done', shell=True, timeout=1) == 'Command timed out!' assert execute('while True: pass', timeout=1, shell=False) == 'Command timed out!' # Test timeout doesn't hinder a normal command assert execute('cd "{0}"; pwd'.format(os.getcwd()), shell=True, timeout=10, cwd=os.getcwd()).strip() == os.getcwd() assert timeit.timeit( ''' from Ganga.Utility.execute import execute execute('cd "{0}"; pwd', shell=True, timeout=10) '''.format(os.getcwd()), number=1) < 11
def __init__(self, setup=None, setup_args=[]): """ THIS EXPECTS THE BASH SHELL TO AT LEAST BE AVAILABLE TO RUN THESE COMMANDS! The setup script is sourced (with possible arguments) and the environment is captured. The environment variables are expanded automatically (this is a fix for bug #44259: GangaLHCb tests fail due to gridProxy check). Example of variable expansion: os.environ['BAR'] = 'rabarbar' os.environ['FOO'] = '$BAR' s = Shell() # with or without the setup script assert s.env['FOO'] == 'rabarbar' # NOT literal string '$BAR' NOTE: the behaviour is not 100% bash compatible: undefined variables in bash are empty strings, Shell() leaves the literals unchanged,so: os.environ['FOO'] = '$NO_BAR' s = Shell() if 'NO_BAR' not in os.environ: assert s.env['FOO'] == '$NO_BAR' will store an env from: source setup setup_args[0] setup_args[1] e.g. source . && myCmd.sh someoption source = '.' source_args = ['&&', 'myCmd.sh', 'someoption'] Args: setup (str): typically a file or '.' being sourced in bash setup_args (list): list of strings which are executed directly with a ' ' character spacing """ if setup is not None: self.env = dict(os.environ) execute('source {0} {1}'.format(setup," ".join(setup_args)), shell=True, env=self.env, update_env=True) else: # bug #44334: Ganga/Utility/Shell.py does not save environ env = dict(os.environ) self.env = expand_vars(env) self.dirname = None
def _store_dirac_environment(): from GangaDirac.Lib.Utilities.DiracUtilities import write_env_cache platform = os.environ['CMTOPT'] diracversion = Ganga.Utility.Config.getConfig('LHCb')['LHCbDiracVersion'] fdir = os.path.join(os.path.expanduser("~/.cache/Ganga/GangaLHCb"), platform) fname = os.path.join(fdir, diracversion) if not os.path.exists(fname) or not os.path.getsize(fname): cmd = 'lb-run LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format(version=diracversion) env = execute(cmd) if isinstance(env, str): try: env_temp = eval(env) env = env_temp except SyntaxError: logger.error("LHCbDirac version {version} does not exist".format(version=diracversion)) raise OptionValueError("LHCbDirac version {version} does not exist".format(version=diracversion)) try: write_env_cache(env, fname) logger.info("Storing new LHCbDirac environment (%s:%s)" % (str(diracversion), str(platform))) except (OSError, IOError, TypeError): logger.error("Unable to store LHCbDirac environment") raise logger.info("Using LHCbDirac version %s", diracversion) os.environ['GANGADIRACENVIRONMENT'] = fname
def test_execute_timeouts(): ''' This tests the timeouts cause the scripts to fail as expected ''' # Test timeouts assert execute('while true; do sleep 1; done', shell=True, timeout=1) == 'Command timed out!' assert execute('while True: pass', timeout=1, shell=False) == 'Command timed out!' # Test timeout doesn't hinder a normal command assert execute('import os\noutput(os.getcwd())', timeout=10, cwd=os.getcwd(), shell=False).strip() == os.getcwd() # Test timeout doesn't delay normal command assert timeit.timeit( """ import os from Ganga.Utility.execute import execute execute('import os\\noutput(os.getcwd())',timeout=10, cwd=os.getcwd(), shell=False) """, number=1) < 11
def test_execute_output(): ''' This tests the abilty to send complex objects back through the output stream to Ganga ''' # Test pkl pipe output is interpreted properly d = execute('import datetime\noutput(datetime.datetime(2013,12,12))', shell=False) assert hasattr(d, 'month') assert d.month == 12
def store_dirac_environment(): """Store the LHCbDIRAC environment in a cache file.""" # platform_env_var = 'CMTCONFIG' # try: # platform = os.environ[platform_env_var] # except KeyError: # logger.error("Environment variable %s is missing. Can't cache LHCbDIRAC environment.", platform_env_var) # raise PluginError #While LHCbDirac is only available for gcc49 we shall unfortunately hard-code the platform. platform = 'x86_64-slc6-gcc49-opt' wildcard = Ganga.Utility.Config.getConfig('LHCb')['LHCbDiracVersion'] diracversion = select_dirac_version(wildcard) fdir = join(expanduser("~/.cache/Ganga/GangaLHCb"), platform) fname = join(fdir, diracversion) if not exists(fname) or not getsize(fname): cmd = 'lb-run -c x86_64-slc6-gcc49-opt LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format(version=diracversion) env = execute(cmd) if isinstance(env, str): try: env_temp = eval(env) env = env_temp except SyntaxError: logger.error("LHCbDirac version %s does not exist", diracversion) raise OptionValueError("LHCbDirac version {version} does not exist".format(version=diracversion)) try: write_env_cache(env, fname) logger.info("Storing new LHCbDirac environment (%s:%s)", str(diracversion), str(platform)) except (OSError, IOError, TypeError): logger.error("Unable to store LHCbDirac environment") raise PluginError logger.info("Using LHCbDirac version %s", diracversion) os.environ['GANGADIRACENVIRONMENT'] = fname
def _store_dirac_environment(): from GangaDirac.Lib.Utilities.DiracUtilities import write_env_cache platform = os.environ['CMTOPT'] diracversion = Ganga.Utility.Config.getConfig('LHCb')['LHCbDiracVersion'] fdir = os.path.join(os.path.expanduser("~/.cache/Ganga/GangaLHCb"), platform) fname = os.path.join(fdir, diracversion) if not os.path.exists(fname) or not os.path.getsize(fname): cmd = 'lb-run LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format( version=diracversion) env = execute(cmd) if isinstance(env, str): try: env_temp = eval(env) env = env_temp except SyntaxError: logger.error( "LHCbDirac version {version} does not exist".format( version=diracversion)) raise OptionValueError( "LHCbDirac version {version} does not exist".format( version=diracversion)) try: write_env_cache(env, fname) logger.info("Storing new LHCbDirac environment (%s:%s)" % (str(diracversion), str(platform))) except (OSError, IOError, TypeError): logger.error("Unable to store LHCbDirac environment") raise logger.info("Using LHCbDirac version %s", diracversion) os.environ['GANGADIRACENVIRONMENT'] = fname
def get_env(env_source): """ Given a source command, return the DIRAC environment that the command created. Args: env_source: a command which can be sourced, providing the desired environment Returns: dict: the environment """ logger.debug('Running DIRAC source command %s', env_source) env = dict(os.environ) execute('source {0}'.format(env_source), shell=True, env=env, update_env=True) if not any(key.startswith('DIRAC') for key in env): raise RuntimeError("'DIRAC*' not found in environment") return env
def test_execute_timeouts(): ''' This tests that the timeouts on scripts cause them to stop as expected ''' # Test timeouts assert execute('while true; do sleep 1; done', shell=True, timeout=1) == 'Command timed out!' assert execute('while True: pass', timeout=1, shell=False) == 'Command timed out!' # Test timeout doesn't hinder a normal command assert execute('cd "{0}"; pwd'.format(os.getcwd()), shell=True, timeout=10, cwd=os.getcwd()).strip() == os.getcwd() assert timeit.timeit(''' from Ganga.Utility.execute import execute execute('cd "{0}"; pwd', shell=True, timeout=10) '''.format(os.getcwd()), number=1) < 11
def test_execute_timeouts(): ''' This tests the timeouts cause the scripts to fail as expected ''' # Test timeouts assert execute('while true; do sleep 1; done', shell=True, timeout=1) == 'Command timed out!' assert execute('while True: pass', timeout=1, shell=False) == 'Command timed out!' # Test timeout doesn't hinder a normal command assert execute('import os\noutput(os.getcwd())', timeout=10, cwd=os.getcwd(), shell=False).strip() == os.getcwd() # Test timeout doesn't delay normal command assert timeit.timeit(""" import os from Ganga.Utility.execute import execute execute('import os\\noutput(os.getcwd())',timeout=10, cwd=os.getcwd(), shell=False) """, number=1) < 11
def _store_dirac_environment(): from GangaDirac.Lib.Utilities.DiracUtilities import write_env_cache, get_env diracversion = _guess_version('LHCBDIRAC') platform = os.environ['CMTOPT'] fdir = os.path.join(os.path.expanduser("~/.cache/Ganga/GangaLHCb"), platform) fname = os.path.join(fdir, diracversion) if not os.path.exists(fname) or not os.path.getsize(fname): logger.info("Storing new LHCbDirac environment (%s:%s)" % (str(diracversion), str(platform))) cmd = 'lb-run LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format(version=diracversion) env = execute(cmd) # grab the stdout text env = eval(env) # env is a string so convert it to a dict write_env_cache(env, fname) os.environ['GANGADIRACENVIRONMENT'] = fname
def test_execute_env(): ''' This tests that the scripts can update an env based upon input and handle an appropriate python_setup sent in execute args ''' # Test correctly picking up env env = {'ALEX': '/hello/world', 'PATH': os.environ.get('PATH', ''), 'PYTHONPATH': os.environ.get('PYTHONPATH', '')} # env.update(os.environ) assert execute('echo $ALEX', shell=True, env=env).strip() == '/hello/world' assert execute('import os\noutput(os.environ.get("ALEX","BROKEN"))', env=env, python_setup='#', shell=False).strip() == '/hello/world' # Test env not updated by default execute('export NEWTEST=/new/test', shell=True, env=env) assert 'NEWTEST' not in env execute('import os\nos.environ["NEWTEST"]="/new/test"', env=env, python_setup='#') assert 'NEWTEST' not in env # Test updating of env execute('export NEWTEST=/new/test', shell=True, env=env, update_env=True) assert 'NEWTEST' in env del env['NEWTEST'] assert 'NEWTEST' not in env execute('import os\nos.environ["NEWTEST"]="/new/test"', env=env, python_setup='#', update_env=True, shell=False) assert 'NEWTEST' in env
def test_execute_output(): ''' This tests the various levels when an import can occur and that complex objects can be returned via stdout ''' # Test printout of pickle dump interpreted correctly d = execute('import pickle, datetime\nprint(pickle.dumps(datetime.datetime(2013,12,12)))', shell=False) assert hasattr(d, 'month') assert d.month == 12 # Test straight printout doesn't work without includes, stdout as str # returned by default d1 = execute('import datetime\nprint(datetime.datetime(2013,12,12))', shell=False) d2 = execute('import datetime\nrepr(datetime.datetime(2013,12,12))', shell=False) assert not hasattr(d1, 'month') assert isinstance(d1, str) assert not hasattr(d2, 'month') assert isinstance(d2, str) # Test printout works with the right includes d1 = execute('print(datetime.datetime(2013,12,12))', python_setup='import datetime', eval_includes='import datetime', shell=False) d2 = execute('print(repr(datetime.datetime(2013,12,12)))', python_setup='import datetime', eval_includes='import datetime', shell=False) assert not hasattr(d1, 'month') assert isinstance(d1, str) assert hasattr(d2, 'month') assert d2.month == 12
def execute(command, timeout=getConfig('DIRAC')['Timeout'], env=None, cwd=None, shell=False, python_setup='', eval_includes=None, update_env=False): """ Execute a command on the local DIRAC server. This function blocks until the server returns. """ if env is None: env = getDiracEnv() if python_setup == '': python_setup = getDiracCommandIncludes() _checkProxy() #logger.debug("Executing command:\n'%s'" % str(command)) #logger.debug("python_setup:\n'%s'" % str(python_setup)) #logger.debug("eval_includes:\n'%s'" % str(eval_includes)) returnable = gexecute.execute(command, timeout=timeout, env=env, cwd=cwd, shell=shell, python_setup=python_setup, eval_includes=eval_includes, update_env=update_env) # rcurrie I've seen problems with just returning this raw object, # expanding it to be sure that an instance remains in memory myObject = {} if hasattr(returnable, 'keys'): # Expand object(s) in dictionaries myObject = _expand_object(returnable) elif type(returnable) == type([]): # Expand object(s) in lists myObject = _expand_list(returnable) else: # Copy object(s) so thet they definately are in memory myObject = copy.deepcopy(returnable) return myObject
def execute(command, timeout=getConfig('DIRAC')['Timeout'], env=None, cwd=None, shell=False, python_setup='', eval_includes=None, update_env=False): """ Execute a command on the local DIRAC server. This function blocks until the server returns. """ if env is None: env = getDiracEnv() if python_setup == '': python_setup = getDiracCommandIncludes() _checkProxy() #logger.debug("Executing command:\n'%s'" % str(command)) #logger.debug("python_setup:\n'%s'" % str(python_setup)) #logger.debug("eval_includes:\n'%s'" % str(eval_includes)) returnable = gexecute.execute(command, timeout=timeout, env=env, cwd=cwd, shell=shell, python_setup=python_setup, eval_includes=eval_includes, update_env=update_env) # rcurrie I've seen problems with just returning this raw object, # expanding it to be sure that an instance remains in memory myObject = {} if hasattr(returnable, 'keys'): # Expand object(s) in dictionaries myObject = _expand_object(returnable) elif type(returnable) in [list, tuple]: # Expand object(s) in lists myObject = _expand_list(returnable) else: # Copy object(s) so thet they definately are in memory myObject = copy.deepcopy(returnable) return myObject
def execute(command, timeout=getConfig('DIRAC')['Timeout'], env=None, cwd=None, shell=False, python_setup='', eval_includes=None, update_env=False): """ Execute a command on the local DIRAC server. This function blocks until the server returns. Args: command (str): This is the command we're running within our DIRAC session timeout (bool): This is the length of time that a DIRAC call has before it's decided some interaction has timed out env (dict): an optional environment to execute the DIRAC code in cwd (str): an optional string to a valid path where this code should be executed shell (bool): Should this code be executed in a new shell environment python_setup (str): Optional extra code to pass to python when executing eval_incldes (???): TODO document me update_env (bool): Should this modify the given env object with the env after the command has executed """ if env is None: env = getDiracEnv() if python_setup == '': python_setup = getDiracCommandIncludes() _checkProxy() #logger.info("Executing command:\n'%s'" % str(command)) #logger.debug("python_setup:\n'%s'" % str(python_setup)) #logger.debug("eval_includes:\n'%s'" % str(eval_includes)) returnable = gexecute.execute(command, timeout=timeout, env=env, cwd=cwd, shell=shell, python_setup=python_setup, eval_includes=eval_includes, update_env=update_env) return deepcopy(returnable)
def select_dirac_version(wildcard): """ Find the LHCbDIRAC version that should be used based on the confuguration system. Wildcards can be used and soflinks are dereferenced. """ cmd = 'lb-run -c x86_64-slc6-gcc49-opt -l LHCbDIRAC' out = execute(cmd) if out == '': raise PluginError("Can't find any LHCbDirac versions from '%s'" % cmd) versions = [s.split() for s in out.splitlines() if fnmatch(s.split()[0], wildcard)] if len(versions) == 0: raise PluginError("Can't find LHCbDIRAC version matching %s.", wildcard) versions.sort(key=lambda v: v[0]) version = versions[-1] dereferenced_version = basename(realpath(version[2]))[10:] return dereferenced_version
def select_dirac_version(wildcard): """ Find the LHCbDIRAC version that should be used based on the confuguration system. Wildcards can be used and soflinks are dereferenced. """ cmd = 'lb-run -c x86_64-slc6-gcc49-opt -l LHCbDIRAC' out = execute(cmd) if out == '': raise PluginError("Can't find any LHCbDirac versions from '%s'" % cmd) versions = [ s.split() for s in out.splitlines() if fnmatch(s.split()[0], wildcard) ] if len(versions) == 0: raise PluginError("Can't find LHCbDIRAC version matching %s.", wildcard) versions.sort(key=lambda v: v[0]) version = versions[-1] dereferenced_version = basename(realpath(version[2]))[10:] return dereferenced_version
def store_dirac_environment(): """Store the LHCbDIRAC environment in a cache file.""" # platform_env_var = 'CMTCONFIG' # try: # platform = os.environ[platform_env_var] # except KeyError: # logger.error("Environment variable %s is missing. Can't cache LHCbDIRAC environment.", platform_env_var) # raise PluginError #While LHCbDirac is only available for gcc49 we shall unfortunately hard-code the platform. platform = 'x86_64-slc6-gcc49-opt' wildcard = Ganga.Utility.Config.getConfig('LHCb')['LHCbDiracVersion'] diracversion = select_dirac_version(wildcard) fdir = join(expanduser("~/.cache/Ganga/GangaLHCb"), platform) fname = join(fdir, diracversion) if not exists(fname) or not getsize(fname): cmd = 'lb-run -c x86_64-slc6-gcc49-opt LHCBDIRAC {version} python -c "import os; print(dict(os.environ))"'.format( version=diracversion) env = execute(cmd) if isinstance(env, str): try: env_temp = eval(env) env = env_temp except SyntaxError: logger.error("LHCbDirac version %s does not exist", diracversion) raise OptionValueError( "LHCbDirac version {version} does not exist".format( version=diracversion)) try: write_env_cache(env, fname) logger.info("Storing new LHCbDirac environment (%s:%s)", str(diracversion), str(platform)) except (OSError, IOError, TypeError): logger.error("Unable to store LHCbDirac environment") raise PluginError logger.info("Using LHCbDirac version %s", diracversion) os.environ['GANGADIRACENVIRONMENT'] = fname
def test_output(): """Test that ``output`` works as expected""" assert execute('output("foo")', shell=False).strip() == 'foo'
def test_execute(): ''' This tests that the script correctly changes the dir as requested via the script ''' assert execute('cd "{0}"; pwd'.format(os.getcwd()), shell=True, cwd=os.getcwd()).strip() == os.getcwd()
def test_execute_cwd(): ''' This tests that the cwd of the script is changed as per the arguments to execute ''' # Test changing dir assert execute('import os\noutput(os.getcwd())', cwd='/', shell=False).strip() == '/'
def test_print(): """Test that ``print`` works as expected""" assert execute('print("foo")', shell=False).strip() == 'foo'
def test_execute(): ''' This tests that the cwd returned from the output is what is requested in the script ''' # Test shell vs python mode assert execute('import os\noutput(os.getcwd())', cwd=os.getcwd(), shell=False).strip() == os.getcwd()
def execute(command, timeout=getConfig('DIRAC')['Timeout'], env=None, cwd=None, shell=False, python_setup='', eval_includes=None, update_env=False, ): """ Execute a command on the local DIRAC server. This function blocks until the server returns. Args: command (str): This is the command we're running within our DIRAC session timeout (int): This is the length of time that a DIRAC call has before it's decided some interaction has timed out env (dict): an optional environment to execute the DIRAC code in cwd (str): an optional string to a valid path where this code should be executed shell (bool): Should this code be executed in a new shell environment python_setup (str): Optional extra code to pass to python when executing eval_includes (???): TODO document me update_env (bool): Should this modify the given env object with the env after the command has executed """ if env is None: env = getDiracEnv() if python_setup == '': python_setup = getDiracCommandIncludes() # We're about to perform an expensive operation so being safe before we run it shouldn't cost too much _checkProxy(force = True) #logger.debug("Executing command:\n'%s'" % str(command)) #logger.debug("python_setup:\n'%s'" % str(python_setup)) #logger.debug("eval_includes:\n'%s'" % str(eval_includes)) if cwd is None: # We can in all likelyhood be in a temp folder on a shared (SLOW) filesystem # If we are we do NOT want to execute commands which will involve any I/O on the system that isn't needed cwd_ = tempfile.mkdtemp() else: # We know were whe want to run, lets just run there cwd_ = cwd global last_modified_valid if not last_modified_valid: return None returnable = gexecute.execute(command, timeout=timeout, env=env, cwd=cwd_, shell=shell, python_setup=python_setup, eval_includes=eval_includes, update_env=update_env) # TODO we would like some way of working out if the code has been executed correctly # Most commands will be OK now that we've added the check for the valid proxy before executing commands here if cwd is None: shutil.rmtree(cwd_, ignore_errors=True) return returnable
def execute(command, timeout=getConfig('DIRAC')['Timeout'], env=None, cwd=None, shell=False, python_setup='', eval_includes=None, update_env=False, return_raw_dict=False, cred_req=None, ): """ Execute a command on the local DIRAC server. This function blocks until the server returns. Args: command (str): This is the command we're running within our DIRAC session timeout (int): This is the length of time that a DIRAC call has before it's decided some interaction has timed out env (dict): an optional environment to execute the DIRAC code in cwd (str): an optional string to a valid path where this code should be executed shell (bool): Should this code be executed in a new shell environment python_setup (str): Optional extra code to pass to python when executing eval_includes (???): TODO document me update_env (bool): Should this modify the given env object with the env after the command has executed return_raw_dict(bool): Should we return the raw dict from the DIRAC interface or parse it here cred_req (ICredentialRequirement): What credentials does this call need """ if env is None: if cred_req is None: env = getDiracEnv() else: env = getDiracEnv(cred_req.dirac_env) if python_setup == '': python_setup = getDiracCommandIncludes() if cred_req is not None: env['X509_USER_PROXY'] = credential_store[cred_req].location if cwd is None: # We can in all likelyhood be in a temp folder on a shared (SLOW) filesystem # If we are we do NOT want to execute commands which will involve any I/O on the system that isn't needed cwd_ = tempfile.mkdtemp() else: # We know were whe want to run, lets just run there cwd_ = cwd returnable = gexecute.execute(command, timeout=timeout, env=env, cwd=cwd_, shell=shell, python_setup=python_setup, eval_includes=eval_includes, update_env=update_env) # If the time if returnable == 'Command timed out!': raise GangaDiracError("DIRAC command timed out") # TODO we would like some way of working out if the code has been executed correctly # Most commands will be OK now that we've added the check for the valid proxy before executing commands here if cwd is None: shutil.rmtree(cwd_, ignore_errors=True) if isinstance(returnable, dict) and not return_raw_dict: # If the output is a dictionary allow for automatic error detection if returnable['OK']: return returnable['Value'] else: raise GangaDiracError(returnable['Message']) elif isinstance(returnable, dict): # If the output is a dictionary return and it has been requested, then return it return returnable else: # Else raise an exception as it should be a dictionary raise GangaDiracError(returnable)
def test_execute_cwd(): ''' This tests that teh cwd is set correctly as requeted in the arguments ''' assert execute('pwd', shell=True, cwd='/').strip() == '/'
def execute( command, timeout=getConfig('DIRAC')['Timeout'], env=None, cwd=None, shell=False, python_setup='', eval_includes=None, update_env=False, ): """ Execute a command on the local DIRAC server. This function blocks until the server returns. Args: command (str): This is the command we're running within our DIRAC session timeout (int): This is the length of time that a DIRAC call has before it's decided some interaction has timed out env (dict): an optional environment to execute the DIRAC code in cwd (str): an optional string to a valid path where this code should be executed shell (bool): Should this code be executed in a new shell environment python_setup (str): Optional extra code to pass to python when executing eval_includes (???): TODO document me update_env (bool): Should this modify the given env object with the env after the command has executed """ if env is None: env = getDiracEnv() if python_setup == '': python_setup = getDiracCommandIncludes() # We're about to perform an expensive operation so being safe before we run it shouldn't cost too much _checkProxy(force=True) #logger.debug("Executing command:\n'%s'" % str(command)) #logger.debug("python_setup:\n'%s'" % str(python_setup)) #logger.debug("eval_includes:\n'%s'" % str(eval_includes)) if cwd is None: # We can in all likelyhood be in a temp folder on a shared (SLOW) filesystem # If we are we do NOT want to execute commands which will involve any I/O on the system that isn't needed cwd_ = tempfile.mkdtemp() else: # We know were whe want to run, lets just run there cwd_ = cwd global last_modified_valid if not last_modified_valid: return None returnable = gexecute.execute(command, timeout=timeout, env=env, cwd=cwd_, shell=shell, python_setup=python_setup, eval_includes=eval_includes, update_env=update_env) # TODO we would like some way of working out if the code has been executed correctly # Most commands will be OK now that we've added the check for the valid proxy before executing commands here if cwd is None: shutil.rmtree(cwd_, ignore_errors=True) return returnable
def __worker_thread(self, thread): """ Code run by worker threads to allow parallelism in Ganga. Can be used for executing non-blocking calls to local DIRAC server """ # Occasionally when shutting down the Queue import at top has been garbage collected # and line "except Queue.Empty: continue" will throw # <type 'exceptions.AttributeError'>: 'NoneType' object has no attribute 'Empty' # im hoping that importing within the thread will avoid this. import Queue oldname = thread.gangaName # Note can use threading.current_thread to get the thread rather than passing it as an arg # easier to unit test this way though with a dummy thread. while not thread.should_stop(): try: item = self.__queue.get(True, 0.05) except Queue.Empty: # wait 0.05 sec then loop again to give shutdown a chance continue # regster as a working thread if isinstance(item, QueueElement): oldname = thread.gangaName thread.gangaName = item.name thread.register() if not isinstance(item, QueueElement): logger.error("Unrecognised queue element: '%s'" % repr(item)) logger.error(" expected: 'QueueElement'") self.__queue.task_done() thread.unregister() continue if isinstance(item.command_input, FunctionInput): thread._command = item.command_input.function.__name__ elif isinstance(item.command_input, CommandInput): thread._command = item.command_input.command thread._timeout = item.command_input.timeout else: logger.error("Unrecognised input command type: '%s'" % repr(item.command_input)) logger.error(" expected: ('FunctionInput' or 'CommandInput')") self.__queue.task_done() thread.unregister() continue try: if isinstance(item.command_input, FunctionInput): these_args = item.command_input.args if isinstance(these_args, str): these_args = (these_args, ) result = item.command_input.function(*these_args, **item.command_input.kwargs) else: result = execute(*item.command_input) except Exception as e: logger.error("Exception raised executing '%s' in Thread '%s':\n%s" % (thread._command, thread.gangaName, traceback.format_exc())) if item.fallback_func.function is not None: if isinstance(item.fallback_func, FunctionInput): thread._command = item.fallback_func.function.__name__ thread._timeout = 'N/A' try: item.fallback_func.function(e, *item.fallback_func.args, **item.fallback_func.kwargs) except Exception as x: logger.error("Exception raised in fallback function '%s' of Thread '%s':\n%s" % (thread._command, thread.gangaName, traceback.format_exc())) else: logger.error("Unrecognised fallback_func type: '%s'" % repr(item.fallback_func)) logger.error(" expected: 'FunctionInput'") else: if item.callback_func.function is not None: if isinstance(item.callback_func, FunctionInput): thread._command = item.callback_func.function.__name__ thread._timeout = 'N/A' try: item.callback_func.function( result, *item.callback_func.args, **item.callback_func.kwargs) except Exception as e: logger.error("Exception raised in callback_func '%s' of Thread '%s': %s" % ( thread._command, thread.gangaName, traceback.format_exc())) else: logger.error("Unrecognised callback_func type: '%s'" % repr(item.callback_func)) logger.error(" expected: 'FunctionInput'") finally: # unregister as a working thread bcoz free thread._command = 'idle' thread._timeout = 'N/A' self.__queue.task_done() thread.unregister() thread.gangaName = oldname
def __worker_thread(self, thread): """ Code run by worker threads to allow parallelism in Ganga. Can be used for executing non-blocking calls to local DIRAC server """ # Occasionally when shutting down the Queue import at top has been garbage collected # and line "except Queue.Empty: continue" will throw # <type 'exceptions.AttributeError'>: 'NoneType' object has no attribute 'Empty' # im hoping that importing within the thread will avoid this. import Queue oldname = thread.gangaName # Note can use threading.current_thread to get the thread rather than passing it as an arg # easier to unit test this way though with a dummy thread. while not thread.should_stop(): try: item = self.__queue.get(True, 0.05) except Queue.Empty: # wait 0.05 sec then loop again to give shutdown a chance continue # regster as a working thread if isinstance(item, QueueElement): oldname = thread.gangaName thread.gangaName = item.name thread.register() if not isinstance(item, QueueElement): logger.error("Unrecognised queue element: '%s'" % repr(item)) logger.error(" expected: 'QueueElement'") self.__queue.task_done() thread.unregister() continue if isinstance(item.command_input, FunctionInput): thread._command = getName(item.command_input.function) elif isinstance(item.command_input, CommandInput): thread._command = item.command_input.command thread._timeout = item.command_input.timeout else: logger.error("Unrecognised input command type: '%s'" % repr(item.command_input)) logger.error( " expected: ('FunctionInput' or 'CommandInput')" ) self.__queue.task_done() thread.unregister() continue try: if isinstance(item.command_input, FunctionInput): these_args = item.command_input.args if isinstance(these_args, str): these_args = (these_args, ) result = item.command_input.function( *these_args, **item.command_input.kwargs) else: result = execute(*item.command_input) except Exception as e: logger.error( "Exception raised executing '%s' in Thread '%s':\n%s" % (thread._command, thread.gangaName, traceback.format_exc())) if item.fallback_func.function is not None: if isinstance(item.fallback_func, FunctionInput): thread._command = getName(item.fallback_func.function) thread._timeout = 'N/A' try: item.fallback_func.function( e, *item.fallback_func.args, **item.fallback_func.kwargs) except Exception as x: logger.error( "Exception raised in fallback function '%s' of Thread '%s':\n%s" % (thread._command, thread.gangaName, traceback.format_exc())) else: logger.error("Unrecognised fallback_func type: '%s'" % repr(item.fallback_func)) logger.error( " expected: 'FunctionInput'") else: if item.callback_func.function is not None: if isinstance(item.callback_func, FunctionInput): thread._command = getName(item.callback_func.function) thread._timeout = 'N/A' try: item.callback_func.function( result, *item.callback_func.args, **item.callback_func.kwargs) except Exception as e: logger.error( "Exception raised in callback_func '%s' of Thread '%s': %s" % (thread._command, thread.gangaName, traceback.format_exc())) else: logger.error("Unrecognised callback_func type: '%s'" % repr(item.callback_func)) logger.error( " expected: 'FunctionInput'") finally: # unregister as a working thread bcoz free thread._command = 'idle' thread._timeout = 'N/A' self.__queue.task_done() thread.unregister() thread.gangaName = oldname