def maybe_activate_venv(): if not environ.get('VIRTUAL_ENV'): from pathlib import Path proj_root = Path(__file__).resolve().parents[1] for venv_name in ('venv', '.venv'): venv_path = proj_root.joinpath(venv_name) if venv_path.exists(): break else: return import platform import sys from subprocess import call on_windows = platform.system().lower() == 'windows' bin_path = venv_path.joinpath('Scripts' if on_windows else 'bin') environ.update( PYTHONHOME='', VIRTUAL_ENV=venv_path.as_posix(), PATH='{}:{}'.format(bin_path.as_posix(), environ['PATH']), ) cmd = [ bin_path.joinpath( 'python.exe' if on_windows else 'python').as_posix() ] + sys.argv sys.exit(call(cmd, env=environ))
def update_env(**kwargs): """Update environment and then set it back.""" start = environ.copy() environ.update(**kwargs) yield environ.clear() environ.update(**start)
def source(script, update=True, clean=True): """ Source variables from a shell script import them in the environment (if update==True) and report only the script variables (if clean==True) """ global environ environ_back = None if clean: environ_back = dict(environ) environ.clear() pipe = Popen(". %s; env" % script, stdout=PIPE, shell=True) data = pipe.communicate()[0] env = dict(line.split("=", 1) for line in data.splitlines()) if clean: # remove unwanted minimal vars env.pop('LINES', None) env.pop('COLUMNS', None) environ.update(environ_back) if update: environ.update(env) return env
def teardown(self, cleanup_test_bench=False): """Tear down the environment""" environ.update(self.old_environ) if cleanup_test_bench: self._cleanup_content(self.dump_directory) else: self.cleanup_index_testbench()
def install(name, keep_prefix=False, keep_stage=False, ignore_deps=False, environs=None, compiler=None): _init_spack() from spack import repo, installed_db from spack.cmd import parse_specs from os import environ if not isinstance(name, str): results = [], [] for pkg in name: a, b = install(pkg, keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, environs=environs, compiler=compiler) results[0].extend(a) results[1].extend(b) return results if environs is not None: environ.update(environs) if compiler is not None: names = name.split() names.insert(1, "%" + compiler.rstrip().lstrip()) name = ' '.join(names) specs = parse_specs(name, concretize=True) packages = [repo.get(spec) for spec in specs] new_pkgs = [u for u in packages if not u.installed] for package in new_pkgs: with installed_db.write_transaction(): package.do_install( keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps ) return [p.name for p in new_pkgs if p.installed], \ [p.name for p in new_pkgs if not p.installed]
def setUp(self): self.defaults = { 'intopt': 11, 'floatopt':11.1, 'boolopttrue': False, 'booloptfalse': False, 'stropt':'stroptval_default', 'intopt_str': 2 } self.filecontent = { 'intopt': 111, 'floatopt':111.1, 'boolopttrue': False, 'booloptfalse': False, 'stropt':'stroptval_file', 'intopt_str': 2 } self.filename = './testsettings.json' with open(self.filename, 'w') as f: f.write(json.dumps(self.filecontent)) environ.update({k: str(self.reference[k]) for k in self.reference.keys()}) environ[self.not_default_key] = 'only_file_value' self.settings = Settings(defaults=self.defaults, filename=self.filename)
def test_strict_parsing(): importlib.reload(expandvars) with pytest.raises( expandvars.ExpandvarsException, match="FOO: parameter null or not set" ) as e: expandvars.expandvars("${FOO:?}") assert isinstance(e.value, expandvars.ParameterNullOrNotSet) with pytest.raises( expandvars.ExpandvarsException, match="FOO: parameter null or not set" ) as e: expandvars.expandvars("${FOO?}") assert isinstance(e.value, expandvars.ParameterNullOrNotSet) with pytest.raises(expandvars.ExpandvarsException, match="FOO: custom error") as e: expandvars.expandvars("${FOO:?custom error}") assert isinstance(e.value, expandvars.ParameterNullOrNotSet) with pytest.raises(expandvars.ExpandvarsException, match="FOO: custom error") as e: expandvars.expandvars("${FOO?custom error}") assert isinstance(e.value, expandvars.ParameterNullOrNotSet) env.update({"FOO": "foo"}) assert expandvars.expandvars("${FOO:?custom err}") == "foo" assert expandvars.expandvars("${FOO?custom err}:bar") == "foo:bar"
def do_install(reqs): from os import environ previously_installed = pip_get_installed() required = pip_parse_requirements(reqs) requirements_as_options = tuple( '--requirement={0}'.format(requirement) for requirement in reqs ) # We put the cache in the directory that pip already uses. # This has better security characteristics than a machine-wide cache, and is a # pattern people can use for open-source projects pipdir = environ['HOME'] + '/.pip' # We could combine these caches to one directory, but pip would search everything twice, going slower. pip_download_cache = pipdir + '/cache' pip_wheels = pipdir + '/wheelhouse' environ.update( PIP_DOWNLOAD_CACHE=pip_download_cache, ) cache_opts = ( '--download-cache=' + pip_download_cache, '--find-links=file://' + pip_wheels, ) # --use-wheel is somewhat redundant here, but it means we get an error if we have a bad version of pip/setuptools. install_opts = ('--upgrade', '--use-wheel',) + cache_opts recently_installed = [] # 1) Bootstrap the install system; setuptools and pip are already installed, just need wheel recently_installed += pip_install(install_opts + BOOTSTRAP_VERSIONS) # 2) Caching: Make sure everything we want is downloaded, cached, and has a wheel. pip( ('wheel', '--wheel-dir=' + pip_wheels) + BOOTSTRAP_VERSIONS + cache_opts + requirements_as_options ) # 3) Install: Use our well-populated cache, to do the installations. install_opts += ('--no-index',) # only use the cache recently_installed += pip_install(install_opts + requirements_as_options) required_with_deps = trace_requirements(required) # TODO-TEST require A==1 then A==2 extraneous = ( reqnames(previously_installed) - reqnames(required_with_deps) - reqnames(recently_installed) ) # 2) Uninstall any extraneous packages. if extraneous: pip(('uninstall', '--yes') + tuple(sorted(extraneous))) return 0 # posix:success!
def clean_state(self): """ Cleans up the state. """ if path.exists(self.state_file): remove(self.state_file) environ.update(self.old_environ)
def __init__(self, *args, **kwargs): if 'nmpi' in kwargs: self.nmpi = kwargs['nmpi'] del kwargs['nmpi'] else: self.nmpi = 1 if 'mpicompiler' in kwargs: self.compiler = kwargs['mpicompiler'] del kwargs['mpicompiler'] else: self.compiler = None super().__init__(*args, **kwargs) environ_backup = dict(environ) self.set_compiler() super().build(*args) environ.clear() environ.update(environ_backup) self.executable = 'start.exe' self.create_executable(**kwargs) super().clean_module()
def on_context_leave(self): ''' Called when the context manager entered with :func:`magic.enter_context` is exited. Undos all of the stuff that :meth:`on_context_enter` did and more. * Stop the Craftr Runtime Server * Restore the :data:`os.environ` dictionary * Removes all ``craftr.ext.`` modules from :data:`sys.modules` and ensures they are in :attr:`Session.modules` (they are expected to be put there from the :class:`ext.CraftrImporter`). ''' self._stop_server() # Restore the original values of os.environ. self.env = environ.copy() environ.clear() environ.update(self._old_environ) del self._old_environ sys.meta_path.remove(self.ext_importer) for key, module in list(sys.modules.items()): if key.startswith('craftr.ext.'): name = key[11:] assert name in self.modules and self.modules[ name] is module, key del sys.modules[key] try: # Remove the module from the `craftr.ext` modules contents, too. delattr(ext, name.split('.')[0]) except AttributeError: pass
def on_context_enter(self, prev): ''' Called when entering the Session context with :func:`magic.enter_context`. Does the following things: * Sets up the :data`os.environ` with the values from :attr:`Session.env` * Adds the :attr:`Session.ext_importer` to :data:`sys.meta_path` * Starts the Craftr Runtime Server (:attr:`Session.server`) and sets the ``CRAFTR_RTS`` environment variable .. note:: A copy of the original :data:`os.environ` is saved and later restored in :meth:`on_context_leave`. The :data:`os.environ` object can not be replaced by another object, that is why we change its values in-place. ''' if prev is not None: raise RuntimeError('session context can not be nested') # We can not change os.environ effectively, we must update the # dictionary instead. self._old_environ = environ.copy() environ.clear() environ.update(self.env) self.env = environ sys.meta_path.append(self.ext_importer) self.update()
def __init__(self, context): # Context associated attributes self.log = context.log self.run_name = context.pipeline_run.pipeline_name self.dagster_run_id = context.run_id # resource config attributes resource_config = context.resource_config self.tracking_uri = resource_config.get("mlflow_tracking_uri") if self.tracking_uri: mlflow.set_tracking_uri(self.tracking_uri) self.parent_run_id = resource_config.get("parent_run_id") self.experiment_name = resource_config["experiment_name"] self.env_tags_to_log = resource_config.get("env_to_tag") or [] self.extra_tags = resource_config.get("extra_tags") # Update env variables if any are given self.env_vars = resource_config.get("env", {}) if self.env_vars: environ.update(self.env_vars) # If the experiment exists then the set won't do anything mlflow.set_experiment(self.experiment_name) self.experiment = mlflow.get_experiment_by_name(self.experiment_name) # Get the client object self.tracking_client = mlflow.tracking.MlflowClient() # Set up the active run and tags self._setup()
def on_context_leave(self): ''' Called when the context manager entered with :func:`magic.enter_context` is exited. Undos all of the stuff that :meth:`on_context_enter` did and more. * Stop the Craftr Runtime Server * Restore the :data:`os.environ` dictionary * Removes all ``craftr.ext.`` modules from :data:`sys.modules` and ensures they are in :attr:`Session.modules` (they are expected to be put there from the :class:`ext.CraftrImporter`). ''' self._stop_server() # Restore the original values of os.environ. self.env = environ.copy() environ.clear() environ.update(self._old_environ) del self._old_environ sys.meta_path.remove(self.ext_importer) for key, module in list(sys.modules.items()): if key.startswith('craftr.ext.'): name = key[11:] assert name in self.modules and self.modules[name] is module, key del sys.modules[key] try: # Remove the module from the `craftr.ext` modules contents, too. delattr(ext, name.split('.')[0]) except AttributeError: pass
def export(envvars): try: environ.update(envvars) yield finally: for key in envvars: del environ[key]
def initONOSEnv(): """Initialize ONOS environment (and module) variables This is ugly and painful, but they have to be set correctly in order for the onos-setup-karaf script to work. nodes: list of ONOS nodes returns: ONOS environment variable dict""" # pylint: disable=global-statement global HOME, ONOS_ROOT, ONOS_USER global ONOS_APPS, ONOS_WEB_USER, ONOS_WEB_PASS env = {} def sd(var, val): "Set default value for environment variable" env[var] = environ.setdefault(var, val) return env[var] assert environ['HOME'] HOME = sd('HOME', environ['HOME']) ONOS_ROOT = sd('ONOS_ROOT', join(HOME, 'onos')) environ['ONOS_USER'] = defaultUser() ONOS_USER = sd('ONOS_USER', defaultUser()) # Set to items passed in the "controllerConfig.ini" config = ConfigParser() config.read('controllerConfig.ini') assert config.get("ONOS", "rules") ONOS_APPS = sd('ONOS_APPS', config.get("ONOS", "rules")) # ONOS_WEB_{USER,PASS} isn't respected by onos-karaf: environ.update(ONOS_WEB_USER='******', ONOS_WEB_PASS='******') ONOS_WEB_USER = sd('ONOS_WEB_USER', 'karaf') ONOS_WEB_PASS = sd('ONOS_WEB_PASS', 'karaf') return env
def on_context_enter(self, prev): ''' Called when entering the Session context with :func:`magic.enter_context`. Does the following things: * Sets up the :data`os.environ` with the values from :attr:`Session.env` * Adds the :attr:`Session.ext_importer` to :data:`sys.meta_path` * Starts the Craftr Runtime Server (:attr:`Session.server`) and sets the ``CRAFTR_RTS`` environment variable .. note:: A copy of the original :data:`os.environ` is saved and later restored in :meth:`on_context_leave`. The :data:`os.environ` object can not be replaced by another object, that is why we change its values in-place. ''' if prev is not None: raise RuntimeError('session context can not be nested') # We can not change os.environ effectively, we must update the # dictionary instead. self._old_environ = environ.copy() environ.clear() environ.update(self.env) self.env = environ sys.meta_path.append(self.ext_importer)
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--use-env', action='store_true', help='Use existing TINYSERV_* environment variables.' ) args = parser.parse_args() if not args.use_env: tinyserv_bin_path = path.normpath(path.join(ROOT, '..')) environ.update({ 'TINYSERV_REMOTE': 'localhost', 'TINYSERV_ROOT': path.join(ROOT, '.tinyserv'), 'TINYSERV_START_PORT': '5000', 'PATH': tinyserv_bin_path + path.pathsep + os.environ['PATH'] }) remote = environ['TINYSERV_REMOTE'] host = remote.split('@')[-1] print "Running smoke test on %s." % host describe("Setting up sample 'tinysmoke' repository.") chdir(ROOT) if path.exists("tinysmoke"): run("rm -rf tinysmoke") try: run("tinyserv apps:destroy tinysmoke", silent=True) except subprocess.CalledProcessError, e: if "invalid choice: 'tinysmoke'" not in e.output: print "Failed process output follows." print e.output raise
def initONOSEnv(): """Initialize ONOS environment (and module) variables This is ugly and painful, but they have to be set correctly in order for the onos-setup-karaf script to work. nodes: list of ONOS nodes returns: ONOS environment variable dict""" # pylint: disable=global-statement global HOME, ONOS_ROOT, ONOS_USER global ONOS_APPS, ONOS_WEB_USER, ONOS_WEB_PASS env = {} def sd( var, val ): "Set default value for environment variable" env[ var ] = environ.setdefault( var, val ) return env[ var ] assert environ[ 'HOME' ] HOME = sd( 'HOME', environ[ 'HOME' ] ) ONOS_ROOT = sd( 'ONOS_ROOT', join( HOME, 'onos' ) ) environ[ 'ONOS_USER' ] = defaultUser() ONOS_USER = sd( 'ONOS_USER', defaultUser() ) ONOS_APPS = sd( 'ONOS_APPS', 'drivers,openflow,fwd,proxyarp,mobility' ) # ONOS_WEB_{USER,PASS} isn't respected by onos-karaf: environ.update( ONOS_WEB_USER='******', ONOS_WEB_PASS='******' ) ONOS_WEB_USER = sd( 'ONOS_WEB_USER', 'karaf' ) ONOS_WEB_PASS = sd( 'ONOS_WEB_PASS', 'karaf' ) return env
def _source(script, replace=None, update=True): """ Source variables from a shell script import them in the environment (if update==True) """ from subprocess import Popen, PIPE from os import environ import os if os.path.isfile(script) is False: errmsg = "'%s' is not available" % (script) raise ValueError(errmsg) else: print(" Reading Environmental Variables from %s" % (script)) pipe = Popen(". %s > /dev/null 2>&1; env" % script, stdout=PIPE, shell=True) data = pipe.communicate()[0] env = dict((line.split("=", 1) for line in data.splitlines())) if replace is not None: for key in env.keys(): value = env[key] if replace[0] in value: env[key] = env[key].replace(replace[0], replace[1]) if update: environ.update(env) else: return env
def init(self, url, targetpath, fullnames=True, branch=None, **kwargs): # verify repo url execcmd("svn", "info", url) topurl = dirname(url) trunk = basename(url) tags = "releases" # cloning svn braches as well should rather be optionalif reenabled.. #cmd = ["svn", "init", topurl, "--trunk="+trunk, "--tags="+tags", targetpath] cmd = ["svn", "init", url, abspath(targetpath)] self._execVcs(*cmd, **kwargs) environ.update({ "GIT_WORK_TREE": abspath(targetpath), "GIT_DIR": join(abspath(targetpath), ".git") }) if fullnames: usermap = UserTagParser() # store configuration in local git config so that'll be reused later when ie. updating gitconfig = { "svn-remote.authorlog.url": usermap.url, "svn-remote.authorlog.defaultmail": usermap.defaultmail } self.configset(gitconfig) if branch: execcmd(("git", "init", "-q", self.path), **kwargs) execcmd(("git", "checkout", "-q", branch), **kwargs) cmd = ["svn", "rebase", "--local"] status, output = self._execVcs(*cmd, **kwargs) return True
def _run(self, runobj: RunObject, execution): handler = runobj.spec.handler self._force_handler(handler) extra_env = self._generate_runtime_env(runobj) environ.update(extra_env) if not inspect.isfunction(handler): if not self.spec.command: raise ValueError( "specified handler (string) without command " "(py file path), specify command or use handler pointer") mod, handler = load_module(self.spec.command, handler) context = MLClientCtx.from_dict( runobj.to_dict(), rundb=self.spec.rundb, autocommit=False, host=socket.gethostname(), ) client = self.client setattr(context, "dask_client", client) sout, serr = exec_from_params(handler, runobj, context) log_std(self._db_conn, runobj, sout, serr, skip=self.is_child, show=False) return context.to_dict()
def initONOSEnv(): """Initialize ONOS environment (and module) variables This is ugly and painful, but they have to be set correctly in order for the onos-setup-karaf script to work. nodes: list of ONOS nodes returns: ONOS environment variable dict""" # pylint: disable=global-statement global HOME, ONOS_ROOT, ONOS_USER global ONOS_APPS, ONOS_WEB_USER, ONOS_WEB_PASS env = {} def sd(var, val): "Set default value for environment variable" env[var] = environ.setdefault(var, val) return env[var] assert environ['HOME'] HOME = sd('HOME', environ['HOME']) ONOS_ROOT = sd('ONOS_ROOT', join(HOME, 'onos')) environ['ONOS_USER'] = defaultUser() ONOS_USER = sd('ONOS_USER', defaultUser()) ONOS_APPS = sd('ONOS_APPS', 'drivers,openflow,fwd,proxyarp,mobility') # ONOS_WEB_{USER,PASS} isn't respected by onos-karaf: environ.update(ONOS_WEB_USER='******', ONOS_WEB_PASS='******') ONOS_WEB_USER = sd('ONOS_WEB_USER', 'karaf') ONOS_WEB_PASS = sd('ONOS_WEB_PASS', 'karaf') return env
def mirror_server(mirrors_dict): mirror_file_path = '/mirrors.json' mirror_corrupt_file_path = '/corrupt_mirrors.json' mirror_json_varname = 'ci_repos' mirror_data = {mirror_json_varname: mirrors_dict} mirror_json = json.dumps(mirror_data).encode('utf8') class MirrorRequestHandler(BaseHTTPRequestHandler): def do_GET(self): if self.path == mirror_file_path: self.send_response(200) self.send_header("Content-type", 'application/json') self.end_headers() self.wfile.write(mirror_json) elif self.path == mirror_corrupt_file_path: self.send_response(200) self.send_header("Content-type", 'application/json') self.end_headers() self.wfile.write('{"this": "is", "bad": "json"') else: self.send_error(404) for attempt in range(0, 20): server_address = ('127.0.0.1', randrange(8765, 8876)) try: server = HTTPServer(server_address, MirrorRequestHandler) except socket.error as e: if e.errno == 98: continue raise break else: raise RuntimeError("Failed to allocate port for mirror_server fixture") server_url = 'http://{0}:{1}'.format(*server_address) sthread = Thread(target=server.serve_forever) sthread.start() try: # Wait for http server to start sleep(0.1) # ensure we won't implictly try to use proxies to access local server old_env = dict(((k, environ.pop(k)) for k in ('http_proxy', 'HTTP_PROXY') if k in environ)) try: yield dict( mirror_url=urljoin(server_url, mirror_file_path), json_varname=mirror_json_varname, bad_path_url=urljoin(server_url, '/bad_file'), bad_port_url=urljoin( 'http://{0}:8764'.format(server_address[0]), mirror_file_path), corrupt_url=urljoin(server_url, mirror_corrupt_file_path), ) finally: environ.update(old_env) finally: server.shutdown() sthread.join()
def teardown_environment(): """Restore things that were remembered by the setup_environment function """ orig_env = GIVEN_ENV['env'] for key in env.keys(): if key not in orig_env: del env[key] env.update(orig_env)
def env(**vars): original = dict(environ) environ.update(vars) try: yield finally: environ.clear() environ.update(original)
def envvars(**kwargs): environ.update(kwargs) try: yield finally: # not trying to restore existing values for key in kwargs: del environ[key]
def source(script, update=1): pipe = Popen(". %s; env" % script, stdout=PIPE, shell=True) data = pipe.communicate()[0] env = dict((line.split("=", 1) for line in data.splitlines())) if update: environ.update(env) return env
def tests_setup_and_teardown(): original_environment = dict(environ) environ.update({ "TWILIO_ACCOUNT_SID": "TEST", "TWILIO_AUTH_TOKEN": "TOKEN", }) yield environ.clear() environ.update(original_environment)
def instance_path(): """Fixture for creating an instance path.""" path = tempfile.mkdtemp() environ.update( INVENIO_INSTANCE_PATH=environ.get('INSTANCE_PATH', path), ) yield path environ.pop('INVENIO_INSTANCE_PATH', None) shutil.rmtree(path)
def __enter__(self): self.old_path = old_path = environ.get("PATH", None) self.p = save_pkg_resources_state() _ = self.p.__enter__() # noqa: F841 self.m = save_modules() _ = self.m.__enter__() # noqa: F841 self.e = CleanEnv() self.e.__enter__() if old_path: first_colon = old_path.index(":") first_part = old_path[:first_colon] if first_part.endswith("venv/bin"): replacement_path = old_path[first_colon + 1:] environ.putenv("PATH", replacement_path) if self.venv_path is False: # if isinstance(args, list): # if args[0] == "python3": # args[0] = "/usr/bin/python3" # elif args[0] == "pip3": # args[0] = "/usr/bin/pip3" # elif args[0] == "python": # args[0] = "/usr/bin/python" # elif args[0] == "pip": # args[0] = "/usr/bin/pip" # elif isinstance(args, str): # if args.startswith("python3 "): # args = args.replace("python3", "/usr/bin/python3", 1) # elif args.startswith("python "): # args = args.replace("python", "/usr/bin/python", 1) # elif args.startswith("pip3 "): # args = args.replace("pip3", "/usr/bin/pip3", 1) # elif args.startswith("pip "): # args = args.replace("pip", "/usr/bin/pip", 1) pass else: venv_parent = path.dirname(self.venv_path) activate_location = path.join(self.venv_path, "bin", "activate") cmd2 = ". {} && echo ~~MARKER~~ && set".format(activate_location) env = (subprocess.Popen( cmd2, shell=True, cwd=venv_parent, stdout=subprocess.PIPE).stdout.read().decode( "utf-8").splitlines()) marker = False new_envs = {} for e in env: if marker: e = e.strip().split("=", 1) if len(e) > 1: name = str(e[0]).upper() if name in ("IFS", "OPTIND"): continue else: new_envs[name] = e[1].lstrip("'").rstrip("'") elif e.strip() == "~~MARKER~~": marker = True environ.update(new_envs)
async def wrapper(*args, **kwargs): # type: ignore _environ_copy = environ.copy() try: return await func(*args, **kwargs) finally: environ.clear() environ.update(_environ_copy) reload(consts) reload(utils)
def _patch_environ(**kwargs): '''Temporarily adds kwargs to os.environ''' try: orig_vars = {k: environ[k] for k in kwargs.keys() if k in environ} environ.update(kwargs) yield finally: environ.update(orig_vars) for extra_key in (kwargs.keys() - orig_vars.keys()): del environ[extra_key]
def teardown_environment(): """Restore things that were remembered by the setup_environment function """ orig_env = GIVEN_ENV['env'] # Pull keys out into list to avoid altering dictionary during iteration, # causing python 3 error for key in list(env.keys()): if key not in orig_env: del env[key] env.update(orig_env)
def init_spark_context(): environ.update({'SPARK_HOME': '/sparkdirectory'}) spark_home = environ.get('SPARK_HOME') sys.path.insert(0, spark_home + "/python") sys.path.insert(0, path.join(spark_home, 'python/lib/py4j-0.10.3-src.zip')) # load spark context conf = SparkConf().setAppName("SparkProject") sc = SparkContext(conf=conf, pyFiles=['main.py', 'modules.py']) return sc
def teardown_environment(): """Restore things that were remebered by the setup_environment function """ orig_env = GIVEN_ENV['env'] for key in env.keys(): if key not in orig_env: del env[key] env.update(orig_env) nud.get_nipy_system_dir = GIVEN_ENV['sys_dir_func'] nud.get_data_path = GIVEN_ENV['path_func']
def cmd_run(app, cmd): """Run a command inside the app, e.g.: ls -- -al""" app = exit_if_invalid(app) config_file = join(ENV_ROOT, app, 'LIVE_ENV') environ.update(parse_settings(config_file)) for f in [stdout, stderr]: fl = fcntl(f, F_GETFL) fcntl(f, F_SETFL, fl | O_NONBLOCK) p = Popen(' '.join(cmd), stdin=stdin, stdout=stdout, stderr=stderr, env=environ, cwd=join(APP_ROOT,app), shell=True) p.communicate()
def source(script,update=1): """ Source a script in a shell environment""" try: pipe = subprocess.Popen('. %s' % script, stdout=subprocess.PIPE, shell=True) data = pipe.communicate()[0] env = dict((line.split("=", 1) for line in data.splitlines())) if update: environ.update(env) return env except: logging.error("Something happened during %s" % funcname()) logging.error(sys.exc_info())
def test_env_var_config_no_split_sequence(self): def make_key(appname, key): return "{0}_{1}".format(appname.upper(), key.upper()) appname = "myapp" test_dict = {} test_dict[make_key(appname, 'channels')] = 'channel1' try: environ.update(test_dict) assert 'MYAPP_CHANNELS' in environ config = SampleConfiguration()._set_env_vars(appname) assert config.channels == ('channel1',) finally: [environ.pop(key) for key in test_dict]
def prepare_git_checkout(account, repo, ref, token): ''' ''' repo_href = 'https://github.com/%s/%s.git' % (account, repo) repo_path = join(getcwd(), 'repos/%s-%s' % (account, repo)) repo_refs = 'https://api.github.com/repos/%s/%s/branches' % (account, repo) repo_sha = 'https://api.github.com/repos/%s/%s/commits/%s' % (account, repo, ref) checkout_path = join(getcwd(), 'checkouts/%s-%s-%s' % (account, repo, ref)) checkout_lock = checkout_path + '.git-lock' if exists(checkout_path) and is_fresh(checkout_path): return checkout_path ref_check = OAuth2Session(github_client_id, token=token).get(repo_refs) if ref_check.status_code == 401: # Github wants authentication. raise PrivateRepoException() elif ref_check.status_code == 404: # This repository might not exist at all? raise MissingRepoException() branches = dict([(b['name'], b['commit']['sha']) for b in ref_check.json()]) ref_sha = branches.get(ref, None) if ref_sha is None: # The ref is not a branch, but it may be a sha. sha_check = OAuth2Session(github_client_id, token=token).get(repo_sha) if sha_check.status_code == 200: # The ref must be a sha hash. ref_sha = sha_check.json()['sha'] else: # The repository exists, but the branch does not? raise MissingRefException() if token: jlogger.debug('Adding Github credentials to environment') environ.update(dict(GIT_ASKPASS=join(dirname(__file__), 'askpass.py'))) environ.update(dict(GIT_USERNAME=token['access_token'], GIT_PASSWORD='')) else: jlogger.debug('Clearing Github credentials from environment') environ.update(dict(GIT_ASKPASS='', GIT_USERNAME='', GIT_PASSWORD='')) with locked_file(checkout_lock): if not exists(repo_path): git_clone(repo_href, repo_path) else: git_fetch(repo_path, ref, ref_sha) git_checkout(repo_path, checkout_path, ref) # Make sure these are gone before we return. environ.update(dict(GIT_ASKPASS='', GIT_USERNAME='', GIT_PASSWORD='')) return checkout_path
def updateNodeIPs( env, nodes ): "Update env dict and environ with node IPs" # Get rid of stale junk for var in 'ONOS_NIC', 'ONOS_CELL', 'ONOS_INSTANCES': env[ var ] = '' for var in environ.keys(): if var.startswith( 'OC' ): env[ var ] = '' for index, node in enumerate( nodes, 1 ): var = 'OC%d' % index env[ var ] = node.IP() env[ 'OCI' ] = env[ 'OCN' ] = env[ 'OC1' ] env[ 'ONOS_INSTANCES' ] = '\n'.join( node.IP() for node in nodes ) environ.update( env ) return env
def test_env_var_config_alias(self): def make_key(appname, key): return "{0}_{1}".format(appname.upper(), key.upper()) appname = "myapp" test_dict = {} test_dict[make_key(appname, 'yes')] = 'yes' test_dict[make_key(appname, 'changeps1')] = 'false' try: environ.update(test_dict) assert 'MYAPP_YES' in environ config = SampleConfiguration()._set_env_vars(appname) assert config.always_yes is True assert config.changeps1 is False finally: [environ.pop(key) for key in test_dict]
def test_env_var_config(self): def make_key(appname, key): return "{0}_{1}".format(appname.upper(), key.upper()) appname = "myapp" test_dict = {} test_dict[make_key(appname, 'always_yes')] = 'yes' test_dict[make_key(appname, 'changeps1')] = 'false' try: environ.update(test_dict) assert 'MYAPP_ALWAYS_YES' in environ raw_data = load_from_string_data('file1', 'file2') config = TestConfiguration(app_name=appname)._add_raw_data(raw_data) assert config.changeps1 is False assert config.always_yes is True finally: [environ.pop(key) for key in test_dict]
def initialize_config(config_file_name='env.yaml'): config_keys = ['DBSERVER', 'DBNAME', 'DBUSER', 'DBPASS', 'DBPORT', 'REDISHOST', 'REDISPORT', 'REDISPASS'] if contains(config_keys, list(environ.keys())): environ['DEBUG'] = 'False' return config_file_path = path.join(path.dirname(path.abspath(__file__)), config_file_name) if not path.exists(config_file_path): raise Exception('env.yaml required for config initialization') with open(config_file_path, 'r') as config_file: config = yaml.load(config_file) config['dbconfig']['DBPORT'] = str(config['dbconfig']['DBPORT']) config['redisconfig']['REDISPORT'] = str(config['redisconfig']['REDISPORT']) environ.update(config['dbconfig']) environ.update(config['redisconfig']) environ['DEBUG'] = 'True'
def __environ(values, remove=[]): """ Modify the environment for a test, adding/updating values in dict `values` and removing any environment variables mentioned in list `remove`. """ new_keys = set(environ.keys()) - set(values.keys()) old_environ = environ.copy() try: environ.update(values) for to_remove in remove: try: del environ[remove] except KeyError: pass yield finally: environ.update(old_environ) for key in new_keys: del environ[key]
def env_restored(unset = []): """ Create a temporary directory, with support for cleanup. """ # preserve the current environment from os import environ old = environ.copy() # arbitrarily modify it for name in unset: del environ[name] yield # then restore the preserved copy environ.clear() environ.update(old)
def replace_variables(input_s, variables=None, extra_env=None): """ Replace variables like ${foo} with matching from `variables`. Also works with environment variables ${env.bar}. :arg input_s :type StringType :arg variables :type DictType :arg extra_env :type DictType :return parsed_string :type StringType """ assert isinstance(input_s, basestring) variables = variables or {} # Linters complain when default arg set to `{}` extra_env = extra_env or {} # Linters complain when default arg set to `{}` if len(input_s) < 3: return input_s environ.update(extra_env) parsed_string = '' possible_var = '' # Scanner for c in input_s: if c == '$': possible_var += c elif len(possible_var) == 1 and c != '{': parsed_string += possible_var + c possible_var = '' elif c == '}' and possible_var: possible_var += c var = possible_var[len('${'):-len('}')] parsed_string += ((lambda res: res.replace(path.sep, path.sep + path.sep) if path.sep == '\\' else res )(environ.get(var[len('env.'):], '')) or possible_var) \ if var.startswith('env.') else variables.get(var, possible_var) possible_var = '' elif possible_var: possible_var += c else: parsed_string += c return parsed_string
def source_bash(setup_script): foop = open("tmp.sh", "w") foop.write("#/bin/bash\nsource $1\nenv|sort") foop.close() out, err, rc = run(["bash tmp.sh %s" % expandvars(setup_script)], useLogging=False, suppressErrors=True) if rc: print 'source encountered error, returning that one' return err lines = [l for l in out.split("\n") if "=" in l] keys, values = [], [] for l in lines: tl = l.split("=") if len(tl) == 2: if tl[0].startswith("_"): continue if tl[0].startswith("BASH_FUNC"): continue keys.append(tl[0]) values.append(tl[1]) environ.update(dict(zip(keys, values))) remove("tmp.sh") return
# Setup the require environmental variables for QSTK # http://wiki.quantsoftware.org/index.php?title=QuantSoftware_ToolKit from os import environ from os.path import join from sys import path HOME = '/home/emilmont' QS = join(HOME, 'Software/QSTK') QSDATA = join(HOME, 'Data/QSData') environ.update({ 'QS' : QS, 'QSDATA' : QSDATA, 'QSDATAPROCESSED': join(QSDATA, 'Processed'), 'QSDATATMP' : join(QSDATA, 'Tmp'), 'QSBIN' : join(QS, 'Bin'), 'QSSCRATCH' : join(QSDATA, 'Scratch'), 'CACHESTALLTIME' : '12', }) path.append(QS)
try: from signal import SIGTERM as sig except ImportError: print "couldn't import signal module" sig = 15 os.kill(pid, sig) except OSError: # process is already dead, so ignore it pass except: traceback.print_exc() if __name__ == '__main__': try: # treat all arguments as environment settings Env.update(getenv(sys.argv[1:])) # user name is used in log file name to avoid permission problems user = Env.get('USER') or Env.get('USERNAME') or \ _winuser or 'nwsuser' f = 'MatlabSleighSentinelLog_' + user + '_' + \ Env.get('MatlabSleighID', 'X') + '.txt' setup(f) if not Env.has_key('MatlabSleighNwsName'): print "MatlabSleighNwsName variable is not set" print >> sys.__stderr__, "MatlabSleighNwsName variable is not set" else: main() except: traceback.print_exc()
from distutils.core import setup, Command from distutils.command.install_data import install_data from distutils.dir_util import remove_tree from distutils import log from os import path, makedirs, walk, environ from shutil import copyfile from subprocess import call from sys import version_info if version_info[0] >= 3: from io import FileIO file = FileIO from falias import __version__ environ.update({'PYTHONPATH': 'falias'}) def find_data_files(directory, targetFolder=""): def skip(name): return (name[0] != '.' and name[-1] != '~') rv = [] for root, dirs, files in walk(directory): if targetFolder: rv.append((targetFolder, list(root+'/'+f for f in files if skip(f)))) else: rv.append((root, list(root+'/'+f for f in files if skip(f)))) log.info(str(rv))
from distutils.dir_util import remove_tree from distutils import log from os import path, makedirs, walk, environ from shutil import copyfile from subprocess import call from sys import version_info if version_info[0] >= 3: from io import FileIO file = FileIO from poorwsgi.state import __version__ environ.update({"PYTHONPATH": "poorwsgi"}) def find_data_files(directory, targetFolder=""): rv = [] for root, dirs, files in walk(directory): if targetFolder: rv.append((targetFolder, list(root + "/" + f for f in files if f[0] != "." and f[-1] != "~"))) else: rv.append((root, list(root + "/" + f for f in files if f[0] != "." and f[-1] != "~"))) log.info(str(rv)) return rv class build_html(Command): description = "build html documentation, need jinja24doc >= 1.1.0"
from dotenv import Dotenv from os import path, environ as env # First, parse env file and splice it into the OS environment envpath = path.join(path.dirname(__file__), '..', '.env') dotenv = Dotenv(envpath) env.update(dotenv) # Now, parse and set the settings dict SETTINGS = { 'DEBUG': env.get('MODUS_DEBUG', '').lower() == 'true', 'DB_FILE': env.get('MODUS_DB_FILE', '').lower(), }
""" Runs Faceoff as a standalone web server. Do not use this is in production. Copyright: (c) 2012-2014 Artem Nezvigin <*****@*****.**> License: MIT, see LICENSE for details """ from sys import path from os import environ from os.path import dirname, join path.append(join(dirname(__file__), '..')) from argparse import ArgumentParser parser = ArgumentParser(usage='%(prog)s [options]', argument_default='') parser.add_argument('--host', metavar='<string>', dest='FACEOFF_HOST', default='0') parser.add_argument('--port', metavar='<string>', dest='FACEOFF_PORT', default='5000') parser.add_argument('--config', metavar='<string>', dest='FACEOFF_CONFIG') parser.add_argument('--debug', action='store_const', const='1', dest='FACEOFF_DEBUG') parser.add_argument('--db-path', metavar='<string>', dest='FACEOFF_DB_PATH') parser.add_argument('--db-fixtures', action='store_const', const='1', dest='FACEOFF_DB_FIXTURES') parser.add_argument('--log-path', metavar='<string>', dest='FACEOFF_LOG_PATH') parser.add_argument('--log-level', metavar='<string>', dest='FACEOFF_LOG_LEVEL') parser.add_argument('--log-filter', metavar='<string>', dest='FACEOFF_LOG_FILTER') parser.add_argument('--log-ignore', metavar='<string>', dest='FACEOFF_LOG_IGNORE') args = parser.parse_args() environ.update(vars(args)) from faceoff import app app.run(args.FACEOFF_HOST, int(args.FACEOFF_PORT), threaded=True)