def maybe_activate_venv():
if not environ.get('VIRTUAL_ENV'):
from pathlib import Path
proj_root = Path(__file__).resolve().parents[1]
for venv_name in ('venv', '.venv'):
venv_path = proj_root.joinpath(venv_name)
if venv_path.exists():
break
else:
return
import platform
import sys
from subprocess import call
on_windows = platform.system().lower() == 'windows'
bin_path = venv_path.joinpath('Scripts' if on_windows else 'bin')
environ.update(
PYTHONHOME='',
VIRTUAL_ENV=venv_path.as_posix(),
PATH='{}:{}'.format(bin_path.as_posix(), environ['PATH']),
)
cmd = [
bin_path.joinpath(
'python.exe' if on_windows else 'python').as_posix()
] + sys.argv
sys.exit(call(cmd, env=environ))
def update_env(**kwargs):
"""Update environment and then set it back."""
start = environ.copy()
environ.update(**kwargs)
yield
environ.clear()
environ.update(**start)
def source(script, update=True, clean=True):
"""
Source variables from a shell script
import them in the environment (if update==True)
and report only the script variables (if clean==True)
"""
global environ
environ_back = None
if clean:
environ_back = dict(environ)
environ.clear()
pipe = Popen(". %s; env" % script, stdout=PIPE, shell=True)
data = pipe.communicate()[0]
env = dict(line.split("=", 1) for line in data.splitlines())
if clean:
# remove unwanted minimal vars
env.pop('LINES', None)
env.pop('COLUMNS', None)
environ.update(environ_back)
if update:
environ.update(env)
return env
def teardown(self, cleanup_test_bench=False):
"""Tear down the environment"""
environ.update(self.old_environ)
if cleanup_test_bench:
self._cleanup_content(self.dump_directory)
else:
self.cleanup_index_testbench()
def install(name, keep_prefix=False, keep_stage=False, ignore_deps=False, environs=None, compiler=None):
_init_spack()
from spack import repo, installed_db
from spack.cmd import parse_specs
from os import environ
if not isinstance(name, str):
results = [], []
for pkg in name:
a, b = install(pkg, keep_prefix=keep_prefix, keep_stage=keep_stage,
ignore_deps=ignore_deps, environs=environs,
compiler=compiler)
results[0].extend(a)
results[1].extend(b)
return results
if environs is not None:
environ.update(environs)
if compiler is not None:
names = name.split()
names.insert(1, "%" + compiler.rstrip().lstrip())
name = ' '.join(names)
specs = parse_specs(name, concretize=True)
packages = [repo.get(spec) for spec in specs]
new_pkgs = [u for u in packages if not u.installed]
for package in new_pkgs:
with installed_db.write_transaction():
package.do_install(
keep_prefix=keep_prefix,
keep_stage=keep_stage,
ignore_deps=ignore_deps
)
return [p.name for p in new_pkgs if p.installed], \
[p.name for p in new_pkgs if not p.installed]
def setUp(self):
self.defaults = {
'intopt': 11,
'floatopt':11.1,
'boolopttrue': False,
'booloptfalse': False,
'stropt':'stroptval_default',
'intopt_str': 2
}
self.filecontent = {
'intopt': 111,
'floatopt':111.1,
'boolopttrue': False,
'booloptfalse': False,
'stropt':'stroptval_file',
'intopt_str': 2
}
self.filename = './testsettings.json'
with open(self.filename, 'w') as f:
f.write(json.dumps(self.filecontent))
environ.update({k: str(self.reference[k]) for k in self.reference.keys()})
environ[self.not_default_key] = 'only_file_value'
self.settings = Settings(defaults=self.defaults, filename=self.filename)
def test_strict_parsing():
importlib.reload(expandvars)
with pytest.raises(
expandvars.ExpandvarsException, match="FOO: parameter null or not set"
) as e:
expandvars.expandvars("${FOO:?}")
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
with pytest.raises(
expandvars.ExpandvarsException, match="FOO: parameter null or not set"
) as e:
expandvars.expandvars("${FOO?}")
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
with pytest.raises(expandvars.ExpandvarsException, match="FOO: custom error") as e:
expandvars.expandvars("${FOO:?custom error}")
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
with pytest.raises(expandvars.ExpandvarsException, match="FOO: custom error") as e:
expandvars.expandvars("${FOO?custom error}")
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
env.update({"FOO": "foo"})
assert expandvars.expandvars("${FOO:?custom err}") == "foo"
assert expandvars.expandvars("${FOO?custom err}:bar") == "foo:bar"
def do_install(reqs):
from os import environ
previously_installed = pip_get_installed()
required = pip_parse_requirements(reqs)
requirements_as_options = tuple(
'--requirement={0}'.format(requirement) for requirement in reqs
)
# We put the cache in the directory that pip already uses.
# This has better security characteristics than a machine-wide cache, and is a
# pattern people can use for open-source projects
pipdir = environ['HOME'] + '/.pip'
# We could combine these caches to one directory, but pip would search everything twice, going slower.
pip_download_cache = pipdir + '/cache'
pip_wheels = pipdir + '/wheelhouse'
environ.update(
PIP_DOWNLOAD_CACHE=pip_download_cache,
)
cache_opts = (
'--download-cache=' + pip_download_cache,
'--find-links=file://' + pip_wheels,
)
# --use-wheel is somewhat redundant here, but it means we get an error if we have a bad version of pip/setuptools.
install_opts = ('--upgrade', '--use-wheel',) + cache_opts
recently_installed = []
# 1) Bootstrap the install system; setuptools and pip are already installed, just need wheel
recently_installed += pip_install(install_opts + BOOTSTRAP_VERSIONS)
# 2) Caching: Make sure everything we want is downloaded, cached, and has a wheel.
pip(
('wheel', '--wheel-dir=' + pip_wheels) +
BOOTSTRAP_VERSIONS +
cache_opts +
requirements_as_options
)
# 3) Install: Use our well-populated cache, to do the installations.
install_opts += ('--no-index',) # only use the cache
recently_installed += pip_install(install_opts + requirements_as_options)
required_with_deps = trace_requirements(required)
# TODO-TEST require A==1 then A==2
extraneous = (
reqnames(previously_installed) -
reqnames(required_with_deps) -
reqnames(recently_installed)
)
# 2) Uninstall any extraneous packages.
if extraneous:
pip(('uninstall', '--yes') + tuple(sorted(extraneous)))
return 0 # posix:success!
def clean_state(self):
"""
Cleans up the state.
"""
if path.exists(self.state_file):
remove(self.state_file)
environ.update(self.old_environ)
def __init__(self, *args, **kwargs):
if 'nmpi' in kwargs:
self.nmpi = kwargs['nmpi']
del kwargs['nmpi']
else:
self.nmpi = 1
if 'mpicompiler' in kwargs:
self.compiler = kwargs['mpicompiler']
del kwargs['mpicompiler']
else:
self.compiler = None
super().__init__(*args, **kwargs)
environ_backup = dict(environ)
self.set_compiler()
super().build(*args)
environ.clear()
environ.update(environ_backup)
self.executable = 'start.exe'
self.create_executable(**kwargs)
super().clean_module()
def teardown(self, cleanup_test_bench=False):
"""Tear down the environment"""
environ.update(self.old_environ)
if cleanup_test_bench:
self._cleanup_content(self.dump_directory)
else:
self.cleanup_index_testbench()
def on_context_leave(self):
''' Called when the context manager entered with
:func:`magic.enter_context` is exited. Undos all of the stuff
that :meth:`on_context_enter` did and more.
* Stop the Craftr Runtime Server
* Restore the :data:`os.environ` dictionary
* Removes all ``craftr.ext.`` modules from :data:`sys.modules` and
ensures they are in :attr:`Session.modules` (they are expected to
be put there from the :class:`ext.CraftrImporter`).
'''
self._stop_server()
# Restore the original values of os.environ.
self.env = environ.copy()
environ.clear()
environ.update(self._old_environ)
del self._old_environ
sys.meta_path.remove(self.ext_importer)
for key, module in list(sys.modules.items()):
if key.startswith('craftr.ext.'):
name = key[11:]
assert name in self.modules and self.modules[
name] is module, key
del sys.modules[key]
try:
# Remove the module from the `craftr.ext` modules contents, too.
delattr(ext, name.split('.')[0])
except AttributeError:
pass
def on_context_enter(self, prev):
''' Called when entering the Session context with
:func:`magic.enter_context`. Does the following things:
* Sets up the :data`os.environ` with the values from :attr:`Session.env`
* Adds the :attr:`Session.ext_importer` to :data:`sys.meta_path`
* Starts the Craftr Runtime Server (:attr:`Session.server`) and sets
the ``CRAFTR_RTS`` environment variable
.. note:: A copy of the original :data:`os.environ` is saved and later
restored in :meth:`on_context_leave`. The :data:`os.environ` object
can not be replaced by another object, that is why we change its
values in-place.
'''
if prev is not None:
raise RuntimeError('session context can not be nested')
# We can not change os.environ effectively, we must update the
# dictionary instead.
self._old_environ = environ.copy()
environ.clear()
environ.update(self.env)
self.env = environ
sys.meta_path.append(self.ext_importer)
self.update()
def __init__(self, context):
# Context associated attributes
self.log = context.log
self.run_name = context.pipeline_run.pipeline_name
self.dagster_run_id = context.run_id
# resource config attributes
resource_config = context.resource_config
self.tracking_uri = resource_config.get("mlflow_tracking_uri")
if self.tracking_uri:
mlflow.set_tracking_uri(self.tracking_uri)
self.parent_run_id = resource_config.get("parent_run_id")
self.experiment_name = resource_config["experiment_name"]
self.env_tags_to_log = resource_config.get("env_to_tag") or []
self.extra_tags = resource_config.get("extra_tags")
# Update env variables if any are given
self.env_vars = resource_config.get("env", {})
if self.env_vars:
environ.update(self.env_vars)
# If the experiment exists then the set won't do anything
mlflow.set_experiment(self.experiment_name)
self.experiment = mlflow.get_experiment_by_name(self.experiment_name)
# Get the client object
self.tracking_client = mlflow.tracking.MlflowClient()
# Set up the active run and tags
self._setup()
def on_context_leave(self):
''' Called when the context manager entered with
:func:`magic.enter_context` is exited. Undos all of the stuff
that :meth:`on_context_enter` did and more.
* Stop the Craftr Runtime Server
* Restore the :data:`os.environ` dictionary
* Removes all ``craftr.ext.`` modules from :data:`sys.modules` and
ensures they are in :attr:`Session.modules` (they are expected to
be put there from the :class:`ext.CraftrImporter`).
'''
self._stop_server()
# Restore the original values of os.environ.
self.env = environ.copy()
environ.clear()
environ.update(self._old_environ)
del self._old_environ
sys.meta_path.remove(self.ext_importer)
for key, module in list(sys.modules.items()):
if key.startswith('craftr.ext.'):
name = key[11:]
assert name in self.modules and self.modules[name] is module, key
del sys.modules[key]
try:
# Remove the module from the `craftr.ext` modules contents, too.
delattr(ext, name.split('.')[0])
except AttributeError:
pass
def export(envvars):
try:
environ.update(envvars)
yield
finally:
for key in envvars:
del environ[key]
def initONOSEnv():
"""Initialize ONOS environment (and module) variables
This is ugly and painful, but they have to be set correctly
in order for the onos-setup-karaf script to work.
nodes: list of ONOS nodes
returns: ONOS environment variable dict"""
# pylint: disable=global-statement
global HOME, ONOS_ROOT, ONOS_USER
global ONOS_APPS, ONOS_WEB_USER, ONOS_WEB_PASS
env = {}
def sd(var, val):
"Set default value for environment variable"
env[var] = environ.setdefault(var, val)
return env[var]
assert environ['HOME']
HOME = sd('HOME', environ['HOME'])
ONOS_ROOT = sd('ONOS_ROOT', join(HOME, 'onos'))
environ['ONOS_USER'] = defaultUser()
ONOS_USER = sd('ONOS_USER', defaultUser())
# Set to items passed in the "controllerConfig.ini"
config = ConfigParser()
config.read('controllerConfig.ini')
assert config.get("ONOS", "rules")
ONOS_APPS = sd('ONOS_APPS', config.get("ONOS", "rules"))
# ONOS_WEB_{USER,PASS} isn't respected by onos-karaf:
environ.update(ONOS_WEB_USER='******', ONOS_WEB_PASS='******')
ONOS_WEB_USER = sd('ONOS_WEB_USER', 'karaf')
ONOS_WEB_PASS = sd('ONOS_WEB_PASS', 'karaf')
return env
def on_context_enter(self, prev):
''' Called when entering the Session context with
:func:`magic.enter_context`. Does the following things:
* Sets up the :data`os.environ` with the values from :attr:`Session.env`
* Adds the :attr:`Session.ext_importer` to :data:`sys.meta_path`
* Starts the Craftr Runtime Server (:attr:`Session.server`) and sets
the ``CRAFTR_RTS`` environment variable
.. note:: A copy of the original :data:`os.environ` is saved and later
restored in :meth:`on_context_leave`. The :data:`os.environ` object
can not be replaced by another object, that is why we change its
values in-place.
'''
if prev is not None:
raise RuntimeError('session context can not be nested')
# We can not change os.environ effectively, we must update the
# dictionary instead.
self._old_environ = environ.copy()
environ.clear()
environ.update(self.env)
self.env = environ
sys.meta_path.append(self.ext_importer)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--use-env',
action='store_true',
help='Use existing TINYSERV_* environment variables.'
)
args = parser.parse_args()
if not args.use_env:
tinyserv_bin_path = path.normpath(path.join(ROOT, '..'))
environ.update({
'TINYSERV_REMOTE': 'localhost',
'TINYSERV_ROOT': path.join(ROOT, '.tinyserv'),
'TINYSERV_START_PORT': '5000',
'PATH': tinyserv_bin_path + path.pathsep + os.environ['PATH']
})
remote = environ['TINYSERV_REMOTE']
host = remote.split('@')[-1]
print "Running smoke test on %s." % host
describe("Setting up sample 'tinysmoke' repository.")
chdir(ROOT)
if path.exists("tinysmoke"):
run("rm -rf tinysmoke")
try:
run("tinyserv apps:destroy tinysmoke", silent=True)
except subprocess.CalledProcessError, e:
if "invalid choice: 'tinysmoke'" not in e.output:
print "Failed process output follows."
print e.output
raise
def initONOSEnv():
"""Initialize ONOS environment (and module) variables
This is ugly and painful, but they have to be set correctly
in order for the onos-setup-karaf script to work.
nodes: list of ONOS nodes
returns: ONOS environment variable dict"""
# pylint: disable=global-statement
global HOME, ONOS_ROOT, ONOS_USER
global ONOS_APPS, ONOS_WEB_USER, ONOS_WEB_PASS
env = {}
def sd( var, val ):
"Set default value for environment variable"
env[ var ] = environ.setdefault( var, val )
return env[ var ]
assert environ[ 'HOME' ]
HOME = sd( 'HOME', environ[ 'HOME' ] )
ONOS_ROOT = sd( 'ONOS_ROOT', join( HOME, 'onos' ) )
environ[ 'ONOS_USER' ] = defaultUser()
ONOS_USER = sd( 'ONOS_USER', defaultUser() )
ONOS_APPS = sd( 'ONOS_APPS',
'drivers,openflow,fwd,proxyarp,mobility' )
# ONOS_WEB_{USER,PASS} isn't respected by onos-karaf:
environ.update( ONOS_WEB_USER='******', ONOS_WEB_PASS='******' )
ONOS_WEB_USER = sd( 'ONOS_WEB_USER', 'karaf' )
ONOS_WEB_PASS = sd( 'ONOS_WEB_PASS', 'karaf' )
return env
def _source(script, replace=None, update=True):
"""
Source variables from a shell script
import them in the environment (if update==True)
"""
from subprocess import Popen, PIPE
from os import environ
import os
if os.path.isfile(script) is False:
errmsg = "'%s' is not available" % (script)
raise ValueError(errmsg)
else:
print(" Reading Environmental Variables from %s" % (script))
pipe = Popen(". %s > /dev/null 2>&1; env" % script,
stdout=PIPE,
shell=True)
data = pipe.communicate()[0]
env = dict((line.split("=", 1) for line in data.splitlines()))
if replace is not None:
for key in env.keys():
value = env[key]
if replace[0] in value:
env[key] = env[key].replace(replace[0], replace[1])
if update:
environ.update(env)
else:
return env
def init(self, url, targetpath, fullnames=True, branch=None, **kwargs):
# verify repo url
execcmd("svn", "info", url)
topurl = dirname(url)
trunk = basename(url)
tags = "releases"
# cloning svn braches as well should rather be optionalif reenabled..
#cmd = ["svn", "init", topurl, "--trunk="+trunk, "--tags="+tags", targetpath]
cmd = ["svn", "init", url, abspath(targetpath)]
self._execVcs(*cmd, **kwargs)
environ.update({
"GIT_WORK_TREE": abspath(targetpath),
"GIT_DIR": join(abspath(targetpath), ".git")
})
if fullnames:
usermap = UserTagParser()
# store configuration in local git config so that'll be reused later when ie. updating
gitconfig = {
"svn-remote.authorlog.url": usermap.url,
"svn-remote.authorlog.defaultmail": usermap.defaultmail
}
self.configset(gitconfig)
if branch:
execcmd(("git", "init", "-q", self.path), **kwargs)
execcmd(("git", "checkout", "-q", branch), **kwargs)
cmd = ["svn", "rebase", "--local"]
status, output = self._execVcs(*cmd, **kwargs)
return True
def _run(self, runobj: RunObject, execution):
handler = runobj.spec.handler
self._force_handler(handler)
extra_env = self._generate_runtime_env(runobj)
environ.update(extra_env)
if not inspect.isfunction(handler):
if not self.spec.command:
raise ValueError(
"specified handler (string) without command "
"(py file path), specify command or use handler pointer")
mod, handler = load_module(self.spec.command, handler)
context = MLClientCtx.from_dict(
runobj.to_dict(),
rundb=self.spec.rundb,
autocommit=False,
host=socket.gethostname(),
)
client = self.client
setattr(context, "dask_client", client)
sout, serr = exec_from_params(handler, runobj, context)
log_std(self._db_conn,
runobj,
sout,
serr,
skip=self.is_child,
show=False)
return context.to_dict()
def initONOSEnv():
"""Initialize ONOS environment (and module) variables
This is ugly and painful, but they have to be set correctly
in order for the onos-setup-karaf script to work.
nodes: list of ONOS nodes
returns: ONOS environment variable dict"""
# pylint: disable=global-statement
global HOME, ONOS_ROOT, ONOS_USER
global ONOS_APPS, ONOS_WEB_USER, ONOS_WEB_PASS
env = {}
def sd(var, val):
"Set default value for environment variable"
env[var] = environ.setdefault(var, val)
return env[var]
assert environ['HOME']
HOME = sd('HOME', environ['HOME'])
ONOS_ROOT = sd('ONOS_ROOT', join(HOME, 'onos'))
environ['ONOS_USER'] = defaultUser()
ONOS_USER = sd('ONOS_USER', defaultUser())
ONOS_APPS = sd('ONOS_APPS', 'drivers,openflow,fwd,proxyarp,mobility')
# ONOS_WEB_{USER,PASS} isn't respected by onos-karaf:
environ.update(ONOS_WEB_USER='******', ONOS_WEB_PASS='******')
ONOS_WEB_USER = sd('ONOS_WEB_USER', 'karaf')
ONOS_WEB_PASS = sd('ONOS_WEB_PASS', 'karaf')
return env
def mirror_server(mirrors_dict):
mirror_file_path = '/mirrors.json'
mirror_corrupt_file_path = '/corrupt_mirrors.json'
mirror_json_varname = 'ci_repos'
mirror_data = {mirror_json_varname: mirrors_dict}
mirror_json = json.dumps(mirror_data).encode('utf8')
class MirrorRequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == mirror_file_path:
self.send_response(200)
self.send_header("Content-type", 'application/json')
self.end_headers()
self.wfile.write(mirror_json)
elif self.path == mirror_corrupt_file_path:
self.send_response(200)
self.send_header("Content-type", 'application/json')
self.end_headers()
self.wfile.write('{"this": "is", "bad": "json"')
else:
self.send_error(404)
for attempt in range(0, 20):
server_address = ('127.0.0.1', randrange(8765, 8876))
try:
server = HTTPServer(server_address, MirrorRequestHandler)
except socket.error as e:
if e.errno == 98:
continue
raise
break
else:
raise RuntimeError("Failed to allocate port for mirror_server fixture")
server_url = 'http://{0}:{1}'.format(*server_address)
sthread = Thread(target=server.serve_forever)
sthread.start()
try:
# Wait for http server to start
sleep(0.1)
# ensure we won't implictly try to use proxies to access local server
old_env = dict(((k, environ.pop(k))
for k in ('http_proxy', 'HTTP_PROXY') if k in environ))
try:
yield dict(
mirror_url=urljoin(server_url, mirror_file_path),
json_varname=mirror_json_varname,
bad_path_url=urljoin(server_url, '/bad_file'),
bad_port_url=urljoin(
'http://{0}:8764'.format(server_address[0]),
mirror_file_path),
corrupt_url=urljoin(server_url, mirror_corrupt_file_path),
)
finally:
environ.update(old_env)
finally:
server.shutdown()
sthread.join()
def teardown_environment():
"""Restore things that were remembered by the setup_environment function
"""
orig_env = GIVEN_ENV['env']
for key in env.keys():
if key not in orig_env:
del env[key]
env.update(orig_env)
def teardown_environment():
"""Restore things that were remembered by the setup_environment function
"""
orig_env = GIVEN_ENV['env']
for key in env.keys():
if key not in orig_env:
del env[key]
env.update(orig_env)
def env(**vars):
original = dict(environ)
environ.update(vars)
try:
yield
finally:
environ.clear()
environ.update(original)
def envvars(**kwargs):
environ.update(kwargs)
try:
yield
finally:
# not trying to restore existing values
for key in kwargs:
del environ[key]
def source(script, update=1):
pipe = Popen(". %s; env" % script, stdout=PIPE, shell=True)
data = pipe.communicate()[0]
env = dict((line.split("=", 1) for line in data.splitlines()))
if update:
environ.update(env)
return env
def tests_setup_and_teardown():
original_environment = dict(environ)
environ.update({
"TWILIO_ACCOUNT_SID": "TEST",
"TWILIO_AUTH_TOKEN": "TOKEN",
})
yield
environ.clear()
environ.update(original_environment)
def instance_path():
"""Fixture for creating an instance path."""
path = tempfile.mkdtemp()
environ.update(
INVENIO_INSTANCE_PATH=environ.get('INSTANCE_PATH', path),
)
yield path
environ.pop('INVENIO_INSTANCE_PATH', None)
shutil.rmtree(path)
def __enter__(self):
self.old_path = old_path = environ.get("PATH", None)
self.p = save_pkg_resources_state()
_ = self.p.__enter__() # noqa: F841
self.m = save_modules()
_ = self.m.__enter__() # noqa: F841
self.e = CleanEnv()
self.e.__enter__()
if old_path:
first_colon = old_path.index(":")
first_part = old_path[:first_colon]
if first_part.endswith("venv/bin"):
replacement_path = old_path[first_colon + 1:]
environ.putenv("PATH", replacement_path)
if self.venv_path is False:
# if isinstance(args, list):
# if args[0] == "python3":
# args[0] = "/usr/bin/python3"
# elif args[0] == "pip3":
# args[0] = "/usr/bin/pip3"
# elif args[0] == "python":
# args[0] = "/usr/bin/python"
# elif args[0] == "pip":
# args[0] = "/usr/bin/pip"
# elif isinstance(args, str):
# if args.startswith("python3 "):
# args = args.replace("python3", "/usr/bin/python3", 1)
# elif args.startswith("python "):
# args = args.replace("python", "/usr/bin/python", 1)
# elif args.startswith("pip3 "):
# args = args.replace("pip3", "/usr/bin/pip3", 1)
# elif args.startswith("pip "):
# args = args.replace("pip", "/usr/bin/pip", 1)
pass
else:
venv_parent = path.dirname(self.venv_path)
activate_location = path.join(self.venv_path, "bin", "activate")
cmd2 = ". {} && echo ~~MARKER~~ && set".format(activate_location)
env = (subprocess.Popen(
cmd2, shell=True, cwd=venv_parent,
stdout=subprocess.PIPE).stdout.read().decode(
"utf-8").splitlines())
marker = False
new_envs = {}
for e in env:
if marker:
e = e.strip().split("=", 1)
if len(e) > 1:
name = str(e[0]).upper()
if name in ("IFS", "OPTIND"):
continue
else:
new_envs[name] = e[1].lstrip("'").rstrip("'")
elif e.strip() == "~~MARKER~~":
marker = True
environ.update(new_envs)
def source(script, update=1):
pipe = Popen(". %s; env" % script, stdout=PIPE, shell=True)
data = pipe.communicate()[0]
env = dict((line.split("=", 1) for line in data.splitlines()))
if update:
environ.update(env)
return env
async def wrapper(*args, **kwargs): # type: ignore
_environ_copy = environ.copy()
try:
return await func(*args, **kwargs)
finally:
environ.clear()
environ.update(_environ_copy)
reload(consts)
reload(utils)
def _patch_environ(**kwargs):
'''Temporarily adds kwargs to os.environ'''
try:
orig_vars = {k: environ[k] for k in kwargs.keys() if k in environ}
environ.update(kwargs)
yield
finally:
environ.update(orig_vars)
for extra_key in (kwargs.keys() - orig_vars.keys()):
del environ[extra_key]
def teardown_environment():
"""Restore things that were remembered by the setup_environment function
"""
orig_env = GIVEN_ENV['env']
# Pull keys out into list to avoid altering dictionary during iteration,
# causing python 3 error
for key in list(env.keys()):
if key not in orig_env:
del env[key]
env.update(orig_env)
def init_spark_context():
environ.update({'SPARK_HOME': '/sparkdirectory'})
spark_home = environ.get('SPARK_HOME')
sys.path.insert(0, spark_home + "/python")
sys.path.insert(0, path.join(spark_home, 'python/lib/py4j-0.10.3-src.zip'))
# load spark context
conf = SparkConf().setAppName("SparkProject")
sc = SparkContext(conf=conf, pyFiles=['main.py', 'modules.py'])
return sc
def teardown_environment():
"""Restore things that were remebered by the setup_environment function
"""
orig_env = GIVEN_ENV['env']
for key in env.keys():
if key not in orig_env:
del env[key]
env.update(orig_env)
nud.get_nipy_system_dir = GIVEN_ENV['sys_dir_func']
nud.get_data_path = GIVEN_ENV['path_func']
def cmd_run(app, cmd):
"""Run a command inside the app, e.g.: ls -- -al"""
app = exit_if_invalid(app)
config_file = join(ENV_ROOT, app, 'LIVE_ENV')
environ.update(parse_settings(config_file))
for f in [stdout, stderr]:
fl = fcntl(f, F_GETFL)
fcntl(f, F_SETFL, fl | O_NONBLOCK)
p = Popen(' '.join(cmd), stdin=stdin, stdout=stdout, stderr=stderr, env=environ, cwd=join(APP_ROOT,app), shell=True)
p.communicate()
def source(script,update=1):
""" Source a script in a shell environment"""
try:
pipe = subprocess.Popen('. %s' % script, stdout=subprocess.PIPE, shell=True)
data = pipe.communicate()[0]
env = dict((line.split("=", 1) for line in data.splitlines()))
if update:
environ.update(env)
return env
except:
logging.error("Something happened during %s" % funcname())
logging.error(sys.exc_info())
def test_env_var_config_no_split_sequence(self):
def make_key(appname, key):
return "{0}_{1}".format(appname.upper(), key.upper())
appname = "myapp"
test_dict = {}
test_dict[make_key(appname, 'channels')] = 'channel1'
try:
environ.update(test_dict)
assert 'MYAPP_CHANNELS' in environ
config = SampleConfiguration()._set_env_vars(appname)
assert config.channels == ('channel1',)
finally:
[environ.pop(key) for key in test_dict]
def prepare_git_checkout(account, repo, ref, token):
'''
'''
repo_href = 'https://github.com/%s/%s.git' % (account, repo)
repo_path = join(getcwd(), 'repos/%s-%s' % (account, repo))
repo_refs = 'https://api.github.com/repos/%s/%s/branches' % (account, repo)
repo_sha = 'https://api.github.com/repos/%s/%s/commits/%s' % (account, repo, ref)
checkout_path = join(getcwd(), 'checkouts/%s-%s-%s' % (account, repo, ref))
checkout_lock = checkout_path + '.git-lock'
if exists(checkout_path) and is_fresh(checkout_path):
return checkout_path
ref_check = OAuth2Session(github_client_id, token=token).get(repo_refs)
if ref_check.status_code == 401:
# Github wants authentication.
raise PrivateRepoException()
elif ref_check.status_code == 404:
# This repository might not exist at all?
raise MissingRepoException()
branches = dict([(b['name'], b['commit']['sha']) for b in ref_check.json()])
ref_sha = branches.get(ref, None)
if ref_sha is None:
# The ref is not a branch, but it may be a sha.
sha_check = OAuth2Session(github_client_id, token=token).get(repo_sha)
if sha_check.status_code == 200:
# The ref must be a sha hash.
ref_sha = sha_check.json()['sha']
else:
# The repository exists, but the branch does not?
raise MissingRefException()
if token:
jlogger.debug('Adding Github credentials to environment')
environ.update(dict(GIT_ASKPASS=join(dirname(__file__), 'askpass.py')))
environ.update(dict(GIT_USERNAME=token['access_token'], GIT_PASSWORD=''))
else:
jlogger.debug('Clearing Github credentials from environment')
environ.update(dict(GIT_ASKPASS='', GIT_USERNAME='', GIT_PASSWORD=''))
with locked_file(checkout_lock):
if not exists(repo_path):
git_clone(repo_href, repo_path)
else:
git_fetch(repo_path, ref, ref_sha)
git_checkout(repo_path, checkout_path, ref)
# Make sure these are gone before we return.
environ.update(dict(GIT_ASKPASS='', GIT_USERNAME='', GIT_PASSWORD=''))
return checkout_path
def updateNodeIPs( env, nodes ):
"Update env dict and environ with node IPs"
# Get rid of stale junk
for var in 'ONOS_NIC', 'ONOS_CELL', 'ONOS_INSTANCES':
env[ var ] = ''
for var in environ.keys():
if var.startswith( 'OC' ):
env[ var ] = ''
for index, node in enumerate( nodes, 1 ):
var = 'OC%d' % index
env[ var ] = node.IP()
env[ 'OCI' ] = env[ 'OCN' ] = env[ 'OC1' ]
env[ 'ONOS_INSTANCES' ] = '\n'.join(
node.IP() for node in nodes )
environ.update( env )
return env
def test_env_var_config_alias(self):
def make_key(appname, key):
return "{0}_{1}".format(appname.upper(), key.upper())
appname = "myapp"
test_dict = {}
test_dict[make_key(appname, 'yes')] = 'yes'
test_dict[make_key(appname, 'changeps1')] = 'false'
try:
environ.update(test_dict)
assert 'MYAPP_YES' in environ
config = SampleConfiguration()._set_env_vars(appname)
assert config.always_yes is True
assert config.changeps1 is False
finally:
[environ.pop(key) for key in test_dict]
def test_env_var_config(self):
def make_key(appname, key):
return "{0}_{1}".format(appname.upper(), key.upper())
appname = "myapp"
test_dict = {}
test_dict[make_key(appname, 'always_yes')] = 'yes'
test_dict[make_key(appname, 'changeps1')] = 'false'
try:
environ.update(test_dict)
assert 'MYAPP_ALWAYS_YES' in environ
raw_data = load_from_string_data('file1', 'file2')
config = TestConfiguration(app_name=appname)._add_raw_data(raw_data)
assert config.changeps1 is False
assert config.always_yes is True
finally:
[environ.pop(key) for key in test_dict]
def initialize_config(config_file_name='env.yaml'):
config_keys = ['DBSERVER', 'DBNAME', 'DBUSER', 'DBPASS', 'DBPORT', 'REDISHOST', 'REDISPORT', 'REDISPASS']
if contains(config_keys, list(environ.keys())):
environ['DEBUG'] = 'False'
return
config_file_path = path.join(path.dirname(path.abspath(__file__)), config_file_name)
if not path.exists(config_file_path):
raise Exception('env.yaml required for config initialization')
with open(config_file_path, 'r') as config_file:
config = yaml.load(config_file)
config['dbconfig']['DBPORT'] = str(config['dbconfig']['DBPORT'])
config['redisconfig']['REDISPORT'] = str(config['redisconfig']['REDISPORT'])
environ.update(config['dbconfig'])
environ.update(config['redisconfig'])
environ['DEBUG'] = 'True'
def __environ(values, remove=[]):
"""
Modify the environment for a test, adding/updating values in dict `values` and
removing any environment variables mentioned in list `remove`.
"""
new_keys = set(environ.keys()) - set(values.keys())
old_environ = environ.copy()
try:
environ.update(values)
for to_remove in remove:
try:
del environ[remove]
except KeyError:
pass
yield
finally:
environ.update(old_environ)
for key in new_keys:
del environ[key]
def env_restored(unset = []):
"""
Create a temporary directory, with support for cleanup.
"""
# preserve the current environment
from os import environ
old = environ.copy()
# arbitrarily modify it
for name in unset:
del environ[name]
yield
# then restore the preserved copy
environ.clear()
environ.update(old)
def replace_variables(input_s, variables=None, extra_env=None):
"""
Replace variables like ${foo} with matching from `variables`. Also works with environment variables ${env.bar}.
:arg input_s :type StringType
:arg variables :type DictType
:arg extra_env :type DictType
:return parsed_string :type StringType
"""
assert isinstance(input_s, basestring)
variables = variables or {} # Linters complain when default arg set to `{}`
extra_env = extra_env or {} # Linters complain when default arg set to `{}`
if len(input_s) < 3:
return input_s
environ.update(extra_env)
parsed_string = ''
possible_var = ''
# Scanner
for c in input_s:
if c == '$':
possible_var += c
elif len(possible_var) == 1 and c != '{':
parsed_string += possible_var + c
possible_var = ''
elif c == '}' and possible_var:
possible_var += c
var = possible_var[len('${'):-len('}')]
parsed_string += ((lambda res: res.replace(path.sep, path.sep + path.sep) if path.sep == '\\' else res
)(environ.get(var[len('env.'):], '')) or possible_var) \
if var.startswith('env.') else variables.get(var, possible_var)
possible_var = ''
elif possible_var:
possible_var += c
else:
parsed_string += c
return parsed_string
def source_bash(setup_script):
foop = open("tmp.sh", "w")
foop.write("#/bin/bash\nsource $1\nenv|sort")
foop.close()
out, err, rc = run(["bash tmp.sh %s" % expandvars(setup_script)], useLogging=False, suppressErrors=True)
if rc:
print 'source encountered error, returning that one'
return err
lines = [l for l in out.split("\n") if "=" in l]
keys, values = [], []
for l in lines:
tl = l.split("=")
if len(tl) == 2:
if tl[0].startswith("_"):
continue
if tl[0].startswith("BASH_FUNC"):
continue
keys.append(tl[0])
values.append(tl[1])
environ.update(dict(zip(keys, values)))
remove("tmp.sh")
return
# Setup the require environmental variables for QSTK
# http://wiki.quantsoftware.org/index.php?title=QuantSoftware_ToolKit
from os import environ
from os.path import join
from sys import path
HOME = '/home/emilmont'
QS = join(HOME, 'Software/QSTK')
QSDATA = join(HOME, 'Data/QSData')
environ.update({
'QS' : QS,
'QSDATA' : QSDATA,
'QSDATAPROCESSED': join(QSDATA, 'Processed'),
'QSDATATMP' : join(QSDATA, 'Tmp'),
'QSBIN' : join(QS, 'Bin'),
'QSSCRATCH' : join(QSDATA, 'Scratch'),
'CACHESTALLTIME' : '12',
})
path.append(QS)
try:
from signal import SIGTERM as sig
except ImportError:
print "couldn't import signal module"
sig = 15
os.kill(pid, sig)
except OSError:
# process is already dead, so ignore it
pass
except:
traceback.print_exc()
if __name__ == '__main__':
try:
# treat all arguments as environment settings
Env.update(getenv(sys.argv[1:]))
# user name is used in log file name to avoid permission problems
user = Env.get('USER') or Env.get('USERNAME') or \
_winuser or 'nwsuser'
f = 'MatlabSleighSentinelLog_' + user + '_' + \
Env.get('MatlabSleighID', 'X') + '.txt'
setup(f)
if not Env.has_key('MatlabSleighNwsName'):
print "MatlabSleighNwsName variable is not set"
print >> sys.__stderr__, "MatlabSleighNwsName variable is not set"
else:
main()
except:
traceback.print_exc()
from distutils.core import setup, Command
from distutils.command.install_data import install_data
from distutils.dir_util import remove_tree
from distutils import log
from os import path, makedirs, walk, environ
from shutil import copyfile
from subprocess import call
from sys import version_info
if version_info[0] >= 3:
from io import FileIO
file = FileIO
from falias import __version__
environ.update({'PYTHONPATH': 'falias'})
def find_data_files(directory, targetFolder=""):
def skip(name):
return (name[0] != '.' and name[-1] != '~')
rv = []
for root, dirs, files in walk(directory):
if targetFolder:
rv.append((targetFolder,
list(root+'/'+f for f in files if skip(f))))
else:
rv.append((root,
list(root+'/'+f for f in files if skip(f))))
log.info(str(rv))
from distutils.dir_util import remove_tree
from distutils import log
from os import path, makedirs, walk, environ
from shutil import copyfile
from subprocess import call
from sys import version_info
if version_info[0] >= 3:
from io import FileIO
file = FileIO
from poorwsgi.state import __version__
environ.update({"PYTHONPATH": "poorwsgi"})
def find_data_files(directory, targetFolder=""):
rv = []
for root, dirs, files in walk(directory):
if targetFolder:
rv.append((targetFolder, list(root + "/" + f for f in files if f[0] != "." and f[-1] != "~")))
else:
rv.append((root, list(root + "/" + f for f in files if f[0] != "." and f[-1] != "~")))
log.info(str(rv))
return rv
class build_html(Command):
description = "build html documentation, need jinja24doc >= 1.1.0"
from dotenv import Dotenv
from os import path, environ as env
# First, parse env file and splice it into the OS environment
envpath = path.join(path.dirname(__file__), '..', '.env')
dotenv = Dotenv(envpath)
env.update(dotenv)
# Now, parse and set the settings dict
SETTINGS = {
'DEBUG': env.get('MODUS_DEBUG', '').lower() == 'true',
'DB_FILE': env.get('MODUS_DB_FILE', '').lower(),
}
"""
Runs Faceoff as a standalone web server. Do not use this is in production.
Copyright: (c) 2012-2014 Artem Nezvigin <*****@*****.**>
License: MIT, see LICENSE for details
"""
from sys import path
from os import environ
from os.path import dirname, join
path.append(join(dirname(__file__), '..'))
from argparse import ArgumentParser
parser = ArgumentParser(usage='%(prog)s [options]', argument_default='')
parser.add_argument('--host', metavar='<string>', dest='FACEOFF_HOST', default='0')
parser.add_argument('--port', metavar='<string>', dest='FACEOFF_PORT', default='5000')
parser.add_argument('--config', metavar='<string>', dest='FACEOFF_CONFIG')
parser.add_argument('--debug', action='store_const', const='1', dest='FACEOFF_DEBUG')
parser.add_argument('--db-path', metavar='<string>', dest='FACEOFF_DB_PATH')
parser.add_argument('--db-fixtures', action='store_const', const='1', dest='FACEOFF_DB_FIXTURES')
parser.add_argument('--log-path', metavar='<string>', dest='FACEOFF_LOG_PATH')
parser.add_argument('--log-level', metavar='<string>', dest='FACEOFF_LOG_LEVEL')
parser.add_argument('--log-filter', metavar='<string>', dest='FACEOFF_LOG_FILTER')
parser.add_argument('--log-ignore', metavar='<string>', dest='FACEOFF_LOG_IGNORE')
args = parser.parse_args()
environ.update(vars(args))
from faceoff import app
app.run(args.FACEOFF_HOST, int(args.FACEOFF_PORT), threaded=True)