def create_virtualenv(where=None, distribute=False, **kw):
"""
Create a virtual Python environment for testing purposes. If
distribute is True, installs the distribute package in place of
setuptools.
Returns the directory of the environment itself, its Python
library directory (which contains site-packages), its 'include'
and binary file directory (bin, or on windows, Scripts)
respectively. Additional keyword arguments are passed to mkdtemp
to create the virtual environment directory.
"""
save_argv = sys.argv
if not where:
where = mkdtemp(**kw)
try:
import virtualenv
distribute_opt = ['--distribute'] if distribute else []
sys.argv = ['virtualenv', '--quiet'] + distribute_opt + ['--no-site-packages', '--unzip-setuptools', where]
virtualenv.main()
finally:
sys.argv = save_argv
return virtualenv.path_locations(where)
def virtualenv(path='.', args=None):
if not args:
args = ['--no-site-packages']
argv = sys.argv[:]
sys.argv = [sys.executable] + args + [path]
m_virtualenv.main()
sys.argv = argv
def virtualenv(path='.', args=None):
if not args:
args = ['--no-site-packages']
argv = sys.argv[:]
sys.argv = [sys.executable] + args + [path]
m_virtualenv.main()
sys.argv=argv
def create_virtualenv(where=None, distribute=False, **kw):
"""
Create a virtual Python environment for testing purposes. If
distribute is True, installs the distribute package in place of
setuptools.
Returns the directory of the environment itself, its Python
library directory (which contains site-packages), its 'include'
and binary file directory (bin, or on windows, Scripts)
respectively. Additional keyword arguments are passed to mkdtemp
to create the virtual environment directory.
"""
save_argv = sys.argv
if not where:
where = mkdtemp(**kw)
try:
import virtualenv
distribute_opt = ['--distribute'] if distribute else []
sys.argv = ['virtualenv', '--quiet'] + distribute_opt + [
'--no-site-packages', '--unzip-setuptools', where
]
virtualenv.main()
finally:
sys.argv = save_argv
return virtualenv.path_locations(where)
def virtualenv_setup(dirpath):
print "Installing virtualenv..."
# add the dirpath to the argument vector for virtualenv to work
sys.argv.append(dirpath)
# setup the virtualenv
main()
return
def create_virtualenv(where):
save_argv = sys.argv
try:
import virtualenv
sys.argv = ['virtualenv', '--quiet', '--no-site-packages', where]
virtualenv.main()
finally:
sys.argv = save_argv
return virtualenv.path_locations(where)
def create_virtualenv(venv_name, dir_path=None):
dir_path = dir_path or tempfile.mkdtemp()
old_args = sys.argv
try:
from virtualenv import main
sys.argv = ['virtualenv', op.join(dir_path, venv_name)]
main()
finally:
sys.argv = old_args
print("Created virtualenv %s at %s" % (venv_name, dir_path))
return dir_path
def create_virtualenv(venv_name, dir_path=None):
dir_path = dir_path or tempfile.mkdtemp()
old_args = sys.argv
try:
from virtualenv import main
sys.argv = ['virtualenv', op.join(dir_path, venv_name)]
main()
finally:
sys.argv = old_args
print("Created virtualenv %s at %s" % (venv_name, dir_path))
return dir_path
def create_virtualenv(where, distribute=False):
save_argv = sys.argv
try:
import virtualenv
distribute_opt = distribute and ['--distribute'] or []
sys.argv = ['virtualenv', '--quiet'] + distribute_opt + ['--no-site-packages', '--unzip-setuptools', where]
virtualenv.main()
finally:
sys.argv = save_argv
return virtualenv.path_locations(where)
def create_virtualenv(where, distribute=False):
save_argv = sys.argv
try:
import virtualenv
distribute_opt = distribute and ['--distribute'] or []
sys.argv = ['virtualenv', '--quiet'] + distribute_opt + [
'--no-site-packages', '--unzip-setuptools', where
]
virtualenv.main()
finally:
sys.argv = save_argv
return virtualenv.path_locations(where)
def _testPackageUpdate(self, directory: Directory) -> None:
""" Test Package Update
:param directory: The directory where the packages are extracted
Since we ARE running on the server, we will test install these packages here
first, this is done by creating a virtualenv.
There isn't a lot of testing for the release at this stage.
Currently we just use pip to try and install the packages off line, if it's happy
we're happy.
"""
# Create the test virtualenv
virtualEnvDir = Directory()
virtualenv.main(['--site-packages', virtualEnvDir])
# Install all the packages from the directory
args = [
'install', # Install the packages
'--ignore-installed ', # Reinstall if they already exist
'--no-cache-dir', # Don't use the local pip cache
'--no-index', # Work offline, don't use pypi
'--find-links',
directory.path, # Look in the directory for dependencies
[f.name for f in directory]
]
# We could use import pip, pip.main(..), except:
# We want to capture, the output, and:
# we can't tell it to use the virtualenv
pipExec = os.path.join(virtualEnvDir.path, 'bin', 'pip')
commandComplete = subprocess.run(['pip'] + args,
executable=pipExec,
stdout=PIPE,
stderr=PIPE,
shell=True)
if commandComplete.returncode:
raise PlatformInstallException(
"Package install test failed",
output=commandComplete.stdout.decode(),
error=commandComplete.stderr.decode())
# Continue normally if it all succeeded
logger.debug("Peek update successfully tested.")
def _create_virtualenv(self, venv_dir):
# The original implementation used Popen(['virtualenv', ...])
# However, it's hard to make assumptions about how a users
# PATH is set up. This could result in using old versions
# of virtualenv that give confusing error messages.
# To fix this issue, we're calling directly into the
# virtualenv package. The main() method doesn't accept
# args, so we need to patch out sys.argv with the venv
# dir. The original sys.argv is replaced on exit.
original = sys.argv
sys.argv = ['', venv_dir, '--quiet']
try:
virtualenv.main()
finally:
sys.argv = original
def _create_virtualenv(self, venv_dir):
# The original implementation used Popen(['virtualenv', ...])
# However, it's hard to make assumptions about how a users
# PATH is set up. This could result in using old versions
# of virtualenv that give confusing error messages.
# To fix this issue, we're calling directly into the
# virtualenv package. The main() method doesn't accept
# args, so we need to patch out sys.argv with the venv
# dir. The original sys.argv is replaced on exit.
original = sys.argv
sys.argv = ['', venv_dir, '--quiet']
try:
virtualenv.main()
finally:
sys.argv = original
def create_virtual_env(project_path, install_path, ve_args):
# remove existing virtual env if exists
ve_path = os.path.join(project_path, cfg.VIRTUAL_ENV_PATH)
if os.path.exists(ve_path):
shutil.rmtree(ve_path)
try:
logger.info('creating virtual env')
sys.argv = ['']
if '--no-wheel' not in ve_args:
sys.argv.append('--no-wheel')
sys.argv.extend(ve_args)
sys.argv.append(ve_path)
try:
virtualenv.main()
except SystemExit as sysext:
if sysext.code != 0:
raise SystemExit(sysext)
except Exception:
logger.exception('failed to create virtualenv: ')
raise Exception('failed to create virtualenv!')
try:
logger.info('installing requirements for virtualenv')
# update pip to latest version
run_command(['{}/{}/bin/pip'.format(project_path,
cfg.VIRTUAL_ENV_PATH),
'install',
'-U',
'pip'])
if not os.path.exists('{}/requirements.txt'.format(project_path)):
logger.warning('requirements.txt not found')
return ve_path
run_command(['{}/{}/bin/pip'.format(project_path,
cfg.VIRTUAL_ENV_PATH),
'install',
'-r',
'{}/requirements.txt'.format(project_path)])
virtualenv.make_environment_relocatable(ve_path)
fixup_scripts(install_path, ve_path)
except Exception:
logger.exception('failed to install requirements! error message:')
raise Exception('fail to install requirements.')
return ve_path
def _deploy_virtenv_init(args):
_virtenv = utils.active_virtualenv()
virtenv = getattr(args, 'virtenv') or _virtenv
# skip if we're already in the targeted virtenv...
if virtenv and virtenv != _virtenv:
# we can't alrady be in a virtenv when running virtualenv.main()
utils.virtualenv_deactivate()
# scratch the existing virtenv directory, if requested
if args.trash:
utils.remove_file(virtenv, force=True)
if args.trash_home:
trash()
# virtualenv.main; pass in only the virtenv path
sys.argv = sys.argv[0:1] + [virtenv]
# run the virtualenv script to install the virtenv
virtualenv.main()
# activate the newly installed virtenv
utils.virtualenv_activate(args.virtenv)
return virtenv
def setup_environment(after_install):
try:
import virtualenv
except:
raise Exception("virtualenv not installed! This is required.")
root_path = os.path.dirname(__file__)
virtualenv_root = os.path.join(root_path, BOOTSTRAP_VIRTUALENV_PATH)
if os.path.exists(virtualenv_root):
logging.info(
"virtualenv already exists at \"%s\". Nothing to do." %
virtualenv_root
)
return virtualenv_root
logging.info("creating virtualenv at \"%s\"" % virtualenv_root)
sys.argv.append("--distribute")
sys.argv.append(virtualenv_root)
virtualenv.after_install = after_install
virtualenv.main()
return virtualenv_root
def create_env():
ENV_NAME = 'biofacenv'
import contextlib
@contextlib.contextmanager
def hack_sys_args():
old_argv = sys.argv[:]
sys.argv = ['vurtualenv.py', ENV_NAME]
yield
sys.argv = old_argv
@contextlib.contextmanager
def add_venv_to_path():
old_path = sys.path[:]
sys.path.append(os.path.join('.', 'virtualenv'))
yield
sys.path = old_path
with add_venv_to_path(), hack_sys_args():
import virtualenv
virtualenv.main()
def _deploy_virtenv_init(args):
_virtenv = utils.active_virtualenv()
virtenv = getattr(args, 'virtenv') or _virtenv
# skip if we're already in the targeted virtenv...
if virtenv and virtenv != _virtenv:
# we can't alrady be in a virtenv when running virtualenv.main()
utils.virtualenv_deactivate()
# scratch the existing virtenv directory, if requested
if args.trash:
utils.remove_file(virtenv, force=True)
if args.trash_home:
trash()
# virtualenv.main; pass in only the virtenv path
sys.argv = sys.argv[0:1] + [virtenv]
# run the virtualenv script to install the virtenv
virtualenv.main()
# activate the newly installed virtenv
utils.virtualenv_activate(args.virtenv)
return virtenv
def create_virtualenv(self, location):
is_win = (sys.platform == 'win32')
is_cygwin = (sys.platform == 'cygwin')
executable = (is_win or is_cygwin) and 'python.exe' or 'python'
binFolder = is_win and 'Scripts' or 'bin'
binLocation = join(location, binFolder)
if is_cygwin:
# Virtualenv doesn't work on cygwin, but create a
# bin/python using the one of buildout
buildoutExecutable = self.buildout['buildout']['executable']
if not buildoutExecutable.endswith('exe'):
buildoutExecutable += '.exe'
unixBinLocation = join(location, 'bin')
if not os.path.isfile(join(unixBinLocation, executable)):
if not os.path.exists(unixBinLocation):
os.mkdir(unixBinLocation)
os.symlink(buildoutExecutable,
join(unixBinLocation, executable))
else:
old = sys.argv
try:
sys.argv = ['virtualenv', '--no-site-packages', location]
virtualenv.main()
if 'eggs' in self.options:
eggs = [e for e in self.options['eggs'].split('\n') if e]
subprocess.call([join(binLocation, 'easy_install'),] + eggs)
finally:
sys.argv = old
if is_win:
# On windows, add a bin/python
unixBinLocation = join(location, 'bin')
if not os.path.isfile(join(unixBinLocation, executable)):
pythons = glob.glob(join(binLocation, 'python*'))
if not os.path.exists(unixBinLocation):
os.mkdir(unixBinLocation)
shutil.copyfile(pythons[0],
join(unixBinLocation, executable))
def package(req_files, target_dir):
venv_dir = os.path.join(target_dir, '.venv')
tmp_dir = os.path.join(target_dir, '.tmp')
for req_file in req_files:
if not os.path.isfile(req_file):
sys.exit('No requirements file found in: {}'.format(req_file))
if os.path.exists(target_dir):
if not os.path.isdir(target_dir):
sys.exit('Existing non-directory found at: {}'.format(target_dir))
else:
os.mkdir(target_dir)
if os.path.exists(venv_dir):
shutil.rmtree(venv_dir)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
original = sys.argv
sys.argv = ['', venv_dir, '--quiet']
try:
virtualenv.main()
finally:
sys.argv = original
if platform.system() == 'Windows':
pip_exe = os.path.join(venv_dir, 'Scripts', 'pip.exe')
deps_dir = os.path.join(venv_dir, 'Lib', 'site-packages')
else:
pip_exe = os.path.join(venv_dir, 'bin', 'pip')
lib_path = os.path.join(venv_dir, 'lib')
libs_dir_path_items = os.listdir(lib_path)
directories = [
d for d in libs_dir_path_items
if os.path.isdir(os.path.join(lib_path, d))
]
if len(directories) > 0:
python_dir = directories[0]
else:
sys.exit('No python directory')
deps_dir = os.path.join(venv_dir, 'lib', python_dir, 'site-packages')
if not os.path.isfile(pip_exe):
sys.exit('Pip not found in: {}'.format(pip_exe))
for req_file in req_files:
p = subprocess.Popen([pip_exe, 'install', '-r', req_file],
stdout=subprocess.PIPE)
p.communicate()
if p.returncode != 0:
sys.exit(
"Failed to install requirements from: {}".format(req_file))
if not os.path.isdir(deps_dir):
sys.exit('Installed packages not found in: {}'.format(deps_dir))
blacklist = [
'pip', 'pip-*', 'wheel', 'wheel-*', 'setuptools', 'setuptools-*',
'easy_install.*'
]
shutil.copytree(deps_dir,
tmp_dir,
symlinks=False,
ignore=shutil.ignore_patterns(*blacklist))
for f in os.listdir(tmp_dir):
target = os.path.join(target_dir, f)
if os.path.isdir(target):
shutil.rmtree(target)
elif os.path.exists(target):
os.remove(target)
shutil.move(os.path.join(tmp_dir, f), target_dir)
shutil.rmtree(venv_dir)
shutil.rmtree(tmp_dir)
def run():
virtualenv.after_install = after_install
virtualenv.adjust_options = adjust_options
virtualenv.main()
def warn(msg):
print (WARNING + msg + ENDC)
def err(msg):
print (FAIL + msg + ENDC)
"""
Check python version
"""
info("checking python version...")
cur_version = sys.version_info
if cur_version < (2,7):
err("Your Python interpreter is too old. Please consider upgrading.")
sys.exit(-1)
if cur_version >= (3,0):
err("Your Python interpreter is 3.x but 2.7 is required.")
sys.exit(-1)
"""
Check virtual enviroment
"""
if not os.path.exists(".py"):
sys.argv = ['virtualenv', '.py']
venv.main()
"\n\nFatal error while installing %s\n" % requirement)
sys.exit(1)
pkg_resources.working_set.add_entry(tmp_eggs)
pkg_resources.working_set.require(requirement)
if options.virtualenv:
python_path = os.path.join(bin_dir, os.path.basename(sys.executable))
if not os.path.isfile(python_path):
install('virtualenv>=1.5')
import virtualenv
print "Running virtualenv"
args = sys.argv[:]
sys.argv = ['bootstrap', os.getcwd(), '--clear', '--no-site-package']
virtualenv.main()
execute([python_path] + args)
sys.exit(0)
if options.profile:
if not os.path.isfile(options.profile):
sys.stderr.write('No such profile file: %s\n' % options.profile)
sys.exit(1)
print "Creating configuration '%s'" % os.path.abspath(options.config)
config = open(options.config, 'w')
config.write("""[buildout]
extends = %s
""" % options.profile)
config.close()
#!/usr/bin/env python
import os
import subprocess
import virtualenv
def after_install(option, home_dir):
pip = os.path.join(home_dir, 'bin', 'pip')
subprocess.call([pip, 'install', '-r', 'requirements.txt'])
virtualenv.after_install = after_install
if __name__ == '__main__':
virtualenv.main()
def create():
if not os.path.isdir(venv_dir()):
tmp = sys.argv
sys.argv = ['', venv_dir()]
main()
sys.argv = tmp
def warn(msg):
print(WARNING + msg + ENDC)
def err(msg):
print(FAIL + msg + ENDC)
"""
Check python version
"""
info("checking python version...")
cur_version = sys.version_info
if cur_version < (2, 7):
err("Your Python interpreter is too old. Please consider upgrading.")
sys.exit(-1)
if cur_version >= (3, 0):
err("Your Python interpreter is 3.x but 2.7 is required.")
sys.exit(-1)
"""
Check virtual enviroment
"""
if not os.path.exists(".py"):
sys.argv = ['virtualenv', '.py']
venv.main()
def create_deploy_artifact(project_dir):
# Create virtualenv
venv_dir = os.path.join(project_dir, '.deploy', 'venv')
original = sys.argv
sys.argv = ['', venv_dir, '--quiet']
try:
virtualenv.main()
finally:
sys.argv = original
# Make sure pip is available independent of platform
pip_exe = compat.pip_script_in_venv(venv_dir)
assert os.path.isfile(pip_exe)
# Process requirements file
requirements_file = os.path.join(project_dir, 'requirements.txt')
if not os.path.isfile(requirements_file):
hash_content = ''
else:
with open(requirements_file, 'r') as f:
hash_content = f.read()
requirements_hash = hashlib.md5(hash_content).hexdigest()
deployment_package_filename = os.path.join(project_dir, '.deploy',
'deployments',
requirements_hash + '.zip')
if has_at_least_one_package(requirements_file) and not \
os.path.isfile(deployment_package_filename):
p = subprocess.Popen([pip_exe, 'install', '-r', requirements_file],
stdout=subprocess.PIPE)
p.communicate()
# Handle new virtualenv dependencies
deps_dir = compat.site_packages_dir_in_venv(venv_dir)
assert os.path.isdir(deps_dir)
if not os.path.isdir(os.path.dirname(deployment_package_filename)):
os.makedirs(os.path.dirname(deployment_package_filename))
with zipfile.ZipFile(deployment_package_filename,
'w',
compression=zipfile.ZIP_DEFLATED) as z:
# add dependencies
prefix_len = len(deps_dir) + 1
for root, dirnames, filenames in os.walk(deps_dir):
if root == deps_dir and 'lambda-deployer' in dirnames:
# we don't want to deploy the deployer, just the project deps
dirnames.remove('lambda-deployer')
for filename in filenames:
full_path = os.path.join(root, filename)
zip_path = full_path[prefix_len:]
z.write(full_path, zip_path)
# add project files
sources_directory = os.path.join(project_dir, 'src')
prefix_len = len(sources_directory) + 1
for root, dirnames, filenames in os.walk(sources_directory):
for filename in filenames:
full_path = os.path.join(root, filename)
zip_path = full_path[prefix_len:]
z.write(full_path, zip_path)
return deployment_package_filename
def install(bin_):
print "Creating virtualenv"
virtualenv.main()
print "Installing AIRi with dependencies"
subprocess.call([os.path.join(bin_, "pip"), "install", "AIRi"])
def install(bin_):
print "Creating virtualenv"
virtualenv.main()
print "Installing AIRi with dependencies"
subprocess.call([os.path.join(bin_, "pip"), "install", "AIRi"])
def create():
if not os.path.isdir(venv_dir()):
tmp = sys.argv
sys.argv = ['', venv_dir()]
main()
sys.argv = tmp
def create_and_exit(virtualenv_path, **kwds):
sys.argv = ["virtualenv", virtualenv_path]
python = kwds.get("python", None)
if python:
sys.argv.extend(["--python", python])
return virtualenv.main()
def package(req_files, target_dir):
venv_dir = os.path.join(target_dir, ".venv")
tmp_dir = os.path.join(target_dir, ".tmp")
for req_file in req_files:
if not os.path.isfile(req_file):
sys.exit("No requirements file found in: {}".format(req_file))
if os.path.exists(target_dir):
if not os.path.isdir(target_dir):
sys.exit("Existing non-directory found at: {}".format(target_dir))
shutil.rmtree(target_dir)
os.mkdir(target_dir)
if os.path.exists(venv_dir):
shutil.rmtree(venv_dir)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
original = sys.argv
sys.argv = ["", venv_dir, "--quiet", "-p", sys.executable]
try:
virtualenv.main()
finally:
sys.argv = original
if platform.system() == "Windows":
pip_exe = os.path.join(venv_dir, "Scripts", "pip.exe")
deps_dir = os.path.join(venv_dir, "Lib", "site-packages")
else:
pip_exe = os.path.join(venv_dir, "bin", "pip")
lib_path = os.path.join(venv_dir, "lib")
libs_dir_path_items = os.listdir(lib_path)
directories = [
d for d in libs_dir_path_items
if os.path.isdir(os.path.join(lib_path, d))
]
if len(directories) > 0:
python_dir = directories[0]
else:
sys.exit("No python directory")
deps_dir = os.path.join(venv_dir, "lib", python_dir, "site-packages")
if not os.path.isfile(pip_exe):
sys.exit("Pip not found in: {}".format(pip_exe))
for req_file in req_files:
p = subprocess.Popen([pip_exe, "install", "-r", req_file],
stdout=subprocess.PIPE)
p.communicate()
if p.returncode != 0:
sys.exit(
"Failed to install requirements from: {}".format(req_file))
if not os.path.isdir(deps_dir):
sys.exit("Installed packages not found in: {}".format(deps_dir))
blacklist = [
"pip",
"pip-*",
"wheel",
"wheel-*",
"setuptools",
"setuptools-*",
"*.dist-info",
"easy_install.*",
"*.pyc",
"__pycache__",
]
shutil.copytree(deps_dir,
tmp_dir,
symlinks=False,
ignore=shutil.ignore_patterns(*blacklist))
for f in os.listdir(tmp_dir):
target = os.path.join(target_dir, f)
if os.path.isdir(target):
shutil.rmtree(target)
elif os.path.exists(target):
os.remove(target)
shutil.move(os.path.join(tmp_dir, f), target_dir)
shutil.rmtree(venv_dir)
shutil.rmtree(tmp_dir)
def run(inst):
virtualenv.after_install = inst.after_install
virtualenv.adjust_options = inst.adjust_options
virtualenv.main()
def create_and_exit(virtualenv_path, **kwds):
sys.argv = ["virtualenv", virtualenv_path]
python = kwds.get("python", None)
if python:
sys.argv.extend(["--python", python])
return virtualenv.main()
