def test_pypy_pre_import(tmp_path):
"""For PyPy, some built-in modules should be pre-imported because
some programs expect them to be in sys.modules on startup.
"""
check_code = inspect.getsource(check_pypy_pre_import)
check_code = textwrap.dedent(check_code[check_code.index("\n") + 1 :])
if six.PY2:
check_code = check_code.decode()
check_prog = tmp_path / "check-pre-import.py"
check_prog.write_text(check_code)
ve_path = str(tmp_path / "venv")
virtualenv.create_environment(ve_path)
bin_dir = virtualenv.path_locations(ve_path)[-1]
try:
cmd = [
os.path.join(bin_dir, "{}{}".format(virtualenv.EXPECTED_EXE, ".exe" if virtualenv.IS_WIN else "")),
str(check_prog),
]
subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exception:
assert not exception.returncode, exception.output
def test_create_environment_from_virtual_environment(tmpdir):
"""Create virtual environment using Python from another virtual environment
"""
venvdir = str(tmpdir / "venv")
home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(venvdir)
virtualenv.create_environment(venvdir)
assert not os.path.islink(os.path.join(lib_dir, "distutils"))
def configure_virtualenv(config, python_worktree, build_worktree=None,
remote_packages=None, site_packages=True):
if not remote_packages:
remote_packages = list()
# create a new virtualenv
python_worktree.config = config
venv_path = python_worktree.venv_path
pip = python_worktree.pip
try:
virtualenv.create_environment(python_worktree.venv_path,
site_packages=site_packages)
except:
ui.error("Failed to create virtualenv")
return
# Install all Python projects using pip install -e .
python_projects = python_worktree.python_projects
for i, project in enumerate(python_projects):
ui.info_count(i, len(python_projects),
ui.green, "Configuring", ui.reset, ui.blue, project.src)
cmd = [pip, "install", "--editable", "."]
qisys.command.call(cmd, cwd=project.path)
# Write a qi.pth file containing path to C/C++ extensions
if build_worktree:
handle_extensions(venv_path, python_worktree, build_worktree)
# Install the extension in the virtualenv
binaries_path = virtualenv.path_locations(venv_path)[-1]
pip_binary = os.path.join(binaries_path, "pip")
if remote_packages:
cmd = [pip_binary, "install"] + remote_packages
subprocess.check_call(cmd)
def handle_extensions(venv_path, python_worktree, build_worktree):
""" Check if there is a build project matching the given source, and
add the correct path to the virtualenv
"""
extensions_projects = list()
build_projects = build_worktree.build_projects
for project in python_worktree.python_projects:
parent_project = qisys.parsers.find_parent_project(build_projects,
project.path)
if parent_project:
extensions_projects.append(parent_project)
to_write = ""
for project in extensions_projects:
qi_pth_src = os.path.join(project.sdk_directory, "qi.pth")
if os.path.exists(qi_pth_src):
with open(qi_pth_src, "r") as fp:
to_write += fp.read()
if not to_write.endswith("\n"):
to_write += "\n"
lib_path = virtualenv.path_locations(venv_path)[1]
qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
with open(qi_pth_dest, "a") as fp:
fp.write(to_write)
def create_env():
if env_exists():
raise Exception('Virtual environment already exists.')
log('Creating virtual environment...')
home_dir, lib_dir, inc_dir, bin_dir = venv.path_locations(ENV_PATH)
python_loc = venv.install_python(
home_dir,
lib_dir,
inc_dir,
bin_dir,
site_packages=False,
clear=False,
symlink=True
)
python_abs_loc = os.path.abspath(python_loc)
venv.install_activate(home_dir, bin_dir)
venv.install_wheel(['setuptools', 'pip'], python_abs_loc, None)
venv.install_distutils(home_dir)
log('Installing requirements...')
req_cmd = '{0}/bin/pip install {1}'.format(
ENV_PATH,
get_config().requirements
)
execute_under_env(req_cmd)
log('Virtual environment created!')
def create_virtualenv(where=None, distribute=False, **kw):
"""
Create a virtual Python environment for testing purposes. If
distribute is True, installs the distribute package in place of
setuptools.
Returns the directory of the environment itself, its Python
library directory (which contains site-packages), its 'include'
and binary file directory (bin, or on windows, Scripts)
respectively. Additional keyword arguments are passed to mkdtemp
to create the virtual environment directory.
"""
save_argv = sys.argv
if not where:
where = mkdtemp(**kw)
try:
import virtualenv
distribute_opt = ['--distribute'] if distribute else []
sys.argv = ['virtualenv', '--quiet'] + distribute_opt + ['--no-site-packages', '--unzip-setuptools', where]
virtualenv.main()
finally:
sys.argv = save_argv
return virtualenv.path_locations(where)
def version_exe(venv, exe_name):
_, _, _, bin_dir = virtualenv.path_locations(str(venv))
exe = os.path.join(bin_dir, exe_name)
script = "import sys; import json; print(json.dumps(dict(v=list(sys.version_info), e=sys.executable)))"
cmd = [exe, "-c", script]
out = json.loads(subprocess.check_output(cmd, universal_newlines=True))
return out["v"], out["e"]
def env_prefix(self):
return ". '{{prefix}}{0}activate' &&".format(
virtualenv.path_locations(
helpers.environment_dir(ENVIRONMENT_DIR, self.language_version)
)[-1].rstrip(os.sep) + os.sep,
'activate',
)
def __init__(self, base_path, *args, **kwargs):
# Make our base_path a test.lib.path.Path object
base_path = Path(base_path)
# Store paths related to the virtual environment
_virtualenv = kwargs.pop("virtualenv")
venv, lib, include, bin = virtualenv.path_locations(_virtualenv)
self.venv_path = venv
self.lib_path = lib
self.include_path = include
self.bin_path = bin
if hasattr(sys, "pypy_version_info"):
self.site_packages_path = self.venv_path.join("site-packages")
else:
self.site_packages_path = self.lib_path.join("site-packages")
self.user_base_path = self.venv_path.join("user")
self.user_bin_path = self.user_base_path.join(self.bin_path - self.venv_path)
self.user_site_path = self.venv_path.join(
"user",
site.USER_SITE[len(site.USER_BASE) + 1:],
)
# Create a Directory to use as a scratch pad
self.scratch_path = base_path.join("scratch").mkdir()
# Set our default working directory
kwargs.setdefault("cwd", self.scratch_path)
# Setup our environment
environ = kwargs.get("environ")
if environ is None:
environ = os.environ.copy()
environ["PIP_LOG_FILE"] = base_path.join("pip-log.txt")
environ["PATH"] = Path.pathsep.join(
[self.bin_path] + [environ.get("PATH", [])],
)
environ["PYTHONUSERBASE"] = self.user_base_path
# Writing bytecode can mess up updated file detection
environ["PYTHONDONTWRITEBYTECODE"] = "1"
kwargs["environ"] = environ
# Call the TestFileEnvironment __init__
super(PipTestEnvironment, self).__init__(base_path, *args, **kwargs)
# Expand our absolute path directories into relative
for name in ["base", "venv", "lib", "include", "bin", "site_packages",
"user_base", "user_site", "user_bin", "scratch"]:
real_name = "%s_path" % name
setattr(self, name, getattr(self, real_name) - self.base_path)
# Ensure the tmp dir exists, things break horribly if it doesn't
self.temp_path.mkdir()
# create easy-install.pth in user_site, so we always have it updated
# instead of created
self.user_site_path.makedirs()
self.user_site_path.join("easy-install.pth").touch()
def env_prefix(self):
return ". '{{prefix}}{0}activate' &&".format(
virtualenv.path_locations(
ENVIRONMENT_DIR,
)[-1].rstrip(os.sep) + os.sep,
'activate',
)
def handle_pure_python(venv_path, python_worktree, env=None):
""" Add the paths of all python projects to the virtualenv """
lib_path = virtualenv.path_locations(venv_path)[1]
qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
res = True
with open(qi_pth_dest, "w") as fp:
fp.write("")
for i, project in enumerate(python_worktree.python_projects):
ui.info_count(i, len(python_worktree.python_projects),
ui.blue, project.name)
if project.setup_with_distutils:
cmd = [python_worktree.pip, "install"]
if not ui.CONFIG["verbose"]:
cmd.append("--quiet")
cmd.extend(["--editable", "."])
rc = qisys.command.call(cmd, cwd=project.path, ignore_ret_code=True,
env=env)
if rc != 0:
ui.warning("Failed to run pip install on", project.src)
res = False
else:
ui.debug("Adding python path for project", project.name, ":\n",
project.python_path)
for path in project.python_path:
fp.write(path + "\n")
return res
def create_virtualenv(where, distribute=False):
import virtualenv
if sys.version_info[0] > 2:
distribute = True
virtualenv.create_environment(
where, use_distribute=distribute, unzip_setuptools=True)
return virtualenv.path_locations(where)
def configure_virtualenv(config, python_worktree, build_worktree=None,
remote_packages=None, site_packages=True):
""" Main entry point. Called by ``qipy bootstrap``
:param: remote_packages List of third-party packages to add in the virtualenv
:param: site_packages Allow access to global site packages
"""
ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root)
if not remote_packages:
remote_packages = list()
# create a new virtualenv
python_worktree.config = config
venv_path = python_worktree.venv_path
pip = python_worktree.pip
try:
virtualenv.create_environment(python_worktree.venv_path,
site_packages=site_packages)
except:
ui.error("Failed to create virtualenv")
return False
ui.info("Adding python projects")
# Write a qi.pth file containing path to C/C++ extensions and
# path to pure python modules or packages
pure_python_ok = handle_pure_python(venv_path, python_worktree)
if build_worktree:
handle_extensions(venv_path, python_worktree, build_worktree)
ui.info("Adding other requirements: " + ", ".join(remote_packages))
binaries_path = virtualenv.path_locations(venv_path)[-1]
pip_binary = os.path.join(binaries_path, "pip")
remote_ok = True
if remote_packages:
cmd = [pip_binary, "install"] + remote_packages
rc = qisys.command.call(cmd, ignore_ret_code=True)
remote_ok = (rc == 0)
if pure_python_ok and remote_ok:
ui.info(ui.green, "Done")
if not pure_python_ok:
ui.info(ui.red, "Failed to add some python projects")
if not remote_ok:
ui.info(ui.red, "Failed to add some third party requirements")
requirements_ok = True
for project in python_worktree.python_projects:
path = os.path.join(project.path, "requirements.txt")
if os.path.isfile( path ):
ui.info(ui.green, " * Installing dependencies from " + path)
cmd = [pip_binary, "install", "--requirement", path]
rc = qisys.command.call(cmd, ignore_ret_code=True)
requirements_ok = (rc == 0)
else:
ui.debug(ui.yellow, " * missing " + path)
return (pure_python_ok and remote_ok and requirements_ok)
def do(args):
build_worktree = qibuild.parsers.get_build_worktree(args)
env = build_worktree.get_env()
build_config = qibuild.parsers.get_build_config(build_worktree, args)
worktree = build_worktree.worktree
cmd = args.command
venvs_path = os.path.join(worktree.dot_qi,
"venvs")
name = build_config.build_directory("py")
venv_root = os.path.join(venvs_path, name)
if not os.path.exists(venv_root):
err = "No Virtualenv found in %s\n" % (venv_root)
err += "Please run `qipy bootstrap`"
raise Exception(err)
binaries_path = virtualenv.path_locations(venv_root)[-1]
if args.dolist:
for f in sorted(os.listdir(binaries_path)):
if os.path.isfile(os.path.join(binaries_path, f)):
print f
return
if not cmd:
cmd = ["ipython"]
if os.path.exists(cmd[0]):
# Assume it is a script we want to run
python_path = os.path.join(binaries_path, "python")
cmd.insert(0, python_path)
else:
script_path = qipy.venv.find_script(venv_root, cmd[0])
if script_path:
cmd[0] = script_path
lib_path = virtualenv.path_locations(venv_root)[1]
env["PYTHONHOME"] = venv_root
env["PYTHONPATH"] = os.path.join(lib_path, "site-packages")
if args.exec_:
ui.debug("exec", cmd)
os.execve(cmd[0], cmd, env)
else:
qisys.command.call(cmd, env=env)
def __init__(self, source, venv=None):
self.dir_source = source
if not venv:
base, source_name = os.path.split(source)
venv = os.path.join(base, 'venv-{}-{}'.format(source_name, os.path.basename(sys.executable)))
venv_home, venv_lib, venv_inc, venv_bin = virtualenv.path_locations(venv)
self.dir_venv = venv_home
self.dir_venv_bin = venv_bin
def get_paths(name):
_, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(name)
venv_home = get_venv_home()
home_dir = get_env_home(name)
lib_dir = os.path.join(venv_home, lib_dir)
pkg_dir = os.path.join(lib_dir, 'site-packages')
inc_dir = os.path.join(venv_home, inc_dir)
bin_dir = os.path.join(venv_home, bin_dir)
return home_dir, lib_dir, pkg_dir, inc_dir, bin_dir
def add_virtualenv_path(venv=os.getenv('VIRTUAL_ENV')):
"""Add virtualenv's site-packages to `sys.path`."""
if not venv:
return
import virtualenv
venv = os.path.abspath(venv)
home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(venv)
path = os.path.join(lib_dir, 'site-packages')
sys.path.insert(0, path)
site.addsitedir(path)
def __init__(self, location, *args, **kwargs):
self.location = Path(location)
self.pip_source_dir = kwargs.pop("pip_source_dir")
self._system_site_packages = kwargs.pop("system_site_packages", False)
home, lib, inc, bin = _virtualenv.path_locations(self.location)
self.lib = Path(lib)
self.bin = Path(bin)
super(VirtualEnvironment, self).__init__(*args, **kwargs)
def create_virtualenv(where):
save_argv = sys.argv
try:
import virtualenv
sys.argv = ['virtualenv', '--quiet', '--no-site-packages', where]
virtualenv.main()
finally:
sys.argv = save_argv
return virtualenv.path_locations(where)
def _update_paths(self):
home, lib, inc, bin = _virtualenv.path_locations(self.location)
self.bin = Path(bin)
self.site = Path(lib) / 'site-packages'
# Workaround for https://github.com/pypa/virtualenv/issues/306
if hasattr(sys, "pypy_version_info"):
version_fmt = '{0}' if six.PY3 else '{0}.{1}'
version_dir = version_fmt.format(*sys.version_info)
self.lib = Path(home, 'lib-python', version_dir)
else:
self.lib = Path(lib)
def custom_prompt_root(tmp_path_factory):
"""Provide Path to root with default and custom venvs created."""
root = tmp_path_factory.mktemp("custom_prompt")
virtualenv.create_environment(
str(root / ENV_CUSTOM), prompt=PREFIX_CUSTOM, no_setuptools=True, no_pip=True, no_wheel=True
)
_, _, _, bin_dir = virtualenv.path_locations(str(root / ENV_DEFAULT))
bin_dir_name = os.path.split(bin_dir)[-1]
return root, bin_dir_name
def create_virtualenv(where, distribute=False):
save_argv = sys.argv
try:
import virtualenv
distribute_opt = distribute and ['--distribute'] or []
sys.argv = ['virtualenv', '--quiet'] + distribute_opt + ['--no-site-packages', '--unzip-setuptools', where]
virtualenv.main()
finally:
sys.argv = save_argv
return virtualenv.path_locations(where)
def __init__(self, dir):
"""Construct
:arg dir: The path of the new virtualenv
"""
create_environment(dir,
site_packages=False,
clear=False,
never_download=True)
home_dir, lib_dir, inc_dir, bin_dir = path_locations(dir)
self.python_path = join(bin_dir, basename(sys.executable))
def _get_env_bin_path(env_path):
"""returns the bin path for a virtualenv
"""
try:
import virtualenv
return virtualenv.path_locations(env_path)[3]
except ImportError:
# this is a fallback for a race condition in which you're trying
# to use the script and create a virtualenv from within
# a virtualenv in which virtualenv isn't installed and so
# is not importable.
return os.path.join(env_path, 'scripts' if IS_WIN else 'bin')
def __init__(self, location, *args, **kwargs):
self.location = Path(location)
self.pip_source_dir = kwargs.pop("pip_source_dir")
self._system_site_packages = kwargs.pop("system_site_packages", False)
home, lib, inc, bin = _virtualenv.path_locations(self.location)
# workaround for https://github.com/pypa/virtualenv/issues/306
if hasattr(sys, "pypy_version_info"):
lib = os.path.join(home, 'lib-python', sys.version[:3])
self.lib = Path(lib)
self.bin = Path(bin)
super(VirtualEnvironment, self).__init__(*args, **kwargs)
def test_commandline_basic(tmpdir):
"""Simple command line usage should work and files should be generated"""
home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(str(tmpdir.join("venv")))
subprocess.check_call([sys.executable, VIRTUALENV_SCRIPT, home_dir])
assert os.path.exists(home_dir)
assert os.path.exists(bin_dir)
assert os.path.exists(os.path.join(bin_dir, "activate"))
assert os.path.exists(os.path.join(bin_dir, "activate_this.py"))
assert os.path.exists(os.path.join(bin_dir, "activate.ps1"))
assert os.path.exists(os.path.join(bin_dir, os.path.basename(sys.executable)))
def get_pip_path():
"""
Try to figure out an explicit path to the Pip executable script.
:return: Pip path
:rtype: str
"""
try:
from virtualenv import path_locations
(home_dir, lib_dir, inc_dir, bin_dir) = path_locations(sys.prefix)
return os.path.join(bin_dir, "pip")
except ImportError:
pass
return "pip"
def build(self):
print('Building Python Virtual Environment: {0}...'.format(self.uuid))
home_dir, lib_dir, inc_dir, bin_dir = venv.path_locations(
self._env_path)
self._python = os.path.abspath(venv.install_python(
home_dir, lib_dir, inc_dir, bin_dir, site_packages=False,
clear=False, symlink=True))
venv.install_wheel(['setuptools', 'pip'], self._python, None)
venv.install_distutils(home_dir)
print('Build complete!')
return self._python
def test_activate_with_powershell(tmpdir, monkeypatch):
monkeypatch.chdir(tmpdir)
home_dir, _, __, bin_dir = virtualenv.path_locations(str(tmpdir.join("env")))
virtualenv.create_environment(home_dir, no_pip=True, no_setuptools=True, no_wheel=True)
monkeypatch.chdir(home_dir)
activate_script = join(bin_dir, "activate.ps1")
cmd = [POWER_SHELL, "-Command", "{0}; {1}; {0}; deactivate; {0}".format(print_python_exe_path(), activate_script)]
output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT)
content = output.split()
assert len(content) == 3, output
before_activate, after_activate, after_deactivate = content
exe = "{}.exe".format(virtualenv.expected_exe) if virtualenv.is_win else virtualenv.expected_exe
assert normcase(long_path(after_activate)) == normcase(long_path(join(bin_dir, exe)))
assert before_activate == after_deactivate
def _get_env_bin_path(env_path):
"""Returns the bin path for a virtualenv
This provides a fallback for a race condition in which you're trying
to use the script and create a virtualenv from within
a virtualenv in which virtualenv isn't installed and so
is not importable.
"""
try:
import virtualenv
return virtualenv.path_locations(env_path)[3]
except ImportError:
return os.path.join(env_path, "scripts" if IS_WIN else "bin")
def ext_pillar(minion_id,
pillar,
pillar_name,
project_path,
settings_module,
django_app,
env=None,
env_file=None,
*args,
**kwargs):
'''
Connect to a Django database through the ORM and retrieve model fields
:type pillar_name: str
:param pillar_name: The name of the pillar to be returned
:type project_path: str
:param project_path: The full path to your Django project (the directory
manage.py is in)
:type settings_module: str
:param settings_module: The settings module for your project. This can be
found in your manage.py file
:type django_app: str
:param django_app: A dictionary containing your apps, models, and fields
:type env: str
:param env: The full path to the virtualenv for your Django project
:type env_file: str
:param env_file: An optional bash file that sets up your environment. The
file is run in a subprocess and the changed variables are then added
'''
if not os.path.isdir(project_path):
log.error(
'Django project dir: {0!r} not a directory!'.format(project_path))
return {}
if HAS_VIRTUALENV and env is not None and os.path.isdir(env):
for path in virtualenv.path_locations(env):
if not os.path.isdir(path):
log.error('Virtualenv {0} not a directory!'.format(path))
return {}
# load the virtualenv first
sys.path.insert(
0,
os.path.join(virtualenv.path_locations(env)[1], 'site-packages'))
# load the django project
sys.path.append(project_path)
os.environ['DJANGO_SETTINGS_MODULE'] = settings_module
if env_file is not None:
import subprocess
base_env = {}
proc = subprocess.Popen(['bash', '-c', 'env'], stdout=subprocess.PIPE)
for line in proc.stdout:
(key, _, value) = line.partition('=')
base_env[key] = value
command = ['bash', '-c', 'source {0} && env'.format(env_file)]
proc = subprocess.Popen(command, stdout=subprocess.PIPE)
for line in proc.stdout:
(key, _, value) = line.partition('=')
# only add a key if it is different or doesn't already exist
if key not in base_env or base_env[key] != value:
os.environ[key] = value.rstrip('\n')
log.debug('Adding {0} = {1} to Django environment'.format(
key, value.rstrip('\n')))
try:
from django.db.models.loading import get_model
django_pillar = {}
for proj_app, models in django_app.iteritems():
_, _, app = proj_app.rpartition('.')
django_pillar[app] = {}
for model_name, model_meta in models.iteritems():
model_orm = get_model(app, model_name)
if model_orm is None:
raise salt.exceptions.SaltException(
"Django model '{0}' not found in app '{1}'.".format(
app, model_name))
pillar_for_model = django_pillar[app][model_orm.__name__] = {}
name_field = model_meta['name']
fields = model_meta['fields']
if 'filter' in model_meta.keys():
qs = (model_orm.objects.filter(
**model_meta['filter']).values(*fields))
else:
qs = model_orm.objects.values(*fields)
for model in qs:
# Check that the human-friendly name given is valid (will
# be able to pick up a value from the query) and unique
# (since we're using it as the key in a dictionary)
if not name_field in model:
raise salt.exceptions.SaltException(
"Name '{0}' not found in returned fields.".format(
name_field))
if model[name_field] in pillar_for_model:
raise salt.exceptions.SaltException(
"Value for '{0}' is not unique: {0}".format(
model[name_field]))
pillar_for_model[model[name_field]] = model
return {pillar_name: django_pillar}
except ImportError as e:
log.error('Failed to import library: {0}'.format(e.message))
return {}
except Exception as e:
log.error('Failed on Error: {0}'.format(e.message))
log.debug('django_orm traceback', exc_info=True)
return {}
def deploy_django(proj):
"""
Deploy a Django project
"""
wsgi_base_path = os.environ.get('WSGI_BASE_PATH', '/var/www/wsgi')
httpd_conf_dir = os.environ.get('HTTPD_CONF_DIR', '/etc/httpd/locations.d')
httpd_host = os.environ.get('HTTPD_HOST', platform.node())
httpd_media_base = os.environ.get('HTTPD_MEDIA_BASE',
'/var/www/html/media')
httpd_static_base = os.environ.get('HTTPD_STATIC_BASE',
'/var/www/html/static')
secret_key_gen = os.environ.get('SECRET_KEY_GEN',
'/usr/bin/pwgen -c -n -y 78 1')
proj_base = os.path.join(wsgi_base_path, proj)
path = lambda p: os.path.join(proj_base, p)
proj_defaults = {
'name':
proj,
'proj_base':
proj_base,
'dst':
'%(name)s-project',
'settings':
'%(name)s_production',
'url':
'/%(name)s',
'build':
'build/build.sh',
'wsgi':
'wsgi.py',
'allowed_hosts':
httpd_host,
'secret_key':
subprocess.check_output(secret_key_gen.split()).strip().replace(
"'", "-"),
'media_root':
os.path.join(httpd_media_base, proj),
'static_root':
os.path.join(httpd_static_base, proj),
'scm':
'/usr/bin/git',
'settings_append':
DEFAULT_SETTINGS_APPEND,
'deploy_requires':
None,
'deploy_commands': ['migrate']
}
# Protect '%' from interpolation
proj_defaults['secret_key'] = re.sub(r'%', r'',
proj_defaults['secret_key'])
# Choose clone command
proj_defaults['scm_clone'] = SCM_DEFAULT_CHECKOUT[os.path.split(
proj_defaults['scm'])[-1]]
# Load defaults
cfg = SafeConfigParser(proj_defaults)
# Force read
cfg.readfp(open(proj + '.cfg', 'r'))
#logger.debug('Final configuration:')
#for k,v in cfg.items(CFG_SECTION):
# logger.debug('\t%s: %s', k, v)
# Create directory
os.mkdir(proj_base)
# Virtualenv
virtualenv.create_environment(proj_base)
# Checkout
subprocess.check_call([
cfg.get(CFG_SECTION, 'scm'),
cfg.get(CFG_SECTION, 'scm_clone'),
cfg.get(CFG_SECTION, 'src'),
path(cfg.get(CFG_SECTION, 'dst')),
])
# Build
activate = path('bin/activate')
build = os.path.join(cfg.get(CFG_SECTION, 'dst'),
cfg.get(CFG_SECTION, 'build'))
subprocess.check_call([build], cwd=proj_base, env={'BASH_ENV': activate})
# Install Deploy Requiremts
deploy_requires = cfg.get(CFG_SECTION, 'deploy_requires')
if deploy_requires:
logger.debug('Installing: %s', deploy_requires)
cmd = [
os.path.join(virtualenv.path_locations(proj_base)[-1], 'pip'),
'install'
]
cmd.extend(parse_list(deploy_requires))
subprocess.check_call(cmd)
# Create settings
settings_file = path(cfg.get(CFG_SECTION, 'settings')) + '.py'
slock = LockFile(settings_file)
slock.acquire()
if os.path.exists(settings_file):
slock.release()
raise IOError([17, 'File exists'])
try:
sfp = open(settings_file, 'w')
print(DJANGO_SETTINGS_TEMPLATE % dict(cfg.items(CFG_SECTION)),
file=sfp)
sfp.close()
finally:
slock.release()
# Create wsgi script
wsgi_file = path(cfg.get(CFG_SECTION, 'wsgi'))
slock = LockFile(wsgi_file)
slock.acquire()
if os.path.exists(wsgi_file):
slock.release()
raise IOError([17, 'File exists'])
try:
wfp = open(wsgi_file, 'w')
print(WSGI_TEMPLATE % dict(cfg.items(CFG_SECTION)), file=wfp)
wfp.close()
finally:
slock.release()
# Create apache conf
conf_file = os.path.join(httpd_conf_dir, cfg.get(CFG_SECTION,
'name')) + '.conf'
slock = LockFile(conf_file)
slock.acquire()
if os.path.exists(conf_file):
slock.release()
raise IOError([17, 'File exists'])
try:
sfp = open(conf_file, 'w')
conf = dict(cfg.items(CFG_SECTION))
conf['site_libs'] = os.path.join(
virtualenv.path_locations(proj_base)[1], 'site-packages')
http_conf = HTTPD_CONF_TEMPLATE % conf
print(http_conf, file=sfp)
sfp.close()
finally:
slock.release()
# Perform django commands
deploy_commands = cfg.get(CFG_SECTION, 'deploy_commands')
if deploy_commands:
manage = [
os.path.join(
virtualenv.path_locations(proj_base)[-1],
virtualenv.expected_exe), 'manage.py'
]
os.chdir(path(cfg.get(CFG_SECTION, 'dst')))
# Deployment django environment
dep_env = os.environ.copy()
dep_env['DJANGO_SETTINGS_MODULE'] = cfg.get(CFG_SECTION, 'settings')
dep_env['PYTHONPATH'] = path('.')
logger.debug('Environment for commands: PYTHONPATH=%s',
dep_env['PYTHONPATH'])
logger.debug(' Django settings: %s',
dep_env['DJANGO_SETTINGS_MODULE'])
for cmd in parse_list(deploy_commands):
logger.debug("Executing '%s'", ' '.join(manage + [cmd]))
subprocess.check_call(manage + cmd.split(), env=dep_env)
# That's it. Remember to reload apache
print('You should reload apache:\n', '\t', 'systemctl reload httpd')
return True
def bin_path(self, name):
""" Path to the virtualenv's binaries """
binaries_path = virtualenv.path_locations(self.venv_path)[-1]
return os.path.join(binaries_path, name)
def __init__(self, base_path, *args, **kwargs):
# Make our base_path a test.lib.path.Path object
base_path = Path(base_path)
# Store paths related to the virtual environment
_virtualenv = kwargs.pop("virtualenv")
path_locations = virtualenv.path_locations(_virtualenv)
# Make sure we have test.lib.path.Path objects
venv, lib, include, bin = map(Path, path_locations)
self.venv_path = venv
self.lib_path = virtualenv_lib_path(venv, lib)
self.include_path = include
self.bin_path = bin
if hasattr(sys, "pypy_version_info"):
self.site_packages_path = self.venv_path.join("site-packages")
else:
self.site_packages_path = self.lib_path.join("site-packages")
self.user_base_path = self.venv_path.join("user")
self.user_site_path = self.venv_path.join(
"user",
site.USER_SITE[len(site.USER_BASE) + 1:],
)
if sys.platform == 'win32':
if sys.version_info >= (3, 5):
scripts_base = self.user_site_path.join('..').normpath
else:
scripts_base = self.user_base_path
self.user_bin_path = scripts_base.join('Scripts')
else:
self.user_bin_path = self.user_base_path.join(self.bin_path -
self.venv_path)
# Create a Directory to use as a scratch pad
self.scratch_path = base_path.join("scratch").mkdir()
# Set our default working directory
kwargs.setdefault("cwd", self.scratch_path)
# Setup our environment
environ = kwargs.get("environ")
if environ is None:
environ = os.environ.copy()
environ["PATH"] = Path.pathsep.join([self.bin_path] +
[environ.get("PATH", [])], )
environ["PYTHONUSERBASE"] = self.user_base_path
# Writing bytecode can mess up updated file detection
environ["PYTHONDONTWRITEBYTECODE"] = "1"
# Make sure we get UTF-8 on output, even on Windows...
environ["PYTHONIOENCODING"] = "UTF-8"
kwargs["environ"] = environ
# Call the TestFileEnvironment __init__
super(PipTestEnvironment, self).__init__(base_path, *args, **kwargs)
# Expand our absolute path directories into relative
for name in [
"base", "venv", "lib", "include", "bin", "site_packages",
"user_base", "user_site", "user_bin", "scratch"
]:
real_name = "%s_path" % name
setattr(self, name, getattr(self, real_name) - self.base_path)
# Make sure temp_path is a Path object
self.temp_path = Path(self.temp_path)
# Ensure the tmp dir exists, things break horribly if it doesn't
self.temp_path.mkdir()
# create easy-install.pth in user_site, so we always have it updated
# instead of created
self.user_site_path.makedirs()
self.user_site_path.join("easy-install.pth").touch()
def __init__(self, environ=None):
import virtualenv
self.root_path = fast_test_env_root
self.backup_path = fast_test_env_backup
self.scratch_path = self.root_path / self.scratch
# We will set up a virtual environment at root_path.
self.venv_path = self.root_path / self.venv
if not environ:
environ = os.environ.copy()
environ = clear_environ(environ)
environ['PIP_DOWNLOAD_CACHE'] = str(download_cache)
environ['PIP_NO_INPUT'] = '1'
environ['PIP_LOG_FILE'] = str(self.root_path / 'pip-log.txt')
TestFileEnvironment.__init__(self,
self.root_path,
ignore_hidden=False,
environ=environ,
split_cmd=False,
start_clear=False,
cwd=self.scratch_path,
capture_temp=True,
assert_no_temp=True)
virtualenv_paths = virtualenv.path_locations(self.venv_path)
for id, path in zip(('venv', 'lib', 'include', 'bin'),
virtualenv_paths):
setattr(self, id + '_path', Path(path))
setattr(self, id, relpath(self.root_path, path))
assert self.venv == TestPipEnvironment.venv # sanity check
self.site_packages = self.lib / 'site-packages'
self.user_base_path = self.venv_path / 'user'
self.user_site_path = self.venv_path / 'user' / 'lib' / self.lib.name / 'site-packages'
self.user_site = relpath(self.root_path, self.user_site_path)
self.environ["PYTHONUSERBASE"] = self.user_base_path
# put the test-scratch virtualenv's bin dir first on the PATH
self.environ['PATH'] = Path.pathsep.join(
(self.bin_path, self.environ['PATH']))
self.use_distribute = os.environ.get('PIP_TEST_USE_DISTRIBUTE', False)
if self.root_path.exists:
rmtree(self.root_path)
if self.backup_path.exists:
shutil.copytree(self.backup_path, self.root_path, True)
else:
demand_dirs(self.venv_path)
demand_dirs(self.scratch_path)
# Create a virtualenv and remember where it's putting things.
create_virtualenv(self.venv_path, distribute=self.use_distribute)
demand_dirs(self.user_site_path)
# create easy-install.pth in user_site, so we always have it updated instead of created
open(self.user_site_path / 'easy-install.pth', 'w').close()
# test that test-scratch virtualenv creation produced sensible venv python
result = self.run('python', '-c',
'import sys; print(sys.executable)')
pythonbin = result.stdout.strip()
if Path(pythonbin).noext != self.bin_path / 'python':
raise RuntimeError(
"Oops! 'python' in our test environment runs %r"
" rather than expected %r" %
(pythonbin, self.bin_path / 'python'))
# make sure we have current setuptools to avoid svn incompatibilities
if not self.use_distribute:
install_setuptools(self)
# Uninstall whatever version of pip came with the virtualenv.
# Earlier versions of pip were incapable of
# self-uninstallation on Windows, so we use the one we're testing.
self.run(
'python', '-c',
'"import sys; sys.path.insert(0, %r); import pip; sys.exit(pip.main());"'
% os.path.dirname(here), 'uninstall', '-vvv', '-y', 'pip')
# Install this version instead
self.run('python',
'setup.py',
'install',
cwd=src_folder,
expect_stderr=True)
shutil.copytree(self.root_path, self.backup_path, True)
self._use_cached_pypi_server()
assert self.root_path.exists