def work(self):
env = VirtualEnvironment(path=self.args['dir'], python="python2.7")
env.open_or_create()
return 0, ""
def copy_deps_to_build(deps, build_dir, dest_dir):
venv_dir = os.path.join(build_dir, 'venv')
env = VirtualEnvironment(venv_dir)
env.open_or_create()
venv_site_packages = os.path.join(venv_dir, 'lib', 'python2.7',
'site-packages')
ignore_paths = []
for path in os.listdir(venv_site_packages):
ignore_paths.append(path)
print("venv ignore paths = %r" % ignore_paths)
for dep in deps:
env.install(dep)
for path in os.listdir(venv_site_packages):
if not path in ignore_paths:
fullpath = os.path.join(venv_site_packages, path)
outpath = os.path.join(dest_dir, path)
logging.info("Copying dependency: %s" % path)
if os.path.isfile(fullpath):
shutil.copy2(fullpath, outpath)
else:
shutil.copytree(fullpath, outpath)
else:
logging.info("Ignoring %s" % path)
class VirtualEnv(object):
modul_pattern = re.compile(r'\.py.?$')
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
@property
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
packages = set([p for p in site_packages if not self.modul_pattern.search(p)])
py_modules = set([os.path.splitext(m)[0] for m in site_packages - packages])
scripts = scripts_filter(list(diff.bindir))
logger.debug('Packages from files differance in virtualenv: {0}.'.format(
packages))
logger.debug('py_modules from files differance in virtualenv: {0}.'.format(
py_modules))
logger.debug('Scripts from files differance in virtualenv: {0}.'.format(scripts))
return (packages, py_modules, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
(self.data['packages'], self.data['py_modules'],
self.data['scripts']) = self.get_dirs_differance
return self.data
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv',
python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException(
'Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
@property
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException(
"Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
logger.debug(
'Site_packages from files differance in virtualenv: {0}.'.format(
site_packages))
scripts = scripts_filter(list(diff.bindir))
logger.debug(
'Scripts from files differance in virtualenv: {0}.'.format(
scripts))
return (site_packages, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
self.data['packages'], self.data['scripts'] = self.get_dirs_differance
return self.data
def install_venv(name, path, requirements, python_path):
print('[*] Setting up %s virtualenv...' % log.color('1;34', name))
try:
# Convert target path to absolute path.
path = os.path.abspath(path)
# Make sure that base path exists.
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
# Create new virtualenv if it doesn't exist.
if not os.path.exists(path):
env = VirtualEnvironment(path, python=python_path)
env.open_or_create()
# Reopen virtualenv to use the correct Python path
env = VirtualEnvironment(path)
assert env.path == os.path.abspath(path)
# Install requirements for the virtualenv.
# TODO: Shebang lines in binaries installed this way
# use the path in python_path instead of this virtualenv's,
# which I haven't been able to solve yet. A workaround
# is just to pip uninstall and then install it back.
requirements = get_requirements(requirements)
for requirement in requirements:
requirement = requirement.split(' ')
package, options = requirement[0], requirement[1:]
if not env.is_installed(package):
print(' Installing %s...' % log.color('1;33', package))
env.install(package, options=options)
except AssertionError as e:
raise e
except VirtualenvCreationException as e:
# Could not create virtualenv, which executes `virtualenv` as a subprocess under the hood.
log.error('Exception occurred while creating virtualenv: %s' % e)
# Check if we can find the virtualenv bin.
which = shutil.which('virtualenv')
if which is None:
log.error(
'Most probable error: Could not locate `virtualenv` in $PATH.')
return False
log.error('Possible errors:\n' + \
'1. Check the shebang of %s' % which)
return False
except Exception as e:
log.error('Exception (%s) occurred while setting up %s: %s' %
(e.__class__, name, e))
traceback.print_exc()
return False
return True
def create(self, **kwargs):
"""
build the virtualenv
"""
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
site_packages = kwargs.get('system-site-packages',
self.use_sitepackages)
upgrade = kwargs.get('upgrade', False)
nosetupdevelop = kwargs.get('nosetupdevelop', False)
venv = VirtualEnvironment(self.venv_path,
python=self.python_bin,
system_site_packages=site_packages)
LOGGER.info("Bootstrapping virtualenv: {}".format(self.venv_path))
venv.open_or_create()
cmd = build_pip_command(self.config,
self.venv_path,
self.reqs_name,
upgrade=upgrade)
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Virtualenv: {3}\n"
"Requirements: {4}\n").format(ex, cmd, self.working_dir,
self.venv_path,
self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = build_pip_command(self.config,
self.venv_path,
req,
upgrade=upgrade)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command extra "
"requirements install: {}\n{}").format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
self.data['has_pth'] = any([x for x in diff.lib_sitepackages if x.endswith('.pth')])
site_packages = site_packages_filter(diff.lib_sitepackages)
self.data['packages'] = set([p for p in site_packages if not p.endswith(MODULE_SUFFIXES)])
self.data['py_modules'] = set([os.path.splitext(m)[0]
for m in site_packages - self.data['packages']])
self.data['scripts'] = scripts_filter(list(diff.bindir))
logger.debug('Data from files differance in virtualenv:')
logger.debug(pprint.pformat(self.data))
@property
def get_venv_data(self):
self.install_package_to_venv()
self.get_dirs_differance()
return self.data
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = "python" + base_python_version
self.env = VirtualEnvironment(temp_dir + "/venv", python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException("Failed to create virtualenv")
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + "/venv/")
self.data = {}
def install_package_to_venv(self):
"""
Installs package given as first argument to virtualenv without
dependencies
"""
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException("Failed to install package to virtualenv")
self.dirs_after_install.fill(self.temp_dir + "/venv/")
@property
def get_dirs_differance(self):
"""
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
"""
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
logger.debug("Site_packages from files differance in virtualenv: {0}.".format(site_packages))
scripts = scripts_filter(list(diff.bindir))
logger.debug("Scripts from files differance in virtualenv: {0}.".format(scripts))
return (site_packages, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
self.data["packages"], self.data["scripts"] = self.get_dirs_differance
return self.data
class VirtualEnv(object):
def __init__(self, name, version, temp_dir, name_convertor,
base_python_version):
self.name = name
self.version = version
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv',
python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install((self.name, self.version),
force=True,
options=["--no-deps"])
except (ve.PackageInstallationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException(
'Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException(
"Some of the DirsContent attributes is uninicialized")
self.data['has_pth'] = \
any([x for x in diff.lib_sitepackages if x.endswith('.pth')])
site_packages = site_packages_filter(diff.lib_sitepackages)
self.data['packages'] = sorted(
[p for p in site_packages if not p.endswith(MODULE_SUFFIXES)])
self.data['py_modules'] = sorted(
set([
os.path.splitext(m)[0]
for m in site_packages - set(self.data['packages'])
]))
self.data['scripts'] = scripts_filter(sorted(diff.bindir))
logger.debug('Data from files differance in virtualenv:')
logger.debug(pprint.pformat(self.data))
@property
def get_venv_data(self):
self.install_package_to_venv()
self.get_dirs_differance()
return self.data
def create(self, **kwargs):
"""
build the virtualenv
"""
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
site_packages = kwargs.get(
'system-site-packages',
self.use_sitepackages
)
upgrade = kwargs.get('upgrade', False)
extras_require = kwargs.get('extras_require', [])
all_extras = kwargs.get('all_extras', False)
if all_extras:
extras_require = self.config.extras_require().keys()
nosetupdevelop = kwargs.get('nosetupdevelop', False)
venv = VirtualEnvironment(
self.venv_path,
python=self.python_bin_for_venv,
system_site_packages=site_packages
)
LOGGER.info("Bootstrapping virtualenv: {}".format(self.venv_path))
venv.open_or_create()
cmd = build_pip_command(
self.config,
self.venv_path,
self.reqs_name,
upgrade=upgrade
)
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Virtualenv: {3}\n"
"Requirements: {4}\n"
).format(ex, cmd, self.working_dir, self.venv_path, self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = build_pip_command(
self.config,
self.venv_path,
req,
upgrade=upgrade
)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install command extra "
"requirements install: {}\n{}"
).format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
if extras_require:
for extra in extras_require:
self._install_extras(extra)
