def example(path=tempfile.mkdtemp('virtualenv.test')):
print('Env path', path)
env = VirtualEnvironment(path)
print('django 1.5 installed?', env.is_installed('django==1.5'))
print('mezzanine installed?', env.is_installed('mezzanine'))
env.install('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
print('cartridge-payments installed?', env.is_installed(payments_repo))
env.install(payments_repo)
print('cartridge-payments installed?', env.is_installed(payments_repo))
print(env.installed_packages)
env.uninstall('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
pkgs = env.search('requests')
print('search for \'requests\':')
print(len(pkgs), 'found:')
print(pkgs)
def example(path='/tmp/virtualenv.test'):
env = VirtualEnvironment(path)
print 'django 1.5 installed?', env.is_installed('django==1.5')
print 'mezzanine installed?', env.is_installed('mezzanine')
env.install('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
print 'cartridge-payments installed?', env.is_installed(payments_repo)
env.install(payments_repo)
print 'cartridge-payments installed?', env.is_installed(payments_repo)
print env.installed_packages
env.uninstall('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
pkgs = env.search('requests')
print 'search for \'requests\':'
print len(pkgs), 'found:'
print pkgs
def example(path='/tmp/virtualenv.test'):
env = VirtualEnvironment(path)
print 'django 1.5 installed?', env.is_installed('django==1.5')
print 'mezzanine installed?', env.is_installed('mezzanine')
env.install('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
print 'cartridge-payments installed?', env.is_installed(payments_repo)
env.install(payments_repo)
print 'cartridge-payments installed?', env.is_installed(payments_repo)
print env.installed_packages
env.uninstall('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
pkgs = env.search('requests')
print 'search for \'requests\':'
print len(pkgs), 'found:'
print pkgs
def copy_deps_to_build(deps, build_dir, dest_dir):
venv_dir = os.path.join(build_dir, 'venv')
env = VirtualEnvironment(venv_dir)
env.open_or_create()
venv_site_packages = os.path.join(venv_dir, 'lib', 'python2.7',
'site-packages')
ignore_paths = []
for path in os.listdir(venv_site_packages):
ignore_paths.append(path)
print("venv ignore paths = %r" % ignore_paths)
for dep in deps:
env.install(dep)
for path in os.listdir(venv_site_packages):
if not path in ignore_paths:
fullpath = os.path.join(venv_site_packages, path)
outpath = os.path.join(dest_dir, path)
logging.info("Copying dependency: %s" % path)
if os.path.isfile(fullpath):
shutil.copy2(fullpath, outpath)
else:
shutil.copytree(fullpath, outpath)
else:
logging.info("Ignoring %s" % path)
def __init__(self, module, environmentDirectory, logging):
self.environmentDirectory = environmentDirectory
self.logging = logging
if not environmentDirectory:
logging.debug('No virtual environment will be set up for %s', module.__name__)
return
if not virtualEnvManagerAvailable:
logging.error(
'%s requires virtualenv, but virtualenvapi is not installed!',
module.__name__
)
sys.exit(10)
if not os.path.exists(environmentDirectory):
try:
os.makedirs(environmentDirectory)
except Exception as e:
logging.error('Could not create virtualenv directory %s: %s', environmentDirectory, e)
venv = VirtualEnvironment(environmentDirectory)
for requirement in set(module.requirements):
logging.debug(
'Checking if %s is already installed in %s',
requirement,
environmentDirectory
)
try:
if not venv.is_installed(requirement):
logging.debug('%s not installed in %s', requirement, environmentDirectory)
venv.install(requirement)
logging.debug('%s installed in %s', requirement, environmentDirectory)
else:
logging.debug('%s already in %s', requirement, environmentDirectory)
except:
logging.error('Could not install dependency %s in %s', requirement, environmentDirectory)
def example(path=tempfile.mkdtemp('virtualenv.test')):
print('Env path', path)
env = VirtualEnvironment(path)
print('django 1.5 installed?', env.is_installed('django==1.5'))
print('mezzanine installed?', env.is_installed('mezzanine'))
env.install('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
repo = 'git+git://github.com/sjkingo/django_auth_ldap3.git'
pkg = repo.split('/')[-1].replace('.git', '')
print('django_auth_ldap3 installed?', env.is_installed(pkg))
env.install(repo)
print('django_auth_ldap3 installed?', env.is_installed(pkg))
print(env.installed_packages)
env.uninstall('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
pkgs = env.search('requests')
print('search for \'requests\':')
print(len(pkgs), 'found:')
print(pkgs)
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv',
python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException(
'Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
@property
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException(
"Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
logger.debug(
'Site_packages from files differance in virtualenv: {0}.'.format(
site_packages))
scripts = scripts_filter(list(diff.bindir))
logger.debug(
'Scripts from files differance in virtualenv: {0}.'.format(
scripts))
return (site_packages, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
self.data['packages'], self.data['scripts'] = self.get_dirs_differance
return self.data
class VirtualEnv(object):
modul_pattern = re.compile(r'\.py.?$')
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
@property
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
packages = set([p for p in site_packages if not self.modul_pattern.search(p)])
py_modules = set([os.path.splitext(m)[0] for m in site_packages - packages])
scripts = scripts_filter(list(diff.bindir))
logger.debug('Packages from files differance in virtualenv: {0}.'.format(
packages))
logger.debug('py_modules from files differance in virtualenv: {0}.'.format(
py_modules))
logger.debug('Scripts from files differance in virtualenv: {0}.'.format(scripts))
return (packages, py_modules, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
(self.data['packages'], self.data['py_modules'],
self.data['scripts']) = self.get_dirs_differance
return self.data
def run(self):
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def work(self):
env = VirtualEnvironment(path=self.args['venv'], python="python2.7")
env.install('wheel')
exit_code, error = self.wheel_from_file(self.args['requirements'], env, self.args['wheel-dir'])
return exit_code, error
def run(self):
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def install_venv(name, path, requirements, python_path):
print('[*] Setting up %s virtualenv...' % log.color('1;34', name))
try:
# Convert target path to absolute path.
path = os.path.abspath(path)
# Make sure that base path exists.
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
# Create new virtualenv if it doesn't exist.
if not os.path.exists(path):
env = VirtualEnvironment(path, python=python_path)
env.open_or_create()
# Reopen virtualenv to use the correct Python path
env = VirtualEnvironment(path)
assert env.path == os.path.abspath(path)
# Install requirements for the virtualenv.
# TODO: Shebang lines in binaries installed this way
# use the path in python_path instead of this virtualenv's,
# which I haven't been able to solve yet. A workaround
# is just to pip uninstall and then install it back.
requirements = get_requirements(requirements)
for requirement in requirements:
requirement = requirement.split(' ')
package, options = requirement[0], requirement[1:]
if not env.is_installed(package):
print(' Installing %s...' % log.color('1;33', package))
env.install(package, options=options)
except AssertionError as e:
raise e
except VirtualenvCreationException as e:
# Could not create virtualenv, which executes `virtualenv` as a subprocess under the hood.
log.error('Exception occurred while creating virtualenv: %s' % e)
# Check if we can find the virtualenv bin.
which = shutil.which('virtualenv')
if which is None:
log.error(
'Most probable error: Could not locate `virtualenv` in $PATH.')
return False
log.error('Possible errors:\n' + \
'1. Check the shebang of %s' % which)
return False
except Exception as e:
log.error('Exception (%s) occurred while setting up %s: %s' %
(e.__class__, name, e))
traceback.print_exc()
return False
return True
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp('test_env')
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp("test_env")
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def run(self):
if sys.hexversion <= 33950192:
# spurious failure on py2.6, so drop out here
return
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def run(self):
if platform.python_implementation(
) == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3:
# spurious failure pypy3 <3.2
return
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
self.data['has_pth'] = any([x for x in diff.lib_sitepackages if x.endswith('.pth')])
site_packages = site_packages_filter(diff.lib_sitepackages)
self.data['packages'] = set([p for p in site_packages if not p.endswith(MODULE_SUFFIXES)])
self.data['py_modules'] = set([os.path.splitext(m)[0]
for m in site_packages - self.data['packages']])
self.data['scripts'] = scripts_filter(list(diff.bindir))
logger.debug('Data from files differance in virtualenv:')
logger.debug(pprint.pformat(self.data))
@property
def get_venv_data(self):
self.install_package_to_venv()
self.get_dirs_differance()
return self.data
def ensure_environment(self, venvs_directory=None):
if not venvs_directory:
venvs_directory = getenv(
"KIRBY_VENV_DIRECTORY",
default=expanduser("~/.kirby/virtualenvs"),
)
venv_path = os.path.join(venvs_directory, self.venv_name)
logging.info(f"creating venv for {self.venv_name} at {venv_path}")
env = VirtualEnvironment(venv_path)
env.install(self.package_name)
return env
def run(self):
if sys.hexversion <= 33950192 or \
(platform.python_implementation() == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3):
# spurious failure on py2.6, so drop out here
# also failing on pypy3 <3.2
return
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def setup_env(self):
env_path = self.env_path
if env_path is None:
# See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving?
# Windows and OSX have their own limits so this may not work 100% of the time
long_path = "".join([random.choice(string.digits) for _ in range(0, 129)])
env_path = tempfile.mkdtemp('test_long_env-'+long_path)
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def run(self):
if sys.hexversion <= 33950192 or \
(platform.python_implementation() == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3):
# spurious failure on py2.6, so drop out here
# also failing on pypy3 <3.2
return
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def setup_env(self):
env_path = self.env_path
if env_path is None:
# See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving?
# Windows and OSX have their own limits so this may not work 100% of the time
long_path = "".join(
[random.choice(string.digits) for _ in range(0, 129)])
env_path = tempfile.mkdtemp('test_long_env-' + long_path)
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def install_project_dependencies_into_virtual_env(project_name):
# Install dependencies defined in requirements.txt (if exists) into virtualenv
repo_dir = get_repo_dir(project_name)
env = VirtualEnvironment('%s/virtual_env' % repo_dir, python='python3')
# Install standard stuff that we want
env.install("nose")
env.install("nose-htmloutput")
env.install("coverage")
requirements_file = "%s/requirements.txt" % repo_dir
if os.path.isfile(requirements_file):
requirements_file = open(requirements_file)
for line in requirements_file.readlines():
if not line.startswith("#"):
try:
env.install(line)
except Exception as error:
print("Error: Could not install package %s. Error: %s" %
(line, repr(error)))
else:
print("Project %s does not have requirements.txt" % project_name)
os.chmod(repo_dir + "/virtual_env/bin/activate", 0o665)
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = "python" + base_python_version
self.env = VirtualEnvironment(temp_dir + "/venv", python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException("Failed to create virtualenv")
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + "/venv/")
self.data = {}
def install_package_to_venv(self):
"""
Installs package given as first argument to virtualenv without
dependencies
"""
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException("Failed to install package to virtualenv")
self.dirs_after_install.fill(self.temp_dir + "/venv/")
@property
def get_dirs_differance(self):
"""
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
"""
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
logger.debug("Site_packages from files differance in virtualenv: {0}.".format(site_packages))
scripts = scripts_filter(list(diff.bindir))
logger.debug("Scripts from files differance in virtualenv: {0}.".format(scripts))
return (site_packages, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
self.data["packages"], self.data["scripts"] = self.get_dirs_differance
return self.data
def main(params):
# initiate the virtualenv, it will be created if it does not exists
env = VirtualEnvironment(os.path.join(PATH, VIRTUALENV))
# use main function to extend the application of the script
# env.upgrade_all()
# env.uninstall()
# rest of the code will be executed with regard to the args
# passed to the script, this part will not be executed if no
# arguments are given
if params['--install_all']:
env.install('petl')
env.install('pysftp')
# env.install('pandas')
# env.install('matplotlib')
# env.install('numpy')
# env.install('scipy')
# env.install('statsmodels')
# env.install('ggplot')
env.install('ipython[notebook]')
print '\n\nInstalled packages:\n'
for package in env.installed_packages:
print package
if params['--list_all']:
for package in env.installed_packages:
print package
if params['--is_installed']:
print env.is_installed(params['--is_installed'])
class TestBase(unittest.TestCase):
"""
Base class for test cases to inherit from.
"""
env_path = None
def setUp(self):
self.env_path = tempfile.mkdtemp()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def tearDown(self):
if os.path.exists(self.env_path):
shutil.rmtree(self.env_path)
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
class TestBase(unittest.TestCase):
"""
Base class for test cases to inherit from.
"""
env_path = None
def setUp(self):
self.env_path = tempfile.mkdtemp()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def tearDown(self):
if os.path.exists(self.env_path):
shutil.rmtree(self.env_path)
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
class VirtualEnv(object):
def __init__(self, name, version, temp_dir, name_convertor,
base_python_version):
self.name = name
self.version = version
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv',
python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install((self.name, self.version),
force=True,
options=["--no-deps"])
except (ve.PackageInstallationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException(
'Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException(
"Some of the DirsContent attributes is uninicialized")
self.data['has_pth'] = \
any([x for x in diff.lib_sitepackages if x.endswith('.pth')])
site_packages = site_packages_filter(diff.lib_sitepackages)
self.data['packages'] = sorted(
[p for p in site_packages if not p.endswith(MODULE_SUFFIXES)])
self.data['py_modules'] = sorted(
set([
os.path.splitext(m)[0]
for m in site_packages - set(self.data['packages'])
]))
self.data['scripts'] = scripts_filter(sorted(diff.bindir))
logger.debug('Data from files differance in virtualenv:')
logger.debug(pprint.pformat(self.data))
@property
def get_venv_data(self):
self.install_package_to_venv()
self.get_dirs_differance()
return self.data
class BaseTest(TestCase):
env_path = None
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp('test_env')
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
def test_installed(self):
for pack in packages_for_pre_install:
self.assertTrue(self.virtual_env_obj.is_installed(pack))
self.assertFalse(self.virtual_env_obj.is_installed(''.join(random.sample(string.ascii_letters, 30))))
def test_install(self):
self._uninstall_packages(packages_for_tests)
for pack in packages_for_tests:
self.virtual_env_obj.install(pack)
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_uninstall(self):
self._install_packages(packages_for_tests)
for pack in packages_for_tests:
self.virtual_env_obj.uninstall(pack)
self.assertFalse(self.virtual_env_obj.is_installed(pack))
def test_search(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search(pack)
self.assertIsInstance(result, list)
self.assertTrue(bool(result))
if result:
self.assertTrue(isinstance(result[0], (tuple, list)))
self.assertIn(pack, (n.lower() for n in dict(result).keys()))
def test_search_names(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search_names(pack)
self.assertIsInstance(result, list)
self.assertIn(pack, (n.lower() for n in result))
def tearDown(self):
if os.path.exists(self.env_path) and self.__class__.env_path is None:
shutil.rmtree(self.env_path)
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('.')
env.install('pandas')
print(env.installed_package_names)
print(env.installed_packages)
print(env.name)
print(env.root)
print(env.is_installed('pandas'))
class BaseTest(TestCase):
env_path = None
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp('test_env')
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
def test_installed(self):
for pack in packages_for_pre_install:
self.assertTrue(self.virtual_env_obj.is_installed(pack))
self.assertFalse(
self.virtual_env_obj.is_installed(''.join(
random.sample(string.ascii_letters, 30))))
def test_install(self):
self._uninstall_packages(packages_for_tests)
for pack in packages_for_tests:
self.virtual_env_obj.install(pack)
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_uninstall(self):
self._install_packages(packages_for_tests)
for pack in packages_for_tests:
if pack.endswith('.git'):
pack = pack.split('/')[-1].replace('.git', '')
self.virtual_env_obj.uninstall(pack)
self.assertFalse(self.virtual_env_obj.is_installed(pack))
def test_wheel(self):
for pack in packages_for_tests:
self.virtual_env_obj.wheel(pack,
options=['--wheel-dir=/tmp/wheelhouse'])
self.virtual_env_obj.install(pack,
options=[
'--no-index',
'--find-links=/tmp/wheelhouse',
'--use-wheel'
])
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_search(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search(pack)
self.assertIsInstance(result, dict)
self.assertTrue(bool(result))
if result:
self.assertIn(pack,
[k.split(' (')[0].lower() for k in result.keys()])
def test_search_names(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search_names(pack)
self.assertIsInstance(result, list)
self.assertIn(pack, [k.split(' (')[0].lower() for k in result])
def tearDown(self):
if os.path.exists(self.env_path) and self.__class__.env_path is None:
shutil.rmtree(self.env_path)
class BaseTest(TestCase):
env_path = None
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp("test_env")
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
def test_installed(self):
for pack in packages_for_pre_install:
self.assertTrue(self.virtual_env_obj.is_installed(pack))
self.assertFalse(self.virtual_env_obj.is_installed("".join(random.sample(string.ascii_letters, 30))))
def test_install(self):
self._uninstall_packages(packages_for_tests)
for pack in packages_for_tests:
self.virtual_env_obj.install(pack)
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_uninstall(self):
self._install_packages(packages_for_tests)
for pack in packages_for_tests:
if pack.endswith(".git"):
pack = pack.split("/")[-1].replace(".git", "")
self.virtual_env_obj.uninstall(pack)
self.assertFalse(self.virtual_env_obj.is_installed(pack))
def test_wheel(self):
for pack in packages_for_tests:
self.virtual_env_obj.wheel(pack, options=["--wheel-dir=/tmp/wheelhouse"])
self.virtual_env_obj.install(pack, options=["--no-index", "--find-links=/tmp/wheelhouse", "--use-wheel"])
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_search(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search(pack)
self.assertIsInstance(result, dict)
self.assertTrue(bool(result))
if result:
self.assertIn(pack, [k.split(" (")[0].lower() for k in result.keys()])
def test_search_names(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search_names(pack)
self.assertIsInstance(result, list)
self.assertIn(pack, [k.split(" (")[0].lower() for k in result])
def tearDown(self):
if os.path.exists(self.env_path) and self.__class__.env_path is None:
shutil.rmtree(self.env_path)
class VirtualEnvHandler(object):
def __init__(self):
self.venvDir = None
self.venv = None
try:
self.venvDir = tempfile.mkdtemp()
LOG.info("Created virtual env dir: %s", self.venvDir)
except Exception as e:
LOG.exception("Exception during virtual env directory create: %s",
e.message)
if not os.path.exists(self.venvDir):
LOG.error("Failed to create virtual env dir: %s. Exiting",
self.venvDir)
return
# create virtual env
self.venv = VirtualEnvironment(self.venvDir)
def isValidVenv(self):
if self.venv != None:
return True
return False
def getVenvDir(self):
return self.venvDir
def timeoutHandler(self, signum, frame):
raise RuntimeError("Error")
def installReqsInVirtualEnv(self, reqFile):
try:
if os.path.exists(reqFile):
LOG.info("Installing pip packages from requirements.txt")
self.venv.install('-r', options=[reqFile])
else:
# Here, 'reqFile' is the package name
self.venv.install(reqFile)
except Exception as e:
LOG.exception("Error installing package(s)")
return False
LOG.info("Successfully installed pip packages")
return True
'''
Upon successful execution of cmdargs (return value 0), output to stdout
is returned in the variable out. A non-zero return value results in None
being returned to the caller. A time out during execution is considered
a success (maybe we are executing a blocking process?). An empty string
is returned to indicate success in this case.
'''
def testAppInVirtualEnv(self, cmdargs=[]):
out = None
# If process doesn't quit in TIMEOUT seconds, raise alarm
signal.signal(signal.SIGALRM, self.timeoutHandler)
signal.alarm(TIMEOUT)
try:
# Private function. We may want to fork the project for stability.
LOG.info("Running python application from %s", self.venvDir)
out = self.venv._execute(cmdargs)
except RuntimeError as e:
LOG.info(
"Timed out waiting for app to finish. Exiting with success.")
out = ''
except Exception as e:
LOG.exception("Exception while executing app in virtual env: %s",
e.message)
# Disable the alarm
signal.alarm(0)
LOG.info("Output from execution: %s", out)
return out
class Node:
def __init__(self,
master_addresses,
process_count=None,
package_cache_dirs=None,
amazon_s3_bucket='ipeterov'):
# This will throw an exception if run not from virtualenv
self.env = VirtualEnvironment()
self.s3_client = boto3.client('s3')
self.amazon_s3_bucket = amazon_s3_bucket
self.thread = threading.Thread(target=self.main)
self.is_alive = True
self.current_master = None
self.master_addresses = master_addresses
if package_cache_dirs == None:
package_cache_dirs = [
'/var/tmp/multiserver/', '~/.multiserver/tmp/'
]
for directory in package_cache_dirs:
if not os.path.exists(directory):
try:
os.makedirs(directory)
except (PermissionError, OSerror) as e:
continue
self.package_cache_dir = directory
break
# Multiprocessing
if process_count == None:
self.process_count = multiprocessing.cpu_count()
else:
self.process_count = process_count
self.manager = multiprocessing.Manager()
self.tasks = multiprocessing.Queue()
self.answers = multiprocessing.Queue()
self.package_lock = multiprocessing.Lock()
self.master_addresses_lock = multiprocessing.Lock()
self.processes = [
_NodeProcess(self) for i in range(self.process_count)
]
# Amazon
try:
self.instance_id = open_url(
'http://instance-data/latest/meta-data/instance-id')
self.instance_type = open_url(
'http://instance-data/latest/meta-data/instance-type')
except URLError:
self.is_amazon = False
else:
self.is_amazon = True
with open('/proc/uptime', 'r') as f:
self.approx_term_time = time_since_epoch() - float(f.readline(
).split()[0]) + 3600 # 3600 так как инстансы работают час
def main(self):
while self.is_alive:
socket = JSONSocket()
with self.master_addresses_lock:
for ip, port in self.master_addresses:
if socket.connect(ip, port):
self.current_master = ip, port
break
else:
continue
startup_msg = {
'header': 'start',
'process_count': len(self.processes),
'is_amazon': self.is_amazon
}
if self.is_amazon:
startup_msg['instance_id'] = self.instance_id
startup_msg['instance_type'] = self.instance_type
startup_msg['approx_term_time'] = self.approx_term_time
try:
socket.send(startup_msg)
except ConnectionLostError:
self.current_master = None
socket.close()
continue
while self.is_alive:
try:
if self.tasks.empty():
socket.send({'header': 'task_request'})
answer = socket.receive()
if answer['header'] == 'task':
task = answer['task']
self.tasks.put(task)
elif answer['header'] == 'no_task':
pass
if not self.answers.empty():
socket.send(self.answers.get())
except ConnectionLostError:
self.current_master = None
socket.close()
break
def start(self):
self.thread.start()
for process in self.processes:
process.start()
def stop(self):
for process in self.processes:
process.terminate()
self.is_alive = False
self.thread.join()
def give_task(self):
return self.tasks.get()
def grab_answer(self, result):
self.answers.put(result)
def add_master_address(self, address):
with self.master_addresses_lock:
self.master_addresses.append(address)
def remove_master_address(self, address):
with self.master_addresses_lock:
self.master_addresses.remove(address)
def ensure_package(self, package_filename, package_name, package_version):
with self.package_lock:
if (package_name,
package_version) not in self.env.installed_packages:
full_package_path = os.path.join(self.package_cache_dir,
package_filename)
self.s3_client.download_file(self.amazon_s3_bucket,
package_filename,
full_package_path)
self.env.install(full_package_path, upgrade=True)
