def example(path=tempfile.mkdtemp('virtualenv.test')): print('Env path', path) env = VirtualEnvironment(path) print('django 1.5 installed?', env.is_installed('django==1.5')) print('mezzanine installed?', env.is_installed('mezzanine')) env.install('mezzanine') print('mezzanine installed?', env.is_installed('mezzanine')) print(env.installed_packages) payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git' print('cartridge-payments installed?', env.is_installed(payments_repo)) env.install(payments_repo) print('cartridge-payments installed?', env.is_installed(payments_repo)) print(env.installed_packages) env.uninstall('mezzanine') print('mezzanine installed?', env.is_installed('mezzanine')) print(env.installed_packages) pkgs = env.search('requests') print('search for \'requests\':') print(len(pkgs), 'found:') print(pkgs)
def example(path='/tmp/virtualenv.test'): env = VirtualEnvironment(path) print 'django 1.5 installed?', env.is_installed('django==1.5') print 'mezzanine installed?', env.is_installed('mezzanine') env.install('mezzanine') print 'mezzanine installed?', env.is_installed('mezzanine') print env.installed_packages payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git' print 'cartridge-payments installed?', env.is_installed(payments_repo) env.install(payments_repo) print 'cartridge-payments installed?', env.is_installed(payments_repo) print env.installed_packages env.uninstall('mezzanine') print 'mezzanine installed?', env.is_installed('mezzanine') print env.installed_packages pkgs = env.search('requests') print 'search for \'requests\':' print len(pkgs), 'found:' print pkgs
def copy_deps_to_build(deps, build_dir, dest_dir): venv_dir = os.path.join(build_dir, 'venv') env = VirtualEnvironment(venv_dir) env.open_or_create() venv_site_packages = os.path.join(venv_dir, 'lib', 'python2.7', 'site-packages') ignore_paths = [] for path in os.listdir(venv_site_packages): ignore_paths.append(path) print("venv ignore paths = %r" % ignore_paths) for dep in deps: env.install(dep) for path in os.listdir(venv_site_packages): if not path in ignore_paths: fullpath = os.path.join(venv_site_packages, path) outpath = os.path.join(dest_dir, path) logging.info("Copying dependency: %s" % path) if os.path.isfile(fullpath): shutil.copy2(fullpath, outpath) else: shutil.copytree(fullpath, outpath) else: logging.info("Ignoring %s" % path)
def __init__(self, module, environmentDirectory, logging): self.environmentDirectory = environmentDirectory self.logging = logging if not environmentDirectory: logging.debug('No virtual environment will be set up for %s', module.__name__) return if not virtualEnvManagerAvailable: logging.error( '%s requires virtualenv, but virtualenvapi is not installed!', module.__name__ ) sys.exit(10) if not os.path.exists(environmentDirectory): try: os.makedirs(environmentDirectory) except Exception as e: logging.error('Could not create virtualenv directory %s: %s', environmentDirectory, e) venv = VirtualEnvironment(environmentDirectory) for requirement in set(module.requirements): logging.debug( 'Checking if %s is already installed in %s', requirement, environmentDirectory ) try: if not venv.is_installed(requirement): logging.debug('%s not installed in %s', requirement, environmentDirectory) venv.install(requirement) logging.debug('%s installed in %s', requirement, environmentDirectory) else: logging.debug('%s already in %s', requirement, environmentDirectory) except: logging.error('Could not install dependency %s in %s', requirement, environmentDirectory)
def example(path=tempfile.mkdtemp('virtualenv.test')): print('Env path', path) env = VirtualEnvironment(path) print('django 1.5 installed?', env.is_installed('django==1.5')) print('mezzanine installed?', env.is_installed('mezzanine')) env.install('mezzanine') print('mezzanine installed?', env.is_installed('mezzanine')) print(env.installed_packages) repo = 'git+git://github.com/sjkingo/django_auth_ldap3.git' pkg = repo.split('/')[-1].replace('.git', '') print('django_auth_ldap3 installed?', env.is_installed(pkg)) env.install(repo) print('django_auth_ldap3 installed?', env.is_installed(pkg)) print(env.installed_packages) env.uninstall('mezzanine') print('mezzanine installed?', env.is_installed('mezzanine')) print(env.installed_packages) pkgs = env.search('requests') print('search for \'requests\':') print(len(pkgs), 'found:') print(pkgs)
class VirtualEnv(object): def __init__(self, name, temp_dir, name_convertor, base_python_version): self.name = name self.temp_dir = temp_dir self.name_convertor = name_convertor if not base_python_version: base_python_version = DEFAULT_PYTHON_VERSION python_version = 'python' + base_python_version self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version) try: self.env.open_or_create() except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException('Failed to create virtualenv') self.dirs_before_install = DirsContent() self.dirs_after_install = DirsContent() self.dirs_before_install.fill(temp_dir + '/venv/') self.data = {} def install_package_to_venv(self): ''' Installs package given as first argument to virtualenv without dependencies ''' try: self.env.install(self.name, options=["--no-deps"]) except (ve.PackageInstallationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException( 'Failed to install package to virtualenv') self.dirs_after_install.fill(self.temp_dir + '/venv/') @property def get_dirs_differance(self): ''' Makes final versions of site_packages and scripts using DirsContent sub method and filters ''' try: diff = self.dirs_after_install - self.dirs_before_install except ValueError: raise VirtualenvFailException( "Some of the DirsContent attributes is uninicialized") site_packages = site_packages_filter(diff.lib_sitepackages) logger.debug( 'Site_packages from files differance in virtualenv: {0}.'.format( site_packages)) scripts = scripts_filter(list(diff.bindir)) logger.debug( 'Scripts from files differance in virtualenv: {0}.'.format( scripts)) return (site_packages, scripts) @property def get_venv_data(self): self.install_package_to_venv() self.data['packages'], self.data['scripts'] = self.get_dirs_differance return self.data
class VirtualEnv(object): modul_pattern = re.compile(r'\.py.?$') def __init__(self, name, temp_dir, name_convertor, base_python_version): self.name = name self.temp_dir = temp_dir self.name_convertor = name_convertor if not base_python_version: base_python_version = DEFAULT_PYTHON_VERSION python_version = 'python' + base_python_version self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version) try: self.env.open_or_create() except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException('Failed to create virtualenv') self.dirs_before_install = DirsContent() self.dirs_after_install = DirsContent() self.dirs_before_install.fill(temp_dir + '/venv/') self.data = {} def install_package_to_venv(self): ''' Installs package given as first argument to virtualenv without dependencies ''' try: self.env.install(self.name, options=["--no-deps"]) except (ve.PackageInstallationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException('Failed to install package to virtualenv') self.dirs_after_install.fill(self.temp_dir + '/venv/') @property def get_dirs_differance(self): ''' Makes final versions of site_packages and scripts using DirsContent sub method and filters ''' try: diff = self.dirs_after_install - self.dirs_before_install except ValueError: raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized") site_packages = site_packages_filter(diff.lib_sitepackages) packages = set([p for p in site_packages if not self.modul_pattern.search(p)]) py_modules = set([os.path.splitext(m)[0] for m in site_packages - packages]) scripts = scripts_filter(list(diff.bindir)) logger.debug('Packages from files differance in virtualenv: {0}.'.format( packages)) logger.debug('py_modules from files differance in virtualenv: {0}.'.format( py_modules)) logger.debug('Scripts from files differance in virtualenv: {0}.'.format(scripts)) return (packages, py_modules, scripts) @property def get_venv_data(self): self.install_package_to_venv() (self.data['packages'], self.data['py_modules'], self.data['scripts']) = self.get_dirs_differance return self.data
def run(self): if os.path.exists('tmp'): shutil.rmtree('tmp') os.mkdir('tmp') from virtualenvapi.manage import VirtualEnvironment env = VirtualEnvironment('tmp/scratch') env.install('stomp.py')
def work(self): env = VirtualEnvironment(path=self.args['venv'], python="python2.7") env.install('wheel') exit_code, error = self.wheel_from_file(self.args['requirements'], env, self.args['wheel-dir']) return exit_code, error
def install_venv(name, path, requirements, python_path): print('[*] Setting up %s virtualenv...' % log.color('1;34', name)) try: # Convert target path to absolute path. path = os.path.abspath(path) # Make sure that base path exists. if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) # Create new virtualenv if it doesn't exist. if not os.path.exists(path): env = VirtualEnvironment(path, python=python_path) env.open_or_create() # Reopen virtualenv to use the correct Python path env = VirtualEnvironment(path) assert env.path == os.path.abspath(path) # Install requirements for the virtualenv. # TODO: Shebang lines in binaries installed this way # use the path in python_path instead of this virtualenv's, # which I haven't been able to solve yet. A workaround # is just to pip uninstall and then install it back. requirements = get_requirements(requirements) for requirement in requirements: requirement = requirement.split(' ') package, options = requirement[0], requirement[1:] if not env.is_installed(package): print(' Installing %s...' % log.color('1;33', package)) env.install(package, options=options) except AssertionError as e: raise e except VirtualenvCreationException as e: # Could not create virtualenv, which executes `virtualenv` as a subprocess under the hood. log.error('Exception occurred while creating virtualenv: %s' % e) # Check if we can find the virtualenv bin. which = shutil.which('virtualenv') if which is None: log.error( 'Most probable error: Could not locate `virtualenv` in $PATH.') return False log.error('Possible errors:\n' + \ '1. Check the shebang of %s' % which) return False except Exception as e: log.error('Exception (%s) occurred while setting up %s: %s' % (e.__class__, name, e)) traceback.print_exc() return False return True
def setup_env(self): env_path = self.env_path if env_path is None: env_path = tempfile.mkdtemp('test_env') virt_env = VirtualEnvironment(env_path) virt_env._create() for pack in packages_for_pre_install: virt_env.install(pack) return env_path
def setup_env(self): env_path = self.env_path if env_path is None: env_path = tempfile.mkdtemp("test_env") virt_env = VirtualEnvironment(env_path) virt_env._create() for pack in packages_for_pre_install: virt_env.install(pack) return env_path
def run(self): if sys.hexversion <= 33950192: # spurious failure on py2.6, so drop out here return if os.path.exists('tmp'): shutil.rmtree('tmp') os.mkdir('tmp') from virtualenvapi.manage import VirtualEnvironment env = VirtualEnvironment('tmp/scratch') env.install('stomp.py')
def run(self): if platform.python_implementation( ) == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3: # spurious failure pypy3 <3.2 return if os.path.exists('tmp'): shutil.rmtree('tmp') os.mkdir('tmp') from virtualenvapi.manage import VirtualEnvironment env = VirtualEnvironment('tmp/scratch') env.install('stomp.py')
class VirtualEnv(object): def __init__(self, name, temp_dir, name_convertor, base_python_version): self.name = name self.temp_dir = temp_dir self.name_convertor = name_convertor if not base_python_version: base_python_version = DEFAULT_PYTHON_VERSION python_version = 'python' + base_python_version self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version) try: self.env.open_or_create() except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException('Failed to create virtualenv') self.dirs_before_install = DirsContent() self.dirs_after_install = DirsContent() self.dirs_before_install.fill(temp_dir + '/venv/') self.data = {} def install_package_to_venv(self): ''' Installs package given as first argument to virtualenv without dependencies ''' try: self.env.install(self.name, options=["--no-deps"]) except (ve.PackageInstallationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException('Failed to install package to virtualenv') self.dirs_after_install.fill(self.temp_dir + '/venv/') def get_dirs_differance(self): ''' Makes final versions of site_packages and scripts using DirsContent sub method and filters ''' try: diff = self.dirs_after_install - self.dirs_before_install except ValueError: raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized") self.data['has_pth'] = any([x for x in diff.lib_sitepackages if x.endswith('.pth')]) site_packages = site_packages_filter(diff.lib_sitepackages) self.data['packages'] = set([p for p in site_packages if not p.endswith(MODULE_SUFFIXES)]) self.data['py_modules'] = set([os.path.splitext(m)[0] for m in site_packages - self.data['packages']]) self.data['scripts'] = scripts_filter(list(diff.bindir)) logger.debug('Data from files differance in virtualenv:') logger.debug(pprint.pformat(self.data)) @property def get_venv_data(self): self.install_package_to_venv() self.get_dirs_differance() return self.data
def ensure_environment(self, venvs_directory=None): if not venvs_directory: venvs_directory = getenv( "KIRBY_VENV_DIRECTORY", default=expanduser("~/.kirby/virtualenvs"), ) venv_path = os.path.join(venvs_directory, self.venv_name) logging.info(f"creating venv for {self.venv_name} at {venv_path}") env = VirtualEnvironment(venv_path) env.install(self.package_name) return env
def run(self): if sys.hexversion <= 33950192 or \ (platform.python_implementation() == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3): # spurious failure on py2.6, so drop out here # also failing on pypy3 <3.2 return if os.path.exists('tmp'): shutil.rmtree('tmp') os.mkdir('tmp') from virtualenvapi.manage import VirtualEnvironment env = VirtualEnvironment('tmp/scratch') env.install('stomp.py')
def setup_env(self): env_path = self.env_path if env_path is None: # See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving? # Windows and OSX have their own limits so this may not work 100% of the time long_path = "".join([random.choice(string.digits) for _ in range(0, 129)]) env_path = tempfile.mkdtemp('test_long_env-'+long_path) virt_env = VirtualEnvironment(env_path) virt_env._create() for pack in packages_for_pre_install: virt_env.install(pack) return env_path
def setup_env(self): env_path = self.env_path if env_path is None: # See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving? # Windows and OSX have their own limits so this may not work 100% of the time long_path = "".join( [random.choice(string.digits) for _ in range(0, 129)]) env_path = tempfile.mkdtemp('test_long_env-' + long_path) virt_env = VirtualEnvironment(env_path) virt_env._create() for pack in packages_for_pre_install: virt_env.install(pack) return env_path
def install_project_dependencies_into_virtual_env(project_name): # Install dependencies defined in requirements.txt (if exists) into virtualenv repo_dir = get_repo_dir(project_name) env = VirtualEnvironment('%s/virtual_env' % repo_dir, python='python3') # Install standard stuff that we want env.install("nose") env.install("nose-htmloutput") env.install("coverage") requirements_file = "%s/requirements.txt" % repo_dir if os.path.isfile(requirements_file): requirements_file = open(requirements_file) for line in requirements_file.readlines(): if not line.startswith("#"): try: env.install(line) except Exception as error: print("Error: Could not install package %s. Error: %s" % (line, repr(error))) else: print("Project %s does not have requirements.txt" % project_name) os.chmod(repo_dir + "/virtual_env/bin/activate", 0o665)
class VirtualEnv(object): def __init__(self, name, temp_dir, name_convertor, base_python_version): self.name = name self.temp_dir = temp_dir self.name_convertor = name_convertor if not base_python_version: base_python_version = DEFAULT_PYTHON_VERSION python_version = "python" + base_python_version self.env = VirtualEnvironment(temp_dir + "/venv", python=python_version) try: self.env.open_or_create() except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException("Failed to create virtualenv") self.dirs_before_install = DirsContent() self.dirs_after_install = DirsContent() self.dirs_before_install.fill(temp_dir + "/venv/") self.data = {} def install_package_to_venv(self): """ Installs package given as first argument to virtualenv without dependencies """ try: self.env.install(self.name, options=["--no-deps"]) except (ve.PackageInstallationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException("Failed to install package to virtualenv") self.dirs_after_install.fill(self.temp_dir + "/venv/") @property def get_dirs_differance(self): """ Makes final versions of site_packages and scripts using DirsContent sub method and filters """ try: diff = self.dirs_after_install - self.dirs_before_install except ValueError: raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized") site_packages = site_packages_filter(diff.lib_sitepackages) logger.debug("Site_packages from files differance in virtualenv: {0}.".format(site_packages)) scripts = scripts_filter(list(diff.bindir)) logger.debug("Scripts from files differance in virtualenv: {0}.".format(scripts)) return (site_packages, scripts) @property def get_venv_data(self): self.install_package_to_venv() self.data["packages"], self.data["scripts"] = self.get_dirs_differance return self.data
def main(params): # initiate the virtualenv, it will be created if it does not exists env = VirtualEnvironment(os.path.join(PATH, VIRTUALENV)) # use main function to extend the application of the script # env.upgrade_all() # env.uninstall() # rest of the code will be executed with regard to the args # passed to the script, this part will not be executed if no # arguments are given if params['--install_all']: env.install('petl') env.install('pysftp') # env.install('pandas') # env.install('matplotlib') # env.install('numpy') # env.install('scipy') # env.install('statsmodels') # env.install('ggplot') env.install('ipython[notebook]') print '\n\nInstalled packages:\n' for package in env.installed_packages: print package if params['--list_all']: for package in env.installed_packages: print package if params['--is_installed']: print env.is_installed(params['--is_installed'])
class TestBase(unittest.TestCase): """ Base class for test cases to inherit from. """ env_path = None def setUp(self): self.env_path = tempfile.mkdtemp() self.virtual_env_obj = VirtualEnvironment(self.env_path) def tearDown(self): if os.path.exists(self.env_path): shutil.rmtree(self.env_path) def _install_packages(self, packages): for pack in packages: self.virtual_env_obj.install(pack) def _uninstall_packages(self, packages): for pack in packages: self.virtual_env_obj.uninstall(pack)
class VirtualEnv(object): def __init__(self, name, version, temp_dir, name_convertor, base_python_version): self.name = name self.version = version self.temp_dir = temp_dir self.name_convertor = name_convertor if not base_python_version: base_python_version = DEFAULT_PYTHON_VERSION python_version = 'python' + base_python_version self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version) try: self.env.open_or_create() except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException('Failed to create virtualenv') self.dirs_before_install = DirsContent() self.dirs_after_install = DirsContent() self.dirs_before_install.fill(temp_dir + '/venv/') self.data = {} def install_package_to_venv(self): ''' Installs package given as first argument to virtualenv without dependencies ''' try: self.env.install((self.name, self.version), force=True, options=["--no-deps"]) except (ve.PackageInstallationException, ve.VirtualenvReadonlyException): raise VirtualenvFailException( 'Failed to install package to virtualenv') self.dirs_after_install.fill(self.temp_dir + '/venv/') def get_dirs_differance(self): ''' Makes final versions of site_packages and scripts using DirsContent sub method and filters ''' try: diff = self.dirs_after_install - self.dirs_before_install except ValueError: raise VirtualenvFailException( "Some of the DirsContent attributes is uninicialized") self.data['has_pth'] = \ any([x for x in diff.lib_sitepackages if x.endswith('.pth')]) site_packages = site_packages_filter(diff.lib_sitepackages) self.data['packages'] = sorted( [p for p in site_packages if not p.endswith(MODULE_SUFFIXES)]) self.data['py_modules'] = sorted( set([ os.path.splitext(m)[0] for m in site_packages - set(self.data['packages']) ])) self.data['scripts'] = scripts_filter(sorted(diff.bindir)) logger.debug('Data from files differance in virtualenv:') logger.debug(pprint.pformat(self.data)) @property def get_venv_data(self): self.install_package_to_venv() self.get_dirs_differance() return self.data
class BaseTest(TestCase): env_path = None def setUp(self): self.env_path = self.setup_env() self.virtual_env_obj = VirtualEnvironment(self.env_path) def setup_env(self): env_path = self.env_path if env_path is None: env_path = tempfile.mkdtemp('test_env') virt_env = VirtualEnvironment(env_path) virt_env._create() for pack in packages_for_pre_install: virt_env.install(pack) return env_path def _install_packages(self, packages): for pack in packages: self.virtual_env_obj.install(pack) def _uninstall_packages(self, packages): for pack in packages: self.virtual_env_obj.uninstall(pack) def test_installed(self): for pack in packages_for_pre_install: self.assertTrue(self.virtual_env_obj.is_installed(pack)) self.assertFalse(self.virtual_env_obj.is_installed(''.join(random.sample(string.ascii_letters, 30)))) def test_install(self): self._uninstall_packages(packages_for_tests) for pack in packages_for_tests: self.virtual_env_obj.install(pack) self.assertTrue(self.virtual_env_obj.is_installed(pack)) def test_uninstall(self): self._install_packages(packages_for_tests) for pack in packages_for_tests: self.virtual_env_obj.uninstall(pack) self.assertFalse(self.virtual_env_obj.is_installed(pack)) def test_search(self): pack = packages_for_tests[0].lower() result = self.virtual_env_obj.search(pack) self.assertIsInstance(result, list) self.assertTrue(bool(result)) if result: self.assertTrue(isinstance(result[0], (tuple, list))) self.assertIn(pack, (n.lower() for n in dict(result).keys())) def test_search_names(self): pack = packages_for_tests[0].lower() result = self.virtual_env_obj.search_names(pack) self.assertIsInstance(result, list) self.assertIn(pack, (n.lower() for n in result)) def tearDown(self): if os.path.exists(self.env_path) and self.__class__.env_path is None: shutil.rmtree(self.env_path)
from virtualenvapi.manage import VirtualEnvironment env = VirtualEnvironment('.') env.install('pandas') print(env.installed_package_names) print(env.installed_packages) print(env.name) print(env.root) print(env.is_installed('pandas'))
class BaseTest(TestCase): env_path = None def setUp(self): self.env_path = self.setup_env() self.virtual_env_obj = VirtualEnvironment(self.env_path) def setup_env(self): env_path = self.env_path if env_path is None: env_path = tempfile.mkdtemp('test_env') virt_env = VirtualEnvironment(env_path) virt_env._create() for pack in packages_for_pre_install: virt_env.install(pack) return env_path def _install_packages(self, packages): for pack in packages: self.virtual_env_obj.install(pack) def _uninstall_packages(self, packages): for pack in packages: self.virtual_env_obj.uninstall(pack) def test_installed(self): for pack in packages_for_pre_install: self.assertTrue(self.virtual_env_obj.is_installed(pack)) self.assertFalse( self.virtual_env_obj.is_installed(''.join( random.sample(string.ascii_letters, 30)))) def test_install(self): self._uninstall_packages(packages_for_tests) for pack in packages_for_tests: self.virtual_env_obj.install(pack) self.assertTrue(self.virtual_env_obj.is_installed(pack)) def test_uninstall(self): self._install_packages(packages_for_tests) for pack in packages_for_tests: if pack.endswith('.git'): pack = pack.split('/')[-1].replace('.git', '') self.virtual_env_obj.uninstall(pack) self.assertFalse(self.virtual_env_obj.is_installed(pack)) def test_wheel(self): for pack in packages_for_tests: self.virtual_env_obj.wheel(pack, options=['--wheel-dir=/tmp/wheelhouse']) self.virtual_env_obj.install(pack, options=[ '--no-index', '--find-links=/tmp/wheelhouse', '--use-wheel' ]) self.assertTrue(self.virtual_env_obj.is_installed(pack)) def test_search(self): pack = packages_for_tests[0].lower() result = self.virtual_env_obj.search(pack) self.assertIsInstance(result, dict) self.assertTrue(bool(result)) if result: self.assertIn(pack, [k.split(' (')[0].lower() for k in result.keys()]) def test_search_names(self): pack = packages_for_tests[0].lower() result = self.virtual_env_obj.search_names(pack) self.assertIsInstance(result, list) self.assertIn(pack, [k.split(' (')[0].lower() for k in result]) def tearDown(self): if os.path.exists(self.env_path) and self.__class__.env_path is None: shutil.rmtree(self.env_path)
class BaseTest(TestCase): env_path = None def setUp(self): self.env_path = self.setup_env() self.virtual_env_obj = VirtualEnvironment(self.env_path) def setup_env(self): env_path = self.env_path if env_path is None: env_path = tempfile.mkdtemp("test_env") virt_env = VirtualEnvironment(env_path) virt_env._create() for pack in packages_for_pre_install: virt_env.install(pack) return env_path def _install_packages(self, packages): for pack in packages: self.virtual_env_obj.install(pack) def _uninstall_packages(self, packages): for pack in packages: self.virtual_env_obj.uninstall(pack) def test_installed(self): for pack in packages_for_pre_install: self.assertTrue(self.virtual_env_obj.is_installed(pack)) self.assertFalse(self.virtual_env_obj.is_installed("".join(random.sample(string.ascii_letters, 30)))) def test_install(self): self._uninstall_packages(packages_for_tests) for pack in packages_for_tests: self.virtual_env_obj.install(pack) self.assertTrue(self.virtual_env_obj.is_installed(pack)) def test_uninstall(self): self._install_packages(packages_for_tests) for pack in packages_for_tests: if pack.endswith(".git"): pack = pack.split("/")[-1].replace(".git", "") self.virtual_env_obj.uninstall(pack) self.assertFalse(self.virtual_env_obj.is_installed(pack)) def test_wheel(self): for pack in packages_for_tests: self.virtual_env_obj.wheel(pack, options=["--wheel-dir=/tmp/wheelhouse"]) self.virtual_env_obj.install(pack, options=["--no-index", "--find-links=/tmp/wheelhouse", "--use-wheel"]) self.assertTrue(self.virtual_env_obj.is_installed(pack)) def test_search(self): pack = packages_for_tests[0].lower() result = self.virtual_env_obj.search(pack) self.assertIsInstance(result, dict) self.assertTrue(bool(result)) if result: self.assertIn(pack, [k.split(" (")[0].lower() for k in result.keys()]) def test_search_names(self): pack = packages_for_tests[0].lower() result = self.virtual_env_obj.search_names(pack) self.assertIsInstance(result, list) self.assertIn(pack, [k.split(" (")[0].lower() for k in result]) def tearDown(self): if os.path.exists(self.env_path) and self.__class__.env_path is None: shutil.rmtree(self.env_path)
class VirtualEnvHandler(object): def __init__(self): self.venvDir = None self.venv = None try: self.venvDir = tempfile.mkdtemp() LOG.info("Created virtual env dir: %s", self.venvDir) except Exception as e: LOG.exception("Exception during virtual env directory create: %s", e.message) if not os.path.exists(self.venvDir): LOG.error("Failed to create virtual env dir: %s. Exiting", self.venvDir) return # create virtual env self.venv = VirtualEnvironment(self.venvDir) def isValidVenv(self): if self.venv != None: return True return False def getVenvDir(self): return self.venvDir def timeoutHandler(self, signum, frame): raise RuntimeError("Error") def installReqsInVirtualEnv(self, reqFile): try: if os.path.exists(reqFile): LOG.info("Installing pip packages from requirements.txt") self.venv.install('-r', options=[reqFile]) else: # Here, 'reqFile' is the package name self.venv.install(reqFile) except Exception as e: LOG.exception("Error installing package(s)") return False LOG.info("Successfully installed pip packages") return True ''' Upon successful execution of cmdargs (return value 0), output to stdout is returned in the variable out. A non-zero return value results in None being returned to the caller. A time out during execution is considered a success (maybe we are executing a blocking process?). An empty string is returned to indicate success in this case. ''' def testAppInVirtualEnv(self, cmdargs=[]): out = None # If process doesn't quit in TIMEOUT seconds, raise alarm signal.signal(signal.SIGALRM, self.timeoutHandler) signal.alarm(TIMEOUT) try: # Private function. We may want to fork the project for stability. LOG.info("Running python application from %s", self.venvDir) out = self.venv._execute(cmdargs) except RuntimeError as e: LOG.info( "Timed out waiting for app to finish. Exiting with success.") out = '' except Exception as e: LOG.exception("Exception while executing app in virtual env: %s", e.message) # Disable the alarm signal.alarm(0) LOG.info("Output from execution: %s", out) return out
class Node: def __init__(self, master_addresses, process_count=None, package_cache_dirs=None, amazon_s3_bucket='ipeterov'): # This will throw an exception if run not from virtualenv self.env = VirtualEnvironment() self.s3_client = boto3.client('s3') self.amazon_s3_bucket = amazon_s3_bucket self.thread = threading.Thread(target=self.main) self.is_alive = True self.current_master = None self.master_addresses = master_addresses if package_cache_dirs == None: package_cache_dirs = [ '/var/tmp/multiserver/', '~/.multiserver/tmp/' ] for directory in package_cache_dirs: if not os.path.exists(directory): try: os.makedirs(directory) except (PermissionError, OSerror) as e: continue self.package_cache_dir = directory break # Multiprocessing if process_count == None: self.process_count = multiprocessing.cpu_count() else: self.process_count = process_count self.manager = multiprocessing.Manager() self.tasks = multiprocessing.Queue() self.answers = multiprocessing.Queue() self.package_lock = multiprocessing.Lock() self.master_addresses_lock = multiprocessing.Lock() self.processes = [ _NodeProcess(self) for i in range(self.process_count) ] # Amazon try: self.instance_id = open_url( 'http://instance-data/latest/meta-data/instance-id') self.instance_type = open_url( 'http://instance-data/latest/meta-data/instance-type') except URLError: self.is_amazon = False else: self.is_amazon = True with open('/proc/uptime', 'r') as f: self.approx_term_time = time_since_epoch() - float(f.readline( ).split()[0]) + 3600 # 3600 так как инстансы работают час def main(self): while self.is_alive: socket = JSONSocket() with self.master_addresses_lock: for ip, port in self.master_addresses: if socket.connect(ip, port): self.current_master = ip, port break else: continue startup_msg = { 'header': 'start', 'process_count': len(self.processes), 'is_amazon': self.is_amazon } if self.is_amazon: startup_msg['instance_id'] = self.instance_id startup_msg['instance_type'] = self.instance_type startup_msg['approx_term_time'] = self.approx_term_time try: socket.send(startup_msg) except ConnectionLostError: self.current_master = None socket.close() continue while self.is_alive: try: if self.tasks.empty(): socket.send({'header': 'task_request'}) answer = socket.receive() if answer['header'] == 'task': task = answer['task'] self.tasks.put(task) elif answer['header'] == 'no_task': pass if not self.answers.empty(): socket.send(self.answers.get()) except ConnectionLostError: self.current_master = None socket.close() break def start(self): self.thread.start() for process in self.processes: process.start() def stop(self): for process in self.processes: process.terminate() self.is_alive = False self.thread.join() def give_task(self): return self.tasks.get() def grab_answer(self, result): self.answers.put(result) def add_master_address(self, address): with self.master_addresses_lock: self.master_addresses.append(address) def remove_master_address(self, address): with self.master_addresses_lock: self.master_addresses.remove(address) def ensure_package(self, package_filename, package_name, package_version): with self.package_lock: if (package_name, package_version) not in self.env.installed_packages: full_package_path = os.path.join(self.package_cache_dir, package_filename) self.s3_client.download_file(self.amazon_s3_bucket, package_filename, full_package_path) self.env.install(full_package_path, upgrade=True)