def work(self):
env = VirtualEnvironment(path=self.args['dir'], python="python2.7")
env.open_or_create()
return 0, ""
def __init__(self, module, environmentDirectory, logging):
self.environmentDirectory = environmentDirectory
self.logging = logging
if not environmentDirectory:
logging.debug('No virtual environment will be set up for %s', module.__name__)
return
if not virtualEnvManagerAvailable:
logging.error(
'%s requires virtualenv, but virtualenvapi is not installed!',
module.__name__
)
sys.exit(10)
if not os.path.exists(environmentDirectory):
try:
os.makedirs(environmentDirectory)
except Exception as e:
logging.error('Could not create virtualenv directory %s: %s', environmentDirectory, e)
venv = VirtualEnvironment(environmentDirectory)
for requirement in set(module.requirements):
logging.debug(
'Checking if %s is already installed in %s',
requirement,
environmentDirectory
)
try:
if not venv.is_installed(requirement):
logging.debug('%s not installed in %s', requirement, environmentDirectory)
venv.install(requirement)
logging.debug('%s installed in %s', requirement, environmentDirectory)
else:
logging.debug('%s already in %s', requirement, environmentDirectory)
except:
logging.error('Could not install dependency %s in %s', requirement, environmentDirectory)
class LongPathTestCase(BaseTest):
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setup_env(self):
env_path = self.env_path
if env_path is None:
# See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving?
# Windows and OSX have their own limits so this may not work 100% of the time
long_path = "".join([random.choice(string.digits) for _ in range(0, 129)])
env_path = tempfile.mkdtemp('test_long_env-'+long_path)
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def test_pip_error(self):
self.assertRaises(OSError, self.virtual_env_obj._execute_pip, ['-V'], raw_pip=True)
try:
self.virtual_env_obj._execute_pip(['-V'])
except OSError:
self.fail("_execute_pip raised OSError unexpectedly")
class VirtualEnv(object):
modul_pattern = re.compile(r'\.py.?$')
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
@property
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
packages = set([p for p in site_packages if not self.modul_pattern.search(p)])
py_modules = set([os.path.splitext(m)[0] for m in site_packages - packages])
scripts = scripts_filter(list(diff.bindir))
logger.debug('Packages from files differance in virtualenv: {0}.'.format(
packages))
logger.debug('py_modules from files differance in virtualenv: {0}.'.format(
py_modules))
logger.debug('Scripts from files differance in virtualenv: {0}.'.format(scripts))
return (packages, py_modules, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
(self.data['packages'], self.data['py_modules'],
self.data['scripts']) = self.get_dirs_differance
return self.data
def work(self):
env = VirtualEnvironment(path=self.args['venv'], python="python2.7")
env.install('wheel')
exit_code, error = self.wheel_from_file(self.args['requirements'], env, self.args['wheel-dir'])
return exit_code, error
def run(self):
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp("test_env")
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def main(params):
# initiate the virtualenv, it will be created if it does not exists
env = VirtualEnvironment(os.path.join(PATH, VIRTUALENV))
# use main function to extend the application of the script
# env.upgrade_all()
# env.uninstall()
# rest of the code will be executed with regard to the args
# passed to the script, this part will not be executed if no
# arguments are given
if params['--install_all']:
env.install('petl')
env.install('pysftp')
# env.install('pandas')
# env.install('matplotlib')
# env.install('numpy')
# env.install('scipy')
# env.install('statsmodels')
# env.install('ggplot')
env.install('ipython[notebook]')
print '\n\nInstalled packages:\n'
for package in env.installed_packages:
print package
if params['--list_all']:
for package in env.installed_packages:
print package
if params['--is_installed']:
print env.is_installed(params['--is_installed'])
def test_system_site_packages(self):
"""
test that creating a venv with system_site_packages=True
results in a venv that does not contain the no-global-site-packages file
"""
venv = VirtualEnvironment(self.dir, system_site_packages=True)
venv._create()
expected = os.path.join(venv.path, self.no_global)
self.assertTrue(
not os.path.exists(expected)
)
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
self.data['has_pth'] = any([x for x in diff.lib_sitepackages if x.endswith('.pth')])
site_packages = site_packages_filter(diff.lib_sitepackages)
self.data['packages'] = set([p for p in site_packages if not p.endswith(MODULE_SUFFIXES)])
self.data['py_modules'] = set([os.path.splitext(m)[0]
for m in site_packages - self.data['packages']])
self.data['scripts'] = scripts_filter(list(diff.bindir))
logger.debug('Data from files differance in virtualenv:')
logger.debug(pprint.pformat(self.data))
@property
def get_venv_data(self):
self.install_package_to_venv()
self.get_dirs_differance()
return self.data
def setup_env(self):
env_path = self.env_path
if env_path is None:
# See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving?
# Windows and OSX have their own limits so this may not work 100% of the time
long_path = "".join([random.choice(string.digits) for _ in range(0, 129)])
env_path = tempfile.mkdtemp('test_long_env-'+long_path)
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def run(self):
if sys.hexversion <= 33950192 or \
(platform.python_implementation() == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3):
# spurious failure on py2.6, so drop out here
# also failing on pypy3 <3.2
return
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = "python" + base_python_version
self.env = VirtualEnvironment(temp_dir + "/venv", python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException("Failed to create virtualenv")
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + "/venv/")
self.data = {}
def install_package_to_venv(self):
"""
Installs package given as first argument to virtualenv without
dependencies
"""
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException("Failed to install package to virtualenv")
self.dirs_after_install.fill(self.temp_dir + "/venv/")
@property
def get_dirs_differance(self):
"""
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
"""
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
logger.debug("Site_packages from files differance in virtualenv: {0}.".format(site_packages))
scripts = scripts_filter(list(diff.bindir))
logger.debug("Scripts from files differance in virtualenv: {0}.".format(scripts))
return (site_packages, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
self.data["packages"], self.data["scripts"] = self.get_dirs_differance
return self.data
class TestBase(unittest.TestCase):
"""
Base class for test cases to inherit from.
"""
env_path = None
def setUp(self):
self.env_path = tempfile.mkdtemp()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def tearDown(self):
if os.path.exists(self.env_path):
shutil.rmtree(self.env_path)
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = "python" + base_python_version
self.env = VirtualEnvironment(temp_dir + "/venv", python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
raise VirtualenvFailException("Failed to create virtualenv")
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + "/venv/")
self.data = {}
def example(path=tempfile.mkdtemp('virtualenv.test')):
print('Env path', path)
env = VirtualEnvironment(path)
print('django 1.5 installed?', env.is_installed('django==1.5'))
print('mezzanine installed?', env.is_installed('mezzanine'))
env.install('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
print('cartridge-payments installed?', env.is_installed(payments_repo))
env.install(payments_repo)
print('cartridge-payments installed?', env.is_installed(payments_repo))
print(env.installed_packages)
env.uninstall('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
pkgs = env.search('requests')
print('search for \'requests\':')
print(len(pkgs), 'found:')
print(pkgs)
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def example(path=tempfile.mkdtemp('virtualenv.test')):
print('Env path', path)
env = VirtualEnvironment(path)
print('django 1.5 installed?', env.is_installed('django==1.5'))
print('mezzanine installed?', env.is_installed('mezzanine'))
env.install('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
repo = 'git+git://github.com/sjkingo/django_auth_ldap3.git'
pkg = repo.split('/')[-1].replace('.git', '')
print('django_auth_ldap3 installed?', env.is_installed(pkg))
env.install(repo)
print('django_auth_ldap3 installed?', env.is_installed(pkg))
print(env.installed_packages)
env.uninstall('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
pkgs = env.search('requests')
print('search for \'requests\':')
print(len(pkgs), 'found:')
print(pkgs)
class BaseTest(TestCase):
env_path = None
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp('test_env')
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
def test_installed(self):
for pack in packages_for_pre_install:
self.assertTrue(self.virtual_env_obj.is_installed(pack))
self.assertFalse(self.virtual_env_obj.is_installed(''.join(random.sample(string.ascii_letters, 30))))
def test_install(self):
self._uninstall_packages(packages_for_tests)
for pack in packages_for_tests:
self.virtual_env_obj.install(pack)
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_uninstall(self):
self._install_packages(packages_for_tests)
for pack in packages_for_tests:
self.virtual_env_obj.uninstall(pack)
self.assertFalse(self.virtual_env_obj.is_installed(pack))
def test_search(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search(pack)
self.assertIsInstance(result, list)
self.assertTrue(bool(result))
if result:
self.assertTrue(isinstance(result[0], (tuple, list)))
self.assertIn(pack, (n.lower() for n in dict(result).keys()))
def test_search_names(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search_names(pack)
self.assertIsInstance(result, list)
self.assertIn(pack, (n.lower() for n in result))
def tearDown(self):
if os.path.exists(self.env_path) and self.__class__.env_path is None:
shutil.rmtree(self.env_path)
class VirtualEnvHandler(object):
def __init__(self):
self.venvDir = None
self.venv = None
try:
self.venvDir = tempfile.mkdtemp()
LOG.info("Created virtual env dir: %s", self.venvDir)
except Exception as e:
LOG.exception("Exception during virtual env directory create: %s",
e.message)
if not os.path.exists(self.venvDir):
LOG.error("Failed to create virtual env dir: %s. Exiting",
self.venvDir)
return
# create virtual env
self.venv = VirtualEnvironment(self.venvDir)
def isValidVenv(self):
if self.venv != None:
return True
return False
def getVenvDir(self):
return self.venvDir
def timeoutHandler(self, signum, frame):
raise RuntimeError("Error")
def installReqsInVirtualEnv(self, reqFile):
try:
if os.path.exists(reqFile):
LOG.info("Installing pip packages from requirements.txt")
self.venv.install('-r', options=[reqFile])
else:
# Here, 'reqFile' is the package name
self.venv.install(reqFile)
except Exception as e:
LOG.exception("Error installing package(s)")
return False
LOG.info("Successfully installed pip packages")
return True
'''
Upon successful execution of cmdargs (return value 0), output to stdout
is returned in the variable out. A non-zero return value results in None
being returned to the caller. A time out during execution is considered
a success (maybe we are executing a blocking process?). An empty string
is returned to indicate success in this case.
'''
def testAppInVirtualEnv(self, cmdargs=[]):
out = None
# If process doesn't quit in TIMEOUT seconds, raise alarm
signal.signal(signal.SIGALRM, self.timeoutHandler)
signal.alarm(TIMEOUT)
try:
# Private function. We may want to fork the project for stability.
LOG.info("Running python application from %s", self.venvDir)
out = self.venv._execute(cmdargs)
except RuntimeError as e:
LOG.info(
"Timed out waiting for app to finish. Exiting with success.")
out = ''
except Exception as e:
LOG.exception("Exception while executing app in virtual env: %s",
e.message)
# Disable the alarm
signal.alarm(0)
LOG.info("Output from execution: %s", out)
return out
def setUp(self):
self.env_path = self.setup_env()
self.python = which('python')
self.assertIsNotNone(self.python)
self.virtual_env_obj = VirtualEnvironment(self.env_path,
python=self.python)
def create(self, **kwargs):
"""
build the virtualenv
"""
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
site_packages = kwargs.get(
'system-site-packages',
self.use_sitepackages
)
upgrade = kwargs.get('upgrade', False)
extras_require = kwargs.get('extras_require', [])
all_extras = kwargs.get('all_extras', False)
if all_extras:
extras_require = self.config.extras_require().keys()
nosetupdevelop = kwargs.get('nosetupdevelop', False)
venv = VirtualEnvironment(
self.venv_path,
python=self.python_bin_for_venv,
system_site_packages=site_packages
)
LOGGER.info("Bootstrapping virtualenv: {}".format(self.venv_path))
venv.open_or_create()
cmd = build_pip_command(
self.config,
self.venv_path,
self.reqs_name,
upgrade=upgrade
)
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Virtualenv: {3}\n"
"Requirements: {4}\n"
).format(ex, cmd, self.working_dir, self.venv_path, self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = build_pip_command(
self.config,
self.venv_path,
req,
upgrade=upgrade
)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install command extra "
"requirements install: {}\n{}"
).format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
if extras_require:
for extra in extras_require:
self._install_extras(extra)
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('.')
env.install('pandas')
print(env.installed_package_names)
print(env.installed_packages)
print(env.name)
print(env.root)
print(env.is_installed('pandas'))
class VirtualEnv(object):
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv',
python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, force=True, options=["--no-deps"])
except (ve.PackageInstallationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException(
'Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException(
"Some of the DirsContent attributes is uninicialized")
self.data['has_pth'] = \
any([x for x in diff.lib_sitepackages if x.endswith('.pth')])
site_packages = site_packages_filter(diff.lib_sitepackages)
self.data['packages'] = sorted(
[p for p in site_packages if not p.endswith(MODULE_SUFFIXES)])
self.data['py_modules'] = sorted(
set([
os.path.splitext(m)[0]
for m in site_packages - set(self.data['packages'])
]))
self.data['scripts'] = scripts_filter(sorted(diff.bindir))
logger.debug('Data from files differance in virtualenv:')
logger.debug(pprint.pformat(self.data))
@property
def get_venv_data(self):
self.install_package_to_venv()
self.get_dirs_differance()
return self.data
class Node:
def __init__(self,
master_addresses,
process_count=None,
package_cache_dirs=None,
amazon_s3_bucket='ipeterov'):
# This will throw an exception if run not from virtualenv
self.env = VirtualEnvironment()
self.s3_client = boto3.client('s3')
self.amazon_s3_bucket = amazon_s3_bucket
self.thread = threading.Thread(target=self.main)
self.is_alive = True
self.current_master = None
self.master_addresses = master_addresses
if package_cache_dirs == None:
package_cache_dirs = [
'/var/tmp/multiserver/', '~/.multiserver/tmp/'
]
for directory in package_cache_dirs:
if not os.path.exists(directory):
try:
os.makedirs(directory)
except (PermissionError, OSerror) as e:
continue
self.package_cache_dir = directory
break
# Multiprocessing
if process_count == None:
self.process_count = multiprocessing.cpu_count()
else:
self.process_count = process_count
self.manager = multiprocessing.Manager()
self.tasks = multiprocessing.Queue()
self.answers = multiprocessing.Queue()
self.package_lock = multiprocessing.Lock()
self.master_addresses_lock = multiprocessing.Lock()
self.processes = [
_NodeProcess(self) for i in range(self.process_count)
]
# Amazon
try:
self.instance_id = open_url(
'http://instance-data/latest/meta-data/instance-id')
self.instance_type = open_url(
'http://instance-data/latest/meta-data/instance-type')
except URLError:
self.is_amazon = False
else:
self.is_amazon = True
with open('/proc/uptime', 'r') as f:
self.approx_term_time = time_since_epoch() - float(f.readline(
).split()[0]) + 3600 # 3600 так как инстансы работают час
def main(self):
while self.is_alive:
socket = JSONSocket()
with self.master_addresses_lock:
for ip, port in self.master_addresses:
if socket.connect(ip, port):
self.current_master = ip, port
break
else:
continue
startup_msg = {
'header': 'start',
'process_count': len(self.processes),
'is_amazon': self.is_amazon
}
if self.is_amazon:
startup_msg['instance_id'] = self.instance_id
startup_msg['instance_type'] = self.instance_type
startup_msg['approx_term_time'] = self.approx_term_time
try:
socket.send(startup_msg)
except ConnectionLostError:
self.current_master = None
socket.close()
continue
while self.is_alive:
try:
if self.tasks.empty():
socket.send({'header': 'task_request'})
answer = socket.receive()
if answer['header'] == 'task':
task = answer['task']
self.tasks.put(task)
elif answer['header'] == 'no_task':
pass
if not self.answers.empty():
socket.send(self.answers.get())
except ConnectionLostError:
self.current_master = None
socket.close()
break
def start(self):
self.thread.start()
for process in self.processes:
process.start()
def stop(self):
for process in self.processes:
process.terminate()
self.is_alive = False
self.thread.join()
def give_task(self):
return self.tasks.get()
def grab_answer(self, result):
self.answers.put(result)
def add_master_address(self, address):
with self.master_addresses_lock:
self.master_addresses.append(address)
def remove_master_address(self, address):
with self.master_addresses_lock:
self.master_addresses.remove(address)
def ensure_package(self, package_filename, package_name, package_version):
with self.package_lock:
if (package_name,
package_version) not in self.env.installed_packages:
full_package_path = os.path.join(self.package_cache_dir,
package_filename)
self.s3_client.download_file(self.amazon_s3_bucket,
package_filename,
full_package_path)
self.env.install(full_package_path, upgrade=True)
def example(path='/tmp/virtualenv.test'):
env = VirtualEnvironment(path)
print 'django 1.5 installed?', env.is_installed('django==1.5')
print 'mezzanine installed?', env.is_installed('mezzanine')
env.install('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
print 'cartridge-payments installed?', env.is_installed(payments_repo)
env.install(payments_repo)
print 'cartridge-payments installed?', env.is_installed(payments_repo)
print env.installed_packages
env.uninstall('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
pkgs = env.search('requests')
print 'search for \'requests\':'
print len(pkgs), 'found:'
print pkgs
def setUp(self):
self.env_path = tempfile.mkdtemp()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def assert_installed_packages(*packages):
from virtualenvapi.manage import VirtualEnvironment
venv = VirtualEnvironment(str(workon_home.join(venv_name)))
for i_package in packages:
assert venv.is_installed(i_package)
Date: 14 February 2018
Purpose: Creates a requirements.txt file from the directory of a python project.
"""
import sys
import os
from virtualenvapi.manage import VirtualEnvironment
venvFiles=['include','bin','local','lib','pip-selfcheck.json']
def envPath(directory_path):
"""
Parameters:
directory_path : Directory of the python project.
Returns the absolute path of the virtual environment folder.
"""
for dir in os.listdir(directory_path):
#Checks for every folder if it contains the virtual env files.
if os.path.isdir(os.path.join(directory_path,dir)) and set(venvFiles).issubset(set(os.listdir(os.path.join(directory_path,dir)))):
return os.path.join(directory_path,dir)
if __name__ == '__main__':
try:
#Checks if the virtual environment is present in the specified path and creates one if not.
env = VirtualEnvironment(envPath(sys.argv[1]))
#Lists the python packages installed in the virtual env.
pkgs=env.installed_packages
with open("requirements.txt","w") as f:
for pkg,pkg_version in pkgs:
f.write(pkg+"==="+pkg_version+"\n")
except:
print "No virtual env folder"
class VirtualEnv(object):
modul_pattern = re.compile(r'\.py.?$')
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv',
python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def install_package_to_venv(self):
'''
Installs package given as first argument to virtualenv without
dependencies
'''
try:
self.env.install(self.name, options=["--no-deps"])
except (ve.PackageInstallationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException(
'Failed to install package to virtualenv')
self.dirs_after_install.fill(self.temp_dir + '/venv/')
@property
def get_dirs_differance(self):
'''
Makes final versions of site_packages and scripts using DirsContent
sub method and filters
'''
try:
diff = self.dirs_after_install - self.dirs_before_install
except ValueError:
raise VirtualenvFailException(
"Some of the DirsContent attributes is uninicialized")
site_packages = site_packages_filter(diff.lib_sitepackages)
packages = set(
[p for p in site_packages if not self.modul_pattern.search(p)])
py_modules = set(
[os.path.splitext(m)[0] for m in site_packages - packages])
scripts = scripts_filter(list(diff.bindir))
logger.debug(
'Packages from files differance in virtualenv: {0}.'.format(
packages))
logger.debug(
'py_modules from files differance in virtualenv: {0}.'.format(
py_modules))
logger.debug(
'Scripts from files differance in virtualenv: {0}.'.format(
scripts))
return (packages, py_modules, scripts)
@property
def get_venv_data(self):
self.install_package_to_venv()
(self.data['packages'], self.data['py_modules'],
self.data['scripts']) = self.get_dirs_differance
return self.data
continue
if r['sucsess'] == True:
yield r
else:
time.sleep(check_interval)
continue
def mainloop(self):
for result in self.pool.imap_unordered(process_task,
self.task_generator()):
self.send_result(**result)
class InvalidCredentialsException(Exception):
pass
if __name__ == '__main__':
# This will throw an exception if run not from virtualenv
env = VirtualEnvironment()
parser = argparse.ArgumentParser(description='A convenient-rpc node.')
parser.add_argument('server_address')
parser.add_argument('login')
parser.add_argument('password')
args = parser.parse_args()
processor = TaskProcessor(args.server_address, args.login, args.password)
processor.mainloop()
class BaseTest(TestCase):
env_path = None
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp("test_env")
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
def test_installed(self):
for pack in packages_for_pre_install:
self.assertTrue(self.virtual_env_obj.is_installed(pack))
self.assertFalse(self.virtual_env_obj.is_installed("".join(random.sample(string.ascii_letters, 30))))
def test_install(self):
self._uninstall_packages(packages_for_tests)
for pack in packages_for_tests:
self.virtual_env_obj.install(pack)
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_uninstall(self):
self._install_packages(packages_for_tests)
for pack in packages_for_tests:
if pack.endswith(".git"):
pack = pack.split("/")[-1].replace(".git", "")
self.virtual_env_obj.uninstall(pack)
self.assertFalse(self.virtual_env_obj.is_installed(pack))
def test_wheel(self):
for pack in packages_for_tests:
self.virtual_env_obj.wheel(pack, options=["--wheel-dir=/tmp/wheelhouse"])
self.virtual_env_obj.install(pack, options=["--no-index", "--find-links=/tmp/wheelhouse", "--use-wheel"])
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_search(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search(pack)
self.assertIsInstance(result, dict)
self.assertTrue(bool(result))
if result:
self.assertIn(pack, [k.split(" (")[0].lower() for k in result.keys()])
def test_search_names(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search_names(pack)
self.assertIsInstance(result, list)
self.assertIn(pack, [k.split(" (")[0].lower() for k in result])
def tearDown(self):
if os.path.exists(self.env_path) and self.__class__.env_path is None:
shutil.rmtree(self.env_path)
def example(path='/tmp/virtualenv.test'):
env = VirtualEnvironment(path)
print 'django 1.5 installed?', env.is_installed('django==1.5')
print 'mezzanine installed?', env.is_installed('mezzanine')
env.install('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
print 'cartridge-payments installed?', env.is_installed(payments_repo)
env.install(payments_repo)
print 'cartridge-payments installed?', env.is_installed(payments_repo)
print env.installed_packages
env.uninstall('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
pkgs = env.search('requests')
print 'search for \'requests\':'
print len(pkgs), 'found:'
print pkgs
def create(self, **kwargs):
"""
build the virtualenv
"""
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
site_packages = kwargs.get('system-site-packages',
self.use_sitepackages)
upgrade = kwargs.get('upgrade', False)
extras_require = kwargs.get('extras_require', [])
all_extras = kwargs.get('all_extras', False)
if all_extras:
extras_require = self.config.extras_require().keys()
nosetupdevelop = kwargs.get('nosetupdevelop', False)
venv = VirtualEnvironment(self.venv_path,
python=self.python_bin_for_venv,
system_site_packages=site_packages)
LOGGER.info("Bootstrapping virtualenv: {}".format(self.venv_path))
venv.open_or_create()
cmd = build_pip_command(self.config,
self.venv_path,
self.reqs_name,
upgrade=upgrade)
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Virtualenv: {3}\n"
"Requirements: {4}\n").format(ex, cmd, self.working_dir,
self.venv_path,
self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = build_pip_command(self.config,
self.venv_path,
req,
upgrade=upgrade)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command extra "
"requirements install: {}\n{}").format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
if extras_require:
for extra in extras_require:
self._install_extras(extra)
class BaseTest(TestCase):
env_path = None
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp('test_env')
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def _install_packages(self, packages):
for pack in packages:
self.virtual_env_obj.install(pack)
def _uninstall_packages(self, packages):
for pack in packages:
self.virtual_env_obj.uninstall(pack)
def test_installed(self):
for pack in packages_for_pre_install:
self.assertTrue(self.virtual_env_obj.is_installed(pack))
self.assertFalse(
self.virtual_env_obj.is_installed(''.join(
random.sample(string.ascii_letters, 30))))
def test_install(self):
self._uninstall_packages(packages_for_tests)
for pack in packages_for_tests:
self.virtual_env_obj.install(pack)
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_uninstall(self):
self._install_packages(packages_for_tests)
for pack in packages_for_tests:
if pack.endswith('.git'):
pack = pack.split('/')[-1].replace('.git', '')
self.virtual_env_obj.uninstall(pack)
self.assertFalse(self.virtual_env_obj.is_installed(pack))
def test_wheel(self):
for pack in packages_for_tests:
self.virtual_env_obj.wheel(pack,
options=['--wheel-dir=/tmp/wheelhouse'])
self.virtual_env_obj.install(pack,
options=[
'--no-index',
'--find-links=/tmp/wheelhouse',
'--use-wheel'
])
self.assertTrue(self.virtual_env_obj.is_installed(pack))
def test_search(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search(pack)
self.assertIsInstance(result, dict)
self.assertTrue(bool(result))
if result:
self.assertIn(pack,
[k.split(' (')[0].lower() for k in result.keys()])
def test_search_names(self):
pack = packages_for_tests[0].lower()
result = self.virtual_env_obj.search_names(pack)
self.assertIsInstance(result, list)
self.assertIn(pack, [k.split(' (')[0].lower() for k in result])
def tearDown(self):
if os.path.exists(self.env_path) and self.__class__.env_path is None:
shutil.rmtree(self.env_path)
