def install_venv(name, path, requirements, python_path):
print('[*] Setting up %s virtualenv...' % log.color('1;34', name))
try:
# Convert target path to absolute path.
path = os.path.abspath(path)
# Make sure that base path exists.
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
# Create new virtualenv if it doesn't exist.
if not os.path.exists(path):
env = VirtualEnvironment(path, python=python_path)
env.open_or_create()
# Reopen virtualenv to use the correct Python path
env = VirtualEnvironment(path)
assert env.path == os.path.abspath(path)
# Install requirements for the virtualenv.
# TODO: Shebang lines in binaries installed this way
# use the path in python_path instead of this virtualenv's,
# which I haven't been able to solve yet. A workaround
# is just to pip uninstall and then install it back.
requirements = get_requirements(requirements)
for requirement in requirements:
requirement = requirement.split(' ')
package, options = requirement[0], requirement[1:]
if not env.is_installed(package):
print(' Installing %s...' % log.color('1;33', package))
env.install(package, options=options)
except AssertionError as e:
raise e
except VirtualenvCreationException as e:
# Could not create virtualenv, which executes `virtualenv` as a subprocess under the hood.
log.error('Exception occurred while creating virtualenv: %s' % e)
# Check if we can find the virtualenv bin.
which = shutil.which('virtualenv')
if which is None:
log.error(
'Most probable error: Could not locate `virtualenv` in $PATH.')
return False
log.error('Possible errors:\n' + \
'1. Check the shebang of %s' % which)
return False
except Exception as e:
log.error('Exception (%s) occurred while setting up %s: %s' %
(e.__class__, name, e))
traceback.print_exc()
return False
return True
def example(path=tempfile.mkdtemp('virtualenv.test')):
print('Env path', path)
env = VirtualEnvironment(path)
print('django 1.5 installed?', env.is_installed('django==1.5'))
print('mezzanine installed?', env.is_installed('mezzanine'))
env.install('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
repo = 'git+git://github.com/sjkingo/django_auth_ldap3.git'
pkg = repo.split('/')[-1].replace('.git', '')
print('django_auth_ldap3 installed?', env.is_installed(pkg))
env.install(repo)
print('django_auth_ldap3 installed?', env.is_installed(pkg))
print(env.installed_packages)
env.uninstall('mezzanine')
print('mezzanine installed?', env.is_installed('mezzanine'))
print(env.installed_packages)
pkgs = env.search('requests')
print('search for \'requests\':')
print(len(pkgs), 'found:')
print(pkgs)
def install_project_dependencies_into_virtual_env(project_name):
# Install dependencies defined in requirements.txt (if exists) into virtualenv
repo_dir = get_repo_dir(project_name)
env = VirtualEnvironment('%s/virtual_env' % repo_dir, python='python3')
# Install standard stuff that we want
env.install("nose")
env.install("nose-htmloutput")
env.install("coverage")
requirements_file = "%s/requirements.txt" % repo_dir
if os.path.isfile(requirements_file):
requirements_file = open(requirements_file)
for line in requirements_file.readlines():
if not line.startswith("#"):
try:
env.install(line)
except Exception as error:
print("Error: Could not install package %s. Error: %s" %
(line, repr(error)))
else:
print("Project %s does not have requirements.txt" % project_name)
os.chmod(repo_dir + "/virtual_env/bin/activate", 0o665)
def copy_deps_to_build(deps, build_dir, dest_dir):
venv_dir = os.path.join(build_dir, 'venv')
env = VirtualEnvironment(venv_dir)
env.open_or_create()
venv_site_packages = os.path.join(venv_dir, 'lib', 'python2.7',
'site-packages')
ignore_paths = []
for path in os.listdir(venv_site_packages):
ignore_paths.append(path)
print("venv ignore paths = %r" % ignore_paths)
for dep in deps:
env.install(dep)
for path in os.listdir(venv_site_packages):
if not path in ignore_paths:
fullpath = os.path.join(venv_site_packages, path)
outpath = os.path.join(dest_dir, path)
logging.info("Copying dependency: %s" % path)
if os.path.isfile(fullpath):
shutil.copy2(fullpath, outpath)
else:
shutil.copytree(fullpath, outpath)
else:
logging.info("Ignoring %s" % path)
def example(path='/tmp/virtualenv.test'):
env = VirtualEnvironment(path)
print 'django 1.5 installed?', env.is_installed('django==1.5')
print 'mezzanine installed?', env.is_installed('mezzanine')
env.install('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
print 'cartridge-payments installed?', env.is_installed(payments_repo)
env.install(payments_repo)
print 'cartridge-payments installed?', env.is_installed(payments_repo)
print env.installed_packages
env.uninstall('mezzanine')
print 'mezzanine installed?', env.is_installed('mezzanine')
print env.installed_packages
pkgs = env.search('requests')
print 'search for \'requests\':'
print len(pkgs), 'found:'
print pkgs
def setUp(self):
# See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving?
# Windows and OSX have their own limits so this may not work 100% of the time
long_path = "".join(
[random.choice(string.digits) for _ in range(0, 129)])
self.env_path = tempfile.mkdtemp(long_path)
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def __init__(self,
master_addresses,
process_count=None,
package_cache_dirs=None,
amazon_s3_bucket='ipeterov'):
# This will throw an exception if run not from virtualenv
self.env = VirtualEnvironment()
self.s3_client = boto3.client('s3')
self.amazon_s3_bucket = amazon_s3_bucket
self.thread = threading.Thread(target=self.main)
self.is_alive = True
self.current_master = None
self.master_addresses = master_addresses
if package_cache_dirs == None:
package_cache_dirs = [
'/var/tmp/multiserver/', '~/.multiserver/tmp/'
]
for directory in package_cache_dirs:
if not os.path.exists(directory):
try:
os.makedirs(directory)
except (PermissionError, OSerror) as e:
continue
self.package_cache_dir = directory
break
# Multiprocessing
if process_count == None:
self.process_count = multiprocessing.cpu_count()
else:
self.process_count = process_count
self.manager = multiprocessing.Manager()
self.tasks = multiprocessing.Queue()
self.answers = multiprocessing.Queue()
self.package_lock = multiprocessing.Lock()
self.master_addresses_lock = multiprocessing.Lock()
self.processes = [
_NodeProcess(self) for i in range(self.process_count)
]
# Amazon
try:
self.instance_id = open_url(
'http://instance-data/latest/meta-data/instance-id')
self.instance_type = open_url(
'http://instance-data/latest/meta-data/instance-type')
except URLError:
self.is_amazon = False
else:
self.is_amazon = True
with open('/proc/uptime', 'r') as f:
self.approx_term_time = time_since_epoch() - float(f.readline(
).split()[0]) + 3600 # 3600 так как инстансы работают час
def run(self):
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def fix_venv(path, python, python_path, force):
# Default to local venv path in current directory
if not path:
path = LOCAL_VIRTUALENV_PATH
path = os.path.abspath(path)
# Make sure virtualenv path exists
if not os.path.exists(path):
log.error('Virtualenv path does not exist: %s' % path)
return False
# Validate either Python interpreter or Python path
if not python and not python_path:
log.error(
'Please specify either a Python interpreter (with --python) or Python path (with --python-path) to recreate the virtualenv with.'
)
return False
if python:
data = virtualenvs.get_virtualenvs()
python_path = virtualenvs.resolve_python_paths(python, data)
if not python_path:
log.error('No valid interpreter path found for %s.' % python)
log.info('Valid interpreters: %s' % ', '.join(data.get('pythonPaths').keys()))
return False
# Confirmation prompt
yes = input('Are you sure you want to fix the virtualenv at %s? [y/n] ' % path).upper() == 'Y'
if not yes:
log.info('Aborting.')
return False
# Remove all symlinks in the virtualenv
log.info('Removing broken symlinks in virtualenv...')
num_broken = 0
for dirpath, dirnames, filenames in os.walk(path):
for filename in filenames:
fullpath = os.path.join(dirpath, filename)
# Find broken symlinks, or any symlink if -f is passed
if os.path.islink(fullpath) and (force or not os.path.exists(fullpath)):
log.debug('Removing symlink %s' % fullpath)
os.unlink(fullpath)
num_broken += 1
if num_broken == 0:
log.error('No broken symlinks found. Aborting.')
return False
# Recreate virtualenv
log.info('Recreating virtualenv using %s...' % python_path)
env = VirtualEnvironment(path, python=python_path)
env._create()
log.success('Done!')
def create(self, **kwargs):
"""
build the virtualenv
"""
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
site_packages = kwargs.get('system-site-packages',
self.use_sitepackages)
upgrade = kwargs.get('upgrade', False)
nosetupdevelop = kwargs.get('nosetupdevelop', False)
venv = VirtualEnvironment(self.venv_path,
python=self.python_bin,
system_site_packages=site_packages)
LOGGER.info("Bootstrapping virtualenv: {}".format(self.venv_path))
venv.open_or_create()
cmd = build_pip_command(self.config,
self.venv_path,
self.reqs_name,
upgrade=upgrade)
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Virtualenv: {3}\n"
"Requirements: {4}\n").format(ex, cmd, self.working_dir,
self.venv_path,
self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = build_pip_command(self.config,
self.venv_path,
req,
upgrade=upgrade)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command extra "
"requirements install: {}\n{}").format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
def setup_env(self):
env_path = self.env_path
if env_path is None:
env_path = tempfile.mkdtemp('test_env')
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def test_system_site_packages(self):
"""
test that creating a venv with system_site_packages=True
results in a venv that does not contain the no-global-site-packages file
"""
venv = VirtualEnvironment(self.dir, system_site_packages=True)
venv._create()
expected = os.path.join(venv.path, self.no_global)
self.assertTrue(not os.path.exists(expected))
def run(self):
if sys.hexversion <= 33950192:
# spurious failure on py2.6, so drop out here
return
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def run(self):
if platform.python_implementation(
) == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3:
# spurious failure pypy3 <3.2
return
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def run(self):
if sys.hexversion <= 33950192 or \
(platform.python_implementation() == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3):
# spurious failure on py2.6, so drop out here
# also failing on pypy3 <3.2
return
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('tmp/scratch')
env.install('stomp.py')
def ensure_environment(self, venvs_directory=None):
if not venvs_directory:
venvs_directory = getenv(
"KIRBY_VENV_DIRECTORY",
default=expanduser("~/.kirby/virtualenvs"),
)
venv_path = os.path.join(venvs_directory, self.venv_name)
logging.info(f"creating venv for {self.venv_name} at {venv_path}")
env = VirtualEnvironment(venv_path)
env.install(self.package_name)
return env
def setup_env(self):
env_path = self.env_path
if env_path is None:
# See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving?
# Windows and OSX have their own limits so this may not work 100% of the time
long_path = "".join(
[random.choice(string.digits) for _ in range(0, 129)])
env_path = tempfile.mkdtemp('test_long_env-' + long_path)
virt_env = VirtualEnvironment(env_path)
virt_env._create()
for pack in packages_for_pre_install:
virt_env.install(pack)
return env_path
def _create_venv(name, python):
from virtualenvapi.manage import VirtualEnvironment
import os
def workon_home(*args):
try:
result = os.environ['WORKON_HOME']
except KeyError:
raise RuntimeError('Environment variables WORKON_HOME not found.')
else:
os.makedirs(result, exist_ok=True)
return os.path.join(result, *args)
venv_path = workon_home(name)
result = VirtualEnvironment(venv_path, python=python)
return result
def __init__(self):
self.venvDir = None
self.venv = None
try:
self.venvDir = tempfile.mkdtemp()
LOG.info("Created virtual env dir: %s", self.venvDir)
except Exception as e:
LOG.exception("Exception during virtual env directory create: %s",
e.message)
if not os.path.exists(self.venvDir):
LOG.error("Failed to create virtual env dir: %s. Exiting",
self.venvDir)
return
# create virtual env
self.venv = VirtualEnvironment(self.venvDir)
def __init__(self, name, temp_dir, name_convertor, base_python_version):
self.name = name
self.temp_dir = temp_dir
self.name_convertor = name_convertor
if not base_python_version:
base_python_version = DEFAULT_PYTHON_VERSION
python_version = 'python' + base_python_version
self.env = VirtualEnvironment(temp_dir + '/venv',
python=python_version)
try:
self.env.open_or_create()
except (ve.VirtualenvCreationException,
ve.VirtualenvReadonlyException):
raise VirtualenvFailException('Failed to create virtualenv')
self.dirs_before_install = DirsContent()
self.dirs_after_install = DirsContent()
self.dirs_before_install.fill(temp_dir + '/venv/')
self.data = {}
def test_install_package_dependencies(self):
print("test install package dependencies")
test_repo_name = "__testrepo__"
repo_dir = get_repo_dir(test_repo_name)
if os.path.exists(repo_dir):
shutil.rmtree(repo_dir)
os.makedirs(repo_dir)
self.assertTrue(os.path.isdir(repo_dir))
requirements = ["virtualenv-api==2.1.16"] #, "six"]
requirements_file = open(repo_dir + "requirements.txt", "w")
requirements_file.write('\n'.join(requirements))
requirements_file.close()
install_project_dependencies_into_virtual_env(test_repo_name)
self.assertTrue(os.path.isdir(repo_dir + "virtual_env"))
# Check that requirements are installed
env = VirtualEnvironment(repo_dir + "virtual_env")
for req in requirements:
self.assertTrue(env.is_installed(req))
def setUp(self):
self.env_path = tempfile.mkdtemp()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
def setUp(self):
self.env_path = tempfile.mkdtemp()
self.python = which('python')
self.assertIsNotNone(self.python)
self.virtual_env_obj = VirtualEnvironment(self.env_path,
python=self.python)
def setUp(self):
self.env_path = self.setup_env()
self.python = which('python')
self.assertIsNotNone(self.python)
self.virtual_env_obj = VirtualEnvironment(self.env_path,
python=self.python)
def assert_installed_packages(*packages):
from virtualenvapi.manage import VirtualEnvironment
venv = VirtualEnvironment(str(workon_home.join(venv_name)))
for i_package in packages:
assert venv.is_installed(i_package)
def setUp(self):
self.env_path = self.setup_env()
self.virtual_env_obj = VirtualEnvironment(self.env_path)
continue
if r['sucsess'] == True:
yield r
else:
time.sleep(check_interval)
continue
def mainloop(self):
for result in self.pool.imap_unordered(process_task,
self.task_generator()):
self.send_result(**result)
class InvalidCredentialsException(Exception):
pass
if __name__ == '__main__':
# This will throw an exception if run not from virtualenv
env = VirtualEnvironment()
parser = argparse.ArgumentParser(description='A convenient-rpc node.')
parser.add_argument('server_address')
parser.add_argument('login')
parser.add_argument('password')
args = parser.parse_args()
processor = TaskProcessor(args.server_address, args.login, args.password)
processor.mainloop()
Date: 14 February 2018
Purpose: Creates a requirements.txt file from the directory of a python project.
"""
import sys
import os
from virtualenvapi.manage import VirtualEnvironment
venvFiles=['include','bin','local','lib','pip-selfcheck.json']
def envPath(directory_path):
"""
Parameters:
directory_path : Directory of the python project.
Returns the absolute path of the virtual environment folder.
"""
for dir in os.listdir(directory_path):
#Checks for every folder if it contains the virtual env files.
if os.path.isdir(os.path.join(directory_path,dir)) and set(venvFiles).issubset(set(os.listdir(os.path.join(directory_path,dir)))):
return os.path.join(directory_path,dir)
if __name__ == '__main__':
try:
#Checks if the virtual environment is present in the specified path and creates one if not.
env = VirtualEnvironment(envPath(sys.argv[1]))
#Lists the python packages installed in the virtual env.
pkgs=env.installed_packages
with open("requirements.txt","w") as f:
for pkg,pkg_version in pkgs:
f.write(pkg+"==="+pkg_version+"\n")
except:
print "No virtual env folder"
