def work(self):

        env = VirtualEnvironment(path=self.args['dir'], python="python2.7")

        env.open_or_create()

        return 0, ""
 def __init__(self, module, environmentDirectory, logging):
     self.environmentDirectory = environmentDirectory
     self.logging = logging
     if not environmentDirectory:
         logging.debug('No virtual environment will be set up for %s', module.__name__)
         return
     if not virtualEnvManagerAvailable:
         logging.error(
             '%s requires virtualenv, but virtualenvapi is not installed!',
             module.__name__
         )
         sys.exit(10)
     if not os.path.exists(environmentDirectory):
         try:
             os.makedirs(environmentDirectory)
         except Exception as e:
             logging.error('Could not create virtualenv directory %s: %s', environmentDirectory, e)
     venv = VirtualEnvironment(environmentDirectory)
     for requirement in set(module.requirements):
         logging.debug(
             'Checking if %s is already installed in %s',
             requirement,
             environmentDirectory
         )
         try:
             if not venv.is_installed(requirement):
                 logging.debug('%s not installed in %s', requirement, environmentDirectory)
                 venv.install(requirement)
                 logging.debug('%s installed in %s', requirement, environmentDirectory)
             else:
                 logging.debug('%s already in %s', requirement, environmentDirectory)
         except:
             logging.error('Could not install dependency %s in %s', requirement, environmentDirectory)
Exemple #3
0
class LongPathTestCase(BaseTest):

    def setUp(self):
        self.env_path = self.setup_env()
        self.virtual_env_obj = VirtualEnvironment(self.env_path)

    def setup_env(self):
        env_path = self.env_path
        if env_path is None:
            # See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving?
            # Windows and OSX have their own limits so this may not work 100% of the time
            long_path = "".join([random.choice(string.digits) for _ in range(0, 129)])
            env_path = tempfile.mkdtemp('test_long_env-'+long_path)
            virt_env = VirtualEnvironment(env_path)
            virt_env._create()
            for pack in packages_for_pre_install:
                virt_env.install(pack)

        return env_path

    def test_pip_error(self):
        self.assertRaises(OSError, self.virtual_env_obj._execute_pip, ['-V'], raw_pip=True)
        try:
            self.virtual_env_obj._execute_pip(['-V'])
        except OSError:
            self.fail("_execute_pip raised OSError unexpectedly")
Exemple #4
0
class VirtualEnv(object):

    modul_pattern = re.compile(r'\.py.?$')

    def __init__(self, name, temp_dir, name_convertor, base_python_version):
        self.name = name
        self.temp_dir = temp_dir
        self.name_convertor = name_convertor
        if not base_python_version:
            base_python_version = DEFAULT_PYTHON_VERSION
        python_version = 'python' + base_python_version
        self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version)
        try:
            self.env.open_or_create()
        except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
            raise VirtualenvFailException('Failed to create virtualenv')
        self.dirs_before_install = DirsContent()
        self.dirs_after_install = DirsContent()
        self.dirs_before_install.fill(temp_dir + '/venv/')
        self.data = {}

    def install_package_to_venv(self):
        '''
        Installs package given as first argument to virtualenv without
        dependencies
        '''
        try:
            self.env.install(self.name, options=["--no-deps"])
        except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
            raise VirtualenvFailException('Failed to install package to virtualenv')
        self.dirs_after_install.fill(self.temp_dir + '/venv/')

    @property
    def get_dirs_differance(self):
        '''
        Makes final versions of site_packages and scripts using DirsContent
        sub method and filters
        '''
        try:
            diff = self.dirs_after_install - self.dirs_before_install
        except ValueError:
            raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
        site_packages = site_packages_filter(diff.lib_sitepackages)
        packages = set([p for p in site_packages if not self.modul_pattern.search(p)])
        py_modules = set([os.path.splitext(m)[0] for m in site_packages - packages])
        scripts = scripts_filter(list(diff.bindir))
        logger.debug('Packages from files differance in virtualenv: {0}.'.format(
            packages))
        logger.debug('py_modules from files differance in virtualenv: {0}.'.format(
            py_modules))
        logger.debug('Scripts from files differance in virtualenv: {0}.'.format(scripts))
        return (packages, py_modules, scripts)

    @property
    def get_venv_data(self):
        self.install_package_to_venv()
        (self.data['packages'], self.data['py_modules'],
         self.data['scripts']) = self.get_dirs_differance
        return self.data
Exemple #5
0
    def work(self):
        env = VirtualEnvironment(path=self.args['venv'], python="python2.7")

        env.install('wheel')

        exit_code, error = self.wheel_from_file(self.args['requirements'], env, self.args['wheel-dir'])

        return exit_code, error
Exemple #6
0
 def run(self):
     if os.path.exists('tmp'):
         shutil.rmtree('tmp')
     os.mkdir('tmp')
     
     from virtualenvapi.manage import VirtualEnvironment
     env = VirtualEnvironment('tmp/scratch')
     env.install('stomp.py')            
Exemple #7
0
    def setup_env(self):
        env_path = self.env_path
        if env_path is None:
            env_path = tempfile.mkdtemp("test_env")
            virt_env = VirtualEnvironment(env_path)
            virt_env._create()
            for pack in packages_for_pre_install:
                virt_env.install(pack)

        return env_path
Exemple #8
0
def main(params):

    # initiate the virtualenv, it will be created if it does not exists
    env = VirtualEnvironment(os.path.join(PATH, VIRTUALENV))

    # use main function to extend the application of the script
    # env.upgrade_all()
    # env.uninstall()

    # rest of the code will be executed with regard to the args
    # passed to the script, this part will not be executed if no
    # arguments are given

    if params['--install_all']:
        env.install('petl')
        env.install('pysftp')
        # env.install('pandas')
        # env.install('matplotlib')
        # env.install('numpy')
        # env.install('scipy')
        # env.install('statsmodels')
        # env.install('ggplot')
        env.install('ipython[notebook]')

        print '\n\nInstalled packages:\n'
        for package in env.installed_packages:
            print package

    if params['--list_all']:
        for package in env.installed_packages:
            print package

    if params['--is_installed']:
        print env.is_installed(params['--is_installed'])
Exemple #9
0
    def test_system_site_packages(self):
        """
        test that creating a venv with system_site_packages=True
        results in a venv that does not contain the no-global-site-packages file

        """
        venv = VirtualEnvironment(self.dir, system_site_packages=True)
        venv._create()
        expected = os.path.join(venv.path, self.no_global)
        self.assertTrue(
            not os.path.exists(expected)
        )
Exemple #10
0
class VirtualEnv(object):

    def __init__(self, name, temp_dir, name_convertor, base_python_version):
        self.name = name
        self.temp_dir = temp_dir
        self.name_convertor = name_convertor
        if not base_python_version:
            base_python_version = DEFAULT_PYTHON_VERSION
        python_version = 'python' + base_python_version
        self.env = VirtualEnvironment(temp_dir + '/venv', python=python_version)
        try:
            self.env.open_or_create()
        except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
            raise VirtualenvFailException('Failed to create virtualenv')
        self.dirs_before_install = DirsContent()
        self.dirs_after_install = DirsContent()
        self.dirs_before_install.fill(temp_dir + '/venv/')
        self.data = {}

    def install_package_to_venv(self):
        '''
        Installs package given as first argument to virtualenv without
        dependencies
        '''
        try:
            self.env.install(self.name, options=["--no-deps"])
        except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
            raise VirtualenvFailException('Failed to install package to virtualenv')
        self.dirs_after_install.fill(self.temp_dir + '/venv/')

    def get_dirs_differance(self):
        '''
        Makes final versions of site_packages and scripts using DirsContent
        sub method and filters
        '''
        try:
            diff = self.dirs_after_install - self.dirs_before_install
        except ValueError:
            raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
        self.data['has_pth'] = any([x for x in diff.lib_sitepackages if x.endswith('.pth')])
        site_packages = site_packages_filter(diff.lib_sitepackages)
        self.data['packages'] = set([p for p in site_packages if not p.endswith(MODULE_SUFFIXES)])
        self.data['py_modules'] = set([os.path.splitext(m)[0]
                                       for m in site_packages - self.data['packages']])
        self.data['scripts'] = scripts_filter(list(diff.bindir))
        logger.debug('Data from files differance in virtualenv:')
        logger.debug(pprint.pformat(self.data))

    @property
    def get_venv_data(self):
        self.install_package_to_venv()
        self.get_dirs_differance()
        return self.data
Exemple #11
0
    def setup_env(self):
        env_path = self.env_path
        if env_path is None:
            # See: https://github.com/pypa/pip/issues/1773 This test may not be 100% accurate, tips on improving?
            # Windows and OSX have their own limits so this may not work 100% of the time
            long_path = "".join([random.choice(string.digits) for _ in range(0, 129)])
            env_path = tempfile.mkdtemp('test_long_env-'+long_path)
            virt_env = VirtualEnvironment(env_path)
            virt_env._create()
            for pack in packages_for_pre_install:
                virt_env.install(pack)

        return env_path
Exemple #12
0
    def run(self):
        if sys.hexversion <= 33950192 or \
            (platform.python_implementation() == 'PyPy' and sys.version_info.major == 3 and sys.version_info.minor < 3):
            # spurious failure on py2.6, so drop out here
            # also failing on pypy3 <3.2
            return
        if os.path.exists('tmp'):
            shutil.rmtree('tmp')
        os.mkdir('tmp')

        from virtualenvapi.manage import VirtualEnvironment
        env = VirtualEnvironment('tmp/scratch')
        env.install('stomp.py')
Exemple #13
0
class VirtualEnv(object):
    def __init__(self, name, temp_dir, name_convertor, base_python_version):
        self.name = name
        self.temp_dir = temp_dir
        self.name_convertor = name_convertor
        if not base_python_version:
            base_python_version = DEFAULT_PYTHON_VERSION
        python_version = "python" + base_python_version
        self.env = VirtualEnvironment(temp_dir + "/venv", python=python_version)
        try:
            self.env.open_or_create()
        except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
            raise VirtualenvFailException("Failed to create virtualenv")
        self.dirs_before_install = DirsContent()
        self.dirs_after_install = DirsContent()
        self.dirs_before_install.fill(temp_dir + "/venv/")
        self.data = {}

    def install_package_to_venv(self):
        """
        Installs package given as first argument to virtualenv without
        dependencies
        """
        try:
            self.env.install(self.name, options=["--no-deps"])
        except (ve.PackageInstallationException, ve.VirtualenvReadonlyException):
            raise VirtualenvFailException("Failed to install package to virtualenv")
        self.dirs_after_install.fill(self.temp_dir + "/venv/")

    @property
    def get_dirs_differance(self):
        """
        Makes final versions of site_packages and scripts using DirsContent
        sub method and filters
        """
        try:
            diff = self.dirs_after_install - self.dirs_before_install
        except ValueError:
            raise VirtualenvFailException("Some of the DirsContent attributes is uninicialized")
        site_packages = site_packages_filter(diff.lib_sitepackages)
        logger.debug("Site_packages from files differance in virtualenv: {0}.".format(site_packages))
        scripts = scripts_filter(list(diff.bindir))
        logger.debug("Scripts from files differance in virtualenv: {0}.".format(scripts))
        return (site_packages, scripts)

    @property
    def get_venv_data(self):
        self.install_package_to_venv()
        self.data["packages"], self.data["scripts"] = self.get_dirs_differance
        return self.data
Exemple #14
0
class TestBase(unittest.TestCase):
    """
    Base class for test cases to inherit from.
    """

    env_path = None

    def setUp(self):
        self.env_path = tempfile.mkdtemp()
        self.virtual_env_obj = VirtualEnvironment(self.env_path)

    def tearDown(self):
        if os.path.exists(self.env_path):
            shutil.rmtree(self.env_path)

    def _install_packages(self, packages):
        for pack in packages:
            self.virtual_env_obj.install(pack)

    def _uninstall_packages(self, packages):
        for pack in packages:
            self.virtual_env_obj.uninstall(pack)
Exemple #15
0
 def __init__(self, name, temp_dir, name_convertor, base_python_version):
     self.name = name
     self.temp_dir = temp_dir
     self.name_convertor = name_convertor
     if not base_python_version:
         base_python_version = DEFAULT_PYTHON_VERSION
     python_version = "python" + base_python_version
     self.env = VirtualEnvironment(temp_dir + "/venv", python=python_version)
     try:
         self.env.open_or_create()
     except (ve.VirtualenvCreationException, ve.VirtualenvReadonlyException):
         raise VirtualenvFailException("Failed to create virtualenv")
     self.dirs_before_install = DirsContent()
     self.dirs_after_install = DirsContent()
     self.dirs_before_install.fill(temp_dir + "/venv/")
     self.data = {}
Exemple #16
0
def example(path=tempfile.mkdtemp('virtualenv.test')):
    print('Env path', path)
    env = VirtualEnvironment(path)

    print('django 1.5 installed?', env.is_installed('django==1.5'))

    print('mezzanine installed?', env.is_installed('mezzanine'))
    env.install('mezzanine')
    print('mezzanine installed?', env.is_installed('mezzanine'))

    print(env.installed_packages)

    payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
    print('cartridge-payments installed?', env.is_installed(payments_repo))
    env.install(payments_repo)
    print('cartridge-payments installed?', env.is_installed(payments_repo))
    print(env.installed_packages)

    env.uninstall('mezzanine')
    print('mezzanine installed?', env.is_installed('mezzanine'))
    print(env.installed_packages)

    pkgs = env.search('requests')
    print('search for \'requests\':')
    print(len(pkgs), 'found:')
    print(pkgs)
Exemple #17
0
 def setUp(self):
     self.env_path = self.setup_env()
     self.virtual_env_obj = VirtualEnvironment(self.env_path)
Exemple #18
0
 def setUp(self):
     self.env_path = self.setup_env()
     self.virtual_env_obj = VirtualEnvironment(self.env_path)
Exemple #19
0
def example(path=tempfile.mkdtemp('virtualenv.test')):
    print('Env path', path)
    env = VirtualEnvironment(path)

    print('django 1.5 installed?', env.is_installed('django==1.5'))

    print('mezzanine installed?', env.is_installed('mezzanine'))
    env.install('mezzanine')
    print('mezzanine installed?', env.is_installed('mezzanine'))

    print(env.installed_packages)

    repo = 'git+git://github.com/sjkingo/django_auth_ldap3.git'
    pkg = repo.split('/')[-1].replace('.git', '')
    print('django_auth_ldap3 installed?', env.is_installed(pkg))
    env.install(repo)
    print('django_auth_ldap3 installed?', env.is_installed(pkg))
    print(env.installed_packages)

    env.uninstall('mezzanine')
    print('mezzanine installed?', env.is_installed('mezzanine'))
    print(env.installed_packages)

    pkgs = env.search('requests')
    print('search for \'requests\':')
    print(len(pkgs), 'found:')
    print(pkgs)
Exemple #20
0
class BaseTest(TestCase):

    env_path = None

    def setUp(self):
        self.env_path = self.setup_env()
        self.virtual_env_obj = VirtualEnvironment(self.env_path)

    def setup_env(self):
        env_path = self.env_path
        if env_path is None:
            env_path = tempfile.mkdtemp('test_env')
            virt_env = VirtualEnvironment(env_path)
            virt_env._create()
            for pack in packages_for_pre_install:
                virt_env.install(pack)

        return env_path

    def _install_packages(self, packages):
        for pack in packages:
            self.virtual_env_obj.install(pack)

    def _uninstall_packages(self, packages):
        for pack in packages:
            self.virtual_env_obj.uninstall(pack)

    def test_installed(self):
        for pack in packages_for_pre_install:
            self.assertTrue(self.virtual_env_obj.is_installed(pack))
        self.assertFalse(self.virtual_env_obj.is_installed(''.join(random.sample(string.ascii_letters, 30))))

    def test_install(self):
        self._uninstall_packages(packages_for_tests)
        for pack in packages_for_tests:
            self.virtual_env_obj.install(pack)
            self.assertTrue(self.virtual_env_obj.is_installed(pack))

    def test_uninstall(self):
        self._install_packages(packages_for_tests)
        for pack in packages_for_tests:
            self.virtual_env_obj.uninstall(pack)
            self.assertFalse(self.virtual_env_obj.is_installed(pack))

    def test_search(self):
        pack = packages_for_tests[0].lower()
        result = self.virtual_env_obj.search(pack)
        self.assertIsInstance(result, list)
        self.assertTrue(bool(result))
        if result:
            self.assertTrue(isinstance(result[0], (tuple, list)))
            self.assertIn(pack, (n.lower() for n in dict(result).keys()))

    def test_search_names(self):
        pack = packages_for_tests[0].lower()
        result = self.virtual_env_obj.search_names(pack)
        self.assertIsInstance(result, list)
        self.assertIn(pack, (n.lower() for n in result))

    def tearDown(self):
        if os.path.exists(self.env_path) and self.__class__.env_path is None:
            shutil.rmtree(self.env_path)
class VirtualEnvHandler(object):
    def __init__(self):
        self.venvDir = None
        self.venv = None

        try:
            self.venvDir = tempfile.mkdtemp()
            LOG.info("Created virtual env dir: %s", self.venvDir)
        except Exception as e:
            LOG.exception("Exception during virtual env directory create: %s",
                          e.message)

        if not os.path.exists(self.venvDir):
            LOG.error("Failed to create virtual env dir: %s. Exiting",
                      self.venvDir)
            return

        # create virtual env
        self.venv = VirtualEnvironment(self.venvDir)

    def isValidVenv(self):
        if self.venv != None:
            return True
        return False

    def getVenvDir(self):
        return self.venvDir

    def timeoutHandler(self, signum, frame):
        raise RuntimeError("Error")

    def installReqsInVirtualEnv(self, reqFile):
        try:
            if os.path.exists(reqFile):
                LOG.info("Installing pip packages from requirements.txt")
                self.venv.install('-r', options=[reqFile])
            else:
                # Here, 'reqFile' is the package name
                self.venv.install(reqFile)

        except Exception as e:
            LOG.exception("Error installing package(s)")
            return False

        LOG.info("Successfully installed pip packages")
        return True

    '''
    Upon successful execution of cmdargs (return value 0), output to stdout
    is returned in the variable out. A non-zero return value results in None
    being returned to the caller. A time out during execution is considered
    a success (maybe we are executing a blocking process?). An empty string
    is returned to indicate success in this case.
    '''

    def testAppInVirtualEnv(self, cmdargs=[]):
        out = None

        # If process doesn't quit in TIMEOUT seconds, raise alarm
        signal.signal(signal.SIGALRM, self.timeoutHandler)
        signal.alarm(TIMEOUT)

        try:
            # Private function. We may want to fork the project for stability.
            LOG.info("Running python application from %s", self.venvDir)
            out = self.venv._execute(cmdargs)

        except RuntimeError as e:
            LOG.info(
                "Timed out waiting for app to finish. Exiting with success.")
            out = ''

        except Exception as e:
            LOG.exception("Exception while executing app in virtual env: %s",
                          e.message)

        # Disable the alarm
        signal.alarm(0)
        LOG.info("Output from execution: %s", out)
        return out
Exemple #22
0
 def setUp(self):
     self.env_path = self.setup_env()
     self.python = which('python')
     self.assertIsNotNone(self.python)
     self.virtual_env_obj = VirtualEnvironment(self.env_path,
                                               python=self.python)
Exemple #23
0
    def create(self, **kwargs):
        """
        build the virtualenv
        """
        clean = kwargs.get('clean', False)
        if clean:
            self.clean(**kwargs)

        site_packages = kwargs.get(
            'system-site-packages',
            self.use_sitepackages
        )
        upgrade = kwargs.get('upgrade', False)
        extras_require = kwargs.get('extras_require', [])
        all_extras = kwargs.get('all_extras', False)
        if all_extras:
            extras_require = self.config.extras_require().keys()
        nosetupdevelop = kwargs.get('nosetupdevelop', False)
        venv = VirtualEnvironment(
            self.venv_path,
            python=self.python_bin_for_venv,
            system_site_packages=site_packages
        )
        LOGGER.info("Bootstrapping virtualenv: {}".format(self.venv_path))

        venv.open_or_create()
        cmd = build_pip_command(
            self.config,
            self.venv_path,
            self.reqs_name,
            upgrade=upgrade
        )

        try:
            local(cmd)
        except OSError as ex:
            msg = (
                "Error running pip install command during build\n"
                "Error was {0}\n"
                "Running command: {1}\n"
                "Working Dir: {2}\n"
                "Virtualenv: {3}\n"
                "Requirements: {4}\n"
            ).format(ex, cmd, self.working_dir, self.venv_path, self.reqs_name)
            LOGGER.error(msg)
            raise

        for req in self.extra_reqs:
            cmd = build_pip_command(
                self.config,
                self.venv_path,
                req,
                upgrade=upgrade
            )
            LOGGER.info("Installing extra requirements... {}".format(cmd))
            try:
                local(cmd)
            except OSError as ex:
                msg = (
                    "Error running pip install command extra "
                    "requirements install: {}\n{}"
                ).format(req, ex)
                LOGGER.error(msg)
                raise
        # setup for development
        if nosetupdevelop:
            msg = "skipping python setup.py develop..."
            LOGGER.info(msg)
        else:
            self.run_setup_develop()
            if extras_require:
                for extra in extras_require:
                    self._install_extras(extra)
Exemple #24
0
from virtualenvapi.manage import VirtualEnvironment
env = VirtualEnvironment('.')
env.install('pandas')
print(env.installed_package_names)
print(env.installed_packages)
print(env.name)
print(env.root)
print(env.is_installed('pandas'))
Exemple #25
0
class VirtualEnv(object):
    def __init__(self, name, temp_dir, name_convertor, base_python_version):
        self.name = name
        self.temp_dir = temp_dir
        self.name_convertor = name_convertor
        if not base_python_version:
            base_python_version = DEFAULT_PYTHON_VERSION
        python_version = 'python' + base_python_version
        self.env = VirtualEnvironment(temp_dir + '/venv',
                                      python=python_version)
        try:
            self.env.open_or_create()
        except (ve.VirtualenvCreationException,
                ve.VirtualenvReadonlyException):
            raise VirtualenvFailException('Failed to create virtualenv')
        self.dirs_before_install = DirsContent()
        self.dirs_after_install = DirsContent()
        self.dirs_before_install.fill(temp_dir + '/venv/')
        self.data = {}

    def install_package_to_venv(self):
        '''
        Installs package given as first argument to virtualenv without
        dependencies
        '''
        try:
            self.env.install(self.name, force=True, options=["--no-deps"])
        except (ve.PackageInstallationException,
                ve.VirtualenvReadonlyException):
            raise VirtualenvFailException(
                'Failed to install package to virtualenv')
        self.dirs_after_install.fill(self.temp_dir + '/venv/')

    def get_dirs_differance(self):
        '''
        Makes final versions of site_packages and scripts using DirsContent
        sub method and filters
        '''
        try:
            diff = self.dirs_after_install - self.dirs_before_install
        except ValueError:
            raise VirtualenvFailException(
                "Some of the DirsContent attributes is uninicialized")
        self.data['has_pth'] = \
            any([x for x in diff.lib_sitepackages if x.endswith('.pth')])

        site_packages = site_packages_filter(diff.lib_sitepackages)
        self.data['packages'] = sorted(
            [p for p in site_packages if not p.endswith(MODULE_SUFFIXES)])
        self.data['py_modules'] = sorted(
            set([
                os.path.splitext(m)[0]
                for m in site_packages - set(self.data['packages'])
            ]))
        self.data['scripts'] = scripts_filter(sorted(diff.bindir))
        logger.debug('Data from files differance in virtualenv:')
        logger.debug(pprint.pformat(self.data))

    @property
    def get_venv_data(self):
        self.install_package_to_venv()
        self.get_dirs_differance()
        return self.data
Exemple #26
0
class Node:
    def __init__(self,
                 master_addresses,
                 process_count=None,
                 package_cache_dirs=None,
                 amazon_s3_bucket='ipeterov'):

        # This will throw an exception if run not from virtualenv
        self.env = VirtualEnvironment()

        self.s3_client = boto3.client('s3')
        self.amazon_s3_bucket = amazon_s3_bucket

        self.thread = threading.Thread(target=self.main)
        self.is_alive = True

        self.current_master = None
        self.master_addresses = master_addresses

        if package_cache_dirs == None:
            package_cache_dirs = [
                '/var/tmp/multiserver/', '~/.multiserver/tmp/'
            ]

        for directory in package_cache_dirs:
            if not os.path.exists(directory):
                try:
                    os.makedirs(directory)
                except (PermissionError, OSerror) as e:
                    continue
            self.package_cache_dir = directory
            break

        # Multiprocessing
        if process_count == None:
            self.process_count = multiprocessing.cpu_count()
        else:
            self.process_count = process_count

        self.manager = multiprocessing.Manager()
        self.tasks = multiprocessing.Queue()
        self.answers = multiprocessing.Queue()
        self.package_lock = multiprocessing.Lock()
        self.master_addresses_lock = multiprocessing.Lock()
        self.processes = [
            _NodeProcess(self) for i in range(self.process_count)
        ]

        # Amazon
        try:
            self.instance_id = open_url(
                'http://instance-data/latest/meta-data/instance-id')
            self.instance_type = open_url(
                'http://instance-data/latest/meta-data/instance-type')
        except URLError:
            self.is_amazon = False
        else:
            self.is_amazon = True
            with open('/proc/uptime', 'r') as f:
                self.approx_term_time = time_since_epoch() - float(f.readline(
                ).split()[0]) + 3600  # 3600 так как инстансы работают час

    def main(self):
        while self.is_alive:

            socket = JSONSocket()

            with self.master_addresses_lock:
                for ip, port in self.master_addresses:
                    if socket.connect(ip, port):
                        self.current_master = ip, port
                        break
                else:
                    continue

            startup_msg = {
                'header': 'start',
                'process_count': len(self.processes),
                'is_amazon': self.is_amazon
            }
            if self.is_amazon:
                startup_msg['instance_id'] = self.instance_id
                startup_msg['instance_type'] = self.instance_type
                startup_msg['approx_term_time'] = self.approx_term_time

            try:
                socket.send(startup_msg)
            except ConnectionLostError:
                self.current_master = None
                socket.close()
                continue

            while self.is_alive:
                try:
                    if self.tasks.empty():
                        socket.send({'header': 'task_request'})
                        answer = socket.receive()
                        if answer['header'] == 'task':
                            task = answer['task']
                            self.tasks.put(task)
                        elif answer['header'] == 'no_task':
                            pass

                    if not self.answers.empty():
                        socket.send(self.answers.get())

                except ConnectionLostError:
                    self.current_master = None
                    socket.close()
                    break

    def start(self):
        self.thread.start()
        for process in self.processes:
            process.start()

    def stop(self):
        for process in self.processes:
            process.terminate()
        self.is_alive = False
        self.thread.join()

    def give_task(self):
        return self.tasks.get()

    def grab_answer(self, result):
        self.answers.put(result)

    def add_master_address(self, address):
        with self.master_addresses_lock:
            self.master_addresses.append(address)

    def remove_master_address(self, address):
        with self.master_addresses_lock:
            self.master_addresses.remove(address)

    def ensure_package(self, package_filename, package_name, package_version):
        with self.package_lock:
            if (package_name,
                    package_version) not in self.env.installed_packages:

                full_package_path = os.path.join(self.package_cache_dir,
                                                 package_filename)

                self.s3_client.download_file(self.amazon_s3_bucket,
                                             package_filename,
                                             full_package_path)

                self.env.install(full_package_path, upgrade=True)
def example(path='/tmp/virtualenv.test'):
    env = VirtualEnvironment(path)

    print 'django 1.5 installed?', env.is_installed('django==1.5')

    print 'mezzanine installed?', env.is_installed('mezzanine')
    env.install('mezzanine')
    print 'mezzanine installed?', env.is_installed('mezzanine')

    print env.installed_packages

    payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
    print 'cartridge-payments installed?', env.is_installed(payments_repo)
    env.install(payments_repo)
    print 'cartridge-payments installed?', env.is_installed(payments_repo)
    print env.installed_packages

    env.uninstall('mezzanine')
    print 'mezzanine installed?', env.is_installed('mezzanine')
    print env.installed_packages

    pkgs = env.search('requests')
    print 'search for \'requests\':'
    print len(pkgs), 'found:'
    print pkgs
Exemple #28
0
 def setUp(self):
     self.env_path = tempfile.mkdtemp()
     self.virtual_env_obj = VirtualEnvironment(self.env_path)
Exemple #29
0
 def assert_installed_packages(*packages):
     from virtualenvapi.manage import VirtualEnvironment
     venv = VirtualEnvironment(str(workon_home.join(venv_name)))
     for i_package in packages:
         assert venv.is_installed(i_package)
Exemple #30
0
	Date: 14 February 2018
	Purpose: Creates a requirements.txt file from the directory of a python project.
"""
import sys
import os
from virtualenvapi.manage import VirtualEnvironment

venvFiles=['include','bin','local','lib','pip-selfcheck.json']

def envPath(directory_path):
	"""
		Parameters:
			directory_path : Directory of the python project. 
		Returns the absolute path of the virtual environment folder.
	"""
	for dir in os.listdir(directory_path):
		#Checks for every folder if it contains the virtual env files.
		if os.path.isdir(os.path.join(directory_path,dir)) and set(venvFiles).issubset(set(os.listdir(os.path.join(directory_path,dir)))):
			return os.path.join(directory_path,dir)

if __name__ == '__main__':
	try:
		#Checks if the virtual environment is present in the specified path and creates one if not.
		env = VirtualEnvironment(envPath(sys.argv[1])) 
		#Lists the python packages installed in the virtual env.
		pkgs=env.installed_packages
		with open("requirements.txt","w") as f:
			for pkg,pkg_version in pkgs:
				f.write(pkg+"==="+pkg_version+"\n")
	except:
		print "No virtual env folder"
Exemple #31
0
class VirtualEnv(object):

    modul_pattern = re.compile(r'\.py.?$')

    def __init__(self, name, temp_dir, name_convertor, base_python_version):
        self.name = name
        self.temp_dir = temp_dir
        self.name_convertor = name_convertor
        if not base_python_version:
            base_python_version = DEFAULT_PYTHON_VERSION
        python_version = 'python' + base_python_version
        self.env = VirtualEnvironment(temp_dir + '/venv',
                                      python=python_version)
        try:
            self.env.open_or_create()
        except (ve.VirtualenvCreationException,
                ve.VirtualenvReadonlyException):
            raise VirtualenvFailException('Failed to create virtualenv')
        self.dirs_before_install = DirsContent()
        self.dirs_after_install = DirsContent()
        self.dirs_before_install.fill(temp_dir + '/venv/')
        self.data = {}

    def install_package_to_venv(self):
        '''
        Installs package given as first argument to virtualenv without
        dependencies
        '''
        try:
            self.env.install(self.name, options=["--no-deps"])
        except (ve.PackageInstallationException,
                ve.VirtualenvReadonlyException):
            raise VirtualenvFailException(
                'Failed to install package to virtualenv')
        self.dirs_after_install.fill(self.temp_dir + '/venv/')

    @property
    def get_dirs_differance(self):
        '''
        Makes final versions of site_packages and scripts using DirsContent
        sub method and filters
        '''
        try:
            diff = self.dirs_after_install - self.dirs_before_install
        except ValueError:
            raise VirtualenvFailException(
                "Some of the DirsContent attributes is uninicialized")
        site_packages = site_packages_filter(diff.lib_sitepackages)
        packages = set(
            [p for p in site_packages if not self.modul_pattern.search(p)])
        py_modules = set(
            [os.path.splitext(m)[0] for m in site_packages - packages])
        scripts = scripts_filter(list(diff.bindir))
        logger.debug(
            'Packages from files differance in virtualenv: {0}.'.format(
                packages))
        logger.debug(
            'py_modules from files differance in virtualenv: {0}.'.format(
                py_modules))
        logger.debug(
            'Scripts from files differance in virtualenv: {0}.'.format(
                scripts))
        return (packages, py_modules, scripts)

    @property
    def get_venv_data(self):
        self.install_package_to_venv()
        (self.data['packages'], self.data['py_modules'],
         self.data['scripts']) = self.get_dirs_differance
        return self.data
Exemple #32
0
                continue

            if r['sucsess'] == True:
                yield r
            else:
                time.sleep(check_interval)
                continue

    def mainloop(self):
        for result in self.pool.imap_unordered(process_task,
                                               self.task_generator()):
            self.send_result(**result)


class InvalidCredentialsException(Exception):
    pass


if __name__ == '__main__':
    # This will throw an exception if run not from virtualenv
    env = VirtualEnvironment()

    parser = argparse.ArgumentParser(description='A convenient-rpc node.')
    parser.add_argument('server_address')
    parser.add_argument('login')
    parser.add_argument('password')
    args = parser.parse_args()

    processor = TaskProcessor(args.server_address, args.login, args.password)
    processor.mainloop()
Exemple #33
0
class BaseTest(TestCase):

    env_path = None

    def setUp(self):
        self.env_path = self.setup_env()
        self.virtual_env_obj = VirtualEnvironment(self.env_path)

    def setup_env(self):
        env_path = self.env_path
        if env_path is None:
            env_path = tempfile.mkdtemp("test_env")
            virt_env = VirtualEnvironment(env_path)
            virt_env._create()
            for pack in packages_for_pre_install:
                virt_env.install(pack)

        return env_path

    def _install_packages(self, packages):
        for pack in packages:
            self.virtual_env_obj.install(pack)

    def _uninstall_packages(self, packages):
        for pack in packages:
            self.virtual_env_obj.uninstall(pack)

    def test_installed(self):
        for pack in packages_for_pre_install:
            self.assertTrue(self.virtual_env_obj.is_installed(pack))
        self.assertFalse(self.virtual_env_obj.is_installed("".join(random.sample(string.ascii_letters, 30))))

    def test_install(self):
        self._uninstall_packages(packages_for_tests)
        for pack in packages_for_tests:
            self.virtual_env_obj.install(pack)
            self.assertTrue(self.virtual_env_obj.is_installed(pack))

    def test_uninstall(self):
        self._install_packages(packages_for_tests)
        for pack in packages_for_tests:
            if pack.endswith(".git"):
                pack = pack.split("/")[-1].replace(".git", "")
            self.virtual_env_obj.uninstall(pack)
            self.assertFalse(self.virtual_env_obj.is_installed(pack))

    def test_wheel(self):
        for pack in packages_for_tests:
            self.virtual_env_obj.wheel(pack, options=["--wheel-dir=/tmp/wheelhouse"])
            self.virtual_env_obj.install(pack, options=["--no-index", "--find-links=/tmp/wheelhouse", "--use-wheel"])
            self.assertTrue(self.virtual_env_obj.is_installed(pack))

    def test_search(self):
        pack = packages_for_tests[0].lower()
        result = self.virtual_env_obj.search(pack)
        self.assertIsInstance(result, dict)
        self.assertTrue(bool(result))
        if result:
            self.assertIn(pack, [k.split(" (")[0].lower() for k in result.keys()])

    def test_search_names(self):
        pack = packages_for_tests[0].lower()
        result = self.virtual_env_obj.search_names(pack)
        self.assertIsInstance(result, list)
        self.assertIn(pack, [k.split(" (")[0].lower() for k in result])

    def tearDown(self):
        if os.path.exists(self.env_path) and self.__class__.env_path is None:
            shutil.rmtree(self.env_path)
Exemple #34
0
def example(path='/tmp/virtualenv.test'):
    env = VirtualEnvironment(path)

    print 'django 1.5 installed?', env.is_installed('django==1.5')

    print 'mezzanine installed?', env.is_installed('mezzanine')
    env.install('mezzanine')
    print 'mezzanine installed?', env.is_installed('mezzanine')

    print env.installed_packages

    payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
    print 'cartridge-payments installed?', env.is_installed(payments_repo)
    env.install(payments_repo)
    print 'cartridge-payments installed?', env.is_installed(payments_repo)
    print env.installed_packages

    env.uninstall('mezzanine')
    print 'mezzanine installed?', env.is_installed('mezzanine')
    print env.installed_packages

    pkgs = env.search('requests')
    print 'search for \'requests\':'
    print len(pkgs), 'found:'
    print pkgs
Exemple #35
0
    def create(self, **kwargs):
        """
        build the virtualenv
        """
        clean = kwargs.get('clean', False)
        if clean:
            self.clean(**kwargs)

        site_packages = kwargs.get('system-site-packages',
                                   self.use_sitepackages)
        upgrade = kwargs.get('upgrade', False)
        extras_require = kwargs.get('extras_require', [])
        all_extras = kwargs.get('all_extras', False)
        if all_extras:
            extras_require = self.config.extras_require().keys()
        nosetupdevelop = kwargs.get('nosetupdevelop', False)
        venv = VirtualEnvironment(self.venv_path,
                                  python=self.python_bin_for_venv,
                                  system_site_packages=site_packages)
        LOGGER.info("Bootstrapping virtualenv: {}".format(self.venv_path))

        venv.open_or_create()
        cmd = build_pip_command(self.config,
                                self.venv_path,
                                self.reqs_name,
                                upgrade=upgrade)

        try:
            local(cmd)
        except OSError as ex:
            msg = ("Error running pip install command during build\n"
                   "Error was {0}\n"
                   "Running command: {1}\n"
                   "Working Dir: {2}\n"
                   "Virtualenv: {3}\n"
                   "Requirements: {4}\n").format(ex, cmd, self.working_dir,
                                                 self.venv_path,
                                                 self.reqs_name)
            LOGGER.error(msg)
            raise

        for req in self.extra_reqs:
            cmd = build_pip_command(self.config,
                                    self.venv_path,
                                    req,
                                    upgrade=upgrade)
            LOGGER.info("Installing extra requirements... {}".format(cmd))
            try:
                local(cmd)
            except OSError as ex:
                msg = ("Error running pip install command extra "
                       "requirements install: {}\n{}").format(req, ex)
                LOGGER.error(msg)
                raise
        # setup for development
        if nosetupdevelop:
            msg = "skipping python setup.py develop..."
            LOGGER.info(msg)
        else:
            self.run_setup_develop()
            if extras_require:
                for extra in extras_require:
                    self._install_extras(extra)
Exemple #36
0
class BaseTest(TestCase):

    env_path = None

    def setUp(self):
        self.env_path = self.setup_env()
        self.virtual_env_obj = VirtualEnvironment(self.env_path)

    def setup_env(self):
        env_path = self.env_path
        if env_path is None:
            env_path = tempfile.mkdtemp('test_env')
            virt_env = VirtualEnvironment(env_path)
            virt_env._create()
            for pack in packages_for_pre_install:
                virt_env.install(pack)

        return env_path

    def _install_packages(self, packages):
        for pack in packages:
            self.virtual_env_obj.install(pack)

    def _uninstall_packages(self, packages):
        for pack in packages:
            self.virtual_env_obj.uninstall(pack)

    def test_installed(self):
        for pack in packages_for_pre_install:
            self.assertTrue(self.virtual_env_obj.is_installed(pack))
        self.assertFalse(
            self.virtual_env_obj.is_installed(''.join(
                random.sample(string.ascii_letters, 30))))

    def test_install(self):
        self._uninstall_packages(packages_for_tests)
        for pack in packages_for_tests:
            self.virtual_env_obj.install(pack)
            self.assertTrue(self.virtual_env_obj.is_installed(pack))

    def test_uninstall(self):
        self._install_packages(packages_for_tests)
        for pack in packages_for_tests:
            if pack.endswith('.git'):
                pack = pack.split('/')[-1].replace('.git', '')
            self.virtual_env_obj.uninstall(pack)
            self.assertFalse(self.virtual_env_obj.is_installed(pack))

    def test_wheel(self):
        for pack in packages_for_tests:
            self.virtual_env_obj.wheel(pack,
                                       options=['--wheel-dir=/tmp/wheelhouse'])
            self.virtual_env_obj.install(pack,
                                         options=[
                                             '--no-index',
                                             '--find-links=/tmp/wheelhouse',
                                             '--use-wheel'
                                         ])
            self.assertTrue(self.virtual_env_obj.is_installed(pack))

    def test_search(self):
        pack = packages_for_tests[0].lower()
        result = self.virtual_env_obj.search(pack)
        self.assertIsInstance(result, dict)
        self.assertTrue(bool(result))
        if result:
            self.assertIn(pack,
                          [k.split(' (')[0].lower() for k in result.keys()])

    def test_search_names(self):
        pack = packages_for_tests[0].lower()
        result = self.virtual_env_obj.search_names(pack)
        self.assertIsInstance(result, list)
        self.assertIn(pack, [k.split(' (')[0].lower() for k in result])

    def tearDown(self):
        if os.path.exists(self.env_path) and self.__class__.env_path is None:
            shutil.rmtree(self.env_path)