Esempio n. 1
0
def test_pypy_pre_import(tmp_path):
    """For PyPy, some built-in modules should be pre-imported because
    some programs expect them to be in sys.modules on startup.
    """
    check_code = inspect.getsource(check_pypy_pre_import)
    check_code = textwrap.dedent(check_code[check_code.index("\n") + 1 :])
    if six.PY2:
        check_code = check_code.decode()

    check_prog = tmp_path / "check-pre-import.py"
    check_prog.write_text(check_code)

    ve_path = str(tmp_path / "venv")
    virtualenv.create_environment(ve_path)

    bin_dir = virtualenv.path_locations(ve_path)[-1]

    try:
        cmd = [
            os.path.join(bin_dir, "{}{}".format(virtualenv.EXPECTED_EXE, ".exe" if virtualenv.IS_WIN else "")),
            str(check_prog),
        ]
        subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as exception:
        assert not exception.returncode, exception.output
Esempio n. 2
0
def test_create_environment_from_virtual_environment(tmpdir):
    """Create virtual environment using Python from another virtual environment
    """
    venvdir = str(tmpdir / "venv")
    home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(venvdir)
    virtualenv.create_environment(venvdir)
    assert not os.path.islink(os.path.join(lib_dir, "distutils"))
Esempio n. 3
0
def configure_virtualenv(config, python_worktree,  build_worktree=None,
                         remote_packages=None, site_packages=True):
    if not remote_packages:
        remote_packages = list()

    # create a new virtualenv
    python_worktree.config = config
    venv_path = python_worktree.venv_path
    pip = python_worktree.pip

    try:
        virtualenv.create_environment(python_worktree.venv_path,
                                      site_packages=site_packages)
    except:
        ui.error("Failed to create virtualenv")
        return

    # Install all Python projects using pip install -e .
    python_projects = python_worktree.python_projects
    for i, project in enumerate(python_projects):
        ui.info_count(i, len(python_projects),
                     ui.green, "Configuring", ui.reset, ui.blue, project.src)
        cmd = [pip, "install", "--editable", "."]
        qisys.command.call(cmd, cwd=project.path)

    # Write a qi.pth file containing path to C/C++ extensions
    if build_worktree:
        handle_extensions(venv_path, python_worktree, build_worktree)

    # Install the extension in the virtualenv
    binaries_path = virtualenv.path_locations(venv_path)[-1]
    pip_binary = os.path.join(binaries_path, "pip")
    if remote_packages:
        cmd = [pip_binary, "install"] + remote_packages
        subprocess.check_call(cmd)
Esempio n. 4
0
def handle_extensions(venv_path, python_worktree, build_worktree):
    """ Check if there is a build project matching the given source, and
    add the correct path to the virtualenv

    """
    extensions_projects = list()
    build_projects = build_worktree.build_projects
    for project in python_worktree.python_projects:
        parent_project = qisys.parsers.find_parent_project(build_projects,
                                                           project.path)
        if parent_project:
            extensions_projects.append(parent_project)

    to_write = ""
    for project in extensions_projects:
        qi_pth_src = os.path.join(project.sdk_directory, "qi.pth")
        if os.path.exists(qi_pth_src):
            with open(qi_pth_src, "r") as fp:
                to_write += fp.read()
                if not to_write.endswith("\n"):
                    to_write += "\n"

    lib_path = virtualenv.path_locations(venv_path)[1]
    qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
    with open(qi_pth_dest, "a") as fp:
        fp.write(to_write)
Esempio n. 5
0
def create_env():
    if env_exists():
        raise Exception('Virtual environment already exists.')

    log('Creating virtual environment...')

    home_dir, lib_dir, inc_dir, bin_dir = venv.path_locations(ENV_PATH)
    python_loc = venv.install_python(
        home_dir,
        lib_dir,
        inc_dir,
        bin_dir,
        site_packages=False,
        clear=False,
        symlink=True
    )

    python_abs_loc = os.path.abspath(python_loc)

    venv.install_activate(home_dir, bin_dir)
    venv.install_wheel(['setuptools', 'pip'], python_abs_loc, None)
    venv.install_distutils(home_dir)

    log('Installing requirements...')
    req_cmd = '{0}/bin/pip install {1}'.format(
        ENV_PATH,
        get_config().requirements
    )
    execute_under_env(req_cmd)

    log('Virtual environment created!')
Esempio n. 6
0
def create_virtualenv(where=None, distribute=False, **kw):
    """
    Create a virtual Python environment for testing purposes.  If
    distribute is True, installs the distribute package in place of
    setuptools.

    Returns the directory of the environment itself, its Python
    library directory (which contains site-packages), its 'include'
    and binary file directory (bin, or on windows, Scripts)
    respectively.  Additional keyword arguments are passed to mkdtemp
    to create the virtual environment directory.
    """
    save_argv = sys.argv

    if not where:
        where = mkdtemp(**kw)

    try:
        import virtualenv
        distribute_opt = ['--distribute'] if distribute else []
        sys.argv = ['virtualenv', '--quiet'] + distribute_opt + ['--no-site-packages', '--unzip-setuptools', where]
        virtualenv.main()
    finally: 
        sys.argv = save_argv

    return virtualenv.path_locations(where)
Esempio n. 7
0
def version_exe(venv, exe_name):
    _, _, _, bin_dir = virtualenv.path_locations(str(venv))
    exe = os.path.join(bin_dir, exe_name)
    script = "import sys; import json; print(json.dumps(dict(v=list(sys.version_info), e=sys.executable)))"
    cmd = [exe, "-c", script]
    out = json.loads(subprocess.check_output(cmd, universal_newlines=True))
    return out["v"], out["e"]
Esempio n. 8
0
 def env_prefix(self):
     return ". '{{prefix}}{0}activate' &&".format(
         virtualenv.path_locations(
             helpers.environment_dir(ENVIRONMENT_DIR, self.language_version)
         )[-1].rstrip(os.sep) + os.sep,
         'activate',
     )
Esempio n. 9
0
    def __init__(self, base_path, *args, **kwargs):
        # Make our base_path a test.lib.path.Path object
        base_path = Path(base_path)

        # Store paths related to the virtual environment
        _virtualenv = kwargs.pop("virtualenv")
        venv, lib, include, bin = virtualenv.path_locations(_virtualenv)
        self.venv_path = venv
        self.lib_path = lib
        self.include_path = include
        self.bin_path = bin

        if hasattr(sys, "pypy_version_info"):
            self.site_packages_path = self.venv_path.join("site-packages")
        else:
            self.site_packages_path = self.lib_path.join("site-packages")

        self.user_base_path = self.venv_path.join("user")
        self.user_bin_path = self.user_base_path.join(self.bin_path - self.venv_path)
        self.user_site_path = self.venv_path.join(
            "user",
            site.USER_SITE[len(site.USER_BASE) + 1:],
        )

        # Create a Directory to use as a scratch pad
        self.scratch_path = base_path.join("scratch").mkdir()

        # Set our default working directory
        kwargs.setdefault("cwd", self.scratch_path)

        # Setup our environment
        environ = kwargs.get("environ")
        if environ is None:
            environ = os.environ.copy()

        environ["PIP_LOG_FILE"] = base_path.join("pip-log.txt")
        environ["PATH"] = Path.pathsep.join(
            [self.bin_path] + [environ.get("PATH", [])],
        )
        environ["PYTHONUSERBASE"] = self.user_base_path
        # Writing bytecode can mess up updated file detection
        environ["PYTHONDONTWRITEBYTECODE"] = "1"
        kwargs["environ"] = environ

        # Call the TestFileEnvironment __init__
        super(PipTestEnvironment, self).__init__(base_path, *args, **kwargs)

        # Expand our absolute path directories into relative
        for name in ["base", "venv", "lib", "include", "bin", "site_packages",
                     "user_base", "user_site", "user_bin", "scratch"]:
            real_name = "%s_path" % name
            setattr(self, name, getattr(self, real_name) - self.base_path)

        # Ensure the tmp dir exists, things break horribly if it doesn't
        self.temp_path.mkdir()

        # create easy-install.pth in user_site, so we always have it updated
        #   instead of created
        self.user_site_path.makedirs()
        self.user_site_path.join("easy-install.pth").touch()
Esempio n. 10
0
 def env_prefix(self):
     return ". '{{prefix}}{0}activate' &&".format(
         virtualenv.path_locations(
             ENVIRONMENT_DIR,
         )[-1].rstrip(os.sep) + os.sep,
         'activate',
     )
Esempio n. 11
0
def handle_pure_python(venv_path, python_worktree, env=None):
    """ Add the paths of all python projects to the virtualenv """
    lib_path = virtualenv.path_locations(venv_path)[1]
    qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
    res = True
    with open(qi_pth_dest, "w") as fp:
        fp.write("")
        for i, project in enumerate(python_worktree.python_projects):
            ui.info_count(i, len(python_worktree.python_projects),
                          ui.blue, project.name)
            if project.setup_with_distutils:
                cmd = [python_worktree.pip, "install"]
                if not ui.CONFIG["verbose"]:
                    cmd.append("--quiet")
                cmd.extend(["--editable", "."])
                rc = qisys.command.call(cmd, cwd=project.path, ignore_ret_code=True,
                                        env=env)
                if rc != 0:
                    ui.warning("Failed to run pip install on", project.src)
                    res = False
            else:
                ui.debug("Adding python path for project", project.name, ":\n",
                         project.python_path)
                for path in project.python_path:
                    fp.write(path + "\n")
    return res
Esempio n. 12
0
def create_virtualenv(where, distribute=False):
    import virtualenv
    if sys.version_info[0] > 2:
        distribute = True
    virtualenv.create_environment(
        where, use_distribute=distribute, unzip_setuptools=True)

    return virtualenv.path_locations(where)
Esempio n. 13
0
def configure_virtualenv(config, python_worktree,  build_worktree=None,
                         remote_packages=None, site_packages=True):
    """ Main entry point. Called by ``qipy bootstrap``

    :param: remote_packages List of third-party packages to add in the virtualenv
    :param: site_packages Allow access to global site packages

    """
    ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root)
    if not remote_packages:
        remote_packages = list()

    # create a new virtualenv
    python_worktree.config = config
    venv_path = python_worktree.venv_path
    pip = python_worktree.pip

    try:
        virtualenv.create_environment(python_worktree.venv_path,
                                      site_packages=site_packages)
    except:
        ui.error("Failed to create virtualenv")
        return False


    ui.info("Adding python projects")
    # Write a qi.pth file containing path to C/C++ extensions and
    # path to pure python modules or packages
    pure_python_ok = handle_pure_python(venv_path, python_worktree)
    if build_worktree:
        handle_extensions(venv_path, python_worktree, build_worktree)

    ui.info("Adding other requirements: " + ", ".join(remote_packages))
    binaries_path = virtualenv.path_locations(venv_path)[-1]
    pip_binary = os.path.join(binaries_path, "pip")
    remote_ok = True
    if remote_packages:
        cmd = [pip_binary, "install"] + remote_packages
        rc = qisys.command.call(cmd, ignore_ret_code=True)
        remote_ok = (rc == 0)
    if pure_python_ok and remote_ok:
        ui.info(ui.green, "Done")
    if not pure_python_ok:
        ui.info(ui.red, "Failed to add some python projects")
    if not remote_ok:
        ui.info(ui.red, "Failed to add some third party requirements")

    requirements_ok = True
    for project in python_worktree.python_projects:
        path = os.path.join(project.path, "requirements.txt")
        if os.path.isfile( path ):
            ui.info(ui.green, " * Installing dependencies from " + path)
            cmd = [pip_binary, "install", "--requirement", path]
            rc = qisys.command.call(cmd, ignore_ret_code=True)
            requirements_ok = (rc == 0)
        else:
            ui.debug(ui.yellow, " * missing " + path)
    return (pure_python_ok and remote_ok and requirements_ok)
Esempio n. 14
0
def do(args):
    build_worktree = qibuild.parsers.get_build_worktree(args)
    env = build_worktree.get_env()
    build_config = qibuild.parsers.get_build_config(build_worktree, args)
    worktree = build_worktree.worktree
    cmd = args.command

    venvs_path = os.path.join(worktree.dot_qi,
                             "venvs")
    name = build_config.build_directory("py")
    venv_root = os.path.join(venvs_path, name)
    if not os.path.exists(venv_root):
        err = "No Virtualenv found in %s\n" % (venv_root)
        err += "Please run `qipy bootstrap`"
        raise Exception(err)

    binaries_path = virtualenv.path_locations(venv_root)[-1]

    if args.dolist:
        for f in sorted(os.listdir(binaries_path)):
            if os.path.isfile(os.path.join(binaries_path, f)):
                print f
        return

    if not cmd:
        cmd = ["ipython"]

    if os.path.exists(cmd[0]):
        # Assume it is a script we want to run
        python_path = os.path.join(binaries_path, "python")
        cmd.insert(0, python_path)
    else:
        script_path = qipy.venv.find_script(venv_root, cmd[0])
        if script_path:
            cmd[0] = script_path

    lib_path = virtualenv.path_locations(venv_root)[1]
    env["PYTHONHOME"] = venv_root
    env["PYTHONPATH"] = os.path.join(lib_path, "site-packages")

    if args.exec_:
        ui.debug("exec", cmd)
        os.execve(cmd[0], cmd, env)
    else:
        qisys.command.call(cmd, env=env)
Esempio n. 15
0
    def __init__(self, source, venv=None):
        self.dir_source = source
        if not venv:
            base, source_name = os.path.split(source)
            venv = os.path.join(base, 'venv-{}-{}'.format(source_name, os.path.basename(sys.executable)))

        venv_home, venv_lib, venv_inc, venv_bin = virtualenv.path_locations(venv)
        self.dir_venv = venv_home
        self.dir_venv_bin = venv_bin
Esempio n. 16
0
def get_paths(name):
    _, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(name)
    venv_home = get_venv_home()
    home_dir = get_env_home(name)
    lib_dir = os.path.join(venv_home, lib_dir)
    pkg_dir = os.path.join(lib_dir, 'site-packages')
    inc_dir = os.path.join(venv_home, inc_dir)
    bin_dir = os.path.join(venv_home, bin_dir)        
    return home_dir, lib_dir, pkg_dir, inc_dir, bin_dir
Esempio n. 17
0
def add_virtualenv_path(venv=os.getenv('VIRTUAL_ENV')):
    """Add virtualenv's site-packages to `sys.path`."""
    if not venv:
        return
    import virtualenv
    venv = os.path.abspath(venv)
    home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(venv)
    path = os.path.join(lib_dir, 'site-packages')
    sys.path.insert(0, path)
    site.addsitedir(path)
Esempio n. 18
0
File: venv.py Progetto: fpytloun/pip
    def __init__(self, location, *args, **kwargs):
        self.location = Path(location)
        self.pip_source_dir = kwargs.pop("pip_source_dir")
        self._system_site_packages = kwargs.pop("system_site_packages", False)

        home, lib, inc, bin = _virtualenv.path_locations(self.location)
        self.lib = Path(lib)
        self.bin = Path(bin)

        super(VirtualEnvironment, self).__init__(*args, **kwargs)
Esempio n. 19
0
def create_virtualenv(where):
    save_argv = sys.argv
    
    try:
        import virtualenv
        sys.argv = ['virtualenv', '--quiet', '--no-site-packages', where]
        virtualenv.main()
    finally: 
        sys.argv = save_argv

    return virtualenv.path_locations(where)
Esempio n. 20
0
File: venv.py Progetto: akaihola/pip
 def _update_paths(self):
     home, lib, inc, bin = _virtualenv.path_locations(self.location)
     self.bin = Path(bin)
     self.site = Path(lib) / 'site-packages'
     # Workaround for https://github.com/pypa/virtualenv/issues/306
     if hasattr(sys, "pypy_version_info"):
         version_fmt = '{0}' if six.PY3 else '{0}.{1}'
         version_dir = version_fmt.format(*sys.version_info)
         self.lib = Path(home, 'lib-python', version_dir)
     else:
         self.lib = Path(lib)
Esempio n. 21
0
def custom_prompt_root(tmp_path_factory):
    """Provide Path to root with default and custom venvs created."""
    root = tmp_path_factory.mktemp("custom_prompt")
    virtualenv.create_environment(
        str(root / ENV_CUSTOM), prompt=PREFIX_CUSTOM, no_setuptools=True, no_pip=True, no_wheel=True
    )

    _, _, _, bin_dir = virtualenv.path_locations(str(root / ENV_DEFAULT))

    bin_dir_name = os.path.split(bin_dir)[-1]

    return root, bin_dir_name
Esempio n. 22
0
def create_virtualenv(where, distribute=False):
    save_argv = sys.argv

    try:
        import virtualenv
        distribute_opt = distribute and ['--distribute'] or []
        sys.argv = ['virtualenv', '--quiet'] + distribute_opt + ['--no-site-packages', '--unzip-setuptools', where]
        virtualenv.main()
    finally:
        sys.argv = save_argv

    return virtualenv.path_locations(where)
Esempio n. 23
0
    def __init__(self, dir):
        """Construct

        :arg dir: The path of the new virtualenv

        """
        create_environment(dir,
                           site_packages=False,
                           clear=False,
                           never_download=True)
        home_dir, lib_dir, inc_dir, bin_dir = path_locations(dir)
        self.python_path = join(bin_dir, basename(sys.executable))
def _get_env_bin_path(env_path):
    """returns the bin path for a virtualenv
    """
    try:
        import virtualenv
        return virtualenv.path_locations(env_path)[3]
    except ImportError:
        # this is a fallback for a race condition in which you're trying
        # to use the script and create a virtualenv from within
        # a virtualenv in which virtualenv isn't installed and so
        # is not importable.
        return os.path.join(env_path, 'scripts' if IS_WIN else 'bin')
Esempio n. 25
0
    def __init__(self, location, *args, **kwargs):
        self.location = Path(location)
        self.pip_source_dir = kwargs.pop("pip_source_dir")
        self._system_site_packages = kwargs.pop("system_site_packages", False)

        home, lib, inc, bin = _virtualenv.path_locations(self.location)
        # workaround for https://github.com/pypa/virtualenv/issues/306
        if hasattr(sys, "pypy_version_info"):
            lib = os.path.join(home, 'lib-python', sys.version[:3])
        self.lib = Path(lib)
        self.bin = Path(bin)

        super(VirtualEnvironment, self).__init__(*args, **kwargs)
Esempio n. 26
0
def test_commandline_basic(tmpdir):
    """Simple command line usage should work and files should be generated"""
    home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(str(tmpdir.join("venv")))
    subprocess.check_call([sys.executable, VIRTUALENV_SCRIPT, home_dir])

    assert os.path.exists(home_dir)
    assert os.path.exists(bin_dir)

    assert os.path.exists(os.path.join(bin_dir, "activate"))
    assert os.path.exists(os.path.join(bin_dir, "activate_this.py"))
    assert os.path.exists(os.path.join(bin_dir, "activate.ps1"))

    assert os.path.exists(os.path.join(bin_dir, os.path.basename(sys.executable)))
Esempio n. 27
0
def get_pip_path():
    """
    Try to figure out an explicit path to the Pip executable script.

    :return: Pip path
    :rtype: str
    """
    try:
        from virtualenv import path_locations
        (home_dir, lib_dir, inc_dir, bin_dir) = path_locations(sys.prefix)
        return os.path.join(bin_dir, "pip")
    except ImportError:
        pass
    return "pip"
Esempio n. 28
0
    def build(self):
        print('Building Python Virtual Environment: {0}...'.format(self.uuid))
        home_dir, lib_dir, inc_dir, bin_dir = venv.path_locations(
            self._env_path)

        self._python = os.path.abspath(venv.install_python(
            home_dir, lib_dir, inc_dir, bin_dir, site_packages=False,
            clear=False, symlink=True))

        venv.install_wheel(['setuptools', 'pip'], self._python, None)
        venv.install_distutils(home_dir)

        print('Build complete!')
        return self._python
def test_activate_with_powershell(tmpdir, monkeypatch):
    monkeypatch.chdir(tmpdir)
    home_dir, _, __, bin_dir = virtualenv.path_locations(str(tmpdir.join("env")))
    virtualenv.create_environment(home_dir, no_pip=True, no_setuptools=True, no_wheel=True)
    monkeypatch.chdir(home_dir)
    activate_script = join(bin_dir, "activate.ps1")
    cmd = [POWER_SHELL, "-Command", "{0}; {1}; {0}; deactivate; {0}".format(print_python_exe_path(), activate_script)]
    output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT)
    content = output.split()
    assert len(content) == 3, output
    before_activate, after_activate, after_deactivate = content
    exe = "{}.exe".format(virtualenv.expected_exe) if virtualenv.is_win else virtualenv.expected_exe
    assert normcase(long_path(after_activate)) == normcase(long_path(join(bin_dir, exe)))
    assert before_activate == after_deactivate
Esempio n. 30
0
def _get_env_bin_path(env_path):
    """Returns the bin path for a virtualenv

    This provides a fallback for a race condition in which you're trying
    to use the script and create a virtualenv from within
    a virtualenv in which virtualenv isn't installed and so
    is not importable.
    """
    try:
        import virtualenv

        return virtualenv.path_locations(env_path)[3]
    except ImportError:
        return os.path.join(env_path, "scripts" if IS_WIN else "bin")
Esempio n. 31
0
def ext_pillar(minion_id,
               pillar,
               pillar_name,
               project_path,
               settings_module,
               django_app,
               env=None,
               env_file=None,
               *args,
               **kwargs):
    '''
    Connect to a Django database through the ORM and retrieve model fields

    :type pillar_name: str
    :param pillar_name: The name of the pillar to be returned

    :type project_path: str
    :param project_path: The full path to your Django project (the directory
        manage.py is in)

    :type settings_module: str
    :param settings_module: The settings module for your project. This can be
        found in your manage.py file

    :type django_app: str
    :param django_app: A dictionary containing your apps, models, and fields

    :type env: str
    :param env: The full path to the virtualenv for your Django project

    :type env_file: str
    :param env_file: An optional bash file that sets up your environment. The
        file is run in a subprocess and the changed variables are then added
    '''

    if not os.path.isdir(project_path):
        log.error(
            'Django project dir: {0!r} not a directory!'.format(project_path))
        return {}
    if HAS_VIRTUALENV and env is not None and os.path.isdir(env):
        for path in virtualenv.path_locations(env):
            if not os.path.isdir(path):
                log.error('Virtualenv {0} not a directory!'.format(path))
                return {}
        # load the virtualenv first
        sys.path.insert(
            0,
            os.path.join(virtualenv.path_locations(env)[1], 'site-packages'))

    # load the django project
    sys.path.append(project_path)

    os.environ['DJANGO_SETTINGS_MODULE'] = settings_module

    if env_file is not None:
        import subprocess

        base_env = {}
        proc = subprocess.Popen(['bash', '-c', 'env'], stdout=subprocess.PIPE)
        for line in proc.stdout:
            (key, _, value) = line.partition('=')
            base_env[key] = value

        command = ['bash', '-c', 'source {0} && env'.format(env_file)]
        proc = subprocess.Popen(command, stdout=subprocess.PIPE)

        for line in proc.stdout:
            (key, _, value) = line.partition('=')
            # only add a key if it is different or doesn't already exist
            if key not in base_env or base_env[key] != value:
                os.environ[key] = value.rstrip('\n')
                log.debug('Adding {0} = {1} to Django environment'.format(
                    key, value.rstrip('\n')))

    try:
        from django.db.models.loading import get_model

        django_pillar = {}

        for proj_app, models in django_app.iteritems():
            _, _, app = proj_app.rpartition('.')
            django_pillar[app] = {}
            for model_name, model_meta in models.iteritems():
                model_orm = get_model(app, model_name)
                if model_orm is None:
                    raise salt.exceptions.SaltException(
                        "Django model '{0}' not found in app '{1}'.".format(
                            app, model_name))

                pillar_for_model = django_pillar[app][model_orm.__name__] = {}

                name_field = model_meta['name']
                fields = model_meta['fields']

                if 'filter' in model_meta.keys():
                    qs = (model_orm.objects.filter(
                        **model_meta['filter']).values(*fields))
                else:
                    qs = model_orm.objects.values(*fields)

                for model in qs:
                    # Check that the human-friendly name given is valid (will
                    # be able to pick up a value from the query) and unique
                    # (since we're using it as the key in a dictionary)
                    if not name_field in model:
                        raise salt.exceptions.SaltException(
                            "Name '{0}' not found in returned fields.".format(
                                name_field))

                    if model[name_field] in pillar_for_model:
                        raise salt.exceptions.SaltException(
                            "Value for '{0}' is not unique: {0}".format(
                                model[name_field]))

                    pillar_for_model[model[name_field]] = model

        return {pillar_name: django_pillar}
    except ImportError as e:
        log.error('Failed to import library: {0}'.format(e.message))
        return {}
    except Exception as e:
        log.error('Failed on Error: {0}'.format(e.message))
        log.debug('django_orm traceback', exc_info=True)
        return {}
Esempio n. 32
0
def deploy_django(proj):
    """
    Deploy a Django project
    """

    wsgi_base_path = os.environ.get('WSGI_BASE_PATH', '/var/www/wsgi')
    httpd_conf_dir = os.environ.get('HTTPD_CONF_DIR', '/etc/httpd/locations.d')
    httpd_host = os.environ.get('HTTPD_HOST', platform.node())
    httpd_media_base = os.environ.get('HTTPD_MEDIA_BASE',
                                      '/var/www/html/media')
    httpd_static_base = os.environ.get('HTTPD_STATIC_BASE',
                                       '/var/www/html/static')
    secret_key_gen = os.environ.get('SECRET_KEY_GEN',
                                    '/usr/bin/pwgen -c -n -y 78 1')

    proj_base = os.path.join(wsgi_base_path, proj)
    path = lambda p: os.path.join(proj_base, p)

    proj_defaults = {
        'name':
        proj,
        'proj_base':
        proj_base,
        'dst':
        '%(name)s-project',
        'settings':
        '%(name)s_production',
        'url':
        '/%(name)s',
        'build':
        'build/build.sh',
        'wsgi':
        'wsgi.py',
        'allowed_hosts':
        httpd_host,
        'secret_key':
        subprocess.check_output(secret_key_gen.split()).strip().replace(
            "'", "-"),
        'media_root':
        os.path.join(httpd_media_base, proj),
        'static_root':
        os.path.join(httpd_static_base, proj),
        'scm':
        '/usr/bin/git',
        'settings_append':
        DEFAULT_SETTINGS_APPEND,
        'deploy_requires':
        None,
        'deploy_commands': ['migrate']
    }

    # Protect '%' from interpolation
    proj_defaults['secret_key'] = re.sub(r'%', r'',
                                         proj_defaults['secret_key'])
    # Choose clone command
    proj_defaults['scm_clone'] = SCM_DEFAULT_CHECKOUT[os.path.split(
        proj_defaults['scm'])[-1]]

    # Load defaults
    cfg = SafeConfigParser(proj_defaults)

    # Force read
    cfg.readfp(open(proj + '.cfg', 'r'))

    #logger.debug('Final configuration:')
    #for k,v in cfg.items(CFG_SECTION):
    #    logger.debug('\t%s: %s', k, v)

    # Create directory
    os.mkdir(proj_base)

    # Virtualenv
    virtualenv.create_environment(proj_base)

    # Checkout
    subprocess.check_call([
        cfg.get(CFG_SECTION, 'scm'),
        cfg.get(CFG_SECTION, 'scm_clone'),
        cfg.get(CFG_SECTION, 'src'),
        path(cfg.get(CFG_SECTION, 'dst')),
    ])

    # Build
    activate = path('bin/activate')
    build = os.path.join(cfg.get(CFG_SECTION, 'dst'),
                         cfg.get(CFG_SECTION, 'build'))
    subprocess.check_call([build], cwd=proj_base, env={'BASH_ENV': activate})

    # Install Deploy Requiremts
    deploy_requires = cfg.get(CFG_SECTION, 'deploy_requires')
    if deploy_requires:
        logger.debug('Installing: %s', deploy_requires)
        cmd = [
            os.path.join(virtualenv.path_locations(proj_base)[-1], 'pip'),
            'install'
        ]
        cmd.extend(parse_list(deploy_requires))
        subprocess.check_call(cmd)

    # Create settings
    settings_file = path(cfg.get(CFG_SECTION, 'settings')) + '.py'
    slock = LockFile(settings_file)
    slock.acquire()

    if os.path.exists(settings_file):
        slock.release()
        raise IOError([17, 'File exists'])
    try:
        sfp = open(settings_file, 'w')
        print(DJANGO_SETTINGS_TEMPLATE % dict(cfg.items(CFG_SECTION)),
              file=sfp)
        sfp.close()
    finally:
        slock.release()

    # Create wsgi script
    wsgi_file = path(cfg.get(CFG_SECTION, 'wsgi'))
    slock = LockFile(wsgi_file)
    slock.acquire()

    if os.path.exists(wsgi_file):
        slock.release()
        raise IOError([17, 'File exists'])
    try:
        wfp = open(wsgi_file, 'w')
        print(WSGI_TEMPLATE % dict(cfg.items(CFG_SECTION)), file=wfp)
        wfp.close()
    finally:
        slock.release()

    # Create apache conf
    conf_file = os.path.join(httpd_conf_dir, cfg.get(CFG_SECTION,
                                                     'name')) + '.conf'
    slock = LockFile(conf_file)
    slock.acquire()

    if os.path.exists(conf_file):
        slock.release()
        raise IOError([17, 'File exists'])
    try:
        sfp = open(conf_file, 'w')
        conf = dict(cfg.items(CFG_SECTION))
        conf['site_libs'] = os.path.join(
            virtualenv.path_locations(proj_base)[1], 'site-packages')
        http_conf = HTTPD_CONF_TEMPLATE % conf
        print(http_conf, file=sfp)
        sfp.close()
    finally:
        slock.release()

    # Perform django commands
    deploy_commands = cfg.get(CFG_SECTION, 'deploy_commands')
    if deploy_commands:
        manage = [
            os.path.join(
                virtualenv.path_locations(proj_base)[-1],
                virtualenv.expected_exe), 'manage.py'
        ]
        os.chdir(path(cfg.get(CFG_SECTION, 'dst')))
        # Deployment django environment
        dep_env = os.environ.copy()
        dep_env['DJANGO_SETTINGS_MODULE'] = cfg.get(CFG_SECTION, 'settings')
        dep_env['PYTHONPATH'] = path('.')
        logger.debug('Environment for commands: PYTHONPATH=%s',
                     dep_env['PYTHONPATH'])
        logger.debug('  Django settings: %s',
                     dep_env['DJANGO_SETTINGS_MODULE'])
        for cmd in parse_list(deploy_commands):
            logger.debug("Executing '%s'", ' '.join(manage + [cmd]))
            subprocess.check_call(manage + cmd.split(), env=dep_env)

    # That's it. Remember to reload apache
    print('You should reload apache:\n', '\t', 'systemctl reload httpd')
    return True
Esempio n. 33
0
 def bin_path(self, name):
     """ Path to the virtualenv's binaries """
     binaries_path = virtualenv.path_locations(self.venv_path)[-1]
     return os.path.join(binaries_path, name)
Esempio n. 34
0
    def __init__(self, base_path, *args, **kwargs):
        # Make our base_path a test.lib.path.Path object
        base_path = Path(base_path)

        # Store paths related to the virtual environment
        _virtualenv = kwargs.pop("virtualenv")
        path_locations = virtualenv.path_locations(_virtualenv)
        # Make sure we have test.lib.path.Path objects
        venv, lib, include, bin = map(Path, path_locations)
        self.venv_path = venv
        self.lib_path = virtualenv_lib_path(venv, lib)
        self.include_path = include
        self.bin_path = bin

        if hasattr(sys, "pypy_version_info"):
            self.site_packages_path = self.venv_path.join("site-packages")
        else:
            self.site_packages_path = self.lib_path.join("site-packages")

        self.user_base_path = self.venv_path.join("user")
        self.user_site_path = self.venv_path.join(
            "user",
            site.USER_SITE[len(site.USER_BASE) + 1:],
        )
        if sys.platform == 'win32':
            if sys.version_info >= (3, 5):
                scripts_base = self.user_site_path.join('..').normpath
            else:
                scripts_base = self.user_base_path
            self.user_bin_path = scripts_base.join('Scripts')
        else:
            self.user_bin_path = self.user_base_path.join(self.bin_path -
                                                          self.venv_path)

        # Create a Directory to use as a scratch pad
        self.scratch_path = base_path.join("scratch").mkdir()

        # Set our default working directory
        kwargs.setdefault("cwd", self.scratch_path)

        # Setup our environment
        environ = kwargs.get("environ")
        if environ is None:
            environ = os.environ.copy()

        environ["PATH"] = Path.pathsep.join([self.bin_path] +
                                            [environ.get("PATH", [])], )
        environ["PYTHONUSERBASE"] = self.user_base_path
        # Writing bytecode can mess up updated file detection
        environ["PYTHONDONTWRITEBYTECODE"] = "1"
        # Make sure we get UTF-8 on output, even on Windows...
        environ["PYTHONIOENCODING"] = "UTF-8"
        kwargs["environ"] = environ

        # Call the TestFileEnvironment __init__
        super(PipTestEnvironment, self).__init__(base_path, *args, **kwargs)

        # Expand our absolute path directories into relative
        for name in [
                "base", "venv", "lib", "include", "bin", "site_packages",
                "user_base", "user_site", "user_bin", "scratch"
        ]:
            real_name = "%s_path" % name
            setattr(self, name, getattr(self, real_name) - self.base_path)

        # Make sure temp_path is a Path object
        self.temp_path = Path(self.temp_path)
        # Ensure the tmp dir exists, things break horribly if it doesn't
        self.temp_path.mkdir()

        # create easy-install.pth in user_site, so we always have it updated
        #   instead of created
        self.user_site_path.makedirs()
        self.user_site_path.join("easy-install.pth").touch()
Esempio n. 35
0
    def __init__(self, environ=None):
        import virtualenv

        self.root_path = fast_test_env_root
        self.backup_path = fast_test_env_backup

        self.scratch_path = self.root_path / self.scratch

        # We will set up a virtual environment at root_path.
        self.venv_path = self.root_path / self.venv

        if not environ:
            environ = os.environ.copy()
            environ = clear_environ(environ)
            environ['PIP_DOWNLOAD_CACHE'] = str(download_cache)

        environ['PIP_NO_INPUT'] = '1'
        environ['PIP_LOG_FILE'] = str(self.root_path / 'pip-log.txt')

        TestFileEnvironment.__init__(self,
                                     self.root_path,
                                     ignore_hidden=False,
                                     environ=environ,
                                     split_cmd=False,
                                     start_clear=False,
                                     cwd=self.scratch_path,
                                     capture_temp=True,
                                     assert_no_temp=True)

        virtualenv_paths = virtualenv.path_locations(self.venv_path)

        for id, path in zip(('venv', 'lib', 'include', 'bin'),
                            virtualenv_paths):
            setattr(self, id + '_path', Path(path))
            setattr(self, id, relpath(self.root_path, path))

        assert self.venv == TestPipEnvironment.venv  # sanity check

        self.site_packages = self.lib / 'site-packages'
        self.user_base_path = self.venv_path / 'user'
        self.user_site_path = self.venv_path / 'user' / 'lib' / self.lib.name / 'site-packages'

        self.user_site = relpath(self.root_path, self.user_site_path)

        self.environ["PYTHONUSERBASE"] = self.user_base_path

        # put the test-scratch virtualenv's bin dir first on the PATH
        self.environ['PATH'] = Path.pathsep.join(
            (self.bin_path, self.environ['PATH']))

        self.use_distribute = os.environ.get('PIP_TEST_USE_DISTRIBUTE', False)

        if self.root_path.exists:
            rmtree(self.root_path)
        if self.backup_path.exists:
            shutil.copytree(self.backup_path, self.root_path, True)
        else:
            demand_dirs(self.venv_path)
            demand_dirs(self.scratch_path)

            # Create a virtualenv and remember where it's putting things.
            create_virtualenv(self.venv_path, distribute=self.use_distribute)

            demand_dirs(self.user_site_path)

            # create easy-install.pth in user_site, so we always have it updated instead of created
            open(self.user_site_path / 'easy-install.pth', 'w').close()

            # test that test-scratch virtualenv creation produced sensible venv python
            result = self.run('python', '-c',
                              'import sys; print(sys.executable)')
            pythonbin = result.stdout.strip()

            if Path(pythonbin).noext != self.bin_path / 'python':
                raise RuntimeError(
                    "Oops! 'python' in our test environment runs %r"
                    " rather than expected %r" %
                    (pythonbin, self.bin_path / 'python'))

            # make sure we have current setuptools to avoid svn incompatibilities
            if not self.use_distribute:
                install_setuptools(self)

            # Uninstall whatever version of pip came with the virtualenv.
            # Earlier versions of pip were incapable of
            # self-uninstallation on Windows, so we use the one we're testing.
            self.run(
                'python', '-c',
                '"import sys; sys.path.insert(0, %r); import pip; sys.exit(pip.main());"'
                % os.path.dirname(here), 'uninstall', '-vvv', '-y', 'pip')

            # Install this version instead
            self.run('python',
                     'setup.py',
                     'install',
                     cwd=src_folder,
                     expect_stderr=True)
            shutil.copytree(self.root_path, self.backup_path, True)
        self._use_cached_pypi_server()
        assert self.root_path.exists