def unpin_and_copy_requirements(ctx, requirement_file, name="requirements.txt"): with TemporaryDirectory() as tempdir: target = Path(tempdir.name).joinpath("requirements.txt") contents = unpin_file(requirement_file.read_text()) target.write_text(contents) env = { "PIPENV_IGNORE_VIRTUALENVS": "1", "PIPENV_NOSPIN": "1", "PIPENV_PYTHON": "2.7", } with ctx.cd(tempdir.name): ctx.run("pipenv install -r {0}".format(target.as_posix()), env=env, hide=True) result = ctx.run("pipenv lock -r", env=env, hide=True).stdout.strip() ctx.run("pipenv --rm", env=env, hide=True) result = list( sorted([line.strip() for line in result.splitlines()[1:]])) new_requirements = requirement_file.parent.joinpath(name) requirement_file.rename( requirement_file.parent.joinpath("{}.bak".format(name))) new_requirements.write_text("\n".join(result)) return result
def install_pyyaml(ctx, vendor_dir): build_dir = vendor_dir / "build" if build_dir.exists() and build_dir.is_dir(): log(f"dropping pre-existing build dir at {build_dir.as_posix()}") drop_dir(build_dir) build_dir.mkdir() with TemporaryDirectory(prefix="pipenv-", suffix="-safety") as download_dir: pip_command = "pip download -b {} --no-binary=:all: --no-clean --no-deps -d {} pyyaml safety".format( build_dir.absolute().as_posix(), str(download_dir.name), ) temp_env = "TEMP" if os.name == "nt" else "TMPDIR" log(f"downloading deps via pip: {pip_command}") ctx.run(pip_command, env={temp_env: str(build_dir)}) yaml_build_dir = next(build_dir.glob('pip-download-*/pyyaml_*')) yaml_dir = vendor_dir / "yaml" path_dict = { "current_path": yaml_build_dir / "lib3/yaml", "destination": vendor_dir / "yaml3", } if yaml_dir.exists(): drop_dir(yaml_dir) path_dict["current_path"].rename(path_dict["destination"]) path_dict["destination"].joinpath("LICENSE").write_text( yaml_build_dir.joinpath("LICENSE").read_text()) drop_dir(build_dir)
def pipenv(self, cmd, block=True): if self.pipfile_path and os.path.isfile(self.pipfile_path): os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path) # a bit of a hack to make sure the virtualenv is created with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir: cmd_args = shlex.split(cmd) env = {**self.env, **{'PIPENV_CACHE_DIR': tempdir.name}} self.capfd.readouterr() r = cli_runner.invoke(cli, cmd_args, env=env) r.returncode = r.exit_code # Pretty output for failing tests. out, err = self.capfd.readouterr() if out: r.stdout_bytes = r.stdout_bytes + out if err: r.stderr_bytes = r.stderr_bytes + err if block: print(f'$ pipenv {cmd}') print(r.stdout) print(r.stderr, file=sys.stderr) if r.exception: print(''.join(traceback.format_exception(*r.exc_info)), file=sys.stderr) if r.returncode != 0: print("Command failed...") # Where the action happens. return r
def install_pyyaml(ctx, vendor_dir): build_dir = vendor_dir / "build" if build_dir.exists() and build_dir.is_dir(): log("dropping pre-existing build dir at {0}".format( build_dir.as_posix())) drop_dir(build_dir) with TemporaryDirectory(prefix="pipenv-", suffix="-safety") as download_dir: pip_command = "pip download -b {0} --no-binary=:all: --no-clean --no-deps -d {1} pyyaml safety".format( build_dir.absolute().as_posix(), str(download_dir.name), ) log("downloading deps via pip: {0}".format(pip_command)) ctx.run(pip_command) safety_build_dir = build_dir / "safety" yaml_build_dir = build_dir / "pyyaml" yaml_dir = vendor_dir / "yaml" yaml_lib_dir_map = { "2": { "current_path": yaml_build_dir / "lib/yaml", "destination": vendor_dir / "yaml2", }, "3": { "current_path": yaml_build_dir / "lib3/yaml", "destination": vendor_dir / "yaml3", }, } if yaml_dir.exists(): drop_dir(yaml_dir) log("Mapping yaml paths for python 2 and 3...") for py_version, path_dict in yaml_lib_dir_map.items(): path_dict["current_path"].rename(path_dict["destination"]) path_dict["destination"].joinpath("LICENSE").write_text( yaml_build_dir.joinpath("LICENSE").read_text()) drop_dir(build_dir)
def pipenv(self, cmd, block=True): if self.pipfile_path and os.path.isfile(self.pipfile_path): os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path) # a bit of a hack to make sure the virtualenv is created with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir: os.environ['PIPENV_CACHE_DIR'] = fs_str(tempdir.name) c = delegator.run('pipenv {0}'.format(cmd), block=block, cwd=os.path.abspath(self.path), env=os.environ.copy()) if 'PIPENV_CACHE_DIR' in os.environ: del os.environ['PIPENV_CACHE_DIR'] if 'PIPENV_PIPFILE' in os.environ: del os.environ['PIPENV_PIPFILE'] # Pretty output for failing tests. if block: print('$ pipenv {0}'.format(cmd)) print(c.out) print(c.err, file=sys.stderr) if c.return_code != 0: print("Command failed...") # Where the action happens. return c
def local_tempdir(request): old_temp = os.environ.get("TEMP", "") new_temp = Path(os.getcwd()).absolute() / "temp" new_temp.mkdir(parents=True, exist_ok=True) os.environ["TEMP"] = new_temp.as_posix() def finalize(): os.environ['TEMP'] = fs_str(old_temp) _rmtree_func(new_temp.as_posix()) request.addfinalizer(finalize) with TemporaryDirectory(dir=new_temp.as_posix()) as temp_dir: yield Path(temp_dir.name)
def update_safety(ctx): ignore_subdeps = [ "pip", "pip-egg-info", "bin", "pipenv", "virtualenv", "virtualenv-clone", "setuptools", ] ignore_files = [ "pip-delete-this-directory.txt", "PKG-INFO", "easy_install.py", "clonevirtualenv.py" ] ignore_patterns = ["*.pyd", "*.so", "**/*.pyc", "*.pyc"] cmd_envvars = { "PIPENV_NO_INHERIT": "true", "PIPENV_IGNORE_VIRTUALENVS": "true", "PIPENV_VENV_IN_PROJECT": "true" } patched_dir = _get_patched_dir(ctx) vendor_dir = _get_vendor_dir(ctx) safety_dir = Path(__file__).absolute().parent.joinpath("safety") log("Using vendor dir: %s" % patched_dir) log("Downloading safety package files...") build_dir = patched_dir / "build" root = _get_git_root(ctx) with TemporaryDirectory(prefix="pipenv-", suffix="-safety") as download_dir: log("generating lockfile...") packages = "\n".join(["safety", "requests[security]"]) env = {"PIPENV_PACKAGES": packages} resolve_cmd = "python {0}".format( root.joinpath("pipenv/resolver.py").as_posix()) py27_resolve_cmd = "python2.7 {0}".format( root.joinpath("pipenv/resolver.py").as_posix()) _, _, resolved = ctx.run(resolve_cmd, hide=True, env=env).stdout.partition("RESULTS:") _, _, resolved_py2 = ctx.run(py27_resolve_cmd, hide=True, env=env).stdout.partition("RESULTS:") resolved = json.loads(resolved.strip()) resolved_py2 = json.loads(resolved_py2.strip()) pkg_dict, pkg_dict_py2 = {}, {} for pkg in resolved: name = pkg.pop("name") pkg["version"] = "=={0}".format(pkg["version"]) pkg_dict[name] = pkg for pkg in resolved_py2: name = pkg.pop("name") pkg["version"] = "=={0}".format(pkg["version"]) pkg_dict_py2[name] = pkg merged = merge_items([pkg_dict, pkg_dict_py2]) lf = Lockfile.create(safety_dir.as_posix()) lf["default"] = merged lf.write() # envvars_no_deps = {"PIP_NO_DEPS": "true"}.update(cmd_envvars) # ctx.run("python -m pipenv run pip install safety", env=envvars_no_deps) # ctx.run("python -m pipenv run pip uninstall -y pipenv", env=cmd_envvars) # ctx.run("python -m pipenv install safety", env=cmd_envvars) # ctx.run("python -m pipenv run pip uninstall -y pipenv", env=cmd_envvars) # ctx.run("python2.7 -m pip install --upgrade --upgrade-strategy=eager -e {}".format(root.as_posix())) # ctx.run("python2.7 -m pipenv install safety", env=cmd_envvars) # requirements_txt = ctx.run("python2.7 -m pipenv lock -r", env=cmd_envvars, quiet=True).out requirements = [ r.as_line(include_hashes=False, include_markers=False) for r in lf.requirements ] safety_dir.joinpath("requirements.txt").write_text( "\n".join(requirements)) if build_dir.exists() and build_dir.is_dir(): log("dropping pre-existing build dir at {0}".format( build_dir.as_posix())) drop_dir(build_dir) pip_command = "pip download -b {0} --no-binary=:all: --no-clean --no-deps -d {1} pyyaml safety".format( build_dir.absolute().as_posix(), str(download_dir.name), ) log("downloading deps via pip: {0}".format(pip_command)) ctx.run(pip_command) safety_build_dir = build_dir / "safety" yaml_build_dir = build_dir / "pyyaml" lib_dir = safety_dir.joinpath("lib") with ctx.cd(str(safety_dir)): lib_dir.mkdir(exist_ok=True) install_cmd = "python2.7 -m pip install --ignore-requires-python -t {0} -r {1}".format( lib_dir.as_posix(), safety_dir.joinpath("requirements.txt").as_posix()) log("installing dependencies: {0}".format(install_cmd)) ctx.run(install_cmd) safety_dir = safety_dir.absolute() yaml_dir = lib_dir / "yaml" yaml_lib_dir_map = { "2": { "current_path": yaml_build_dir / "lib/yaml", "destination": lib_dir / "yaml2", }, "3": { "current_path": yaml_build_dir / "lib3/yaml", "destination": lib_dir / "yaml3", }, } if yaml_dir.exists(): drop_dir(yaml_dir) log("Mapping yaml paths for python 2 and 3...") for py_version, path_dict in yaml_lib_dir_map.items(): path_dict["current_path"].rename(path_dict["destination"]) log("Ensuring certificates are available...") requests_dir = lib_dir / "requests" cacert = vendor_dir / "certifi" / "cacert.pem" if not cacert.exists(): from pipenv.vendor import requests cacert = Path(requests.certs.where()) target_cert = requests_dir / "cacert.pem" target_cert.write_bytes(cacert.read_bytes()) log("dropping ignored files...") for pattern in ignore_patterns: for path in lib_dir.rglob(pattern): log("removing {0!s}".format(path)) path.unlink() for dep in ignore_subdeps: if lib_dir.joinpath(dep).exists(): log("cleaning up {0}".format(dep)) drop_dir(lib_dir.joinpath(dep)) for path in itertools.chain.from_iterable( (lib_dir.rglob("{0}*.egg-info".format(dep)), lib_dir.rglob("{0}*.dist-info".format(dep)))): log("cleaning up {0}".format(path)) drop_dir(path) for fn in ignore_files: for path in lib_dir.rglob(fn): log("cleaning up {0}".format(path)) path.unlink() zip_name = "{0}/safety.zip".format(str(patched_dir)) log("writing zipfile...") with zipfile.ZipFile(zip_name, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=6) as zf: _recursive_write_to_zip(zf, safety_dir) drop_dir(build_dir) drop_dir(lib_dir)
def __init__(self, pypi=None, pipfile=True, chdir=True, path=None, capfd=None, venv_root=None, ignore_virtualenvs=True, venv_in_project=True, name=None): self.index_url = os.getenv("PIPENV_TEST_INDEX") self.pypi = None self.env = {} self.capfd = capfd if pypi: self.pypi = pypi.url elif self.index_url is not None: self.pypi, _, _ = self.index_url.rpartition( "/") if self.index_url else "" self.index = os.getenv("PIPENV_PYPI_INDEX") self.env["PYTHONWARNINGS"] = "ignore:DEPRECATION" if ignore_virtualenvs: self.env["PIPENV_IGNORE_VIRTUALENVS"] = fs_str("1") if venv_root: self.env["VIRTUAL_ENV"] = venv_root if venv_in_project: self.env["PIPENV_VENV_IN_PROJECT"] = fs_str("1") else: self.env.pop("PIPENV_VENV_IN_PROJECT", None) self.original_dir = Path(__file__).parent.parent.parent path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None) if name is not None: path = Path(os.environ["HOME"]) / "projects" / name path.mkdir(exist_ok=True) if not path: path = TemporaryDirectory(suffix='-project', prefix='pipenv-') if isinstance(path, TemporaryDirectory): self._path = path path = Path(self._path.name) try: self.path = str(path.resolve()) except OSError: self.path = str(path.absolute()) elif isinstance(path, Path): self._path = path try: self.path = str(path.resolve()) except OSError: self.path = str(path.absolute()) else: self._path = path self.path = path # set file creation perms self.pipfile_path = None self.chdir = chdir if self.pypi and "PIPENV_PYPI_URL" not in os.environ: self.env['PIPENV_PYPI_URL'] = fs_str(f'{self.pypi}') # os.environ['PIPENV_PYPI_URL'] = fs_str('{0}'.format(self.pypi.url)) # os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) if pipfile: p_path = os.sep.join([self.path, 'Pipfile']) with open(p_path, 'a'): os.utime(p_path, None) self.chdir = False or chdir self.pipfile_path = p_path self._pipfile = _Pipfile(Path(p_path))
def update_safety(ctx): ignore_subdeps = ['pip', 'pip-egg-info', 'bin'] ignore_files = ['pip-delete-this-directory.txt', 'PKG-INFO'] vendor_dir = _get_patched_dir(ctx) log('Using vendor dir: %s' % vendor_dir) log('Downloading safety package files...') build_dir = vendor_dir / 'build' download_dir = TemporaryDirectory(prefix='pipenv-', suffix='-safety') if build_dir.exists() and build_dir.is_dir(): drop_dir(build_dir) ctx.run( 'pip download -b {0} --no-binary=:all: --no-clean -d {1} safety pyyaml'.format( str(build_dir), str(download_dir.name), ) ) safety_dir = build_dir / 'safety' yaml_build_dir = build_dir / 'pyyaml' main_file = safety_dir / '__main__.py' main_content = """ import sys yaml_lib = 'yaml{0}'.format(sys.version_info[0]) locals()[yaml_lib] = __import__(yaml_lib) sys.modules['yaml'] = sys.modules[yaml_lib] from safety.cli import cli # Disable insecure warnings. import urllib3 from urllib3.exceptions import InsecureRequestWarning urllib3.disable_warnings(InsecureRequestWarning) cli(prog_name="safety") """.strip() with open(str(main_file), 'w') as fh: fh.write(main_content) with ctx.cd(str(safety_dir)): ctx.run('pip install --no-compile --no-binary=:all: -t . .') safety_dir = safety_dir.absolute() yaml_dir = safety_dir / 'yaml' if yaml_dir.exists(): version_choices = ['2', '3'] version_choices.remove(str(sys.version_info[0])) mkdir_p(str(safety_dir / 'yaml{0}'.format(sys.version_info[0]))) for fn in yaml_dir.glob('*.py'): fn.rename(str(safety_dir.joinpath('yaml{0}'.format(sys.version_info[0]), fn.name))) if version_choices[0] == '2': lib = yaml_build_dir / 'lib' / 'yaml' else: lib = yaml_build_dir / 'lib3' / 'yaml' shutil.copytree(str(lib.absolute()), str(safety_dir / 'yaml{0}'.format(version_choices[0]))) requests_dir = safety_dir / 'requests' cacert = vendor_dir / 'requests' / 'cacert.pem' if not cacert.exists(): from pipenv.vendor import requests cacert = Path(requests.certs.where()) target_cert = requests_dir / 'cacert.pem' target_cert.write_bytes(cacert.read_bytes()) ctx.run("sed -i 's/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers)/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, verify=False)/g' {0}".format(str(safety_dir / 'safety' / 'safety.py'))) for egg in safety_dir.glob('*.egg-info'): drop_dir(egg.absolute()) for dep in ignore_subdeps: dep_dir = safety_dir / dep if dep_dir.exists(): drop_dir(dep_dir) for dep in ignore_files: fn = safety_dir / dep if fn.exists(): fn.unlink() zip_name = '{0}/safety'.format(str(vendor_dir)) shutil.make_archive(zip_name, format='zip', root_dir=str(safety_dir), base_dir='./') drop_dir(build_dir) download_dir.cleanup()