def unpin_and_copy_requirements(ctx,
requirement_file,
name="requirements.txt"):
with TemporaryDirectory() as tempdir:
target = Path(tempdir.name).joinpath("requirements.txt")
contents = unpin_file(requirement_file.read_text())
target.write_text(contents)
env = {
"PIPENV_IGNORE_VIRTUALENVS": "1",
"PIPENV_NOSPIN": "1",
"PIPENV_PYTHON": "2.7",
}
with ctx.cd(tempdir.name):
ctx.run("pipenv install -r {0}".format(target.as_posix()),
env=env,
hide=True)
result = ctx.run("pipenv lock -r", env=env,
hide=True).stdout.strip()
ctx.run("pipenv --rm", env=env, hide=True)
result = list(
sorted([line.strip() for line in result.splitlines()[1:]]))
new_requirements = requirement_file.parent.joinpath(name)
requirement_file.rename(
requirement_file.parent.joinpath("{}.bak".format(name)))
new_requirements.write_text("\n".join(result))
return result
def install_pyyaml(ctx, vendor_dir):
build_dir = vendor_dir / "build"
if build_dir.exists() and build_dir.is_dir():
log(f"dropping pre-existing build dir at {build_dir.as_posix()}")
drop_dir(build_dir)
build_dir.mkdir()
with TemporaryDirectory(prefix="pipenv-",
suffix="-safety") as download_dir:
pip_command = "pip download -b {} --no-binary=:all: --no-clean --no-deps -d {} pyyaml safety".format(
build_dir.absolute().as_posix(),
str(download_dir.name),
)
temp_env = "TEMP" if os.name == "nt" else "TMPDIR"
log(f"downloading deps via pip: {pip_command}")
ctx.run(pip_command, env={temp_env: str(build_dir)})
yaml_build_dir = next(build_dir.glob('pip-download-*/pyyaml_*'))
yaml_dir = vendor_dir / "yaml"
path_dict = {
"current_path": yaml_build_dir / "lib3/yaml",
"destination": vendor_dir / "yaml3",
}
if yaml_dir.exists():
drop_dir(yaml_dir)
path_dict["current_path"].rename(path_dict["destination"])
path_dict["destination"].joinpath("LICENSE").write_text(
yaml_build_dir.joinpath("LICENSE").read_text())
drop_dir(build_dir)
def pipenv(self, cmd, block=True):
if self.pipfile_path and os.path.isfile(self.pipfile_path):
os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path)
# a bit of a hack to make sure the virtualenv is created
with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir:
cmd_args = shlex.split(cmd)
env = {**self.env, **{'PIPENV_CACHE_DIR': tempdir.name}}
self.capfd.readouterr()
r = cli_runner.invoke(cli, cmd_args, env=env)
r.returncode = r.exit_code
# Pretty output for failing tests.
out, err = self.capfd.readouterr()
if out:
r.stdout_bytes = r.stdout_bytes + out
if err:
r.stderr_bytes = r.stderr_bytes + err
if block:
print(f'$ pipenv {cmd}')
print(r.stdout)
print(r.stderr, file=sys.stderr)
if r.exception:
print(''.join(traceback.format_exception(*r.exc_info)),
file=sys.stderr)
if r.returncode != 0:
print("Command failed...")
# Where the action happens.
return r
def install_pyyaml(ctx, vendor_dir):
build_dir = vendor_dir / "build"
if build_dir.exists() and build_dir.is_dir():
log("dropping pre-existing build dir at {0}".format(
build_dir.as_posix()))
drop_dir(build_dir)
with TemporaryDirectory(prefix="pipenv-",
suffix="-safety") as download_dir:
pip_command = "pip download -b {0} --no-binary=:all: --no-clean --no-deps -d {1} pyyaml safety".format(
build_dir.absolute().as_posix(),
str(download_dir.name),
)
log("downloading deps via pip: {0}".format(pip_command))
ctx.run(pip_command)
safety_build_dir = build_dir / "safety"
yaml_build_dir = build_dir / "pyyaml"
yaml_dir = vendor_dir / "yaml"
yaml_lib_dir_map = {
"2": {
"current_path": yaml_build_dir / "lib/yaml",
"destination": vendor_dir / "yaml2",
},
"3": {
"current_path": yaml_build_dir / "lib3/yaml",
"destination": vendor_dir / "yaml3",
},
}
if yaml_dir.exists():
drop_dir(yaml_dir)
log("Mapping yaml paths for python 2 and 3...")
for py_version, path_dict in yaml_lib_dir_map.items():
path_dict["current_path"].rename(path_dict["destination"])
path_dict["destination"].joinpath("LICENSE").write_text(
yaml_build_dir.joinpath("LICENSE").read_text())
drop_dir(build_dir)
def pipenv(self, cmd, block=True):
if self.pipfile_path and os.path.isfile(self.pipfile_path):
os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path)
# a bit of a hack to make sure the virtualenv is created
with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir:
os.environ['PIPENV_CACHE_DIR'] = fs_str(tempdir.name)
c = delegator.run('pipenv {0}'.format(cmd),
block=block,
cwd=os.path.abspath(self.path),
env=os.environ.copy())
if 'PIPENV_CACHE_DIR' in os.environ:
del os.environ['PIPENV_CACHE_DIR']
if 'PIPENV_PIPFILE' in os.environ:
del os.environ['PIPENV_PIPFILE']
# Pretty output for failing tests.
if block:
print('$ pipenv {0}'.format(cmd))
print(c.out)
print(c.err, file=sys.stderr)
if c.return_code != 0:
print("Command failed...")
# Where the action happens.
return c
def local_tempdir(request):
old_temp = os.environ.get("TEMP", "")
new_temp = Path(os.getcwd()).absolute() / "temp"
new_temp.mkdir(parents=True, exist_ok=True)
os.environ["TEMP"] = new_temp.as_posix()
def finalize():
os.environ['TEMP'] = fs_str(old_temp)
_rmtree_func(new_temp.as_posix())
request.addfinalizer(finalize)
with TemporaryDirectory(dir=new_temp.as_posix()) as temp_dir:
yield Path(temp_dir.name)
def update_safety(ctx):
ignore_subdeps = [
"pip",
"pip-egg-info",
"bin",
"pipenv",
"virtualenv",
"virtualenv-clone",
"setuptools",
]
ignore_files = [
"pip-delete-this-directory.txt", "PKG-INFO", "easy_install.py",
"clonevirtualenv.py"
]
ignore_patterns = ["*.pyd", "*.so", "**/*.pyc", "*.pyc"]
cmd_envvars = {
"PIPENV_NO_INHERIT": "true",
"PIPENV_IGNORE_VIRTUALENVS": "true",
"PIPENV_VENV_IN_PROJECT": "true"
}
patched_dir = _get_patched_dir(ctx)
vendor_dir = _get_vendor_dir(ctx)
safety_dir = Path(__file__).absolute().parent.joinpath("safety")
log("Using vendor dir: %s" % patched_dir)
log("Downloading safety package files...")
build_dir = patched_dir / "build"
root = _get_git_root(ctx)
with TemporaryDirectory(prefix="pipenv-",
suffix="-safety") as download_dir:
log("generating lockfile...")
packages = "\n".join(["safety", "requests[security]"])
env = {"PIPENV_PACKAGES": packages}
resolve_cmd = "python {0}".format(
root.joinpath("pipenv/resolver.py").as_posix())
py27_resolve_cmd = "python2.7 {0}".format(
root.joinpath("pipenv/resolver.py").as_posix())
_, _, resolved = ctx.run(resolve_cmd, hide=True,
env=env).stdout.partition("RESULTS:")
_, _, resolved_py2 = ctx.run(py27_resolve_cmd, hide=True,
env=env).stdout.partition("RESULTS:")
resolved = json.loads(resolved.strip())
resolved_py2 = json.loads(resolved_py2.strip())
pkg_dict, pkg_dict_py2 = {}, {}
for pkg in resolved:
name = pkg.pop("name")
pkg["version"] = "=={0}".format(pkg["version"])
pkg_dict[name] = pkg
for pkg in resolved_py2:
name = pkg.pop("name")
pkg["version"] = "=={0}".format(pkg["version"])
pkg_dict_py2[name] = pkg
merged = merge_items([pkg_dict, pkg_dict_py2])
lf = Lockfile.create(safety_dir.as_posix())
lf["default"] = merged
lf.write()
# envvars_no_deps = {"PIP_NO_DEPS": "true"}.update(cmd_envvars)
# ctx.run("python -m pipenv run pip install safety", env=envvars_no_deps)
# ctx.run("python -m pipenv run pip uninstall -y pipenv", env=cmd_envvars)
# ctx.run("python -m pipenv install safety", env=cmd_envvars)
# ctx.run("python -m pipenv run pip uninstall -y pipenv", env=cmd_envvars)
# ctx.run("python2.7 -m pip install --upgrade --upgrade-strategy=eager -e {}".format(root.as_posix()))
# ctx.run("python2.7 -m pipenv install safety", env=cmd_envvars)
# requirements_txt = ctx.run("python2.7 -m pipenv lock -r", env=cmd_envvars, quiet=True).out
requirements = [
r.as_line(include_hashes=False, include_markers=False)
for r in lf.requirements
]
safety_dir.joinpath("requirements.txt").write_text(
"\n".join(requirements))
if build_dir.exists() and build_dir.is_dir():
log("dropping pre-existing build dir at {0}".format(
build_dir.as_posix()))
drop_dir(build_dir)
pip_command = "pip download -b {0} --no-binary=:all: --no-clean --no-deps -d {1} pyyaml safety".format(
build_dir.absolute().as_posix(),
str(download_dir.name),
)
log("downloading deps via pip: {0}".format(pip_command))
ctx.run(pip_command)
safety_build_dir = build_dir / "safety"
yaml_build_dir = build_dir / "pyyaml"
lib_dir = safety_dir.joinpath("lib")
with ctx.cd(str(safety_dir)):
lib_dir.mkdir(exist_ok=True)
install_cmd = "python2.7 -m pip install --ignore-requires-python -t {0} -r {1}".format(
lib_dir.as_posix(),
safety_dir.joinpath("requirements.txt").as_posix())
log("installing dependencies: {0}".format(install_cmd))
ctx.run(install_cmd)
safety_dir = safety_dir.absolute()
yaml_dir = lib_dir / "yaml"
yaml_lib_dir_map = {
"2": {
"current_path": yaml_build_dir / "lib/yaml",
"destination": lib_dir / "yaml2",
},
"3": {
"current_path": yaml_build_dir / "lib3/yaml",
"destination": lib_dir / "yaml3",
},
}
if yaml_dir.exists():
drop_dir(yaml_dir)
log("Mapping yaml paths for python 2 and 3...")
for py_version, path_dict in yaml_lib_dir_map.items():
path_dict["current_path"].rename(path_dict["destination"])
log("Ensuring certificates are available...")
requests_dir = lib_dir / "requests"
cacert = vendor_dir / "certifi" / "cacert.pem"
if not cacert.exists():
from pipenv.vendor import requests
cacert = Path(requests.certs.where())
target_cert = requests_dir / "cacert.pem"
target_cert.write_bytes(cacert.read_bytes())
log("dropping ignored files...")
for pattern in ignore_patterns:
for path in lib_dir.rglob(pattern):
log("removing {0!s}".format(path))
path.unlink()
for dep in ignore_subdeps:
if lib_dir.joinpath(dep).exists():
log("cleaning up {0}".format(dep))
drop_dir(lib_dir.joinpath(dep))
for path in itertools.chain.from_iterable(
(lib_dir.rglob("{0}*.egg-info".format(dep)),
lib_dir.rglob("{0}*.dist-info".format(dep)))):
log("cleaning up {0}".format(path))
drop_dir(path)
for fn in ignore_files:
for path in lib_dir.rglob(fn):
log("cleaning up {0}".format(path))
path.unlink()
zip_name = "{0}/safety.zip".format(str(patched_dir))
log("writing zipfile...")
with zipfile.ZipFile(zip_name,
'w',
compression=zipfile.ZIP_DEFLATED,
compresslevel=6) as zf:
_recursive_write_to_zip(zf, safety_dir)
drop_dir(build_dir)
drop_dir(lib_dir)
def __init__(self,
pypi=None,
pipfile=True,
chdir=True,
path=None,
capfd=None,
venv_root=None,
ignore_virtualenvs=True,
venv_in_project=True,
name=None):
self.index_url = os.getenv("PIPENV_TEST_INDEX")
self.pypi = None
self.env = {}
self.capfd = capfd
if pypi:
self.pypi = pypi.url
elif self.index_url is not None:
self.pypi, _, _ = self.index_url.rpartition(
"/") if self.index_url else ""
self.index = os.getenv("PIPENV_PYPI_INDEX")
self.env["PYTHONWARNINGS"] = "ignore:DEPRECATION"
if ignore_virtualenvs:
self.env["PIPENV_IGNORE_VIRTUALENVS"] = fs_str("1")
if venv_root:
self.env["VIRTUAL_ENV"] = venv_root
if venv_in_project:
self.env["PIPENV_VENV_IN_PROJECT"] = fs_str("1")
else:
self.env.pop("PIPENV_VENV_IN_PROJECT", None)
self.original_dir = Path(__file__).parent.parent.parent
path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None)
if name is not None:
path = Path(os.environ["HOME"]) / "projects" / name
path.mkdir(exist_ok=True)
if not path:
path = TemporaryDirectory(suffix='-project', prefix='pipenv-')
if isinstance(path, TemporaryDirectory):
self._path = path
path = Path(self._path.name)
try:
self.path = str(path.resolve())
except OSError:
self.path = str(path.absolute())
elif isinstance(path, Path):
self._path = path
try:
self.path = str(path.resolve())
except OSError:
self.path = str(path.absolute())
else:
self._path = path
self.path = path
# set file creation perms
self.pipfile_path = None
self.chdir = chdir
if self.pypi and "PIPENV_PYPI_URL" not in os.environ:
self.env['PIPENV_PYPI_URL'] = fs_str(f'{self.pypi}')
# os.environ['PIPENV_PYPI_URL'] = fs_str('{0}'.format(self.pypi.url))
# os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url))
if pipfile:
p_path = os.sep.join([self.path, 'Pipfile'])
with open(p_path, 'a'):
os.utime(p_path, None)
self.chdir = False or chdir
self.pipfile_path = p_path
self._pipfile = _Pipfile(Path(p_path))
def update_safety(ctx):
ignore_subdeps = ['pip', 'pip-egg-info', 'bin']
ignore_files = ['pip-delete-this-directory.txt', 'PKG-INFO']
vendor_dir = _get_patched_dir(ctx)
log('Using vendor dir: %s' % vendor_dir)
log('Downloading safety package files...')
build_dir = vendor_dir / 'build'
download_dir = TemporaryDirectory(prefix='pipenv-', suffix='-safety')
if build_dir.exists() and build_dir.is_dir():
drop_dir(build_dir)
ctx.run(
'pip download -b {0} --no-binary=:all: --no-clean -d {1} safety pyyaml'.format(
str(build_dir), str(download_dir.name),
)
)
safety_dir = build_dir / 'safety'
yaml_build_dir = build_dir / 'pyyaml'
main_file = safety_dir / '__main__.py'
main_content = """
import sys
yaml_lib = 'yaml{0}'.format(sys.version_info[0])
locals()[yaml_lib] = __import__(yaml_lib)
sys.modules['yaml'] = sys.modules[yaml_lib]
from safety.cli import cli
# Disable insecure warnings.
import urllib3
from urllib3.exceptions import InsecureRequestWarning
urllib3.disable_warnings(InsecureRequestWarning)
cli(prog_name="safety")
""".strip()
with open(str(main_file), 'w') as fh:
fh.write(main_content)
with ctx.cd(str(safety_dir)):
ctx.run('pip install --no-compile --no-binary=:all: -t . .')
safety_dir = safety_dir.absolute()
yaml_dir = safety_dir / 'yaml'
if yaml_dir.exists():
version_choices = ['2', '3']
version_choices.remove(str(sys.version_info[0]))
mkdir_p(str(safety_dir / 'yaml{0}'.format(sys.version_info[0])))
for fn in yaml_dir.glob('*.py'):
fn.rename(str(safety_dir.joinpath('yaml{0}'.format(sys.version_info[0]), fn.name)))
if version_choices[0] == '2':
lib = yaml_build_dir / 'lib' / 'yaml'
else:
lib = yaml_build_dir / 'lib3' / 'yaml'
shutil.copytree(str(lib.absolute()), str(safety_dir / 'yaml{0}'.format(version_choices[0])))
requests_dir = safety_dir / 'requests'
cacert = vendor_dir / 'requests' / 'cacert.pem'
if not cacert.exists():
from pipenv.vendor import requests
cacert = Path(requests.certs.where())
target_cert = requests_dir / 'cacert.pem'
target_cert.write_bytes(cacert.read_bytes())
ctx.run("sed -i 's/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers)/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, verify=False)/g' {0}".format(str(safety_dir / 'safety' / 'safety.py')))
for egg in safety_dir.glob('*.egg-info'):
drop_dir(egg.absolute())
for dep in ignore_subdeps:
dep_dir = safety_dir / dep
if dep_dir.exists():
drop_dir(dep_dir)
for dep in ignore_files:
fn = safety_dir / dep
if fn.exists():
fn.unlink()
zip_name = '{0}/safety'.format(str(vendor_dir))
shutil.make_archive(zip_name, format='zip', root_dir=str(safety_dir), base_dir='./')
drop_dir(build_dir)
download_dir.cleanup()
