def make_wheel(capsys, mocker, scratch_dir="/tmp/jdtest", python_tag=None, callback=None, **extra):
kwargs = default_setup_kwargs.copy()
kwargs.update(extra)
name = kwargs["name"]
version = kwargs["version"]
fname_prefix = "-".join([name, version])
script_args = ["--no-user-cfg", "bdist_wheel"]
if python_tag is None:
script_args.append("--universal")
python_tag = "py2.py3"
else:
script_args.extend(["--python-tag", python_tag])
mocker.patch("sys.dont_write_bytecode", False)
with working_directory(scratch_dir):
for fname in kwargs["py_modules"]:
if os.path.exists(fname):
raise Exception("already exists: {}".format(fname))
with open(fname + ".py", "w"):
pass
dist = setup(script_args=script_args, **kwargs)
out, err = capsys.readouterr()
lines = out.splitlines() + err.splitlines()
for line in lines:
if "warning" in line.lower():
raise Exception("setup warning: {}".format(line))
dist_dir = os.path.join(scratch_dir, "dist")
dist_files = [f for f in os.listdir(dist_dir) if fname_prefix in f]
[dist_fname] = [f for f in dist_files if "-" + python_tag + "-" in f]
dist_path = os.path.join(dist_dir, dist_fname)
with open(dist_path, mode="rb") as f:
md5 = hashlib.md5(f.read()).hexdigest()
if callback is not None:
# contribute to test index
callback(name=name, path=dist_path, urlfragment="#md5=" + md5)
return dist, dist_path, md5
def get(dist_name,
index_url=None,
env=None,
extra_index_url=None,
tmpdir=None):
args = _get_wheel_args(index_url, env, extra_index_url) + [dist_name]
scratch_dir = tempfile.mkdtemp(dir=tmpdir)
log.debug("wheeling and dealing",
scratch_dir=scratch_dir,
args=" ".join(args))
try:
out = subprocess.check_output(args,
stderr=subprocess.STDOUT,
cwd=scratch_dir)
except subprocess.CalledProcessError as err:
output = getattr(err, "output", b"").decode("utf-8")
log.warning(output)
raise
log.debug("wheel command completed ok")
out = out.decode("utf-8")
links = []
for line in out.splitlines():
line = line.strip()
if line.startswith("Downloading from URL"):
link = line.split()[3]
links.append(link)
elif line.startswith(
"Source in ") and "which satisfies requirement" in line:
link = line.split()[-1]
links.append(link)
links = list(OrderedDict.fromkeys(links)) # order-preserving dedupe
if not links:
log.warning("could not find download link", out=out)
raise Exception("failed to collect dist")
if len(links) > 1:
log.debug("more than 1 link collected", out=out, links=links)
# Since PEP 517, maybe an sdist will also need to collect other distributions
# for the build system, even with --no-deps specified. pendulum==1.4.4 is one
# example, which uses poetry and doesn't publish any python37 wheel to PyPI.
# However, the dist itself should still be the first one downloaded.
link = links[0]
with working_directory(scratch_dir):
[whl] = [
os.path.abspath(x) for x in os.listdir(".") if x.endswith(".whl")
]
url, _sep, checksum = link.partition("#")
if not checksum.startswith("md5=") and not checksum.startswith("sha256="):
# PyPI gives you the checksum in url fragment, as a convenience. But not all indices are so kind.
algorithm = "md5"
if os.path.basename(whl) == url.rsplit("/")[-1]:
target = whl
else:
scratch_file = os.path.join(scratch_dir, os.path.basename(url))
target, _headers = urlretrieve(url, scratch_file)
checksum = compute_checksum(target=target, algorithm=algorithm)
checksum = "=".join([algorithm, checksum])
result = {"path": whl, "url": url, "checksum": checksum}
return result
def test_pipper_main(mocker, capsys, make_dist, tmpdir):
make_dist(name="fakedist", version="1.2.3")
mocker.patch("sys.argv", "pipper.py fakedist".split())
with working_directory(tmpdir):
johnnydep.pipper.main()
out, err = capsys.readouterr()
output = json.loads(out)
path = output.pop("path")
checksum = output.pop("checksum")
assert output == {"url": "http://fakeindex/fakedist-1.2.3-py2.py3-none-any.whl"}
assert os.path.isfile(path)
assert os.path.basename(path) == "fakedist-1.2.3-py2.py3-none-any.whl"
assert len(checksum) == 4 + 32
assert checksum.startswith("md5=")
assert err == ""
def get(dist_name, index_url=None, env=None):
args = _get_wheel_args(index_url, env) + [dist_name]
scratch_dir = tempfile.mkdtemp()
log.debug("wheeling and dealing",
scratch_dir=scratch_dir,
args=" ".join(args))
try:
out = subprocess.check_output(args,
stderr=subprocess.STDOUT,
cwd=scratch_dir)
except subprocess.CalledProcessError as err:
output = getattr(err, "output", b"").decode("utf-8")
log.warning(output)
raise
log.debug("wheel command completed ok")
out = out.decode("utf-8")
links = set()
for line in out.splitlines():
line = line.strip()
if line.startswith("Downloading from URL"):
link = line.split()[3]
links.add(link)
elif line.startswith(
"Source in ") and "which satisfies requirement" in line:
link = line.split()[-1]
links.add(link)
if len(links) != 1:
log.warning(out, links=links)
raise Exception("Expected exactly 1 link downloaded")
with working_directory(scratch_dir):
[whl] = [
os.path.abspath(x) for x in os.listdir(".") if x.endswith(".whl")
]
url, _sep, checksum = link.partition("#")
if not checksum.startswith("md5=") and not checksum.startswith("sha256="):
# PyPI gives you the checksum in url fragment, as a convenience. But not all indices are so kind.
algorithm = "md5"
if os.path.basename(whl) == url.rsplit("/")[-1]:
target = whl
else:
scratch_file = os.path.join(scratch_dir, os.path.basename(url))
target, _headers = urlretrieve(url, scratch_file)
checksum = compute_checksum(target=target, algorithm=algorithm)
checksum = "=".join([algorithm, checksum])
result = {"path": whl, "url": url, "checksum": checksum}
return result
def __init__(self, req_string, parent=None, index_url=None, env=None, extra_index_url=None):
self.dist_path = None
if req_string.endswith(".whl") and os.path.isfile(req_string):
self.dist_path = req_string
whl = WheelFile(req_string)
whl_name_info = whl.parsed_filename.groupdict()
self.name = canonicalize_name(whl_name_info["name"])
self.specifier = "==" + canonicalize_version(whl_name_info["ver"])
self.req = pkg_resources.Requirement.parse(self.name + self.specifier)
else:
self.req = pkg_resources.Requirement.parse(req_string)
self.name = canonicalize_name(self.req.name)
self.specifier = str(self.req.specifier)
self.extras_requested = sorted(self.req.extras)
log = self.log = logger.bind(dist=str(self.req))
log.info("init johnnydist", parent=parent and str(parent.req))
if parent is not None:
self.index_url = parent.index_url
self.extra_index_url = parent.extra_index_url
self.required_by = [str(parent.req)]
self.env = parent.env
self.env_data = parent.env_data
else:
self.index_url = index_url
self.extra_index_url = extra_index_url
self.required_by = []
self.env = env
if self.env is None:
self.env_data = default_environment()
else:
self.env_data = dict(self.env)
log.debug("target env", **self.env_data)
if self.dist_path is None:
log.debug("fetching best wheel")
with wimpy.working_directory(self.tmp()):
data = pipper.get(
req_string,
index_url=self.index_url,
env=self.env,
extra_index_url=self.extra_index_url,
)
self.dist_path = data["path"]
self.parent = parent
self._recursed = False
def _get_info(dist_name, index_url=None, env=None, extra_index_url=None):
log = logger.bind(dist_name=dist_name)
tmpdir = tempfile.mkdtemp()
log.debug("created scratch", tmpdir=tmpdir)
try:
with wimpy.working_directory(tmpdir):
data = pipper.get(
dist_name,
index_url=index_url,
env=env,
extra_index_url=extra_index_url,
tmpdir=".",
)
dist_path = data["path"]
# extract any info we may need from downloaded dist right now, so the
# downloaded file can be cleaned up immediately
import_names = _discover_import_names(dist_path)
metadata = _extract_metadata(dist_path)
finally:
log.debug("removing scratch", tmpdir=tmpdir)
shutil.rmtree(tmpdir)
return import_names, metadata
def __init__(self, req_string, parent=None, index_url=DEFAULT_INDEX):
self.req = pkg_resources.Requirement.parse(req_string)
self.name = canonicalize_name(str(self.req.name))
self.specifier = str(self.req.specifier)
self.extras_requested = sorted(self.req.extras)
log = self.log = logger.bind(dist=str(self.req))
log.info('init johnnydist', parent=parent and str(parent.req))
if parent is not None:
self.index_url = parent.index_url
self.required_by = [str(parent.req)]
else:
self.index_url = index_url
self.required_by = []
try:
log.debug('checking if installed already')
dist = pkg_resources.get_distribution(req_string)
log.debug('existing installation found', version=dist.version)
self.version_installed = dist.version
dist = pkg_resources.get_distribution(self.pinned)
except (pkg_resources.DistributionNotFound, pkg_resources.VersionConflict) as err:
log.debug('fetching best wheel')
with wimpy.working_directory(self.tmp()):
local_whl_data = get_wheel(req_string, index_url=self.index_url)
self.zip = ZipFile(file=local_whl_data['path'])
self.namelist = self.zip.namelist()
if isinstance(err, pkg_resources.VersionConflict):
self.version_installed = pkg_resources.get_distribution(self.name).version
else:
self.version_installed = None
else:
log.debug('existing installation is best!', version=dist.version)
self.namelist = [os.path.join(dist.egg_info, x) for x in os.listdir(dist.egg_info)]
self.zip = None
self.parent = parent
self._recursed = False
def __init__(self,
req_string,
parent=None,
index_url=None,
env=None,
extra_index_url=None):
self.dist_path = None
if req_string.endswith(".whl") and os.path.isfile(req_string):
self.dist_path = req_string
whl = WheelFile(req_string)
whl_name_info = whl.parsed_filename.groupdict()
self.name = canonicalize_name(whl_name_info["name"])
self.specifier = "==" + canonicalize_version(whl_name_info["ver"])
self.req = pkg_resources.Requirement.parse(self.name +
self.specifier)
else:
self.req = pkg_resources.Requirement.parse(req_string)
self.name = canonicalize_name(self.req.name)
self.specifier = str(self.req.specifier)
self.extras_requested = sorted(self.req.extras)
log = self.log = logger.bind(dist=str(self.req))
log.info("init johnnydist", parent=parent and str(parent.req))
if parent is not None:
self.index_url = parent.index_url
self.extra_index_url = parent.extra_index_url
self.required_by = [str(parent.req)]
self.env = parent.env
self.env_data = parent.env_data
else:
self.index_url = index_url
self.extra_index_url = extra_index_url
self.required_by = []
self.env = env
if self.env is None:
self.env_data = default_environment()
else:
self.env_data = dict(self.env)
log.debug("target env", **self.env_data)
if self.dist_path is None:
log.debug("fetching best wheel")
tmpdir = tempfile.mkdtemp()
log.debug("created scratch", tmpdir=tmpdir)
try:
with wimpy.working_directory(tmpdir):
data = pipper.get(
req_string,
index_url=self.index_url,
env=self.env,
extra_index_url=self.extra_index_url,
tmpdir=tmpdir,
)
self.dist_path = data["path"]
# triggers retrieval of any info we need from downloaded dist
self.import_names
self.metadata
finally:
log.debug("removing scratch", tmpdir=tmpdir)
shutil.rmtree(tmpdir)
self.parent = parent
self._recursed = False