def add_blacklist(request):
project_name = request.POST.get("project")
if project_name is None:
raise HTTPBadRequest("Have a project to confirm.")
comment = request.POST.get("comment", "")
# Verify that the user has confirmed the request to blacklist.
confirm = request.POST.get("confirm")
if not confirm:
request.session.flash("Confirm the blacklist request", queue="error")
return HTTPSeeOther(request.current_route_path())
elif canonicalize_name(confirm) != canonicalize_name(project_name):
request.session.flash(
f"{confirm!r} is not the same as {project_name!r}", queue="error"
)
return HTTPSeeOther(request.current_route_path())
# Check to make sure the object doesn't already exist.
if (
request.db.query(literal(True))
.filter(
request.db.query(BlacklistedProject)
.filter(BlacklistedProject.name == project_name)
.exists()
)
.scalar()
):
request.session.flash(
f"{project_name!r} has already been blacklisted.", queue="error"
)
return HTTPSeeOther(request.route_path("admin.blacklist.list"))
# Add our requested blacklist.
request.db.add(
BlacklistedProject(
name=project_name, comment=comment, blacklisted_by=request.user
)
)
# Go through and delete the project and everything related to it so that
# our blacklist actually blocks things and isn't ignored (since the
# blacklist only takes effect on new project registration).
project = (
request.db.query(Project)
.filter(Project.normalized_name == func.normalize_pep426_name(project_name))
.first()
)
if project is not None:
remove_project(project, request)
request.session.flash(f"Blacklisted {project_name!r}", queue="success")
return HTTPSeeOther(request.route_path("admin.blacklist.list"))
def find_extra_reqs(options):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug("installed package: %s (at %s)", package["name"], package["location"])
for f in package.get("files", []):
path = os.path.realpath(os.path.join(package["location"], f))
installed_files[path] = package["name"]
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package["name"]
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug("used module: %s (from package %s)", modname, installed_files[info.filename])
used[used_name].append(info)
else:
log.debug("used module: %s (from file %s, assuming stdlib or local)", modname, info.filename)
# 4. compare with requirements.txt
explicit = common.find_required_modules(options)
return [name for name in explicit if name not in used]
def package_or_url_from_pep508(
requirement: Requirement, remove_version_specifiers: bool = False
) -> str:
requirement.marker = None
requirement.name = canonicalize_name(requirement.name)
if remove_version_specifiers:
requirement.specifier = SpecifierSet("")
return str(requirement)
def __init__(self, name, serial, mirror):
self.name = name
self.serial = serial
self.normalized_name = canonicalize_name(name)
# This is really only useful for pip 8.0 -> 8.1.1
self.normalized_name_legacy = pkg_resources.safe_name(name).lower()
self.mirror = mirror
self.customized_host = environ.get("CUSTOMIZED_HOST") # Hack
def __init__(self, name, version, url=None, extras=None):
self.name = canonicalize_name(name)
self.version = version
self.url = url
self.extras = extras
self._metadata = None
self._dependencies = None
def filter(self, metadata: Dict) -> bool:
"""
Returns False if version fails the filter,
i.e. doesn't matches an allowlist version specifier
"""
name = metadata["info"]["name"]
version = metadata["version"]
return self._check_match(canonicalize_name(name), version)
def add(req: Requirement) -> Node:
key = Node.key(req)
if key in nodes:
return nodes[key]
dist = installed_dists.get(canonicalize_name(req.name))
node = Node(req, dist)
nodes[key] = node
if not dist:
# not installed
return node
for dep_req in dist.requires:
node.children.append(add(dep_req))
for extra in req.extras:
extra = canonicalize_name(extra)
for dep_req in dist.extra_requires.get(extra, []):
node.children.append(add(dep_req))
return node
def _get_distribution_version(normalized_package_name: str) -> str:
name: str
version: str
for name, version in _get_distributions_names_versions().items():
# If the package name is a match, we will return the version found
if name and canonicalize_name(name) == normalized_package_name:
return version
return ""
def __init__(self, req_string, parent=None, index_url=None, env=None, extra_index_url=None):
self.dist_path = None
if req_string.endswith(".whl") and os.path.isfile(req_string):
self.dist_path = req_string
whl = WheelFile(req_string)
whl_name_info = whl.parsed_filename.groupdict()
self.name = canonicalize_name(whl_name_info["name"])
self.specifier = "==" + canonicalize_version(whl_name_info["ver"])
self.req = pkg_resources.Requirement.parse(self.name + self.specifier)
else:
self.req = pkg_resources.Requirement.parse(req_string)
self.name = canonicalize_name(self.req.name)
self.specifier = str(self.req.specifier)
self.extras_requested = sorted(self.req.extras)
log = self.log = logger.bind(dist=str(self.req))
log.info("init johnnydist", parent=parent and str(parent.req))
if parent is not None:
self.index_url = parent.index_url
self.extra_index_url = parent.extra_index_url
self.required_by = [str(parent.req)]
self.env = parent.env
self.env_data = parent.env_data
else:
self.index_url = index_url
self.extra_index_url = extra_index_url
self.required_by = []
self.env = env
if self.env is None:
self.env_data = default_environment()
else:
self.env_data = dict(self.env)
log.debug("target env", **self.env_data)
if self.dist_path is None:
log.debug("fetching best wheel")
with wimpy.working_directory(self.tmp()):
data = pipper.get(
req_string,
index_url=self.index_url,
env=self.env,
extra_index_url=self.extra_index_url,
)
self.dist_path = data["path"]
self.parent = parent
self._recursed = False
def _add_responses_mocks(self):
for package in ['Django', 'celery', 'django-rest-auth', 'ipython']:
with open('tests/fixtures/{}.json'.format(package)) as fh:
body = fh.read()
responses.add(
responses.GET,
"https://pypi.python.org/pypi/{}/json".format(package),
body=body,
content_type="application/json")
with open('tests/fixtures/{}.html'.format(
canonicalize_name(package))) as fh:
body_html = fh.read()
responses.add(responses.GET,
"https://pypi.python.org/simple/{}".format(
canonicalize_name(package)),
body=body_html)
def get_req_name(requirement: str) -> Optional[NormalizedName]:
name: Optional[str] = None
try:
name = Requirement(requirement).name
except InvalidRequirement:
name = _get_egg_name(requirement)
if not name:
return None
return canonicalize_name(name)
def get_project_name(python: str, project_root: Path) -> NormalizedName:
log_info("Getting project name..", nl=False)
pyproject_toml = _load_pyproject_toml(project_root)
name = (get_project_name_from_pyproject_toml_pep621(pyproject_toml)
or get_project_name_from_setup_cfg(project_root, pyproject_toml)
or get_project_name_from_pyproject_toml_flit(pyproject_toml)
or get_project_name_from_pep517(python, project_root))
log_info(" " + name)
return canonicalize_name(name)
def get_pypi_uploaders(dist_name):
roles = _get_pypi_roles(dist_name)
if not roles:
canonical_name = packaging_utils.canonicalize_name(dist_name)
roles = _get_pypi_roles(canonical_name)
uploaders = set(acct for role, acct in roles
if role in ('Owner', 'Maintainer'))
LOG.debug('found: {}'.format(sorted(uploaders)))
return uploaders
def __init__(self, impl):
self.groups = []
self.depends = {}
for depend in impl.depends:
if isinstance(depend, tuple):
self.groups.append(frozenset(map(canonicalize_name, depend)))
else:
self.depends[canonicalize_name(depend)] = depend
self.impl = impl
def get_infos(requires):
for require in requires:
info = PackageInfo(
require.name,
installed.get(canonicalize_name(require.name))
)
if check_outdated and not info.check_update():
continue
yield info
def __init__(
self, name: str, serial: str, mirror: "Mirror", *, cleanup: bool = False
) -> None:
self.name = canonicalize_name(name)
self.raw_name = name
self.normalized_name_legacy = utils.bandersnatch_safe_name(name)
self.serial = serial
self.mirror = mirror
self.cleanup = cleanup
def test_no_query(self, db_request):
db_request.db.query(ProhibitedProjectName).delete()
prohibited = sorted(
[ProhibitedProjectFactory.create() for _ in range(30)],
key=lambda b: canonicalize_name(b.name),
)
result = views.prohibited_project_names(db_request)
assert result == {"prohibited_project_names": prohibited[:25], "query": None}
def fix_package_name(package_or_url: str, package: str) -> str:
try:
package_req = Requirement(package_or_url)
except InvalidRequirement:
# not a valid PEP508 package specification
return package_or_url
if canonicalize_name(package_req.name) != canonicalize_name(package):
logging.warning(
textwrap.fill(
f"{hazard} Name supplied in package specifier was "
f"{package_req.name!r} but package found has name "
f"{package!r}. Using {package!r}.",
subsequent_indent=" ",
))
package_req.name = package
return str(package_req)
def from_pypi_name(self, name: str) -> Sequence[NixPackage]:
try:
data = self.__data[canonicalize_name(name)]
except KeyError:
raise PackageNotFound(f'{name} is not defined in nixpkgs')
return [
NixPackage(attr=drv['attr'], version=parse(drv['version']))
for drv in data
]
def test_no_query(self, db_request):
db_request.db.query(BlacklistedProject).delete()
blacklisted = sorted(
[BlacklistedProjectFactory.create() for _ in range(30)],
key=lambda b: canonicalize_name(b.name),
)
result = views.blacklist(db_request)
assert result == {"blacklist": blacklisted[:25], "query": None}
def test_no_query(self, db_request):
db_request.db.query(BlacklistedProject).delete()
blacklisted = sorted(
[BlacklistedProjectFactory.create() for _ in range(30)],
key=lambda b: canonicalize_name(b.name),
)
result = views.blacklist(db_request)
assert result == {"blacklist": blacklisted[:25], "query": None}
def find_missing_reqs(options):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug('installed package: %s (at %s)', package['name'],
package['location'])
for file in package.get('files', []) or []:
path = os.path.realpath(os.path.join(package['location'], file))
installed_files[path] = package['name']
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package['name']
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements.txt
explicit = set()
for requirement in parse_requirements('requirements.txt',
session=PipSession()):
log.debug('found requirement: %s', requirement.name)
explicit.add(canonicalize_name(requirement.name))
return [(name, used[name]) for name in used
if name not in explicit]
def add_blacklist(request):
project_name = request.POST.get("project")
if project_name is None:
raise HTTPBadRequest("Must have a project to confirm.")
comment = request.POST.get("comment", "")
# Verify that the user has confirmed the request to blacklist.
confirm = request.POST.get("confirm")
if not confirm:
request.session.flash(
"Must confirm the blacklist request.",
queue="error",
)
return HTTPSeeOther(request.current_route_path())
elif canonicalize_name(confirm) != canonicalize_name(project_name):
request.session.flash(
f"{confirm!r} is not the same as {project_name!r}",
queue="error",
)
return HTTPSeeOther(request.current_route_path())
# Add our requested blacklist.
request.db.add(
BlacklistedProject(
name=project_name,
comment=comment,
blacklisted_by=request.user,
))
# Go through and delete the project and everything related to it so that
# our blacklist actually blocks things and isn't ignored (since the
# blacklist only takes effect on new project registration).
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
project_name)).first())
if project is not None:
remove_project(project, request)
request.session.flash(
f"Successfully blacklisted {project_name!r}",
queue="success",
)
return HTTPSeeOther(request.route_path("admin.blacklist.list"))
def __init__(self, req_string, parent=None, index_url=None, env=None, extra_index_url=None):
log = self.log = logger.bind(dist=req_string)
log.info("init johnnydist", parent=parent and str(parent.req))
self.parent = parent
self.index_url = index_url
self.env = env
self.extra_index_url = extra_index_url
self._recursed = False
if req_string.endswith(".whl") and os.path.isfile(req_string):
whl = distlib.wheel.Wheel(req_string)
self.name = canonicalize_name(whl.name)
self.specifier = "==" + canonicalize_version(whl.version)
self.req = pkg_resources.Requirement.parse(self.name + self.specifier)
self.import_names = _discover_import_names(req_string)
self.metadata = _extract_metadata(req_string)
elif req_string.endswith(".txt") and os.path.isfile(req_string):
self.name = req_string
self.specifier = ""
self.req = pkg_resources.Requirement.parse("requirements")
self.import_names = ["requirements"]
with open(self.name, 'r') as f:
lines = f.readlines()
lines = [line for line in lines if not line.startswith('-') and not line.startswith('git+')]
self.metadata = {"requires_dist": lines}
else:
self.req = pkg_resources.Requirement.parse(req_string)
self.name = canonicalize_name(self.req.name)
self.specifier = str(self.req.specifier)
log.debug("fetching best wheel")
self.import_names, self.metadata = _get_info(
dist_name=req_string,
index_url=index_url,
env=env,
extra_index_url=extra_index_url,
)
self.extras_requested = sorted(self.req.extras)
if parent is None:
if env:
log.debug("root node target env", **dict(env))
self.required_by = []
else:
self.required_by = [str(parent.req)]
def validate_package_import_name(answer: str,
_: columbo.Answers) -> Optional[str]:
canonical_name = canonicalize_name(answer).replace("-", "_")
if not canonical_name == answer:
return f"Import names should follow PEP-8 naming conventions. Did you mean {canonical_name}?"
if not answer.replace("_", "").isalpha():
return (
"Import names may only contain alphabetical characters and underscores. "
"They may not contain spaces, numbers, or other characters.")
return None
def find_required_modules(options):
explicit = set()
for requirement in parse_requirements('requirements.txt',
session=PipSession()):
if options.ignore_reqs(requirement):
log.debug('ignoring requirement: %s', requirement.name)
else:
log.debug('found requirement: %s', requirement.name)
explicit.add(canonicalize_name(requirement.name))
return explicit
def find_missing_reqs(options):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug('installed package: %s (at %s)', package['name'],
package['location'])
for file in package.get('files', []) or []:
path = os.path.realpath(os.path.join(package['location'], file))
installed_files[path] = package['name']
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package['name']
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements.txt
explicit = set()
for requirement in parse_requirements('requirements.txt',
session=PipSession()):
log.debug('found requirement: %s', requirement.name)
explicit.add(canonicalize_name(requirement.name))
return [(name, used[name]) for name in used if name not in explicit]
def get_reverse_dependencies(pkg_name: str) -> List[str]:
pkg_name = canonicalize_name(pkg_name)
dataset_path = config.get_reverse_dependencies_path()
with dataset_path.open("r") as fd:
dataset = loads(fd.read())
if pkg_name in dataset:
return dataset[pkg_name]
else:
return []
def __call__(self) -> bool:
name = canonicalize_name(self.args.name)
data = dict(
total=get_total_downloads(name=name),
pythons=get_downloads_by_category(category='pythons', name=name),
systems=get_downloads_by_category(category='systems', name=name),
)
print(make_json(data=data, key=self.config.get('filter')))
return True
def get_pypi_info(dist_name):
"Return PyPI information for the distribution."
canonical_name = packaging_utils.canonicalize_name(dist_name)
LOG.debug('looking at PyPI for {!r}'.format(canonical_name))
url = 'https://pypi.org/project/{}/json'.format(canonical_name)
LOG.debug(url)
try:
return requests.get(url).json()
except json.decoder.JSONDecodeError:
return {}
def from_local_mirror(cls, name: str, *args, **kwargs):
name = canonicalize_name(name)
if cls.mirror is None:
cls.mirror = LocalMirror()
kwargs["source"] = "local_mirror"
kwargs["info"] = cls.mirror.get_json(name)
return cls(name, *args, **kwargs)
def test_basic_query(self, db_request):
db_request.db.query(BlacklistedProject).delete()
blacklisted = sorted(
[BlacklistedProjectFactory.create() for _ in range(30)],
key=lambda b: canonicalize_name(b.name),
)
db_request.GET["q"] = blacklisted[0].name
result = views.blacklist(db_request)
assert result == {"blacklist": [blacklisted[0]], "query": blacklisted[0].name}
def find_package_indexes_in_dir(self, simple_dir: Path) -> List[str]:
"""Given a directory that contains simple packages indexes, return
a sorted list of normalized package names. This presumes every
directory within is a simple package index directory."""
simple_path = self.storage_backend.PATH_BACKEND(str(simple_dir))
return sorted({
canonicalize_name(str(x.parent.relative_to(simple_path)))
for x in simple_path.glob("**/index.html")
if str(x.parent.relative_to(simple_path)) != "."
})
def find_required_modules(options):
explicit = set()
for requirement in parse_requirements('requirements.txt',
session=PipSession()):
if options.ignore_reqs(requirement):
log.debug('ignoring requirement: %s', requirement.name)
else:
log.debug('found requirement: %s', requirement.name)
explicit.add(canonicalize_name(requirement.name))
return explicit
def get_package_location(name: str = "") -> str:
"""
Find the directory in which a package is installed
"""
name = canonicalize_name(name)
try:
distribution: pkg_resources.Distribution = get_distribution(name)
return distribution.location
except pkg_resources.DistributionNotFound:
return os.path.dirname(_get_editable_distribution_setup_script(name))
def test_basic_query(self, db_request):
db_request.db.query(BlacklistedProject).delete()
blacklisted = sorted(
[BlacklistedProjectFactory.create() for _ in range(30)],
key=lambda b: canonicalize_name(b.name),
)
db_request.GET["q"] = blacklisted[0].name
result = views.blacklist(db_request)
assert result == {"blacklist": [blacklisted[0]], "query": blacklisted[0].name}
def __init__(self, name, serial, mirror):
self.name = name
self.serial = serial
self.normalized_name = canonicalize_name(name).encode("utf-8")
# This is really only useful for pip 8.0 -> 8.1.1
self.normalized_name_legacy = \
pkg_resources.safe_name(name).lower().encode("utf-8")
self.encoded_name = self.name.encode('utf-8')
self.encoded_first = self.name[0].encode('utf-8')
self.quoted_name = quote(self.encoded_name)
self.mirror = mirror
def confirm_project(project, request, fail_route):
confirm = request.POST.get("confirm_project_name")
project_name = project.normalized_name
if not confirm:
request.session.flash("Confirm the request", queue="error")
raise HTTPSeeOther(request.route_path(fail_route, project_name=project_name))
if canonicalize_name(confirm) != project.normalized_name:
request.session.flash(
"Could not delete project - "
+ f"{confirm!r} is not the same as {project.normalized_name!r}",
queue="error",
)
raise HTTPSeeOther(request.route_path(fail_route, project_name=project_name))
def test_with_page(self, db_request):
db_request.db.query(BlacklistedProject).delete()
blacklisted = sorted(
[BlacklistedProjectFactory.create() for _ in range(30)],
key=lambda b: canonicalize_name(b.name),
)
db_request.GET["page"] = "2"
result = views.blacklist(db_request)
assert result == {
"blacklist": blacklisted[25:],
"query": None,
}
def sync_index_page(self):
if not self.need_index_sync:
return
logger.info(u'Generating global index page.')
simple_dir = os.path.join(self.webdir, 'simple')
with rewrite(os.path.join(simple_dir, 'index.html')) as f:
f.write('<html><head><title>Simple Index</title></head><body>\n')
for pkg in sorted(set(
# Filter out all of the "non" normalized names here
canonicalize_name(x)
for x in os.listdir(simple_dir))):
if not os.path.isdir(os.path.join(simple_dir, pkg)):
continue
# We're really trusty that this is all encoded in UTF-8. :/
f.write('<a href="{0}/">{1}</a><br/>\n'.format(pkg, pkg))
f.write('</body></html>')
def write_simple_api(publication):
"""
Write metadata for the simple API.
Writes metadata mimicking the simple api of PyPI for all python packages
in the repository version.
https://wiki.python.org/moin/PyPISimple
Args:
publication (pulpcore.plugin.models.Publication): A publication to generate metadata for
"""
simple_dir = 'simple/'
os.mkdir(simple_dir)
project_names = (
python_models.PythonPackageContent.objects.filter(
pk__in=publication.repository_version.content
)
.order_by('name')
.values_list('name', flat=True)
.distinct()
)
index_names = [(name, canonicalize_name(name)) for name in project_names]
# write the root index, which lists all of the projects for which there is a package available
index_path = '{simple_dir}index.html'.format(simple_dir=simple_dir)
with open(index_path, 'w') as index:
context = Context({'projects': index_names})
template = Template(simple_index_template)
index.write(template.render(context))
index_metadata = models.PublishedMetadata(
relative_path=simple_dir,
publication=publication,
file=File(open(index_path, 'rb'))
)
index_metadata.save()
for (name, canonical_name) in index_names:
project_dir = '{simple_dir}{name}/'.format(simple_dir=simple_dir, name=canonical_name)
os.mkdir(project_dir)
packages = python_models.PythonPackageContent.objects.filter(name=name)
package_detail_data = []
for package in packages:
artifact_set = package.contentartifact_set.all()
for content_artifact in artifact_set:
published_artifact = models.PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=publication,
content_artifact=content_artifact
)
published_artifact.save()
checksum = content_artifact.artifact.sha256
path = "../../{}".format(package.filename)
package_detail_data.append((package.filename, path, checksum))
metadata_relative_path = '{project_dir}index.html'.format(project_dir=project_dir)
with open(metadata_relative_path, 'w') as simple_metadata:
context = Context({
'project_name': name,
'project_packages': package_detail_data
})
template = Template(simple_detail_template)
simple_metadata.write(template.render(context))
project_metadata = models.PublishedMetadata(
relative_path=project_dir,
publication=publication,
file=File(open(metadata_relative_path, 'rb'))
)
project_metadata.save()
def test_canonicalize_name(name, expected):
assert canonicalize_name(name) == expected