def _fetch_specified_metadata(remote, project_specifiers): """ Fetch metadata for content units matching project specifiers from the remote. Args: project_specifiers (dict): Information about a project and which versions of a project to filter Returns: list: of contentunit metadata. """ remote_units = [] for project in project_specifiers: digests = python_models.DistributionDigest.objects.filter( project_specifier=project) metadata_url = urljoin(remote.url, 'pypi/%s/json' % project.name) downloader = remote.get_downloader(metadata_url) downloader.fetch() metadata = json.load(open(downloader.path)) for version, packages in metadata['releases'].items(): for package in packages: # If neither specifiers nor digests have been set, then we should add the unit if not project.version_specifier and not digests.exists(): remote_units.append( parse_metadata(metadata['info'], version, package)) continue specifier = specifiers.SpecifierSet(project.version_specifier) # Note: SpecifierSet("").contains(version) will return true for released versions # SpecifierSet("").contains('3.0.0') returns True # SpecifierSet("").contains('3.0.0b1') returns False if specifier.contains(version): # add the package if the project specifier does not have an associated digest if not digests.exists(): remote_units.append( parse_metadata(metadata['info'], version, package)) # otherwise check each digest to see if it matches the specifier else: for type, digest in package['digests'].items(): if digests.filter(type=type, digest=digest).exists(): remote_units.append( parse_metadata(metadata['info'], version, package)) break return remote_units
async def create_content(self, pkg): """ Take the filtered package, separate into releases and create a Content Unit to put into the pipeline """ for version, dists in pkg.releases.items(): for package in dists: entry = parse_metadata(pkg.info, version, package) url = entry.pop("url") artifact = Artifact(sha256=entry["sha256"]) package = PythonPackageContent(**entry) da = DeclarativeArtifact( artifact, url, entry["filename"], self.python_stage.remote, deferred_download=self.deferred_download, ) dc = DeclarativeContent(content=package, d_artifacts=[da]) await self.python_stage.put(dc)
async def get_relevant_packages(self, metadata, includes, excludes, prereleases): """ Provided project metadata and specifiers, return the matching packages. Compare the defined specifiers against the project metadata and create a deduplicated list of metadata for the packages matching the criteria. Args: metadata (dict): Metadata about the project from PyPI. includes (iterable): An iterable of project_specifiers for package versions to include. excludes (iterable): An iterable of project_specifiers for package versions to exclude. prereleases (bool): Whether or not to include pre-release package versions in the sync. Returns: list: List of dictionaries containing Python package metadata """ # The set of project release metadata, in the format {"version": [package1, package2, ...]} releases = metadata['releases'] # The packages we want to return remote_packages = [] # Delete versions/packages matching the exclude specifiers. for exclude_specifier in excludes: # Fast path: If one of the specifiers matches all versions and we don't have any # digests to reference, clear the whole dict, we're done. if not exclude_specifier.version_specifier: releases.clear() break # Slow path: We have to check all the metadata. for version, packages in list( releases.items()): # Prevent iterator invalidation. specifier = specifiers.SpecifierSet( exclude_specifier.version_specifier, prereleases=prereleases) # First check the version specifer, if it matches, check the digests and delete # matching packages. If there are no digests, delete them all. if specifier.contains(version): del releases[version] for version, packages in releases.items(): for include_specifier in includes: # Fast path: If one of the specifiers matches all versions and we don't have any # digests to reference, return all of the packages for the version. if prereleases and not include_specifier.version_specifier: for package in packages: remote_packages.append( parse_metadata(metadata['info'], version, package)) # This breaks the inner loop, e.g. don't check any other include_specifiers. # We want to continue the outer loop. break specifier = specifiers.SpecifierSet( include_specifier.version_specifier, prereleases=prereleases) # First check the version specifer, if it matches, check the digests and include # matching packages. If there are no digests, include them all. if specifier.contains(version): for package in packages: remote_packages.append( parse_metadata(metadata['info'], version, package)) return remote_packages
async def get_relevant_packages(self, metadata, includes, excludes, prereleases): """ Provided project metadata and specifiers, return the matching packages. Compare the defined specifiers against the project metadata and create a deduplicated list of metadata for the packages matching the criteria. Args: metadata (dict): Metadata about the project from PyPI. includes (iterable): An iterable of project_specifiers for package versions to include. excludes (iterable): An iterable of project_specifiers for package versions to exclude. prereleases (bool): Whether or not to include pre-release package versions in the sync. Returns: list: List of dictionaries containing Python package metadata """ # The set of project release metadata, in the format {"version": [package1, package2, ...]} releases = metadata['releases'] # The packages we want to return remote_packages = [] # Delete versions/packages matching the exclude specifiers. for exclude_specifier in excludes: # Fast path: If one of the specifiers matches all versions and we don't have any # digests to reference, clear the whole dict, we're done. if not exclude_specifier.version_specifier: releases.clear() break # Slow path: We have to check all the metadata. for version, packages in list(releases.items()): # Prevent iterator invalidation. specifier = specifiers.SpecifierSet( exclude_specifier.version_specifier, prereleases=prereleases ) # First check the version specifer, if it matches, check the digests and delete # matching packages. If there are no digests, delete them all. if specifier.contains(version): del releases[version] for version, packages in releases.items(): for include_specifier in includes: # Fast path: If one of the specifiers matches all versions and we don't have any # digests to reference, return all of the packages for the version. if prereleases and not include_specifier.version_specifier: for package in packages: remote_packages.append(parse_metadata(metadata['info'], version, package)) # This breaks the inner loop, e.g. don't check any other include_specifiers. # We want to continue the outer loop. break specifier = specifiers.SpecifierSet( include_specifier.version_specifier, prereleases=prereleases ) # First check the version specifer, if it matches, check the digests and include # matching packages. If there are no digests, include them all. if specifier.contains(version): for package in packages: remote_packages.append( parse_metadata(metadata['info'], version, package) ) return remote_packages