def _get_package_url(package_name):
import pkg_resources
from infi.pypi_manager import DjangoPyPI, PyPI
pypi_url = get_pypi_index_url().replace("/simple", "")
pkg_info = pkg_resources.get_distribution(package_name)
pypi = PyPI() if is_official_pypi(pypi_url) else DjangoPyPI(pypi_url)
return pypi.get_source_distribution_url_of_specific_release_version(package_name, pkg_info.version).split("#")[0]
def mirror_package(server_name, package_name, version=None):
pypi = PyPI()
if version is None:
version = pypi.get_latest_version(package_name)
version_data = pypi._client.release_data(package_name, version)
release_dataset = pypi._client.release_urls(package_name, version)
repository_config = get_repository_config(server_name)
final_result = True
for release_data in release_dataset:
result = mirror_release(repository_config, package_name, version, version_data, release_data)
final_result = final_result and result
return final_result
def crawl(conn, crawl_count=1, new_only=False):
existing_packages = set([x[0] for x in list(conn.execute("SELECT name FROM packages"))])
client = PyPI()._client
packages = client.list_packages()
remove_deleted_packages(conn, existing_packages, packages)
total_count = len(packages)
progress = Progress("packages", crawl_count, total_count)
progress.start()
for package in packages:
progress.parse_count += 1
# for the first crawl skip the packages that we already know of. Only the second crawl onward will replace data
if new_only and package in existing_packages:
progress.preloaded += 1
continue
package_data = per_package(package)
save_package_data(conn, package, package_data)
progress.stop()
def mirror_package(server_name, package_name, version=None):
pypi = PyPI()
version = version or pypi.get_latest_version(package_name)
version_data = pypi.get_release_data(package_name, version)
release_dataset = pypi._client.release_urls(package_name, version)
repository_config = get_repository_config(server_name)
final_result = True
if not release_dataset:
msg = "No distributions found for {} {} (maybe you should try to build from download url?)"
raise DistributionNotFound(msg.format(package_name, version))
for release_data in release_dataset:
result = mirror_release(repository_config, package_name, version, version_data, release_data)
final_result = final_result and result
return final_result