def test_get_file_links(shared_datadir: Path):
print(shared_datadir)
with (shared_datadir / "click.html").open() as f:
html_doc = f.read()
base_url = "https://mirrors.aliyun.com/pypi/simple/click/"
python_package_local = PythonPackage("Click", "7.0")
res = get_file_links(html_doc, base_url, python_package_local)
assert len(res) == 2
python_package_local = PythonPackage("click", "6.7")
res = get_file_links(html_doc, base_url, python_package_local)
assert len(res) == 2
def pipdownload(packages, index_url, requirement_file, dest_dir, whl_suffixes,
platform_tags, python_versions, quiet, no_source, show_config,
show_urls):
"""
pip-download is a tool which can be used to download python projects and their dependencies listed on
pypi's `download files` page. It can be used to download Python packages across system platforms and
Python versions.
"""
if show_config:
if not Path(settings.SETTINGS_FILE).exists():
Path(settings.SETTINGS_FILE).parent.mkdir(parents=True,
exist_ok=True)
# Path(SETTINGS_FILE).touch()
with open(settings.SETTINGS_FILE, "w", encoding="utf8") as f:
json.dump({}, f)
click.echo(f"The config file is {settings.SETTINGS_FILE}.")
sys.exit(0)
if Path(settings.SETTINGS_FILE).exists():
with open(settings.SETTINGS_FILE, "r") as f:
try:
settings_dict = json.loads(f.read(),
object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError:
logger.error(
f"The config file {settings.SETTINGS_FILE} is not correct, it should be a json object."
)
sys.exit(-2)
if not python_versions:
python_versions = settings_dict.get("python-versions", None)
if python_versions:
click.echo(f"Using `python-versions` in config file.")
if not (platform_tags or whl_suffixes):
platform_tags = settings_dict.get("platform-tags", None)
if platform_tags:
click.echo(f"Using `platform-tags` in config file.")
tz = get_localzone()
if tz.zone in ["Asia/Shanghai", "Asia/Chongqing"]:
index_url = "https://mirrors.aliyun.com/pypi/simple/"
if whl_suffixes:
warnings.warn(
"Option '-s/--suffix' has been deprecated. Please use '-p/--platform-tag' instead."
)
platform_tags = whl_suffixes
if quiet:
logger.setLevel(logging.ERROR)
download = quiet_download
else:
download = normal_download
url_list = []
if not dest_dir:
dest_dir = os.getcwd()
else:
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
# dest_dir = os.path.abspath(dest_dir)
if requirement_file:
packages_extra_dict = pip_api.parse_requirements(requirement_file)
packages_extra = {str(value) for value in packages_extra_dict.values()}
else:
packages_extra = set()
for package in itertools.chain(packages_extra, packages):
with TempDirectory(delete=True) as directory:
logger.info(
"We are using pip download command to download package %s" %
package)
logger.info("-" * 50)
try:
command = [
sys.executable,
"-m",
"pip",
"download",
"-i",
index_url,
"--dest",
directory.path,
package,
]
if quiet:
command.extend(["--progress-bar", "off", "-qqq"])
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
logger.error(
"Sorry, we can not use pip download to download the package %s,"
" and Exception is below" % package)
logger.error(e)
raise
file_names = os.listdir(directory.path)
for file_name in file_names:
python_package = resolve_package_file(file_name)
url_list.append(python_package)
if python_package.name is None:
logger.warning(
"Can not resolve a package's name and version from a downloaded package. You shuold "
"create an issue maybe.")
continue
url = mkurl_pypi_url(index_url, python_package.name)
try:
r = session.get(url)
for file in get_file_links(r.text, url, python_package):
url_list.append(file)
if "none-any" in file:
download(file, dest_dir)
continue
if ".tar.gz" in file or ".zip" in file:
if not no_source:
download(file, dest_dir)
continue
eligible = True
if platform_tags:
for tag in platform_tags:
if tag in file:
eligible = True
break
else:
eligible = False
if not eligible:
continue
if python_versions:
for version in python_versions:
if version in file:
eligible = True
break
else:
eligible = False
if eligible:
download(file, dest_dir)
except ConnectionError as e:
logger.error(
"Can not get information about package %s, and the Exception is below.",
python_package.name,
)
logger.error(e)
raise
logger.info("All packages have been downloaded successfully!")
if show_urls:
logger.setLevel(logging.INFO)
logger.error("List of files downloaded :")
for entry in url_list:
logger.info(entry)
return url_list
def pipdownload(packages, index_url, requirement_file, dest_dir, whl_suffixes,
python_versions):
"""
pip-download is a tool which can be used to download python projects and their dependencies listed on
pypi's `download files` page.
"""
if not dest_dir:
dest_dir = os.getcwd()
# dest_dir = os.path.abspath(dest_dir)
if requirement_file:
packages_extra = {req.strip() for req in requirement_file}
packages_extra.discard('')
else:
packages_extra = set()
for package in itertools.chain(packages_extra, packages):
with TempDirectory(delete=True) as directory:
logger.info(
'We are using pip download command to download package %s' %
package)
logger.info('-' * 50)
try:
subprocess.check_call([
sys.executable, '-m', 'pip', 'download', '-i', index_url,
'--dest', directory.path, package
])
except subprocess.CalledProcessError as e:
logger.error(
'Sorry, we can not use pip download to download the package %s,'
' and Exception is below' % package)
logger.error(e)
raise
logger.info('-' * 50)
logger.info('At First, we copy these files to %s' % dest_dir)
file_names = os.listdir(directory.path)
for file_name in file_names:
shutil.copy(os.path.join(directory.path, file_name), dest_dir)
logger.info('All file have been copied!')
for python_package in resolve_package_files(file_names):
url = mkurl_pypi_url(index_url, python_package.name)
try:
r = session.get(url)
for file in get_file_links(r.text, index_url,
python_package):
if 'tar.gz' in file:
download(file, dest_dir)
continue
if 'none-any' in file:
download(file, dest_dir)
continue
if 'zip' in file:
download(file, dest_dir)
continue
eligible = True
if whl_suffixes:
for suffix in whl_suffixes:
if suffix in file:
eligible = True
break
else:
eligible = False
if not eligible:
continue
if python_versions:
for version in python_versions:
if version in file:
eligible = True
break
else:
eligible = False
if eligible:
download(file, dest_dir)
except ConnectionError as e:
logger.error(
'Can not get information about package %s, and the Exception is below.',
python_package.name)
logger.error(e)
raise
logger.info('All packages have been downloaded successfully!')
def pipdownload(packages, index_url, requirement_file, dest_dir, whl_suffixes,
python_versions, quiet):
"""
pip-download is a tool which can be used to download python projects and their dependencies listed on
pypi's `download files` page.
"""
tz = get_localzone()
if tz.zone in ['Asia/Shanghai', 'Asia/Chongqing']:
index_url = 'https://pypi.tuna.tsinghua.edu.cn/simple'
if quiet:
logger.setLevel(logging.ERROR)
download = quiet_download
else:
download = normal_download
if not dest_dir:
dest_dir = os.getcwd()
else:
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
# dest_dir = os.path.abspath(dest_dir)
if requirement_file:
packages_extra_dict = pip_api.parse_requirements(requirement_file)
packages_extra = {str(value) for value in packages_extra_dict.values()}
else:
packages_extra = set()
for package in itertools.chain(packages_extra, packages):
with TempDirectory(delete=True) as directory:
logger.info(
'We are using pip download command to download package %s' %
package)
logger.info('-' * 50)
try:
command = [
sys.executable, '-m', 'pip', 'download', '-i', index_url,
'--dest', directory.path, package
]
if quiet:
command.extend(['--progress-bar', 'off', '-qqq'])
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
logger.error(
'Sorry, we can not use pip download to download the package %s,'
' and Exception is below' % package)
logger.error(e)
raise
file_names = os.listdir(directory.path)
# if not exact:
# logger.info('-' * 50)
# logger.info('At First, we copy these files to %s' % dest_dir)
# for file_name in file_names:
# shutil.copy(os.path.join(directory.path, file_name), dest_dir)
# logger.info('All file have been copied!')
for python_package in resolve_package_files(file_names):
url = mkurl_pypi_url(index_url, python_package.name)
try:
r = session.get(url)
for file in get_file_links(r.text, index_url,
python_package):
if 'tar.gz' in file:
download(file, dest_dir)
continue
if 'none-any' in file:
download(file, dest_dir)
continue
if 'zip' in file:
download(file, dest_dir)
continue
eligible = True
if whl_suffixes:
for suffix in whl_suffixes:
if suffix in file:
eligible = True
break
else:
eligible = False
if not eligible:
continue
if python_versions:
for version in python_versions:
if version in file:
eligible = True
break
else:
eligible = False
if eligible:
download(file, dest_dir)
except ConnectionError as e:
logger.error(
'Can not get information about package %s, and the Exception is below.',
python_package.name)
logger.error(e)
raise
logger.info('All packages have been downloaded successfully!')