async def verify(
config: ConfigParser,
json_file: str,
mirror_base_path: Path,
all_package_files: List[Path],
args: argparse.Namespace,
executor: concurrent.futures.ThreadPoolExecutor,
releases_key: str = "releases",
) -> None:
json_base = mirror_base_path / "web" / "json"
json_full_path = json_base / json_file
loop = asyncio.get_event_loop()
logger.info(f"Parsing {json_file}")
if args.json_update:
if not args.dry_run:
await get_latest_json(json_full_path, config, executor, args.delete)
else:
logger.info(f"[DRY RUN] Would of grabbed latest json for {json_file}")
if not json_full_path.exists():
logger.debug(f"Not trying to sync package as {json_full_path} does not exist")
return
try:
with json_full_path.open("r") as jfp:
pkg = json.load(jfp)
except json.decoder.JSONDecodeError as jde:
logger.error(f"Failed to load {json_full_path}: {jde} - skipping ...")
return
# apply releases filter plugins like class Package
for plugin in filter_release_plugins() or []:
plugin.filter(pkg["info"])
for release_version in pkg[releases_key]:
for jpkg in pkg[releases_key][release_version]:
pkg_file = mirror_base_path / "web" / convert_url_to_path(jpkg["url"])
if not pkg_file.exists():
if args.dry_run:
logger.info(f"{jpkg['url']} would be fetched")
all_package_files.append(pkg_file)
continue
else:
await url_fetch(jpkg["url"], pkg_file, executor)
calc_sha256 = await loop.run_in_executor(executor, hash, str(pkg_file))
if calc_sha256 != jpkg["digests"]["sha256"]:
if not args.dry_run:
await loop.run_in_executor(None, pkg_file.unlink)
await url_fetch(jpkg["url"], pkg_file, executor)
else:
logger.info(
f"[DRY RUN] {jpkg['info']['name']} has a sha256 mismatch."
)
all_package_files.append(pkg_file)
logger.info(f"Finished validating {json_file}")
def test__filter_no_plugin(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write("""\
[plugins]
enabled =
""")
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
plugins = filter_release_plugins()
self.assertEqual(len(plugins), 0)
plugins = filter_project_plugins()
self.assertEqual(len(plugins), 0)
def test__filter_release_plugins__default__loads(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(
"""\
[blacklist]
"""
)
builtin_plugin_names = ["blacklist_release"]
instance = BandersnatchConfig()
instance.config_file = "test.conf"
instance.load_configuration()
plugins = filter_release_plugins()
names = [plugin.name for plugin in plugins]
for name in builtin_plugin_names:
self.assertIn(name, names)
def test__filter_release_plugins__loads(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write("""\
[blacklist]
plugins = all
""")
builtin_plugin_names = [
"blacklist_release",
"prerelease_release",
"regex_release",
]
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
plugins = filter_release_plugins()
names = [plugin.name for plugin in plugins]
for name in builtin_plugin_names:
self.assertIn(name, names)
def test__mirror_run__filter_package(self):
# Create a configuration that mirrors only the aiida-core package and verify that the created index file
# does not contain the links for the releases that are in the safety_db for that package.
# The aiida-core rule in the safety_db is currently "aiida-core": [ "<0.12.3"]
config = f"""[mirror]
directory = {os.getcwd()}
json = false
master = https://test.pypi.org
timeout = 10
workers = 1
hash-index = false
stop-on-error = false
verifiers = 1
[blacklist]
plugins =
safety_db_release
whitelist_project
[whitelist]
packages =
aiohttp
aiida-core
"""
print(f'Config: \n{config}')
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write(config)
bandersnatch_config = BandersnatchConfig()
bandersnatch_config.config_file = TEST_CONF
bandersnatch_config.load_configuration()
self._dump_config(bandersnatch_config)
print('Release plugins:', [_.name for _ in filter_release_plugins()])
bandersnatch.main.mirror(bandersnatch_config.config)
print(os.listdir('.'))
self.assertTrue(os.path.exists('web/simple/aiida-core/index.html'))
with open('web/simple/aiida-core/index.html') as fh:
index = fh.read()
self.assertIn('aiida-core-0.12.3.tar.gz', index)
self.assertNotIn('aiida-core-0.6.0.1.tar.gz', index) # //NOSONAR