async def verify( config: ConfigParser, json_file: str, mirror_base_path: Path, all_package_files: List[Path], args: argparse.Namespace, executor: concurrent.futures.ThreadPoolExecutor, releases_key: str = "releases", ) -> None: json_base = mirror_base_path / "web" / "json" json_full_path = json_base / json_file loop = asyncio.get_event_loop() logger.info(f"Parsing {json_file}") if args.json_update: if not args.dry_run: await get_latest_json(json_full_path, config, executor, args.delete) else: logger.info(f"[DRY RUN] Would of grabbed latest json for {json_file}") if not json_full_path.exists(): logger.debug(f"Not trying to sync package as {json_full_path} does not exist") return try: with json_full_path.open("r") as jfp: pkg = json.load(jfp) except json.decoder.JSONDecodeError as jde: logger.error(f"Failed to load {json_full_path}: {jde} - skipping ...") return # apply releases filter plugins like class Package for plugin in filter_release_plugins() or []: plugin.filter(pkg["info"]) for release_version in pkg[releases_key]: for jpkg in pkg[releases_key][release_version]: pkg_file = mirror_base_path / "web" / convert_url_to_path(jpkg["url"]) if not pkg_file.exists(): if args.dry_run: logger.info(f"{jpkg['url']} would be fetched") all_package_files.append(pkg_file) continue else: await url_fetch(jpkg["url"], pkg_file, executor) calc_sha256 = await loop.run_in_executor(executor, hash, str(pkg_file)) if calc_sha256 != jpkg["digests"]["sha256"]: if not args.dry_run: await loop.run_in_executor(None, pkg_file.unlink) await url_fetch(jpkg["url"], pkg_file, executor) else: logger.info( f"[DRY RUN] {jpkg['info']['name']} has a sha256 mismatch." ) all_package_files.append(pkg_file) logger.info(f"Finished validating {json_file}")
def test__filter_no_plugin(self): with open(TEST_CONF, "w") as testconfig_handle: testconfig_handle.write("""\ [plugins] enabled = """) instance = BandersnatchConfig() instance.config_file = TEST_CONF instance.load_configuration() plugins = filter_release_plugins() self.assertEqual(len(plugins), 0) plugins = filter_project_plugins() self.assertEqual(len(plugins), 0)
def test__filter_release_plugins__default__loads(self): with open("test.conf", "w") as testconfig_handle: testconfig_handle.write( """\ [blacklist] """ ) builtin_plugin_names = ["blacklist_release"] instance = BandersnatchConfig() instance.config_file = "test.conf" instance.load_configuration() plugins = filter_release_plugins() names = [plugin.name for plugin in plugins] for name in builtin_plugin_names: self.assertIn(name, names)
def test__filter_release_plugins__loads(self): with open(TEST_CONF, "w") as testconfig_handle: testconfig_handle.write("""\ [blacklist] plugins = all """) builtin_plugin_names = [ "blacklist_release", "prerelease_release", "regex_release", ] instance = BandersnatchConfig() instance.config_file = TEST_CONF instance.load_configuration() plugins = filter_release_plugins() names = [plugin.name for plugin in plugins] for name in builtin_plugin_names: self.assertIn(name, names)
def test__mirror_run__filter_package(self): # Create a configuration that mirrors only the aiida-core package and verify that the created index file # does not contain the links for the releases that are in the safety_db for that package. # The aiida-core rule in the safety_db is currently "aiida-core": [ "<0.12.3"] config = f"""[mirror] directory = {os.getcwd()} json = false master = https://test.pypi.org timeout = 10 workers = 1 hash-index = false stop-on-error = false verifiers = 1 [blacklist] plugins = safety_db_release whitelist_project [whitelist] packages = aiohttp aiida-core """ print(f'Config: \n{config}') with open(TEST_CONF, "w") as testconfig_handle: testconfig_handle.write(config) bandersnatch_config = BandersnatchConfig() bandersnatch_config.config_file = TEST_CONF bandersnatch_config.load_configuration() self._dump_config(bandersnatch_config) print('Release plugins:', [_.name for _ in filter_release_plugins()]) bandersnatch.main.mirror(bandersnatch_config.config) print(os.listdir('.')) self.assertTrue(os.path.exists('web/simple/aiida-core/index.html')) with open('web/simple/aiida-core/index.html') as fh: index = fh.read() self.assertIn('aiida-core-0.12.3.tar.gz', index) self.assertNotIn('aiida-core-0.6.0.1.tar.gz', index) # //NOSONAR