async def get_package_from_pypi(package_name, plugin_path):
"""
Download a package from PyPI.
:param name: name of the package to download from PyPI
:return: String path to the package
"""
config = BandersnatchConfig().config
config["mirror"]["master"] = "https://pypi.org"
config["mirror"]["workers"] = "1"
config["mirror"]["directory"] = plugin_path
if not config.has_section("plugins"):
config.add_section("plugins")
config["plugins"]["enabled"] = "blocklist_release\n"
if not config.has_section("allowlist"):
config.add_section("allowlist")
config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n"
config["allowlist"]["packages"] = "\n".join([package_name])
os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True)
async with Master("https://pypi.org/") as master:
mirror = BandersnatchMirror(homedir=plugin_path, master=master)
name = Requirement(package_name).name
result = await mirror.synchronize([name])
package_found = False
for package in result[name]:
current_path = os.path.join(plugin_path, package)
destination_path = os.path.join(plugin_path, "dist",
os.path.basename(package))
shutil.move(current_path, destination_path)
package_found = True
return package_found
def test__plugin__loads__explicitly_enabled(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write("""\
[blacklist]
plugins =
blacklist_release
""")
instance = BandersnatchConfig()
instance.config_file = "test.conf"
instance.load_configuration()
plugins = bandersnatch.filter.filter_release_plugins()
names = [plugin.name for plugin in plugins]
self.assertListEqual(names, ["blacklist_release"])
self.assertEqual(len(plugins), 1)
def test__plugin__loads__explicitly_enabled(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write("""\
[plugins]
enabled =
whitelist_project
""")
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
plugins = bandersnatch.filter.filter_project_plugins()
names = [plugin.name for plugin in plugins]
self.assertListEqual(names, ["whitelist_project"])
self.assertEqual(len(plugins), 1)
async def async_verify(
config: ConfigParser,
all_package_files: List[Path],
mirror_base_path: Path,
json_files: List[str],
args: argparse.Namespace,
executor: concurrent.futures.ThreadPoolExecutor,
) -> None:
queue = asyncio.Queue() # type: Queue
for jf in json_files:
queue.put_nowait(jf)
async def consume(q: Queue) -> None:
while not q.empty():
json_file = q.get_nowait()
await verify(
config, json_file, mirror_base_path, all_package_files, args, executor
)
# TODO: See if we can use passed in config
config = BandersnatchConfig().config
verifiers = config.getint("mirror", "verifiers", fallback=3)
consumers = [consume(queue)] * verifiers
await asyncio.gather(*consumers)
def test__plugin__doesnt_load__explicitly__disabled(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write(
"""\
[plugins]
enabled =
blacklist_release
"""
)
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
plugins = bandersnatch.filter.filter_project_plugins()
names = [plugin.name for plugin in plugins]
self.assertNotIn("blacklist_project", names)
def test__filter__pyversions(self):
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
with open(TEST_JSON) as json_file:
package_json = json.load(json_file)
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.info = package_json["info"]
pkg.releases = package_json["releases"]
pkg._filter_releases()
self.assertEqual(set([v['python_version'] for v in pkg.releases["0.4.3"]]), set(['cp35', 'cp36']))
def test__filter_no_plugin(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write("""\
[plugins]
enabled =
""")
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
plugins = filter_release_plugins()
self.assertEqual(len(plugins), 0)
plugins = filter_project_plugins()
self.assertEqual(len(plugins), 0)
def test_single_config__default__all_sections_present(self) -> None:
with importlib.resources.path( # type: ignore
"bandersnatch", "unittest.conf") as config_file:
instance = BandersnatchConfig(str(config_file))
# All default values should at least be present and be the write types
for section in ["mirror", "plugins", "blocklist"]:
self.assertIn(section, instance.config.sections())
def test__filter_release_plugins__default__loads(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(
"""\
[blacklist]
"""
)
builtin_plugin_names = ["blacklist_release"]
instance = BandersnatchConfig()
instance.config_file = "test.conf"
instance.load_configuration()
plugins = filter_release_plugins()
names = [plugin.name for plugin in plugins]
for name in builtin_plugin_names:
self.assertIn(name, names)
async def async_verify(config, all_package_files, mirror_base, json_files,
args, executor) -> None:
queue = asyncio.Queue() # type: Queue
for jf in json_files:
queue.put_nowait(jf)
async def consume(q: Queue):
while not q.empty():
json_file = q.get_nowait()
await verify(config, json_file, mirror_base, all_package_files,
args, executor)
config = BandersnatchConfig().config
verifiers = config.getint("mirror", "verifiers", fallback=3)
consumers = [consume(queue)] * verifiers
await asyncio.gather(*consumers)
def test__filter__nomatch_package(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write("""\
[blacklist]
plugins =
blacklist_project
packages =
foo
""")
instance = BandersnatchConfig()
instance.config_file = "test.conf"
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo2": {}}
mirror._filter_packages()
self.assertIn("foo2", mirror.packages_to_sync.keys())
def test__filter_project_plugins__loads(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write("""\
[plugins]
enabled = all
""")
builtin_plugin_names = [
"blacklist_project",
"regex_project",
"whitelist_project",
]
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
plugins = filter_project_plugins()
names = [plugin.name for plugin in plugins]
for name in builtin_plugin_names:
self.assertIn(name, names)
def test__mirror_run__filter_package(self):
# Create a configuration that mirrors only the aiida-core package and verify that the created index file
# does not contain the links for the releases that are in the safety_db for that package.
# The aiida-core rule in the safety_db is currently "aiida-core": [ "<0.12.3"]
config = f"""[mirror]
directory = {os.getcwd()}
json = false
master = https://test.pypi.org
timeout = 10
workers = 1
hash-index = false
stop-on-error = false
verifiers = 1
[blacklist]
plugins =
safety_db_release
whitelist_project
[whitelist]
packages =
aiohttp
aiida-core
"""
print(f'Config: \n{config}')
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write(config)
bandersnatch_config = BandersnatchConfig()
bandersnatch_config.config_file = TEST_CONF
bandersnatch_config.load_configuration()
self._dump_config(bandersnatch_config)
print('Release plugins:', [_.name for _ in filter_release_plugins()])
bandersnatch.main.mirror(bandersnatch_config.config)
print(os.listdir('.'))
self.assertTrue(os.path.exists('web/simple/aiida-core/index.html'))
with open('web/simple/aiida-core/index.html') as fh:
index = fh.read()
self.assertIn('aiida-core-0.12.3.tar.gz', index)
self.assertNotIn('aiida-core-0.6.0.1.tar.gz', index) # //NOSONAR
def test__filter__matches__package(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write("""\
[plugins]
enabled =
whitelist_project
[whitelist]
packages =
foo
""")
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo": {}}
mirror._filter_packages()
self.assertIn("foo", mirror.packages_to_sync.keys())
def test__filter__matches__release(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write("""\
[blacklist]
plugins =
blacklist_release
packages =
foo==1.2.0
""")
instance = BandersnatchConfig()
instance.config_file = "test.conf"
instance.load_configuration()
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg.releases = {"1.2.0": {}, "1.2.1": {}}
pkg._filter_releases()
self.assertEqual(pkg.releases, {"1.2.1": {}})
def test__filter_release_plugins__loads(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write("""\
[blacklist]
plugins = all
""")
builtin_plugin_names = [
"blacklist_release",
"prerelease_release",
"regex_release",
"exclude_platform",
"latest_release",
]
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
plugins = filter_release_plugins()
names = [plugin.name for plugin in plugins]
for name in builtin_plugin_names:
self.assertIn(name, names)
def test__filter__matches__release(self):
with open(TEST_CONF, "w") as testconfig_handle:
testconfig_handle.write(
"""\
[blacklist]
plugins =
safety_db_release
"""
)
instance = BandersnatchConfig()
instance.config_file = TEST_CONF
instance.load_configuration()
self._dump_config(instance)
mirror = Mirror(".", Master(url="https://foo.bar.com"))
pkg = Package("aiohttp", 1, mirror)
pkg.info = {"name": "aiohttp"}
pkg.releases = {"0.16.3": {}, "0.16.0": {}, "0.15.1": {}}
pkg._filter_releases()
self.assertEqual(pkg.releases, {"0.16.3": {}})
def test_multiple_instances_custom_setting_str(self):
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write("[mirror]\nmaster=https://foo.bar.baz\n")
instance1 = BandersnatchConfig()
instance1.config_file = "test.conf"
instance1.load_configuration()
instance2 = BandersnatchConfig()
self.assertEqual(instance2.config["mirror"]["master"], "https://foo.bar.baz")
def test_single_config__default__mirror__setting_attributes(self):
instance = BandersnatchConfig()
options = [option for option in instance.config["mirror"]]
options.sort()
self.assertListEqual(
options,
[
"directory",
"hash-index",
"json",
"master",
"stop-on-error",
"timeout",
"verifiers",
"workers",
],
)
def test_single_config__default__mirror__setting__types(self):
"""
Make sure all default mirror settings will cast to the correct types
"""
instance = BandersnatchConfig()
for option, option_type in [
("directory", str),
("hash-index", bool),
("json", bool),
("master", str),
("stop-on-error", bool),
("timeout", int),
("workers", int),
]:
self.assertIsInstance(
option_type(instance.config["mirror"].get(option)), option_type
)
def from_bandersnatch(self, request):
"""
<!-- User-facing documentation, rendered as html-->
Takes the fields specified in the Bandersnatch config and creates a Python Remote from
it.
"""
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
bander_config_file = serializer.validated_data.get("config")
name = serializer.validated_data.get("name")
policy = serializer.validated_data.get("policy")
bander_config = BandersnatchConfig(bander_config_file.file.name).config
data = {
"name": name,
"policy": policy,
"url": bander_config.get("mirror", "master"),
"download_concurrency": bander_config.get("mirror", "workers"),
}
enabled = bander_config.get("plugins", "enabled")
enabled_all = "all" in enabled
data["prereleases"] = not (enabled_all
or "prerelease_release" in enabled)
if bander_config.has_option("allowlist", "packages") and \
(enabled_all or "allowlist_project" in enabled):
data["includes"] = bander_config.get("allowlist",
"packages").split()
if bander_config.has_option("blocklist", "packages") and \
(enabled_all or "blocklist_project" in enabled):
data["excludes"] = bander_config.get("blocklist",
"packages").split()
remote = python_serializers.PythonRemoteSerializer(
data=data, context={"request": request})
remote.is_valid(raise_exception=True)
remote.save()
headers = self.get_success_headers(remote.data)
return Response(remote.data,
status=status.HTTP_201_CREATED,
headers=headers)
def from_bandersnatch(self, request):
"""
<!-- User-facing documentation, rendered as html-->
Takes the fields specified in the Bandersnatch config and creates a Python Remote from
it.
"""
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
bander_config_file = serializer.validated_data.get("config")
name = serializer.validated_data.get("name")
policy = serializer.validated_data.get("policy")
bander_config = BandersnatchConfig(bander_config_file.file.name).config
data = {
"name": name,
"policy": policy,
"url": bander_config.get("mirror", "master"),
"download_concurrency": bander_config.get("mirror", "workers"),
}
enabled = bander_config.get("plugins", "enabled")
enabled_all = "all" in enabled
data["prereleases"] = not (enabled_all
or "prerelease_release" in enabled)
# TODO refactor to use a translation object
plugin_filters = { # plugin : (section_name, bander_option, pulp_option)
"allowlist_project": ("allowlist", "packages", "includes"),
"blocklist_project": ("blocklist", "packages", "excludes"),
"regex_release_file_metadata": (
"regex_release_file_metadata",
"any:release_file.packagetype",
"package_types",
),
"latest_release": ("latest_release", "keep", "keep_latest_packages"),
"exclude_platform": ("blocklist", "platforms", "exclude_platforms"),
}
for plugin, options in plugin_filters.items():
if (enabled_all or plugin in enabled) and \
bander_config.has_option(options[0], options[1]):
data[options[2]] = bander_config.get(options[0],
options[1]).split()
remote = python_serializers.PythonRemoteSerializer(
data=data, context={"request": request})
remote.is_valid(raise_exception=True)
remote.save()
headers = self.get_success_headers(remote.data)
return Response(remote.data,
status=status.HTTP_201_CREATED,
headers=headers)
def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir):
"""
Package lines with a PEP440 spec on them should not be filtered from the
list of packages.
"""
test_configuration = """\
[blacklist]
packages =
example3>2.0.0
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
for plugin in filter_project_plugins():
plugin.initialize_plugin()
m = Mirror(str(tmpdir), mock.Mock())
m.packages_to_sync = {"example1": None, "example3": None}
m._filter_packages()
assert "example3" in m.packages_to_sync.keys()
def test_mirror_filter_packages_match(tmpdir):
"""
Packages that exist in the blacklist should be removed from the list of
packages to sync.
"""
test_configuration = """\
[blacklist]
packages =
example1
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
for plugin in filter_project_plugins():
plugin.initialize_plugin()
m = Mirror(str(tmpdir), mock.Mock())
m.packages_to_sync = {"example1": None, "example2": None}
m._filter_packages()
assert "example1" not in m.packages_to_sync.keys()
def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir: Path) -> None:
"""
Package lines with a PEP440 spec on them should not be filtered from the
list of packages.
"""
test_configuration = """\
[plugins]
enable =
blocklist_project
[blocklist]
packages =
example3>2.0.0
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
m = BandersnatchMirror(tmpdir, mock.Mock())
m.packages_to_sync = {"example1": "", "example3": ""}
m._filter_packages()
assert "example3" in m.packages_to_sync.keys()
def test_mirror_filter_packages_match(tmpdir: Path) -> None:
"""
Packages that exist in the blocklist should be removed from the list of
packages to sync.
"""
test_configuration = """\
[plugins]
enabled =
blocklist_project
[blocklist]
packages =
example1
"""
Singleton._instances = {}
with open("test.conf", "w") as testconfig_handle:
testconfig_handle.write(test_configuration)
BandersnatchConfig("test.conf")
m = BandersnatchMirror(tmpdir, mock.Mock())
m.packages_to_sync = {"example1": "", "example2": ""}
m._filter_packages()
assert "example1" not in m.packages_to_sync.keys()
def test_single_config__default__mirror__setting_attributes(self) -> None:
instance = BandersnatchConfig()
options = [option for option in instance.config["mirror"]]
options.sort()
self.assertListEqual(
options,
[
"cleanup",
"directory",
"global-timeout",
"hash-index",
"json",
"master",
"release-files",
"stop-on-error",
"storage-backend",
"timeout",
"verifiers",
"workers",
],
)
def test_deprecation_warning_raised(self) -> None:
# Remove in 5.0 once we deprecate whitelist/blacklist
config_file = "test.conf"
instance = BandersnatchConfig()
instance.config_file = config_file
# Test no warning if new plugins used
with open(config_file, "w") as f:
f.write("[allowlist]\npackages=foo\n")
instance.load_configuration()
with warnings.catch_warnings(record=True) as w:
instance.check_for_deprecations()
self.assertEqual(len(w), 0)
# Test warning if old plugins used
instance.SHOWN_DEPRECATIONS = False
with open(config_file, "w") as f:
f.write("[whitelist]\npackages=foo\n")
instance.load_configuration()
with warnings.catch_warnings(record=True) as w:
instance.check_for_deprecations()
instance.check_for_deprecations()
# Assert we only throw 1 warning
self.assertEqual(len(w), 1)
def create_bandersnatch_config(remote):
"""Modifies the global Bandersnatch config state for this sync"""
config = BandersnatchConfig().config
config["mirror"]["master"] = remote.url
config["mirror"]["workers"] = str(remote.download_concurrency)
if not config.has_section("plugins"):
config.add_section("plugins")
config["plugins"]["enabled"] = "blocklist_release\n"
if remote.includes:
if not config.has_section("allowlist"):
config.add_section("allowlist")
config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n"
config["allowlist"]["packages"] = "\n".join(remote.includes)
if remote.excludes:
if not config.has_section("blocklist"):
config.add_section("blocklist")
config["plugins"]["enabled"] += "blocklist_project\n"
config["blocklist"]["packages"] = "\n".join(remote.excludes)
if not remote.prereleases:
config["plugins"]["enabled"] += "prerelease_release\n"
if remote.package_types:
rrfm = "regex_release_file_metadata"
config["plugins"]["enabled"] += rrfm
if not config.has_section(rrfm):
config.add_section(rrfm)
config[rrfm]["any:release_file.packagetype"] = "\n".join(remote.package_types)
if remote.keep_latest_packages:
config["plugins"]["enabled"] += "latest_release\n"
if not config.has_section("latest_release"):
config.add_section("latest_release")
config["latest_release"]["keep"] = str(remote.keep_latest_packages)
if remote.exclude_platforms:
config["plugins"]["enabled"] += "exclude_platform\n"
if not config.has_section("blocklist"):
config.add_section("blocklist")
config["blocklist"]["platforms"] = "\n".join(remote.exclude_platforms)
def test_is_singleton(self) -> None:
instance1 = BandersnatchConfig()
instance2 = BandersnatchConfig()
self.assertEqual(id(instance1), id(instance2))
