def test__filter_project_blocklist_allowlist__pep503_normalize(self) -> None:
mock_config(
"""\
[plugins]
enabled =
blocklist_project
allowlist_project
[blocklist]
packages =
SampleProject
trove----classifiers
[allowlist]
packages =
SampleProject
trove----classifiers
"""
)
plugins = {
plugin.name: plugin for plugin in LoadedFilters().filter_project_plugins()
}
self.assertTrue(plugins["blocklist_project"].check_match(name="sampleproject"))
self.assertTrue(
plugins["blocklist_project"].check_match(name="trove-classifiers")
)
self.assertFalse(plugins["allowlist_project"].check_match(name="sampleproject"))
self.assertFalse(
plugins["allowlist_project"].check_match(name="trove-classifiers")
)
def test_latest_releases_keep_stable(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
}, # stable version
"releases": {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {}, # <= stable version, keep it
"2.0.1b1": {},
"2.0.1b2": {}, # <= most recent, keep it
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {"2.0.1b2": {}, "2.0.0": {}}
def test__casing__no__affect(self) -> None:
mock_config("""\
[plugins]
enabled =
blocklist_release
[blocklist]
packages =
Foo<=1.2.0
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {
"name": "foo"
},
"releases": {
"1.2.0": {},
"1.2.1": {}
},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.2.1": {}})
def test__filter__matches__release(self) -> None:
with open(Path(self.tempdir.name) / "requirements.txt", "w") as fh:
fh.write(
"""\
# This is needed for workshop 1
#
foo==1.2.0 # via -r requirements.in
"""
)
mock_config(
f"""\
[plugins]
enabled =
project_requirements
project_requirements_pinned
[allowlist]
requirements_path = {self.tempdir.name}
requirements =
requirements.txt
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {"name": "foo"},
"releases": {"1.2.0": {}, "1.2.1": {}},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual({"1.2.0": {}}, pkg.releases)
def test__filter__find_files(self) -> None:
absolute_file_path = Path(self.tempdir.name) / "requirements.txt"
with open(absolute_file_path, "w") as fh:
fh.write(
"""\
# This is needed for workshop 1
#
foo==1.2.0 # via -r requirements.in
"""
)
mock_config(
f"""\
[plugins]
enabled =
project_requirements
[allowlist]
requirements =
{absolute_file_path}
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {
"foo": "",
"bar": "",
"baz": "",
}
mirror._filter_packages()
self.assertEqual({"foo": ""}, mirror.packages_to_sync)
def test__filter__commented__out(self) -> None:
mock_config(
"""\
[mirror]
storage-backend = filesystem
[plugins]
enabled =
allowlist_project
[allowlist]
packages =
foo==1.2.3 # inline comment
# bar
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {
"foo": "",
"bar": "",
"snu": "",
}
mirror._filter_packages()
self.assertEqual({"foo": ""}, mirror.packages_to_sync)
def test__dont__filter__prereleases(self) -> None:
mock_config(
"""\
[plugins]
enabled =
allowlist_release
[allowlist]
packages =
foo<=1.2.0
"""
)
mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1)
pkg._metadata = {
"info": {"name": "foo"},
"releases": {
"1.1.0a2": {},
"1.1.1beta1": {},
"1.2.0": {},
"1.2.1": {},
"1.2.2alpha3": {},
"1.2.3rc1": {},
},
}
pkg.filter_all_releases(mirror.filters.filter_release_plugins())
self.assertEqual(pkg.releases, {"1.1.0a2": {}, "1.1.1beta1": {}, "1.2.0": {}})
def test_latest_releases_uninitialized(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
},
"releases": {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {
"1.0.0": {},
"1.1.0": {},
"1.1.1": {},
"1.1.2": {},
"1.1.3": {},
"2.0.0": {},
}
def test__filter__varying__specifiers(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
[plugins]
enabled =
blocklist_project
[blocklist]
packages =
foo==1.2.3
bar~=3.0,<=1.5
snu
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {
"foo": "",
"foo2": "",
"bar": "",
"snu": "",
}
mirror._filter_packages()
self.assertEqual({
"foo": "",
"foo2": "",
"bar": ""
}, mirror.packages_to_sync)
def test_plugin_compiles_patterns(self) -> None:
mock_config(self.config_contents)
plugins = bandersnatch.filter.LoadedFilters().filter_release_file_plugins()
assert any(
type(plugin) == filename_name.ExcludePlatformFilter for plugin in plugins
)
def test_plugin_check_match(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo-good": "", "foo-evil": "", "foo-neutral": ""}
mirror._filter_packages()
assert list(mirror.packages_to_sync.keys()) == ["foo-good"]
def test__plugin__loads__default(self) -> None:
mock_config("""\
[blocklist]
""")
plugins = bandersnatch.filter.LoadedFilters().filter_project_plugins()
names = [plugin.name for plugin in plugins]
self.assertNotIn("blocklist_project", names)
def test__plugin__doesnt_load__explicitly__disabled(self) -> None:
mock_config("""\
[plugins]
enabled =
allowlist_package
""")
plugins = bandersnatch.filter.LoadedFilters().filter_release_plugins()
names = [plugin.name for plugin in plugins]
self.assertNotIn("allowlist_release", names)
def test__filter_no_plugin(self) -> None:
mock_config("""\
[plugins]
enabled =
""")
plugins = LoadedFilters().filter_release_plugins()
self.assertEqual(len(plugins), 0)
plugins = LoadedFilters().filter_project_plugins()
self.assertEqual(len(plugins), 0)
def test_plugin_compiles_patterns(self) -> None:
mock_config(self.config_contents)
plugins = bandersnatch.filter.LoadedFilters().filter_release_plugins()
assert any(
type(plugin) == latest_name.LatestReleaseFilter
for plugin in plugins)
plugin = next(plugin for plugin in plugins
if isinstance(plugin, latest_name.LatestReleaseFilter))
assert plugin.keep == 2
def test__plugin__loads__explicitly_enabled(self) -> None:
mock_config(contents="""\
[plugins]
enabled =
whitelist_project
""")
plugins = bandersnatch.filter.LoadedFilters().filter_project_plugins()
names = [plugin.name for plugin in plugins]
self.assertListEqual(names, ["whitelist_project"])
self.assertEqual(len(plugins), 1)
def test__plugin__loads__default(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
[plugins]
""")
plugins = bandersnatch.filter.LoadedFilters().filter_project_plugins()
names = [plugin.name for plugin in plugins]
self.assertNotIn("whitelist_project", names)
def test__plugin__loads__explicitly_enabled(self) -> None:
mock_config("""\
[plugins]
enabled =
allowlist_release
""")
plugins = bandersnatch.filter.LoadedFilters().filter_release_plugins()
names = [plugin.name for plugin in plugins]
self.assertListEqual(names, ["allowlist_release"])
self.assertEqual(len(plugins), 1)
def test_plugin_compiles_patterns(self) -> None:
mock_config(self.config_contents)
plugins = bandersnatch.filter.LoadedFilters().filter_project_plugins()
assert any(type(plugin) == regex_name.RegexProjectFilter for plugin in plugins)
plugin = next(
plugin
for plugin in plugins
if isinstance(plugin, regex_name.RegexProjectFilter)
)
assert plugin.patterns == [re.compile(r".+-evil$"), re.compile(r".+-neutral$")]
def test_plugin_compiles_patterns(self) -> None:
mock_config(self.config_contents)
plugins = bandersnatch.filter.LoadedFilters().filter_release_plugins()
assert any(type(plugin) == regex_name.RegexReleaseFilter for plugin in plugins)
plugin = next(
plugin
for plugin in plugins
if isinstance(plugin, regex_name.RegexReleaseFilter)
)
assert plugin.patterns == [re.compile(r".+rc\d$"), re.compile(r".+alpha\d$")]
def test_plugin_init(s3_mock: S3Path) -> None:
config_loader = mock_config("""
[mirror]
directory = /tmp/pypi
json = true
master = https://pypi.org
timeout = 60
global-timeout = 18000
workers = 3
hash-index = true
stop-on-error = true
storage-backend = swift
verifiers = 3
keep_index_versions = 2
compare-method = hash
[s3]
region_name = us-east-1
aws_access_key_id = 123456
aws_secret_access_key = 123456
endpoint_url = http://localhost:9090
signature_version = s3v4
""")
backend = s3.S3Storage(config=config_loader.config)
backend.initialize_plugin()
path = s3.S3Path("/tmp/pypi")
resource, _ = path._accessor.configuration_map.get_configuration(path)
assert resource.meta.client.meta.endpoint_url == "http://localhost:9090"
config_loader = mock_config("""
[mirror]
directory = /tmp/pypi
json = true
master = https://pypi.org
timeout = 60
global-timeout = 18000
workers = 3
hash-index = true
stop-on-error = true
storage-backend = swift
verifiers = 3
keep_index_versions = 2
compare-method = hash
[s3]
endpoint_url = http://localhost:9090
""")
backend = s3.S3Storage(config=config_loader.config)
backend.initialize_plugin()
path = s3.S3Path("/tmp/pypi")
resource, _ = path._accessor.configuration_map.get_configuration(path)
assert resource.meta.client.meta.endpoint_url == "http://localhost:9090"
def test_plugin_check_match(self) -> None:
mock_config(self.config_contents)
mirror = Mirror(Path("."), Master(url="https://foo.bar.com"))
pkg = Package("foo", 1, mirror)
pkg._metadata = {
"info": {"name": "foo", "version": "foo-1.2.0"},
"releases": {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}},
}
pkg._filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg.releases == {"foo-1.2.0": {}}
def test__plugin__doesnt_load__explicitly__disabled(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
workers = 2
[plugins]
enabled =
allowlist_package
""")
plugins = bandersnatch.filter.LoadedFilters().filter_release_plugins()
names = [plugin.name for plugin in plugins]
self.assertNotIn("project_requirements", names)
def test__filter__nomatch_package(self) -> None:
mock_config("""\
[blocklist]
plugins =
blocklist_project
packages =
foo
""")
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
mirror.packages_to_sync = {"foo2": ""}
mirror._filter_packages()
self.assertIn("foo2", mirror.packages_to_sync.keys())
def test__plugin__loads__explicitly_enabled(self) -> None:
mock_config("""\
[mirror]
storage-backend = filesystem
workers = 2
[plugins]
enabled =
project_requirements_pinned
""")
plugins = bandersnatch.filter.LoadedFilters().filter_release_plugins()
names = [plugin.name for plugin in plugins]
self.assertListEqual(names, ["project_requirements_pinned"])
self.assertEqual(len(plugins), 1)
def test__plugin__loads__explicitly_enabled(self) -> None:
mock_config(contents="""\
[mirror]
storage-backend = filesystem
workers = 2
[plugins]
enabled =
allowlist_project
""")
plugins = bandersnatch.filter.LoadedFilters().filter_project_plugins()
names = [plugin.name for plugin in plugins]
self.assertListEqual(names, ["allowlist_project"])
self.assertEqual(len(plugins), 1)
def test_plugin_includes_predefined_patterns(self) -> None:
mock_config(self.config_contents)
plugins = bandersnatch.filter.LoadedFilters().filter_release_plugins()
assert any(
type(plugin) == prerelease_name.PreReleaseFilter
for plugin in plugins)
plugin = next(plugin for plugin in plugins
if isinstance(plugin, prerelease_name.PreReleaseFilter))
expected_patterns = [
re.compile(pattern_string)
for pattern_string in plugin.PRERELEASE_PATTERNS
]
assert plugin.patterns == expected_patterns
def test__filter_project_plugins__loads(self) -> None:
mock_config("""\
[plugins]
enabled = all
""")
builtin_plugin_names = [
"blacklist_project",
"regex_project",
"whitelist_project",
]
plugins = LoadedFilters().filter_project_plugins()
names = [plugin.name for plugin in plugins]
for name in builtin_plugin_names:
self.assertIn(name, names)
def test_latest_releases_ensure_reusable(self) -> None:
"""
Tests the filter multiple times to ensure no state is preserved and
thus is reusable between packages
"""
mock_config(self.config_contents)
mirror = BandersnatchMirror(Path("."),
Master(url="https://foo.bar.com"))
pkg1 = Package("foo", 1)
pkg1._metadata = {
"info": {
"name": "foo",
"version": "2.0.0"
},
"releases": {
"0.1.1": {},
"0.1.2": {},
"0.1.3": {},
"1.0.0": {},
"1.1.0": {},
"1.2.0": {},
"2.0.0": {},
},
}
pkg2 = Package("bar", 1)
pkg2._metadata = {
"info": {
"name": "bar",
"version": "0.3.0"
},
"releases": {
"0.1.0": {},
"0.1.1": {},
"0.1.2": {},
"0.1.3": {},
"0.1.4": {},
"0.1.5": {},
"0.2.0": {},
"0.3.0": {},
},
}
pkg1.filter_all_releases(mirror.filters.filter_release_plugins())
pkg2.filter_all_releases(mirror.filters.filter_release_plugins())
assert pkg1.releases == {"1.2.0": {}, "2.0.0": {}}
assert pkg2.releases == {"0.2.0": {}, "0.3.0": {}}
def test__filter_release_plugins__loads(self) -> None:
mock_config("""\
[plugins]
enabled = all
""")
builtin_plugin_names = [
"blacklist_release",
"prerelease_release",
"regex_release",
"latest_release",
]
plugins = LoadedFilters().filter_release_plugins()
names = [plugin.name for plugin in plugins]
for name in builtin_plugin_names:
self.assertIn(name, names)
