def test_bad_symbol_shadows():
volume = {"shadow_academic.cli.zippy"}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=get_symbol_table_dummy_func)
assert intersection == {}
assert bad == volume
def test_cycle():
volume = {"astropy.A"}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=get_symbol_table_dummy_func)
assert intersection == {
"astropy": {"cchardet/conda-forge/linux-64/cchardet-2.1.1-py27_0"}
}
def test_symbol_doesnt_exist_recursion_error():
volume = {"cchardet._cchardet.detect_with_confidence"}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=get_symbol_table_dummy_func)
assert intersection == {
"cchardet": {"cchardet/conda-forge/linux-64/cchardet-2.1.1-py27_0"}
}
def test_find_supplying_version_null_set():
volume = {
"academic.cli",
"academic.cli.AcademicError",
"academic.cli.clean_bibtex_authors",
}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=lambda x: SAMPLE_TABLE)
assert intersection == {"academic": set()}
def inner_loop(artifact):
symbols = web_interface.get_artifact_symbols(artifact)
if not symbols:
return None
volume = set()
for v in symbols.values():
volume.update(v.get("data", {}).get("symbols_in_volume", set()))
# pull out self symbols, since we assume those are gotten locally and self consistent
volume -= set(symbols)
volume -= set(builtin_symbols)
deps, bad = find_supplying_version_set(volume)
return deps, bad
def main(n_to_pull=1000):
if not os.path.exists("audit"):
os.makedirs("audit")
existing_artifacts = glob.glob("audit/**/*.json", recursive=True)
existing_names = {k.partition("/")[2] for k in existing_artifacts}
existing_pkg_names = {k.partition("/")[0] for k in existing_names}
def not_already_audited(k):
return k.partition("/")[2] not in existing_names
artifacts = [
k for k in requests.get(
"https://raw.githubusercontent.com/symbol-management/ast-symbol-table/master/.file_listing.json"
).json() if k.startswith("symbols") and not_already_audited(k)
]
# Don't have the artifacts in alphabetical order
shuffle(artifacts)
def diff_sort(val):
_, package, channel, arch, name = val.split("/")
return (
package in existing_pkg_names,
sort_arch_ordering.index(arch),
)
for i, artifact in tqdm(enumerate(sorted(artifacts, key=diff_sort)),
total=n_to_pull):
if i >= n_to_pull:
break
print(artifact)
symbols = requests.get(
f"https://raw.githubusercontent.com/symbol-management/ast-symbol-table/master/{artifact}"
).json()
if not symbols:
continue
volume = set()
for v in symbols.values():
volume.update(v.get("symbols_in_volume", set()))
deps, bad = find_supplying_version_set(volume)
dep_sets = [list(sorted(k)) for k in deps]
outname = artifact.replace("symbols/", "audit/")
os.makedirs(os.path.dirname(outname), exist_ok=True)
with open(outname, "w") as f:
json.dump(
{
"deps": dep_sets,
"bad": list(sorted(bad))
},
f,
indent=1,
sort_keys=True,
)
def test_find_supplying_version_set():
volume = {"academic.cli", "academic.cli.AcademicError"}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=lambda x: SAMPLE_TABLE)
assert intersection == {
"academic": {
"academic-0.5.1-py_0",
"academic-0.6.1-py_0",
"academic-0.6.2-py_0",
"academic-0.7.0-py_0",
}
}
def test_find_supplying_version_null_set_shadows():
volume = {
"academic.cli",
"shadow_academic.cli.AcademicError",
"academic.cli.clean_bibtex_authors",
}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=get_symbol_table_dummy_func)
assert intersection == {
"academic": set(),
"shadow_academic": {"shadow_academic-0.0.1", "shadow_academic-0.0.2"}
}
def test_find_supplying_version_set_shadows():
volume = {"shadow_academic.cli", "shadow_academic.cli.AcademicError"}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=get_symbol_table_dummy_func)
assert intersection == {
"shadow_academic": {"shadow_academic-0.0.1", "shadow_academic-0.0.2"},
"academic": {
"academic-0.5.1-py_0",
"academic-0.6.1-py_0",
"academic-0.6.2-py_0",
"academic-0.7.0-py_0",
},
}
def test_find_supplying_version_multi_pkg_set():
volume = {"academic.cli", "academic.cli.AcademicError", "zappy"}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=get_symbol_table_dummy_func)
assert intersection == {
"academic": {
"academic-0.5.1-py_0",
"academic-0.6.1-py_0",
"academic-0.6.2-py_0",
"academic-0.7.0-py_0",
},
"zappy":
{"zappy-0.1.0-py_0", "zappy-0.2.0-py_0", "fork-zappy-0.2.0-py_0"},
}
def test_find_supplying_version_multi_pkg_set():
volume = {"academic.cli", "academic.cli.AcademicError", "zappy"}
intersection, bad = find_supplying_version_set(
volume, get_symbol_table_func=lambda x: SAMPLE_TABLE
)
assert sorted(intersection) == sorted(
[
{
"academic-0.5.1-py_0",
"academic-0.6.1-py_0",
"academic-0.6.2-py_0",
"academic-0.7.0-py_0",
},
{"zappy-0.1.0-py_0", "zappy-0.2.0-py_0"},
]
)
