def test_apifail_inventory_dictimport_toobig(self, path_fxn, res_dec):
"""Confirm error raised when JSON dict passed w/too many objects."""
inv = soi.Inventory(path_fxn(res_dec))
d = inv.json_dict()
d.update({"57": d["23"]})
with pytest.raises(ValueError):
soi.Inventory(dict_json=d)
def test_apifail_inventory_dictimport_toosmall(self, path_fxn, res_dec):
"""Confirm error raised when JSON dict passed w/too few objects."""
inv = soi.Inventory(path_fxn(res_dec))
d = inv.json_dict()
d.pop("12")
with pytest.raises(ValueError):
soi.Inventory(d)
def test_apifail_inventory_dictimport_badrootobject(
self, path_fxn, res_dec):
"""Confirm error raised when spurious extra root object present."""
inv = soi.Inventory(path_fxn(res_dec))
d = inv.json_dict()
d.update({"bad_foo": "angry_bar"})
with pytest.raises(ValidationError):
soi.Inventory(dict_json=d)
def test_apifail_inventory_dictimport_badobj(self, bad_key, path_fxn,
res_dec,
jsonschema_validator):
"""Confirm error raised when JSON dict passed w/an invalid object."""
inv = soi.Inventory(path_fxn(res_dec))
d = inv.json_dict()
d.update({bad_key: "foobarbazquux"})
with pytest.raises(ValidationError):
soi.Inventory(dict_json=d)
def test_apifail_inventory_dictimport_missingdataobjmember(
self, res_cmp, jsonschema_validator, path_fxn):
"""Confirm inventory load failure on missing key in data object."""
inv = soi.Inventory(path_fxn(res_cmp))
d = inv.json_dict()
d["0"].pop("domain")
with pytest.raises(ValidationError):
soi.Inventory(dict_json=d)
assert not jsonschema_validator(soi.json_schema).is_valid(d)
def test_apifail_inventory_dictimport_baddataobjmember(
self, res_cmp, jsonschema_validator, path_fxn):
"""Confirm inventory load failure on spurious key in a data object."""
inv = soi.Inventory(path_fxn(res_cmp))
d = inv.json_dict()
d["0"].update({"foo": "bar"})
with pytest.raises(ValidationError):
soi.Inventory(dict_json=d)
assert not jsonschema_validator(soi.json_schema).is_valid(d)
def test_apifail_inventory_no_object_invs(self, check):
"""Confirm no-objects inventories don't import."""
inv = soi.Inventory()
with pytest.raises(TypeError):
soi.Inventory(inv.data_file())
with pytest.raises((TypeError, ValueError)):
soi.Inventory(soi.compress(inv.data_file()))
d = {"project": "test", "version": "0.0", "count": 0}
with pytest.raises(ValueError):
soi.Inventory(d)
def test_sphinx_import_complete_url(workdir):
"""Ensure that if the importer is called with a complete objects.inv url it is used
as is.
"""
filepath = str(pathlib.Path(workdir.name, "sphinx.db"))
inv = soi.Inventory()
inv.project = "Example"
inv.version = "1.0"
inv.objects.append(
soi.DataObjStr(
name="foo",
domain="py",
priority="1",
role="function",
uri="foo.html",
dispname="-",
))
with mock.patch("llyfrau.importers.soi.Inventory",
return_value=inv) as m_inv:
sphinx(filepath, "https://docs.python.org/3/objects.inv")
m_inv.assert_called_with(url="https://docs.python.org/3/objects.inv")
def test_apifail_error_decompressing_plaintext(self, path_fxn, res_dec):
"""Confirm error raised on attempt to decompress plaintext."""
# OS-dependent. VersionError on Windows b/c bytes import of the
# text-file objects_attrs.txt has the first line ending with b'2\r\n',
# whereas *nix will pass the version check but choke in the zlib
# decompression process
with pytest.raises((zlib_error, soi.VersionError)):
soi.Inventory(fname_zlib=path_fxn(res_dec))
def test_apifail_inventory_dictimport_noitems(self):
"""Confirm ValueError with no-items dict passed to json_dict."""
d = {
soi.HeaderFields.Project.value: "proj",
soi.HeaderFields.Version.value: "v3.3",
soi.HeaderFields.Count.value: 5,
}
with pytest.raises(ValueError):
soi.Inventory()._import_json_dict(d)
def test_apifail_changing_immutable_dataobj(self, path_fxn, use_bytes,
res_cmp):
"""Confirm DataObj's are immutable."""
inv = soi.Inventory(path_fxn(res_cmp))
with pytest.raises(FrozenInstanceError): # pragma: no cover (XFAIL)
if use_bytes:
inv.objects[0].as_bytes.name = b"newname"
else:
inv.objects[0].name = "newname"
def main():
import argparse
parser = argparse.ArgumentParser(usage=__doc__)
parser.add_argument("--input",
dest="filename",
required=False,
help="sphinx inventory file (objects.inv)",
metavar="FILE",
type=lambda x: is_valid_file(parser, x))
parser.add_argument(
"--output",
dest="output",
default="../../../release/scripts/modules/rna_manual_reference.py",
required=False,
help="path to output including filename and extentsion",
metavar="FILE")
parser.add_argument(
"--url",
dest="url",
default="https://docs.blender.org/manual/en/dev/objects.inv",
required=False,
help="url to sphinx inventory file (objects.inv)",
metavar="FILE")
args = parser.parse_args()
if args.filename:
# Download and decode objects.inv
print("Loading from file...")
inv = soi.Inventory(args.filename)
else:
# Load and decode objects.inv
print("Downloading...")
inv = soi.Inventory(url=args.url)
write_mappings(inv, args.output)
print("Done!")
def current_version(build_dir):
"""Return the current version of an objects.inv file"""
objects_file = build_dir.joinpath("objects.inv")
inventory = sphobjinv.Inventory(objects_file)
if not inventory.version:
raise RuntimeError(
f"The versjon tool requires a version number "
"in the {objects_file.parent}. Add one to conf.py or pass "
'it to sphinx build using "-D version=X.Y.Z".')
return inventory.version
def _get_doc_url(obj_name):
doc_base_url = 'https://lisa-linux-integrated-system-analysis.readthedocs.io/en/master/'
# Use the inventory built by RTD
inv_url = urllib.parse.urljoin(doc_base_url, 'objects.inv')
inv = sphobjinv.Inventory(url=inv_url)
for inv_obj in inv.objects:
if inv_obj.name == obj_name:
doc_page = inv_obj.uri.replace('$', inv_obj.name)
doc_url = urllib.parse.urljoin(doc_base_url, doc_page)
return doc_url
raise ValueError('Could not find the doc of: {}'.format(obj_name))
def test_api_inventory_many_url_imports(
testall_inv_path,
res_path,
scratch_path,
misc_info,
sphinx_load_test,
pytestconfig,
):
"""Confirm a plethora of .inv files downloads properly via url arg.
This test is SLOW, and so does not run by default. Invoke with `--nonloc`
to run it; invoke with `--testall` to test over all .inv files in
tests/resource.
"""
fname = testall_inv_path.name
scr_fpath = scratch_path / fname
# Drop most unless testall
if not pytestconfig.getoption("--testall") and fname != "objects_attrs.inv":
pytest.skip("'--testall' not specified")
# Construct inventories for comparison
mch = misc_info.p_inv.match(fname)
proj_name = mch.group(1)
inv1 = soi.Inventory(str(res_path / fname))
inv2 = soi.Inventory(url=misc_info.remote_url.format(proj_name))
# Test the things
assert inv1 == inv2
# Ensure sphinx likes the regenerated inventory
data = inv2.data_file()
cmp_data = soi.compress(data)
soi.writebytes(scr_fpath, cmp_data)
sphinx_load_test(scr_fpath)
def main():
inv = soi.Inventory()
inv.project = "TensorFlow-Probability"
inv.version = "2.0"
with open("./tfp_docs_scraper/core_symbols.json") as fp:
symbols = json.load(fp)
for symbol in symbols:
add_to_inventory(inv, symbol)
text = inv.data_file(contract=True)
ztext = soi.compress(text)
soi.writebytes("objects.inv", ztext)
print("All done!")
def test_sphinx_import(workdir):
"""Ensure that the sphinx importer imports links correctly"""
filepath = str(pathlib.Path(workdir.name, "sphinx.db"))
inv = soi.Inventory()
inv.project = "Python"
inv.version = "1.0"
inv.objects.append(
soi.DataObjStr(
name="print",
domain="py",
priority="1",
role="function",
uri="builtins.html#$",
dispname="-",
))
inv.objects.append(
soi.DataObjStr(
name="enumeration",
domain="py",
priority="1",
role="label",
uri="concepts.html#$",
dispname="Enumeration",
))
with mock.patch("llyfrau.importers.soi.Inventory",
return_value=inv) as m_inv:
sphinx(filepath, "https://docs.python.org/")
m_inv.assert_called_with(url="https://docs.python.org/objects.inv")
db = Database(filepath)
source = Source.search(db, name="Python v1.0")[0]
assert source.name == "Python v1.0 Documentation"
assert source.prefix == "https://docs.python.org/"
links = Link.search(db, source=source)
assert links[0].name == "print"
assert links[0].url == "builtins.html#print"
assert {t.name for t in links[0].tags} == {"sphinx", "py", "function"}
assert links[1].name == "Enumeration"
assert links[1].url == "concepts.html#enumeration"
assert {t.name for t in links[1].tags} == {"sphinx", "py", "label"}
def main(
project="TensorFlow",
version="2.0",
symbols_file="./tf_docs_scraper/core_symbols.json",
common_url="https://www.tensorflow.org/api_docs/python/",
inv_file="tf2_py_objects.inv",
):
inv = soi.Inventory()
inv.project = project
inv.version = version
with open(symbols_file) as fp:
symbols = json.load(fp)
for symbol in symbols:
add_to_inventory(inv, symbol, common_url)
text = inv.data_file(contract=True)
ztext = soi.compress(text)
soi.writebytes(inv_file, ztext)
print("All done!")
def sphinx(uri: str, collection: Collection):
"""Import links from a sphinx documentation site."""
if uri.endswith("/"):
uri = f"{uri}objects.inv"
if not uri.endswith("objects.inv"):
uri = f"{uri}/objects.inv"
print(f"Trying index url : {uri}", end="\r")
inv = soi.Inventory(url=uri)
print(f"Found object index: {uri} with {inv.count} entries")
collection.name = f"{inv.project} v{inv.version} Documentation"
collection.prefix = uri.replace("objects.inv", "")
for item in inv.objects:
name = item.dispname_expanded
url = item.uri_expanded
tags = [t for t in [item.domain, item.role] if t is not None]
collection.add_link(name=name, url=url, tags=tags)
def test_mapping_objects_inv(mapping_tuple):
"""Check docs objects.inv link validity."""
inv_link = mapping_tuple[1] or (mapping_tuple[0].removesuffix("/") +
"/objects.inv")
inv = soi.Inventory(url=inv_link)
log_append(f"{repr(inv)}\n")
# Interestingly, (some?) module objects don't craft accurate URIs?
data_obj = [
obj for obj in dropwhile(lambda o: o.role == "module", inv.objects)
][0]
obj_link = mapping_tuple[0] + data_obj.uri_expanded
obj_resp = rq.get(obj_link)
log_append(f"{obj_link}: {obj_resp.status_code} {obj_resp.reason}\n")
assert obj_resp.ok
soup = BSoup(obj_resp.text, "html.parser")
if anchor := obj_link.partition("#")[2]:
anchor_check = soup.find_all("dt", id=anchor)
log_append(f"Anchor '{anchor}' found? {bool(anchor_check)}\n\n")
assert anchor_check
def docs_inventory(self) -> sphobjinv.Inventory:
return sphobjinv.Inventory(url=self.docs_url + "objects.inv")
shutil.rmtree("api", ignore_errors=True)
subprocess.call(
" ".join([
"sphinx-apidoc",
f"../src/{package}/",
"-o api/",
"--force",
"--no-toc",
"--templatedir _templates",
"--separate",
]),
shell=True,
)
# -- Convert sphinx object inventory -----------------------------------------
inv = soi.Inventory()
inv.project = "constraint"
constraint_object_names = [
"Constraint",
"Domain",
"Problem",
"Solver",
"Variable",
]
for object_name in constraint_object_names:
inv.objects.append(
soi.DataObjStr(
name=f"{inv.project}.{object_name}",
domain="py",
role="class",
# manual/ --> manual/ru/
#
# The table in the script, contains all of the languages we have in the
# Blender manual website, for those other languages that still
# does not have a team of translators,
# and/or don't have a manual for their languages we commented the lines below,
# you should add them to the language table when they have a proper manual,
# or added to the Blender UI translation table.
#
# URL is the: url_manual_prefix + url_manual_mapping[#id]
import os
import sphobjinv as soi
# Download and decode objects.inv
inv = soi.Inventory(url="https://docs.blender.org/manual/en/dev/objects.inv")
# Write the fire
filepath = os.path.join("rna_manual_reference.py")
file = open(filepath, "w", encoding="utf-8")
fw = file.write
fw("# Do not edit this file.")
fw(" This file is auto generated from rna_manual_reference_updater.py\n\n")
fw("import bpy\n\n")
fw("if bpy.app.version_cycle in {'rc', 'release'}:\n")
fw(" manual_version = '%d.%d' % bpy.app.version[:2]\n")
fw("else:\n")
fw(" manual_version = 'dev'\n\n")
fw("url_manual_prefix = \"https://docs.blender.org/manual/en/\" + manual_version + \"/\"\n\n"
)
def test_api_inventory_known_header_required(name, url):
"""Confirm URL load works on docs pages requiring HTTP header config."""
inv = soi.Inventory(url=url)
assert inv.count > 0
def inventory_from_index(project, version, index_json):
inv = soi.Inventory()
inv.project = project
inv.version = version
inv.objects.extend(objects_from_index(index_json))
return inv
# or added to the Blender UI translation table.
#
# URL is the: url_manual_prefix + url_manual_mapping[#id]
import os
import sphobjinv as soi
import urllib.request
# Download the objects.inv file
urlretrieve = urllib.request.urlretrieve
urlretrieve("https://docs.blender.org/manual/en/dev/objects.inv",
"objects.inv")
# Decode objects.inv
inv = soi.Inventory('objects.inv')
objects_data = inv.data_file()
soi.writebytes('objects.tmp', objects_data) # TODO leave in memory
os.remove("objects.inv")
# Write the fire
filepath = os.path.join("rna_manual_reference.py")
file = open(filepath, "w", encoding="utf-8")
fw = file.write
fw("# Do not edit this file.")
fw(" This file is auto generated from rna_manual_reference_updater.py\n\n")
fw("import bpy\n\n")
fw("if bpy.app.version_cycle in {'rc', 'release'}:\n")
fw(" manual_version = '%d.%d' % bpy.app.version[:2]\n")
fw("else:\n")
def test_apifail_inventory_invalidsource(self):
"""Confirm error raised when invalid source provided."""
with pytest.raises(TypeError):
soi.Inventory("abcdefg")
def test_apifail_inventory_dictimport_toomanysrcargs(self, ):
"""Confirm error if >1 sources passed."""
with pytest.raises(RuntimeError):
soi.Inventory(source="foo", plaintext="bar")
def test_apifail_invalid_inventory_init_arg(self, bad_arg):
"""Confirm non-__init__ Inventory members raise exceptions when passed."""
with pytest.raises(TypeError):
soi.Inventory(**{bad_arg: "foo"})
