def test_apifail_inventory_dictimport_toobig(self, path_fxn, res_dec): """Confirm error raised when JSON dict passed w/too many objects.""" inv = soi.Inventory(path_fxn(res_dec)) d = inv.json_dict() d.update({"57": d["23"]}) with pytest.raises(ValueError): soi.Inventory(dict_json=d)
def test_apifail_inventory_dictimport_toosmall(self, path_fxn, res_dec): """Confirm error raised when JSON dict passed w/too few objects.""" inv = soi.Inventory(path_fxn(res_dec)) d = inv.json_dict() d.pop("12") with pytest.raises(ValueError): soi.Inventory(d)
def test_apifail_inventory_dictimport_badrootobject( self, path_fxn, res_dec): """Confirm error raised when spurious extra root object present.""" inv = soi.Inventory(path_fxn(res_dec)) d = inv.json_dict() d.update({"bad_foo": "angry_bar"}) with pytest.raises(ValidationError): soi.Inventory(dict_json=d)
def test_apifail_inventory_dictimport_badobj(self, bad_key, path_fxn, res_dec, jsonschema_validator): """Confirm error raised when JSON dict passed w/an invalid object.""" inv = soi.Inventory(path_fxn(res_dec)) d = inv.json_dict() d.update({bad_key: "foobarbazquux"}) with pytest.raises(ValidationError): soi.Inventory(dict_json=d)
def test_apifail_inventory_dictimport_missingdataobjmember( self, res_cmp, jsonschema_validator, path_fxn): """Confirm inventory load failure on missing key in data object.""" inv = soi.Inventory(path_fxn(res_cmp)) d = inv.json_dict() d["0"].pop("domain") with pytest.raises(ValidationError): soi.Inventory(dict_json=d) assert not jsonschema_validator(soi.json_schema).is_valid(d)
def test_apifail_inventory_dictimport_baddataobjmember( self, res_cmp, jsonschema_validator, path_fxn): """Confirm inventory load failure on spurious key in a data object.""" inv = soi.Inventory(path_fxn(res_cmp)) d = inv.json_dict() d["0"].update({"foo": "bar"}) with pytest.raises(ValidationError): soi.Inventory(dict_json=d) assert not jsonschema_validator(soi.json_schema).is_valid(d)
def test_apifail_inventory_no_object_invs(self, check): """Confirm no-objects inventories don't import.""" inv = soi.Inventory() with pytest.raises(TypeError): soi.Inventory(inv.data_file()) with pytest.raises((TypeError, ValueError)): soi.Inventory(soi.compress(inv.data_file())) d = {"project": "test", "version": "0.0", "count": 0} with pytest.raises(ValueError): soi.Inventory(d)
def test_sphinx_import_complete_url(workdir): """Ensure that if the importer is called with a complete objects.inv url it is used as is. """ filepath = str(pathlib.Path(workdir.name, "sphinx.db")) inv = soi.Inventory() inv.project = "Example" inv.version = "1.0" inv.objects.append( soi.DataObjStr( name="foo", domain="py", priority="1", role="function", uri="foo.html", dispname="-", )) with mock.patch("llyfrau.importers.soi.Inventory", return_value=inv) as m_inv: sphinx(filepath, "https://docs.python.org/3/objects.inv") m_inv.assert_called_with(url="https://docs.python.org/3/objects.inv")
def test_apifail_error_decompressing_plaintext(self, path_fxn, res_dec): """Confirm error raised on attempt to decompress plaintext.""" # OS-dependent. VersionError on Windows b/c bytes import of the # text-file objects_attrs.txt has the first line ending with b'2\r\n', # whereas *nix will pass the version check but choke in the zlib # decompression process with pytest.raises((zlib_error, soi.VersionError)): soi.Inventory(fname_zlib=path_fxn(res_dec))
def test_apifail_inventory_dictimport_noitems(self): """Confirm ValueError with no-items dict passed to json_dict.""" d = { soi.HeaderFields.Project.value: "proj", soi.HeaderFields.Version.value: "v3.3", soi.HeaderFields.Count.value: 5, } with pytest.raises(ValueError): soi.Inventory()._import_json_dict(d)
def test_apifail_changing_immutable_dataobj(self, path_fxn, use_bytes, res_cmp): """Confirm DataObj's are immutable.""" inv = soi.Inventory(path_fxn(res_cmp)) with pytest.raises(FrozenInstanceError): # pragma: no cover (XFAIL) if use_bytes: inv.objects[0].as_bytes.name = b"newname" else: inv.objects[0].name = "newname"
def main(): import argparse parser = argparse.ArgumentParser(usage=__doc__) parser.add_argument("--input", dest="filename", required=False, help="sphinx inventory file (objects.inv)", metavar="FILE", type=lambda x: is_valid_file(parser, x)) parser.add_argument( "--output", dest="output", default="../../../release/scripts/modules/rna_manual_reference.py", required=False, help="path to output including filename and extentsion", metavar="FILE") parser.add_argument( "--url", dest="url", default="https://docs.blender.org/manual/en/dev/objects.inv", required=False, help="url to sphinx inventory file (objects.inv)", metavar="FILE") args = parser.parse_args() if args.filename: # Download and decode objects.inv print("Loading from file...") inv = soi.Inventory(args.filename) else: # Load and decode objects.inv print("Downloading...") inv = soi.Inventory(url=args.url) write_mappings(inv, args.output) print("Done!")
def current_version(build_dir): """Return the current version of an objects.inv file""" objects_file = build_dir.joinpath("objects.inv") inventory = sphobjinv.Inventory(objects_file) if not inventory.version: raise RuntimeError( f"The versjon tool requires a version number " "in the {objects_file.parent}. Add one to conf.py or pass " 'it to sphinx build using "-D version=X.Y.Z".') return inventory.version
def _get_doc_url(obj_name): doc_base_url = 'https://lisa-linux-integrated-system-analysis.readthedocs.io/en/master/' # Use the inventory built by RTD inv_url = urllib.parse.urljoin(doc_base_url, 'objects.inv') inv = sphobjinv.Inventory(url=inv_url) for inv_obj in inv.objects: if inv_obj.name == obj_name: doc_page = inv_obj.uri.replace('$', inv_obj.name) doc_url = urllib.parse.urljoin(doc_base_url, doc_page) return doc_url raise ValueError('Could not find the doc of: {}'.format(obj_name))
def test_api_inventory_many_url_imports( testall_inv_path, res_path, scratch_path, misc_info, sphinx_load_test, pytestconfig, ): """Confirm a plethora of .inv files downloads properly via url arg. This test is SLOW, and so does not run by default. Invoke with `--nonloc` to run it; invoke with `--testall` to test over all .inv files in tests/resource. """ fname = testall_inv_path.name scr_fpath = scratch_path / fname # Drop most unless testall if not pytestconfig.getoption("--testall") and fname != "objects_attrs.inv": pytest.skip("'--testall' not specified") # Construct inventories for comparison mch = misc_info.p_inv.match(fname) proj_name = mch.group(1) inv1 = soi.Inventory(str(res_path / fname)) inv2 = soi.Inventory(url=misc_info.remote_url.format(proj_name)) # Test the things assert inv1 == inv2 # Ensure sphinx likes the regenerated inventory data = inv2.data_file() cmp_data = soi.compress(data) soi.writebytes(scr_fpath, cmp_data) sphinx_load_test(scr_fpath)
def main(): inv = soi.Inventory() inv.project = "TensorFlow-Probability" inv.version = "2.0" with open("./tfp_docs_scraper/core_symbols.json") as fp: symbols = json.load(fp) for symbol in symbols: add_to_inventory(inv, symbol) text = inv.data_file(contract=True) ztext = soi.compress(text) soi.writebytes("objects.inv", ztext) print("All done!")
def test_sphinx_import(workdir): """Ensure that the sphinx importer imports links correctly""" filepath = str(pathlib.Path(workdir.name, "sphinx.db")) inv = soi.Inventory() inv.project = "Python" inv.version = "1.0" inv.objects.append( soi.DataObjStr( name="print", domain="py", priority="1", role="function", uri="builtins.html#$", dispname="-", )) inv.objects.append( soi.DataObjStr( name="enumeration", domain="py", priority="1", role="label", uri="concepts.html#$", dispname="Enumeration", )) with mock.patch("llyfrau.importers.soi.Inventory", return_value=inv) as m_inv: sphinx(filepath, "https://docs.python.org/") m_inv.assert_called_with(url="https://docs.python.org/objects.inv") db = Database(filepath) source = Source.search(db, name="Python v1.0")[0] assert source.name == "Python v1.0 Documentation" assert source.prefix == "https://docs.python.org/" links = Link.search(db, source=source) assert links[0].name == "print" assert links[0].url == "builtins.html#print" assert {t.name for t in links[0].tags} == {"sphinx", "py", "function"} assert links[1].name == "Enumeration" assert links[1].url == "concepts.html#enumeration" assert {t.name for t in links[1].tags} == {"sphinx", "py", "label"}
def main( project="TensorFlow", version="2.0", symbols_file="./tf_docs_scraper/core_symbols.json", common_url="https://www.tensorflow.org/api_docs/python/", inv_file="tf2_py_objects.inv", ): inv = soi.Inventory() inv.project = project inv.version = version with open(symbols_file) as fp: symbols = json.load(fp) for symbol in symbols: add_to_inventory(inv, symbol, common_url) text = inv.data_file(contract=True) ztext = soi.compress(text) soi.writebytes(inv_file, ztext) print("All done!")
def sphinx(uri: str, collection: Collection): """Import links from a sphinx documentation site.""" if uri.endswith("/"): uri = f"{uri}objects.inv" if not uri.endswith("objects.inv"): uri = f"{uri}/objects.inv" print(f"Trying index url : {uri}", end="\r") inv = soi.Inventory(url=uri) print(f"Found object index: {uri} with {inv.count} entries") collection.name = f"{inv.project} v{inv.version} Documentation" collection.prefix = uri.replace("objects.inv", "") for item in inv.objects: name = item.dispname_expanded url = item.uri_expanded tags = [t for t in [item.domain, item.role] if t is not None] collection.add_link(name=name, url=url, tags=tags)
def test_mapping_objects_inv(mapping_tuple): """Check docs objects.inv link validity.""" inv_link = mapping_tuple[1] or (mapping_tuple[0].removesuffix("/") + "/objects.inv") inv = soi.Inventory(url=inv_link) log_append(f"{repr(inv)}\n") # Interestingly, (some?) module objects don't craft accurate URIs? data_obj = [ obj for obj in dropwhile(lambda o: o.role == "module", inv.objects) ][0] obj_link = mapping_tuple[0] + data_obj.uri_expanded obj_resp = rq.get(obj_link) log_append(f"{obj_link}: {obj_resp.status_code} {obj_resp.reason}\n") assert obj_resp.ok soup = BSoup(obj_resp.text, "html.parser") if anchor := obj_link.partition("#")[2]: anchor_check = soup.find_all("dt", id=anchor) log_append(f"Anchor '{anchor}' found? {bool(anchor_check)}\n\n") assert anchor_check
def docs_inventory(self) -> sphobjinv.Inventory: return sphobjinv.Inventory(url=self.docs_url + "objects.inv")
shutil.rmtree("api", ignore_errors=True) subprocess.call( " ".join([ "sphinx-apidoc", f"../src/{package}/", "-o api/", "--force", "--no-toc", "--templatedir _templates", "--separate", ]), shell=True, ) # -- Convert sphinx object inventory ----------------------------------------- inv = soi.Inventory() inv.project = "constraint" constraint_object_names = [ "Constraint", "Domain", "Problem", "Solver", "Variable", ] for object_name in constraint_object_names: inv.objects.append( soi.DataObjStr( name=f"{inv.project}.{object_name}", domain="py", role="class",
# manual/ --> manual/ru/ # # The table in the script, contains all of the languages we have in the # Blender manual website, for those other languages that still # does not have a team of translators, # and/or don't have a manual for their languages we commented the lines below, # you should add them to the language table when they have a proper manual, # or added to the Blender UI translation table. # # URL is the: url_manual_prefix + url_manual_mapping[#id] import os import sphobjinv as soi # Download and decode objects.inv inv = soi.Inventory(url="https://docs.blender.org/manual/en/dev/objects.inv") # Write the fire filepath = os.path.join("rna_manual_reference.py") file = open(filepath, "w", encoding="utf-8") fw = file.write fw("# Do not edit this file.") fw(" This file is auto generated from rna_manual_reference_updater.py\n\n") fw("import bpy\n\n") fw("if bpy.app.version_cycle in {'rc', 'release'}:\n") fw(" manual_version = '%d.%d' % bpy.app.version[:2]\n") fw("else:\n") fw(" manual_version = 'dev'\n\n") fw("url_manual_prefix = \"https://docs.blender.org/manual/en/\" + manual_version + \"/\"\n\n" )
def test_api_inventory_known_header_required(name, url): """Confirm URL load works on docs pages requiring HTTP header config.""" inv = soi.Inventory(url=url) assert inv.count > 0
def inventory_from_index(project, version, index_json): inv = soi.Inventory() inv.project = project inv.version = version inv.objects.extend(objects_from_index(index_json)) return inv
# or added to the Blender UI translation table. # # URL is the: url_manual_prefix + url_manual_mapping[#id] import os import sphobjinv as soi import urllib.request # Download the objects.inv file urlretrieve = urllib.request.urlretrieve urlretrieve("https://docs.blender.org/manual/en/dev/objects.inv", "objects.inv") # Decode objects.inv inv = soi.Inventory('objects.inv') objects_data = inv.data_file() soi.writebytes('objects.tmp', objects_data) # TODO leave in memory os.remove("objects.inv") # Write the fire filepath = os.path.join("rna_manual_reference.py") file = open(filepath, "w", encoding="utf-8") fw = file.write fw("# Do not edit this file.") fw(" This file is auto generated from rna_manual_reference_updater.py\n\n") fw("import bpy\n\n") fw("if bpy.app.version_cycle in {'rc', 'release'}:\n") fw(" manual_version = '%d.%d' % bpy.app.version[:2]\n") fw("else:\n")
def test_apifail_inventory_invalidsource(self): """Confirm error raised when invalid source provided.""" with pytest.raises(TypeError): soi.Inventory("abcdefg")
def test_apifail_inventory_dictimport_toomanysrcargs(self, ): """Confirm error if >1 sources passed.""" with pytest.raises(RuntimeError): soi.Inventory(source="foo", plaintext="bar")
def test_apifail_invalid_inventory_init_arg(self, bad_arg): """Confirm non-__init__ Inventory members raise exceptions when passed.""" with pytest.raises(TypeError): soi.Inventory(**{bad_arg: "foo"})