def func(path):
"""Perform the 'live' inventory load test."""
with path.open("rb") as f:
try:
IFile.load(f, "", osp.join)
except Exception as e: # noqa: PIE786
# An exception here is a failing test, not a test error.
pytest.fail(e)
def sphinx_load_test(testcase, path):
"""Perform 'live' Sphinx inventory load test."""
# Easier to have the file open the whole time
with open(path, "rb") as f:
from sphinx.util.inventory import InventoryFile as IFile
# Attempt the load operation
try:
IFile.load(f, "", osp.join)
except Exception:
with testcase.subTest("sphinx_load_ok"):
testcase.fail()
def test_dirhtml(app, status, warning):
app.build()
assert (app.outdir / 'index.html').exists()
assert (app.outdir / 'foo/index.html').exists()
assert (app.outdir / 'foo/foo_1/index.html').exists()
assert (app.outdir / 'foo/foo_2/index.html').exists()
assert (app.outdir / 'bar/index.html').exists()
content = (app.outdir / 'index.html').text()
assert 'href="foo/"' in content
assert 'href="foo/foo_1/"' in content
assert 'href="foo/foo_2/"' in content
assert 'href="bar/"' in content
# objects.inv (refs: #7095)
f = (app.outdir / 'objects.inv').open('rb')
invdata = InventoryFile.load(f, 'path/to', posixpath.join)
assert 'index' in invdata.get('std:doc')
assert ('Python', '', 'path/to/', '-') == invdata['std:doc']['index']
assert 'foo/index' in invdata.get('std:doc')
assert ('Python', '', 'path/to/foo/',
'-') == invdata['std:doc']['foo/index']
assert 'index' in invdata.get('std:label')
assert ('Python', '', 'path/to/#index',
'-') == invdata['std:label']['index']
assert 'foo' in invdata.get('std:label')
assert ('Python', '', 'path/to/foo/#foo',
'foo/index') == invdata['std:label']['foo']
async def update_docs(self) -> None:
await self._update_official_docs()
response = await self._httpx_client.get(
url=urljoin(DOCS_URL, "objects.inv"),
headers=DEFAULT_HEADERS,
follow_redirects=True,
)
data = InventoryFile.load(BytesIO(response.content), DOCS_URL, urljoin)
self._docs = []
for entry_type, items in data.items():
for name, (_, _, url, display_name) in items.items():
if "._" in name:
# For some reason both `ext._application.Application` and `ext.Application`
# are present ...
continue
tg_url, tg_test, tg_name = "", "", ""
name_bits = name.split(".")
if entry_type == "py:method" and ("telegram.Bot" in name
or "telegram.ext.ExtBot"
in name):
tg_test = name_bits[-1]
if entry_type == "py:attribute":
tg_test = name_bits[-2]
if entry_type == "py:class":
tg_test = name_bits[-1]
elif entry_type == "py:parameter":
tg_test = name_bits[-4]
tg_test = tg_test.replace("_", "").lower()
if tg_test in self._official:
tg_name = self._official[tg_test]
tg_url = urljoin(OFFICIAL_URL, "#" + tg_name.lower())
if entry_type == "py:parameter":
self._docs.append(
ParamDocEntry(
name=name,
url=url,
display_name=display_name
if display_name.strip() != "-" else None,
entry_type=entry_type,
telegram_url=tg_url,
telegram_name=tg_name,
))
else:
self._docs.append(
DocEntry(
name=name,
url=url,
display_name=display_name
if display_name.strip() != "-" else None,
entry_type=entry_type,
telegram_url=tg_url,
telegram_name=tg_name,
))
def __init__(self, url, pre=None):
if pre is None: pre = url + "/"
invs = urlread(f'{url}/objects.inv')
self.idx = InventoryFile.load(stream=BytesIO(invs),
uri=pre,
joinfunc=urljoin)
typs = 'module', 'class', 'method', 'function'
self.d = {o: self._get(o) for o in typs}
self.syms = defaultdict(dict)
for o in typs:
for k, v in self.d[o].items():
modparts = k.split(".")[:-2 if o == 'method' else -1]
if modparts: self.syms['.'.join(modparts)][k] = v
def fetch_docs(self) -> None:
self.fetch_official_docs()
request = Request(urljoin(DOCS_URL, "objects.inv"),
headers={"User-Agent": USER_AGENT})
docs_data = urlopen(request)
data = InventoryFile.load(docs_data, DOCS_URL, urljoin)
self._docs = []
for entry_type, items in data.items():
for name, (_, _, url, display_name) in items.items():
tg_url, tg_test, tg_name = "", "", ""
name_bits = name.split(".")
if entry_type in ["py:method", "py:attribute", "py:property"]:
if "telegram.Bot" in name or "telegram.ext.ExtBot" in name:
tg_test = name_bits[-1]
else:
tg_test = name_bits[-2]
if entry_type == "py:class":
tg_test = name_bits[-1]
elif entry_type == "py:parameter":
tg_test = name_bits[-4]
tg_test = tg_test.replace("_", "").lower()
if tg_test in self._official.keys():
tg_name = self._official[tg_test]
tg_url = urljoin(OFFICIAL_URL, "#" + tg_name.lower())
if entry_type == "py:parameter":
self._docs.append(
ParamDocEntry(
name=name,
url=url,
display_name=display_name
if display_name.strip() != "-" else None,
entry_type=entry_type,
telegram_url=tg_url,
telegram_name=tg_name,
))
else:
self._docs.append(
DocEntry(
name=name,
url=url,
display_name=display_name
if display_name.strip() != "-" else None,
entry_type=entry_type,
telegram_url=tg_url,
telegram_name=tg_name,
))
def main(args):
base_url = gen_base_url(args.objects_inv_path)
def joinfunc(uri, location):
return uri + '/' + location
with open(args.objects_inv_path, 'rb') as fin:
inv = InventoryFile.load(fin, base_url, joinfunc)
if args.type:
for typ in inv:
print(typ)
else:
for typ, indices in inv.items():
for _, (_, _, uri, title) in indices.items():
print(typ, uri, title, sep='\t')
def load_inventory(uris):
import requests
from sphinx.util.inventory import InventoryFile
uris = list(uris)
inventory = {}
for base_uri in uris:
object_inv_uri = f"{base_uri}/objects.inv"
r = requests.get(object_inv_uri)
r.raise_for_status()
with io.BytesIO(r.content) as fobj:
sub_inventory = InventoryFile.load(fobj, base_uri,
lambda a, b: f"{a}/{b}")
for section, data in sub_inventory.items():
inventory.setdefault(section, {}).update(data)
return dict(uris=uris, inventory=inventory)
def get_sphinx_inventory(url):
"""
get the sphinx inventory from a url.
Parameters
----------
url : url of the package where find objects.inv file
Returns
-------
the inventory : dict
"""
inventory_url = urllib.parse.urljoin(url, INVENTORY_FILENAME)
f = urllib.request.urlopen(inventory_url)
inventory = InventoryFile.load(f, url, urllib.parse.urljoin)
f.close()
return inventory
def process(path):
r = [
"<html>",
"<head>",
"<title>NOC go</title>",
"</head>",
"<body>",
"<script>",
JS,
"redirect({",
]
with open(path) as f:
data = InventoryFile.load(f, "", os.path.join) or {}
rr = []
for entry, einfo in sorted(data["std:label"].items()):
rr += ["'%s': '%s'" % (entry, einfo[2])]
r += [",".join(rr), "});", "</script>", "</body>", "</html>"]
base = os.path.dirname(path)
go_path = os.path.join(base, "go.html")
with open(go_path, "w") as f:
f.write("".join(r))
def fetch_inventory(app, uri, inv):
# type: (Sphinx, unicode, Any) -> Any
"""Fetch, parse and return an intersphinx inventory file."""
# both *uri* (base URI of the links to generate) and *inv* (actual
# location of the inventory file) can be local or remote URIs
localuri = '://' not in uri
if not localuri:
# case: inv URI points to remote resource; strip any existing auth
uri = _strip_basic_auth(uri)
try:
if '://' in inv:
f = _read_from_url(inv, config=app.config)
else:
f = open(path.join(app.srcdir, inv), 'rb')
except Exception as err:
err.args = ('intersphinx inventory %r not fetchable due to %s: %s',
inv, err.__class__, str(err))
raise
try:
if hasattr(f, 'url'):
newinv = f.url # type: ignore
if inv != newinv:
logger.info('intersphinx inventory has moved: %s -> %s', inv,
newinv)
if uri in (inv, path.dirname(inv), path.dirname(inv) + '/'):
uri = path.dirname(newinv)
with f:
try:
join = localuri and path.join or posixpath.join
invdata = InventoryFile.load(f, uri, join)
except ValueError as exc:
raise ValueError(
'unknown or unsupported inventory version: %r' % exc)
except Exception as err:
err.args = ('intersphinx inventory %r not readable due to %s: %s', inv,
err.__class__.__name__, str(err))
raise
else:
return invdata
def _get_data(url):
"""
Loads file ``objects.inv`` generated by
extension :epkg:`sphinx.ext.intersphinx`.
@param url url of documentation, example
``https://pandas.pydata.org/docs/``
@return instance of `InventoryFile`
"""
url_inv = urljoin(url, "objects.inv")
if urllib_parse.urlparse(url_inv).scheme in ('http', 'https'):
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11' # noqa: E501
headers = {'User-Agent': user_agent}
req = urllib_request.Request(url_inv, None, headers)
resp = urllib_request.urlopen(req)
data = resp.read()
else:
with open(url, 'rb') as fid:
data = fid.read()
inv = InventoryFile.load(io.BytesIO(data), url, urljoin)
return inv
def fetch_inventory(app, uri, inv):
# type: (Sphinx, unicode, Any) -> Any
"""Fetch, parse and return an intersphinx inventory file."""
# both *uri* (base URI of the links to generate) and *inv* (actual
# location of the inventory file) can be local or remote URIs
localuri = '://' not in uri
if not localuri:
# case: inv URI points to remote resource; strip any existing auth
uri = _strip_basic_auth(uri)
try:
if '://' in inv:
f = _read_from_url(inv, config=app.config)
else:
f = open(path.join(app.srcdir, inv), 'rb')
except Exception as err:
err.args = ('intersphinx inventory %r not fetchable due to %s: %s',
inv, err.__class__, err)
raise
try:
if hasattr(f, 'url'):
newinv = f.url # type: ignore
if inv != newinv:
logger.info('intersphinx inventory has moved: %s -> %s', inv, newinv)
if uri in (inv, path.dirname(inv), path.dirname(inv) + '/'):
uri = path.dirname(newinv)
with f:
try:
join = localuri and path.join or posixpath.join
invdata = InventoryFile.load(f, uri, join)
except ValueError as exc:
raise ValueError('unknown or unsupported inventory version: %r' % exc)
except Exception as err:
err.args = ('intersphinx inventory %r not readable due to %s: %s',
inv, err.__class__.__name__, err)
raise
else:
return invdata
def parse_docs(self):
docs_data = urlopen(urljoin(DOCS_URL, "objects.inv"))
self._docs = InventoryFile.load(docs_data, DOCS_URL, urljoin)
def worker(inv=inv, key=inv[0]):
rv[key] = InventoryFile.load(
IterIO(requests.get(inv[1], stream=True).iter_content()),
inv[0], posixpath.join)
def parse_docs(self):
request = Request(urljoin(DOCS_URL, "objects.inv"),
headers={'User-Agent': USER_AGENT})
docs_data = urlopen(request)
self._docs = InventoryFile.load(docs_data, DOCS_URL, urljoin)
import json
import os
import urllib.request
from pathlib import Path
import kalong.utils.doc_lookup
from sphinx.util.inventory import InventoryFile
uris = ['https://docs.python.org/']
out = Path(kalong.utils.doc_lookup.__file__).parent / 'lookup.json'
lookup = {}
for uri in uris:
f = urllib.request.urlopen(os.path.join(uri, 'objects.inv'))
invdata = InventoryFile.load(f, uri, os.path.join)
for category, data in invdata.items():
domain, name = category.split(':')
if domain != 'py':
continue
for attr, (_, _, link, _) in data.items():
if attr in lookup:
print(f'Dup {attr} in {category}')
lookup[attr] = link
with open(out, 'w') as f:
json.dump(lookup, f)
def parse_docs(self):
docs_data = urlopen(urljoin(DOCS_URL, "objects.inv"))
self._docs = InventoryFile.load(docs_data, DOCS_URL, urljoin)
