def sphinx_load_test(testcase, path):
"""Perform 'live' Sphinx inventory load test."""
# Easier to have the file open the whole time
with open(path, "rb") as f:
from sphinx.util.inventory import InventoryFile as IFile
# Attempt the load operation
try:
IFile.load(f, "", osp.join)
except Exception:
with testcase.subTest("sphinx_load_ok"):
testcase.fail()
def fetch_inventory(app, uri, inv):
# type: (Sphinx, unicode, Any) -> Any
"""Fetch, parse and return an intersphinx inventory file."""
# both *uri* (base URI of the links to generate) and *inv* (actual
# location of the inventory file) can be local or remote URIs
localuri = '://' not in uri
if not localuri:
# case: inv URI points to remote resource; strip any existing auth
uri = _strip_basic_auth(uri)
try:
if '://' in inv:
f = _read_from_url(inv, config=app.config)
else:
f = open(path.join(app.srcdir, inv), 'rb')
except Exception as err:
err.args = ('intersphinx inventory %r not fetchable due to %s: %s',
inv, err.__class__, err)
raise
try:
if hasattr(f, 'url'):
newinv = f.url # type: ignore
if inv != newinv:
logger.info('intersphinx inventory has moved: %s -> %s', inv, newinv)
if uri in (inv, path.dirname(inv), path.dirname(inv) + '/'):
uri = path.dirname(newinv)
with f:
try:
join = localuri and path.join or posixpath.join
invdata = InventoryFile.load(f, uri, join)
except ValueError as exc:
raise ValueError('unknown or unsupported inventory version: %r' % exc)
except Exception as err:
err.args = ('intersphinx inventory %r not readable due to %s: %s',
inv, err.__class__.__name__, err)
raise
else:
return invdata
def fetch_inventory(app, uri, inv):
# type: (Sphinx, unicode, Any) -> Any
"""Fetch, parse and return an intersphinx inventory file."""
# both *uri* (base URI of the links to generate) and *inv* (actual
# location of the inventory file) can be local or remote URIs
localuri = '://' not in uri
if not localuri:
# case: inv URI points to remote resource; strip any existing auth
uri = _strip_basic_auth(uri)
try:
if '://' in inv:
f = _read_from_url(inv, config=app.config)
else:
f = open(path.join(app.srcdir, inv), 'rb')
except Exception as err:
logger.warning('intersphinx inventory %r not fetchable due to %s: %s',
inv, err.__class__, err)
return
try:
if hasattr(f, 'url'):
newinv = f.url # type: ignore
if inv != newinv:
logger.info('intersphinx inventory has moved: %s -> %s', inv, newinv)
if uri in (inv, path.dirname(inv), path.dirname(inv) + '/'):
uri = path.dirname(newinv)
with f:
try:
join = localuri and path.join or posixpath.join
invdata = InventoryFile.load(f, uri, join)
except ValueError as exc:
raise ValueError('unknown or unsupported inventory version: %r' % exc)
except Exception as err:
logger.warning('intersphinx inventory %r not readable due to %s: %s',
inv, err.__class__.__name__, err)
else:
return invdata
def parse_docs(self):
docs_data = urlopen(urljoin(DOCS_URL, "objects.inv"))
self._docs = InventoryFile.load(docs_data, DOCS_URL, urljoin)
def worker(inv=inv, key=inv[0]):
rv[key] = InventoryFile.load(
IterIO(requests.get(inv[1], stream=True).iter_content()),
inv[0], posixpath.join)
def parse_docs(self):
request = Request(urljoin(DOCS_URL, "objects.inv"),
headers={'User-Agent': USER_AGENT})
docs_data = urlopen(request)
self._docs = InventoryFile.load(docs_data, DOCS_URL, urljoin)
def dump_inventory(self) -> None:
logger.info(bold(__('dumping object inventory... ')), nonl=True)
InventoryFile.dump(path.join(self.outdir, INVENTORY_FILENAME), self.env, self)
logger.info(__('done'))
def finish(self) -> None:
InventoryFile.dump(Path(self.outdir) / 'objects.inv', self.env, self)
def parse_docs(self):
docs_data = urlopen(urljoin(DOCS_URL, "objects.inv"))
self._docs = InventoryFile.load(docs_data, DOCS_URL, urljoin)
def dump_inventory(self) -> None:
InventoryFile.dump(path.join(self.outdir, INVENTORY_FILENAME), self.env, self)
import json
import os
import urllib.request
from pathlib import Path
import kalong.utils.doc_lookup
from sphinx.util.inventory import InventoryFile
uris = ['https://docs.python.org/']
out = Path(kalong.utils.doc_lookup.__file__).parent / 'lookup.json'
lookup = {}
for uri in uris:
f = urllib.request.urlopen(os.path.join(uri, 'objects.inv'))
invdata = InventoryFile.load(f, uri, os.path.join)
for category, data in invdata.items():
domain, name = category.split(':')
if domain != 'py':
continue
for attr, (_, _, link, _) in data.items():
if attr in lookup:
print(f'Dup {attr} in {category}')
lookup[attr] = link
with open(out, 'w') as f:
json.dump(lookup, f)
def finish(self):
InventoryFile.dump(path.join(self.outdir, 'objects.inv'), self.env,
self)
