def main():
"""Main script to verify modules included."""
mock_uri = ''
inventory = fetch_inventory(SphinxApp, mock_uri,
OBJECT_INVENTORY_RELPATH)
sphinx_mods = set(inventory['py:module'].keys())
library_dir = os.path.join(BASE_DIR, 'gcloud')
public_mods = get_public_modules(library_dir,
base_package='gcloud')
public_mods = set(public_mods)
if not sphinx_mods <= public_mods:
unexpected_mods = sphinx_mods - public_mods
message = ['Unexpected error. There were modules referenced by '
'Sphinx that are not among the public modules.']
message.extend(['- %s' % (mod,) for mod in unexpected_mods])
print('\n'.join(message), file=sys.stderr)
sys.exit(1)
undocumented_mods = public_mods - sphinx_mods
# Remove ignored modules.
undocumented_mods -= IGNORED_MODULES
if undocumented_mods:
message_parts = ['Found undocumented public modules:']
message_parts.extend(['- ' + mod_name
for mod_name in sorted(undocumented_mods)])
print('\n'.join(message_parts), file=sys.stderr)
sys.exit(1)
def get_inventory(args):
app = DummyApp()
inventory = intersphinx.fetch_inventory(app, "", args.uri)
if args.dump:
dump_inventory(inventory)
if args.create_intersphinx_reference:
with SphinxPage(args.create_intersphinx_reference,
"w",
app=args.project) as f:
sections = sorted(inventory.keys())
for k in sections:
term_name = k.split(':')[1]
term = {
'attribute': 'attr',
'function': 'func'
}.get(term_name, term_name)
section = inventory[k]
f.section(term)
if term == 'label':
pattern_as = "``:ref:{1[0]}:{2}``"
pattern_test = ":ref:`{2}`"
else:
pattern_as = "``:{0}:{1[0]}:{2}``"
pattern_test = ":{0}:`{2}`"
entries = sorted(section, key=lambda key: section[key])
for name in entries:
values = section[name]
access_as = pattern_as.format(term, values, name)
access_test = pattern_test.format(term, values, name)
f.writeln(" * {0} : '{1}' ({2})".format(
name, access_as, access_test))
f.writeln("", "")
return "Success"
def update_intersphinx(version_pk):
version = Version.objects.get(pk=version_pk)
path = version.project.rtd_build_path(version.slug)
if not path:
print "ERR: %s has no path" % version
return None
app = DictObj()
app.srcdir = path
try:
inv = fetch_inventory(app, app.srcdir, 'objects.inv')
except TypeError:
print "Failed to fetch inventory for %s" % version
return None
# I'm entirelty not sure this is even close to correct.
# There's a lot of info I'm throwing away here; revisit later?
for keytype in inv:
for term in inv[keytype]:
try:
_, _, url, title = inv[keytype][term]
if not title or title == '-':
if '#' in url:
title = url.rsplit('#')[-1]
else:
title = url
find_str = "rtd-builds/latest"
latest = url.find(find_str)
url = url[latest + len(find_str) + 1:]
url = "http://%s.readthedocs.org/en/latest/%s" % (
version.project.slug, url)
save_term(version, term, url, title)
if '.' in term:
save_term(version, term.split('.')[-1], url, title)
except Exception, e: #Yes, I'm an evil person.
print "*** Failed updating %s: %s" % (term, e)
def fetch_list(version=None):
"""
For the given version of Python (or all versions if no version is set), this function:
- Uses the `fetch_inventory` function of :py:mod`sphinx.ext.intersphinx` to
grab and parse the Sphinx object inventory
(ie ``http://docs.python.org/<version>/objects.inv``) for the given version.
- Grabs the names of all of the modules in the parsed inventory data.
- Writes the sorted list of module names to file (within the `lists` subfolder).
:param str|None version: A specified version of Python. If not specified, then all
available versions of Python will have their inventory objects fetched
and parsed, and have their module names written to file.
(one of ``"2.6"``, ``"2.7"``, ``"3.2"``, ``"3.3"``, ``"3.4"``, ``"3.5"``, ``"3.6"``, ``"3.7"``, ``"3.8"``, ``"3.9"`` or ``None``)
"""
if version is None:
versions = short_versions
else:
versions = [get_canonical_version(version)]
for version in versions:
url = "http://docs.python.org/{}/objects.inv".format(version)
modules = sorted(
list(fetch_inventory(DummyApp(), "", url).get("py:module").keys()))
with open(os.path.join(list_dir, "{}.txt".format(version)), "w") as f:
for module in modules:
f.write(module)
f.write("\n")
def update_intersphinx(version_pk):
version = Version.objects.get(pk=version_pk)
path = version.project.rtd_build_path(version.slug)
if not path:
print "ERR: %s has no path" % version
return None
app = DictObj()
app.srcdir = path
try:
inv = fetch_inventory(app, app.srcdir, 'objects.inv')
except TypeError:
print "Failed to fetch inventory for %s" % version
return None
# I'm entirelty not sure this is even close to correct.
# There's a lot of info I'm throwing away here; revisit later?
for keytype in inv:
for term in inv[keytype]:
try:
_, _, url, title = inv[keytype][term]
if not title or title == '-':
if '#' in url:
title = url.rsplit('#')[-1]
else:
title = url
find_str = "rtd-builds/latest"
latest = url.find(find_str)
url = url[latest + len(find_str) + 1:]
url = "http://%s.readthedocs.org/en/latest/%s" % (
version.project.slug, url)
save_term(version, term, url, title)
if '.' in term:
save_term(version, term.split('.')[-1], url, title)
except Exception, e: #Yes, I'm an evil person.
print "*** Failed updating %s: %s" % (term, e)
def main():
"""Main script to verify modules included."""
mock_uri = ''
inventory = fetch_inventory(SphinxApp, mock_uri, OBJECT_INVENTORY_RELPATH)
sphinx_mods = set(inventory['py:module'].keys())
library_dir = os.path.join(BASE_DIR, 'gcloud')
public_mods = get_public_modules(library_dir, base_package='gcloud')
public_mods = set(public_mods)
if not sphinx_mods <= public_mods:
message = ('Unexpected error. There were modules referenced by '
'Sphinx that are not among the public modules.')
print(message, file=sys.stderr)
sys.exit(1)
undocumented_mods = public_mods - sphinx_mods
# Remove ignored modules.
undocumented_mods -= IGNORED_MODULES
if undocumented_mods:
message_parts = ['Found undocumented public modules:']
message_parts.extend(
['- ' + mod_name for mod_name in sorted(undocumented_mods)])
print('\n'.join(message_parts), file=sys.stderr)
sys.exit(1)
async def convert(ctx, url: str):
try:
intersphinx.fetch_inventory(SphinxConfiguration(), '', url)
except AttributeError:
raise commands.BadArgument(f"Failed to fetch Intersphinx inventory from URL `{url}`.")
except ConnectionError:
if url.startswith('https'):
raise commands.BadArgument(
f"Cannot establish a connection to `{url}`. Does it support HTTPS?"
)
raise commands.BadArgument(f"Cannot connect to host with URL `{url}`.")
except ValueError:
raise commands.BadArgument(
f"Failed to read Intersphinx inventory from URL `{url}`. "
"Are you sure that it's a valid inventory file?"
)
return url
async def convert(ctx: commands.Context, url: str) -> str:
"""Convert url to Intersphinx inventory URL."""
try:
intersphinx.fetch_inventory(SPHINX_MOCK_APP, '', url)
except AttributeError:
raise commands.BadArgument(f"Failed to fetch Intersphinx inventory from URL `{url}`.")
except ConnectionError:
if url.startswith('https'):
raise commands.BadArgument(
f"Cannot establish a connection to `{url}`. Does it support HTTPS?"
)
raise commands.BadArgument(f"Cannot connect to host with URL `{url}`.")
except ValueError:
raise commands.BadArgument(
f"""Failed to read Intersphinx inventory from URL `{url}`.
Are you sure that it's a valid inventory file?"""
)
return url
def main():
for package_name, (uri, inv) in intersphinx_mapping.items():
if inv is None:
inv = "objects.inv"
inv_uri = os.path.join(uri, inv)
app = App(package_name)
inventory = intersphinx.fetch_inventory(app, "", inv_uri)
for k in inventory.keys():
print(f"{app.name} {k}")
for name, value in inventory[k].items():
print("{} {} is <{}:{}>".format(k, value[2], app.name, name))
def main():
for package_name, (uri, inv) in intersphinx_mapping.items():
if inv is None:
inv = 'objects.inv'
inv_uri = os.path.join(uri, inv)
app = App(package_name)
inventory = intersphinx.fetch_inventory(app, '', inv_uri)
for k in inventory.keys():
print "%s %s" % (app.name, k)
for name, value in inventory[k].items():
print "%s %s is <%s:%s>" % (k, value[2], app.name, name)
def main():
for package_name, (uri, inv) in intersphinx_mapping.items():
if inv is None:
inv = 'objects.inv'
inv_uri = os.path.join(uri, inv)
app = App(package_name)
inventory = intersphinx.fetch_inventory(app, '', inv_uri)
for k in inventory.keys():
print("{} {}".format(app.name, k))
for name, value in inventory[k].items():
print("{} {} is <{}:{}>".format(k, value[2], app.name, name))
def test_fetch_inventory_redirection(app, status, warning, _read_from_url,
read_inventory_v2):
_read_from_url(
).readline.return_value = '# Sphinx inventory version 2'.encode('utf-8')
# same uri and inv, not redirected
_read_from_url(
).geturl.return_value = 'http://hostname/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/',
'http://hostname/' + INVENTORY_FILENAME)
assert 'intersphinx inventory has moved' not in status.getvalue()
assert read_inventory_v2.call_args[0][1] == 'http://hostname/'
# same uri and inv, redirected
status.seek(0)
status.truncate(0)
_read_from_url(
).geturl.return_value = 'http://hostname/new/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/',
'http://hostname/' + INVENTORY_FILENAME)
assert status.getvalue() == (
'intersphinx inventory has moved: '
'http://hostname/%s -> http://hostname/new/%s\n' %
(INVENTORY_FILENAME, INVENTORY_FILENAME))
assert read_inventory_v2.call_args[0][1] == 'http://hostname/new'
# different uri and inv, not redirected
status.seek(0)
status.truncate(0)
_read_from_url(
).geturl.return_value = 'http://hostname/new/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/',
'http://hostname/new/' + INVENTORY_FILENAME)
assert 'intersphinx inventory has moved' not in status.getvalue()
assert read_inventory_v2.call_args[0][1] == 'http://hostname/'
# different uri and inv, redirected
status.seek(0)
status.truncate(0)
_read_from_url(
).geturl.return_value = 'http://hostname/other/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/',
'http://hostname/new/' + INVENTORY_FILENAME)
assert status.getvalue() == (
'intersphinx inventory has moved: '
'http://hostname/new/%s -> http://hostname/other/%s\n' %
(INVENTORY_FILENAME, INVENTORY_FILENAME))
assert read_inventory_v2.call_args[0][1] == 'http://hostname/'
def main():
app = DummyApp()
# baseurl to use
uri = ""
inv = sys.argv[1]
inventory = intersphinx.fetch_inventory(app, uri, inv)
for k in inventory.keys():
print "Type: %s" % k
for name, value in inventory[k].items():
print " %s -> '%s'" % (name, value[2])
return 0
def main():
for package_name, (uri, inv) in intersphinx_mapping.items():
if inv is None:
inv = 'objects.inv'
inv_uri = os.path.join(uri, inv)
app = App(package_name)
inventory = intersphinx.fetch_inventory(app, '', inv_uri)
for k in inventory.keys():
print "%s %s" % (app.name, k)
for name, value in inventory[k].items():
print "%s %s is <%s:%s>" % (
k, value[2], app.name, name)
def fetch_local_inventory(inventory_file):
"""Fetch the inventory file from the build output of the source directory."""
class MockConfig:
intersphinx_timeout = None
tls_verify = False
user_agent = None
class MockApp:
srcdir = ""
config = MockConfig()
return fetch_inventory(MockApp(), "", str(inventory_file))
def make_cache(name, uri):
dest_dir = join(cache_dir, name)
if not exists(dest_dir):
mkdir(dest_dir)
inv = fetch_inventory(warnings, uri, join(uri, 'objects.inv'))
for k, v in inv.items():
filename = join(dest_dir, k.replace(':', '-'))
line_counter = 0
with open(filename, 'w+') as f:
for sk, sv in v.items():
line_counter +=1
if line_counter % 100 == 0:
print('File: %s, Line: %s' % (basename(filename), line_counter))
f.write('%s\t%s\n' % (sk, sv))
def main():
app = DummyApp()
# baseurl to use
uri = ""
for invname in conf.intersphinx_mapping.keys():
# inv = sys.argv[1]
inv = conf.intersphinx_mapping[invname][0] + "objects.inv"
inventory = intersphinx.fetch_inventory(app, uri, inv)
for k in inventory.keys():
print "Type: %s" % k
for name, value in inventory[k].items():
print " %s -> '%s'" % (name, value[2])
return 0
def test_fetch_inventory_redirection(_read_from_url, InventoryFile, app,
status, warning):
intersphinx_setup(app)
_read_from_url().readline.return_value = b'# Sphinx inventory version 2'
# same uri and inv, not redirected
_read_from_url().url = 'http://hostname/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/',
'http://hostname/' + INVENTORY_FILENAME)
assert 'intersphinx inventory has moved' not in status.getvalue()
assert InventoryFile.load.call_args[0][1] == 'http://hostname/'
# same uri and inv, redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = 'http://hostname/new/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/',
'http://hostname/' + INVENTORY_FILENAME)
assert status.getvalue() == (
'intersphinx inventory has moved: '
'http://hostname/%s -> http://hostname/new/%s\n' %
(INVENTORY_FILENAME, INVENTORY_FILENAME))
assert InventoryFile.load.call_args[0][1] == 'http://hostname/new'
# different uri and inv, not redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = 'http://hostname/new/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/',
'http://hostname/new/' + INVENTORY_FILENAME)
assert 'intersphinx inventory has moved' not in status.getvalue()
assert InventoryFile.load.call_args[0][1] == 'http://hostname/'
# different uri and inv, redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = 'http://hostname/other/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/',
'http://hostname/new/' + INVENTORY_FILENAME)
assert status.getvalue() == (
'intersphinx inventory has moved: '
'http://hostname/new/%s -> http://hostname/other/%s\n' %
(INVENTORY_FILENAME, INVENTORY_FILENAME))
assert InventoryFile.load.call_args[0][1] == 'http://hostname/'
def make_cache(name, uri):
dest_dir = join(cache_dir, name)
if not exists(dest_dir):
mkdir(dest_dir)
inv = fetch_inventory(warnings, uri, join(uri, 'objects.inv'))
for k, v in inv.items():
filename = join(dest_dir, k.replace(':', '-'))
line_counter = 0
with open(filename, 'w+') as f:
for sk, sv in v.items():
line_counter += 1
if line_counter % 100 == 0:
print('File: %s, Line: %s' %
(basename(filename), line_counter))
f.write('%s\t%s\n' % (sk, sv))
def fetch_inventory(uri):
from sphinx.ext import intersphinx
import warnings
"""Read a Sphinx inventory file into a dictionary."""
class MockConfig(object):
intersphinx_timeout = None # type: int
tls_verify = False
class MockApp(object):
srcdir = ''
config = MockConfig()
def warn(self, msg):
warnings.warn(msg)
return intersphinx.fetch_inventory(MockApp(), '', uri)
def fetch_local_inventory(source_dir, html_output_dirs):
"""Fetch the inventory file from the build output of the source directory."""
class MockConfig:
intersphinx_timeout = None
tls_verify = False
user_agent = None
class MockApp:
srcdir = ""
config = MockConfig()
inventory_file = Path("objects.inv")
for output_dir in html_output_dirs:
path = source_dir / output_dir / inventory_file
if path.exists():
return fetch_inventory(MockApp(), "", str(path))
return {}
def test_fetch_inventory_redirection(app, status, warning, _read_from_url, read_inventory):
intersphinx_setup(app)
_read_from_url().readline.return_value = "# Sphinx inventory version 2".encode("utf-8")
# same uri and inv, not redirected
_read_from_url().url = "http://hostname/" + INVENTORY_FILENAME
fetch_inventory(app, "http://hostname/", "http://hostname/" + INVENTORY_FILENAME)
assert "intersphinx inventory has moved" not in status.getvalue()
assert read_inventory.call_args[0][1] == "http://hostname/"
# same uri and inv, redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = "http://hostname/new/" + INVENTORY_FILENAME
fetch_inventory(app, "http://hostname/", "http://hostname/" + INVENTORY_FILENAME)
assert status.getvalue() == (
"intersphinx inventory has moved: "
"http://hostname/%s -> http://hostname/new/%s\n" % (INVENTORY_FILENAME, INVENTORY_FILENAME)
)
assert read_inventory.call_args[0][1] == "http://hostname/new"
# different uri and inv, not redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = "http://hostname/new/" + INVENTORY_FILENAME
fetch_inventory(app, "http://hostname/", "http://hostname/new/" + INVENTORY_FILENAME)
assert "intersphinx inventory has moved" not in status.getvalue()
assert read_inventory.call_args[0][1] == "http://hostname/"
# different uri and inv, redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = "http://hostname/other/" + INVENTORY_FILENAME
fetch_inventory(app, "http://hostname/", "http://hostname/new/" + INVENTORY_FILENAME)
assert status.getvalue() == (
"intersphinx inventory has moved: "
"http://hostname/new/%s -> http://hostname/other/%s\n" % (INVENTORY_FILENAME, INVENTORY_FILENAME)
)
assert read_inventory.call_args[0][1] == "http://hostname/"
def update_intersphinx(version_pk):
version_data = api.version(version_pk).get()
del version_data["resource_uri"]
project_data = version_data["project"]
del project_data["users"]
del project_data["resource_uri"]
del project_data["absolute_url"]
project = Project(**project_data)
version_data["project"] = project
version = Version(**version_data)
object_file = version.project.find("objects.inv", version.slug)[0]
path = version.project.rtd_build_path(version.slug)
if not path:
print "ERR: %s has no path" % version
return None
app = DictObj()
app.srcdir = path
try:
inv = fetch_inventory(app, path, object_file)
except TypeError:
print "Failed to fetch inventory for %s" % version
return None
# I'm entirelty not sure this is even close to correct.
# There's a lot of info I'm throwing away here; revisit later?
for keytype in inv:
for term in inv[keytype]:
try:
_, _, url, title = inv[keytype][term]
if not title or title == "-":
if "#" in url:
title = url.rsplit("#")[-1]
else:
title = url
find_str = "rtd-builds/latest"
latest = url.find(find_str)
url = url[latest + len(find_str) + 1 :]
url = "http://%s.readthedocs.org/en/latest/%s" % (version.project.slug, url)
save_term(version, term, url, title)
if "." in term:
save_term(version, term.split(".")[-1], url, title)
except Exception, e: # Yes, I'm an evil person.
print "*** Failed updating %s: %s" % (term, e)
def fetch_intersphinx_inventory(uri):
"""
Fetch and read an intersphinx inventory file at a specified uri, which
can either be a url (e.g. http://...) or a local file system filename.
"""
# See https://stackoverflow.com/a/30981554
class MockConfig(object):
intersphinx_timeout = None
tls_verify = False
class MockApp(object):
srcdir = ''
config = MockConfig()
def warn(self, msg):
warnings.warn(msg)
return intersphinx.fetch_inventory(MockApp(), '', uri)
def test_fetch_inventory_redirection(_read_from_url, InventoryFile, app, status, warning):
intersphinx_setup(app)
_read_from_url().readline.return_value = '# Sphinx inventory version 2'.encode('utf-8')
# same uri and inv, not redirected
_read_from_url().url = 'http://hostname/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/', 'http://hostname/' + INVENTORY_FILENAME)
assert 'intersphinx inventory has moved' not in status.getvalue()
assert InventoryFile.load.call_args[0][1] == 'http://hostname/'
# same uri and inv, redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = 'http://hostname/new/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/', 'http://hostname/' + INVENTORY_FILENAME)
assert status.getvalue() == ('intersphinx inventory has moved: '
'http://hostname/%s -> http://hostname/new/%s\n' %
(INVENTORY_FILENAME, INVENTORY_FILENAME))
assert InventoryFile.load.call_args[0][1] == 'http://hostname/new'
# different uri and inv, not redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = 'http://hostname/new/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/', 'http://hostname/new/' + INVENTORY_FILENAME)
assert 'intersphinx inventory has moved' not in status.getvalue()
assert InventoryFile.load.call_args[0][1] == 'http://hostname/'
# different uri and inv, redirected
status.seek(0)
status.truncate(0)
_read_from_url().url = 'http://hostname/other/' + INVENTORY_FILENAME
fetch_inventory(app, 'http://hostname/', 'http://hostname/new/' + INVENTORY_FILENAME)
assert status.getvalue() == ('intersphinx inventory has moved: '
'http://hostname/new/%s -> http://hostname/other/%s\n' %
(INVENTORY_FILENAME, INVENTORY_FILENAME))
assert InventoryFile.load.call_args[0][1] == 'http://hostname/'
def handle(self, *args, **options):
"""
Docs urls for Classes can differ between Django versions.
This script sets correct urls for specific Classes using bits from
`sphinx.ext.intersphinx` to fetch docs inventory data.
"""
for v in self.django_versions:
cnt = 1
ver_url = self.django_doc_url.format(version=v)
ver_inv_url = ver_url + '/' + self.inv_filename
# get flat list of CBV classes per Django version
qs_lookups = {'module__project_version__version_number': v}
ver_classes = Klass.objects.filter(**qs_lookups).values_list(
'name', flat=True)
self.bless_prints(v, 'Found {0} classes'.format(len(ver_classes)))
self.bless_prints(v, 'Getting inventory @ {0}'.format(ver_inv_url))
# fetch some inventory dataz
# the arg `r.raw` should be a Sphinx instance object..
r = requests.get(ver_inv_url, stream=True)
r.raise_for_status()
invdata = fetch_inventory(r.raw, ver_url, ver_inv_url)
# we only want classes..
for item in invdata[u'py:class']:
# ..which come from one of our sources
if any(
[source in item
for source in settings.CBV_SOURCES.keys()]):
# get class name
inv_klass = item.split('.')[-1]
# save hits to db and update only required classes
for vc in ver_classes:
if vc == inv_klass:
url = invdata[u'py:class'][item][2]
qs_lookups.update({'name': inv_klass})
Klass.objects.filter(**qs_lookups).update(
docs_url=url)
cnt += 1
continue
self.bless_prints(v, 'Updated {0} classes\n'.format(cnt))
def handle(self, *args, **options):
"""
Docs urls for Classes can differ between Django versions.
This script sets correct urls for specific Classes using bits from
`sphinx.ext.intersphinx` to fetch docs inventory data.
"""
for v in self.django_versions:
cnt = 1
ver_url = f'https://docs.djangoproject.com/en/{v}'
ver_inv_url = ver_url + '/' + self.inv_filename
# get flat list of CBV classes per Django version
qs_lookups = {'module__project_version__version_number': v}
ver_classes = Klass.objects.filter(**qs_lookups).values_list(
'name', flat=True)
self.bless_prints(v, f'Found {len(ver_classes)} classes')
self.bless_prints(v, f'Getting inventory @ {ver_inv_url}')
# fetch some inventory dataz
# the arg `r.raw` should be a Sphinx instance object..
r = requests.get(ver_inv_url, stream=True)
r.raise_for_status()
invdata = fetch_inventory(r.raw, ver_url, ver_inv_url)
# we only want classes..
for item in invdata['py:class']:
# ..which come from one of our sources
if any(source in item for source in settings.CBV_SOURCES.keys()):
# get class name
inv_klass = item.split('.')[-1]
# save hits to db and update only required classes
for vc in ver_classes:
if vc == inv_klass:
url = invdata['py:class'][item][2]
qs_lookups.update({
'name': inv_klass
})
Klass.objects.filter(**qs_lookups).update(
docs_url=url)
cnt += 1
continue
self.bless_prints(v, f'Updated {cnt} classes\n')
def fetch_intersphinx_inventory(uri):
"""
Fetch and read an intersphinx inventory file at a specified uri,
which can either be a url (e.g. http://...) or a local file system
filename.
"""
# See https://stackoverflow.com/a/30981554
class MockConfig(object):
intersphinx_timeout = None
tls_verify = False
class MockApp(object):
srcdir = ''
config = MockConfig()
def warn(self, msg):
warnings.warn(msg)
return intersphinx.fetch_inventory(MockApp(), '', uri)
def load_cache_from_url(url):
"""Load some inventory file as raw data.
Args:
url (str):
The website address that points to a objects.inv file.
e.g. "https://foo_bar_name.readthedocs.io/en/latest/objects.inv".
Returns:
dict[str, dict[str, tuple[str, str, str, str]]]:
Each directive target type, its namespace, and it's file-path/URL information.
e.g. {
"py:method": {
"fake_project.basic.MyKlass.get_method": (
"fake_project",
"",
"api/fake_project.html#fake_project.basic.MyKlass.get_method",
"-",
)
}
}
"""
class MockConfiguration(object): # pylint: disable=too-few-public-methods
"""A fake set of settings for intersphinx to pass-through."""
intersphinx_timeout = None # type: int
tls_verify = False
class MockApplication(object): # pylint: disable=too-few-public-methods
"""A fake state machine for intersphinx to consume and pass-through."""
srcdir = ""
config = MockConfiguration()
@staticmethod
def warn(message):
"""Send a warning if bad-formatted text is encountered."""
warnings.warn(message)
return intersphinx.fetch_inventory(MockApplication(), "", url)
def handle(self, *args, **options):
"""
Docs urls for Classes can differ between Django versions.
This script sets correct urls for specific Classes using bits from
`sphinx.ext.intersphinx` to fetch docs inventory data.
"""
for v in self.django_versions:
cnt = 1
ver_url = self.django_doc_url.format(version=v)
ver_inv_url = ver_url + '/' + self.inv_filename
# get flat list of CBV classes per Django version
qs_lookups = {'module__project_version__version_number': v}
ver_classes = Klass.objects.filter(**qs_lookups).values_list(
'name', flat=True)
self.bless_prints(v, 'Found {0} classes'.format(len(ver_classes)))
self.bless_prints(v, 'Getting inventory @ {0}'.format(ver_inv_url))
# fetch some inventory dataz
# the arg `None` should be a Sphinx instance object..
invdata = fetch_inventory(None, ver_url, ver_inv_url)
# we only want classes..
for item in invdata[u'py:class']:
# ..which come from one of our sources
if any ([source in item for source in settings.CBV_SOURCES.keys()]):
# get class name
inv_klass = item.split('.')[-1]
# save hits to db and update only required classes
for vc in ver_classes:
if vc == inv_klass:
url = invdata[u'py:class'][item][2]
qs_lookups.update({
'name': inv_klass
})
Klass.objects.filter(**qs_lookups).update(
docs_url=url)
cnt += 1
continue
self.bless_prints(v, 'Updated {0} classes\n'.format(cnt))
def load_cache(path):
"""Load some inventory file as raw data.
Args:
path (str):
The absolute path where an inventory file can be found.
Returns:
dict[str, dict[str, tuple[str, str, str, str]]]:
Each directive target type, its namespace, and it's file-path/URL information.
e.g. {
"py:method": {
"fake_project.basic.MyKlass.get_method": (
"fake_project",
"",
"api/fake_project.html#fake_project.basic.MyKlass.get_method",
"-",
)
}
}
"""
class MockConfiguration(object): # pylint: disable=too-few-public-methods
"""A fake set of settings for intersphinx to pass-through."""
intersphinx_timeout = None # type: int
tls_verify = False
class MockApplication(object): # pylint: disable=too-few-public-methods
"""A fake state machine for intersphinx to consume and pass-through."""
srcdir = ""
config = MockConfiguration()
@staticmethod
def warn(message):
"""Send a warning if bad-formatted text is encountered."""
warnings.warn(message)
return intersphinx.fetch_inventory(MockApplication(), "", path)
def get_known_blocks():
from sphinx.ext.intersphinx import fetch_inventory #@UnresolvedImport
''' Returns a map block -> url '''
bases = [
'http://andreacensi.github.com/be1008/',
'http://andreacensi.github.com/procgraph/',
'http://andreacensi.github.com/procgraph_rawseeds/']
result = {}
for base in bases:
url = base + '/objects.inv'
map = fetch_inventory(None, base, url);
labels = map['std:label']
for key in labels:
if key.startswith('block:'):
block = key[6:]
url = labels[key][2].split()[0]
result[block] = url
return result
def verify_modules(build_root="_build"):
"""Verify modules included.
:type build_root: str
:param build_root: The root of the directory where docs are built into.
Defaults to ``_build``.
"""
object_inventory_relpath = os.path.join(build_root, "html", "objects.inv")
mock_uri = ""
inventory = fetch_inventory(SphinxApp, mock_uri, object_inventory_relpath)
sphinx_mods = set(inventory["py:module"].keys())
public_mods = set()
for package in PACKAGES:
library_dir = os.path.join(BASE_DIR, package, "google", "cloud")
package_mods = get_public_modules(library_dir,
base_package="google.cloud")
public_mods.update(package_mods)
if not sphinx_mods <= public_mods:
unexpected_mods = sphinx_mods - public_mods
message = [
"Unexpected error. There were modules referenced by "
"Sphinx that are not among the public modules."
]
message.extend(["- %s" % (mod, ) for mod in unexpected_mods])
print("\n".join(message), file=sys.stderr)
sys.exit(1)
undocumented_mods = public_mods - sphinx_mods
# Remove ignored modules.
undocumented_mods -= IGNORED_MODULES
if undocumented_mods:
message_parts = ["Found undocumented public modules:"]
message_parts.extend(
["- " + mod_name for mod_name in sorted(undocumented_mods)])
print("\n".join(message_parts), file=sys.stderr)
sys.exit(1)
def verify_modules(build_root='_build'):
"""Verify modules included.
:type build_root: str
:param build_root: The root of the directory where docs are built into.
Defaults to ``_build``.
"""
object_inventory_relpath = os.path.join(build_root, 'html', 'objects.inv')
mock_uri = ''
inventory = fetch_inventory(SphinxApp, mock_uri, object_inventory_relpath)
sphinx_mods = set(inventory['py:module'].keys())
public_mods = set()
for package in PACKAGES:
library_dir = os.path.join(PROJECT_ROOT, package, 'google', 'cloud')
package_mods = get_public_modules(library_dir,
base_package='google.cloud')
public_mods.update(package_mods)
if not sphinx_mods <= public_mods:
unexpected_mods = sphinx_mods - public_mods
message = [
'Unexpected error. There were modules referenced by '
'Sphinx that are not among the public modules.'
]
message.extend(['- %s' % (mod, ) for mod in unexpected_mods])
print('\n'.join(message), file=sys.stderr)
sys.exit(1)
undocumented_mods = public_mods - sphinx_mods
# Remove ignored modules.
undocumented_mods -= IGNORED_MODULES
if undocumented_mods:
message_parts = ['Found undocumented public modules:']
message_parts.extend(
['- ' + mod_name for mod_name in sorted(undocumented_mods)])
print('\n'.join(message_parts), file=sys.stderr)
sys.exit(1)
def verify_modules(build_root='_build'):
"""Verify modules included.
:type build_root: str
:param build_root: The root of the directory where docs are built into.
Defaults to ``_build``.
"""
object_inventory_relpath = os.path.join(build_root, 'html', 'objects.inv')
mock_uri = ''
inventory = fetch_inventory(SphinxApp, mock_uri,
object_inventory_relpath)
sphinx_mods = set(inventory['py:module'].keys())
public_mods = set()
for package in PACKAGES:
library_dir = os.path.join(PROJECT_ROOT, package, 'google', 'cloud')
package_mods = get_public_modules(library_dir,
base_package='google.cloud')
public_mods.update(package_mods)
if not sphinx_mods <= public_mods:
unexpected_mods = sphinx_mods - public_mods
message = ['Unexpected error. There were modules referenced by '
'Sphinx that are not among the public modules.']
message.extend(['- %s' % (mod,) for mod in unexpected_mods])
print('\n'.join(message), file=sys.stderr)
sys.exit(1)
undocumented_mods = public_mods - sphinx_mods
# Remove ignored modules.
undocumented_mods -= IGNORED_MODULES
if undocumented_mods:
message_parts = ['Found undocumented public modules:']
message_parts.extend(['- ' + mod_name
for mod_name in sorted(undocumented_mods)])
print('\n'.join(message_parts), file=sys.stderr)
sys.exit(1)
def main():
"""Main script to verify modules included."""
mock_uri = ""
inventory = fetch_inventory(SphinxApp, mock_uri, OBJECT_INVENTORY_RELPATH)
sphinx_mods = set(inventory["py:module"].keys())
library_dir = os.path.join(BASE_DIR, "gcloud")
public_mods = get_public_modules(library_dir, base_package="gcloud")
public_mods = set(public_mods)
if not sphinx_mods <= public_mods:
message = "Unexpected error. There were modules referenced by " "Sphinx that are not among the public modules."
print(message, file=sys.stderr)
sys.exit(1)
undocumented_mods = public_mods - sphinx_mods
# Remove ignored modules.
undocumented_mods -= IGNORED_MODULES
if undocumented_mods:
message_parts = ["Found undocumented public modules:"]
message_parts.extend(["- " + mod_name for mod_name in sorted(undocumented_mods)])
print("\n".join(message_parts), file=sys.stderr)
sys.exit(1)
def update_intersphinx(version_pk):
version_data = api.version(version_pk).get()
del version_data['resource_uri']
project_data = version_data['project']
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
version_data['project'] = project
version = Version(**version_data)
object_file = version.project.find('objects.inv', version.slug)[0]
path = version.project.rtd_build_path(version.slug)
if not path:
log.warning("%s has no path" % version)
return None
app = DictObj()
app.srcdir = path
try:
inv = fetch_inventory(app, path, object_file)
except TypeError, e:
log.error("Failed to fetch inventory for %s" % version, exc_info=True)
return None
def update_intersphinx(version_pk):
version_data = api.version(version_pk).get()
del version_data['resource_uri']
project_data = version_data['project']
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
version_data['project'] = project
version = Version(**version_data)
object_file = version.project.find('objects.inv', version.slug)[0]
path = version.project.rtd_build_path(version.slug)
if not path:
log.warning("%s has no path" % version)
return None
app = DictObj()
app.srcdir = path
try:
inv = fetch_inventory(app, path, object_file)
except TypeError, e:
log.error("Failed to fetch inventory for %s" % version, exc_info=True)
return None
def get_python_standard_modules(version=None):
version = '{}.{}'.format(sys.version_info[0],
sys.version_info[1]) if not version else version
module_cache_file = 'python{}_modules.csv'.format(version.replace(
'.', '_'))
if os.path.exists(module_cache_file):
print('read python {} standard modules'.format(version))
modules = list()
with open(module_cache_file, 'r') as fr:
while True:
line = fr.readline()
if not line:
break
modules.append(line.strip())
else:
from sphinx.ext.intersphinx import fetch_inventory
print('fetch python {} standard modules'.format(version))
url = "http://docs.python.org/{}/objects.inv".format(version)
modules = sorted(
list(fetch_inventory(DummyApp(), "", url).get("py:module").keys()))
with open(module_cache_file, 'w') as fw:
fw.write('\n'.join(modules))
return modules
def __prase_inv_file(filename):
class MockConfig:
intersphinx_timeout: int = None
tls_verify = False
user_agent = None
class MockApp:
srcdir = ''
config = MockConfig()
def warn(self, msg: str) -> None:
print(msg, file=sys.stderr)
invdata = fetch_inventory(MockApp(), '', filename) # type: ignore
result = {}
for key in sorted(invdata or {}):
result[key] = {}
data = result[key]
for entry, einfo in sorted(invdata[key].items()):
data[entry] = {
"title": einfo[3] if einfo[3] != '-' else '',
"link": einfo[2]
}
return result
def assert_fetch_inventory_success(self, filename):
app = MockApp()
inv_data = fetch_inventory(app, '', filename)
self.assertEqual(0, app.warning_count, app.warnings)
self.assertIsInstance(inv_data, dict)
return inv_data
logging.basicConfig(level=level, format="[%(levelname)s]: %(message)s")
# Be kind and cache results while we're woking on this locally.
requests_cache.install_cache("sphinx-doc")
# Rather than trying to use Sphinx directly, just mock out the bare minimum.
app = mock_app()
# objects.inv is amazing, it's useful in so many ways! With it we can
# - automatically discover all the (documented!) roles and directives that Sphinx provides.
# - ensure all cross-referencing roles resolve correctly, by providing implementations that
# use it look up the correct url.
# - ensure all descriptive directives produce sections with sensible ids we can use when
# selecting subsections of doctrees to render out.
base_url = "https://www.sphinx-doc.org/en/master/"
objects_inv = fetch_inventory(app, base_url, f"{base_url}objects.inv")
# Register mocks for the additional roles and directives that Sphinx provides
directives_, roles_ = mock_directives_roles(app, objects_inv)
files = [
("directives.json", "rst:directive", directives_),
("roles.json", "rst:role", roles_),
]
for filename, obj_type, objects in files:
logger.info("Generating %s documentation", obj_type)
mapping = build_map(base_url, objects_inv, obj_type)
docs = generate_documentation(mapping, objects)
with (out / filename).open("w") as f:
def _create_intersphinx_data(version, commit, build):
"""
Create intersphinx data for this version.
:param version: Version instance
:param commit: Commit that updated path
:param build: Build id
"""
if not version.is_sphinx_type:
return
html_storage_path = version.project.get_storage_path(
type_='html', version_slug=version.slug, include_file=False
)
json_storage_path = version.project.get_storage_path(
type_='json', version_slug=version.slug, include_file=False
)
object_file = build_media_storage.join(html_storage_path, 'objects.inv')
if not build_media_storage.exists(object_file):
log.debug('No objects.inv, skipping intersphinx indexing.')
return
type_file = build_media_storage.join(json_storage_path, 'readthedocs-sphinx-domain-names.json')
types = {}
titles = {}
if build_media_storage.exists(type_file):
try:
data = json.load(build_media_storage.open(type_file))
types = data['types']
titles = data['titles']
except Exception:
log.exception('Exception parsing readthedocs-sphinx-domain-names.json')
# These classes are copied from Sphinx
# https://github.com/sphinx-doc/sphinx/blob/d79d041f4f90818e0b495523fdcc28db12783caf/sphinx/ext/intersphinx.py#L400-L403 # noqa
class MockConfig:
intersphinx_timeout = None
tls_verify = False
user_agent = None
class MockApp:
srcdir = ''
config = MockConfig()
def warn(self, msg):
log.warning('Sphinx MockApp.', msg=msg)
# Re-create all objects from the new build of the version
object_file_url = build_media_storage.url(object_file)
if object_file_url.startswith('/'):
# Filesystem backed storage simply prepends MEDIA_URL to the path to get the URL
# This can cause an issue if MEDIA_URL is not fully qualified
object_file_url = settings.RTD_INTERSPHINX_URL + object_file_url
invdata = intersphinx.fetch_inventory(MockApp(), '', object_file_url)
for key, value in sorted(invdata.items() or {}):
domain, _type = key.split(':', 1)
for name, einfo in sorted(value.items()):
# project, version, url, display_name
# ('Sphinx', '1.7.9', 'faq.html#epub-faq', 'Epub info')
try:
url = einfo[2]
if '#' in url:
doc_name, anchor = url.split(
'#',
# The anchor can contain ``#`` characters
maxsplit=1
)
else:
doc_name, anchor = url, ''
display_name = einfo[3]
except Exception:
log.exception(
'Error while getting sphinx domain information. Skipping...',
project_slug=version.project.slug,
version_slug=version.slug,
sphinx_domain='{domain}->{name}',
)
continue
# HACK: This is done because the difference between
# ``sphinx.builders.html.StandaloneHTMLBuilder``
# and ``sphinx.builders.dirhtml.DirectoryHTMLBuilder``.
# They both have different ways of generating HTML Files,
# and therefore the doc_name generated is different.
# More info on: http://www.sphinx-doc.org/en/master/usage/builders/index.html#builders
# Also see issue: https://github.com/readthedocs/readthedocs.org/issues/5821
if doc_name.endswith('/'):
doc_name += 'index.html'
html_file = HTMLFile.objects.filter(
project=version.project, version=version,
path=doc_name, build=build,
).first()
if not html_file:
log.debug(
'HTMLFile object not found.',
project_slug=version.project.slug,
version_slug=version.slug,
build_id=build,
doc_name=doc_name
)
# Don't create Sphinx Domain objects
# if the HTMLFile object is not found.
continue
SphinxDomain.objects.create(
project=version.project,
version=version,
html_file=html_file,
domain=domain,
name=name,
display_name=display_name,
type=_type,
type_display=types.get(f'{domain}:{_type}', ''),
doc_name=doc_name,
doc_display=titles.get(doc_name, ''),
anchor=anchor,
commit=commit,
build=build,
)
version_data['project'] = project
version = Version(**version_data)
try:
object_file = version.project.find('objects.inv', version.slug)[0]
except IndexError, e:
print "Failed to find objects file"
return None
path = version.project.rtd_build_path(version.slug)
if not path:
log.warning("%s has no path" % version)
return None
app = DictObj()
app.srcdir = path
try:
inv = fetch_inventory(app, path, object_file)
except TypeError, e:
log.error("Failed to fetch inventory for %s" % version, exc_info=True)
return None
# TODO: I'm entirelty not sure this is even close to correct.
# There's a lot of info I'm throwing away here; revisit later?
for keytype in inv:
for term in inv[keytype]:
try:
_, _, url, title = inv[keytype][term]
if not title or title == '-':
if '#' in url:
title = url.rsplit('#')[-1]
else:
title = url
find_str = "rtd-builds/latest"
class FakeConfig:
intersphinx_timeout = None
tls_verify = True
class FakeApp:
srcdir = ""
config = FakeConfig()
for version_info in VERSIONS:
version = ".".join(version_info)
url = URL.format(version)
invdata = fetch_inventory(FakeApp(), "", url)
modules = set()
for module in invdata["py:module"]:
root, *_ = module.split(".")
if root not in ["__future__", "__main__"]:
modules.add(root)
path = PATH.format("".join(version_info))
with open(path, "w") as fp:
docstring = DOCSTRING.format(version)
fp.write('"""{}"""\n\n'.format(docstring))
fp.write("stdlib = [\n")
for module in sorted(modules):
fp.write(' "{}",\n'.format(module))
fp.write("]\n")