def _redirect_first_link_in_contents(request,
content_id,
version=None,
lang=None,
is_raw=False):
"""
Given a version and a content service, redirect to the first link in it's
navigation.
"""
if not lang:
lang = portal_helper.get_preferred_language(request)
# Get the directory paths on the filesystem, AND of the URL.
content_path, url_prefix = url_helper.get_full_content_path(
content_id, lang, version)
# If the content doesn't exist yet, try generating it.
navigation = None
try:
navigation, menu_path = menu_helper.get_menu(content_id, lang, version)
assert os.path.exists(content_path)
except Exception, e:
if type(e) in [AssertionError, IOError]:
if type(e) == IOError:
menu_path = e[1]
_generate_content(os.path.dirname(menu_path), content_path,
content_id, lang, version)
if not navigation:
navigation, menu_path = menu_helper.get_menu(
content_id, lang, version)
else:
raise e
def _find_matching_equivalent_page_for(path, request, lang=None, version=None):
content_id, old_lang, old_version = url_helper.get_parts_from_url_path(
path)
# Try to find the page in this content's navigation.
menu_path = menu_helper.get_menu_path_cache(content_id, old_lang,
old_version)
if content_id in ['book']:
path = os.path.join(
os.path.dirname(path),
'README.%smd' % ('' if old_lang == 'en' else 'cn.'))
matching_link = None
if menu_path.endswith('.json'):
with open(menu_path, 'r') as menu_file:
menu = json.loads(menu_file.read())
path_to_seek = url_helper.get_raw_page_path_from_html(path)
if lang:
# We are switching to new language
matching_link = menu_helper.find_all_languages_for_link(
path_to_seek, old_lang, menu['sections'], lang)
version = old_version
else:
# We are switching to new version
new_menu_path = menu_helper.get_menu_path_cache(
content_id, old_lang, version)
with open(new_menu_path, 'r') as new_menu_file:
new_menu = json.loads(new_menu_file.read())
# Try to find this link in the new menu path.
# NOTE: We account for the first and last '/'.
matching_link = menu_helper.find_link_in_sections(
new_menu['sections'], path_to_seek)
lang = old_lang
if matching_link:
content_path, url_prefix = url_helper.get_full_content_path(
content_id, lang, version)
# Because READMEs get replaced by index.htmls, so we have to undo that.
if content_id in ['book'] and old_lang != lang:
matching_link = os.path.join(
os.path.dirname(matching_link),
'index.%shtml' % ('' if lang == 'en' else 'cn.'))
return redirect(url_helper.get_url_path(url_prefix, matching_link))
# If no such page is found, redirect to first link in the content.
else:
return _redirect_first_link_in_contents(request, content_id, version,
lang)
def reload_docs(request):
try:
path = urlparse(request.META.get('HTTP_REFERER')).path
# Get all the params from the URL and settings to generate new content.
content_id, lang, version = url_helper.get_parts_from_url_path(path)
menu_path = menu_helper.get_menu_path_cache(content_id, lang, version)
content_path, url_prefix = url_helper.get_full_content_path(
content_id, lang, version)
# Generate new content.
_generate_content(os.path.dirname(menu_path), content_path, content_id,
lang, version)
return redirect(path)
except Exception as e:
return HttpResponseServerError("Cannot reload docs: %s" % e)
class DocumentationGenerator():
GITHUB_REPO_URL = 'https://github.com/PaddlePaddle/Paddle/blob/'
def __init__(self,
source_dir,
destination_dir,
content_id,
raw_version='develop',
lang=None):
self.source_dir = source_dir
self.destination_dir = destination_dir
if content_id == 'paddle-mobile':
content_id = 'mobile'
self.content_id = content_id
self.lang = lang
self.raw_version = raw_version
self.version = sanitize_version(raw_version)
def docs(self):
"""
Strip out the static and extract the body contents, ignoring the TOC,
headers, and body.
"""
try:
menu_path = menu_helper.get_menu('docs', self.lang or 'en',
self.version)[1]
except IOError, e:
menu_path = e[1]
langs = [self.lang] if self.lang else ['en', 'zh']
new_menu = None
# Set up new_menu to indicate that we need to parse html to create new menu
if not settings.SUPPORT_MENU_JSON:
new_menu = {'sections': []}
destination_dir = self.destination_dir
for lang in langs:
if not destination_dir:
destination_dir = url_helper.get_full_content_path(
'docs', lang, self.version)[0]
self.generated_dir = _get_new_generated_dir('docs', lang)
if not new_menu:
sphinx_utils.build_sphinx_index_from_menu(menu_path, lang)
print "Use Sphinx comment: sphinx-build -b html -c %s %s %s" % (
os.path.join(settings.SPHINX_CONFIG_DIR,
lang), self.source_dir, self.generated_dir)
call([
'sphinx-build', '-b', 'html', '-c',
os.path.join(settings.SPHINX_CONFIG_DIR, lang),
self.source_dir, self.generated_dir
])
# Generate a menu from the rst root menu if it doesn't exist.
if new_menu:
# FORCEFULLY generate for both languages.
for lang in (['en', 'zh'] if settings.ENV
in ['production', 'staging'] else langs):
sphinx_utils.create_sphinx_menu(
self.source_dir, 'docs', lang, self.version, new_menu,
_get_new_generated_dir(self.content_id, lang))
for lang in langs:
if self.lang:
lang_destination_dir = destination_dir
else:
lang_destination_dir = os.path.join(destination_dir, 'docs',
lang, self.version)
self.strip_sphinx_documentation(lang, lang_destination_dir)
# shutil.rmtree(generated_dir)
if new_menu:
self.save_menu(new_menu, menu_path)
else:
for lang in (['en', 'zh'] if settings.ENV
in ['production', 'staging'] else langs):
sphinx_utils.remove_sphinx_menu(menu_path, lang)
def documentation(source_dir, destination_dir, content_id, version,
original_lang):
"""
Strip out the static and extract the body contents, ignoring the TOC,
headers, and body.
"""
menu_path = source_dir + '/menu.json'
if original_lang:
langs = [original_lang]
else:
langs = ['en', 'zh']
new_menu = None
# Set up new_menu to indicate that we need to parse html to create new menu
if not settings.SUPPORT_MENU_JSON:
new_menu = {'sections': []}
for lang in langs:
if not destination_dir:
destination_dir = url_helper.get_full_content_path(
'docs', lang, version)[0]
generated_dir = _get_new_generated_dir(content_id, lang)
if not new_menu:
_build_sphinx_index_from_menu(menu_path, lang)
# HACK: If this is chinese API folder, make a copy of api/index_en.rst.
if lang == 'zh' and content_id == 'api':
copyfile(os.path.join(source_dir, 'index_en.rst'),
os.path.join(source_dir, 'index_cn.rst'))
print "Use Sphinx comment: sphinx-build -b html -c %s %s %s" % (
os.path.join(settings.SPHINX_CONFIG_DIR,
lang), source_dir, generated_dir)
call([
'sphinx-build', '-b', 'html', '-c',
os.path.join(settings.SPHINX_CONFIG_DIR, lang), source_dir,
generated_dir
])
# Generate a menu from the rst root menu if it doesn't exist.
if new_menu:
# FORCEFULLY generate for both languages.
for lang in (['en', 'zh']
if settings.ENV in ['production', 'staging'] else langs):
generated_dir = _get_new_generated_dir(content_id, lang)
with open(
os.path.join(
generated_dir, 'index_%s.html' %
('cn' if lang == 'zh' else 'en'))) as index_file:
navs = BeautifulSoup(index_file, 'lxml').findAll(
'nav', class_='doc-menu-vertical')
assert navs > 0
links_container = navs[0].find('ul', recursive=False)
if links_container:
for link in links_container.find_all('li',
recursive=False):
_create_sphinx_menu(new_menu['sections'], link,
'documentation', lang, version,
source_dir, content_id == 'docs')
for lang in langs:
if original_lang:
lang_destination_dir = destination_dir
else:
lang_destination_dir = os.path.join(destination_dir, content_id,
lang, version)
generated_dir = _get_new_generated_dir(content_id, lang)
strip_sphinx_documentation(source_dir, generated_dir,
lang_destination_dir, lang, version)
# shutil.rmtree(generated_dir)
if new_menu:
with open(menu_path, 'w') as menu_file:
menu_file.write(json.dumps(new_menu, indent=4))
else:
_remove_sphinx_menu(menu_path, lang)
def _find_matching_equivalent_page_for(path, request, lang=None, version=None):
content_id, old_lang, old_version = url_helper.get_parts_from_url_path(
path)
# Try to find the page in this content's navigation.
menu_path = menu_helper.get_menu_path_cache(content_id, old_lang,
old_version)
if content_id in ['book']:
path = os.path.join(
os.path.dirname(path),
'README.%smd' % ('' if old_lang == 'en' else 'cn.'))
matching_link = None
if menu_path.endswith('.json'):
with open(menu_path, 'r') as menu_file:
menu = json.loads(menu_file.read())
path_to_seek = url_helper.get_raw_page_path_from_html(path)
# HACK: If this is an API lookup, forcefully adapt to the naming
# convention of api_cn/name_cn (and vice versa) for the paths to seek.
# This is a result of the Chinese API introduction in v1.2
if not old_version < '1.2' and path_to_seek[0].startswith(
'api/') or path_to_seek[0].startswith('api_') and lang:
new_path_to_seek = []
for p2s in list(path_to_seek):
extensionless_path, extension = os.path.splitext(
p2s.replace('api/', 'api_cn/') if old_lang ==
'en' else p2s.replace('api_cn/', 'api/'))
new_path_to_seek.append((
(extensionless_path + '_cn'
) if old_lang == 'en' else extensionless_path[:-3]) +
extension)
path_to_seek = tuple(new_path_to_seek)
if lang:
# HACK: Since we failed to find a way make a merged menu.json.
new_menu_path = menu_helper.get_menu_path_cache(
content_id, lang, old_version)
with open(new_menu_path, 'r') as new_menu_file:
new_menu = json.loads(new_menu_file.read())
# We are switching to new language
matching_link = menu_helper.find_all_languages_for_link(
path_to_seek, old_lang, new_menu['sections'], lang)
version = old_version
else:
# We are switching to new version
new_menu_path = menu_helper.get_menu_path_cache(
content_id, old_lang, version)
with open(new_menu_path, 'r') as new_menu_file:
new_menu = json.loads(new_menu_file.read())
# Try to find this link in the new menu path.
# NOTE: We account for the first and last '/'.
matching_link = menu_helper.find_link_in_sections(
new_menu['sections'], path_to_seek)
lang = old_lang
if matching_link:
content_path, url_prefix = url_helper.get_full_content_path(
content_id, lang, version)
# Because READMEs get replaced by index.htmls, so we have to undo that.
if content_id in ['book'] and old_lang != lang:
matching_link = os.path.join(
os.path.dirname(matching_link),
'index.%shtml' % ('' if lang == 'en' else 'cn.'))
return redirect(url_helper.get_url_path(url_prefix, matching_link))
# If no such page is found, redirect to first link in the content.
else:
return _redirect_first_link_in_contents(request, content_id, version,
lang)
menu_path = e[1]
if original_lang:
langs = [original_lang]
else:
langs = ['en', 'zh']
new_menu = None
# Set up new_menu to indicate that we need to parse html to create new menu
if not settings.SUPPORT_MENU_JSON:
new_menu = { 'sections': [] }
for lang in langs:
if not destination_dir:
destination_dir = url_helper.get_full_content_path(
'docs', lang, version)[0]
generated_dir = _get_new_generated_dir('docs', lang)
if not new_menu:
_build_sphinx_index_from_menu(menu_path, lang)
print "Use Sphinx comment: sphinx-build -b html -c %s %s %s" % (
os.path.join(settings.SPHINX_CONFIG_DIR, lang), source_dir, generated_dir)
call(['sphinx-build', '-b', 'html', '-c',
os.path.join(settings.SPHINX_CONFIG_DIR, lang),
source_dir, generated_dir])
# Generate a menu from the rst root menu if it doesn't exist.
if new_menu:
