def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact=False,
charset='utf-8', force_all=False,
excluded=Matcher([])):
# type: (List[unicode], unicode, List[unicode], bool, unicode, bool, Matcher) -> Set[CatalogInfo] # NOQA
"""
:param list locale_dirs:
list of path as `['locale_dir1', 'locale_dir2', ...]` to find
translation catalogs. Each path contains a structure such as
`<locale>/LC_MESSAGES/domain.po`.
:param str locale: a language as `'en'`
:param list domains: list of domain names to get. If empty list or None
is specified, get all domain names. default is None.
:param boolean gettext_compact:
* False: keep domains directory structure (default).
* True: domains in the sub directory will be merged into 1 file.
:param boolean force_all:
Set True if you want to get all catalogs rather than updated catalogs.
default is False.
:return: [CatalogInfo(), ...]
"""
catalogs = set() # type: Set[CatalogInfo]
if not locale:
return catalogs # locale is not specified
for locale_dir in locale_dirs:
if not locale_dir:
continue # skip system locale directory
base_dir = path.join(locale_dir, locale, 'LC_MESSAGES')
if not path.exists(base_dir):
continue # locale path is not found
for dirpath, dirnames, filenames in walk(base_dir, followlinks=True):
filenames = [f for f in filenames if f.endswith('.po')]
for filename in filenames:
if excluded(path.join(relpath(dirpath, base_dir), filename)):
continue
base = path.splitext(filename)[0]
domain = relpath(path.join(dirpath, base), base_dir)
if gettext_compact and path.sep in domain:
domain = path.split(domain)[0]
domain = domain.replace(path.sep, SEP)
if domains and domain not in domains:
continue
cat = CatalogInfo(base_dir, domain, charset)
if force_all or cat.is_outdated():
catalogs.add(cat)
return catalogs
def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact=None,
charset='utf-8', force_all=False,
excluded=Matcher([])):
# type: (List[str], str, List[str], bool, str, bool, Matcher) -> Set[CatalogInfo]
"""
:param list locale_dirs:
list of path as `['locale_dir1', 'locale_dir2', ...]` to find
translation catalogs. Each path contains a structure such as
`<locale>/LC_MESSAGES/domain.po`.
:param str locale: a language as `'en'`
:param list domains: list of domain names to get. If empty list or None
is specified, get all domain names. default is None.
:param boolean force_all:
Set True if you want to get all catalogs rather than updated catalogs.
default is False.
:return: [CatalogInfo(), ...]
"""
if gettext_compact is not None:
warnings.warn('gettext_compact argument for find_catalog_source_files() '
'is deprecated.', RemovedInSphinx30Warning, stacklevel=2)
catalogs = set() # type: Set[CatalogInfo]
if not locale:
return catalogs # locale is not specified
for locale_dir in locale_dirs:
if not locale_dir:
continue # skip system locale directory
base_dir = path.join(locale_dir, locale, 'LC_MESSAGES')
if not path.exists(base_dir):
continue # locale path is not found
for dirpath, dirnames, filenames in os.walk(base_dir, followlinks=True):
filenames = [f for f in filenames if f.endswith('.po')]
for filename in filenames:
if excluded(path.join(relpath(dirpath, base_dir), filename)):
continue
base = path.splitext(filename)[0]
domain = relpath(path.join(dirpath, base), base_dir).replace(path.sep, SEP)
if domains and domain not in domains:
continue
cat = CatalogInfo(base_dir, domain, charset)
if force_all or cat.is_outdated():
catalogs.add(cat)
return catalogs
def find_files(root, suffix=None):
# type: (unicode, bool) -> Generator
for dirpath, dirs, files in os.walk(root, followlinks=True):
dirpath = path(dirpath)
for f in [f for f in files if not suffix or f.endswith(suffix)]: # type: ignore
fpath = dirpath / f
yield relpath(fpath, root)
def build_project_file(self) -> None:
"""Create a project file (.hhp) on outdir."""
# scan project files
project_files = [] # type: List[str]
for root, dirs, files in os.walk(self.outdir):
dirs.sort()
files.sort()
in_staticdir = root.startswith(path.join(self.outdir, '_static'))
for fn in sorted(files):
if (in_staticdir and not fn.endswith('.js')) or fn.endswith('.html'):
fn = relpath(path.join(root, fn), self.outdir)
project_files.append(fn.replace(os.sep, '\\'))
filename = path.join(self.outdir, self.config.htmlhelp_basename + '.hhp')
with open(filename, 'w', encoding=self.encoding, errors='xmlcharrefreplace') as f:
context = {
'outname': self.config.htmlhelp_basename,
'title': self.config.html_title,
'version': self.config.version,
'project': self.config.project,
'lcid': self.lcid,
'master_doc': self.config.master_doc + self.out_suffix,
'files': project_files,
}
body = self.render('project.hhp', context)
f.write(body)
def get_filename_for_node(self, node, docname):
# type: (nodes.Node, unicode) -> unicode
"""Try to get the file which actually contains the doctest, not the
filename of the document it's included in."""
try:
filename = relpath(node.source, self.env.srcdir)\
.rsplit(':docstring of ', maxsplit=1)[0]
except Exception:
filename = self.env.doc2path(docname, base=None)
return filename
def find_catalog_files(docname, srcdir, locale_dirs, lang, compaction):
# type: (unicode, unicode, List[unicode], unicode, bool) -> List[unicode]
if not(lang and locale_dirs):
return []
domain = find_catalog(docname, compaction)
files = [gettext.find(domain, path.join(srcdir, dir_), [lang]) # type: ignore
for dir_ in locale_dirs]
files = [relpath(f, srcdir) for f in files if f] # type: ignore
return files # type: ignore
def find_catalog_files(docname, srcdir, locale_dirs, lang, compaction):
# type: (str, str, List[str], str, bool) -> List[str]
if not(lang and locale_dirs):
return []
domain = find_catalog(docname, compaction)
files = [gettext.find(domain, path.join(srcdir, dir_), [lang])
for dir_ in locale_dirs]
files = [relpath(f, srcdir) for f in files if f]
return files
def path2doc(self, filename):
# type: (unicode) -> Optional[unicode]
"""Return the docname for the filename if the file is document.
*filename* should be absolute or relative to the source directory.
"""
if filename.startswith(self.srcdir):
filename = relpath(filename, self.srcdir)
for suffix in self.config.source_suffix:
if filename.endswith(suffix):
return filename[:-len(suffix)]
return None
def build(self,
force_all: bool = False,
filenames: List[str] = None) -> None:
self.phase = BuildPhase.READING
try:
if force_all:
self.builder.compile_all_catalogs()
self.builder.build_all()
elif filenames:
self.builder.compile_specific_catalogs(filenames)
self.builder.build_specific(filenames)
else:
self.builder.compile_update_catalogs()
self.builder.build_update()
if self._warncount and self.keep_going:
self.statuscode = 1
status = (__('succeeded') if self.statuscode == 0 else
__('finished with problems'))
if self._warncount:
if self.warningiserror:
msg = __(
'build %s, %s warning (with warnings treated as errors).',
'build %s, %s warnings (with warnings treated as errors).',
self._warncount)
else:
msg = __('build %s, %s warning.', 'build %s, %s warnings.',
self._warncount)
logger.info(bold(msg % (status, self._warncount)))
else:
logger.info(bold(__('build %s.') % status))
if self.statuscode == 0 and self.builder.epilog:
logger.info('')
logger.info(self.builder.epilog % {
'outdir': relpath(self.outdir),
'project': self.config.project
})
except Exception as err:
# delete the saved env to force a fresh build next time
envfile = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
if path.isfile(envfile):
os.unlink(envfile)
self.events.emit('build-finished', err)
raise
else:
self.events.emit('build-finished', None)
self.builder.cleanup()
def path2doc(self, filename: str) -> Optional[str]:
"""Return the docname for the filename if the file is a document.
*filename* should be absolute or relative to the source directory.
"""
if filename.startswith(self.srcdir):
filename = relpath(filename, self.srcdir)
for suffix in self.source_suffix:
if filename.endswith(suffix):
filename = path_stabilize(filename)
return filename[:-len(suffix)]
# the file does not have docname
return None
def finish(self):
# type: () -> None
super().finish()
data = {
'version':
self.config.version,
'copyright':
self.config.copyright,
'project':
self.config.project,
'ctime':
datetime.fromtimestamp(timestamp,
ltz).strftime('%Y-%m-%d %H:%M%z'),
}
for textdomain, catalog in status_iterator(
self.catalogs.items(), __("writing message catalogs... "),
"darkgreen", len(self.catalogs), self.app.verbosity,
lambda textdomain__: textdomain__[0]):
# noop if config.gettext_compact is set
ensuredir(path.join(self.outdir, path.dirname(textdomain)))
pofn = path.join(self.outdir, textdomain + '.pot')
output = StringIO()
output.write(POHEADER % data)
for message in catalog.messages:
positions = catalog.metadata[message]
if self.config.gettext_location:
# generate "#: file1:line1\n#: file2:line2 ..."
output.write("#: %s\n" % "\n#: ".join(
"%s:%s" %
(canon_path(relpath(source, self.outdir)), line)
for source, line, _ in positions))
if self.config.gettext_uuid:
# generate "# uuid1\n# uuid2\n ..."
output.write("# %s\n" %
"\n# ".join(uid for _, _, uid in positions))
# message contains *one* line of text ready for translation
message = message.replace('\\', r'\\'). \
replace('"', r'\"'). \
replace('\n', '\\n"\n"')
output.write('msgid "%s"\nmsgstr ""\n\n' % message)
content = output.getvalue()
if should_write(pofn, content):
with open(pofn, 'w', encoding='utf-8') as pofile:
pofile.write(content)
def pofiles(self):
# type: () -> Generator[Tuple[str, str], None, None]
for locale_dir in self.locale_dirs:
basedir = path.join(locale_dir, self.language, 'LC_MESSAGES')
for root, dirnames, filenames in os.walk(basedir):
# skip dot-directories
for dirname in dirnames:
if dirname.startswith('.'):
dirnames.remove(dirname)
for filename in filenames:
if filename.endswith('.po'):
fullpath = path.join(root, filename)
yield basedir, relpath(fullpath, basedir)
def path2doc(self, filename):
# type: (str) -> str
"""Return the docname for the filename if the file is document.
*filename* should be absolute or relative to the source directory.
"""
if filename.startswith(self.srcdir):
filename = relpath(filename, self.srcdir)
for suffix in self.source_suffix:
if filename.endswith(suffix):
return filename[:-len(suffix)]
# the file does not have docname
return None
def path2doc(self, filename):
# type: (unicode) -> unicode
"""Return the docname for the filename if the file is document.
*filename* should be absolute or relative to the source directory.
"""
if filename.startswith(self.srcdir):
filename = relpath(filename, self.srcdir)
for suffix in self.source_suffix:
if filename.endswith(suffix):
return filename[:-len(suffix)]
# the file does not have docname
return None
def find_catalog_files(docname, srcdir, locale_dirs, lang, compaction):
# type: (str, str, List[str], str, bool) -> List[str]
warnings.warn('find_catalog_files() is deprecated.',
RemovedInSphinx40Warning,
stacklevel=2)
if not (lang and locale_dirs):
return []
domain = find_catalog(docname, compaction)
files = [
gettext.find(domain, path.join(srcdir, dir_), [lang])
for dir_ in locale_dirs
]
files = [relpath(f, srcdir) for f in files if f]
return files
def run(self) -> List[Node]:
if 'file' in self.options and self.options['file'].startswith((SEP, os.sep)):
env = self.state.document.settings.env
filename = self.options['file']
if path.exists(filename):
logger.warning(__('":file:" option for csv-table directive now recognizes '
'an absolute path as a relative path from source directory. '
'Please update your document.'),
location=(env.docname, self.lineno))
else:
abspath = path.join(env.srcdir, os_path(self.options['file'][1:]))
docdir = path.dirname(env.doc2path(env.docname))
self.options['file'] = relpath(abspath, docdir)
return super().run()
def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact=False,
charset='utf-8', force_all=False):
# type: (List[unicode], unicode, List[unicode], bool, unicode, bool) -> Set[CatalogInfo]
"""
:param list locale_dirs:
list of path as `['locale_dir1', 'locale_dir2', ...]` to find
translation catalogs. Each path contains a structure such as
`<locale>/LC_MESSAGES/domain.po`.
:param str locale: a language as `'en'`
:param list domains: list of domain names to get. If empty list or None
is specified, get all domain names. default is None.
:param boolean gettext_compact:
* False: keep domains directory structure (default).
* True: domains in the sub directory will be merged into 1 file.
:param boolean force_all:
Set True if you want to get all catalogs rather than updated catalogs.
default is False.
:return: [CatalogInfo(), ...]
"""
catalogs = set() # type: Set[CatalogInfo]
if not locale:
return catalogs # locale is not specified
for locale_dir in locale_dirs:
if not locale_dir:
continue # skip system locale directory
base_dir = path.join(locale_dir, locale, 'LC_MESSAGES')
if not path.exists(base_dir):
continue # locale path is not found
for dirpath, dirnames, filenames in walk(base_dir, followlinks=True):
filenames = [f for f in filenames if f.endswith('.po')]
for filename in filenames:
base = path.splitext(filename)[0]
domain = relpath(path.join(dirpath, base), base_dir)
if gettext_compact and path.sep in domain:
domain = path.split(domain)[0]
domain = domain.replace(path.sep, SEP)
if domains and domain not in domains:
continue
cat = CatalogInfo(base_dir, domain, charset)
if force_all or cat.is_outdated():
catalogs.add(cat)
return catalogs
def finish(self):
# type: () -> None
I18nBuilder.finish(self)
data = dict(
version = self.config.version,
copyright = self.config.copyright,
project = self.config.project,
ctime = datetime.fromtimestamp(
timestamp, ltz).strftime('%Y-%m-%d %H:%M%z'),
)
for textdomain, catalog in status_iterator(self.catalogs.items(), # type: ignore
__("writing message catalogs... "),
"darkgreen", len(self.catalogs),
self.app.verbosity,
lambda textdomain__: textdomain__[0]):
# noop if config.gettext_compact is set
ensuredir(path.join(self.outdir, path.dirname(textdomain)))
pofn = path.join(self.outdir, textdomain + '.pot')
output = StringIO()
output.write(POHEADER % data) # type: ignore
for message in catalog.messages:
positions = catalog.metadata[message]
if self.config.gettext_location:
# generate "#: file1:line1\n#: file2:line2 ..."
output.write("#: %s\n" % "\n#: ".join( # type: ignore
"%s:%s" % (canon_path(relpath(source, self.outdir)), line)
for source, line, _ in positions))
if self.config.gettext_uuid:
# generate "# uuid1\n# uuid2\n ..."
output.write("# %s\n" % "\n# ".join( # type: ignore
uid for _, _, uid in positions))
# message contains *one* line of text ready for translation
message = message.replace('\\', r'\\'). \
replace('"', r'\"'). \
replace('\n', '\\n"\n"')
output.write('msgid "%s"\nmsgstr ""\n\n' % message) # type: ignore
content = output.getvalue()
if should_write(pofn, content):
with open(pofn, 'w', encoding='utf-8') as pofile: # type: ignore
pofile.write(content)
def build(self, force_all=False, filenames=None):
# type: (bool, List[str]) -> None
self.phase = BuildPhase.READING
try:
if force_all:
self.builder.compile_all_catalogs()
self.builder.build_all()
elif filenames:
self.builder.compile_specific_catalogs(filenames)
self.builder.build_specific(filenames)
else:
self.builder.compile_update_catalogs()
self.builder.build_update()
if self._warncount and self.keep_going:
self.statuscode = 1
status = (self.statuscode == 0 and
__('succeeded') or __('finished with problems'))
if self._warncount:
logger.info(bold(__('build %s, %s warning.',
'build %s, %s warnings.', self._warncount) %
(status, self._warncount)))
else:
logger.info(bold(__('build %s.') % status))
if self.statuscode == 0 and self.builder.epilog:
logger.info('')
logger.info(self.builder.epilog % {
'outdir': relpath(self.outdir),
'project': self.config.project
})
except Exception as err:
# delete the saved env to force a fresh build next time
envfile = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
if path.isfile(envfile):
os.unlink(envfile)
self.emit('build-finished', err)
raise
else:
self.emit('build-finished', None)
self.builder.cleanup()
def discover(self, exclude_paths: List[str] = []) -> Set[str]:
"""Find all document files in the source directory and put them in
:attr:`docnames`.
"""
self.docnames = set()
excludes = compile_matchers(exclude_paths + EXCLUDE_PATHS)
for filename in get_matching_files(self.srcdir, excludes): # type: ignore
docname = self.path2doc(filename)
if docname:
if docname in self.docnames:
pattern = os.path.join(self.srcdir, docname) + '.*'
files = [relpath(f, self.srcdir) for f in glob(pattern)]
logger.warning(__('multiple files found for the document "%s": %r\n'
'Use %r for the build.'),
docname, files, self.doc2path(docname), once=True)
elif os.access(os.path.join(self.srcdir, filename), os.R_OK):
self.docnames.add(docname)
else:
logger.warning(__("document not readable. Ignored."), location=docname)
return self.docnames
def _path2doc(self, filename):
# type: (str) -> str
"""Return the docname for the filename if the file is document.
*filename* should be absolute or relative to the source directory.
"""
if filename.startswith(self.srcdir):
filename = relpath(filename, self.srcdir)
for suffix in self.source_suffix:
if filename.endswith(suffix):
if sphinx.version_info[:3] >= (2, 3, 0):
# This line was added in https://github.com/sphinx-doc/sphinx/commit/155f4b0d00e72d16eed47581f2fee75e41c452cf, starting in v2.3.0. It's a patch to fix https://github.com/sphinx-doc/sphinx/issues/6813.
filename = path_stabilize(filename)
return filename[:-len(suffix)]
# The following code was added.
if is_supported_language(filename):
return filename
# This was the existing code.
# the file does not have docname
return None
def build(self, force_all=False, filenames=None):
# type: (bool, List[unicode]) -> None
try:
if force_all:
self.builder.compile_all_catalogs()
self.builder.build_all()
elif filenames:
self.builder.compile_specific_catalogs(filenames)
self.builder.build_specific(filenames)
else:
self.builder.compile_update_catalogs()
self.builder.build_update()
status = (self.statuscode == 0 and
__('succeeded') or __('finished with problems'))
if self._warncount:
logger.info(bold(__('build %s, %s warning.',
'build %s, %s warnings.', self._warncount) %
(status, self._warncount)))
else:
logger.info(bold(__('build %s.') % status))
if self.statuscode == 0 and self.builder.epilog:
logger.info('')
logger.info(self.builder.epilog % {
'outdir': relpath(self.outdir),
'project': self.config.project
})
except Exception as err:
# delete the saved env to force a fresh build next time
envfile = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
if path.isfile(envfile):
os.unlink(envfile)
self.emit('build-finished', err)
raise
else:
self.emit('build-finished', None)
self.builder.cleanup()
def cat2relpath(cat: CatalogInfo) -> str:
return relpath(cat.mo_path, self.env.srcdir).replace(path.sep, SEP)
def cat2relpath(cat):
# type: (CatalogInfo) -> unicode
return relpath(cat.mo_path, self.env.srcdir).replace(path.sep, SEP)
def find_files(root: str, suffix: bool = None) -> Generator[str, None, None]:
for dirpath, dirs, files in os.walk(root, followlinks=True):
dirpath = path(dirpath)
for f in [f for f in files if not suffix or f.endswith(suffix)]: # type: ignore
fpath = dirpath / f
yield relpath(fpath, root)
def cat2relpath(cat):
# type: (CatalogInfo) -> unicode
return relpath(cat.mo_path, self.env.srcdir).replace(path.sep, SEP)
def find_catalog_source_files(locale_dirs,
locale,
domains=None,
gettext_compact=None,
charset='utf-8',
force_all=False,
excluded=Matcher([])):
# type: (List[str], str, List[str], bool, str, bool, Matcher) -> Set[CatalogInfo]
"""
:param list locale_dirs:
list of path as `['locale_dir1', 'locale_dir2', ...]` to find
translation catalogs. Each path contains a structure such as
`<locale>/LC_MESSAGES/domain.po`.
:param str locale: a language as `'en'`
:param list domains: list of domain names to get. If empty list or None
is specified, get all domain names. default is None.
:param boolean force_all:
Set True if you want to get all catalogs rather than updated catalogs.
default is False.
:return: [CatalogInfo(), ...]
"""
warnings.warn('find_catalog_source_files() is deprecated.',
RemovedInSphinx40Warning,
stacklevel=2)
if gettext_compact is not None:
warnings.warn(
'gettext_compact argument for find_catalog_source_files() '
'is deprecated.',
RemovedInSphinx30Warning,
stacklevel=2)
catalogs = set() # type: Set[CatalogInfo]
if not locale:
return catalogs # locale is not specified
for locale_dir in locale_dirs:
if not locale_dir:
continue # skip system locale directory
base_dir = path.join(locale_dir, locale, 'LC_MESSAGES')
if not path.exists(base_dir):
continue # locale path is not found
for dirpath, dirnames, filenames in os.walk(base_dir,
followlinks=True):
filenames = [f for f in filenames if f.endswith('.po')]
for filename in filenames:
if excluded(path.join(relpath(dirpath, base_dir), filename)):
continue
base = path.splitext(filename)[0]
domain = relpath(path.join(dirpath, base),
base_dir).replace(path.sep, SEP)
if domains and domain not in domains:
continue
cat = CatalogInfo(base_dir, domain, charset)
if force_all or cat.is_outdated():
catalogs.add(cat)
return catalogs
def _find_misspellings(self, docname, doctree):
excluded = Matcher(self.config.spelling_exclude_patterns)
if excluded(self.env.doc2path(docname, None)):
return
# Build the document-specific word filter based on any good
# words listed in spelling directives. If we have no such
# words, we want to push an empty list of filters so that we
# can always safely pop the filter stack when we are done with
# this document.
doc_filters = []
good_words = self.env.spelling_document_words.get(docname)
if good_words:
logger.info('Extending local dictionary for %s', docname)
doc_filters.append(filters.IgnoreWordsFilterFactory(good_words))
self.checker.push_filters(doc_filters)
for node in doctree.traverse(docutils.nodes.Text):
if (node.tagname == '#text' and
node.parent and
node.parent.tagname in self.TEXT_NODES):
# Get the location of the text being checked so we can
# report it in the output file. Nodes from text that
# comes in via an 'include' directive does not include
# the full path, so convert all to relative path
# for consistency.
source, node_lineno = docutils.utils.get_source_line(node)
source = osutil.relpath(source)
# Check the text of the node.
misspellings = self.checker.check(node.astext())
for (
word,
suggestions,
context_line,
line_offset
) in misspellings:
# Avoid TypeError on nodes lacking a line number
# This happens for some node originating from docstrings
lineno = node_lineno
if lineno is not None:
lineno += line_offset
msg_parts = [
f'{source}:{lineno}: ',
'Spell check',
red(word),
]
if self.format_suggestions(suggestions) != '':
msg_parts.append(self.format_suggestions(suggestions))
msg_parts.append(context_line)
msg = ': '.join(msg_parts) + '.'
if self.config.spelling_warning:
logger.warning(msg)
elif self.config.spelling_verbose:
logger.info(msg)
yield "%s:%s: (%s) %s %s\n" % (
source, lineno, word,
self.format_suggestions(suggestions),
context_line,
)
self.checker.pop_filters()
return
def _relpath(s: str) -> str:
return canon_path(relpath(s, self.outdir))
def find_files(root, suffix=None):
for dirpath, dirs, files in os.walk(root, followlinks=True):
dirpath = path(dirpath)
for f in [f for f in files if not suffix or f.endswith(suffix)]:
fpath = dirpath / f
yield relpath(fpath, root)
