def is_available(self):
# type: () -> bool
builders = self.config.smartquotes_excludes.get('builders', [])
languages = self.config.smartquotes_excludes.get('languages', [])
if self.document.settings.smart_quotes is False:
# disabled by 3rd party extension (workaround)
return False
elif self.config.smartquotes is False:
# disabled by confval smartquotes
return False
elif self.app.builder.name in builders:
# disabled by confval smartquotes_excludes['builders']
return False
elif self.config.language in languages:
# disabled by confval smartquotes_excludes['languages']
return False
# confirm selected language supports smart_quotes or not
language = self.env.settings['language_code']
for tag in normalize_language_tag(language):
if tag in smartchars.quotes:
return True
else:
return False
def prepare_settings(self, docname):
# type: (unicode) -> None
"""Prepare to set up environment for reading."""
self.temp_data['docname'] = docname
# defaults to the global default, but can be re-set in a document
self.temp_data['default_role'] = self.config.default_role
self.temp_data['default_domain'] = \
self.domains.get(self.config.primary_domain)
self.settings['input_encoding'] = self.config.source_encoding
self.settings['trim_footnote_reference_space'] = \
self.config.trim_footnote_reference_space
self.settings['gettext_compact'] = self.config.gettext_compact
language = self.config.language or 'en'
self.settings['language_code'] = language
if 'smart_quotes' not in self.settings:
self.settings['smart_quotes'] = True
# confirm selected language supports smart_quotes or not
for tag in normalize_language_tag(language):
if tag in smartchars.quotes:
break
else:
self.settings['smart_quotes'] = False
def apply(self):
smart_quotes = self.document.settings.smart_quotes
if not smart_quotes:
return
try:
alternative = smart_quotes.startswith('alt')
except AttributeError:
alternative = False
# print repr(alternative)
document_language = self.document.settings.language_code
lc_smartquotes = self.document.settings.smartquotes_locales
if lc_smartquotes:
smartquotes.smartchars.quotes.update(dict(lc_smartquotes))
# "Educate" quotes in normal text. Handle each block of text
# (TextElement node) as a unit to keep context around inline nodes:
for node in self.document.traverse(nodes.TextElement):
# skip preformatted text blocks and special elements:
if isinstance(node, self.nodes_to_skip):
continue
# nested TextElements are not "block-level" elements:
if isinstance(node.parent, nodes.TextElement):
continue
# list of text nodes in the "text block":
txtnodes = [txtnode for txtnode in node.traverse(nodes.Text)
if not isinstance(txtnode.parent,
nodes.option_string)]
# language: use typographical quotes for language "lang"
lang = node.get_language_code(document_language)
# use alternative form if `smart-quotes` setting starts with "alt":
if alternative:
if '-x-altquot' in lang:
lang = lang.replace('-x-altquot', '')
else:
lang += '-x-altquot'
# drop unsupported subtags:
for tag in utils.normalize_language_tag(lang):
if tag in smartquotes.smartchars.quotes:
lang = tag
break
else: # language not supported: (keep ASCII quotes)
if lang not in self.unsupported_languages:
self.document.reporter.warning('No smart quotes '
'defined for language "%s".'%lang, base_node=node)
self.unsupported_languages.add(lang)
lang = ''
# Iterator educating quotes in plain text:
# (see "utils/smartquotes.py" for the attribute setting)
teacher = smartquotes.educate_tokens(self.get_tokens(txtnodes),
attr=self.smartquotes_action, language=lang)
for txtnode, newtext in zip(txtnodes, teacher):
txtnode.parent.replace(txtnode, nodes.Text(newtext,
rawsource=txtnode.rawsource))
self.unsupported_languages = set() # reset
def apply(self):
smart_quotes = self.document.settings.smart_quotes
if not smart_quotes:
return
try:
alternative = smart_quotes.startswith('alt')
except AttributeError:
alternative = False
# print repr(alternative)
document_language = self.document.settings.language_code
# "Educate" quotes in normal text. Handle each block of text
# (TextElement node) as a unit to keep context around inline nodes:
for node in self.document.traverse(nodes.TextElement):
# skip preformatted text blocks and special elements:
if isinstance(node, (nodes.FixedTextElement, nodes.Special)):
continue
# nested TextElements are not "block-level" elements:
if isinstance(node.parent, nodes.TextElement):
continue
# list of text nodes in the "text block":
txtnodes = [
txtnode for txtnode in node.traverse(nodes.Text)
if not isinstance(txtnode.parent, nodes.option_string)
]
# language: use smart-quotes for language "lang"
lang = node.get_language_code(document_language)
# use alternative form if `smart-quotes` setting starts with "alt":
if alternative:
if '-x-altquot' in lang:
lang = lang.replace('-x-altquot', '')
else:
lang += '-x-altquot'
# drop subtags missing in quotes:
for tag in utils.normalize_language_tag(lang):
if tag in smartquotes.smartchars.quotes:
lang = tag
break
else: # language not supported: (keep ASCII quotes)
if lang not in self.unsupported_languages:
self.document.reporter.warning(
'No smart quotes '
'defined for language "%s".' % lang,
base_node=node)
self.unsupported_languages.add(lang)
lang = ''
# Iterator educating quotes in plain text:
# '2': set all, using old school en- and em- dash shortcuts
teacher = smartquotes.educate_tokens(self.get_tokens(txtnodes),
attr='2',
language=lang)
for txtnode, newtext in zip(txtnodes, teacher):
txtnode.parent.replace(txtnode, nodes.Text(newtext))
self.unsupported_languages = set() # reset
def get_language(language_code, reporter=None):
"""Return module with language localizations.
`language_code` is a "BCP 47" language tag.
If there is no matching module, warn and fall back to English.
"""
# TODO: use a dummy module returning emtpy strings?, configurable?
for tag in normalize_language_tag(language_code):
tag = tag.replace('-','_') # '-' not valid in module names
if tag in _languages:
return _languages[tag]
try:
module = __import__(tag, globals(), locals(), level=1)
except ImportError:
try:
module = __import__(tag, globals(), locals(), level=0)
except ImportError:
continue
_languages[tag] = module
return module
if reporter is not None:
reporter.warning(
'language "%s" not supported: ' % language_code +
'Docutils-generated text will be in English.')
module = __import__('en', globals(), locals(), level=1)
_languages[tag] = module # warn only one time!
return module
def get_language(language_code, reporter=None):
"""Return module with language localizations.
`language_code` is a "BCP 47" language tag.
If there is no matching module, warn and fall back to English.
"""
# TODO: use a dummy module returning emtpy strings?, configurable?
for tag in normalize_language_tag(language_code):
tag = tag.replace('-','_') # '-' not valid in module names
if tag in _languages:
return _languages[tag]
try:
module = __import__(tag, globals(), locals(), level=0)
except ImportError:
try:
module = __import__(tag, globals(), locals(), level=1)
except ImportError:
continue
_languages[tag] = module
return module
if reporter is not None:
reporter.warning(
'language "%s" not supported: ' % language_code +
'Docutils-generated text will be in English.')
module = __import__('en', globals(), locals(), level=1)
_languages[tag] = module # warn only one time!
return module
def is_available(self):
# type: () -> bool
builders = self.config.smartquotes_excludes.get('builders', [])
languages = self.config.smartquotes_excludes.get('languages', [])
if self.document.settings.smart_quotes is False:
# disabled by 3rd party extension (workaround)
return False
elif self.config.smartquotes is False:
# disabled by confval smartquotes
return False
elif self.app.builder.name in builders:
# disabled by confval smartquotes_excludes['builders']
return False
elif self.config.language in languages:
# disabled by confval smartquotes_excludes['languages']
return False
# confirm selected language supports smart_quotes or not
language = self.env.settings['language_code']
for tag in normalize_language_tag(language):
if tag in smartchars.quotes:
return True
else:
return False
def prepare_settings(self, docname):
# type: (unicode) -> None
"""Prepare to set up environment for reading."""
self.temp_data['docname'] = docname
# defaults to the global default, but can be re-set in a document
self.temp_data['default_role'] = self.config.default_role
self.temp_data['default_domain'] = \
self.domains.get(self.config.primary_domain)
self.settings['input_encoding'] = self.config.source_encoding
self.settings['trim_footnote_reference_space'] = \
self.config.trim_footnote_reference_space
self.settings['gettext_compact'] = self.config.gettext_compact
language = self.config.language or 'en'
self.settings['language_code'] = language
if 'smart_quotes' not in self.settings:
self.settings['smart_quotes'] = True
# confirm selected language supports smart_quotes or not
for tag in normalize_language_tag(language):
if tag in smartchars.quotes:
break
else:
self.settings['smart_quotes'] = False
def apply(self):
# We are using our own config variable instead of
# self.document.settings.smart_quotes in order to avoid the builtin
# SmartQuotes to be executed as well
if not settings['M_HTMLSANITY_SMART_QUOTES']:
return
try:
alternative = settings['M_HTMLSANITY_SMART_QUOTES'].startswith('alt')
except AttributeError:
alternative = False
# print repr(alternative)
document_language = extract_document_language(self.document)
# "Educate" quotes in normal text. Handle each block of text
# (TextElement node) as a unit to keep context around inline nodes:
for node in self.document.traverse(nodes.TextElement):
# skip preformatted text blocks and special elements:
if isinstance(node, (nodes.FixedTextElement, nodes.Special)):
continue
# nested TextElements are not "block-level" elements:
if isinstance(node.parent, nodes.TextElement):
continue
# list of text nodes in the "text block":
# Patched here to exclude more stuff.
txtnodes = []
for txtnode in node.traverse(nodes.Text):
if not can_apply_typography(txtnode): continue
# Don't convert -- in option strings
if isinstance(txtnode.parent, nodes.option_string): continue
txtnodes += [txtnode]
# language: use typographical quotes for language "lang"
lang = node.get_language_code(document_language)
# use alternative form if `smart-quotes` setting starts with "alt":
if alternative:
if '-x-altquot' in lang:
lang = lang.replace('-x-altquot', '')
else:
lang += '-x-altquot'
# drop subtags missing in quotes:
for tag in utils.normalize_language_tag(lang):
if tag in smartquotes.smartchars.quotes:
lang = tag
break
else: # language not supported: (keep ASCII quotes)
lang = ''
# Iterator educating quotes in plain text:
# '2': set all, using old school en- and em- dash shortcuts
teacher = smartquotes.educate_tokens(self.get_tokens(txtnodes),
attr='2', language=lang)
for txtnode, newtext in zip(txtnodes, teacher):
txtnode.parent.replace(txtnode, nodes.Text(newtext))
self.unsupported_languages = set() # reset
def apply(self):
smart_quotes = self.document.settings.smart_quotes
if not smart_quotes:
return
try:
alternative = smart_quotes.startswith("alt")
except AttributeError:
alternative = False
# print repr(alternative)
document_language = self.document.settings.language_code
# "Educate" quotes in normal text. Handle each block of text
# (TextElement node) as a unit to keep context around inline nodes:
for node in self.document.traverse(nodes.TextElement):
# skip preformatted text blocks and special elements:
if isinstance(node, (nodes.FixedTextElement, nodes.Special)):
continue
# nested TextElements are not "block-level" elements:
if isinstance(node.parent, nodes.TextElement):
continue
# list of text nodes in the "text block":
txtnodes = [
txtnode for txtnode in node.traverse(nodes.Text) if not isinstance(txtnode.parent, nodes.option_string)
]
# language: use smart-quotes for language "lang"
lang = node.get_language_code(document_language)
# use alternative form if `smart-quotes` setting starts with "alt":
if alternative:
if "-x-altquot" in lang:
lang = lang.replace("-x-altquot", "")
else:
lang += "-x-altquot"
# drop subtags missing in quotes:
for tag in utils.normalize_language_tag(lang):
if tag in smartquotes.smartchars.quotes:
lang = tag
break
else: # language not supported: (keep ASCII quotes)
if lang not in self.unsupported_languages:
self.document.reporter.warning(
"No smart quotes " 'defined for language "%s".' % lang, base_node=node
)
self.unsupported_languages.add(lang)
lang = ""
# Iterator educating quotes in plain text:
# '2': set all, using old school en- and em- dash shortcuts
teacher = smartquotes.educate_tokens(self.get_tokens(txtnodes), attr="2", language=lang)
for txtnode, newtext in zip(txtnodes, teacher):
txtnode.parent.replace(txtnode, nodes.Text(newtext))
self.unsupported_languages = set() # reset
def get_language(language_code):
for tag in normalize_language_tag(language_code):
if tag in _languages:
return _languages[tag]
try:
module = __import__(tag, globals(), locals())
except ImportError:
continue
_languages[tag] = module
return module
return None
def get_language(language_code):
for tag in normalize_language_tag(language_code):
if tag in _languages:
return _languages[tag]
try:
module = __import__(tag, globals(), locals())
except ImportError:
continue
_languages[tag] = module
return module
return None
def langcode2name(self, lang_code):
"""Return `polyglossia`_ (XeTeX) language name of `language_code`"""
retVal = ""
for tag in utils.normalize_language_tag(lang_code):
retVal = self.language_codes.get(tag, "")
if retVal:
break
if not retVal and self.builder is not None:
self.builder.warn(
'Language "%s" not supported by LaTeX (polyglossia)' %
lang_code)
return retVal
def get_language(language_code):
for tag in normalize_language_tag(language_code):
tag = tag.replace('-', '_') # '-' not valid in module names
if tag in _languages:
return _languages[tag]
try:
module = __import__(tag, globals(), locals(), level=1)
except ImportError:
try:
module = __import__(tag, globals(), locals(), level=0)
except ImportError:
continue
_languages[tag] = module
return module
return None
def get_language(language_code):
for tag in normalize_language_tag(language_code):
tag = tag.replace('-','_') # '-' not valid in module names
if tag in _languages:
return _languages[tag]
try:
module = __import__(tag, globals(), locals(), level=1)
except ImportError:
try:
module = __import__(tag, globals(), locals(), level=0)
except ImportError:
continue
_languages[tag] = module
return module
return None
def __call__(self, language_code, reporter=None):
try:
return self.cache[language_code]
except KeyError:
pass
for tag in normalize_language_tag(language_code):
tag = tag.replace('-', '_') # '-' not valid in module names
module = self.import_from_packages(tag, reporter)
if module is not None:
break
else:
if reporter:
reporter.warning(self.warn_msg % language_code)
if self.fallback:
module = self.import_from_packages(self.fallback)
if reporter and (language_code != 'en'):
reporter.info('Using %s for language "%s".' %
(module, language_code))
self.cache[language_code] = module
return module
def prepare_settings(self, docname):
# type: (unicode) -> None
"""Prepare to set up environment for reading."""
self.temp_data['docname'] = docname
# defaults to the global default, but can be re-set in a document
self.temp_data['default_role'] = self.config.default_role
self.temp_data['default_domain'] = \
self.domains.get(self.config.primary_domain)
self.settings['input_encoding'] = self.config.source_encoding
self.settings['trim_footnote_reference_space'] = \
self.config.trim_footnote_reference_space
self.settings['gettext_compact'] = self.config.gettext_compact
language = self.config.language or 'en'
self.settings['language_code'] = language
if 'smart_quotes' not in self.settings:
self.settings['smart_quotes'] = self.config.smartquotes
# some conditions exclude smart quotes, overriding smart_quotes
for valname, vallist in iteritems(
self.config.smartquotes_excludes):
if valname == 'builders':
# this will work only for checking first build target
if self.app.builder.name in vallist:
self.settings['smart_quotes'] = False
break
elif valname == 'languages':
if self.config.language in vallist:
self.settings['smart_quotes'] = False
break
# confirm selected language supports smart_quotes or not
for tag in normalize_language_tag(language):
if tag in smartchars.quotes:
break
else:
self.settings['smart_quotes'] = False
def apply(self):
smart_quotes = self.document.settings.smart_quotes
if not smart_quotes:
return
try:
alternative = smart_quotes.startswith('alt')
except AttributeError:
alternative = False
document_language = self.document.settings.language_code
lc_smartquotes = self.document.settings.smartquotes_locales
if lc_smartquotes:
smartquotes.smartchars.quotes.update(dict(lc_smartquotes))
# "Educate" quotes in normal text. Handle each block of text
# (TextElement node) as a unit to keep context around inline nodes:
for node in self.document.traverse(nodes.TextElement):
# skip preformatted text blocks and special elements:
if isinstance(node, self.nodes_to_skip):
continue
# nested TextElements are not "block-level" elements:
if isinstance(node.parent, nodes.TextElement):
continue
# list of text nodes in the "text block":
txtnodes = [
txtnode for txtnode in node.traverse(nodes.Text)
if not isinstance(txtnode.parent, nodes.option_string)
]
# language: use typographical quotes for language "lang"
lang = node.get_language_code(document_language)
# use alternative form if `smart-quotes` setting starts with "alt":
if alternative:
if '-x-altquot' in lang:
lang = lang.replace('-x-altquot', '')
else:
lang += '-x-altquot'
# drop unsupported subtags:
for tag in utils.normalize_language_tag(lang):
if tag in smartquotes.smartchars.quotes:
lang = tag
break
else: # language not supported: (keep ASCII quotes)
if lang not in self.unsupported_languages:
self.document.reporter.warning(
'No smart quotes '
'defined for language "%s".' % lang,
base_node=node)
self.unsupported_languages.add(lang)
lang = ''
# Iterator educating quotes in plain text:
# (see "utils/smartquotes.py" for the attribute setting)
teacher = smartquotes.educate_tokens(self.get_tokens(txtnodes),
attr=self.smartquotes_action,
language=lang)
for txtnode, newtext in zip(txtnodes, teacher):
txtnode.parent.replace(
txtnode, nodes.Text(newtext, rawsource=txtnode.rawsource))
self.unsupported_languages = set() # reset
def read_doc(self, docname, app=None):
# type: (unicode, Sphinx) -> None
"""Parse a file and add/update inventory entries for the doctree."""
self.temp_data['docname'] = docname
# defaults to the global default, but can be re-set in a document
self.temp_data['default_domain'] = \
self.domains.get(self.config.primary_domain)
self.settings['input_encoding'] = self.config.source_encoding
self.settings['trim_footnote_reference_space'] = \
self.config.trim_footnote_reference_space
self.settings['gettext_compact'] = self.config.gettext_compact
language = self.config.language or 'en'
self.settings['language_code'] = language
self.settings['smart_quotes'] = True
if self.config.html_use_smartypants is not None:
warnings.warn("html_use_smartypants option is deprecated. Smart "
"quotes are on by default; if you want to disable "
"or customize them, use the smart_quotes option in "
"docutils.conf.",
RemovedInSphinx17Warning)
self.settings['smart_quotes'] = self.config.html_use_smartypants
for tag in normalize_language_tag(language):
if tag in smartchars.quotes:
break
else:
self.settings['smart_quotes'] = False
docutilsconf = path.join(self.srcdir, 'docutils.conf')
# read docutils.conf from source dir, not from current dir
OptionParser.standard_config_files[1] = docutilsconf
if path.isfile(docutilsconf):
self.note_dependency(docutilsconf)
with sphinx_domains(self):
if self.config.default_role:
role_fn, messages = roles.role(self.config.default_role, english,
0, dummy_reporter)
if role_fn:
roles._roles[''] = role_fn
else:
logger.warning('default role %s not found', self.config.default_role,
location=docname)
codecs.register_error('sphinx', self.warn_and_replace) # type: ignore
# publish manually
reader = SphinxStandaloneReader(self.app,
parsers=self.app.registry.get_source_parsers())
pub = Publisher(reader=reader,
writer=SphinxDummyWriter(),
destination_class=NullOutput)
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, self.settings, None)
src_path = self.doc2path(docname)
source = SphinxFileInput(app, self, source=None, source_path=src_path,
encoding=self.config.source_encoding)
pub.source = source
pub.settings._source = src_path
pub.set_destination(None, None)
pub.publish()
doctree = pub.document
# post-processing
for domain in itervalues(self.domains):
domain.process_doc(self, docname, doctree)
# allow extension-specific post-processing
if app:
app.emit('doctree-read', doctree)
# store time of reading, for outdated files detection
# (Some filesystems have coarse timestamp resolution;
# therefore time.time() can be older than filesystem's timestamp.
# For example, FAT32 has 2sec timestamp resolution.)
self.all_docs[docname] = max(
time.time(), path.getmtime(self.doc2path(docname)))
if self.versioning_condition:
old_doctree = None
if self.versioning_compare:
# get old doctree
try:
with open(self.doc2path(docname,
self.doctreedir, '.doctree'), 'rb') as f:
old_doctree = pickle.load(f)
except EnvironmentError:
pass
# add uids for versioning
if not self.versioning_compare or old_doctree is None:
list(add_uids(doctree, self.versioning_condition))
else:
list(merge_doctrees(
old_doctree, doctree, self.versioning_condition))
# make it picklable
doctree.reporter = None
doctree.transformer = None
doctree.settings.warning_stream = None
doctree.settings.env = None
doctree.settings.record_dependencies = None
# cleanup
self.temp_data.clear()
self.ref_context.clear()
roles._roles.pop('', None) # if a document has set a local default role
# save the parsed doctree
doctree_filename = self.doc2path(docname, self.doctreedir,
'.doctree')
ensuredir(path.dirname(doctree_filename))
with open(doctree_filename, 'wb') as f:
pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
def read_doc(self, docname, app=None):
# type: (unicode, Sphinx) -> None
"""Parse a file and add/update inventory entries for the doctree."""
self.temp_data['docname'] = docname
# defaults to the global default, but can be re-set in a document
self.temp_data['default_domain'] = \
self.domains.get(self.config.primary_domain)
self.settings['input_encoding'] = self.config.source_encoding
self.settings['trim_footnote_reference_space'] = \
self.config.trim_footnote_reference_space
self.settings['gettext_compact'] = self.config.gettext_compact
language = self.config.language or 'en'
self.settings['language_code'] = language
self.settings['smart_quotes'] = True
for tag in normalize_language_tag(language):
if tag in smartchars.quotes:
break
else:
self.settings['smart_quotes'] = False
docutilsconf = path.join(self.srcdir, 'docutils.conf')
# read docutils.conf from source dir, not from current dir
OptionParser.standard_config_files[1] = docutilsconf
if path.isfile(docutilsconf):
self.note_dependency(docutilsconf)
with sphinx_domains(self):
if self.config.default_role:
role_fn, messages = roles.role(self.config.default_role,
english, 0, dummy_reporter)
if role_fn:
roles._roles[''] = role_fn
else:
logger.warning('default role %s not found',
self.config.default_role,
location=docname)
codecs.register_error('sphinx',
self.warn_and_replace) # type: ignore
# publish manually
reader = SphinxStandaloneReader(
self.app, parsers=self.app.registry.get_source_parsers())
pub = Publisher(reader=reader,
writer=SphinxDummyWriter(),
destination_class=NullOutput)
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, self.settings, None)
src_path = self.doc2path(docname)
source = SphinxFileInput(app,
self,
source=None,
source_path=src_path,
encoding=self.config.source_encoding)
pub.source = source
pub.settings._source = src_path
pub.set_destination(None, None)
pub.publish()
doctree = pub.document
# post-processing
for domain in itervalues(self.domains):
domain.process_doc(self, docname, doctree)
# allow extension-specific post-processing
if app:
app.emit('doctree-read', doctree)
# store time of reading, for outdated files detection
# (Some filesystems have coarse timestamp resolution;
# therefore time.time() can be older than filesystem's timestamp.
# For example, FAT32 has 2sec timestamp resolution.)
self.all_docs[docname] = max(time.time(),
path.getmtime(self.doc2path(docname)))
if self.versioning_condition:
old_doctree = None
if self.versioning_compare:
# get old doctree
try:
with open(
self.doc2path(docname, self.doctreedir,
'.doctree'), 'rb') as f:
old_doctree = pickle.load(f)
except EnvironmentError:
pass
# add uids for versioning
if not self.versioning_compare or old_doctree is None:
list(add_uids(doctree, self.versioning_condition))
else:
list(
merge_doctrees(old_doctree, doctree,
self.versioning_condition))
# make it picklable
doctree.reporter = None
doctree.transformer = None
doctree.settings.warning_stream = None
doctree.settings.env = None
doctree.settings.record_dependencies = None
# cleanup
self.temp_data.clear()
self.ref_context.clear()
roles._roles.pop('',
None) # if a document has set a local default role
# save the parsed doctree
doctree_filename = self.doc2path(docname, self.doctreedir, '.doctree')
ensuredir(path.dirname(doctree_filename))
with open(doctree_filename, 'wb') as f:
pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
