def build_quick_start():
replace = {'# pyremap': '# Quick Start'}
skip = [('## Documentation', '## Installation')]
outContent = ''
skipMode = False
with open('../README.md', 'r') as inFile:
for line in inFile.readlines():
for skipStart, skipEnd in skip:
if not skipMode and skipStart in line:
skipMode = True
if skipMode and skipEnd in line:
skipMode = False
if not skipMode:
for replaceString in replace:
if replaceString in line:
line = replace[replaceString]
break
outContent = outContent + line
outContent = convert(outContent)
with open('quick_start.rst', 'w') as outFile:
outFile.write('.. _quick_start:\n\n')
outFile.write(outContent)
def proc_dir(self, path):
import m2r
for (dirpath, dirnames, filenames) in os.walk(path):
dirnames.sort()
for fn in sorted(filenames):
fn = dirpath + '/' + fn
if not fn.endswith('.md'):
continue
rst_fn = fn[:-3] + '.rst'
with open(fn, 'rb') as f:
md = f.read().decode('utf-8')
for kw in ['docs/help-']:
md = md.replace(
'({0}'.format(kw),
'(https://github.com/9001/r0c/blob/master/{0}'.format(
kw))
for kw in ['docs', 'clients']:
md = md.replace(
'({0}/'.format(kw),
'(https://ocv.me/static/r0c/{0}/'.format(kw))
md = md.replace('* **[', '* [').replace(')** <-', ') <-')
rst = m2r.convert(md)
rst = rst.replace(':raw-html-m2r:`<del>', ':sub:`')
rst = rst.replace('</del>`', '`')
with open(rst_fn, 'wb') as f:
f.write(rst.encode('utf-8'))
def render_changelog(format="rst"):
tags = releases(ascending=False)
pull_requests = defaultdict(list)
release_date = dict()
for start, stop in zip(tags[1:], tags[:-1]):
if stop.startswith("v"):
version = ".".join(
parse_version(stop[1:]).base_version.split(".")[:2])
else:
version = stop
for commit in git_list_merges(start=start, stop=stop):
pull_requests[version].append(PullRequest(commit))
release_date[version] = git_tag_date(stop)
changelog = OrderedDict()
for version, pulls in pull_requests.items():
groups = defaultdict(list)
for pull in pulls:
groups[pull.topic()].append(str(pull))
changelog[version] = groups
env = jinja2.Environment(
loader=jinja2.DictLoader({"changelog": CHANGELOG}))
contents = env.get_template("changelog").render(releases=changelog,
release_date=release_date)
if format == "rst":
contents = m2r.convert(contents)
return contents
def build_quick_start():
replace = {
'# MPAS-Analysis':
'# Quick Start Guide\n',
'[![Build Status]':
'',
'[![Documentation Status]':
'',
'':
'\n'
}
skip = [('## conda-forge', '## Installation')]
outContent = ''
skipMode = False
with open('../README.md', 'r') as inFile:
for line in inFile.readlines():
for skipStart, skipEnd in skip:
if not skipMode and skipStart in line:
skipMode = True
if skipMode and skipEnd in line:
skipMode = False
if not skipMode:
for replaceString in replace:
if replaceString in line:
line = replace[replaceString]
break
outContent = outContent + line
outContent = convert(outContent)
with open('quick_start.rst', 'w') as outFile:
outFile.write('.. _quick_start:\n\n')
outFile.write(outContent)
def _process_description(self, description):
description = m2r.convert(description)
description = description.replace(m2r.prolog, '')
description = description.replace(":raw-html-m2r:", ":raw-html:")
description = description.replace(r'\ ,', ',')
description = description.replace(r'\ ', ' ')
return description.strip()
def _make_property(self, spec, name=None, indent=None, declare=True):
yield ''
if declare:
type = get_param_type(spec)
if name:
if 'required' in spec or 'x-alertlogic-required' in spec:
yield f'{indent}- **{name}** *({type}) --* **[REQUIRED]**'
else:
yield f'{indent}- **{name}** *({type}) --*'
else:
yield f'{indent}- *({type}) --*'
indent = self._indent(indent)
if declare:
yield ''
for line in convert(spec.get('description', "")).splitlines():
yield f'{indent}{line}'
if 'enum' in spec:
valid_values = ', '.join([f'``{v}``' for v in spec['enum']])
yield ''
yield f'{indent}*Valid values*: {valid_values}'
if 'default' in spec:
yield ''
yield f'{indent}*Default*: ``{spec["default"]}``'
def convert_release_notes():
"""Convert the release notes to an RST page with links to PRs"""
this_dir = os.path.dirname(os.path.realpath(__file__))
notes = os.path.join(this_dir, '..', '..', 'HISTORY.md')
with open(notes, 'r') as f:
notes = f.read()
rst = convert(notes)
with_ref = ''
for line in rst.split('\n'):
# Get URL for PR
if '#' in line:
pr_number = line.split('#')[1]
while len(pr_number):
try:
pr_number = int(pr_number)
break
except ValueError:
# Too many tailing characters to be an int
pr_number = pr_number[:-1]
if pr_number:
line = line.replace(
f'#{pr_number}',
f'`#{pr_number} <https://github.com/XENONnT/straxen/pull/{pr_number}>`_'
)
with_ref += line + '\n'
target = os.path.join(this_dir, 'reference', 'release_notes.rst')
with open(target, 'w') as f:
f.write(header + with_ref)
def iter_properties(cls):
"""Iterate over (property, type, description)"""
import m2r # convert markdown to rst
schema = cls.resolve_references(cls._schema)
properties = schema.get('properties', {})
for prop, propschema in properties.items():
yield (prop, type_description(propschema),
m2r.convert(propschema.get('description', ' ')))
def convert_to_rst(string_to_convert: str) -> str:
"""Convert that string from MD to RST.
"""
try:
return m2r.convert(string_to_convert,
renderer=AutorestRender()).strip()
except Exception: # pylint: disable=broad-except
return string_to_convert
def docstring(app, what, name, obj, options, lines):
md = '\n'.join(lines)
regex = re.compile(r'(.+?)\n:\s+(.*\n)')
md = regex.sub(r'\1\n \2', md)
regex = re.compile(r'(\s|\n)(\$[^\$\n]+\$)([^\$])')
md = regex.sub(r'\1`\2`\3', md)
# FIX DEFINITIONS
regex = re.compile(r'(( |\t)+)- ``([^\n]+)`` -')
md = regex.sub(r"6473829123- `\3` -", md)
# REMOVE STRIKETHROUGH
regex = re.compile(r'~~([^~\n]+)~~ ?')
md = regex.sub(r"", md)
# ALLOW FOR CITATIONS TO SEMI-WORK (AS FOOTNOTES)
regex = re.compile(r'\[#(.*?)\]')
md = regex.sub(r"[^cn\1]", md)
# SUBSCRIPT
regex = re.compile(r'([!~]*\S)~(\S)([!~]*\n)')
md = regex.sub(r"\1~\2~\3", md)
regex = re.compile(r'([^\~])\~([^\~\n]+)\~([^\~])')
index = 0
while index < 100 and "~" in md:
index += 1
md = regex.sub(r'\1\\ :sub:`\2`\\\3', md, count=1)
# SUPERSCRIPT
regex = re.compile(r'([!\^]*\S)\^(\S)([!\^]*\n)')
md = regex.sub(r"\1^\2^\3", md)
regex = re.compile(r'([^\^])\^([^\^\n]+)\^([^\^])')
index = 0
while index < 100 and "^" in md:
index += 1
md = regex.sub(r'\1\\ :sup:`\2`\\\3', md, count=1)
# HR
regex = re.compile(r'\n---')
md = regex.sub(r"\n\n----------\n\n", md)
# FIX LINKS
regex = re.compile(r'\[(.*?)\]\(\/?(\_autosummary\/)?(\S*?)(\.html)?\)')
md = regex.sub(r'[\1](\3.html)', md)
rst = md
rst = m2r.convert(md)
rst = rst.replace("6473829123", " ")
rst = rst.replace(".. code-block:: eval_rst", "")
# REPLACE THE DOCSTRING LINES WITH OUR NEW RST
lines.clear()
for line in rst.split("\n"):
lines.append(line)
def md_to_rst(filepath):
with open(filepath, 'r') as f:
lines = f.readlines()
md = '\n'.join(lines)
rst = m2r.convert(md)
lines.clear()
for line in rst.splitlines():
lines.append(line)
return "\n".join(lines)
def as_rst(self) -> str:
if self.markup == 'rst':
return self.path.read_text()
if self.markup == 'md':
content = convert(self.path.read_text())
content = content.replace('.. code-block:: toml', '.. code-block::')
return content
if self.markup == 'txt':
return self.path.read_text()
raise ValueError('invalid markup')
def convert_file(file_path: str, target_path: str = None):
if target_path is None:
target_path = file_path.replace('.md', '.rst')
with open(file_path, 'r') as f:
md_content = f.read()
with open(target_path, 'w') as f:
f.write(convert(md_content))
print(f'Saved RST file to {target_path}')
def setup(app):
"""Register this configuration as a Sphinx extension."""
subprocess.check_call(
['sphinx-apidoc', '-o',
os.path.join(ROOT_PATH, 'docs'), 'backuppy'])
with open(os.path.join(ROOT_PATH, 'README.md')) as f:
readme_rst = convert(f.read())
with open(os.path.join(ROOT_PATH, 'docs', 'README.rst'), mode='w+t') as f:
f.write(readme_rst)
def pandoc(source, fmt, to, extra_args=None, encoding='utf-8'):
"""Convert an input string using pandoc.
Pandoc converts an input string `from` a format `to` a target format.
Parameters
----------
source : string
Input string, assumed to be valid format `from`.
fmt : string
The name of the input format (markdown, etc.)
to : string
The name of the output format (html, etc.)
Returns
-------
out : unicode
Output as returned by pandoc.
Raises
------
PandocMissing
If pandoc is not installed.
Any error messages generated by pandoc are printed to stderr.
"""
cmd = ['pandoc', '-f', fmt, '-t', to]
if extra_args:
cmd.extend(extra_args)
# iOS: we cannot call pandoc, so we just don't convert markdown cells.
# This is not perfect (...) but it lets the conversion machine work.
# iOS: we replaced pandoc with a mistune plugin. It's not as good but it works
# iOS, TODO: tables in LaTeX, html in LaTeX
if (sys.platform == 'darwin' and platform.machine().startswith('iP')):
if (fmt.startswith('markdown') and to.startswith('latex')):
markdown_to_latex = mistune.Markdown(renderer=LatexRenderer())
return markdown_to_latex(source)
elif (fmt.startswith('markdown') and to.startswith('rst')):
return convert(source) # m2r markdown to rst conversion
elif (fmt.startswith('markdown') and to.startswith('asciidoc')):
markdown_to_asciidoc = mistune.Markdown(renderer=AsciidocRenderer())
return markdown_to_asciidoc(source)
else:
return source
# this will raise an exception that will pop us out of here
check_pandoc_version()
# we can safely continue
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
out, _ = p.communicate(cast_bytes(source, encoding))
out = TextIOWrapper(BytesIO(out), encoding, 'replace').read()
return out.rstrip('\n')
def create_changelog_rst():
logging.info("Creating changelog.rst")
with open(os.path.join(_REPO_ROOT, "CHANGELOG.md"), encoding="utf8") as f:
changelog_md = f.read()
changelog_rst = (
".. _changelog:\n\n" + "Changelog\n=========\n" +
m2r.convert(changelog_md[changelog_md.index("## [Unreleased]"):]))
with open(os.path.join(_DOCS_SOURCE, "changelog.rst"),
"w",
encoding="utf8") as f:
f.write(changelog_rst)
def writeDocs(cls):
obj, name = pydoc.resolve(cls)
html = pydoc.html.page(pydoc.describe(obj), pydoc.html.document(obj, name))
markdown = html2text.html2text(html)
markdown = markdown.replace('`', '')
markdown = re.sub(r'^[ \t|]+', '', markdown, flags=re.MULTILINE)
rst = m2r.convert(markdown)
with open(os.path.join(apiDocsDir, f'{cls.__name__}.rst'), 'w') as file:
file.write(rst)
def makedocs(
c, doctype="html", docs_build_path=get_docs_build_path(),
):
from sphinx.cmd.build import build_main
from m2r import convert
with open("./README.md", "r") as _file:
markdown_contents = _file.read()
with open(os.path.join(get_docs_source_path(), "readme.rst"), "w+") as _file:
_file.write(convert(markdown_contents))
build_main(["-b", doctype, get_docs_source_path(), docs_build_path])
def gen():
for markdown_line in source:
rst_lines = m2r.convert(markdown_line).split('\n')
skip_flag = True
for line in rst_lines:
if line == '' and skip_flag and False:
#
# Suppress empty lines at the start of each section, they
# are not needed.
#
continue
yield line
skip_flag = bool(line)
def translate_docstring(app, what, name, obj, options, lines):
"""
Translates a docstring from Markdown (the default format, since we are generating Markdown docstrings) to
reStructuredText, which is what the rest of Sphinx expects.
See http://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#event-autodoc-process-docstring for the full
semantics of the arguments. The upshot is that `lines` must be modified in-place.
"""
if not app.config.markdown_docstring_convert or not lines:
return
input_text = os.linesep.join(lines)
output_text = convert(input_text)
lines[:] = output_text.split(os.linesep)
def _process_description(self, description):
description = ''.join([
reSpecial.sub('',d) if i%2 else d
for i,d in enumerate(reLink.split(description))
]) # remove formatting from links
description = m2r.convert(description)
description = description.replace(m2r.prolog, '')
description = description.replace(":raw-html-m2r:", ":raw-html:")
description = description.replace(r'\ ,', ',')
description = description.replace(r'\ ', ' ')
# turn explicit references into anonymous references
description = description.replace('>`_', '>`__')
description += '\n'
return description.strip()
def _process_description(self, description):
description = ''.join([
reSpecial.sub('',d) if i%2 else d
for i,d in enumerate(reLink.split(description))
]) # remove formatting from links
description = m2r.convert(description)
description = description.replace(m2r.prolog, '')
description = description.replace(":raw-html-m2r:", ":raw-html:")
description = description.replace(r'\ ,', ',')
description = description.replace(r'\ ', ' ')
# turn explicit references into anonymous references
description = description.replace('>`_', '>`__')
description += '\n'
return description.strip()
def _process_description(self, description):
description = "".join([
reSpecial.sub("", d) if i % 2 else d
for i, d in enumerate(reLink.split(description))
]) # remove formatting from links
description = m2r.convert(description)
description = description.replace(m2r.prolog, "")
description = description.replace(":raw-html-m2r:", ":raw-html:")
description = description.replace(r"\ ,", ",")
description = description.replace(r"\ ", " ")
# turn explicit references into anonymous references
description = description.replace(">`_", ">`__")
description += "\n"
return description.strip()
def get_readme():
with open(os.path.join(THIS_DIR, 'README.md'), encoding='utf-8') as fobj:
out = fobj.read()
try:
import m2r
except ImportError:
# must be running on fellow dev computer - no need to do fancy
# markdown conversion
return out
else:
# Probably an sqlalchemy-fsm dev environment.
# Might be publishing to pypi.
# Better to perform the conversion
return m2r.convert(out)
def _make_methods(self):
indent = self._initial_indent
for op_name, op_spec in self._spec['operations'].items():
yield ''
yield f'{indent}.. py:method:: {op_name}(**kwargs)'
for line in convert(op_spec.get('description', "")).splitlines():
yield f'{indent}{indent}{line}'
parameters = op_spec.get('parameters', {})
for line in self._make_request_syntax(op_name, parameters,
self._indent(indent)):
yield line
yield from self._make_request_parameters(op_name, parameters,
indent)
yield from self._make_response(op_spec.get('response', {}), indent)
def run(self):
parameter = self.arguments[0]
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
markdown = self.get_parameter_doc(parameter)
rst = m2r.convert(markdown)
description = statemachine.string2lines(rst,
4,
convert_whitespace=True)
lines = [f'.. _{parameter}:', '', parameter, '`' * len(parameter), ''
] + description
self.state_machine.insert_input(lines, source)
return []
def processTocRow(docTitle, section, title, content):
#print(f'\tSection: {section} Heading Level: {level} Title: {title} Content File: {content}')
output = ""
level = len(section.split(".")) + 1
if (len(title) > 0):
# Add section heading
output += ('#' * level) + " " + title + "\n"
convertToRst = len(content) == 0 or content.endswith(".md")
# Process content (if present)
if (len(content) > 0):
reader = open(content, encoding="utf8")
try:
# strip off any Jekyll metadata at the top of the file
inMetadata = False
for line in reader.readlines():
if (line.strip() == "---"):
inMetadata = not inMetadata
continue
if (not inMetadata):
if (line.startswith("#")):
# make sure heading level is correct-ish
output += ('#' * level) + line + "\n"
else:
# append line to output
output += line
finally:
reader.close()
rst = ""
if (convertToRst):
rst = convert(output)
else:
rst = output
# add a page break
rst += "\n.. raw:: pdf\n \n PageBreak\n"
return rst
def setup(app):
import pathlib
from m2r import convert
import typing
import shutil
typing.TYPE_CHECKING = True
docs_path = pathlib.Path(__file__).parent
original_readme = os.path.join(docs_path.parent, 'README.md')
rst_readme = os.path.join(docs_path, 'README.rst')
# Copy Examples to docs folder for rendering
original_examples_folder = os.path.join(docs_path.parent, 'examples')
target_examples_folder = os.path.join(docs_path, 'examples')
shutil.rmtree(target_examples_folder, ignore_errors=True)
shutil.copytree(original_examples_folder,
target_examples_folder,
symlinks=True)
# Change readme to rst file, and include in Sphinx index
with open(rst_readme, 'w') as f:
md_content = open(original_readme, 'r').read()
md_content = md_content.replace('(./docs/', '(../')
md_content = md_content.replace('(./examples/',
'(https://github.com/BrikerMan/Kashgari/blob/v2-trunk/examples/')
md_content = md_content.replace('index.md)', ')')
md_content = md_content.replace('.md)', ')')
f.write(convert(md_content))
print(f'Saved RST file to {rst_readme}')
# Update all .md files， for fixing links
update_markdown_content(docs_path)
app.add_css_file('css/modify.css')
app.add_css_file('css/extra.css')
app.config['set_type_checking_flag'] = True
app.connect('autodoc-skip-member', skip_some_classes_members)
def setup(app):
import pathlib
from m2r import convert
docs_path = pathlib.Path(__file__).parent
original_readme = os.path.join(docs_path.parent, 'README.md')
rst_readme = os.path.join(docs_path, 'README.rst')
# Update all .md files， for fixing links
update_markdown_content(docs_path)
# Change readme to rst file, and include in Sphinx index
with open(rst_readme, 'w') as f:
md_content = open(original_readme, 'r').read()
md_content = md_content.replace('(./docs/', '(./')
f.write(convert(md_content))
print(f'Saved RST file to {rst_readme}')
app.add_stylesheet('css/modify.css')
app.add_stylesheet('css/extra.css')
def _make_request_parameters(self, op_name, parameters, indent):
indent = self._indent(indent)
for name, spec in sorted(parameters.items()):
yield ''
if 'content' in spec:
type = ' | '.join(
[get_param_type(v) for v in spec['content'].values()])
else:
type = get_param_type(spec)
yield f'{indent}:type {name}: {type}'
if 'required' in spec or 'x-alertlogic-required' in spec:
yield f'{indent}:param {name}: **[REQUIRED]**'
else:
yield f'{indent}:param {name}:'
yield ''
for line in convert(spec.get('description', "")).splitlines():
yield f'{indent}{indent}{line}'
yield from self._make_property(spec,
name=name,
declare=False,
indent=self._indent(indent))
if name == 'content_type':
yield from self._make_content_type(
op_name,
spec.pop('x-alertlogic-payload-content'),
indent=indent)
elif 'content' in spec:
yield from self._make_request_body_parameter(
spec.pop('content'), name, indent=self._indent(indent))
else:
yield from self._make_properties(spec=spec,
indent=self._indent(indent),
declare=False)
def build_quick_start():
replace = {'# MPAS-Analysis': '# Quick Start Guide\n',
'[![Build Status]': '',
'[![Documentation Status]': '',
'## Documentation': '',
'[http://mpas-analysis.readthedocs.io]': '',
'':
'\n'}
outContent = ''
with open('../README.md', 'r') as inFile:
for line in inFile.readlines():
for replaceString in replace:
if replaceString in line:
line = replace[replaceString]
break
outContent = outContent + line
outContent = convert(outContent)
with open('quick_start.rst', 'w') as outFile:
outFile.write('.. _quick_start:\n\n')
outFile.write(outContent)
def readme():
"""print long description"""
import m2r
with open('README.md') as f:
return m2r.convert(f.read())
def conv(self, src, **kwargs):
out = convert(src, **kwargs)
self.check_rst(out)
return out
def conv_no_check(self, src, **kwargs):
out = convert(src, **kwargs)
return out