def _reformat_returns(title, text):
lines = text.split('\n')
new_text = ''
if len(lines) == 0:
return new_text
indent = get_indent(text)
param = ''
description = ''
while len(lines) > 0:
line = lines[0]
line = line.strip()
# Check if it is continuation of parameter description from previous lines
if param != '':
# It is continuation of multi-line parameter description
description += reindent(line, indent + 4) + '\n'
else:
# This is the first line of ``return`` description
param = _get_param_text(title, '') + ' ' + line
new_text += reindent(param, indent) + '\n'
lines.pop(0)
if param != '' and description != '':
new_text += reindent(description, indent + 4) + '\n'
return new_text + '\n'
def _reformat_parameters(title, text):
lines = text.split('\n')
new_text = ''
if len(lines) == 0:
return new_text
indent = get_indent(text)
param = ''
description = ''
while len(lines) > 0:
line = lines[0]
line = line.strip()
idx = line.find(' : ')
if idx >= 0 & line[0:idx].isalnum():
# Check if previous parameter existed. If so, need to add it to reformatted text
if param != '':
new_text += _get_param_text(
title, param) + '\n' + reindent(
description, indent + 4) + '\n'
# Found parameter. Extract the description (can be multi-line)
param = line[0:idx]
description = line[idx + 3:] + '\n'
lines.pop(0)
else:
# There is no parameter description starting in this line.
# Check if it is continuation of parameter description from previous lines
if param != '':
# It is continuation of multi-line parameter description
description += reindent(line, indent + 4) + '\n'
else:
# This is not the description of parameter. Copy as is
new_text += reindent(line, indent) + '\n'
lines.pop(0)
if param != '' and description != '':
new_text += _get_param_text(title, param) + '\n' + reindent(
description, indent + 4) + '\n'
return new_text
def get_obj_examples(pandas_name):
"""
Get list of examples for Pandas object.
:param pandas_name: Pandas object for which documentation to be generated.
:return: Generated docstring.
"""
sdc_obj = get_sdc_object_by_pandas_name(pandas_name)
sdc_doc = get_docstring(sdc_obj)
sections = split_in_sections(reindent(sdc_doc, 0))
sections_as_dict = {title.strip(): text for title, text in sections}
example_section = sections_as_dict.get('Examples')
if not example_section:
return None
examples = []
section_names = ['literalinclude', 'command-output']
for subsection in example_section.strip().split('\n\n'):
subsection = subsection.strip()
if any(subsection.startswith(f'.. {name}') for name in section_names):
# remove a directory level from path to examples
examples.append(subsection.replace(' ../', ' '))
return reformat('\n\n'.join(examples))
def reformat_bullet_list(text):
lines = text.split('\n')
new_text = ''
bullet_indent = -1
while len(lines) > 0:
line = lines[0]
if line.strip().startswith('- '):
# Here if met new bullet
bullet_indent = get_indent(
line) # We need to know indent to identify multi-line bullets
new_text += line + '\n'
elif line.strip() == '':
bullet_indent = -1 # We finished parsing multi-line bullet
new_text += '\n'
else:
if bullet_indent >= 0:
# Here if we're parsing multi-line bullet
new_text += reindent(line, bullet_indent + 4) + '\n'
else:
# Here if we are not in bullet list
new_text += line + '\n'
lines.pop(0)
return new_text
def parse_templ_rst(fname_templ):
"""
Parses input template rst file and outputs the final rst file
Template document must have the following structure:
Heading or subheading
*********************
Any text (if any)
Another heading or subheading
-----------------------------
Any text (if any)
.. currentmodule:: <module name>
.. sdc_toctree
<api1>
<api2>
<api3>
...
Any text (if any)
Any text (if any)
Another heading or subheading
-----------------------------
Any text (if any)
...
:param fname_templ:
"""
path, fname_out = os.path.split(fname_templ)
fname_out = fname_out.replace('_templ', '')
fname_out = fname_out.replace('_', '', 1)
fout = open_file_for_write(APIREF_REL_PATH + fname_out)
with open(fname_templ, 'r', encoding='utf-8') as fin:
doc = fin.readlines()
while len(doc) > 0:
# Parsing lines until ``.. sdc_toctree`` section is met
while len(doc) > 0 and not doc[0].startswith('.. sdc_toctree'):
line = doc[0]
if line.startswith('.. currentmodule::'):
current_module_name = line[19:].strip()
fout.write(line)
doc.pop(0)
if len(doc) == 0:
return
doc.pop(0) # Skipping ``.. sdc_toctree``
# Parsing the list of APIs
while len(doc) > 0 and doc[0].strip() != '':
line = doc[0]
indent = get_indent(line)
line = line.strip()
full_name = current_module_name + '.' + line
short_description = generate_simple_object_doc(
full_name, short_doc_flag=True).strip()
new_line = reindent(':ref:`', indent) + line + ' <' + full_name + '>`\n' + \
reindent(short_description, indent+4) + '\n'
fout.write(new_line)
doc.pop(0)
full_description = generate_simple_object_doc(
full_name, short_doc_flag=False)
f = open_file_for_write(APIREF_REL_PATH + full_name + '.rst')
f.write('.. _' + full_name + ':\n\n:orphan:\n\n')
f.write(create_heading_str(full_name, '*') + '\n\n')
f.write(full_description)
f.close()
if len(doc) == 0:
return
fout.close()
def generate_simple_object_doc(pandas_name,
short_doc_flag=False,
doc_from_pandas_flag=True,
add_sdc_sections=True,
unsupported_warning=True,
reformat_pandas=True):
"""
Generates documentation for Pandas object obj according to flags.
For complex objects such as modules and classes the function does not go to sub-objects,
i.e. to class attributes and sub-modules of the module.
:param pandas_name: Pandas object for which documentation to be generated.
:param short_doc_flag: Flag to indicate that only short description for the object is needed.
:param doc_from_pandas_flag: Flag to indicate that the documentation must be taken from Pandas docstring.
This docstring can be extended with Intel SDC specific sections. These are See Also, Examples,
Notes, Warning, Limitations, etc. if ``add_sdc_sections`` flag is set.
:param add_sdc_sections: Flag to indicate that extra sections of the documentation need to be taken from Intel SDC.
If ``doc_from_pandas_flag==False`` then the description section is taken from Intel SDC too. Otherwise
Intel SDC description section will be cut and Pandas API description will be used instead.
:param unsupported_warning: Flag, if ``True`` includes warning message if corresponding Intel SDC object is not
found. This indicates that given SDC method is unsupported.
:param reformat_pandas: Flag, if ``True`` re-formats Parameters section to :param: style. Needed to work around
Sphinx generator issues for Pandas Parameters section written in NumPy style
:return: Generated docstring.
"""
doc = ''
pandas_obj = get_obj(pandas_name)
if pandas_obj is None:
return doc # Empty documentation for no-object
if doc_from_pandas_flag: # Check if documentation needs to be generated from Pandas docstring
if short_doc_flag: # Check if only short description is needed
doc = get_short_description(
pandas_obj) # Short description is requested
else:
# Exclude Examples, Notes, See Also, References sections
sections = split_in_sections(reindent(get_docstring(pandas_obj),
0))
while len(sections) > 0:
title, text = sections[0]
if title.strip() == '': # Description sections
doc += text + '\n\n'
sections.pop(0)
elif title.strip() == 'Examples': # Exclude Examples section
sections.pop(0)
elif title.strip(
) == 'Notes': # Exclude Notes section (may be too specific to Pandas)
sections.pop(0)
elif title.strip().lower(
) == 'see also': # Exclude See Also section (may be too specific to Pandas)
sections.pop(0)
elif title.strip(
) == 'References': # Exclude References section (may be too specific to Pandas)
sections.pop(0)
elif title.strip() == 'Parameters' or title.strip() == 'Raises' or title.strip() == 'Return' or \
title.strip() == 'Returns':
if reformat_pandas:
doc += reformat_pandas_params(title, text)
sections.pop(0)
else:
doc += create_heading_str(
title) + '\n\n' + text + '\n\n'
sections.pop(0)
else:
doc += create_heading_str(title) + '\n\n' + text + '\n\n'
sections.pop(0)
if not add_sdc_sections:
if reformat_pandas:
return reformat(doc)
else:
return doc
# Here if additional sections from Intel SDC object needs to be added to pandas_obj docstring
sdc_obj = get_sdc_object_by_pandas_name(pandas_name)
if sdc_obj is None:
if unsupported_warning:
if reformat_pandas:
doc = reformat(doc)
if short_doc_flag:
return doc + ' **Unsupported by Intel SDC**.'
else:
return doc + '\n\n.. warning::\n This feature is currently unsupported ' \
'by Intel Scalable Dataframe Compiler\n\n'
if not short_doc_flag:
sdc_doc = get_docstring(sdc_obj)
sdc_doc = cut_sdc_dev_guide(sdc_doc)
# Cut description section from ``sdc_doc``
if is_sdc_user_guide_header(
sdc_doc[0]): # First section is SDC User Guide header
sdc_doc.pop(0)
if doc_from_pandas_flag:
# Ignore description from Intel SDC, keep Pandas description only
while len(sdc_doc) > 0:
title, text = sdc_doc[0]
if title.strip() != '':
break
sdc_doc.pop(0)
indent = get_indent(doc)
for title, text in sdc_doc:
if title.strip() == '':
doc += '\n' + reindent(text, indent)
else:
doc += '\n' + reindent(create_heading_str(title), indent) + '\n' + \
reindent(text, indent) + '\n'
return reformat(doc)