def sub_block(match) -> str:
if chapter:
meta = chapter.get_section_by_offset(match.start())
meta_config = meta.data.get('confluence', {}).get('codeblocks', {})
options = CombinedOptions(
{
'backend': config,
'meta': meta_config
},
priority='meta'
)
else:
options = config
language = None
if 'syntax' in match.groupdict():
language = match.group('syntax')
source = match.group('content')
if source.endswith('\n'):
source = source[:-1]
logger.debug(f'Found code block ({language}):\n{source[:150]}\n...')
return gen_code_macro(source,
language=language,
theme=options.get('theme'),
title=options.get('title'),
linenumbers=options.get('linenumbers'),
collapse=options.get('collapse'))
def test_priority_list(self):
options1 = {'key1': 'val1', 'key2': 'val2', 'key3': 'val3'}
options2 = {'key2': 'val22', 'key3': 'val32', 'key4': 'val42'}
options3 = {'key2': 'val23', 'key4': 'val43'}
expected = {
'key1': 'val1',
'key2': 'val23',
'key3': 'val32',
'key4': 'val43'
}
coptions = CombinedOptions(
{
'o1': options1,
'o2': options2,
'o3': options3
},
priority=['o3', 'o2', 'o1'])
self.assertEqual(coptions.options, expected)
coptions = CombinedOptions(
{
'o1': options1,
'o2': options2,
'o3': options3
},
priority=['o3', 'o2'])
self.assertEqual(coptions.options, expected)
def process_template_tag(self, block) -> str:
"""
Function for processing tag. Send the contents to the corresponging
template engine along with parameters from tag and config, and
<content_file> path. Replace the tag with output from the engine.
"""
tag_options = Options(
self.get_options(block.group('options')),
validators={'engine': validate_in(self.engines.keys())})
options = CombinedOptions({
'config': self.options,
'tag': tag_options
},
priority='tag')
tag = block.group('tag')
if tag == 'template': # if "template" tag is used — engine must be specified
if 'engine' not in options:
self._warning(
'Engine must be specified in the <template> tag. Skipping.',
self.get_tag_context(block))
return block.group(0)
engine = self.engines[options['engine']]
else:
engine = self.engines[tag]
current_pos = block.start()
chapter = get_meta_for_chapter(self.current_filepath)
section = chapter.get_section_by_offset(current_pos)
_foliant_vars = {
'meta': section.data,
'meta_object': self.meta,
'config': self.config,
'target': self.context['target'],
'backend': self.context['backend'],
'project_path': self.context['project_path']
}
context = {}
# external context is loaded first, it has lowest priority
if 'ext_context' in options:
context.update(
self.load_external_context(options['ext_context'], options))
# all unrecognized params are redirected to template engine params
context.update(
{p: options[p]
for p in options if p not in self.tag_params})
# add options from "context" param
context.update(options.get('context', {}))
template = engine(block.group('body'), context,
options.get('engine_params', {}),
self.current_filepath, _foliant_vars)
return template.build()
def process(self, tag_options) -> str:
self.options = CombinedOptions(
{
'config': self.config,
'tag': tag_options
},
priority='tag',
defaults=self.defaults)
self.connect()
return self.gen_docs()
def _get_config(self, tag_options: dict = {}) -> CombinedOptions:
'''
Get merged config from (decreasing priority):
- tag options,
- preprocessir options,
- backend options.
'''
def filter_uncommon(val: dict) -> dict:
uncommon_options = ['title', 'id']
return {k: v for k, v in val.items() if k not in uncommon_options}
backend_config = self.config.get('backend_config',
{}).get('confluence', {})
options = CombinedOptions(
{
'tag': tag_options,
'config': filter_uncommon(self.options),
'backend_config': filter_uncommon(backend_config)
},
priority=['tag', 'config', 'backend_config'],
required=[(
'host',
'id',
), ('host', 'title', 'space_key')])
return options
def test_combine_override(self):
options1 = {'key1': 'val1', 'key2': 'val2'}
options2 = {'key2': 'val21', 'key4': 'val4'}
coptions = CombinedOptions({'o1': options1, 'o2': options2})
expected = {**options2, **options1}
self.assertEqual(coptions.options, expected)
def _sub(block) -> str:
anchor = block.group('body').strip()
if anchor in self.applied_anchors:
self._warning(
f"Can't apply dublicate anchor \"{anchor}\", skipping.",
context=self.get_tag_context(block))
return ''
if anchor in header_anchors:
self._warning(
f'anchor "{anchor}" may conflict with header "{header_anchors[anchor]}".',
context=self.get_tag_context(block))
options = CombinedOptions(
{
'main': self.options,
'tag': self.get_options(block.group('options'))
},
convertors={'tex': boolean_convertor},
priority='tag')
illegal_char = contains_illegal_chars(anchor)
if illegal_char:
self._warning(
f"Can't apply anchor \"{anchor}\", because it contains illegal symbol: {illegal_char}.",
context=self.get_tag_context(block))
return ''
self.applied_anchors.append(anchor)
if self.context['target'] == 'pdf' and options['tex']:
return get_tex_anchor(anchor)
else:
return get_anchor(anchor, options, self.context['target'])
def process_swaggerdoc_blocks(self, block) -> str:
tag_options = Options(
self.get_options(block.group('options')),
convertors={
'json_path':
rel_path_convertor(self.current_filepath.parent),
'spec_path':
rel_path_convertor(self.current_filepath.parent),
'additional_json_path':
rel_path_convertor(self.current_filepath.parent)
})
options = CombinedOptions(
options={
'config': self.options,
'tag': tag_options
},
priority='tag',
required=[('json_url', ), ('json_path', ), ('spec_url', ),
('spec_path', )],
validators={'mode': validate_in(self._modes)},
defaults=self.defaults)
self.logger.debug(
f'Processing swaggerdoc tag in {self.current_filepath}')
spec_url = options['spec_url'] or options['json_url']
if spec_url and isinstance(spec_url, str):
spec_url = [spec_url]
spec_path = options['spec_path'] or options['json_path']
spec = self._gather_specs(spec_url, spec_path)
if not spec:
raise RuntimeError("No valid swagger spec file specified")
return self._modes[options['mode']](spec, options)
def _create_default_templates(self, options: CombinedOptions):
"""
Copy default templates to project dir if their names in options are
same as default.
"""
if options.is_default('doc_template'):
source = self.project_path / options['doc_template']
to_copy = resource_filename(
__name__, f"templates/{options.defaults['doc_template']}")
copy_if_not_exists(source, to_copy)
if options.is_default('scheme_template'):
source = self.project_path / options['scheme_template']
to_copy = resource_filename(
__name__, f"templates/{options.defaults['scheme_template']}")
copy_if_not_exists(source, to_copy)
def _sub_tags(diagram) -> str:
'''Sub function for <plantuml> tags.'''
options = CombinedOptions(
{
'config': self.options,
'tag': self.get_options(diagram.group('options'))
},
priority='tag')
return self._process_diagram(options, diagram.group('body'))
def _process_jinja(self, spec: PosixPath, options: CombinedOptions) -> str:
"""Process dbml spec with jinja and return the resulting string"""
data = PyDBML(spec)
result = ''
if options['doc']:
if options.is_default('template') and not Path(
options['template']).exists():
# copy default template to project dir if it's doesn't exist there
copyfile(
resource_filename(__name__,
'template/' + self.defaults['template']),
options['template'])
try:
template = self._env.get_template(str(options['template']))
result += template.render(data=data)
except Exception as e:
self._warning(
f'\nFailed to render doc template {options["template"]}',
error=e)
return result
if options['scheme']:
if options.is_default('scheme_template') and\
not Path(options['scheme_template']).exists():
# copy default template to project dir if it's doesn't exist there
copyfile(
resource_filename(
__name__,
'template/' + self.defaults['scheme_template']),
options['scheme_template'])
try:
template = self._env.get_template(
str(options['scheme_template']))
result += template.render(data=data)
except Exception as e:
self._warning(
f'\nFailed to render scheme template {options["scheme_template"]}',
error=e)
return result
return result
def _process_jinja(self, spec: PosixPath, options: CombinedOptions) -> str:
"""Process swagger.json with jinja and return the resulting string"""
self.logger.debug('Using jinja mode')
data = yaml.safe_load(open(spec, encoding="utf8"))
additional = options.get('additional_json_path')
if additional:
if not additional.exists():
self._warning(
f'Additional swagger spec file {additional} is missing. Skipping'
)
else:
add = yaml.safe_load(open(additional, encoding="utf8"))
data = {**add, **data}
if options.is_default('template') and not Path(
options['template']).exists():
copyfile(
resource_filename(__name__,
'template/' + self.defaults['template']),
options['template'])
return self._to_md(data, options['template'])
def test_single_priority(self):
options1 = {'key1': 'val1', 'key2': 'val2'}
options2 = {'key2': 'val21', 'key4': 'val4'}
coptions = CombinedOptions({
'o1': options1,
'o2': options2
},
priority='o2')
expected = {**options1, **options2}
self.assertEqual(coptions.options, expected)
def test_nothing_creates_with_undefault_template_names(self):
options_dict = {
'doc_template': 'undefault_doc',
'scheme_template': 'undefault_scheme'
}
defaults_dict = {'doc_template': 'doc', 'scheme_template': 'scheme'}
options = CombinedOptions(options_dict, defaults=defaults_dict)
with patch.multiple('pgsqldoc.pgsqldoc',
copy_if_not_exists=DEFAULT,
resource_filename=DEFAULT) as mocks:
Preprocessor._create_default_templates(self.preprocessor, options)
self.assertEqual(mocks['copy_if_not_exists'].call_count, 0)
self.assertEqual(mocks['resource_filename'].call_count, 0)
def get_diagram_format(options: CombinedOptions) -> str:
'''
Parse options and get the final diagram format. Format stated in params
(e.g. tsvg) has higher priority.
:param options: the options object to be parsed
:returns: the diagram format string
'''
result = None
for key in options.get('params', {}):
if key.lower().startswith('t'):
result = key[1:]
return result or options['format']
def _sub(block) -> str:
tag_options = self.get_options(block.group('options'))
options = CombinedOptions(
{
'config': self.options,
'tag': tag_options
},
priority='tag',
convertors={'filters': yaml_to_dict_convertor},
defaults=self.defaults)
self._connect(options)
if not self._con:
return ''
self._create_default_templates(options)
return self._gen_docs(options)
def test_create_both_default_templates(self):
options_dict = {'doc_template': 'doc', 'scheme_template': 'scheme'}
options = CombinedOptions(options_dict, defaults=options_dict)
mock_resource = Mock(side_effect=['doc_resource', 'scheme_resource'])
with patch.multiple('pgsqldoc.pgsqldoc',
copy_if_not_exists=DEFAULT,
resource_filename=mock_resource) as mocks:
Preprocessor._create_default_templates(self.preprocessor, options)
self.assertEqual(mocks['copy_if_not_exists'].mock_calls, [
call(
self.preprocessor.project_path /
options_dict['doc_template'], 'doc_resource'),
call(
self.preprocessor.project_path /
options_dict['scheme_template'], 'scheme_resource')
])
def _process_widdershins(self, spec: PosixPath,
options: CombinedOptions) -> str:
"""
Process swagger.json with widdershins and return the resulting string
"""
self.logger.debug('Using widdershins mode')
environment = options.get('environment')
if environment:
if isinstance(environment, str) or isinstance(
environment, PosixPath):
env_str = f'--environment {environment}'
else: # inline config in foliant.yaml
env_yaml = str(self._swagger_tmp / 'env.yaml')
with open(env_yaml, 'w') as f:
yaml.dump(environment, f)
env_str = f'--environment {env_yaml}'
else: # not environment
env_str = ''
log_path = self._swagger_tmp / 'widdershins.log'
in_str = str(spec)
out_str = str(self._swagger_tmp / f'swagger{self._counter}.md')
cmd = f'widdershins {env_str} {in_str} -o {out_str}'
self.logger.info(f'Constructed command: \n {cmd}')
result = run(cmd, shell=True, check=True, stdout=PIPE, stderr=PIPE)
with open(log_path, 'w') as f:
f.write(result.stdout.decode())
f.write('\n\n')
f.write(result.stderr.decode())
self.logger.info(f'Build log saved at {log_path}')
if result.stderr:
error_fragment = '\n'.join(result.stderr.decode().split("\n")[:3])
self._warning('Widdershins builder returned error or warning:\n'
f'{error_fragment}\n...\n'
f'Full build log at {log_path.absolute()}')
with open(out_str) as f:
return f.read()
def process_dbmldoc_blocks(self, block) -> str:
tag_options = Options(self.get_options(block.group('options')),
convertors={
'spec_path':
rel_path_convertor(
self.current_filepath.parent)
})
options = CombinedOptions(options={
'config': self.options,
'tag': tag_options
},
priority='tag',
required=[('spec_url', ), ('spec_path', )],
defaults=self.defaults)
self.logger.debug(f'Processing dbmldoc tag in {self.current_filepath}')
spec_urls = options['spec_url']
if spec_urls and isinstance(spec_urls, str):
spec_urls = [spec_urls]
spec_path = options['spec_path']
spec = self._gather_specs(spec_urls, spec_path)
if not spec:
raise RuntimeError("No valid dbml spec file specified")
return self._process_jinja(spec, options)
def _process_diagrams(self, block) -> str:
'''
Process mermaid tag.
Save Mermaid diagram body to .mmd file, generate an image from it,
and return the image ref.
If the image for this diagram has already been generated, the existing version
is used.
:returns: Image ref
'''
tag_options = Options(self.get_options(block.group('options')))
options = CombinedOptions({'config': self.options,
'tag': tag_options},
priority='tag')
body = block.group('body')
self.logger.debug(f'Processing Mermaid diagram, options: {options}, body: {body}')
body_hash = md5(f'{body}'.encode())
body_hash.update(str(options.options).encode())
diagram_src_path = self._cache_path / 'mermaid' / f'{body_hash.hexdigest()}.mmd'
self.logger.debug(f'Diagram definition file path: {diagram_src_path}')
diagram_path = diagram_src_path.with_suffix(f'.{options["format"]}')
self.logger.debug(f'Diagram image path: {diagram_path}')
if diagram_path.exists():
self.logger.debug('Diagram image found in cache')
return f'.as_posix()})'
diagram_src_path.parent.mkdir(parents=True, exist_ok=True)
with open(diagram_src_path, 'w', encoding='utf8') as diagram_src_file:
diagram_src_file.write(body)
self.logger.debug(f'Diagram definition written into the file')
command = self._get_command(options, diagram_src_path, diagram_path)
self.logger.debug(f'Constructed command: {command}')
# when Mermaid encounters errors in diagram code, it throws error text
# into stderr but doesn't terminate the process, so we have to do it
# manually
p = subprocess.Popen(command, shell=True, stderr=subprocess.PIPE)
error_text = b''
for line in p.stderr:
error_text += line
# I know this is horrible and some day I will find a better solution
if b"DeprecationWarning" in line: # this is usually the list line
p.terminate()
p.kill()
raise RuntimeError(f'Failed to render diagram:\n\n{error_text.decode()}'
'\n\nSkipping')
self.logger.debug(f'Diagram image saved')
return f'.as_posix()})'
class DBRendererBase:
defaults = {}
module_name = __name__
def __init__(self, config):
self.config = config
def process(self, tag_options) -> str:
self.options = CombinedOptions(
{
'config': self.config,
'tag': tag_options
},
priority='tag',
defaults=self.defaults)
self.connect()
return self.gen_docs()
def connect(self):
"""
Connect to database using parameters from options.
"""
raise NotImplementedError
def get_template(self, key: str, default_name: str):
template_path = self.options.get(key)
if template_path:
return template_path
else:
return resource_filename(self.module_name,
f"templates/{default_name}")
def get_doc_template(self):
KEY = 'doc_template'
DEFAULT_NAME = 'doc.j2'
return self.get_template(KEY, DEFAULT_NAME)
def get_scheme_template(self):
KEY = 'scheme_template'
DEFAULT_NAME = 'scheme.j2'
return self.get_template(KEY, DEFAULT_NAME)
def gen_docs(self) -> str:
data = self.collect_datasets()
docs = ''
if self.options['doc']:
docs += self.to_md(data, self.get_doc_template())
if self.options['scheme']:
docs += '\n\n' + self.to_diag(data, self.get_scheme_template())
return docs
def collect_datasets(self) -> dict:
raise NotImplementedError
def to_md(self, data: dict, template: str) -> str:
template_root, template_name = os.path.split(template)
with open(template, encoding='utf8') as f:
# template = Template(f.read())
template = Environment(
loader=FileSystemLoader(template_root)).from_string(f.read())
return template.render(**data)
def to_diag(self, data: dict, template: str) -> str:
template_root, template_name = os.path.split(template)
with open(template, encoding='utf8') as f:
# template = Template(f.read())
template = Environment(
loader=FileSystemLoader(template_root)).from_string(f.read())
return template.render(tables=data['tables'])
def _process_diagrams(self, block) -> str:
'''
Process graphviz tag.
Save GraphViz diagram body to .gv file, generate an image from it,
and return the image ref.
If the image for this diagram has already been generated, the existing version
is used.
:returns: Image ref
'''
tag_options = Options(
self.get_options(block.group('options')),
validators={'engine': validate_in(self.supported_engines)},
convertors={
'as_image': boolean_convertor,
'fix_svg_size': boolean_convertor
})
options = CombinedOptions({
'config': self.options,
'tag': tag_options
},
priority='tag')
body = block.group('body')
self.logger.debug(
f'Processing GraphViz diagram, options: {options}, body: {body}')
body_hash = md5(f'{body}'.encode())
body_hash.update(str(options.options).encode())
diagram_src_path = self._cache_path / 'graphviz' / f'{body_hash.hexdigest()}.gv'
self.logger.debug(f'Diagram definition file path: {diagram_src_path}')
diagram_path = diagram_src_path.with_suffix(f'.{options["format"]}')
self.logger.debug(f'Diagram image path: {diagram_path}')
if diagram_path.exists():
self.logger.debug('Diagram image found in cache')
return self._get_result(diagram_path, options)
diagram_src_path.parent.mkdir(parents=True, exist_ok=True)
with open(diagram_src_path, 'w', encoding='utf8') as diagram_src_file:
diagram_src_file.write(body)
self.logger.debug(f'Diagram definition written into the file')
command = self._get_command(options, diagram_src_path, diagram_path)
self.logger.debug(f'Constructed command: {command}')
result = run(command, shell=True, stdout=PIPE, stderr=PIPE)
if result.returncode != 0:
self._warning(
f'Processing of GraphViz diagram failed:\n{result.stderr.decode()}',
context=self.get_tag_context(block))
return block.group(0)
if options['format'] == 'svg' and options['fix_svg_size']:
self._fix_svg_size(diagram_path)
self.logger.debug(f'Diagram image saved')
return self._get_result(diagram_path, options)