def test_is_default(self):
original = {'key': 'val', 'def1': 12, 'def3': 'overridden'}
defaults = {'def1': 12, 'def2': 'Default', 'def3': 'Default'}
options = Options(original, defaults=defaults)
self.assertTrue(options.is_default('def1'))
self.assertTrue(options.is_default('def2'))
self.assertFalse(options.is_default('def3'))
self.assertFalse(options.is_default('key'))
def test_required_flat(self):
required = ['req1', 'req2']
set1 = {'key': 'val', 'req1': 1, 'req2': False}
options = Options(set1, required=required)
set2 = {'key': 'val', 'req2': False}
with self.assertRaises(RequiredParamsMissingError):
options = Options(set2, required=required)
def test_required_combinations(self):
required = [['req1', 'req2'], ['req21', 'req22'], ['req']]
set1 = {'key': 'val', 'req1': 1, 'req2': False}
options = Options(set1, required=required)
set2 = {'key': 'val', 'req2': False}
with self.assertRaises(RequiredParamsMissingError):
options = Options(set2, required=required)
set3 = {'key': 'val', 'req21': False, 'req22': ''}
options = Options(set3, required=required)
set4 = {'key': 'val', 'req': '1'}
options = Options(set4, required=required)
def _sub(block) -> str:
custom_id = block.group('custom_id').strip()
if custom_id in self.applied_anchors:
self._warning(
f"Can't apply dublicate custom ID \"{custom_id}\", skipping.",
context=self.get_tag_context(block))
return ''
if custom_id in header_anchors:
self._warning(
f'Custom ID "{custom_id}" may conflict with header "{header_anchors[custom_id]}".',
context=self.get_tag_context(block))
options = Options(self.options)
illegal_char = contains_illegal_chars(custom_id)
if illegal_char:
self._warning(
f"Can't apply custom ID \"{custom_id}\", because it contains illegal symbol: {illegal_char}.",
context=self.get_tag_context(block))
return ''
self.applied_anchors.append(custom_id)
if self.context['target'] == 'pdf' and options['tex']:
element = get_tex_anchor(custom_id)
else:
element = get_anchor(custom_id, options,
self.context['target'])
return f'{element}\n\n{block.group("heading")}\n'
def test_convert(self):
mock_convertor = Mock(return_value='converted')
original = {'key': 'val', 'int': 12, 'bool': True}
options = Options(original, convertors={'int': mock_convertor})
self.assertEqual(options['int'], 'converted')
mock_convertor.assert_called_once_with(12)
def process_swaggerdoc_blocks(self, block) -> str:
tag_options = Options(
self.get_options(block.group('options')),
convertors={
'json_path':
rel_path_convertor(self.current_filepath.parent),
'spec_path':
rel_path_convertor(self.current_filepath.parent),
'additional_json_path':
rel_path_convertor(self.current_filepath.parent)
})
options = CombinedOptions(
options={
'config': self.options,
'tag': tag_options
},
priority='tag',
required=[('json_url', ), ('json_path', ), ('spec_url', ),
('spec_path', )],
validators={'mode': validate_in(self._modes)},
defaults=self.defaults)
self.logger.debug(
f'Processing swaggerdoc tag in {self.current_filepath}')
spec_url = options['spec_url'] or options['json_url']
if spec_url and isinstance(spec_url, str):
spec_url = [spec_url]
spec_path = options['spec_path'] or options['json_path']
spec = self._gather_specs(spec_url, spec_path)
if not spec:
raise RuntimeError("No valid swagger spec file specified")
return self._modes[options['mode']](spec, options)
def _get_options(self, *configs, fallback_title=None) -> Options:
'''
Get a list of dictionaries, all of which will be merged in one and
transfered to an Options object with necessary checks.
Returns the resulting Options object.
'''
options = {}
if fallback_title:
options['title'] = fallback_title
for config in configs:
options.update(config)
options = Options(options,
validators={'host': val_type(str),
'login': val_type(str),
'password': val_type(str),
'id': val_type([str, int]),
'parent_id': val_type([str, int]),
'title': val_type(str),
'space_key': val_type(str),
'pandoc_path': val_type(str),
},
required=[('id',),
('space_key', 'title')])
return options
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = self.logger.getChild('swaggerdoc')
self.logger.debug(f'Preprocessor inited: {self.__dict__}')
# '/' for abspaths
self._env = \
Environment(loader=FileSystemLoader([str(self.project_path), '/']),
extensions=["jinja2.ext.do"])
self._modes = {
'jinja': self._process_jinja,
'widdershins': self._process_widdershins
}
self._swagger_tmp = self.project_path / '.swaggercache/'
if self._swagger_tmp.exists():
remove_tree(self._swagger_tmp)
os.makedirs(self._swagger_tmp)
self._counter = 0
self.options = Options(self.options,
validators={
'json_path': validate_exists,
'spec_path': validate_exists
})
def process_template_tag(self, block) -> str:
"""
Function for processing tag. Send the contents to the corresponging
template engine along with parameters from tag and config, and
<content_file> path. Replace the tag with output from the engine.
"""
tag_options = Options(
self.get_options(block.group('options')),
validators={'engine': validate_in(self.engines.keys())})
options = CombinedOptions({
'config': self.options,
'tag': tag_options
},
priority='tag')
tag = block.group('tag')
if tag == 'template': # if "template" tag is used — engine must be specified
if 'engine' not in options:
self._warning(
'Engine must be specified in the <template> tag. Skipping.',
self.get_tag_context(block))
return block.group(0)
engine = self.engines[options['engine']]
else:
engine = self.engines[tag]
current_pos = block.start()
chapter = get_meta_for_chapter(self.current_filepath)
section = chapter.get_section_by_offset(current_pos)
_foliant_vars = {
'meta': section.data,
'meta_object': self.meta,
'config': self.config,
'target': self.context['target'],
'backend': self.context['backend'],
'project_path': self.context['project_path']
}
context = {}
# external context is loaded first, it has lowest priority
if 'ext_context' in options:
context.update(
self.load_external_context(options['ext_context'], options))
# all unrecognized params are redirected to template engine params
context.update(
{p: options[p]
for p in options if p not in self.tag_params})
# add options from "context" param
context.update(options.get('context', {}))
template = engine(block.group('body'), context,
options.get('engine_params', {}),
self.current_filepath, _foliant_vars)
return template.build()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._cachedir = (self.project_path / CACHEDIR_NAME).resolve()
self._cachedir.mkdir(exist_ok=True)
self._debug_dir = self._cachedir / DEBUG_DIR_NAME
shutil.rmtree(self._debug_dir, ignore_errors=True)
self._debug_dir.mkdir(exist_ok=True)
self._flat_src_file_path = self._cachedir / self._flat_src_file_name
self._attachments_dir = self._cachedir / ATTACHMENTS_DIR_NAME
config = self.config.get('backend_config', {}).get('confluence', {})
self.options = {**self.defaults, **config}
self.options = Options(self.options, required=['host'])
self.logger = self.logger.getChild('confluence')
self.logger.debug(f'Backend inited: {self.__dict__}')
def _sub_raw(diagram) -> str:
'''
Sub function for raw diagrams replacement (without ``<plantuml>``
tags). Handles alternation and returns spaces which were used to
filter out inline mentions of ``@startuml``.
'''
spaces = diagram.group('spaces')
body = diagram.group('body')
return spaces + self._process_diagram(Options(self.options), body)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.config = Options(self.options,
defaults=self.defaults)
self._cache_path = self.project_path / self.config['cache_dir']
self._puppeteer_config_file = self._cache_path / 'puppeteer-config.json'
self.logger = self.logger.getChild('mermaid')
self.logger.debug(f'Preprocessor inited: {self.__dict__}')
def _process_diagram(self, options: Options, body: str) -> str:
'''
Add PlantUML diagram to execution queue if it was not found in cache. Save the diagram
source into .diag file for debug.
Finally, replace diagram definition with a buffer tag. After the queue is processed
this tag will be replaced by Markdown image or inline diagram code.
:param options: Options extracted from the diagram definition
:param body: PlantUML diagram body
:returns: Buffer tag to be processed on the second pass.
'''
diagram_source = parse_diagram_source(body)
if not diagram_source:
self._warning(
'Cannot parse diagram body. Have you forgotten @startuml or @enduml?'
)
return ''
self._cache_path.mkdir(parents=True, exist_ok=True)
self.logger.debug(
f'Processing PlantUML diagram, options: {options}, body: {body}')
diag_format = get_diagram_format(options)
original_params = options.get('params', {})
if not isinstance(original_params, dict):
raise ValueError(
f'"params" should be dict, got {type(original_params).__name__}'
)
cmd_args = generate_args(original_params, diag_format,
options['plantuml_path'])
body_hash = md5(f'{cmd_args}{body}'.encode())
diag_output_path = self._cache_path / f'{body_hash.hexdigest()}.{diag_format}'
if diag_output_path.exists():
self.logger.debug('Diagram image found in cache')
else:
self.logger.debug('Adding diagram to queue')
self._add_to_queue(cmd_args, diagram_source, diag_output_path)
self.logger.debug('Diagram added to queue')
# saving diagram source into file for debug
diag_src_path = diag_output_path.with_suffix('.diag')
with open(diag_src_path, 'w', encoding='utf8') as diag_src_file:
diag_src_file.write(body)
buffer_tag = generate_buffer_tag(diag_output_path, options)
self.logger.debug(f'Generated buffer tag: {buffer_tag}')
return buffer_tag
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.config = Options(
self.options,
defaults=self.defaults,
validators={'engine': validate_in(self.supported_engines)})
self._cache_path = self.project_path / self.config['cache_dir']
self.logger = self.logger.getChild('graphviz')
self.logger.debug(f'Preprocessor inited: {self.__dict__}')
def test_defaults(self):
original = {'key': 'val', 'int': 12, 'bool': True, 'overridden': 42}
defaults = {'defaultkey': 'defaultvalue', 'overridden': 0}
expected = {
'key': 'val',
'int': 12,
'bool': True,
'overridden': 42,
'defaultkey': 'defaultvalue'
}
options = Options(original, defaults=defaults)
self.assertEqual(options.defaults, defaults)
self.assertEqual(options.options, expected)
def test_validate(self):
mock_validator1 = Mock(return_value=None)
mock_validator2 = Mock(return_value=None)
original = {'key': 'val', 'int': 12, 'bool': True}
options = Options(original,
validators={
'key': mock_validator1,
'bool': mock_validator2
})
self.assertEqual(options['key'], 'val')
self.assertEqual(options['int'], 12)
self.assertEqual(options['bool'], True)
mock_validator1.assert_called_once_with('val')
mock_validator2.assert_called_once_with(True)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = self.logger.getChild('dbmldoc')
self.logger.debug(f'Preprocessor inited: {self.__dict__}')
# '/' for abspaths
self._env = \
Environment(loader=FileSystemLoader([str(self.project_path), '/']),
extensions=["jinja2.ext.do"])
self._dbml_tmp = self.project_path / '.dbmlcache/'
if self._dbml_tmp.exists():
remove_tree(self._dbml_tmp)
os.makedirs(self._dbml_tmp)
self.options = Options(self.options,
validators={'spec_path': validate_exists})
def generate_buffer_tag(diagram_path: PosixPath, options: Options) -> str:
'''
Generate a buffer tag which should be put in place of a PlantUML diagram in Markdown source.
After processing the queue these tags should be replaced by Markdown image tags or inline
diagram code.
:param diagram_path: path to the generated diagram image file (if not in cache, it doesn't exist
at this stage).
:param options: diagram options.
:returns string with a generated buffer tag:
'''
allow_inline = ('.svg', )
inline = diagram_path.suffix in allow_inline and options[
'as_image'] is False
caption = options.get('caption', '')
result = f'<{BUFFER_TAG} file="{diagram_path}" inline="{inline}" caption="{caption}"></{BUFFER_TAG}>'
return result
def process_dbmldoc_blocks(self, block) -> str:
tag_options = Options(self.get_options(block.group('options')),
convertors={
'spec_path':
rel_path_convertor(
self.current_filepath.parent)
})
options = CombinedOptions(options={
'config': self.options,
'tag': tag_options
},
priority='tag',
required=[('spec_url', ), ('spec_path', )],
defaults=self.defaults)
self.logger.debug(f'Processing dbmldoc tag in {self.current_filepath}')
spec_urls = options['spec_url']
if spec_urls and isinstance(spec_urls, str):
spec_urls = [spec_urls]
spec_path = options['spec_path']
spec = self._gather_specs(spec_urls, spec_path)
if not spec:
raise RuntimeError("No valid dbml spec file specified")
return self._process_jinja(spec, options)
def test_no_processing(self):
original = {'key': 'val', 'int': 12, 'bool': True}
options = Options(original)
self.assertEqual(options.options, original)
def _process_diagrams(self, block) -> str:
'''
Process mermaid tag.
Save Mermaid diagram body to .mmd file, generate an image from it,
and return the image ref.
If the image for this diagram has already been generated, the existing version
is used.
:returns: Image ref
'''
tag_options = Options(self.get_options(block.group('options')))
options = CombinedOptions({'config': self.options,
'tag': tag_options},
priority='tag')
body = block.group('body')
self.logger.debug(f'Processing Mermaid diagram, options: {options}, body: {body}')
body_hash = md5(f'{body}'.encode())
body_hash.update(str(options.options).encode())
diagram_src_path = self._cache_path / 'mermaid' / f'{body_hash.hexdigest()}.mmd'
self.logger.debug(f'Diagram definition file path: {diagram_src_path}')
diagram_path = diagram_src_path.with_suffix(f'.{options["format"]}')
self.logger.debug(f'Diagram image path: {diagram_path}')
if diagram_path.exists():
self.logger.debug('Diagram image found in cache')
return f'.as_posix()})'
diagram_src_path.parent.mkdir(parents=True, exist_ok=True)
with open(diagram_src_path, 'w', encoding='utf8') as diagram_src_file:
diagram_src_file.write(body)
self.logger.debug(f'Diagram definition written into the file')
command = self._get_command(options, diagram_src_path, diagram_path)
self.logger.debug(f'Constructed command: {command}')
# when Mermaid encounters errors in diagram code, it throws error text
# into stderr but doesn't terminate the process, so we have to do it
# manually
p = subprocess.Popen(command, shell=True, stderr=subprocess.PIPE)
error_text = b''
for line in p.stderr:
error_text += line
# I know this is horrible and some day I will find a better solution
if b"DeprecationWarning" in line: # this is usually the list line
p.terminate()
p.kill()
raise RuntimeError(f'Failed to render diagram:\n\n{error_text.decode()}'
'\n\nSkipping')
self.logger.debug(f'Diagram image saved')
return f'.as_posix()})'
def test_validation_error(self):
mock_validator = Mock(side_effect=[ValidationError])
original = {'key': 'val', 'int': 12, 'bool': True}
with self.assertRaises(ValidationError):
options = Options(original, validators={'int': mock_validator})
def _process_diagrams(self, block) -> str:
'''
Process graphviz tag.
Save GraphViz diagram body to .gv file, generate an image from it,
and return the image ref.
If the image for this diagram has already been generated, the existing version
is used.
:returns: Image ref
'''
tag_options = Options(
self.get_options(block.group('options')),
validators={'engine': validate_in(self.supported_engines)},
convertors={
'as_image': boolean_convertor,
'fix_svg_size': boolean_convertor
})
options = CombinedOptions({
'config': self.options,
'tag': tag_options
},
priority='tag')
body = block.group('body')
self.logger.debug(
f'Processing GraphViz diagram, options: {options}, body: {body}')
body_hash = md5(f'{body}'.encode())
body_hash.update(str(options.options).encode())
diagram_src_path = self._cache_path / 'graphviz' / f'{body_hash.hexdigest()}.gv'
self.logger.debug(f'Diagram definition file path: {diagram_src_path}')
diagram_path = diagram_src_path.with_suffix(f'.{options["format"]}')
self.logger.debug(f'Diagram image path: {diagram_path}')
if diagram_path.exists():
self.logger.debug('Diagram image found in cache')
return self._get_result(diagram_path, options)
diagram_src_path.parent.mkdir(parents=True, exist_ok=True)
with open(diagram_src_path, 'w', encoding='utf8') as diagram_src_file:
diagram_src_file.write(body)
self.logger.debug(f'Diagram definition written into the file')
command = self._get_command(options, diagram_src_path, diagram_path)
self.logger.debug(f'Constructed command: {command}')
result = run(command, shell=True, stdout=PIPE, stderr=PIPE)
if result.returncode != 0:
self._warning(
f'Processing of GraphViz diagram failed:\n{result.stderr.decode()}',
context=self.get_tag_context(block))
return block.group(0)
if options['format'] == 'svg' and options['fix_svg_size']:
self._fix_svg_size(diagram_path)
self.logger.debug(f'Diagram image saved')
return self._get_result(diagram_path, options)
class Backend(BaseBackend):
_flat_src_file_name = '__all__.md'
targets = ('confluence')
required_preprocessors_after = [
'unescapecode',
{
'confluence_final': {
'cachedir': CACHEDIR_NAME,
'escapedir': ESCAPE_DIR_NAME
}
}
]
defaults = {'mode': 'single',
'toc': False,
'pandoc_path': 'pandoc',
'restore_comments': True,
'resolve_if_changed': False,
'notify_watchers': False,
'test_run': False,
'verify_ssl': True,
'passfile': 'confluence_secrets.yml',
'cloud': False}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._cachedir = (self.project_path / CACHEDIR_NAME).resolve()
self._cachedir.mkdir(exist_ok=True)
self._debug_dir = self._cachedir / DEBUG_DIR_NAME
shutil.rmtree(self._debug_dir, ignore_errors=True)
self._debug_dir.mkdir(exist_ok=True)
self._flat_src_file_path = self._cachedir / self._flat_src_file_name
self._attachments_dir = self._cachedir / ATTACHMENTS_DIR_NAME
config = self.config.get('backend_config', {}).get('confluence', {})
self.options = {**self.defaults, **config}
self.options = Options(self.options, required=['host'])
self.logger = self.logger.getChild('confluence')
self.logger.debug(f'Backend inited: {self.__dict__}')
def _get_options(self, *configs, fallback_title=None) -> Options:
'''
Get a list of dictionaries, all of which will be merged in one and
transfered to an Options object with necessary checks.
Returns the resulting Options object.
'''
options = {}
if fallback_title:
options['title'] = fallback_title
for config in configs:
options.update(config)
options = Options(options,
validators={'host': val_type(str),
'login': val_type(str),
'password': val_type(str),
'id': val_type([str, int]),
'parent_id': val_type([str, int]),
'title': val_type(str),
'space_key': val_type(str),
'pandoc_path': val_type(str),
},
required=[('id',),
('space_key', 'title')])
return options
def _connect(self, host: str, login: str, password: str, verify_ssl: bool) -> Confluence:
"""Connect to Confluence server and test connection"""
self.logger.debug(f'Trying to connect to confluence server at {host}')
host = host.rstrip('/')
self.con = Confluence(host, login, password, verify_ssl=verify_ssl)
try:
res = self.con.get('rest/api/space')
except UnicodeEncodeError:
raise RuntimeError('Sorry, non-ACSII passwords are not supported')
if isinstance(res, str) or 'statusCode' in res:
raise RuntimeError(f'Cannot connect to {host}:\n{res}')
def _get_credentials(self, host: str) -> tuple:
def get_password_for_login(login: str) -> str:
if 'password' in self.options:
return self.options['password']
else:
password = passdict.get(host.rstrip('/'), {}).get(login)
if password:
return password
else:
msg = '\n!!! User input required !!!\n'
msg += f"Please input password for {login}:\n"
return getpass(msg)
self.logger.debug(f'Loading passfile {self.options["passfile"]}')
if os.path.exists(self.options['passfile']):
self.logger.debug(f'Found passfile at {self.options["passfile"]}')
with open(self.options['passfile'], encoding='utf8') as f:
passdict = yaml.load(f, yaml.Loader)
else:
passdict = {}
if 'login' in self.options:
login = self.options['login']
password = get_password_for_login(login)
else: # login not in self.options
host_dict = passdict.get(host, {})
if host_dict:
# getting first login from passdict
login = next(iter(host_dict.keys()))
else:
msg = '\n!!! User input required !!!\n'
msg += f"Please input login for {host}:\n"
login = input(msg)
password = get_password_for_login(login)
return login, password
def _build(self):
'''
Main method. Builds confluence XHTML document from flat md source and
uploads it into the confluence server.
'''
host = self.options['host']
credentials = self._get_credentials(host)
self.logger.debug(f'Got credentials for host {host}: login {credentials[0]}, '
f'password {credentials[1]}')
self._connect(host,
*credentials,
self.options['verify_ssl'])
result = []
if 'id' in self.options or ('title' in self.options and 'space_key' in self.options):
self.logger.debug('Uploading flat project to confluence')
output(f'Building main project', self.quiet)
flatten.Preprocessor(
self.context,
self.logger,
self.quiet,
self.debug,
{'flat_src_file_name': self._flat_src_file_name,
'keep_sources': True}
).apply()
unescapecode.Preprocessor(
self.context,
self.logger,
self.quiet,
self.debug,
{}
).apply()
shutil.move(self.working_dir / self._flat_src_file_name,
self._flat_src_file_path)
with open(self._flat_src_file_path, encoding='utf8') as f:
md_source = f.read()
options = self._get_options(self.options)
self.logger.debug(f'Options: {options}')
uploader = PageUploader(
self._flat_src_file_path,
options,
self.con,
self._cachedir,
self._debug_dir,
self._attachments_dir,
self.logger
)
try:
result.append(uploader.upload(md_source))
except HTTPError as e:
# reraising HTTPError with meaningful message
raise HTTPError(e.response.text, e.response)
self.logger.debug('Searching metadata for confluence properties')
chapters = self.config['chapters']
meta = load_meta(chapters, self.working_dir)
for section in meta.iter_sections():
if not isinstance(section.data.get('confluence'), dict):
self.logger.debug(f'No "confluence" section in {section}), skipping.')
continue
self.logger.debug(f'Found "confluence" section in {section}), preparing to build.')
# getting common options from foliant.yml and merging them with meta fields
common_options = {}
uncommon_options = ['title', 'id', 'space_key', 'parent_id', 'attachments']
common_options = {k: v for k, v in self.options.items()
if k not in uncommon_options}
try:
options = self._get_options(common_options,
section.data['confluence'],
fallback_title=section.title)
except Exception as e:
# output(f'Skipping section {section}, wrong params: {e}', self.quiet)
self.logger.debug(f'Skipping section {section}, wrong params: {e}')
continue
self.logger.debug(f'Building {section.chapter.filename}: {section.title}')
output(f'Building {section.title}', self.quiet)
md_source = section.get_source()
self.logger.debug(f'Options: {options}')
original_file = self.project_path / section.chapter.filename
uploader = PageUploader(
original_file,
options,
self.con,
self._cachedir,
self._debug_dir,
self._attachments_dir,
self.logger
)
try:
result.append(uploader.upload(md_source))
except HTTPError as e:
# reraising HTTPError with meaningful message
raise HTTPError(e.response.text, e.response)
if result:
return '\n' + '\n'.join(result)
else:
return 'nothing to upload'
def make(self, target: str) -> str:
with spinner(f'Making {target}', self.logger, self.quiet, self.debug):
output('', self.quiet) # empty line for better output
try:
if target == 'confluence':
return self._build()
else:
raise ValueError(f'Confluence cannot make {target}')
except Exception as exception:
raise RuntimeError(f'Build failed: {exception}')