def test_fix_mdx():
res = fix_mdx('this<br>is<hr>')
assert '<br/>' in res
assert '<hr/>' in res
res = fix_mdx('<!-- html comment \n here --> some text <!-- another comment here-->')
assert 'some text ' in res
assert '<!--' not in res
assert '-->' not in res
assert 'html comment' not in res
res = fix_mdx('multiple<br>values<br>')
assert '<br>' not in res
res = fix_mdx('valide br: <br></br>')
assert '<br></br>' in res
def process_readme_doc(target_dir: str, content_dir: str, readme_file: str) -> DocInfo:
try:
base_dir = os.path.dirname(readme_file)
if readme_file.endswith('_README.md'):
ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml'
else:
ymlfiles = glob.glob(base_dir + '/*.yml')
if not ymlfiles:
raise ValueError('no yml file found')
if len(ymlfiles) > 1:
raise ValueError(f'mulitple yml files found: {ymlfiles}')
ymlfile = ymlfiles[0]
with open(ymlfile, 'r', encoding='utf-8') as f:
yml_data = yaml.safe_load(f)
id = yml_data.get('commonfields', {}).get('id') or yml_data['id']
id = normalize_id(id)
name = yml_data.get('display') or yml_data['name']
desc = yml_data.get('description') or yml_data.get('comment')
if desc:
word_break = False
for word in re.split(r'\s|-', desc):
if len(word) > 40:
word_break = True
desc = html.escape(desc)
if word_break: # long words tell browser to break in the midle
desc = '<span style={{wordBreak: "break-word"}}>' + desc + '</span>'
doc_info = DocInfo(id, name, desc, readme_file)
with open(readme_file, 'r', encoding='utf-8') as f:
content = f.read()
if not content.strip():
raise ValueError('empty file')
if is_html_doc(content):
print(f'{readme_file}: detect html file')
content = gen_html_doc(content)
else:
content = fix_mdx(content)
# check if we have a header
lines = content.splitlines(True)
has_header = len(lines) >= 2 and lines[0].startswith('---') and lines[1].startswith('id:')
if not has_header:
readme_repo_path = readme_file
if readme_repo_path.startswith(content_dir):
readme_repo_path = readme_repo_path[len(content_dir):]
edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}'
header = f'---\nid: {id}\ntitle: {json.dumps(doc_info.name)}\ncustom_edit_url: {edit_url}\n---\n\n'
content = get_deprecated_data(yml_data, desc, readme_file) + content
content = get_beta_data(yml_data, content) + content
content = header + content
verify_mdx_server(content)
with open(f'{target_dir}/{id}.md', mode='w', encoding='utf-8') as f: # type: ignore
f.write(content)
return doc_info
except Exception as ex:
print(f'fail: {readme_file}. Exception: {traceback.format_exc()}')
return DocInfo('', '', '', readme_file, str(ex).splitlines()[0])
finally:
sys.stdout.flush()
sys.stderr.flush()
def process_readme_doc(target_dir: str, content_dir: str, prefix: str,
imgs_dir: str, relative_images_dir: str,
readme_file: str) -> DocInfo:
try:
base_dir = os.path.dirname(readme_file)
if readme_file.endswith('_README.md'):
ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml'
else:
ymlfiles = glob.glob(base_dir + '/*.yml')
if not ymlfiles:
raise ValueError('no yml file found')
if len(ymlfiles) > 1:
raise ValueError(f'mulitple yml files found: {ymlfiles}')
ymlfile = ymlfiles[0]
with open(ymlfile, 'r', encoding='utf-8') as f:
yml_data = yaml.safe_load(f)
id = yml_data.get('commonfields', {}).get('id') or yml_data['id']
id = normalize_id(id)
name = yml_data.get('display') or yml_data['name']
desc = yml_data.get('description') or yml_data.get('comment')
if desc:
desc = handle_desc_field(desc)
doc_info = DocInfo(id, name, desc, readme_file)
with open(readme_file, 'r', encoding='utf-8') as f:
content = f.read()
if not content.strip():
raise ValueError(EMPTY_FILE_MSG)
if is_html_doc(content):
print(f'{readme_file}: detect html file')
content = gen_html_doc(content)
else:
content = fix_mdx(content)
content = fix_relative_images(content, base_dir, f'{prefix}-{id}',
imgs_dir, relative_images_dir)
# check if we have a header
lines = content.splitlines(True)
has_header = len(lines) >= 2 and lines[0].startswith(
'---') and lines[1].startswith('id:')
if not has_header:
readme_repo_path = readme_file
if readme_repo_path.startswith(content_dir):
readme_repo_path = readme_repo_path[len(content_dir):]
edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}'
header = f'---\nid: {id}\ntitle: {json.dumps(doc_info.name)}\ncustom_edit_url: {edit_url}\n---\n\n'
content = add_content_info(content, yml_data, desc, readme_file)
content = header + content
verify_mdx_server(content)
with open(f'{target_dir}/{id}.md', mode='w',
encoding='utf-8') as f: # type: ignore
f.write(content)
return doc_info
except Exception as ex:
print(f'fail: {readme_file}. Exception: {traceback.format_exc()}')
return DocInfo('', '', '', readme_file, str(ex).splitlines()[0])
finally:
sys.stdout.flush()
sys.stderr.flush()
def index_doc_infos(doc_infos: List[DocInfo], link_prefix: str):
if not doc_infos:
return ''
table_items = []
for d in doc_infos:
table_items.append({
'Name': f'[{d.name}]({link_prefix}/{d.id})',
'Description': d.description
})
res = tableToMarkdown('', table_items, headers=['Name', 'Description'])
return fix_mdx(res)
def index_doc_infos(doc_infos: List[DocInfo],
link_prefix: str,
headers: Optional[Tuple[str, str]] = None):
if not headers:
headers = ('Name', 'Description')
if not doc_infos:
return ''
table_items = []
for d in doc_infos:
name = html.escape(d.name)
link_name = f'[{name}]({link_prefix}/{d.id})'
for word in re.split(r'\s|-', name):
if len(word
) > 25: # we have a long word tell browser ok to break it
link_name = '<span style={{wordBreak: "break-word"}}>' + link_name + '</span>'
break
table_items.append({headers[0]: link_name, headers[1]: d.description})
res = tableToMarkdown('', table_items, headers=headers)
return fix_mdx(res)
def process_integration_doc(readme_file: str, target_dir: str, content_dir: str) -> DocInfo:
base_dir = os.path.dirname(readme_file)
if readme_file.endswith('_README.md'):
ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml'
else:
ymlfiles = glob.glob(base_dir + '/*.yml')
if not ymlfiles:
raise ValueError(f'no yml file found')
if len(ymlfiles) > 1:
raise ValueError(f'mulitple yml files found: {ymlfiles}')
ymlfile = ymlfiles[0]
with open(ymlfile, 'r', encoding='utf-8') as f:
yml_data = yaml.safe_load(f)
id = yml_data['commonfields']['id']
id = inflection.dasherize(inflection.underscore(id)).replace(' ', '-')
doc_info = DocInfo(id, yml_data['display'], yml_data.get('description'))
with open(readme_file, 'r', encoding='utf-8') as f:
content = f.read()
if not content.strip():
raise ValueError(f'empty file')
if is_html_doc(content):
print(f'{readme_file}: detect html file')
content = gen_html_doc(content)
else:
content = fix_mdx(content)
# check if we have a header
lines = content.splitlines(True)
has_header = len(lines) >= 2 and lines[0].startswith('---') and lines[1].startswith('id:')
if not has_header:
readme_repo_path = readme_file
if readme_repo_path.startswith(content_dir):
readme_repo_path = readme_repo_path[len(content_dir):]
edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}'
content = f'---\nid: {id}\ntitle: {doc_info.name}\ncustom_edit_url: {edit_url}\n---\n\n' + content
with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f: # type: ignore
f.write(content)
f.flush()
verify_mdx(f.name)
shutil.copy(f.name, f'{target_dir}/{id}.md')
return doc_info
