def test_fix_mdx(): res = fix_mdx('this<br>is<hr>') assert '<br/>' in res assert '<hr/>' in res res = fix_mdx('<!-- html comment \n here --> some text <!-- another comment here-->') assert 'some text ' in res assert '<!--' not in res assert '-->' not in res assert 'html comment' not in res res = fix_mdx('multiple<br>values<br>') assert '<br>' not in res res = fix_mdx('valide br: <br></br>') assert '<br></br>' in res
def process_readme_doc(target_dir: str, content_dir: str, readme_file: str) -> DocInfo: try: base_dir = os.path.dirname(readme_file) if readme_file.endswith('_README.md'): ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml' else: ymlfiles = glob.glob(base_dir + '/*.yml') if not ymlfiles: raise ValueError('no yml file found') if len(ymlfiles) > 1: raise ValueError(f'mulitple yml files found: {ymlfiles}') ymlfile = ymlfiles[0] with open(ymlfile, 'r', encoding='utf-8') as f: yml_data = yaml.safe_load(f) id = yml_data.get('commonfields', {}).get('id') or yml_data['id'] id = normalize_id(id) name = yml_data.get('display') or yml_data['name'] desc = yml_data.get('description') or yml_data.get('comment') if desc: word_break = False for word in re.split(r'\s|-', desc): if len(word) > 40: word_break = True desc = html.escape(desc) if word_break: # long words tell browser to break in the midle desc = '<span style={{wordBreak: "break-word"}}>' + desc + '</span>' doc_info = DocInfo(id, name, desc, readme_file) with open(readme_file, 'r', encoding='utf-8') as f: content = f.read() if not content.strip(): raise ValueError('empty file') if is_html_doc(content): print(f'{readme_file}: detect html file') content = gen_html_doc(content) else: content = fix_mdx(content) # check if we have a header lines = content.splitlines(True) has_header = len(lines) >= 2 and lines[0].startswith('---') and lines[1].startswith('id:') if not has_header: readme_repo_path = readme_file if readme_repo_path.startswith(content_dir): readme_repo_path = readme_repo_path[len(content_dir):] edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}' header = f'---\nid: {id}\ntitle: {json.dumps(doc_info.name)}\ncustom_edit_url: {edit_url}\n---\n\n' content = get_deprecated_data(yml_data, desc, readme_file) + content content = get_beta_data(yml_data, content) + content content = header + content verify_mdx_server(content) with open(f'{target_dir}/{id}.md', mode='w', encoding='utf-8') as f: # type: ignore f.write(content) return doc_info except Exception as ex: print(f'fail: {readme_file}. Exception: {traceback.format_exc()}') return DocInfo('', '', '', readme_file, str(ex).splitlines()[0]) finally: sys.stdout.flush() sys.stderr.flush()
def process_readme_doc(target_dir: str, content_dir: str, prefix: str, imgs_dir: str, relative_images_dir: str, readme_file: str) -> DocInfo: try: base_dir = os.path.dirname(readme_file) if readme_file.endswith('_README.md'): ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml' else: ymlfiles = glob.glob(base_dir + '/*.yml') if not ymlfiles: raise ValueError('no yml file found') if len(ymlfiles) > 1: raise ValueError(f'mulitple yml files found: {ymlfiles}') ymlfile = ymlfiles[0] with open(ymlfile, 'r', encoding='utf-8') as f: yml_data = yaml.safe_load(f) id = yml_data.get('commonfields', {}).get('id') or yml_data['id'] id = normalize_id(id) name = yml_data.get('display') or yml_data['name'] desc = yml_data.get('description') or yml_data.get('comment') if desc: desc = handle_desc_field(desc) doc_info = DocInfo(id, name, desc, readme_file) with open(readme_file, 'r', encoding='utf-8') as f: content = f.read() if not content.strip(): raise ValueError(EMPTY_FILE_MSG) if is_html_doc(content): print(f'{readme_file}: detect html file') content = gen_html_doc(content) else: content = fix_mdx(content) content = fix_relative_images(content, base_dir, f'{prefix}-{id}', imgs_dir, relative_images_dir) # check if we have a header lines = content.splitlines(True) has_header = len(lines) >= 2 and lines[0].startswith( '---') and lines[1].startswith('id:') if not has_header: readme_repo_path = readme_file if readme_repo_path.startswith(content_dir): readme_repo_path = readme_repo_path[len(content_dir):] edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}' header = f'---\nid: {id}\ntitle: {json.dumps(doc_info.name)}\ncustom_edit_url: {edit_url}\n---\n\n' content = add_content_info(content, yml_data, desc, readme_file) content = header + content verify_mdx_server(content) with open(f'{target_dir}/{id}.md', mode='w', encoding='utf-8') as f: # type: ignore f.write(content) return doc_info except Exception as ex: print(f'fail: {readme_file}. Exception: {traceback.format_exc()}') return DocInfo('', '', '', readme_file, str(ex).splitlines()[0]) finally: sys.stdout.flush() sys.stderr.flush()
def index_doc_infos(doc_infos: List[DocInfo], link_prefix: str): if not doc_infos: return '' table_items = [] for d in doc_infos: table_items.append({ 'Name': f'[{d.name}]({link_prefix}/{d.id})', 'Description': d.description }) res = tableToMarkdown('', table_items, headers=['Name', 'Description']) return fix_mdx(res)
def index_doc_infos(doc_infos: List[DocInfo], link_prefix: str, headers: Optional[Tuple[str, str]] = None): if not headers: headers = ('Name', 'Description') if not doc_infos: return '' table_items = [] for d in doc_infos: name = html.escape(d.name) link_name = f'[{name}]({link_prefix}/{d.id})' for word in re.split(r'\s|-', name): if len(word ) > 25: # we have a long word tell browser ok to break it link_name = '<span style={{wordBreak: "break-word"}}>' + link_name + '</span>' break table_items.append({headers[0]: link_name, headers[1]: d.description}) res = tableToMarkdown('', table_items, headers=headers) return fix_mdx(res)
def process_integration_doc(readme_file: str, target_dir: str, content_dir: str) -> DocInfo: base_dir = os.path.dirname(readme_file) if readme_file.endswith('_README.md'): ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml' else: ymlfiles = glob.glob(base_dir + '/*.yml') if not ymlfiles: raise ValueError(f'no yml file found') if len(ymlfiles) > 1: raise ValueError(f'mulitple yml files found: {ymlfiles}') ymlfile = ymlfiles[0] with open(ymlfile, 'r', encoding='utf-8') as f: yml_data = yaml.safe_load(f) id = yml_data['commonfields']['id'] id = inflection.dasherize(inflection.underscore(id)).replace(' ', '-') doc_info = DocInfo(id, yml_data['display'], yml_data.get('description')) with open(readme_file, 'r', encoding='utf-8') as f: content = f.read() if not content.strip(): raise ValueError(f'empty file') if is_html_doc(content): print(f'{readme_file}: detect html file') content = gen_html_doc(content) else: content = fix_mdx(content) # check if we have a header lines = content.splitlines(True) has_header = len(lines) >= 2 and lines[0].startswith('---') and lines[1].startswith('id:') if not has_header: readme_repo_path = readme_file if readme_repo_path.startswith(content_dir): readme_repo_path = readme_repo_path[len(content_dir):] edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}' content = f'---\nid: {id}\ntitle: {doc_info.name}\ncustom_edit_url: {edit_url}\n---\n\n' + content with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f: # type: ignore f.write(content) f.flush() verify_mdx(f.name) shutil.copy(f.name, f'{target_dir}/{id}.md') return doc_info