def process_readme_doc(target_dir: str, content_dir: str, prefix: str, imgs_dir: str, relative_images_dir: str, readme_file: str) -> DocInfo: try: base_dir = os.path.dirname(readme_file) if readme_file.endswith('_README.md'): ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml' else: ymlfiles = glob.glob(base_dir + '/*.yml') if not ymlfiles: raise ValueError('no yml file found') if len(ymlfiles) > 1: raise ValueError(f'mulitple yml files found: {ymlfiles}') ymlfile = ymlfiles[0] with open(ymlfile, 'r', encoding='utf-8') as f: yml_data = yaml.safe_load(f) id = yml_data.get('commonfields', {}).get('id') or yml_data['id'] id = normalize_id(id) name = yml_data.get('display') or yml_data['name'] desc = yml_data.get('description') or yml_data.get('comment') if desc: desc = handle_desc_field(desc) doc_info = DocInfo(id, name, desc, readme_file) with open(readme_file, 'r', encoding='utf-8') as f: content = f.read() if not content.strip(): raise ValueError(EMPTY_FILE_MSG) if is_html_doc(content): print(f'{readme_file}: detect html file') content = gen_html_doc(content) else: content = fix_mdx(content) content = fix_relative_images(content, base_dir, f'{prefix}-{id}', imgs_dir, relative_images_dir) # check if we have a header lines = content.splitlines(True) has_header = len(lines) >= 2 and lines[0].startswith( '---') and lines[1].startswith('id:') if not has_header: readme_repo_path = readme_file if readme_repo_path.startswith(content_dir): readme_repo_path = readme_repo_path[len(content_dir):] edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}' header = f'---\nid: {id}\ntitle: {json.dumps(doc_info.name)}\ncustom_edit_url: {edit_url}\n---\n\n' content = add_content_info(content, yml_data, desc, readme_file) content = header + content verify_mdx_server(content) with open(f'{target_dir}/{id}.md', mode='w', encoding='utf-8') as f: # type: ignore f.write(content) return doc_info except Exception as ex: print(f'fail: {readme_file}. Exception: {traceback.format_exc()}') return DocInfo('', '', '', readme_file, str(ex).splitlines()[0]) finally: sys.stdout.flush() sys.stderr.flush()
def get_link_for_ref_file(base_url: str, file: str): if 'releases' in file: name = os.path.splitext(os.path.basename(file))[0] return (f'Content Release {name}', f'{base_url}/docs/reference/releases/{name}') # articles/integrations yml_data = get_front_matter_data(file) name = yml_data.get('title') or file id = yml_data.get('id') or normalize_id(name) relative_path = os.path.relpath(file, ROOT_DIR).replace( 'content-repo/extra-docs', 'docs/reference') path = f'{base_url}/{os.path.dirname(relative_path)}/{id}' return (name, path)
def process_extra_readme_doc(target_dir: str, prefix: str, readme_file: str, private_packs=False) -> DocInfo: try: with open(readme_file, 'r', encoding='utf-8') as f: content = f.read() front_matter_match = re.match(r'---\n(.*?)\n---', content, re.DOTALL) if not front_matter_match: raise ValueError( f'No front matter. Extra docs must have description and title front matter. File: {readme_file}' ) yml_matter = front_matter_match[1] yml_data = yaml.safe_load(yml_matter) name = yml_data['title'] file_id = yml_data.get('id') or normalize_id(name) desc = yml_data.get('description') if desc: desc = handle_desc_field(desc) readme_file_name = os.path.basename(readme_file) content = content.replace(front_matter_match[0], '') if private_packs: print(f'Process README Private file: {readme_file}') header = f'---\nid: {file_id}\ntitle: "{name}"\ncustom_edit_url: null\n---\n\n' else: edit_url = f'https://github.com/demisto/content-docs/blob/master/content-repo/extra-docs/{prefix}/{readme_file_name}' header = f'---\nid: {file_id}\ntitle: "{name}"\ncustom_edit_url: {edit_url}\n---\n\n' content = get_deprecated_data(yml_data, desc, readme_file) + content content = get_beta_data(yml_data, content) + content content = get_fromversion_data(yml_data) + content content = get_pack_link(readme_file) + content content = header + content verify_mdx_server(content) with open(f'{target_dir}/{file_id}.md', mode='w', encoding='utf-8') as f: f.write(content) return DocInfo(file_id, name, desc, readme_file) except Exception as ex: print(f'fail: {readme_file}. Exception: {traceback.format_exc()}') return DocInfo('', '', '', readme_file, str(ex).splitlines()[0])
def test_normalize_id(): assert normalize_id("that's not good") == 'thats-not-good' assert normalize_id("have i been pwned? v2") == 'have-i-been-pwned-v2' assert normalize_id( "path/with/slash/and..-dots") == 'pathwithslashand-dots'