def process_extra_readme_doc(target_dir: str, prefix: str,
readme_file: str) -> DocInfo:
try:
with open(readme_file, 'r', encoding='utf-8') as f:
content = f.read()
front_matter_match = re.match(r'---\n(.*?)\n---', content, re.DOTALL)
if not front_matter_match:
raise ValueError(
f'No front matter. Extra docs must have description and title front matter. File: {readme_file}'
)
yml_matter = front_matter_match[1]
yml_data = yaml.safe_load(yml_matter)
name = yml_data['title']
file_id = yml_data.get('id') or normalize_id(name)
desc = yml_data.get('description')
readme_file_name = os.path.basename(readme_file)
edit_url = f'https://github.com/demisto/content-docs/blob/master/content-repo/extra-docs/{prefix}/{readme_file_name}'
content = content.replace(front_matter_match[0], '')
content = f'---\nid: {file_id}\ntitle: "{name}"\ncustom_edit_url: {edit_url}\n---\n\n' + content
verify_mdx_server(content)
with open(f'{target_dir}/{file_id}.md', mode='w',
encoding='utf-8') as f:
f.write(content)
return DocInfo(file_id, name, desc, readme_file)
except Exception as ex:
print(f'fail: {readme_file}. Exception: {traceback.format_exc()}')
return DocInfo('', '', '', readme_file, str(ex).splitlines()[0])
def process_release_doc(target_dir: str, release_file: str) -> DocInfo:
try:
name = os.path.splitext(os.path.basename(release_file))[0]
with open(release_file, 'r', encoding='utf-8') as f:
content = f.read()
desc_match = re.search(r'Published on .*', content, re.IGNORECASE)
if not desc_match:
raise ValueError('Published on... not found for release: ' + name)
doc_info = DocInfo(name, f'Content Release {name}', desc_match[0],
release_file)
edit_url = f'https://github.com/demisto/content-docs/blob/master/content-repo/extra-docs/releases/{name}.md'
# replace the title to be with one # so it doesn't appear in the TOC
content = re.sub(r'^## Demisto Content Release Notes',
'# Demisto Content Release Notes', content)
content = f'---\nid: {name}\ntitle: "{name}"\ncustom_edit_url: {edit_url}\nhide_title: true\n---\n\n' + content
content = content + \
f'\n\n---\n### Assets\n\n* **Download:** ' + \
f'[content_new.zip](https://github.com/demisto/content/releases/download/{name}/content_new.zip)\n' + \
f'* **Browse the Source Code:** [Content Repo @ {name}](https://github.com/demisto/content/tree/{name})\n'
verify_mdx_server(content)
with open(f'{target_dir}/{name}.md', mode='w', encoding='utf-8') as f:
f.write(content)
return doc_info
except Exception as ex:
print(
f'fail: {release_file}. Exception: {traceback.format_exc()}. Message: {ex}'
)
# We shouldn't have failing release docs. Breack the build
raise
finally:
sys.stdout.flush()
sys.stderr.flush()
def process_readme_doc(target_dir: str, content_dir: str, readme_file: str) -> DocInfo:
try:
base_dir = os.path.dirname(readme_file)
if readme_file.endswith('_README.md'):
ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml'
else:
ymlfiles = glob.glob(base_dir + '/*.yml')
if not ymlfiles:
raise ValueError('no yml file found')
if len(ymlfiles) > 1:
raise ValueError(f'mulitple yml files found: {ymlfiles}')
ymlfile = ymlfiles[0]
with open(ymlfile, 'r', encoding='utf-8') as f:
yml_data = yaml.safe_load(f)
id = yml_data.get('commonfields', {}).get('id') or yml_data['id']
id = normalize_id(id)
name = yml_data.get('display') or yml_data['name']
desc = yml_data.get('description') or yml_data.get('comment')
if desc:
word_break = False
for word in re.split(r'\s|-', desc):
if len(word) > 40:
word_break = True
desc = html.escape(desc)
if word_break: # long words tell browser to break in the midle
desc = '<span style={{wordBreak: "break-word"}}>' + desc + '</span>'
doc_info = DocInfo(id, name, desc, readme_file)
with open(readme_file, 'r', encoding='utf-8') as f:
content = f.read()
if not content.strip():
raise ValueError('empty file')
if is_html_doc(content):
print(f'{readme_file}: detect html file')
content = gen_html_doc(content)
else:
content = fix_mdx(content)
# check if we have a header
lines = content.splitlines(True)
has_header = len(lines) >= 2 and lines[0].startswith('---') and lines[1].startswith('id:')
if not has_header:
readme_repo_path = readme_file
if readme_repo_path.startswith(content_dir):
readme_repo_path = readme_repo_path[len(content_dir):]
edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}'
header = f'---\nid: {id}\ntitle: {json.dumps(doc_info.name)}\ncustom_edit_url: {edit_url}\n---\n\n'
content = get_deprecated_data(yml_data, desc, readme_file) + content
content = get_beta_data(yml_data, content) + content
content = header + content
verify_mdx_server(content)
with open(f'{target_dir}/{id}.md', mode='w', encoding='utf-8') as f: # type: ignore
f.write(content)
return doc_info
except Exception as ex:
print(f'fail: {readme_file}. Exception: {traceback.format_exc()}')
return DocInfo('', '', '', readme_file, str(ex).splitlines()[0])
finally:
sys.stdout.flush()
sys.stderr.flush()
def process_readme_doc(target_dir: str, content_dir: str, prefix: str,
imgs_dir: str, relative_images_dir: str,
readme_file: str) -> DocInfo:
try:
base_dir = os.path.dirname(readme_file)
if readme_file.endswith('_README.md'):
ymlfile = readme_file[0:readme_file.index('_README.md')] + '.yml'
else:
ymlfiles = glob.glob(base_dir + '/*.yml')
if not ymlfiles:
raise ValueError('no yml file found')
if len(ymlfiles) > 1:
raise ValueError(f'mulitple yml files found: {ymlfiles}')
ymlfile = ymlfiles[0]
with open(ymlfile, 'r', encoding='utf-8') as f:
yml_data = yaml.safe_load(f)
id = yml_data.get('commonfields', {}).get('id') or yml_data['id']
id = normalize_id(id)
name = yml_data.get('display') or yml_data['name']
desc = yml_data.get('description') or yml_data.get('comment')
if desc:
desc = handle_desc_field(desc)
doc_info = DocInfo(id, name, desc, readme_file)
with open(readme_file, 'r', encoding='utf-8') as f:
content = f.read()
if not content.strip():
raise ValueError(EMPTY_FILE_MSG)
if is_html_doc(content):
print(f'{readme_file}: detect html file')
content = gen_html_doc(content)
else:
content = fix_mdx(content)
content = fix_relative_images(content, base_dir, f'{prefix}-{id}',
imgs_dir, relative_images_dir)
# check if we have a header
lines = content.splitlines(True)
has_header = len(lines) >= 2 and lines[0].startswith(
'---') and lines[1].startswith('id:')
if not has_header:
readme_repo_path = readme_file
if readme_repo_path.startswith(content_dir):
readme_repo_path = readme_repo_path[len(content_dir):]
edit_url = f'https://github.com/demisto/content/blob/{BRANCH}/{readme_repo_path}'
header = f'---\nid: {id}\ntitle: {json.dumps(doc_info.name)}\ncustom_edit_url: {edit_url}\n---\n\n'
content = add_content_info(content, yml_data, desc, readme_file)
content = header + content
verify_mdx_server(content)
with open(f'{target_dir}/{id}.md', mode='w',
encoding='utf-8') as f: # type: ignore
f.write(content)
return doc_info
except Exception as ex:
print(f'fail: {readme_file}. Exception: {traceback.format_exc()}')
return DocInfo('', '', '', readme_file, str(ex).splitlines()[0])
finally:
sys.stdout.flush()
sys.stderr.flush()
def test_verify_mdx_server(mdx_server):
with open(f'{BASE_DIR}/test_data/good-readme.md', mode='r', encoding='utf-8') as f:
data = f.read()
verify_mdx_server(data)
# test bad readme
try:
with open(f'{BASE_DIR}/test_data/bad-mdx-readme.md', mode='r', encoding='utf-8') as f:
data = f.read()
verify_mdx_server(data)
assert False, 'should fail verify'
except Exception as ex:
assert 'Expected corresponding JSX closing tag' in str(ex)
def process_release_doc(target_dir: str,
release_file: str) -> Optional[DocInfo]:
try:
name = os.path.splitext(os.path.basename(release_file))[0]
if name < MIN_RELEASE_VERSION:
print(
f'Skipping release notes: {release_file} as it is older than: {MIN_RELEASE_VERSION}'
)
return None
with open(release_file, 'r', encoding='utf-8') as f:
content = f.read()
desc_match = re.search(r'Published on .*', content, re.IGNORECASE)
if not desc_match:
raise ValueError('Published on... not found for release: ' + name)
doc_info = DocInfo(name, f'Content Release {name}', desc_match[0],
release_file)
edit_url = f'https://github.com/demisto/content-docs/blob/master/content-repo/extra-docs/releases/{name}.md'
# replace the title to be with one # so it doesn't appear in the TOC
content = re.sub(r'^## Demisto Content Release Notes',
'# Demisto Content Release Notes', content)
content = f'---\nid: {name}\nsidebar_label: "{name}"\ncustom_edit_url: {edit_url}\n---\n\n' + content
download_msg = "Download"
packs_download = ""
if name > StrictVersion('20.8.0'):
# from 20.8.1 we also add a link to the marketplace zips
download_msg = "Download Content Zip (Cortex XSOAR 5.5 and earlier)"
packs_download = '* **Download Marketplace Packs (Cortex XSOAR 6.0 and later):** ' + \
f'[content_marketplace_packs.zip](https://github.com/demisto/content/releases/download/{name}/content_marketplace_packs.zip)\n'
content = content + \
f'\n\n---\n### Assets\n\n* **{download_msg}:** ' + \
f'[content_new.zip](https://github.com/demisto/content/releases/download/{name}/content_new.zip)\n'
if packs_download:
content = content + packs_download
content = content + \
f'* **Browse the Source Code:** [Content Repo @ {name}](https://github.com/demisto/content/tree/{name})\n'
verify_mdx_server(content)
with open(f'{target_dir}/{name}.md', mode='w', encoding='utf-8') as f:
f.write(content)
return doc_info
except Exception as ex:
print(
f'fail: {release_file}. Exception: {traceback.format_exc()}. Message: {ex}'
)
# We shouldn't have failing release docs. Breack the build
raise
finally:
sys.stdout.flush()
sys.stderr.flush()
