def test_format_link(tmp_path):
tmp_directory = tmp_path / "tests"
tmp_directory.mkdir()
bad_file = tmp_directory / "bad_file.md"
bad_file.write_text(data_to_test)
assert expected_data == format_link_file(bad_file,
regex_skip_sections_start,
regex_skip_sections_end)
def process_integration_readme(self, file_name):
"""
Take a single README.md file and
1. extract the first h1, if this isn't a merge item
2. add tabs if they exist
3. inject metrics after ### Metrics header if metrics exists for file
4. inject service checks after ### Service Checks if file exists
5. inject hugo front matter params at top of file
6. write out file to content/integrations with filename changed to integrationname.md
:param file_name: path to a readme md file
"""
no_integration_issue = True
metrics = glob.glob(
"{path}{sep}*metadata.csv".format(
path=dirname(file_name), sep=sep
)
)
metrics = metrics[0] if len(metrics) > 0 else None
metrics_exist = (metrics and exists(metrics)
and linecache.getline(metrics, 2))
service_check = glob.glob("{file}.json".format(
file=self.data_service_checks_dir + basename(dirname(file_name))))
service_check = (
service_check[0]
if len(service_check) > 0
else None
)
service_check_exist = service_check and exists(
service_check
)
manifest = "{0}{1}{2}".format(
dirname(file_name), sep, "manifest.json"
)
if exists(manifest):
try:
manifest_json = json.load(open(manifest))
except JSONDecodeError:
no_integration_issue = False
manifest_json = {}
if getenv("LOCAL") == 'True':
print(
"\x1b[33mWARNING\x1b[0m: manifest could not be parsed {}".format(manifest))
else:
print(
"\x1b[31mERROR\x1b[0m: manifest could not be parsed {}".format(manifest))
raise JSONDecodeError
else:
no_integration_issue = False
manifest_json = {}
print(
"\x1b[33mWARNING\x1b[0m: No manifest found for {}".format(file_name))
dependencies = self.add_dependencies(file_name)
new_file_name = "{}.md".format(
basename(dirname(file_name))
)
exist_already = exists(
self.content_integrations_dir + new_file_name
)
regex_skip_sections_end = r"(```|\{\{< \/code-block >\}\})"
regex_skip_sections_start = r"(```|\{\{< code-block)"
## Formating all link as reference to avoid any corner cases
try:
result = format_link_file(file_name,regex_skip_sections_start,regex_skip_sections_end)
except Exception as e:
print(e)
## Check if there is a integration tab logic in the integration file:
if "<!-- xxx tabs xxx -->" in result:
tab_logic = True
## Inlining all links
result = self.inline_references(result,regex_skip_sections_start,regex_skip_sections_end)
else:
tab_logic= False
title = manifest_json.get("name", "").lower()
if title not in [
k
for k, v in self.integration_mutations.items()
if v.get("action") == "merge"
]:
result = re.sub(
self.regex_h1, "", result, 1
)
result = re.sub(
self.regex_tabs_open, "{{< tabs >}}", result, 0
)
result = re.sub(
self.regex_tabs_close, "{{< /tabs >}}", result, 0
)
result = re.sub(
self.regex_tab_open, "{{% tab", result, 0
)
result = re.sub(
self.regex_tab_close, "{{% /tab %}}", result, 0
)
result = re.sub(
self.regex_tab_end, " %}}", result, 0
)
if metrics_exist:
result = re.sub(
self.regex_metrics,
r'\1{{< get-metrics-from-git "%s" >}}\n\3\4'
% format(title),
result,
0,
)
if service_check_exist:
result = re.sub(
self.regex_service_check,
r'\1{{< get-service-checks-from-git "%s" >}}\n\3\4'
% format(title),
result,
0,
)
result = self.add_integration_frontmatter(
new_file_name, result, dependencies
)
if not exist_already and no_integration_issue:
with open(self.content_integrations_dir + new_file_name, "w", ) as out:
out.write(result)
## Reformating all links now that all processing is done
if tab_logic:
final_text = format_link_file(self.content_integrations_dir + new_file_name,regex_skip_sections_start,regex_skip_sections_end)
with open(self.content_integrations_dir + new_file_name, 'w') as final_file:
final_file.write(final_text)
def merge_integrations(self):
""" Merges integrations that come under one """
for (
name,
action_obj,
) in self.integration_mutations.items():
if name not in self.initial_integration_files:
action = action_obj.get("action")
target = action_obj.get("target")
input_file = "{}{}.md".format(self.content_integrations_dir,
name)
output_file = "{}{}.md".format(self.content_integrations_dir,
target)
if action == "merge":
with open(input_file,
"r") as content_file, open(output_file,
"a") as target_file:
content = content_file.read()
content = re.sub(
self.regex_fm,
r"\2",
content,
count=0,
)
if action_obj.get("remove_header", False):
content = re.sub(
self.regex_h1,
"",
content,
count=0,
)
else:
content = re.sub(
self.regex_h1_replace,
r"##\2",
content,
count=0,
)
regex_skip_sections_end = r"(```|\{\{< \/code-block >\}\})"
regex_skip_sections_start = r"(```|\{\{< code-block)"
## Inlining all link from the file to merge
## to avoid link ref colision with the existing references.
content = self.inline_references(
content, regex_skip_sections_start,
regex_skip_sections_end)
target_file.write(content)
## Formating all link as reference in the new merged integration file
try:
final_text = format_link_file(
output_file, regex_skip_sections_start,
regex_skip_sections_end)
with open(output_file, 'w') as final_file:
final_file.write(final_text)
except Exception as e:
print(e)
try:
remove(input_file)
except OSError:
print(
"\x1b[31mERROR\x1b[0m: The file {} was not found and could not be removed during merge action"
.format(input_file))
elif action == "truncate":
if exists(output_file):
with open(output_file, "r+") as target_file:
content = target_file.read()
content = re.sub(
self.regex_fm,
r"---\n\1\n---\n",
content,
count=0,
)
target_file.truncate(0)
target_file.seek(0)
target_file.write(content)
else:
open(output_file, "w").close()
elif action == "discard":
try:
remove(input_file)
except OSError:
print(
"\x1b[31mERROR\x1b[0m: The file {} was not found and could not be removed during discard action"
.format(input_file))
elif action == "create":
with open(output_file, "w+") as f:
fm = yaml.dump(
action_obj.get("fm"),
default_flow_style=False,
).rstrip()
data = "---\n{0}\n---\n".format(fm)
f.write(data)
def process_integration_readme(self, file_name, marketplace=False):
"""
Take a single README.md file and
1. extract the first h1, if this isn't a merge item
2. add tabs if they exist
3. inject metrics after ### Metrics header if metrics exists for file
4. inject service checks after ### Service Checks if file exists
5. inject hugo front matter params at top of file
6. write out file to content/integrations with filename changed to integrationname.md
:param file_name: path to a readme md file
"""
no_integration_issue = True
tab_logic = False
metrics = glob.glob("{path}{sep}*metadata.csv".format(
path=dirname(file_name), sep=sep))
metrics = metrics[0] if len(metrics) > 0 else None
metrics_exist = (metrics and exists(metrics)
and linecache.getline(metrics, 2))
service_check = glob.glob("{file}.json".format(
file=self.data_service_checks_dir + basename(dirname(file_name))))
service_check = (service_check[0] if len(service_check) > 0 else None)
service_check_exist = service_check and exists(service_check)
manifest = "{0}{1}{2}".format(dirname(file_name), sep, "manifest.json")
if exists(manifest):
try:
manifest_json = json.load(open(manifest))
except JSONDecodeError:
no_integration_issue = False
manifest_json = {}
if getenv("LOCAL") == 'True':
print(
"\x1b[33mWARNING\x1b[0m: manifest could not be parsed {}"
.format(manifest))
else:
print(
"\x1b[31mERROR\x1b[0m: manifest could not be parsed {}"
.format(manifest))
raise JSONDecodeError
else:
no_integration_issue = False
manifest_json = {}
print("\x1b[33mWARNING\x1b[0m: No manifest found for {}".format(
file_name))
dependencies = self.add_dependencies(file_name)
new_file_name = "{}.md".format(basename(dirname(file_name)))
# is this the same as a committed hardcoded integration
exist_already = (self.content_integrations_dir + new_file_name
in self.initial_integration_files)
# is this overwriting another generated integration
exist_collision = exists(self.content_integrations_dir + new_file_name)
regex_skip_sections_end = r"(```|\{\{< \/code-block >\}\})"
regex_skip_sections_start = r"(```|\{\{< code-block)"
## Formating all link as reference to avoid any corner cases
## Replace image filenames in markdown for marketplace interations
if not marketplace:
try:
result = format_link_file(file_name, regex_skip_sections_start,
regex_skip_sections_end)
except Exception as e:
print(e)
else:
with open(file_name, 'r+') as f:
markdown_string = f.read()
markdown_with_replaced_images = self.replace_image_src(
markdown_string, basename(dirname(file_name)))
updated_markdown = self.remove_markdown_section(
markdown_with_replaced_images, '## Setup')
is_marketplace_integration_markdown_valid = self.validate_marketplace_integration_markdown(
updated_markdown)
if not is_marketplace_integration_markdown_valid:
raise Exception(
'Potential setup or pricing information included in Marketplace Integration markdown. Check {} for Setup or Pricing sections.'
.format(file_name))
else:
result = updated_markdown
## Check if there is a integration tab logic in the integration file:
if "<!-- xxx tabs xxx -->" in result:
tab_logic = True
## Inlining all links
result = self.inline_references(result, regex_skip_sections_start,
regex_skip_sections_end)
else:
tab_logic = False
title = manifest_json.get("name", "").lower()
if title not in [
k for k, v in self.integration_mutations.items()
if v.get("action") == "merge"
]:
result = re.sub(self.regex_h1, "", result, 1)
result = re.sub(self.regex_tabs_open, "{{< tabs >}}", result, 0)
result = re.sub(self.regex_tabs_close, "{{< /tabs >}}", result, 0)
result = re.sub(self.regex_tab_open, "{{% tab", result, 0)
result = re.sub(self.regex_tab_close, "{{% /tab %}}", result, 0)
result = re.sub(self.regex_tab_end, " %}}", result, 0)
if metrics_exist:
result = re.sub(
self.regex_metrics,
r'\1{{< get-metrics-from-git "%s" >}}\n\3\4' % format(title),
result,
0,
)
if service_check_exist:
result = re.sub(
self.regex_service_check,
r'\1{{< get-service-checks-from-git "%s" >}}\n\3\4' %
format(title),
result,
0,
)
if not exist_already and no_integration_issue:
# lets only write out file.md if its going to be public
if manifest_json.get("is_public", False):
out_name = self.content_integrations_dir + new_file_name
# if the same integration exists in multiple locations try name md after manifest name entry
if exist_collision:
f_name = manifest_json.get("name", new_file_name)
f_name = f_name if f_name.endswith(
'.md') else f_name + ".md"
out_name = self.content_integrations_dir + f_name
print(
"\x1b[33mWARNING\x1b[0m: Collision, duplicate integration {} trying as {}"
.format(new_file_name, f_name))
new_file_name = f_name
result = self.add_integration_frontmatter(
new_file_name, result, dependencies)
with open(
out_name,
"w",
) as out:
out.write(result)
## Reformating all links now that all processing is done
if tab_logic:
final_text = format_link_file(out_name,
regex_skip_sections_start,
regex_skip_sections_end)
with open(out_name, 'w') as final_file:
final_file.write(final_text)
def process_integration_readme(self, file_name, marketplace=False):
"""
Take a single README.md file and
1. extract the first h1, if this isn't a merge item
2. add tabs if they exist
3. inject metrics after ### Metrics header if metrics exists for file
4. inject service checks after ### Service Checks if file exists
5. inject hugo front matter params at top of file
6. write out file to content/integrations with filename changed to integrationname.md
:param file_name: path to a readme md file
"""
no_integration_issue = True
tab_logic = False
metrics = glob.glob("{path}{sep}*metadata.csv".format(
path=dirname(file_name), sep=sep))
metrics = metrics[0] if len(metrics) > 0 else None
metrics_exist = (metrics and exists(metrics)
and linecache.getline(metrics, 2))
service_check = glob.glob("{file}.json".format(
file=self.data_service_checks_dir + basename(dirname(file_name))))
service_check = (service_check[0] if len(service_check) > 0 else None)
service_check_exist = service_check and exists(service_check)
manifest = "{0}{1}{2}".format(dirname(file_name), sep, "manifest.json")
if exists(manifest):
try:
manifest_json = json.load(open(manifest))
manifest_json = self.process_manifest(
manifest_json, basename(dirname(file_name)))
except JSONDecodeError:
no_integration_issue = False
manifest_json = {}
if getenv("LOCAL") == 'True':
print(
"\x1b[33mWARNING\x1b[0m: manifest could not be parsed {}"
.format(manifest))
else:
print(
"\x1b[31mERROR\x1b[0m: manifest could not be parsed {}"
.format(manifest))
raise JSONDecodeError
else:
no_integration_issue = False
manifest_json = {}
print("\x1b[33mWARNING\x1b[0m: No manifest found for {}".format(
file_name))
dependencies = self.add_dependencies(file_name)
new_file_name = "{}.md".format(basename(dirname(file_name)))
# is this the same as a committed hardcoded integration
exist_already = (self.content_integrations_dir + new_file_name
in self.initial_integration_files)
# is this overwriting another generated integration
exist_collision = exists(self.content_integrations_dir + new_file_name)
regex_skip_sections_end = r"(```|\{\{< \/code-block |\{\{< \/site-region >\}\})"
regex_skip_sections_start = r"(```|\{\{< code-block |\{\{< site-region)"
## Formating all link as reference to avoid any corner cases
## Replace image filenames in markdown for marketplace interations
result = ''
if not marketplace:
try:
result = format_link_file(file_name, regex_skip_sections_start,
regex_skip_sections_end)
except Exception as e:
print(e)
else:
with open(file_name, 'r+') as f:
markdown_string = f.read()
markdown_with_replaced_images = self.replace_image_src(
markdown_string, basename(dirname(file_name)))
markdown_setup_removed = self.remove_markdown_section(
markdown_with_replaced_images, '## Setup')
updated_markdown = self.remove_h3_markdown_section(
markdown_setup_removed, '### Pricing')
is_marketplace_integration_markdown_valid = self.validate_marketplace_integration_markdown(
updated_markdown)
if not is_marketplace_integration_markdown_valid:
raise Exception(
'Potential setup or pricing information included in Marketplace Integration markdown. Check {} for Setup or Pricing sections.'
.format(file_name))
else:
result = updated_markdown
## Check if there is a integration tab logic in the integration file:
if "<!-- xxx tabs xxx -->" in result:
tab_logic = True
## Inlining all links
result = self.inline_references(result, regex_skip_sections_start,
regex_skip_sections_end)
else:
tab_logic = False
title = manifest_json.get("name", "").lower()
if title not in [
k for k, v in self.integration_mutations.items()
if v.get("action") == "merge"
]:
result = re.sub(self.regex_h1, "", result, 1)
result = re.sub(self.regex_tabs_open, "{{< tabs >}}", result, 0)
result = re.sub(self.regex_tabs_close, "{{< /tabs >}}", result, 0)
result = re.sub(self.regex_tab_open, "{{% tab", result, 0)
result = re.sub(self.regex_tab_close, "{{% /tab %}}", result, 0)
result = re.sub(self.regex_tab_end, " %}}", result, 0)
result = re.sub(self.regex_partial_open, "", result, 0)
result = re.sub(self.regex_partial_close, "", result, 0)
if metrics_exist:
result = re.sub(
self.regex_metrics,
r'\1{{< get-metrics-from-git "%s" >}}\n\3\4' % format(title),
result,
0,
)
if service_check_exist:
result = re.sub(
self.regex_service_check,
r'\1{{< get-service-checks-from-git "%s" >}}\n\3\4' %
format(title),
result,
0,
)
# if __init__.py exists lets grab the integration id
integration_id = manifest_json.get("integration_id", "") or ""
initpy = "{0}{1}{2}".format(dirname(file_name), sep, "__init__.py")
if exists(initpy):
with open(initpy) as f:
# look for ID = "integration-name" and extract
matches = re.search(
"^ID\s*=\s*(?:\'|\")([A-Z-a-z-_0-9]+)(?:\'|\")$", f.read(),
re.MULTILINE)
if matches:
integration_id = matches.group(1)
# if __about__.py exists lets grab the integration version
integration_version = manifest_json.get("integration_version",
"") or ""
integration_name = basename(dirname(file_name))
aboutpy = "{0}{1}{2}{3}{4}{5}{6}".format(dirname(file_name), sep,
"datadog_checks", sep,
integration_name, sep,
"__about__.py")
if exists(aboutpy):
with open(aboutpy) as f:
# look for version = "integration-version" and extract
matches = re.search(
"^__version__\s*=\s*(?:\'|\")([0-9.]+)(?:\'|\")$",
f.read(), re.MULTILINE)
if matches:
integration_version = matches.group(1)
if not exist_already and no_integration_issue:
# lets only write out file.md if its going to be public
if manifest_json.get("is_public", False):
out_name = self.content_integrations_dir + new_file_name
# lets make relative app links to integrations tile absolute
regex = r"(?<!https://app.datadoghq.com)(/account/settings#integrations[^.)\s]*)"
regex_result = re.sub(regex, "https://app.datadoghq.com\\1",
result, 0, re.MULTILINE)
if regex_result:
result = regex_result
# if the same integration exists in multiple locations try name md file differently
# integration_id.md -> name.md -> original_collision_name.md
if exist_collision:
f_name = integration_id.replace(
'-', '_') or manifest_json.get("name",
"") or new_file_name
manifest_json["name"] = f_name
f_name = f_name if f_name.endswith(
'.md') else f_name + ".md"
out_name = self.content_integrations_dir + f_name
print(
"\x1b[33mWARNING\x1b[0m: Collision, duplicate integration {} trying as {}"
.format(new_file_name, f_name))
result = self.add_integration_frontmatter(
f_name, result, dependencies, integration_id,
integration_version, manifest_json)
else:
result = self.add_integration_frontmatter(
new_file_name, result, dependencies, integration_id,
integration_version)
with open(
out_name,
"w",
) as out:
out.write(result)
## Reformating all links now that all processing is done
if tab_logic:
final_text = format_link_file(out_name,
regex_skip_sections_start,
regex_skip_sections_end)
with open(out_name, 'w') as final_file:
final_file.write(final_text)
