def get_multiple_systems_html_report(orig_sents, sys_sents_list, refs_sents, system_names, test_set, lowercase, tokenizer, metrics):
doc = Doc()
doc.asis('<!doctype html>')
with doc.tag('html', lang='en'):
doc.asis(get_head_html())
with doc.tag('body', klass='container-fluid m-2 mb-5'):
doc.line('h1', 'EASSE report', klass='mt-4')
with doc.tag('a', klass='btn btn-link', href='https://forms.gle/J8KVkJsqYe8GvYW46'):
doc.text('Any feedback welcome!')
doc.stag('hr')
doc.line('h2', 'Test set')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.asis(get_test_set_description_html(
test_set=test_set,
orig_sents=orig_sents,
refs_sents=refs_sents,
))
doc.line('h2', 'Scores')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.line('h3', 'System vs. Reference')
doc.stag('hr')
doc.asis(get_score_table_html_multiple_systems(orig_sents, sys_sents_list, refs_sents, system_names, lowercase, tokenizer, metrics))
doc.line('h2', 'Qualitative evaluation')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.asis(get_multiple_systems_qualitative_examples_html(orig_sents, sys_sents_list, refs_sents, system_names))
return indent(doc.getvalue())
def get_test_set_description_html(test_set, orig_sents, refs_sents):
doc = Doc()
test_set = test_set.capitalize()
doc.line('h4', test_set)
orig_sents = np.array(orig_sents)
refs_sents = np.array(refs_sents)
df = pd.DataFrame()
df.loc[test_set, '# of samples'] = str(len(orig_sents))
df.loc[test_set, '# of references'] = str(len(refs_sents))
df.loc[test_set, 'Words / source'] = np.average(np.vectorize(count_words)(orig_sents))
df.loc[test_set, 'Words / reference'] = np.average(np.vectorize(count_words)(refs_sents.flatten()))
def modified_count_sentences(sent):
return max(count_sentences(sent), 1)
orig_sent_counts = np.vectorize(modified_count_sentences)(orig_sents)
expanded_orig_sent_counts = np.expand_dims(orig_sent_counts, 0).repeat(len(refs_sents), axis=0)
refs_sent_counts = np.vectorize(modified_count_sentences)(refs_sents)
ratio = np.average((expanded_orig_sent_counts == 1) & (refs_sent_counts == 1))
df.loc[test_set, '1-to-1 alignments*'] = f'{ratio*100:.1f}%'
ratio = np.average((expanded_orig_sent_counts == 1) & (refs_sent_counts > 1))
df.loc[test_set, '1-to-N alignments*'] = f'{ratio*100:.1f}%'
ratio = np.average((expanded_orig_sent_counts > 1) & (refs_sent_counts > 1))
df.loc[test_set, 'N-to-N alignments*'] = f'{ratio*100:.1f}%'
ratio = np.average((expanded_orig_sent_counts > 1) & (refs_sent_counts == 1))
df.loc[test_set, 'N-to-1 alignments*'] = f'{ratio*100:.1f}%'
doc.asis(get_table_html_from_dataframe(df.round(2)))
doc.line('p', klass='text-muted', text_content='* Alignment detection is not 100% accurate')
return doc.getvalue()
def get_score_table_html_multiple_systems(orig_sents, sys_sents_list,
refs_sents, system_names, lowercase,
tokenizer, metrics):
def truncate(sentence):
# Take first 80% words
words = to_words(sentence)
return ' '.join(words[:int(len(words) * 0.8)]) + '.'
doc = Doc()
# We don't want changes to propagate out of this scope
sys_sents_list = sys_sents_list.copy()
system_names = system_names.copy()
# Add the identity baseline
sys_sents_list.append(orig_sents)
system_names.append('Identity baseline')
# Add the truncate baseline
sys_sents_list.append([truncate(sentence) for sentence in orig_sents])
system_names.append('Truncate baseline')
# Evaluate systems
sys_scores_list = [
get_all_scores(orig_sents,
sys_sents,
refs_sents,
lowercase=lowercase,
tokenizer=tokenizer,
metrics=metrics) for sys_sents in sys_sents_list
]
rows = [sys_scores.values() for sys_scores in sys_scores_list]
if len(refs_sents) > 1:
# Evaluate the first reference against all the others (the second reference is duplicated to have the same number of reference as for systems).
# TODO: Ideally the system and references should be evaluated with exactly the same number of references.
ref_scores = get_all_scores(
orig_sents,
refs_sents[0],
[refs_sents[1]] + refs_sents[1:],
lowercase=lowercase,
tokenizer=tokenizer,
metrics=metrics,
)
assert all([
sys_scores.keys() == ref_scores.keys()
for sys_scores in sys_scores_list
])
rows.append(ref_scores.values())
system_names.append('Reference*')
doc.asis(
get_table_html(
header=list(sys_scores_list[0].keys()),
rows=rows,
row_names=system_names,
))
doc.line(
'p',
klass='text-muted',
text_content=
('* The Reference row represents one of the references (picked randomly) evaluated'
' against the others.'),
)
return doc.getvalue()
def _make_toc(entries, max_index_len):
doc, tag, text = Doc().tagtext()
for i, entry in enumerate(filter(_is_single_line_md_heading, entries)):
line = entry.content.strip()
level = re.search(r"[^#]", line).start()
with tag("div", klass=f"indexindent_{level}"):
link_text = line[level:] if len(line) < max_index_len \
or max_index_len == 0 else\
(line[level: max_index_len - 3] + "...")
doc.line("a", link_text, href=f"#{i}")
return indent(doc.getvalue())
def get_header(self, entry):
doc = Doc()
line = entry.content.strip()
level = _get_h_indention_lvl(line)
text = line[level:]
doc.line(f'h{level}',
"\u2798" + text,
id=str(self.counter),
klass='hdr')
self.counter += 1
return doc.getvalue()
def generateFile():
"""Generates final php page."""
begin = (
"<?php\ndefine('PROJECT_DIR', realpath('../'));\ndefine('LOCALE_DIR', PROJECT_DIR . '\\Locale');\ndefine('DEFAULT_LOCALE', 'en');\n\nrequire('../GetText/gettext.inc');\n\n$encoding = 'UTF-8';\n\n$locale = (isset($_COOKIE['lang'])) ? $_COOKIE['lang'] : DEFAULT_LOCALE;\n\nT_setlocale(LC_MESSAGES, $locale);\n\nT_bindtextdomain($locale, LOCALE_DIR);\nT_bind_textdomain_codeset($locale, $encoding);\nT_textdomain($locale);\n\nrequire('../postgresql.php');\n$number = basename(__FILE__, '.php');\n$title = '';\n$stmt = getTests('"
+ str(subject) +
"');\nwhile ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {\n if ($row['id'] == $number) {\n $title = $row['name'];\n break;\n }\n}\nrequire('../Templates/head.php');\n?>\n"
)
end = "\n<?php\nrequire('../Templates/foot.php');\n?>"
# pylint: disable=unused-variable
doc, tag, text, line = Doc().ttl()
with tag("form",
action="../Pages/checker",
method="post",
autocomplete="off"):
doc.line("input", "", type="hidden", name="Lang", value=str(subject))
doc.line("input",
"",
type="hidden",
name="Name",
value=str(Make.name.text))
num = 0
for i in questions:
with tag("fieldset"):
doc.line(
"input",
"",
type="hidden",
name="Count[]",
value=str(len(questions[i])),
)
doc.line("h2", i)
with tag("ol"):
for j in range(len(questions[i])):
with tag("li"):
doc.line(
"input",
questions[i][j],
type="checkbox",
name=str(num) + "[]",
value=str(j),
)
num += 1
doc.stag("input", type="submit", text="send")
global php_file
php_file = begin + indent(doc.getvalue(), indentation=" ",
newline="\r") + end
def get_qualitative_examples_html(orig_sents, sys_sents, refs_sents):
title_key_print = [
('Randomly sampled simplifications',
lambda c, s, refs: 0,
lambda value: ''),
('Best simplifications according to SARI',
lambda c, s, refs: -corpus_sari([c], [s], [refs]),
lambda value: f'SARI={-value:.2f}'),
('Worst simplifications according to SARI',
lambda c, s, refs: corpus_sari([c], [s], [refs]),
lambda value: f'SARI={value:.2f}'),
('Simplifications with the most compression',
lambda c, s, refs: get_compression_ratio(c, s),
lambda value: f'compression_ratio={value:.2f}'),
('Simplifications with a high amount of paraphrasing',
lambda c, s, refs: get_levenshtein_similarity(c, s) / get_compression_ratio(c, s),
lambda value: f'levenshtein_similarity={value:.2f}'),
('Simplifications with the most sentence splits (if any)',
lambda c, s, refs: -(count_sentences(s) - count_sentences(c)),
lambda value: f'#sentence_splits={-value:.2f}'),
]
def get_one_sample_html(orig_sent, sys_sent, ref_sents, sort_key, print_func):
orig_sent, sys_sent, *ref_sents = [html.escape(sent) for sent in [orig_sent, sys_sent, *ref_sents]]
doc = Doc()
with doc.tag('div', klass='mb-2 p-1'):
# Sort key
with doc.tag('div', klass='text-muted small'):
doc.asis(print_func(sort_key(orig_sent, sys_sent, ref_sents)))
with doc.tag('div', klass='ml-2'):
orig_sent_bold, sys_sent_bold = make_differing_words_bold(orig_sent, sys_sent, make_text_bold_html)
# Source
with doc.tag('div'):
doc.asis(orig_sent_bold)
# Prediction
with doc.tag('div'):
doc.asis(sys_sent_bold)
# References
collapse_id = get_random_html_id()
with doc.tag('div', klass='position-relative'):
with doc.tag('a', ('data-toggle', 'collapse'), ('href', f'#{collapse_id}'),
klass='stretched-link small'):
doc.text('References')
with doc.tag('div', klass='collapse', id=collapse_id):
for ref_sent in refs:
_, ref_sent_bold = make_differing_words_bold(orig_sent, ref_sent, make_text_bold_html)
with doc.tag('div', klass='text-muted'):
doc.asis(ref_sent_bold)
return doc.getvalue()
doc = Doc()
for title, sort_key, print_func in title_key_print:
with doc.tag('div', klass='container-fluid mt-4 p-2 border'):
collapse_id = get_random_html_id()
with doc.tag('a', ('data-toggle', 'collapse'), ('href', f'#{collapse_id}')):
doc.line('h3', klass='m-2', text_content=title)
# Now lets print the examples
sample_generator = sorted(
zip(orig_sents, sys_sents, zip(*refs_sents)),
key=lambda args: sort_key(*args),
)
# Samples displayed by default
with doc.tag('div', klass='collapse', id=collapse_id):
n_samples = 50
for i, (orig_sent, sys_sent, refs) in enumerate(sample_generator):
if i >= n_samples:
break
doc.asis(get_one_sample_html(orig_sent, sys_sent, refs, sort_key, print_func))
return doc.getvalue()
def make_text_bold_html(text):
doc = Doc()
doc.line('strong', text)
return doc.getvalue()
def get_multiple_systems_qualitative_examples_html(orig_sents, sys_sents_list, refs_sents, system_names):
def get_relative_sari(orig_sent, sys_sents, refs_sents, system_idx):
saris = [corpus_sari([orig_sent], [sys_sent], refs_sents) for sys_sent in sys_sents]
return saris[system_idx] / np.average(saris)
title_key_print = [
('Randomly sampled simplifications',
lambda c, s, refs: 0,
lambda value: ''),
] + [
(f'Worst relative simplifications (SARI) for {system_names[i]}',
lambda c, sys_sents, refs_sents: get_relative_sari(c, sys_sents, refs_sents, system_idx=i),
lambda value: f'Relative SARI={value:.2f}')
for i in range(len(system_names))
]
def get_one_sample_html(orig_sent, sys_sents, ref_sents, system_names, sort_key, print_func):
def get_one_sentence_html(sentence, system_name):
doc = Doc()
with doc.tag('div', klass='row'):
with doc.tag('div', klass='col-2'):
doc.text(system_name)
with doc.tag('div', klass='col'):
doc.asis(sentence)
return doc.getvalue()
doc = Doc()
with doc.tag('div', klass='mb-2 p-1'):
# Sort key
with doc.tag('div', klass='text-muted small'):
doc.asis(print_func(sort_key(orig_sent, sys_sents, ref_sents)))
with doc.tag('div', klass='ml-2'):
# Source
with doc.tag('div'):
doc.asis(get_one_sentence_html(orig_sent, 'Original'))
# Predictions
for sys_sent, system_name in zip(sys_sents, system_names):
_, sys_sent_bold = make_differing_words_bold(orig_sent, sys_sent, make_text_bold_html)
doc.asis(get_one_sentence_html(sys_sent_bold, system_name))
# References
collapse_id = get_random_html_id()
with doc.tag('div', klass='position-relative'):
with doc.tag('a', ('data-toggle', 'collapse'), ('href', f'#{collapse_id}'),
klass='stretched-link small'):
doc.text('References')
with doc.tag('div', klass='collapse', id=collapse_id):
for ref_sent in refs:
_, ref_sent_bold = make_differing_words_bold(orig_sent, ref_sent, make_text_bold_html)
with doc.tag('div', klass='text-muted'):
doc.asis(ref_sent_bold)
return doc.getvalue()
doc = Doc()
for title, sort_key, print_func in title_key_print:
with doc.tag('div', klass='container-fluid mt-4 p-2 border'):
collapse_id = get_random_html_id()
with doc.tag('a', ('data-toggle', 'collapse'), ('href', f'#{collapse_id}')):
doc.line('h3', klass='m-2', text_content=title)
# Now lets print the examples
sample_generator = sorted(
zip(orig_sents, zip(*sys_sents_list), zip(*refs_sents)),
key=lambda args: sort_key(*args),
)
# Samples displayed by default
with doc.tag('div', klass='collapse', id=collapse_id):
n_samples = 50
for i, (orig_sent, sys_sents, refs) in enumerate(sample_generator):
if i >= n_samples:
break
doc.asis(get_one_sample_html(orig_sent, sys_sents, refs, system_names, sort_key, print_func))
return doc.getvalue()
def get_html_report(orig_sents: List[str], sys_sents: List[str], refs_sents: List[List[str]], test_set: str,
lowercase: bool = False, tokenizer: str = '13a', metrics: List[str] = DEFAULT_METRICS):
doc = Doc()
doc.asis('<!doctype html>')
with doc.tag('html', lang='en'):
doc.asis(get_head_html())
with doc.tag('body', klass='container-fluid m-2 mb-5'):
doc.line('h1', 'EASSE report', klass='mt-4')
with doc.tag('a', klass='btn btn-link', href='https://forms.gle/J8KVkJsqYe8GvYW46'):
doc.text('Any feedback welcome!')
doc.stag('hr')
doc.line('h2', 'Test set')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.asis(get_test_set_description_html(
test_set=test_set,
orig_sents=orig_sents,
refs_sents=refs_sents,
))
doc.line('h2', 'Scores')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.line('h3', 'System vs. Reference')
doc.stag('hr')
doc.asis(get_score_table_html_single_system(orig_sents, sys_sents, refs_sents, lowercase, tokenizer, metrics))
doc.line('h3', 'By sentence length (characters)')
doc.stag('hr')
doc.asis(get_scores_by_length_html(orig_sents, sys_sents, refs_sents))
doc.line('h2', 'Plots')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.asis(get_plots_html(orig_sents, sys_sents, refs_sents[0]))
doc.line('h2', 'Qualitative evaluation')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.asis(get_qualitative_examples_html(orig_sents, sys_sents, refs_sents))
return indent(doc.getvalue())
class IDontSpeaksSSLReporterHTML(IDontSpeaksSSLReporter):
pass
def run(self):
self.doc = Doc()
copy_js_css(self.report_folder)
cprint("[-] Generating the HTML report...", 'blue')
self.doc.asis('<!DOCTYPE html>')
self.create_header()
self.create_body()
self.write_report()
cprint("[+] Report generated.", 'green')
cprint(
"[+] All results can be found in {}/report.html.".format(
self.report_folder), 'green')
def create_body(self):
with self.doc.tag('body'):
with self.doc.tag('div'):
self.doc.attr(klass='container')
self.doc.line('h1', 'IDontSpeakSSL Report')
self.add_summary_section(
"Findings Summary",
self.results['analyzer results']["summary"])
self.add_hosts_section(
"Findings per Host",
self.results['analyzer results']["hosts"])
def add_summary_section(self, section, subsections):
self.doc.line('h2', section)
with self.doc.tag('div'):
self.doc.attr(klass='panel-group')
for subsection, instances in subsections.items():
if instances:
with self.doc.tag('div', klass='panel panel-default'):
with self.doc.tag('div', klass='panel-heading'):
with self.doc.tag('div', klass='panel-title'):
with self.doc.tag('h4', klass='panel-title'):
with self.doc.tag(
'a', ("data-toggle", "collapse"),
("href", "#{}".format(
capitalize_sentence(subsection)))):
self.doc.text("{}".format(
capitalize_sentence(subsection)))
with self.doc.tag('div'):
self.doc.attr(klass='panel-collapse collapse',
id='{}'.format(
capitalize_sentence(subsection)))
with self.doc.tag('div', klass='panel-body'):
if (subsection != "vulnerabilities"):
with self.doc.tag('ul'):
for instance in instances:
self.doc.line(
'li',
"Host: {}, Port: {}".format(
instance['host'],
instance['port']))
else:
with self.doc.tag('ul'):
for vulnerability_name, vulnerability_instance in instances.items(
):
self.doc.line(
'li', "{}".format(
vulnerability_name))
with self.doc.tag('ul'):
for location in vulnerability_instance:
self.doc.line(
'li',
"Host: {}, Port: {}".
format(
location['host'],
location['port']))
def add_hosts_section(self, section, hosts):
self.doc.line('h2', section)
with self.doc.tag('div'):
self.doc.attr(klass='panel-group')
for host, findings in hosts.items():
if findings:
with self.doc.tag('div', klass='panel panel-default'):
with self.doc.tag('div', klass='panel-heading'):
with self.doc.tag('div', klass='panel-title'):
with self.doc.tag('h4', klass='panel-title'):
with self.doc.tag(
'a', ("data-toggle", "collapse"),
("href", "#{}".format(
host.replace(".", "_")))):
instance_host, instance_port = extract_host_and_port(
host)
self.doc.text(
"Host: {}, Port: {}".format(
instance_host, instance_port))
with self.doc.tag('div'):
self.doc.attr(klass='panel-collapse collapse',
id='{}'.format(host.replace(".", "_")))
with self.doc.tag('div', klass='panel-body'):
for finding_type, finding_instance in findings.items(
):
self.doc.text("{}".format(finding_type))
if (isinstance(finding_instance, list)):
with self.doc.tag('ul'):
for instance in finding_instance:
self.doc.line(
'li', "{}".format(instance))
elif (isinstance(finding_instance, bool)):
if (finding_instance):
self.doc.text(": Vulnerable")
def create_header(self):
with self.doc.tag('head'):
self.doc.line('title', 'IDontSpeakSSL Report')
self.doc.stag('link', ("rel", "stylesheet"),
("href", "html/css/bootstrap.min.css"))
self.doc.line('script', '', src="html/js/jquery.min.js")
self.doc.line('script', '', src="html/js/bootstrap.min.js")
def write_report(self):
with open("{}/report.html".format(self.report_folder), 'w') as report:
report.write(indent(self.doc.getvalue()))
def save(self, path):
print("Adding {} new changelogs.".format(self.counter))
self.save_yaml(path)
doc, tag, text = Doc().tagtext()
doc.asis('<!DOCTYPE html>')
doc.asis("<!-- AUTOMATICALLY GENERATED - NOT TO BE MODIFIED MANUALLY -->")
with tag('html'):
with tag("head"):
doc.stag("meta", charset="utf-8")
doc.line("title", "GJAR IoT Changelog")
with open(os.path.join(path, "changelog.css"), "r") as f:
doc.line("style", f.read())
with tag('body'):
doc.line("h1", "GJAR IoT Changelog")
doc.line("i", "Last generated: {}".format(date))
for d in sorted(self.logs.keys()):
with tag("div"):
doc.line("h2", d)
with tag("ul"):
for item in self.logs[d]:
with tag("li"):
doc.line("h3", item.author)
with tag("ul"):
for ch in item.changes:
doc.line("li", ch.value, klass="type-" + ch.type)
if not args.no_changes:
with open(os.path.join(path, "changelog.html"), "w") as f:
f.write(indent(doc.getvalue()))
verbose("Saving changelog.html")
else:
verbose("Would save changelog.html")
class HelpHandler(RequestHandler):
log = get_logger('HelpHandler')
# noinspection PyAttributeOutsideInit
def initialize(self) -> None:
self.doc = Doc()
self.nesting_level = 0
self.f_nesting = 2 # == <html><body>
self.footer = Doc()
self.footnote_counter = 1
if self.application.settings.get('debug'):
def nl(main=True):
doc = self.doc if main else self.footer
doc.nl()
nest = self.nesting_level if main else self.f_nesting
doc.text(' ' * nest)
def go_in(main=True):
doc = self.doc if main else self.footer
doc.nl()
if main:
self.nesting_level += 1
nest = self.nesting_level
else:
self.f_nesting += 1
nest = self.f_nesting
doc.text(' ' * nest)
def go_out(main=True):
doc = self.doc if main else self.footer
if main:
self.nesting_level -= 1
nest = self.nesting_level
else:
self.f_nesting -= 1
nest = self.f_nesting
doc.nl()
doc.text(' ' * nest)
def line(tag_name, text_content, *args, **kwargs):
self.doc.line(tag_name, text_content, *args, **kwargs)
nl()
def f_line(tag_name, text_content, *args, **kwargs):
self.footer.line(tag_name, text_content, *args, **kwargs)
nl(False)
def stag(tag_name, *args, **kwargs):
self.doc.stag(tag_name, *args, **kwargs)
nl()
else:
def no_op(*_a, **_ka):
pass
nl = go_in = go_out = no_op
line = self.doc.line
f_line = self.footer.line
stag = self.doc.stag
self.nl = nl
self.line = line
self.f_line = f_line
self.stag = stag
self.go_in = go_in
self.go_out = go_out
# Prepare footer
self.nl(False)
self.footer.stag('hr')
self.nl(False)
def get_fc(self) -> str:
temp = self.footnote_counter
self.footnote_counter += 1
return str(temp)
def footnote(self, fn_text: str) -> None:
c = self.get_fc()
sup_id = 'ref' + c
note_id = 'fn' + c
tag = self.doc.tag
text = self.doc.text
line = self.line
with tag('sup'):
self.go_in()
line('a', href='#' + note_id, id=sup_id, text_content=c)
self.go_out()
self.nl()
del tag
del text
del line
f_tag = self.footer.tag
f_text = self.footer.asis
f_line = self.f_line
with f_tag('p', id=note_id):
self.go_in(False)
with f_tag('small'):
self.go_in(False)
f_text(c + '. ')
f_line('a',
href='#' + sup_id,
title='Go back',
text_content='^')
f_text(' ' + fn_text)
self.go_out(False)
self.nl(False)
self.go_out(False)
self.nl(False)
def head(self):
tag, line, stag = self.doc.tag, self.line, self.stag
with tag('head'):
self.go_in()
stag(
'link',
rel="stylesheet",
href=
"https://cdnjs.cloudflare.com/ajax/libs/mini.css/3.0.1/mini-default.min.css"
)
line('title', 'General purpose VNF configuration server')
request = self.request # type: HTTPServerRequest
uri = request.protocol + '://' + request.host + '/favicon.png'
stag('link', rel="icon", href=uri, type="image.png")
self.go_out()
self.nl()
def description(self):
tag, text, line, fn = self.doc.tag, self.doc.text, self.doc.line, self.footnote
line('h3', id='description', text_content='Description')
with tag('p'):
self.go_in()
text("This is a general-purpose HTTP server serving VNF")
fn("<a href=https://www.etsi.org/deliver/etsi_gs/nfv-ifa/"
"001_099/011/03.01.01_60/gs_nfv-ifa011v030101p.pdf> "
"https://www.etsi.org/deliver/etsi_gs/nfv-ifa/"
"001_099/011/03.01.01_60/gs_nfv-ifa011v030101p.pdf</a>")
text(" configuration information. "
"It offers an ETSI IFA 008 compliant")
fn("<a href=https://www.etsi.org/deliver/etsi_gs/NFV-IFA/"
"001_099/008/03.01.01_60/gs_nfv-ifa008v030101p.pdf> "
"https://www.etsi.org/deliver/etsi_gs/NFV-IFA/"
"001_099/008/03.01.01_60/gs_nfv-ifa008v030101p.pdf</a>"
" - paragraph 6.2.3")
text(" NBI.")
self.go_out()
self.nl()
def nbi(self):
tag, text, line = self.doc.tag, self.doc.text, self.line
line('h3', id='nbi', text_content='North bound interface')
with tag('p'):
self.go_in()
text('To configure this VNF, perform an HTTP PATCH request to ')
request = self.request # type: HTTPServerRequest
uri = request.protocol + '://' + request.host + '/vnfconfig/v1/configuration'
line('a', uri, href=uri)
text('.')
self.go_out()
self.nl()
line('p', 'The format is as follows:')
self.format_example()
def format_example(self):
ex = """{
"vnfConfigurationData": {
"vnfSpecificData": [
{
"key": "vnf.hss.vdu.hss_vdu.hostname",
"value": "some-value",
},
{
"key": "uservnf.hss.vdu.hss.domainname",
"value": "some-other-value"
}
]
}
}"""
ex = ex.replace('\n', '<br>')
with self.doc.tag('pre'):
self.doc.asis(ex)
self.nl()
def config(self):
tag, text, line, fn, stag = self.doc.tag, self.doc.text, self.doc.line, self.footnote, self.stag
line('h3', id='config', text_content='Configuration file')
self.nl()
line(
'p', "The configuration file (by default the 'settings.json' file "
"in the same current working directory) is a json file "
"following this format: ")
self.nl()
self.config_example()
line(
'p',
"where 'port' is the TCP port on on which the server should bind itself, "
"'script' is the command which should be run to enforce the actual "
"configuration, taking as arguments the parameter values, and "
"'params' are the name of the parameters which are required for configuration."
)
self.nl()
line(
'p',
"For example, if there are two parameters configured with values '1' and 'stop' "
"respectively, the command called will be ")
self.nl()
with tag('pre'):
text('<script> 1 stop')
self.go_out()
self.nl()
def config_example(self):
ex = """{
"port": 8080,
"script": "/opt/VnfServer/hss.sh",
"params": [
"vnf.hss.vdu.hss_vdu.hostname",
"uservnf.hss.vdu.hss.domainname"
]
}"""
ex = ex.replace('\n', '<br>')
with self.doc.tag('pre'):
self.doc.asis(ex)
self.nl()
def body(self):
doc, tag, text, line, fn = self.doc, self.doc.tag, self.doc.text, self.doc.line, self.footnote
with tag('body'):
self.go_in()
line('h1', 'General purpose VNF indicator server')
self.description()
self.nbi()
self.config()
self.doc.asis(self.footer.getvalue())
self.nl()
self.go_out()
self.nl()
def get(self) -> None:
self.log.debug('Received call')
doc, tag, text, line, fn = self.doc, self.doc.tag, self.doc.text, self.doc.line, self.footnote
with tag('html'):
self.go_in()
self.head()
self.body()
self.go_out()
self.nl()
self.write(doc.getvalue())
def get_qualitative_html_examples(orig_sents, sys_sents, refs_sents):
title_key_print = [
('Randomly sampled simplifications',
lambda c, s, refs: 0,
lambda value: ''),
('Best simplifications according to SARI',
lambda c, s, refs: -corpus_sari([c], [s], [refs]),
lambda value: f'SARI={-value:.2f}'),
('Worst simplifications according to SARI',
lambda c, s, refs: corpus_sari([c], [s], [refs]),
lambda value: f'SARI={value:.2f}'),
('Simplifications with only one differing word',
lambda c, s, refs: -(count_words(c) == count_words(s) == len(get_lcs(to_words(c), to_words(s))) + 1),
lambda value: ''),
('Simplifications with the most compression',
lambda c, s, refs: get_compression_ratio(c, s),
lambda value: f'compression_ratio={value:.2f}'),
('Simplifications that are longer than the source',
lambda c, s, refs: -get_compression_ratio(c, s),
lambda value: f'compression_ratio={-value:.2f}'),
('Simplifications that paraphrase the source',
lambda c, s, refs: get_levenshtein_similarity(c, s) / get_compression_ratio(c, s),
lambda value: f'levenshtein_similarity={value:.2f}'),
('Simplifications that are the most similar to the source (excluding exact matches)',
lambda c, s, refs: -get_levenshtein_similarity(c, s) * int(c != s),
lambda value: f'levenshtein_similarity={-value:.2f}'),
('Simplifications with the most sentence splits (if there are any)',
lambda c, s, refs: -count_sentence_splits(c, s),
lambda value: f'nb_sentences_ratio={-value:.2f}'),
]
def get_one_sample_html(orig_sent, sys_sent, ref_sents, sort_key, print_func):
doc = Doc()
with doc.tag('div', klass='mb-2 p-1'):
# Sort key
with doc.tag('div', klass='text-muted small'):
doc.asis(print_func(sort_key(orig_sent, sys_sent, ref_sents)))
with doc.tag('div', klass='ml-2'):
orig_sent_bold, sys_sent_bold = make_differing_words_bold(orig_sent, sys_sent, make_text_bold_html)
# Source
with doc.tag('div'):
doc.asis(orig_sent_bold)
# Prediction
with doc.tag('div'):
doc.asis(sys_sent_bold)
# References
collapse_id = get_random_html_id()
with doc.tag('div', klass='position-relative'):
with doc.tag('a', ('data-toggle', 'collapse'), ('href', f'#{collapse_id}'),
klass='stretched-link small'):
doc.text('References')
with doc.tag('div', klass='collapse', id=collapse_id):
for ref_sent in refs:
_, ref_sent_bold = make_differing_words_bold(orig_sent, ref_sent, make_text_bold_html)
with doc.tag('div', klass='text-muted'):
doc.asis(ref_sent_bold)
return doc.getvalue()
doc = Doc()
for title, sort_key, print_func in title_key_print:
# stretched-link needs position-relative
with doc.tag('div', klass='container-fluid mt-4 p-2 position-relative border'):
doc.line('h3', klass='m-2', text_content=title)
# Make whole div clickable to collapse / uncollapse examples
collapse_id = get_random_html_id()
with doc.tag('a', ('data-toggle', 'collapse'), ('href', f'#{collapse_id}'), klass='stretched-link'):
pass # doc.stag and doc.line don't seem to work with stretched-link
# Now lets print the examples
sample_generator = sorted(
zip(orig_sents, sys_sents, zip(*refs_sents)),
key=lambda args: sort_key(*args),
)
# Samples displayed by default
with doc.tag('div', klass='collapse show', id=collapse_id):
n_samples = 10
for i, (orig_sent, sys_sent, refs) in enumerate(sample_generator):
if i >= n_samples:
break
doc.asis(get_one_sample_html(orig_sent, sys_sent, refs, sort_key, print_func))
return doc.getvalue()
def get_html_report(orig_sents: List[str], sys_sents: List[str], refs_sents: List[List[str]], test_set_name,
lowercase: bool = False, tokenizer: str = '13a', metrics: List[str] = DEFAULT_METRICS):
sns.set_style('darkgrid')
doc = Doc()
doc.asis('<!doctype html>')
with doc.tag('html', lang='en'):
doc.asis(get_head_html())
with doc.tag('body', klass='container-fluid m-2 mb-5'):
doc.line('h1', 'EASSE report', klass='mt-4')
with doc.tag('a', klass='btn btn-link', href='https://forms.gle/J8KVkJsqYe8GvYW46'):
doc.text('Any feedback welcome!')
doc.stag('hr')
doc.line('h2', 'Test set')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.asis(get_test_set_description_html(
test_set_name=test_set_name,
orig_sents=orig_sents,
refs_sents=refs_sents,
))
doc.line('h2', 'Scores')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.line('h3', 'System vs. Reference')
doc.stag('hr')
sys_scores = get_all_scores(orig_sents, sys_sents, refs_sents,
lowercase=lowercase, tokenizer=tokenizer, metrics=metrics)
# TODO: The system and references should be evaluated with the same number of references
ref_scores = get_all_scores(orig_sents, refs_sents[0], refs_sents[1:],
lowercase=lowercase, tokenizer=tokenizer, metrics=metrics)
assert sys_scores.keys() == ref_scores.keys()
doc.asis(get_table_html(
header=list(sys_scores.keys()),
rows=[sys_scores.values(), ref_scores.values()],
row_names=['System output', 'Reference*'],
))
doc.line(
'p',
klass='text-muted',
text_content=('* The Reference row represents one of the references (picked randomly) evaluated'
' against the others.'),
)
doc.line('h3', 'By sentence length (characters)')
doc.stag('hr')
doc.asis(get_scores_by_length_html(orig_sents, sys_sents, refs_sents))
doc.line('h2', 'Plots')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.asis(get_plots_html(orig_sents, sys_sents, refs_sents[0]))
doc.line('h2', 'Qualitative evaluation')
doc.stag('hr')
with doc.tag('div', klass='container-fluid'):
doc.asis(get_qualitative_html_examples(orig_sents, sys_sents, refs_sents))
return indent(doc.getvalue())
class HelpHandler(RequestHandler):
log = get_logger('HelpHandler')
# noinspection PyAttributeOutsideInit
def initialize(self) -> None:
self.doc = Doc()
self.nesting_level = 0
self.f_nesting = 0
self.footer = Doc()
self.footnote_counter = 1
self.footer.stag('hr')
self.footer.nl()
if self.application.settings.get('debug'):
def nl(main=True):
doc = self.doc if main else self.footer
doc.nl()
nest = self.nesting_level if main else self.f_nesting
doc.text(' ' * nest)
def go_in(main=True):
doc = self.doc if main else self.footer
doc.nl()
if main:
self.nesting_level += 1
nest = self.nesting_level
else:
self.f_nesting += 1
nest = self.f_nesting
doc.text(' ' * nest)
def go_out(main=True):
doc = self.doc if main else self.footer
if main:
self.nesting_level -= 1
nest = self.nesting_level
else:
self.f_nesting -= 1
nest = self.f_nesting
doc.nl()
doc.text(' ' * nest)
def line(tag_name, text_content, *args, **kwargs):
self.doc.line(tag_name, text_content, *args, **kwargs)
nl()
def f_line(tag_name, text_content, *args, **kwargs):
self.footer.line(tag_name, text_content, *args, **kwargs)
nl(False)
def stag(tag_name, *args, **kwargs):
self.doc.stag(tag_name, *args, **kwargs)
nl()
else:
def no_op(*_a, **_ka):
pass
nl = go_in = go_out = no_op
line = self.doc.line
f_line = self.footer.line
stag = self.doc.stag
self.nl = nl
self.line = line
self.f_line = f_line
self.stag = stag
self.go_in = go_in
self.go_out = go_out
def get_fc(self) -> str:
temp = self.footnote_counter
self.footnote_counter += 1
return str(temp)
def footnote(self, fn_text: str) -> None:
c = self.get_fc()
sup_id = 'ref' + c
note_id = 'fn' + c
tag = self.doc.tag
text = self.doc.text
line = self.line
with tag('sup'):
self.go_in()
line('a', href='#' + note_id, id=sup_id, text_content=c)
self.go_out()
self.nl()
del tag
del text
del line
f_tag = self.footer.tag
f_text = self.footer.asis
f_line = self.f_line
with f_tag('p', id=note_id):
self.go_in(False)
with f_tag('small'):
self.go_in(False)
f_text(c + '. ')
f_line('a',
href='#' + sup_id,
title='Go back',
text_content='^')
f_text(' ' + fn_text)
self.go_out(False)
self.nl(False)
self.go_out(False)
self.nl(False)
def head(self):
tag, line, stag = self.doc.tag, self.line, self.stag
with tag('head'):
self.go_in()
stag(
'link',
rel="stylesheet",
href=
"https://cdnjs.cloudflare.com/ajax/libs/mini.css/3.0.1/mini-default.min.css"
)
line('title', 'General purpose VNF indicator server')
request = self.request # type: HTTPServerRequest
uri = request.protocol + '://' + request.host + '/favicon.png'
stag('link', rel="icon", href=uri, type="image.png")
self.go_out()
self.nl()
def description(self):
tag, text, line, fn = self.doc.tag, self.doc.text, self.doc.line, self.footnote
line('h3', id='description', text_content='Description')
with tag('p'):
self.go_in()
text("This is a general-purpose HTTP server serving VNF indicator")
fn("<a href=https://www.etsi.org/deliver/etsi_gs/nfv-ifa/"
"001_099/011/03.01.01_60/gs_nfv-ifa011v030101p.pdf> "
"https://www.etsi.org/deliver/etsi_gs/nfv-ifa/"
"001_099/011/03.01.01_60/gs_nfv-ifa011v030101p.pdf</a> "
"- paragraph 7.1.11.2")
text(
" information. It offers several simple APIs to store data and an ETSI IFA 008 compliant"
)
fn("<a href=https://www.etsi.org/deliver/etsi_gs/NFV-IFA/"
"001_099/008/03.01.01_60/gs_nfv-ifa008v030101p.pdf> "
"https://www.etsi.org/deliver/etsi_gs/NFV-IFA/"
"001_099/008/03.01.01_60/gs_nfv-ifa008v030101p.pdf</a>"
"- paragraph 6.3.4")
text(" NBI.")
self.go_out()
self.nl()
def nbi(self):
tag, text, line = self.doc.tag, self.doc.text, self.line
line('h3', id='nbi', text_content='North bound interface')
with tag('p'):
self.go_in()
text(
'To check VNF indicator values from this server, perform an HTTP GET request to '
)
request = self.request # type: HTTPServerRequest
uri = request.protocol + '://' + request.host + '/vnfind/v1'
line('a', uri, href=uri)
text('.')
self.go_out()
line(
'p',
'The actual call performed by the PNFM will be a POST request to the same URL with body '
)
with tag('pre'):
self.doc.asis('{"filter": null}')
self.nl()
line('p', 'The format of the response is as follows:')
self.format_example()
def format_example(self):
ex = """{
"indicatorInformation": [
{
"vnfInstanceId": "VNF_1",
"indicatorId": "firstParameter",
"indicatorValue": "123321"
"indicatorName": "firstParameter"
},
{
"vnfInstanceId": "VNF_1",
"indicatorId": "secondParameter",
"indicatorValue": "true"
"indicatorName": "secondParameter"
}
]
}"""
ex = ex.replace('\n', '<br>')
with self.doc.tag('pre'):
self.doc.asis(ex)
self.nl()
def sbi(self):
line = self.line
tag = self.doc.tag
text = self.doc.text
line('h3', 'South bound interface')
with tag('p'):
self.go_in()
text(
'To set the VNF indicator values returned by the server, send a POST request to '
)
request = self.request # type: HTTPServerRequest
uri = request.protocol + '://' + request.host + '/vnfind/v1/<indicator-id>'
line('a', uri, href=uri)
text('.')
self.go_out()
self.nl()
line('p', 'The format is as follows:')
self.sbi_example()
line(
'p',
'In case the "indicatorId" field is present, it MUST be equal to'
'the ID provided as a path parameter.')
line(
'p',
'The other optional fields are ignored: they are just accepted in order to'
'allow the exact same format of the response to be a valid input.')
def sbi_example(self):
ex = """{
"vnfInstanceId": "VNF_1", // Optional
"indicatorId": "firstParameter", // Optional
"indicatorValue": "123321",
"indicatorName": "firstParameter" // Optional
}"""
ex = ex.replace('\n', '<br>')
with self.doc.tag('pre'):
self.doc.asis(ex)
self.nl()
def body(self):
doc, tag, text, line, fn = self.doc, self.doc.tag, self.doc.text, self.doc.line, self.footnote
with tag('body'):
self.go_in()
line('h1', 'General purpose VNF indicator server')
self.description()
self.nbi()
self.sbi()
self.go_out()
self.nl()
def get(self) -> None:
self.log.debug('Received call')
doc, tag, text, line, fn = self.doc, self.doc.tag, self.doc.text, self.doc.line, self.footnote
with tag('html'):
self.go_in()
self.head()
self.body()
self.go_out()
self.nl()
self.write(doc.getvalue() + '\n' + self.footer.getvalue())
