def format_recipe(recipe, display_opts):
""" Return the recipe in a paragraph in a samepage
"""
recipe_page = SamepageEnvironment()
name_line = LargeText(recipe.name)
if display_opts.origin and 'schubar original' in recipe.origin.lower():
name_line.append(superscript('*'))
#name_line.append(superscript(NoEscape('\dag')))
if display_opts.prices and recipe.max_cost:
price = util.calculate_price(recipe.max_cost, display_opts.markup)
name_line.append(DotFill())
name_line.append(superscript('$'))
name_line.append(price)
name_line.append('\n')
recipe_page.append(name_line)
if display_opts.prep_line:
recipe_page.append(
FootnoteText(recipe.prep_line(extended=True, caps=False) + '\n'))
if display_opts.info and recipe.info:
recipe_page.append(SmallText(italic(recipe.info + '\n')))
for item in recipe.ingredients:
recipe_page.append(item.str() + '\n')
if display_opts.variants:
for variant in recipe.variants:
recipe_page.append(HorizontalSpace('8pt'))
recipe_page.append(SmallText(italic(variant + '\n')))
if display_opts.examples and recipe.examples: # and recipe.name != 'The Cocktail':
for e in recipe.examples:
recipe_page.append(
FootnoteText(
"${cost:.2f} | {abv:.2f}% | {std_drinks:.2f} | {kinds}\n".
format(**e._asdict())))
recipe_page.append(Command('par'))
return recipe_page
def test_basic():
# Tests the basic commands and environments
# Basic commands
new_page = NewPage()
repr(new_page)
new_line = NewLine()
repr(new_line)
line_break = LineBreak()
repr(line_break)
h_fill = HFill()
repr(h_fill)
# Basic environments
huge = HugeText("Huge")
huge.append("Huge 2")
repr(huge)
large = LargeText("Large")
large.append("Large 2")
repr(large)
medium = MediumText("Medium")
medium.append("Medium 2")
repr(medium)
small = SmallText("Small")
small.append("Small 2")
repr(small)
footnote = FootnoteText("Footnote")
footnote.append("Footnote 2")
repr(footnote)
text_color = TextColor("green", "GreenText")
text_color.append("More Green Text")
repr(text_color)
def test_basic():
# Tests the basic commands and environments
# Basic commands
new_page = NewPage()
repr(new_page)
new_line = NewLine()
repr(new_line)
line_break = LineBreak()
repr(line_break)
h_fill = HFill()
repr(h_fill)
# Basic environments
huge = HugeText("Huge")
huge.append("Huge 2")
repr(huge)
large = LargeText("Large")
large.append("Large 2")
repr(large)
medium = MediumText("Medium")
medium.append("Medium 2")
repr(medium)
small = SmallText("Small")
small.append("Small 2")
repr(small)
footnote = FootnoteText("Footnote")
footnote.append("Footnote 2")
repr(footnote)
text_color = TextColor("green", "GreenText")
text_color.append("More Green Text")
repr(text_color)
def setup_header_footer(doc, pdf_opts, display_opts):
# Header with title, tagline, page number right, date left
# Footer with key to denote someting about drinks
title = pdf_opts.title or '@Schubar'
if display_opts.prices:
tagline = 'Tips never required, always appreciated'
tagline = pdf_opts.tagline or 'Tips for your drinks never required, always appreciated'
else:
tagline = 'Get Fubar at Schubar on the good stuff'
tagline = pdf_opts.tagline or 'Get Fubar at Schubar, but, like, in a classy way'
hf = PageStyle("schubarheaderfooter",
header_thickness=0.4,
footer_thickness=0.4)
with hf.create(Head('L')):
hf.append(TitleText(title))
hf.append(Command('\\'))
hf.append(FootnoteText(italic(tagline)))
with hf.create(Head('R')):
hf.append(FootnoteText(time.strftime("%b %d, %Y")))
if display_opts.origin:
with hf.create(Foot('L')):
hf.append(superscript("*"))
#hf.append(superscript(NoEscape("\dag")))
hf.append(FootnoteText(r"Schubar Original"))
with hf.create(Foot('C')):
if display_opts.prices:
hf.append(HorizontalSpace('12pt'))
hf.append(
FootnoteText(
NoEscape(
r"\$ amount shown is recommended tip, calculated from cost of ingredients"
)))
with hf.create(Foot('R')):
hf.append(FootnoteText(Command('thepage')))
doc.preamble.append(hf)
doc.change_document_style("schubarheaderfooter")
def dumps(self):
# Clear the old data
self.clear()
assessment = self.assessment
if hasattr(self.assessment, 'course'):
course = self.assessment.course
else:
course = None
if course:
# Create left header
with self.create(Head('L')):
self.append(course.name)
self.append(LineBreak())
self.append(SmallText(course.kind))
self.append(self.padding)
# Create center header
with self.create(Head('C')):
self.append(NoEscape(r'\ifbool{instructorKey}{'))
self.append(SmallText(bold('FOR INSTRUCTORS ONLY')))
self.append(LineBreak())
self.append(SmallText(bold(f'(Max. of {assessment.maxpoints} points)')))
self.append(NoEscape(r'}{}'))
self.append(self.padding)
# Create right header
with self.create(Head('R')):
self.append(f'{assessment.kind} {assessment.number}.{assessment.version} - {assessment.fullpoints} Points')
self.append(LineBreak())
self.append(SmallText(f'Assessing Period: {assessment.period}'))
self.append(self.padding)
if course:
# Create left footer
with self.create(Foot('L')):
self.append(course.term)
# Create center footer
with self.create(Foot('C')):
now = datetime.now().strftime('%Y%m%d%H%M')
self.append(FootnoteText(now))
# Create right footer
with self.create(Foot('R')):
self.append(simple_page_number())
return super().dumps()
def save_report(model, file_name, detailed_traces=2):
print('Saving analytics report to {}.tex and {}.pdf'.format(
file_name, file_name))
inference_network = model._inference_network
iter_per_sec = inference_network._total_train_iterations / inference_network._total_train_seconds
traces_per_sec = inference_network._total_train_traces / inference_network._total_train_seconds
traces_per_iter = inference_network._total_train_traces / inference_network._total_train_iterations
train_loss_initial = inference_network._history_train_loss[0]
train_loss_final = inference_network._history_train_loss[-1]
train_loss_change = train_loss_final - train_loss_initial
train_loss_change_per_sec = train_loss_change / inference_network._total_train_seconds
train_loss_change_per_iter = train_loss_change / inference_network._total_train_iterations
train_loss_change_per_trace = train_loss_change / inference_network._total_train_traces
valid_loss_initial = inference_network._history_valid_loss[0]
valid_loss_final = inference_network._history_valid_loss[-1]
valid_loss_change = valid_loss_final - valid_loss_initial
valid_loss_change_per_sec = valid_loss_change / inference_network._total_train_seconds
valid_loss_change_per_iter = valid_loss_change / inference_network._total_train_iterations
valid_loss_change_per_trace = valid_loss_change / inference_network._total_train_traces
sys.stdout.write(
'Generating report... \r')
sys.stdout.flush()
geometry_options = {
'tmargin': '1.5cm',
'lmargin': '1cm',
'rmargin': '1cm',
'bmargin': '1.5cm'
}
doc = Document('basic', geometry_options=geometry_options)
doc.preamble.append(NoEscape(r'\usepackage[none]{hyphenat}'))
doc.preamble.append(NoEscape(r'\usepackage{float}'))
# doc.preamble.append(NoEscape(r'\renewcommand{\familydefault}{\ttdefault}'))
doc.preamble.append(Command('title', 'pyprob analytics: ' + model.name))
doc.preamble.append(
Command(
'date',
NoEscape(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))
doc.append(NoEscape(r'\maketitle'))
# doc.append(NoEscape(r'\small'))
print('Analytics: Current system')
with doc.create(Section('Current system', numbering=False)):
with doc.create(Tabularx('ll')) as table:
table.add_row(('pyprob version', __version__))
table.add_row(('PyTorch version', torch.__version__))
# doc.append(NoEscape(r'\newpage'))
print('Analytics: Inference network')
with doc.create(Section('Inference network', numbering=False)):
print('Analytics: Inference network.File')
with doc.create(Section('File')):
with doc.create(Tabularx('ll')) as table:
# table.add_row(('File name', file_name))
# file_size = '{:,}'.format(os.path.getsize(file_name))
# table.add_row(('File size', file_size + ' Bytes'))
table.add_row(('Created', inference_network._created))
table.add_row(('Modified', inference_network._modified))
table.add_row(('Updates to file', inference_network._updates))
print('Analytics: Inference network.Training')
with doc.create(Section('Training')):
with doc.create(Tabularx('ll')) as table:
table.add_row(
('pyprob version', inference_network._pyprob_version))
table.add_row(
('PyTorch version', inference_network._torch_version))
table.add_row(('Trained on', inference_network._trained_on))
table.add_row(('Total training time', '{0}'.format(
util.days_hours_mins_secs_str(
inference_network._total_train_seconds))))
table.add_row(
('Total training traces',
'{:,}'.format(inference_network._total_train_traces)))
table.add_row(('Traces / s', '{:,.2f}'.format(traces_per_sec)))
table.add_row(
('Traces / iteration', '{:,.2f}'.format(traces_per_iter)))
table.add_row(
('Iterations',
'{:,}'.format(inference_network._total_train_iterations)))
table.add_row(
('Iterations / s', '{:,.2f}'.format(iter_per_sec)))
table.add_row(('Optimizer', inference_network._optimizer_type))
table.add_row(('Validation set size',
inference_network._valid_batch.length))
print('Analytics: Inference network.Training loss')
with doc.create(Subsection('Training loss')):
with doc.create(Tabularx('ll')) as table:
table.add_row(
('Initial loss', '{:+.6e}'.format(train_loss_initial)))
table.add_row(
('Final loss', '{:+.6e}'.format(train_loss_final)))
table.add_row(('Loss change / s',
'{:+.6e}'.format(train_loss_change_per_sec)))
table.add_row(('Loss change / iteration',
'{:+.6e}'.format(train_loss_change_per_iter)))
table.add_row(('Loss change / trace',
'{:+.6e}'.format(train_loss_change_per_trace)))
print('Analytics: Inference network.Validation loss')
with doc.create(Subsection('Validation loss')):
with doc.create(Tabularx('ll')) as table:
table.add_row(
('Initial loss', '{:+.6e}'.format(valid_loss_initial)))
table.add_row(
('Final loss', '{:+.6e}'.format(valid_loss_final)))
table.add_row(('Loss change / s',
'{:+.6e}'.format(valid_loss_change_per_sec)))
table.add_row(('Loss change / iteration',
'{:+.6e}'.format(valid_loss_change_per_iter)))
table.add_row(('Loss change / trace',
'{:+.6e}'.format(valid_loss_change_per_trace)))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 6))
ax = plt.subplot(111)
ax.plot(inference_network._history_train_loss_trace,
inference_network._history_train_loss,
label='Training')
ax.plot(inference_network._history_valid_loss_trace,
inference_network._history_valid_loss,
label='Validation')
ax.legend()
plt.xlabel('Training traces')
plt.ylabel('Loss')
plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Loss plot.')
print('Analytics: Inference network.Neural network modules')
with doc.create(Section('Neural network modules')):
with doc.create(Tabularx('ll')) as table:
table.add_row(
('Total trainable parameters',
'{:,}'.format(inference_network._history_num_params[-1])))
# table.add_row(('Softmax boost', inference_network.softmax_boost))
# table.add_row(('Dropout', inference_network.dropout))
# table.add_row(('Standardize inputs', inference_network.standardize_observes))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 4))
ax = plt.subplot(111)
ax.plot(inference_network._history_num_params_trace,
inference_network._history_num_params)
plt.xlabel('Training traces')
plt.ylabel('Number of parameters')
plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Number of parameters.')
doc.append(NoEscape(r'\newpage'))
print(
'Analytics: Inference network.Neural network modules.All modules'
)
with doc.create(Subsection('All modules')):
doc.append(str(inference_network))
for m_name, m in inference_network.named_modules():
if (m_name != ''):
regex = r'(sample_embedding_layer\(\S*\)._)|(proposal_layer\(\S*\)._)|(_observe_embedding_layer.)|(_lstm)'
if len(list(re.finditer(regex, m_name))) > 0:
# if ('_observe_embedding_layer.' in m_name) or ('sample_embedding_layer.' in m_name) or ('proposal_layer.' in m_name):
doc.append(NoEscape(r'\newpage'))
with doc.create(Subsubsection(m_name)):
doc.append(str(m))
for p_name, p in m.named_parameters():
if not 'bias' in p_name:
with doc.create(
Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 10))
ax = plt.subplot(111)
plt.imshow(np.transpose(
util.weights_to_image(p),
(1, 2, 0)),
interpolation='none')
plt.axis('off')
plot.add_plot(
width=NoEscape(r'\textwidth'))
plot.add_caption(m_name + '_' + p_name)
# doc.append(NoEscape(r'\newpage'))
# print('Analytics: Inference network.Neural network modules.Address embeddings')
# with doc.create(Subsection('Address embeddings')):
# for p_name, p in inference_network.named_parameters():
# if ('address_embedding' in p_name):
# with doc.create(Figure(position='H')) as plot:
# fig = plt.figure(figsize=(10,10))
# ax = plt.subplot(111)
# plt.imshow(np.transpose(util.weights_to_image(p),(1,2,0)), interpolation='none')
# plt.axis('off')
# plot.add_plot(width=NoEscape(r'\textwidth'))
# plot.add_caption(FootnoteText(p_name.replace('::', ':: ')))
gc.collect()
doc.append(NoEscape(r'\newpage'))
print('Analytics: Inference network.Traces')
with doc.create(Section('Traces')):
with doc.create(Tabularx('ll')) as table:
table.add_row(
('Total training traces',
'{:,}'.format(inference_network._total_train_traces)))
print(
'Analytics: Inference network.Traces.Distributions encountered'
)
with doc.create(Subsection('Distributions encountered')):
with doc.create(Tabularx('ll')) as table:
# print([v[2] for v in inference_network._address_stats.values()])
distributions = set([
v[2]
for v in inference_network._address_stats.values()
])
num_distributions = len(distributions)
table.add_row(
('Number of distributions', num_distributions))
table.add_empty_row()
for distribution in distributions:
table.add_row((distribution, ''))
print('Analytics: Inference network.Traces.Addresses encountered')
with doc.create(Subsection('Addresses encountered')):
with doc.create(Tabularx('lX')) as table:
num_addresses_all = len(
inference_network._address_stats.keys())
table.add_row(('Number of addresses', num_addresses_all))
num_addresses_controlled = len([
k for k, v in inference_network._address_stats.items()
if v[3]
])
num_addresses_replaced = len([
k for k, v in inference_network._address_stats.items()
if v[3] and v[4]
])
num_addresses_observed = len([
k for k, v in inference_network._address_stats.items()
if v[5]
])
table.add_row(
(TextColor('red', 'Number of addresses (controlled)'),
TextColor('red', num_addresses_controlled)))
table.add_row((TextColor('green',
'Number of addresses (replaced)'),
TextColor('green', num_addresses_replaced)))
table.add_row((TextColor('blue',
'Number of addresses (observed)'),
TextColor('blue', num_addresses_observed)))
table.add_row(
('Number of addresses (uncontrolled)',
num_addresses_all - num_addresses_controlled -
num_addresses_observed))
table.add_empty_row()
doc.append('\n')
with doc.create(LongTable('llllllp{12cm}')) as table:
# table.add_empty_row()
table.add_row(FootnoteText('Count'), FootnoteText('ID'),
FootnoteText('Distrib.'),
FootnoteText('Ctrl.'),
FootnoteText('Replace'),
FootnoteText('Obs.'),
FootnoteText('Address'))
table.add_hline()
# address_to_abbrev = {}
# abbrev_to_address =
# abbrev_i = 0
# sorted_addresses = sorted(inference_network.address_histogram.items(), key=lambda x:x[1], reverse=True)
plt_all_addresses = []
plt_all_counts = []
plt_all_colors = []
plt_controlled_addresses = []
plt_controlled_counts = []
plt_controlled_colors = []
address_id_to_count = {}
address_id_to_color = {}
address_id_count_total = 0
for address, vals in inference_network._address_stats.items(
):
address = address.replace('::', ':: ')
count = vals[0]
address_id = vals[1]
distribution = vals[2]
control = vals[3]
replace = vals[4]
observed = vals[5]
plt_all_addresses.append(address_id)
plt_all_counts.append(1 if replace else count)
address_id_to_count[address_id] = count
address_id_count_total += count
if control:
if replace:
color = 'green'
plt_controlled_counts.append(1)
else:
color = 'red'
plt_controlled_counts.append(count)
plt_controlled_addresses.append(address_id)
plt_controlled_colors.append(color)
elif observed:
color = 'blue'
plt_controlled_addresses.append(address_id)
plt_controlled_colors.append(color)
plt_controlled_counts.append(count)
else:
color = 'black'
table.add_row(
(TextColor(color,
FootnoteText('{:,}'.format(count))),
TextColor(color, FootnoteText(address_id)),
TextColor(color, FootnoteText(distribution)),
TextColor(color, FootnoteText(control)),
TextColor(color, FootnoteText(replace)),
TextColor(color, FootnoteText(observed)),
TextColor(color, FootnoteText(address))))
plt_all_colors.append(color)
address_id_to_color[address_id] = color
gc.collect()
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
plt_x = range(len(plt_all_addresses))
ax.bar(plt_x, plt_all_counts, color=plt_all_colors)
plt.xticks(plt_x, plt_all_addresses)
plt.xlabel('Address ID')
plt.ylabel('Count')
# plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption(
'Histogram of all addresses. Red: controlled, green: replaced, black: uncontrolled, blue: observed.'
)
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
plt_x = range(len(plt_controlled_addresses))
ax.bar(plt_x,
plt_controlled_counts,
color=plt_controlled_colors)
plt.xticks(plt_x, plt_controlled_addresses)
plt.xlabel('Address ID')
plt.ylabel('Count')
# plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption(
'Histogram of controlled and observed addresses. Red: controlled, green: replaced, blue: observed.'
)
gc.collect()
print('Analytics: Inference network.Traces.Trace lengths')
with doc.create(Subsection('Trace lengths')):
with doc.create(Tabularx('ll')) as table:
trace_lengths_controlled = [
v[3] for v in inference_network._trace_stats.values()
]
trace_lengths_controlled_min = min(
trace_lengths_controlled)
trace_lengths_controlled_max = max(
trace_lengths_controlled)
trace_lengths_all = [
v[2] for v in inference_network._trace_stats.values()
]
trace_lengths_all_min = min(trace_lengths_all)
trace_lengths_all_max = max(trace_lengths_all)
s_controlled = 0
s_all = 0
total_count = 0
for _, v in inference_network._trace_stats.items():
trace_length_controlled = v[3]
trace_length_all = v[2]
count = v[0]
s_controlled += trace_length_controlled * count
total_count += count
s_all += trace_length_all * count
trace_length_controlled_mean = s_controlled / total_count
trace_length_all_mean = s_all / total_count
table.add_row(('Trace length min',
'{:,}'.format(trace_lengths_all_min)))
table.add_row(('Trace length max',
'{:,}'.format(trace_lengths_all_max)))
table.add_row(('Trace length mean',
'{:.2f}'.format(trace_length_all_mean)))
table.add_row(
('Controlled trace length min',
'{:,}'.format(trace_lengths_controlled_min)))
table.add_row(
('Controlled trace length max',
'{:,}'.format(trace_lengths_controlled_max)))
table.add_row(
('Controlled trace length mean',
'{:.2f}'.format(trace_length_controlled_mean)))
with doc.create(Figure(position='H')) as plot:
plt_counter = dict(Counter(trace_lengths_all))
plt_lengths = [
i for i in range(0, trace_lengths_all_max + 1)
]
plt_counts = [
plt_counter[i] if i in plt_counter else 0
for i in range(0, trace_lengths_all_max + 1)
]
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
ax.bar(plt_lengths,
plt_counts,
width=trace_lengths_all_max / 500.)
plt.xlabel('Length')
plt.ylabel('Count')
# plt.yscale('log')
# plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Histogram of trace lengths.')
with doc.create(Figure(position='H')) as plot:
plt_counter = dict(Counter(trace_lengths_controlled))
plt_lengths = [
i for i in range(0, trace_lengths_controlled_max + 1)
]
plt_counts = [
plt_counter[i] if i in plt_counter else 0
for i in range(0, trace_lengths_controlled_max + 1)
]
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
ax.bar(plt_lengths, plt_counts)
plt.xlabel('Length')
plt.ylabel('Count')
# plt.yscale('log')
# plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Histogram of controlled trace lengths.')
gc.collect()
print(
'Analytics: Inference network.Traces.Unique traces encountered'
)
with doc.create(Subsection('Unique traces encountered')):
detailed_traces = min(len(inference_network._trace_stats),
detailed_traces)
with doc.create(Tabularx('ll')) as table:
table.add_row(
('Unique traces encountered',
'{:,}'.format(len(inference_network._trace_stats))))
table.add_row(('Unique traces rendered in detail',
'{:,}'.format(detailed_traces)))
doc.append('\n')
with doc.create(LongTable('llllp{15cm}')) as table:
# table.add_empty_row()
table.add_row(FootnoteText('Count'), FootnoteText('ID'),
FootnoteText('Len.'),
FootnoteText('Ctrl. len.'),
FootnoteText('Unique trace'))
table.add_hline()
plt_traces = []
plt_counts = []
for trace_str, vals in inference_network._trace_stats.items(
):
count = vals[0]
trace_id = vals[1]
length_all = vals[2]
length_controlled = vals[3]
addresses_controlled = vals[4]
addresses_controlled_str = ' '.join(
addresses_controlled)
plt_traces.append(trace_id)
plt_counts.append(count)
table.add_row(
(FootnoteText('{:,}'.format(count)),
FootnoteText(trace_id),
FootnoteText('{:,}'.format(length_all)),
FootnoteText('{:,}'.format(length_controlled)),
FootnoteText(addresses_controlled_str)))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
plt_x = range(len(plt_traces))
ax.bar(plt_x, plt_counts)
plt.xticks(plt_x, plt_traces)
plt.xlabel('Trace ID')
plt.ylabel('Count')
# plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Histogram of unique traces.')
with doc.create(Figure(position='H')) as plot:
sorted_trace_stats = OrderedDict(
sorted(dict(inference_network._trace_stats).items(),
key=lambda x: x[1],
reverse=True))
master_trace_pairs = {}
transition_count_total = 0
for trace_str, vals in sorted_trace_stats.items():
count = vals[0]
ta = vals[4]
for left, right in zip(ta, ta[1:]):
if (left, right) in master_trace_pairs:
master_trace_pairs[(left, right)] += count
else:
master_trace_pairs[(left, right)] = count
transition_count_total += count
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
master_graph = pydotplus.graphviz.Dot(graph_type='digraph',
rankdir='LR')
transition_count_max = 0
for p, count in master_trace_pairs.items():
if count > transition_count_max:
transition_count_max = count
nodes = master_graph.get_node(p[0])
if len(nodes) > 0:
n0 = nodes[0]
else:
n0 = pydotplus.Node(p[0])
master_graph.add_node(n0)
nodes = master_graph.get_node(p[1])
if len(nodes) > 0:
n1 = nodes[0]
else:
n1 = pydotplus.Node(p[1])
master_graph.add_node(n1)
master_graph.add_edge(
pydotplus.Edge(n0, n1, weight=count))
for node in master_graph.get_nodes():
node.set_color('gray')
node.set_fontcolor('gray')
for edge in master_graph.get_edges():
edge.set_color('gray')
master_graph_annotated = pydotplus.graphviz.graph_from_dot_data(
master_graph.to_string())
for node in master_graph_annotated.get_nodes():
# color = util.rgb_to_hex(util.rgb_blend((1, 1, 1), (1, 0, 0), address_id_to_count[node.obj_dict['name']] / address_id_count_total))
address_id = node.obj_dict['name']
node.set_style('filled')
node.set_fillcolor(address_id_to_color[address_id])
node.set_color('black')
node.set_fontcolor('black')
for edge in master_graph_annotated.get_edges():
(left, right) = edge.obj_dict['points']
count = master_trace_pairs[(left, right)]
edge.set_label(count)
# color = util.rgb_to_hex((1.5*(count/transition_count_total), 0, 0))
edge.set_color('black')
edge.set_penwidth(2.5 * count / transition_count_max)
png_str = master_graph_annotated.create_png(
prog=['dot', '-Gsize=90', '-Gdpi=600'])
bio = BytesIO()
bio.write(png_str)
bio.seek(0)
img = np.asarray(mpimg.imread(bio))
plt.imshow(util.crop_image(img), interpolation='bilinear')
plt.axis('off')
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption(
'Succession of controlled addresses (accumulated over all traces). Red: controlled, green: replaced, blue: observed.'
)
for trace_str, vals in OrderedDict(
islice(sorted_trace_stats.items(), 0,
detailed_traces)).items():
count = vals[0]
trace_id = vals[1]
doc.append(NoEscape(r'\newpage'))
with doc.create(Subsubsection('Unique trace ' + trace_id)):
sys.stdout.write(
'Rendering unique trace {0}... \r'
.format(trace_id))
sys.stdout.flush()
addresses = len(plt_controlled_addresses)
trace_addresses = vals[4]
with doc.create(Tabularx('ll')) as table:
table.add_row(FootnoteText('Count'),
FootnoteText('{:,}'.format(count)))
table.add_row(
FootnoteText('Controlled length'),
FootnoteText('{:,}'.format(
len(trace_addresses))))
doc.append('\n')
im = np.zeros((addresses, len(trace_addresses)))
for i in range(len(trace_addresses)):
address = trace_addresses[i]
address_i = plt_controlled_addresses.index(address)
im[address_i, i] = 1
truncate = 100
for col_start in range(0, len(trace_addresses),
truncate):
col_end = min(col_start + truncate,
len(trace_addresses))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(20 * (
(col_end + 4 - col_start) / truncate), 4))
ax = plt.subplot(111)
# ax.imshow(im,cmap=plt.get_cmap('Greys'))
sns.heatmap(
im[:, col_start:col_end],
cbar=False,
linecolor='lightgray',
linewidths=.5,
cmap='Greys',
yticklabels=plt_controlled_addresses,
xticklabels=np.arange(col_start, col_end))
plt.yticks(rotation=0)
fig.tight_layout()
plot.add_plot(
width=NoEscape(r'{0}\textwidth'.format(
(col_end + 4 - col_start) / truncate)),
placement=NoEscape(r'\raggedright'))
with doc.create(Figure(position='H')) as plot:
pairs = {}
for left, right in zip(trace_addresses,
trace_addresses[1:]):
if (left, right) in pairs:
pairs[(left, right)] += 1
else:
pairs[(left, right)] = 1
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
graph = pydotplus.graphviz.graph_from_dot_data(
master_graph.to_string())
trace_address_to_count = {}
for address in trace_addresses:
if address in trace_address_to_count:
trace_address_to_count[address] += 1
else:
trace_address_to_count[address] = 1
transition_count_max = 0
for p, count in pairs.items():
if count > transition_count_max:
transition_count_max = count
left_node = graph.get_node(p[0])[0]
right_node = graph.get_node(p[1])[0]
edge = graph.get_edge(p[0], p[1])[0]
# color = util.rgb_to_hex(util.rgb_blend((1,1,1), (1,0,0), trace_address_to_count[p[0]] / len(trace_addresses)))
left_node.set_style('filled')
left_node.set_fillcolor(
address_id_to_color[p[0]])
left_node.set_color('black')
left_node.set_fontcolor('black')
# color = util.rgb_to_hex(util.rgb_blend((1,1,1), (1,0,0), trace_address_to_count[p[0]] / len(trace_addresses)))
right_node.set_style('filled')
right_node.set_fillcolor(
address_id_to_color[p[1]])
right_node.set_color('black')
right_node.set_fontcolor('black')
# (left, right) = edge.obj_dict['points']
edge.set_label(count)
# color = util.rgb_to_hex((1.5*(count/len(trace_addresses)),0,0))
edge.set_color('black')
edge.set_penwidth(2.5 * count /
transition_count_max)
png_str = graph.create_png(
prog=['dot', '-Gsize=90', '-Gdpi=600'])
bio = BytesIO()
bio.write(png_str)
bio.seek(0)
img = np.asarray(mpimg.imread(bio))
plt.imshow(util.crop_image(img),
interpolation='bilinear')
plt.axis('off')
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption(
'Succession of controlled addresses (for one trace of type '
+ trace_id +
'). Red: controlled, green: replaced, blue: observed.'
)
with doc.create(Tabularx('lp{16cm}')) as table:
table.add_row(
FootnoteText('Trace'),
FootnoteText(' '.join(trace_addresses)))
doc.generate_pdf(file_name, clean_tex=False)
sys.stdout.write(
' \r')
sys.stdout.flush()
def doit(self):
def render_metric(value, best, second, decimals, format_string):
if isinstance(value, numbers.Number):
rendered = format_string.format(value, prec=decimals)
if value == best:
rendered = bold(rendered)
elif value == second:
rendered = italic(rendered)
return rendered
else:
return value
values = np.empty((self.num_metrics, self.num_seqs, self.num_exps))
for i, seq in enumerate(self.seq_names):
for j, s in enumerate(self.experiment_specs):
values[:, i, j] = np.array(
self.get_metrics(self.exps[s.name], seq, s.it))
means = np.empty((self.num_metrics, self.num_exps))
for i, m in enumerate(self.metrics):
if m.geometric_mean:
means[i, :] = scipy.stats.gmean(values[i, :, :], axis=0)
else:
means[i, :] = np.mean(values[i, :, :], axis=0)
t = Tabular('l' + 'c' * self.num_exps)
t.add_hline()
escape_header_fun = lambda text: text if self.spec.escape_latex_header else NoEscape(
text)
if self.spec.rotate_header:
t.add_row([self.spec.header] + [
NoEscape(r"\rotatebox{90}{%s}" % escape_latex(
escape_header_fun(s.display_name(self.exps[s.name]))))
for s in self.experiment_specs
])
else:
t.add_row([self.spec.header] + [
escape_header_fun(s.display_name(self.exps[s.name]))
for s in self.experiment_specs
])
t.add_hline()
for i, m in enumerate(self.metrics):
row_values = np.around(means[i, :], m.decimals)
top_values = best_two_non_repeating(row_values,
reverse=m.larger_is_better)
row = [m.display_name]
for v in row_values:
# TODO: use NoEscape only if certain flag is enabled?
row.append(
NoEscape(
render_metric(v, top_values[0], top_values[1],
m.effective_display_decimals(),
m.format_string)))
t.add_row(row)
t.add_hline()
if self.spec.export_latex:
os.makedirs(self.export_basepath, exist_ok=True)
t.generate_tex(
os.path.join(self.export_basepath, self.spec.export_latex))
with self.create(Subsection(self.spec.name, numbering=False)) as p:
p.append(FootnoteText(t))
def doit(self):
is_multirow = self.num_metrics > 1 and self.spec.multirow
def render_metric(value, best, second, decimals, format_string,
highlight_top, relative_to):
if isinstance(value, numbers.Number):
if relative_to is None or relative_to == 0 or not np.isfinite(
relative_to):
# absolute number
rendered = format_string.format(value, prec=decimals)
else:
# percent
rendered = format_ratio_percent(value,
relative_to,
decimals=decimals)
if highlight_top:
if value == best:
rendered = bold(rendered)
elif value == second:
rendered = italic(rendered)
return rendered
else:
return value
if self.spec.export_latex:
row_height = None
else:
row_height = 0.65 if is_multirow and self.num_metrics >= 3 else 1
column_spec = '|r' if self.spec.vertical_bars else 'r'
t = Tabular('l' + column_spec * self.num_exps,
row_height=row_height,
pos=['t'])
escape_header_fun = lambda text: text if self.spec.escape_latex_header else NoEscape(
text)
if self.spec.rotate_header:
t.add_row([''] + [
NoEscape(r"\rotatebox{90}{%s}" % escape_latex(
escape_header_fun(s.display_name(self.exps[s.name]))))
for s in self.experiment_specs
])
else:
t.add_row([''] + [
escape_header_fun(s.display_name(self.exps[s.name]))
for s in self.experiment_specs
])
t.add_hline()
for seq in self.seq_names:
fails = [
self.is_failed(self.exps[s.name], seq)
for s in self.experiment_specs
]
failure_strings = [
self.render_failure(self.exps[s.name], seq)
for s in self.experiment_specs
]
values = np.array([
self.get_metrics(self.exps[s.name], seq, s.it)
for s in self.experiment_specs
])
top_values = list(range(self.num_metrics))
for c, m in enumerate(self.metrics):
try:
values[:, c] = np.around(values[:, c], m.decimals)
except IndexError:
pass
non_excluded_values = np.array(values[:, c])
for i in m.exclude_columns_highlight:
non_excluded_values[i] = math.nan
top_values[c] = best_two_non_repeating(
non_excluded_values, reverse=m.larger_is_better)
if is_multirow:
rows = [[
MultiRow(self.num_metrics, data=self.seq_displayname(seq))
]] + [list(['']) for _ in range(1, self.num_metrics)]
else:
rows = [[self.seq_displayname(seq)]]
for c, (fail, failure_str,
value_col) in enumerate(zip(fails, failure_strings,
values)):
if failure_str is not None:
if self.spec.color_failed:
failure_str = TextColor(self.spec.color_failed,
failure_str)
if is_multirow:
rows[0].append(
MultiRow(self.num_metrics, data=failure_str))
for r in range(1, self.num_metrics):
rows[r].append('')
else:
rows[0].append(failure_str)
else:
tmp_data = [None] * self.num_metrics
for r, m in enumerate(self.metrics):
if m.failed_threshold and value_col[
r] > m.failed_threshold:
obj = "x"
if self.spec.color_failed:
obj = TextColor(self.spec.color_failed, obj)
else:
relative_to = None
if m.relative_to_column is not None and m.relative_to_column != c:
relative_to = values[m.relative_to_column, r]
obj = render_metric(value_col[r],
top_values[r][0],
top_values[r][1],
m.effective_display_decimals(),
m.format_string,
m.highlight_top,
relative_to=relative_to)
if fail and self.spec.color_failed:
obj = TextColor(self.spec.color_failed, obj)
tmp_data[r] = obj
if self.num_metrics == 1 or is_multirow:
for r, obj in enumerate(tmp_data):
rows[r].append(obj)
else:
entry = []
for v in tmp_data:
entry.append(v)
entry.append(NoEscape("~/~"))
entry.pop()
rows[0].append(dumps_list(entry))
for row in rows:
t.add_row(row)
if is_multirow:
t.add_hline()
if self.spec.export_latex:
os.makedirs(self.export_basepath, exist_ok=True)
t.generate_tex(
os.path.join(self.export_basepath, self.spec.export_latex))
with self.create(Subsection(self.spec.name, numbering=False)) as p:
if self.spec.metrics_legend:
legend = Tabular('|c|', row_height=row_height, pos=['t'])
legend.add_hline()
legend.add_row(["Metrics"])
legend.add_hline()
for m in self.metrics:
legend.add_row([m.display_name])
legend.add_hline()
tab = Tabular("ll")
tab.add_row([t, legend])
content = tab
else:
content = t
if True:
content = FootnoteText(content)
p.append(content)
if __name__ == '__main__':
geometry_options = {"margin": "0.5in", "paperwidth": "11in"}
doc = Document(indent=False,
documentclass="article",
document_options="a4paper,landscape",
geometry_options=geometry_options,
page_numbers=False)
doc.packages.append(Package("array"))
# Header
with doc.create(Center()) as centered:
centered.append(FootnoteText(bold(lorem(4))))
centered.append(FootnoteText(lorem(12)))
centered.append(LineBreak())
centered.append(FootnoteText(lorem(19)))
centered.append(LineBreak())
centered.append(FootnoteText(lorem(17)))
centered.append(LineBreak())
centered.append(FootnoteText(lorem(17)))
centered.append(LineBreak())
centered.append(LineBreak())
centered.append(MediumText(bold(lorem(5))))
# Title
doc.append(HorizontalSpace("250pt"))
def main():
try:
parser = argparse.ArgumentParser(
description='pyprob ' + pyprob.__version__ + ' (Analytics)',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-v',
'--version',
help='show version information',
action='store_true')
parser.add_argument(
'--dir',
help='directory for loading artifacts and saving logs',
default='.')
parser.add_argument('--cuda', help='use CUDA', action='store_true')
parser.add_argument(
'--device',
help=
'selected CUDA device (-1: all, 0: 1st device, 1: 2nd device, etc.)',
default=-1,
type=int)
parser.add_argument('--seed',
help='random seed',
default=123,
type=int)
parser.add_argument(
'--structure',
help='show extra information about artifact structure',
action='store_true')
parser.add_argument('--saveReport',
help='save a full analytics report (tex and pdf)',
type=str)
parser.add_argument(
'--maxTraces',
help=
'maximum number of unique traces to plot in the full analytics report',
default=20,
type=int)
parser.add_argument(
'--saveLoss',
help='save training and validation loss history (csv)',
type=str)
parser.add_argument('--saveAddresses',
help='save histogram of addresses (csv)',
type=str)
parser.add_argument('--saveTraceLengths',
help='save histogram of trace lengths (csv)',
type=str)
opt = parser.parse_args()
if opt.version:
print(pyprob.__version__)
quit()
util.set_random_seed(opt.seed)
util.set_cuda(opt.cuda, opt.device)
util.logger.reset()
util.logger.log_config()
file_name = util.file_starting_with(
'{0}/{1}'.format(opt.dir, 'pyprob-artifact'), -1)
util.logger.log(
colored('Resuming previous artifact: {}'.format(file_name),
'blue',
attrs=['bold']))
artifact = util.load_artifact(file_name, util.cuda_enabled,
util.cuda_device)
util.logger.log(artifact.get_info())
util.logger.log()
if opt.structure:
util.logger.log()
util.logger.log(
colored('Artifact structure', 'blue', attrs=['bold']))
util.logger.log()
util.logger.log(artifact.get_structure_str())
util.logger.log(artifact.get_parameter_str())
if opt.saveLoss:
util.logger.log(
'Saving training and validation loss history to file: ' +
opt.saveLoss)
with open(opt.saveLoss, 'w') as f:
data = [
artifact.train_history_trace, artifact.train_history_loss,
artifact.valid_history_trace, artifact.valid_history_loss
]
writer = csv.writer(f)
writer.writerow(
['train_trace', 'train_loss', 'valid_trace', 'valid_loss'])
for values in zip_longest(*data):
writer.writerow(values)
if opt.saveAddresses:
util.logger.log('Saving address histogram to file: ' +
opt.saveAddresses)
with open(opt.saveAddresses, 'w') as f:
data_count = []
data_address = []
data_abbrev = []
abbrev_i = 0
for address, count in sorted(
artifact.address_histogram.items(),
key=lambda x: x[1],
reverse=True):
abbrev_i += 1
data_abbrev.append('A' + str(abbrev_i))
data_address.append(address)
data_count.append(count)
data = [data_count, data_abbrev, data_address]
writer = csv.writer(f)
writer.writerow(['count', 'unique_address_id', 'full_address'])
for values in zip_longest(*data):
writer.writerow(values)
if opt.saveTraceLengths:
util.logger.log('Saving trace length histogram to file: ' +
opt.saveTraceLengths)
with open(opt.saveTraceLengths, 'w') as f:
data_trace_length = []
data_count = []
for trace_length in artifact.trace_length_histogram:
data_trace_length.append(trace_length)
data_count.append(
artifact.trace_length_histogram[trace_length])
data = [data_trace_length, data_count]
writer = csv.writer(f)
writer.writerow(['trace_length', 'count'])
for values in zip_longest(*data):
writer.writerow(values)
if opt.saveReport:
util.logger.log('Saving analytics report to files: ' +
opt.saveReport + '.tex and ' + opt.saveReport +
'.pdf')
iter_per_sec = artifact.total_iterations / artifact.total_training_seconds
traces_per_sec = artifact.total_traces / artifact.total_training_seconds
traces_per_iter = artifact.total_traces / artifact.total_iterations
train_loss_initial = artifact.train_history_loss[0]
train_loss_final = artifact.train_history_loss[-1]
train_loss_change = train_loss_final - train_loss_initial
train_loss_change_per_sec = train_loss_change / artifact.total_training_seconds
train_loss_change_per_iter = train_loss_change / artifact.total_iterations
train_loss_change_per_trace = train_loss_change / artifact.total_traces
valid_loss_initial = artifact.valid_history_loss[0]
valid_loss_final = artifact.valid_history_loss[-1]
valid_loss_change = valid_loss_final - valid_loss_initial
valid_loss_change_per_sec = valid_loss_change / artifact.total_training_seconds
valid_loss_change_per_iter = valid_loss_change / artifact.total_iterations
valid_loss_change_per_trace = valid_loss_change / artifact.total_traces
sys.stdout.write(
'Generating report... \r'
)
sys.stdout.flush()
geometry_options = {
'tmargin': '1.5cm',
'lmargin': '1cm',
'rmargin': '1cm',
'bmargin': '1.5cm'
}
doc = Document('basic', geometry_options=geometry_options)
doc.preamble.append(NoEscape(r'\usepackage[none]{hyphenat}'))
doc.preamble.append(NoEscape(r'\usepackage{float}'))
# doc.preamble.append(NoEscape(r'\renewcommand{\familydefault}{\ttdefault}'))
doc.preamble.append(
Command('title', 'Inference Compilation Analytics'))
doc.preamble.append(
Command(
'date',
NoEscape(datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S"))))
doc.append(NoEscape(r'\maketitle'))
# doc.append(NoEscape(r'\small'))
with doc.create(Section('Current system')):
with doc.create(Tabularx('ll')) as table:
table.add_row(('pyprob version', pyprob.__version__))
table.add_row(('PyTorch version', torch.__version__))
# doc.append(NoEscape(r'\newpage'))
with doc.create(Section('Artifact')):
with doc.create(Subsection('File')):
with doc.create(Tabularx('ll')) as table:
table.add_row(('File name', file_name))
file_size = '{:,}'.format(os.path.getsize(file_name))
table.add_row(('File size', file_size + ' Bytes'))
table.add_row(('Created', artifact.created))
table.add_row(('Modified', artifact.modified))
table.add_row(('Updates to file', artifact.updates))
with doc.create(Subsection('Training system')):
with doc.create(Tabularx('ll')) as table:
table.add_row(
('pyprob version', artifact.code_version))
table.add_row(
('PyTorch version', artifact.pytorch_version))
table.add_row(('Trained on', artifact.trained_on))
with doc.create(Subsection('Neural network')):
with doc.create(Tabularx('ll')) as table:
table.add_row(('Trainable parameters', '{:,}'.format(
artifact.num_params_history_num_params[-1])))
table.add_row(
('Softmax boost', artifact.softmax_boost))
table.add_row(('Dropout', artifact.dropout))
table.add_row(('Standardize inputs',
artifact.standardize_observes))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 4))
ax = plt.subplot(111)
ax.plot(artifact.num_params_history_trace,
artifact.num_params_history_num_params)
plt.xlabel('Training traces')
plt.ylabel('Number of parameters')
plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Number of parameters.')
for m_name, m in artifact.named_modules():
if not ('.' in m_name or m_name == ''):
doc.append(NoEscape(r'\newpage'))
with doc.create(Subsubsection(m_name)):
doc.append(str(m))
for p_name, p in m.named_parameters():
if not 'bias' in p_name:
with doc.create(
Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 10))
ax = plt.subplot(111)
plt.imshow(np.transpose(
util.weights_to_image(p),
(1, 2, 0)),
interpolation='none')
plt.axis('off')
plot.add_plot(
width=NoEscape(r'\textwidth'))
plot.add_caption(m_name + '_' +
p_name)
doc.append(NoEscape(r'\newpage'))
with doc.create(Section('Training')):
with doc.create(Tabularx('ll')) as table:
table.add_row(('Total training time', '{0}'.format(
util.days_hours_mins_secs(
artifact.total_training_seconds))))
table.add_row(('Total training traces',
'{:,}'.format(artifact.total_traces)))
table.add_row(
('Traces / s', '{:,.2f}'.format(traces_per_sec)))
table.add_row(('Traces / iteration',
'{:,.2f}'.format(traces_per_iter)))
table.add_row(('Iterations',
'{:,}'.format(artifact.total_iterations)))
table.add_row(
('Iterations / s', '{:,.2f}'.format(iter_per_sec)))
table.add_row(('Optimizer', artifact.optimizer))
table.add_row(('Validation set size', artifact.valid_size))
with doc.create(Subsection('Training loss')):
with doc.create(Tabularx('ll')) as table:
table.add_row(('Initial loss',
'{:+.6e}'.format(train_loss_initial)))
table.add_row(
('Final loss', '{:+.6e}'.format(train_loss_final)))
table.add_row(
('Loss change / s',
'{:+.6e}'.format(train_loss_change_per_sec)))
table.add_row(
('Loss change / iteration',
'{:+.6e}'.format(train_loss_change_per_iter)))
table.add_row(
('Loss change / trace',
'{:+.6e}'.format(train_loss_change_per_trace)))
with doc.create(Subsection('Validation loss')):
with doc.create(Tabularx('ll')) as table:
table.add_row(('Initial loss',
'{:+.6e}'.format(valid_loss_initial)))
table.add_row(
('Final loss', '{:+.6e}'.format(valid_loss_final)))
table.add_row(
('Loss change / s',
'{:+.6e}'.format(valid_loss_change_per_sec)))
table.add_row(
('Loss change / iteration',
'{:+.6e}'.format(valid_loss_change_per_iter)))
table.add_row(
('Loss change / trace',
'{:+.6e}'.format(valid_loss_change_per_trace)))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 6))
ax = plt.subplot(111)
ax.plot(artifact.train_history_trace,
artifact.train_history_loss,
label='Training')
ax.plot(artifact.valid_history_trace,
artifact.valid_history_loss,
label='Validation')
ax.legend()
plt.xlabel('Training traces')
plt.ylabel('Loss')
plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Loss plot.')
doc.append(NoEscape(r'\newpage'))
with doc.create(Section('Traces')):
with doc.create(Tabularx('ll')) as table:
table.add_row(('Total training traces',
'{:,}'.format(artifact.total_traces)))
with doc.create(Subsection('Distributions encountered')):
with doc.create(Tabularx('ll')) as table:
num_distributions = len(
artifact.one_hot_distribution.keys())
table.add_row(
('Number of distributions', num_distributions))
table.add_empty_row()
for distribution in artifact.one_hot_distribution.keys(
):
table.add_row((distribution, ''))
with doc.create(Subsection('Unique addresses encountered')):
with doc.create(Tabularx('lX')) as table:
num_addresses = len(artifact.one_hot_address.keys())
table.add_row(('Number of addresses', num_addresses))
address_collisions = max(
0, num_addresses - artifact.one_hot_address_dim)
table.add_row(
('Address collisions', address_collisions))
table.add_empty_row()
doc.append('\n')
with doc.create(LongTable('llp{16cm}')) as table:
# table.add_empty_row()
table.add_row('Count', 'ID', 'Unique address')
table.add_hline()
address_to_abbrev = {}
abbrev_to_address = {}
abbrev_i = 0
sorted_addresses = sorted(
artifact.address_histogram.items(),
key=lambda x: x[1],
reverse=True)
plt_addresses = []
plt_counts = []
address_to_count = {}
address_count_total = 0
for address, count in sorted_addresses:
abbrev_i += 1
abbrev = 'A' + str(abbrev_i)
address_to_abbrev[address] = abbrev
abbrev_to_address[abbrev] = address
plt_addresses.append(abbrev)
plt_counts.append(count)
address_to_count[abbrev] = count
address_count_total += count
table.add_row(('{:,}'.format(count), abbrev,
FootnoteText(address)))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
plt_x = range(len(plt_addresses))
ax.bar(plt_x, plt_counts)
plt.xticks(plt_x, plt_addresses)
plt.xlabel('Unique address ID')
plt.ylabel('Count')
plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Histogram of address hits.')
with doc.create(Subsection('Lengths')):
with doc.create(Tabularx('ll')) as table:
table.add_row(
('Min trace length',
'{:,}'.format(artifact.trace_length_min)))
table.add_row(
('Max trace length',
'{:,}'.format(artifact.trace_length_max)))
s = 0
total_count = 0
for trace_length in artifact.trace_length_histogram:
count = artifact.trace_length_histogram[
trace_length]
s += trace_length * count
total_count += count
trace_length_mean = s / total_count
table.add_row(('Mean trace length',
'{:.2f}'.format(trace_length_mean)))
with doc.create(Figure(position='H')) as plot:
plt_lengths = [
i for i in range(0, artifact.trace_length_max + 1)
]
plt_counts = [
artifact.trace_length_histogram[i]
if i in artifact.trace_length_histogram else 0
for i in range(0, artifact.trace_length_max + 1)
]
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
ax.bar(plt_lengths, plt_counts)
plt.xlabel('Length')
plt.ylabel('Count')
# plt.yscale('log')
plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption(
'Histogram of trace lengths (of all traces used during training).'
)
with doc.create(Subsection('Unique traces encountered')):
with doc.create(Tabularx('ll')) as table:
table.add_row(
('Unique traces encountered', '{:,}'.format(
len(artifact.trace_examples_histogram))))
table.add_row(
('Unique trace memory capacity',
'{:,}'.format(artifact.trace_examples_limit)))
table.add_row(
('Unique traces rendered in detail', '{:,}'.format(
min(len(artifact.trace_examples_histogram),
opt.maxTraces))))
doc.append('\n')
with doc.create(LongTable('lllp{16cm}')) as table:
# table.add_empty_row()
table.add_row('Count', 'ID', 'Len.', 'Unique trace')
table.add_hline()
trace_to_abbrev = {}
abbrev_to_trace = {}
abbrev_to_addresses = {}
abbrev_i = 0
sorted_traces = sorted(
artifact.trace_examples_histogram.items(),
key=lambda x: x[1],
reverse=True)
plt_traces = []
plt_counts = []
trace_to_count = {}
trace_count_total = 0
for trace, count in sorted_traces:
abbrev_i += 1
abbrev = 'T' + str(abbrev_i)
trace_to_abbrev[trace] = abbrev
abbrev_to_trace[abbrev] = trace
abbrev_to_addresses[abbrev] = list(
map(lambda x: address_to_abbrev[x],
artifact.trace_examples_addresses[trace]))
trace_addresses = abbrev_to_addresses[abbrev]
trace_addresses_repetitions = util.pack_repetitions(
trace_addresses)
plt_traces.append(abbrev)
plt_counts.append(count)
trace_to_count[trace] = count
trace_count_total += count
length = len(
artifact.trace_examples_addresses[trace])
table.add_row(
('{:,}'.format(count), abbrev,
'{:,}'.format(length),
FootnoteText('-'.join([
a + 'x' + str(i) if i > 1 else a
for a, i in trace_addresses_repetitions
]))))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
plt_x = range(len(plt_traces))
ax.bar(plt_x, plt_counts)
plt.xticks(plt_x, plt_traces)
plt.xlabel('Unique trace ID')
plt.ylabel('Count')
plt.grid()
fig.tight_layout()
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption('Histogram of unique traces.')
with doc.create(Figure(position='H')) as plot:
master_trace_pairs = {}
transition_count_total = 0
for trace, count in sorted_traces:
ta = abbrev_to_addresses[trace_to_abbrev[trace]]
for left, right in zip(ta, ta[1:]):
if (left, right) in master_trace_pairs:
master_trace_pairs[(left, right)] += count
else:
master_trace_pairs[(left, right)] = count
transition_count_total += count
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
master_graph = pydotplus.graphviz.Dot(
graph_type='digraph', rankdir='LR')
for p, w in master_trace_pairs.items():
nodes = master_graph.get_node(p[0])
if len(nodes) > 0:
n0 = nodes[0]
else:
n0 = pydotplus.Node(p[0])
master_graph.add_node(n0)
nodes = master_graph.get_node(p[1])
if len(nodes) > 0:
n1 = nodes[0]
else:
n1 = pydotplus.Node(p[1])
master_graph.add_node(n1)
master_graph.add_edge(
pydotplus.Edge(n0, n1, weight=w))
for node in master_graph.get_nodes():
node.set_color('gray')
node.set_fontcolor('gray')
for edge in master_graph.get_edges():
edge.set_color('gray')
master_graph_annotated = pydotplus.graphviz.graph_from_dot_data(
master_graph.to_string())
for node in master_graph_annotated.get_nodes():
color = util.rgb_to_hex(
util.rgb_blend(
(1, 1, 1), (1, 0, 0),
address_to_count[node.obj_dict['name']] /
address_count_total))
node.set_style('filled')
node.set_fillcolor(color)
node.set_color('black')
node.set_fontcolor('black')
for edge in master_graph_annotated.get_edges():
(left, right) = edge.obj_dict['points']
count = master_trace_pairs[(left, right)]
edge.set_label(count)
color = util.rgb_to_hex(
(1.5 * (count / transition_count_total), 0, 0))
edge.set_color(color)
png_str = master_graph_annotated.create_png(
prog=['dot', '-Gsize=15', '-Gdpi=600'])
bio = BytesIO()
bio.write(png_str)
bio.seek(0)
img = np.asarray(mpimg.imread(bio))
plt.imshow(util.crop_image(img),
interpolation='bilinear')
plt.axis('off')
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption(
'Succession of unique address IDs (accumulated over all traces).'
)
for trace, count in sorted_traces[:opt.maxTraces]:
trace = trace_to_abbrev[trace]
doc.append(NoEscape(r'\newpage'))
with doc.create(Subsubsection('Unique trace ' +
trace)):
sys.stdout.write(
'Rendering unique trace {0}... \r'
.format(trace))
sys.stdout.flush()
addresses = len(address_to_abbrev)
trace_addresses = abbrev_to_addresses[trace]
with doc.create(Tabularx('ll')) as table:
table.add_row(
FootnoteText('Count'),
FootnoteText('{:,}'.format(count)))
table.add_row(
FootnoteText('Length'),
FootnoteText('{:,}'.format(
len(trace_addresses))))
doc.append('\n')
im = np.zeros((addresses, len(trace_addresses)))
for i in range(len(trace_addresses)):
address = trace_addresses[i]
address_i = plt_addresses.index(address)
im[address_i, i] = 1
truncate = 100
for col_start in range(0, len(trace_addresses),
truncate):
col_end = min(col_start + truncate,
len(trace_addresses))
with doc.create(Figure(position='H')) as plot:
fig = plt.figure(figsize=(20 * (
(col_end + 4 - col_start) / truncate),
4))
ax = plt.subplot(111)
# ax.imshow(im,cmap=plt.get_cmap('Greys'))
sns.heatmap(im[:, col_start:col_end],
cbar=False,
linecolor='lightgray',
linewidths=.5,
cmap='Greys',
yticklabels=plt_addresses,
xticklabels=np.arange(
col_start, col_end))
plt.yticks(rotation=0)
fig.tight_layout()
plot.add_plot(
width=NoEscape(r'{0}\textwidth'.format(
(col_end + 4 - col_start) /
truncate)),
placement=NoEscape(r'\raggedright'))
with doc.create(Figure(position='H')) as plot:
pairs = {}
for left, right in zip(trace_addresses,
trace_addresses[1:]):
if (left, right) in pairs:
pairs[(left, right)] += 1
else:
pairs[(left, right)] = 1
fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(111)
graph = pydotplus.graphviz.graph_from_dot_data(
master_graph.to_string())
trace_address_to_count = {}
for address in trace_addresses:
if address in trace_address_to_count:
trace_address_to_count[address] += 1
else:
trace_address_to_count[address] = 1
for p, w in pairs.items():
left_node = graph.get_node(p[0])[0]
right_node = graph.get_node(p[1])[0]
edge = graph.get_edge(p[0], p[1])[0]
color = util.rgb_to_hex(
util.rgb_blend(
(1, 1, 1), (1, 0, 0),
trace_address_to_count[p[0]] /
len(trace_addresses)))
left_node.set_style('filled')
left_node.set_fillcolor(color)
left_node.set_color('black')
left_node.set_fontcolor('black')
color = util.rgb_to_hex(
util.rgb_blend(
(1, 1, 1), (1, 0, 0),
trace_address_to_count[p[0]] /
len(trace_addresses)))
right_node.set_style('filled')
right_node.set_fillcolor(color)
right_node.set_color('black')
right_node.set_fontcolor('black')
(left, right) = edge.obj_dict['points']
edge.set_label(w)
color = util.rgb_to_hex(
(1.5 * (w / len(trace_addresses)), 0,
0))
edge.set_color(color)
png_str = graph.create_png(
prog=['dot', '-Gsize=30', '-Gdpi=600'])
bio = BytesIO()
bio.write(png_str)
bio.seek(0)
img = np.asarray(mpimg.imread(bio))
plt.imshow(util.crop_image(img),
interpolation='bilinear')
plt.axis('off')
plot.add_plot(width=NoEscape(r'\textwidth'))
plot.add_caption(
'Succession of unique address IDs (for one trace of type '
+ trace + ').')
with doc.create(Tabularx('lp{16cm}')) as table:
trace_addresses_repetitions = util.pack_repetitions(
trace_addresses)
table.add_row(
FootnoteText('Trace'),
FootnoteText('-'.join([
a + 'x' + str(i) if i > 1 else a
for a, i in trace_addresses_repetitions
])))
doc.generate_pdf(opt.saveReport, clean_tex=False)
sys.stdout.write(
' \r'
)
sys.stdout.flush()
except KeyboardInterrupt:
util.logger.log('Stopped')
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(0)
def rodape(doc, frase):
doc.append(Foot('R', data=FootnoteText(frase)))
doc.append(Foot('L', data=Command('thepage')))
doc.append(Foot('C'))