def generate_report(graph):
print(f'Generating training report... {graph.report}')
report = graph.session.run(graph.report)
compilation_report = ipu_utils.extract_compile_reports(report)
execution_report = ipu_utils.extract_execute_reports(report)
with open("report.txt", "w") as f:
f.write(ipu_utils.extract_all_strings_from_event_trace(report))
with open("compilation_report.json", "w") as f:
json.dump(compilation_report, f)
with open("execution_report.json", "w") as f:
json.dump(execution_report, f)
print('Reports saved to .')
def extract_runtimes_from_report(raw_report, opts, start_time=0, display=True):
"""Returns timing information from IpuTraceEvent
report -- Array of text encoded IpuTraceEvent
"""
if len(raw_report) is 0:
return
# Timings from tf xla event timestamps
from tensorflow.compiler.plugin.poplar.driver.trace_pb2 import IpuTraceEvent
events = list(map(IpuTraceEvent.FromString, raw_report))
first = start_time == 0
if first:
start_time = events[0].timestamp
events = events[1:]
# Retrieve IpuEvents, poplar report and cycles
if display:
evt_str = "\nIPU Timings\n"
compile_started = False
for evt in events:
extra_str = ""
if evt.type == IpuTraceEvent.COMPILE_BEGIN:
compile_started = True
evt_name = "Compile start"
elif evt.type == IpuTraceEvent.COMPILE_END:
if compile_started:
compile_started = False
evt_name = "Compile"
else:
continue
elif evt.type == IpuTraceEvent.HOST_TO_DEVICE_TRANSFER:
evt_name = "Host->Device"
extra_str = "\n Tensors:"
transfered_tensors = json.loads(
evt.data_transfer.data_transfer.decode('utf-8'))
for t in transfered_tensors["tensors"]:
extra_str += "\n handle: {:>6}, size: {}".format(
t["name"], t["size"])
extra_str += "\n Total_size: {}".format(
transfered_tensors["total_size"])
elif evt.type == IpuTraceEvent.DEVICE_TO_HOST_TRANSFER:
evt_name = "Device->Host"
extra_str = "\n Tensors:"
transfered_tensors = json.loads(
evt.data_transfer.data_transfer.decode('utf-8'))
for t in transfered_tensors["tensors"]:
extra_str += "\n handle: {:>6}, size: {}".format(
t["name"], t["size"])
extra_str += "\n Total_size: {}".format(
transfered_tensors["total_size"])
elif evt.type == IpuTraceEvent.LOAD_ENGINE:
evt_name = "Load engine"
elif evt.type == IpuTraceEvent.EXECUTE:
evt_name = "Execute"
else:
evt_name = "Unknown event"
evt_str += "{:<15s}: {:<8.3g} s {}\n".format(
evt_name, (evt.timestamp - start_time), extra_str)
start_time = evt.timestamp
print(evt_str)
# Write Report to file
if first:
with open("graph.json", "w") as f:
graph_report = utils.extract_compile_reports(raw_report)[0][1]
f.write(graph_report)
with open("execution.json", "w") as f:
execution_report = utils.extract_execute_reports(raw_report)[0][1]
f.write(execution_report)
print("\nWritten to file: graph.json, execution.json")
if opts.tile_activity_report:
generate_tile_activity(raw_report)
return {"graph": graph_report, "execution": execution_report}, start_time
# Executing the trace op flushes everything it has recorded up to that point and outputs it.
# Therefore if the variable initializer graph runs on IPU, we can prevent it from being included
# in the reports with a "session.run(trace)" after it has been run (line above).
if args.no_var_init_profiling and not args.var_init_on_cpu:
session.run(trace)
# Create dummy data
training_data = np.zeros([1, NUM_UNITS_IN])
# Run the main graph
session.run(logits, feed_dict={x: training_data})
# Execute the event trace op: the result is a list of trace event serialized protobufs.
raw_report = session.run(trace)
# These objects can be converted to strings with utility functions, as shown below.
ext = ".json" if args.json_report else ".txt"
if args.split_reports:
compile_reports = utils.extract_compile_reports(raw_report)
execution_reports = utils.extract_execute_reports(raw_report)
# These are lists, as long as the number of graphs profiled, except that the
# execution_reports list will be empty if execution profiling is not enabled.
# You could save only the last (i.e. relative to the main graph); in this case we save everything.
with open("compile" + ext, "w", encoding="utf-8") as f:
for report in compile_reports:
# Each element of the list is a tuple of 2 elements:
# the first is a string representing an auto-generated name of the xla graph
# the second is a string containing the actual report relative to the graph
xla_name, report_string = report
f.write(xla_name + "\n")
f.write(report_string + "\n")
if len(execution_reports) > 0:
with open("execution" + ext, "w", encoding="utf-8") as f:
for report in execution_reports: