def _parse(self, source_file):
"""Parse source step trace files."""
log.info("Start to parse step trace file.")
fp_start, bp_end, iter_end, iter_start = 0, 1, 2, 3
reduce_start = 4
start_time, end_time = 0, 1
STEP_TRACE_POINT_COUNT = 3
source_file = validate_and_normalize_path(source_file)
try:
with open(source_file, 'r') as f:
lines = f.readlines()
if len(lines) < STEP_TRACE_POINT_COUNT:
raise ProfilerRawFileException(
f"Failed to parse {source_file} file. The FP_POINT/BP_POINT/ITER_END_POINT "
f"do not recognized correctly. Try to set the environment variable'PROFILING_FP_START' "
f"and 'PROFILING_BP_END' to solve this problem. For example, "
f"'export PROFILING_FP_START=Default/xxx/Conv2d-op1' ")
step_trace_info_all = [
line.strip().split()[1:] for line in lines
]
num_of_step = len(step_trace_info_all[0])
for step_trace_point in step_trace_info_all:
if len(step_trace_point) != num_of_step:
raise ProfilerRawFileException(
f"Failed to parse {source_file} file. Due to the profiled "
f"step_num of FP/BP/ITER_END Point are not equal")
iter_start_info = [step_trace_info_all[fp_start][0]] + \
step_trace_info_all[iter_end][:num_of_step]
step_trace_info_all.insert(iter_start, iter_start_info)
except (IOError, OSError) as err:
log.warning(f'Failed to read {source_file}', err)
raise ProfilerIOException
for step_num in range(num_of_step):
step_trace = {
'start':
int(step_trace_info_all[iter_start][step_num].split(',')
[start_time]),
'fp':
int(step_trace_info_all[fp_start][step_num].split(',')
[start_time]),
'bp':
int(step_trace_info_all[bp_end][step_num].split(',')
[end_time]),
'end':
int(step_trace_info_all[iter_end][step_num].split(',')
[end_time]),
'reduce': {}
}
num_of_step_point = len(step_trace_info_all)
if num_of_step_point > reduce_start:
reduce_info = {}
reduce_time_info = []
for reduce_idx in range(reduce_start, num_of_step_point):
cur_reduce_time = step_trace_info_all[reduce_idx][step_num]
reduce_time_info += cur_reduce_time.split(',')
reduce_info['ops'] = reduce_time_info
step_trace['reduce'] = reduce_info
self._record_trace_event(step_trace)
self._record_average_info()
log.info("Finish to parse step trace file.")