def execute(source_path, output_path, device_id):
"""
Execute the parser.
Args:
source_path (str): the source file path.
output_path (str): the output file path.
device_id (str): the device id.
"""
col_names = ["node_name", "start_time", "end_time", "queue_size"]
minddata_aicpu_source_path = get_file_join_name(
input_path=source_path, file_name='DATA_PREPROCESS.dev.AICPUMI')
if not minddata_aicpu_source_path:
minddata_aicpu_source_path = get_file_join_name(
input_path=os.path.join(source_path, "data"),
file_name='DATA_PREPROCESS.dev.AICPUMI')
if not minddata_aicpu_source_path:
return
minddata_aicpu_output_path = os.path.join(
output_path, "minddata_aicpu_" + device_id + ".txt")
minddata_aicpu_data = MinddataParser.parse_minddata_aicpu_data(
minddata_aicpu_source_path)
if minddata_aicpu_data:
fwrite_format(minddata_aicpu_output_path,
tabulate(minddata_aicpu_data,
col_names,
tablefmt='simple'),
is_start=True)
def _analyser_op_info(self):
"""Analyse the operator information."""
integrator = Integrator(self._output_path, self._dev_id)
integrator.integrate()
aicore_type_result = self._query_op_type_info()
detail_file_path = os.path.join(
self._output_path,
'output_op_compute_time_detail_{}.txt'.format(self._dev_id)
)
fwrite_format(detail_file_path, data_source='title:op compute time')
display_names = [
'optype_name', 'compute_time(ms, per-step)',
'called_times(per-step)', 'percent'
]
data_source = tabulate(aicore_type_result, display_names)
fwrite_format(detail_file_path, data_source=data_source, is_print=True)
if self._detail:
op_type_order = [item[0] for item in aicore_type_result]
aicore_detail_result = self._query_op_detail_info(op_type_order)
fwrite_format(detail_file_path, data_source='', is_print=True)
fwrite_format(detail_file_path, data_source='Detail:', is_print=True)
data_source = tabulate(
aicore_detail_result.get('object'),
aicore_detail_result.get('col_name')
)
fwrite_format(detail_file_path, data_source=data_source, is_print=True)
def execute(self):
"""Execute the parser, get result data, and write it to the output file."""
if not os.path.exists(self._source_file_name):
logger.info("Did not find the aicpu profiling source file")
return
with open(self._source_file_name, 'rb') as ai_cpu_data:
ai_cpu_str = str(ai_cpu_data.read().replace(b'\n\x00', b' ___ ')
.replace(b'\x00', b' ___ '))[2:-1]
ai_cpu_lines = ai_cpu_str.split(" ___ ")
node_list = list()
ai_cpu_total_time_summary = 0
# node serial number
serial_number = 1
for i in range(len(ai_cpu_lines)-1):
node_line = ai_cpu_lines[i]
thread_line = ai_cpu_lines[i+1]
if "Node" in node_line and "Thread" in thread_line:
# get the node data from node_line
node_name = node_line.split(',')[0].split(':')[-1]
run_v2_start = node_line.split(',')[1].split(':')[-1]
compute_start = node_line.split(',')[2].split(':')[-1]
mercy_start = node_line.split(',')[3].split(':')[-1]
mercy_end = node_line.split(',')[4].split(':')[-1]
run_v2_end = node_line.split(',')[5].split(':')[-1]
# get total_time and dispatch_time from thread line
total_time = thread_line.split(',')[-1].split('=')[-1].split()[0]
dispatch_time = thread_line.split(',')[-2].split('=')[-1].split()[0]
node_data = [serial_number, node_name, total_time, dispatch_time, run_v2_start, compute_start,
mercy_start, mercy_end, run_v2_end]
node_list.append(node_data)
# calculate the total time
ai_cpu_total_time_summary += int(total_time)
# increase node serial number
serial_number += 1
elif "Node" in node_line and "Thread" not in thread_line:
node_name = node_line.split(',')[0].split(':')[-1]
logger.warning("The node:%s cannot find thread data", node_name)
node_list.append(["AI CPU Total Time(us):", ai_cpu_total_time_summary])
if node_list:
fwrite_format(self._output_filename, data_source=_dst_file_title, is_print=True,
is_start=True)
fwrite_format(self._output_filename,
data_source=tabulate(node_list, _dst_file_column_title,
tablefmt='simple'),
is_start=True, is_print=True)
def _write_op_time_into_file(self, result_data):
"""
Write the metadata of operators into the file, including
op name, average time, and stream id.
Args:
result_data (str): The metadata to be written into the file.
'op_name_1', 'avg_time_1', 'stream_id_1',
'op_name_2', 'avg_time_2', 'stream_id_2',
...
"""
fwrite_format(self._output_filename,
data_source=self._dst_file_title,
is_start=True)
fwrite_format(self._output_filename,
data_source=self._dst_file_column_title)
fwrite_format(self._output_filename, data_source=result_data)
def execute(self):
"""Execute the parser, compute all op, get op time, and write it to the output file."""
result_data = []
tmp_result_data = []
op_map_list = self._get_op_task_id_map()
cur_index = 0
length = len(op_map_list)
while cur_index < length:
if cur_index + 1 == length:
break
op_start = op_map_list[cur_index]
op_end = op_map_list[cur_index + 1]
if op_start[1] == "Start" and op_end[1] == "End"\
and op_start[0] == op_end[0]:
# op_name, task_id, cycle counter, stream_id
tmp_result_data.append([
op_start[0], op_start[2],
int(op_end[3]) - int(op_start[3]), op_start[4]
])
cur_index += 2
else:
cur_index += 1
op_name_time_dict = {}
op_name_steamid_dict = {}
op_name_count_dict = {}
op_name_task_dict = {}
# compute all op
for item in tmp_result_data:
if item[0] in op_name_time_dict.keys():
op_name_time_dict[item[0]] += float(
item[2]) / 1e5 # cycle counter/1*10^5 ms
if item[1] == op_name_task_dict[item[0]]:
op_name_count_dict[item[0]] += 1
else:
op_name_time_dict[item[0]] = float(item[2]) / 1e5
op_name_steamid_dict[item[0]] = item[-1]
op_name_task_dict[item[0]] = item[1]
op_name_count_dict[item[0]] = 1
for op_name, time in op_name_time_dict.items():
if op_name in op_name_steamid_dict.keys():
stream_id = op_name_steamid_dict[op_name]
avg_time = time / op_name_count_dict[op_name]
result_data.append([op_name, avg_time, stream_id])
result_data.sort(key=lambda x: x[0])
total_time = 0
for item in result_data:
total_time += item[1]
result_data.append(["total op", total_time, 0])
fwrite_format(self._output_filename,
data_source=self._dst_file_title,
is_start=True)
fwrite_format(self._output_filename,
data_source=tabulate(result_data,
self._dst_file_column_title,
tablefmt='simple'))
def execute(self):
"""
Execute the parser, get result data, and write it to the output file.
Returns:
bool, whether succeed to analyse hwts log.
"""
content_format = ['QIIIIIIIIIIII', 'QIIQIIIIIIII', 'IIIIQIIIIIIII']
log_type = [
'Start of task', 'End of task', 'Start of block', 'End of block',
'Block PMU'
]
result_data = ""
with open(self._source_flie_name, 'rb') as hwts_data:
while True:
line = hwts_data.read(64)
if line:
if not line.strip():
continue
else:
break
byte_first_four = struct.unpack('BBHHH', line[0:8])
byte_first = bin(byte_first_four[0]).replace('0b', '').zfill(8)
ms_type = byte_first[-3:]
is_warn_res0_ov = byte_first[4]
cnt = int(byte_first[0:4], 2)
core_id = byte_first_four[1]
blk_id, task_id = byte_first_four[3], byte_first_four[4]
if ms_type in ['000', '001', '010']: # log type 0,1,2
result = struct.unpack(content_format[0], line[8:])
syscnt = result[0]
stream_id = result[1]
elif ms_type == '011': # log type 3
result = struct.unpack(content_format[1], line[8:])
syscnt = result[0]
stream_id = result[1]
elif ms_type == '100': # log type 4
result = struct.unpack(content_format[2], line[8:])
stream_id = result[2]
if is_warn_res0_ov == '0':
syscnt = result[4]
else:
syscnt = None
else:
logger.info("Profiling: invalid hwts log record type %s",
ms_type)
continue
if int(task_id) < 25000:
task_id = str(stream_id) + "_" + str(task_id)
result_data += ("%-14s %-4s %-8s %-9s %-8s %-15s %s\n" %
(log_type[int(ms_type, 2)], cnt, core_id,
blk_id, task_id, syscnt, stream_id))
fwrite_format(self._output_filename,
data_source=self._dst_file_title,
is_start=True)
fwrite_format(self._output_filename,
data_source=self._dst_file_column_title)
fwrite_format(self._output_filename, data_source=result_data)
return True