def _get_pipeline_path(self, source_dir): """ Get the minddata pipeline file path. Args: source_dir (str): The minddata pipeline source dir. Returns: str, the minddata pipeline file path. """ pipeline_path = os.path.join( source_dir, self._raw_pipeline_file_name.format(self._device_id)) try: pipeline_path = validate_and_normalize_path(pipeline_path) except RuntimeError: logger.warning('Minddata pipeline file is invalid.') raise ProfilerPathErrorException( 'Minddata pipeline file is invalid.') if not os.path.isfile(pipeline_path): logger.warning('The minddata pipeline file <%s> not found.', pipeline_path) raise ProfilerFileNotFoundException(pipeline_path) return pipeline_path
def _get_op_task_id_map(self): """ Read hwts data file, get the task time info. Returns: list: all hwts task time info. """ op_map_result = [] hwts_list = [] if not os.path.exists(self._hwts_output_file): logger.error('The hwts output file does not exist.') raise ProfilerFileNotFoundException('hwts output file') with open(self._hwts_output_file, 'r') as data_file: lines = data_file.readlines() for line in lines: if line.startswith("Start of task") or line.startswith("End of task"): line_split = line.split() container = HWTSContainer(line_split) hwts_list.append(container) # hwts op map by taskId for hwts in hwts_list: if hwts.task_id in self._op_task_info.keys(): hwts.op_name = self._op_task_info[hwts.task_id] op_map_result.append(hwts) return op_map_result
def _get_and_validate_path(self, file_name): """Generate op or activity file path from file name, and validate this path.""" file_path = os.path.join(self._profiling_dir, file_name.format(self._device_id)) file_path = validate_and_normalize_path(file_path) if not os.path.exists(file_path): logger.error(f"Failed to find parsed timeline file {file_path}.") raise ProfilerFileNotFoundException('parsed timeline file') return file_path
def _get_file_path(self): """Get the proto file path.""" file_path = os.path.join(self._profiling_dir, self._proto_file_path.format(self._device_id)) file_path = validate_and_normalize_path(file_path) if not os.path.exists(file_path): msg = 'The memory file does not exist!' logger.error(msg) raise ProfilerFileNotFoundException(msg=msg) return file_path
def _get_file_path(self): """Get the proto file path.""" file_path = os.path.join(self._profiling_dir, self._proto_file_path.format(self._device_id)) file_path = validate_and_normalize_path(file_path) if not os.path.exists(file_path): logger.warning( 'The memory file does not exist! Please ignore the warning ' 'if you are running heterogeneous training.') raise ProfilerFileNotFoundException(msg=file_path) return file_path
def _aicore_trace_data_load(self): """Load data according to the parsed AICORE operator types file.""" file_path = query_latest_trace_time_file(self._profiling_dir, int(self._device_id)) if not file_path: logger.error("Failed to find parsed trace time file.") raise ProfilerFileNotFoundException('parsed step trace time file') file_path = validate_and_normalize_path(file_path) with open(file_path, 'r') as handle: csv_reader = csv.reader(handle) self.__column__ = next(csv_reader) self._aicore_trace_data = list(csv_reader) self._size = len(self._aicore_trace_data) - 1 self._load_point_info()
def _search_file(self, profiling_id, device_id): """ Search all framework files in raw profiling path. Args: profiling_id (str): The profiling ID. device_id (str): The device ID. Raises: ProfilerFileNotFoundException: If the framework files are not found. """ # first search in the JOB dir, and if not, search in the sub directory # in the JOB self._search_file_from_job_path(device_id, search_in_sub_path=False) if self._backend_type is None: self._search_file_from_job_path(device_id, search_in_sub_path=True) self._search_file_from_data_path(profiling_id, device_id) if self._backend_type is None: raise ProfilerFileNotFoundException('Framework') self._framework_path['graph'].sort() self._framework_path['task'].sort()
def _load_timeline_data(self): """Load timeline data from file.""" file_path = os.path.join( self._profiling_dir, self._output_timeline_data_file_path.format(self._device_id)) file_path = validate_and_normalize_path(file_path) if not os.path.exists(file_path): logger.error("Failed to find parsed timeline file.") raise ProfilerFileNotFoundException('parsed timeline file') timeline_list = [] try: with open(file_path, 'r') as f_obj: for line in f_obj: if not line.startswith('op_name'): line_list = line.strip('\n').split(',') timeline_list.append(line_list) except (IOError, OSError) as err: logger.error( 'Error occurred when read timeline intermediate file: %s', err) raise ProfilerIOException return timeline_list