Example #1
0
def tool_proto_to_tool_data(tool_proto, tool, tqx):
    """Converts the serialized tool proto to tool data string.

  Args:
    tool_proto: A serialized XSpace proto string.
    tool: A string of tool name.
    tqx: Gviz output format.

  Returns:
    Returns a string of tool data.
  """
    data = ''
    if tool == 'trace_viewer':
        data = process_raw_trace(tool_proto)
    elif tool == 'tensorflow_stats':
        if tqx == 'out:csv;':
            data = tf_stats_proto_to_gviz.to_csv(tool_proto)
        else:
            data = tf_stats_proto_to_gviz.to_json(tool_proto)
    elif tool == 'overview_page@':
        data = overview_page_proto_to_gviz.to_json(tool_proto)
    elif tool == 'input_pipeline_analyzer@':
        data = input_pipeline_proto_to_gviz.to_json(tool_proto)
    elif tool == 'kernel_stats':
        if tqx == 'out:csv;':
            data = kernel_stats_proto_to_gviz.to_csv(tool_proto)
        else:
            data = kernel_stats_proto_to_gviz.to_json(tool_proto)
    else:
        logger.warning('%s is not a known tool', tool)
    return data
Example #2
0
def xspace_to_tool_data(xspace, tool, tqx):
    """Converts the serialized XSpace proto to tool data string.

  Args:
    xspace: A serialized XSpace proto string.
    tool: A string of tool name.
    tqx: Gviz output format.

  Returns:
    Returns a string of tool data.
  """
    assert tool[-1] == '^'
    tool = tool[:-1]  # xplane tool name ends with '^'
    data = ''
    if tool == 'trace_viewer':
        data = process_raw_trace(
            _pywrap_profiler.xspace_to_trace_events(xspace))
    elif tool == 'overview_page':
        data = overview_page_proto_to_gviz.to_json(
            _pywrap_profiler.xspace_to_overview_page(xspace))
    elif tool == 'input_pipeline_analyzer':
        data = input_pipeline_proto_to_gviz.to_json(
            _pywrap_profiler.xspace_to_input_pipeline(xspace))
    elif tool == 'tensorflow_stats':
        data = _pywrap_profiler.xspace_to_tf_stats(xspace)
        if tqx == 'out:csv;':
            data = tf_stats_proto_to_gviz.to_csv(data)
        else:
            data = tf_stats_proto_to_gviz.to_json(data)
    elif tool == 'kernel_stats':
        data = _pywrap_profiler.xspace_to_kernel_stats(xspace)
        if tqx == 'out:csv;':
            data = kernel_stats_proto_to_gviz.to_csv(data)
        else:
            data = kernel_stats_proto_to_gviz.to_json(data)
    elif tool == 'memory_profile':
        data = _pywrap_profiler.xspace_to_memory_profile(xspace)
    else:
        logger.warning('%s is not a known xplane tool', tool)
    return data
def xspace_to_tool_data(xspace_paths, tool, tqx):
    """Converts XSpace to tool data string.

  Args:
    xspace_paths: A list of XSpace paths.
    tool: A string of tool name.
    tqx: Gviz output format.

  Returns:
    Returns a string of tool data.
  """
    assert tool[-1] == '^'
    tool = tool[:-1]  # xplane tool name ends with '^'
    data = None
    if tool == 'trace_viewer':
        # Trace viewer handles one host at a time.
        assert len(xspace_paths) == 1
        raw_data, success = _pywrap_profiler.xspace_to_tools_data(
            xspace_paths, tool)
        if success:
            data = process_raw_trace(raw_data)
    elif tool == 'overview_page':
        raw_data, success = _pywrap_profiler.xspace_to_tools_data(
            xspace_paths, tool)
        if success:
            data = overview_page_proto_to_gviz.to_json(raw_data)
    elif tool == 'input_pipeline_analyzer':
        raw_data, success = _pywrap_profiler.xspace_to_tools_data(
            xspace_paths, tool)
        if success:
            data = input_pipeline_proto_to_gviz.to_json(raw_data)
    elif tool == 'tensorflow_stats':
        raw_data, success = _pywrap_profiler.xspace_to_tools_data(
            xspace_paths, tool)
        if success:
            if tqx == 'out:csv':
                data = tf_stats_proto_to_gviz.to_csv(raw_data)
            else:
                data = tf_stats_proto_to_gviz.to_json(raw_data)
    elif tool == 'kernel_stats':
        raw_data, success = _pywrap_profiler.xspace_to_tools_data(
            xspace_paths, tool)
        if success:
            if tqx == 'out:csv;':
                data = kernel_stats_proto_to_gviz.to_csv(raw_data)
            else:
                data = kernel_stats_proto_to_gviz.to_json(raw_data)
    elif tool == 'memory_profile':
        # Memory profile handles one host at a time.
        assert len(xspace_paths) == 1
        raw_data, success = _pywrap_profiler.xspace_to_tools_data(
            xspace_paths, tool)
        if success:
            data = raw_data
    elif tool == 'pod_viewer':
        raw_data, success = _pywrap_profiler.xspace_to_tools_data(
            xspace_paths, tool)
        if success:
            data = raw_data
    else:
        logger.warning('%s is not a known xplane tool', tool)
    return data
Example #4
0
    def data_impl(self, request):
        """Retrieves and processes the tool data for a run and a host.

    Args:
      request: XMLHttpRequest

    Returns:
      A string that can be served to the frontend tool or None if tool,
        run or host is invalid.
    """
        run = request.args.get('run')
        tool = request.args.get('tag')
        host = request.args.get('host')
        tqx = request.args.get('tqx')
        run_dir = self._run_dir(run)
        # Profile plugin "run" is the last component of run dir.
        profile_run = os.path.basename(run_dir)

        if tool not in TOOLS:
            return None, None

        self.start_grpc_stub_if_necessary()
        if tool == 'trace_viewer@' and self.stub is not None:
            # Streaming trace viewer needs profiler_analysis service, which is only
            # supported in Cloud TPU. This code is unused when data was produced by
            # open-source TensorFlow. Only import the library when needed.
            # pylint: disable=g-import-not-at-top
            # pylint: disable=g-direct-tensorflow-import
            from tensorflow.core.profiler import profiler_analysis_pb2
            # pylint: enable=g-import-not-at-top
            # pylint: enable=g-direct-tensorflow-import
            grpc_request = profiler_analysis_pb2.ProfileSessionDataRequest()
            grpc_request.repository_root = os.path.dirname(run_dir)
            grpc_request.session_id = profile_run
            grpc_request.tool_name = 'trace_viewer'
            # Remove the trailing dot if present
            grpc_request.host_name = host.rstrip('.')

            grpc_request.parameters['resolution'] = request.args.get(
                'resolution')
            if request.args.get('start_time_ms') is not None:
                grpc_request.parameters['start_time_ms'] = request.args.get(
                    'start_time_ms')
            if request.args.get('end_time_ms') is not None:
                grpc_request.parameters['end_time_ms'] = request.args.get(
                    'end_time_ms')
            grpc_response = self.stub.GetSessionToolData(grpc_request)
            return grpc_response.output, None

        asset_path = os.path.join(run_dir, _make_filename(host, tool))
        raw_data = None
        try:
            with tf.io.gfile.GFile(asset_path, 'rb') as f:
                raw_data = f.read()
        except tf.errors.NotFoundError:
            logger.warning('Asset path %s not found', asset_path)
        except tf.errors.OpError as e:
            logger.warning("Couldn't read asset path: %s, OpError %s",
                           asset_path, e)

        if raw_data is None:
            return None, None
        data, content_encoding = None, None
        if tool == 'trace_viewer':
            data = process_raw_trace(raw_data)
        elif tool == 'tensorflow_stats':
            if tqx == 'out:csv;':
                data = tf_stats_proto_to_gviz.to_csv(raw_data)
            else:
                data = tf_stats_proto_to_gviz.to_json(raw_data)
        elif tool == 'overview_page@':
            data = overview_page_proto_to_gviz.to_json(raw_data)
        elif tool == 'input_pipeline_analyzer@':
            data = input_pipeline_proto_to_gviz.to_json(raw_data)
        elif tool == 'kernel_stats':
            if tqx == 'out:csv;':
                data = kernel_stats_proto_to_gviz.to_csv(raw_data)
            else:
                data = kernel_stats_proto_to_gviz.to_json(raw_data)
        elif tool == 'memory_profile':
            logger.info('Accessing memory_profile data')
            data = raw_data
            content_encoding = 'gzip'
        elif tool in _RAW_DATA_TOOLS:
            data = raw_data
            if tool[-1] == '#':
                content_encoding = 'gzip'
        return data, content_encoding