Example #1
0
def _get_tick_formatter() -> DatetimeTickFormatter:
    """Return tick formatting for different zoom levels."""
    # '%H:%M:%S.%3Nms
    tick_format = DatetimeTickFormatter()
    tick_format.days = ["%m-%d %H:%M"]
    tick_format.hours = ["%H:%M:%S"]
    tick_format.minutes = ["%H:%M:%S"]
    tick_format.seconds = ["%H:%M:%S"]
    tick_format.milliseconds = ["%H:%M:%S.%3N"]
    return tick_format
Example #2
0
def display_timeline(data,
                     alert=None,
                     overlay_data=None,
                     title: str = None,
                     time_column: str = 'TimeGenerated',
                     source_columns: list = None,
                     overlay_colums: list = None,
                     height: int = 300):
    """
    Display a timeline of events.

    Arguments:
        data {pd.DataFrame} -- Input DataFrame

    Keyword Arguments:
        alert {SecurityAlert} -- Input alert (optional) (default: {None})
        overlay_data {pd.DataFrame} -- Second event stream (DataFrame)
            to display as overlay (default: {None})
        title {str} -- [description] (default: {None})
        time_column {str} -- The name of the time
            property used in the Dataframe(s) (default: {'TimeGenerated'})
        source_columns {list} -- List of source columns to use in
            tooltips (default: {None})
        overlay_colums {list} -- List of source columns to use in
            overlay data tooltips (default: {None})
        heigh {int} -- the height of the plot figure (under 300 limits access
            to Bokeh tools)
    """
    reset_output()
    output_notebook()

    # pylint: disable=C0103
    WRAP = 50
    WRAP_CMDL = 'WrapCmdl'
    # pylint: enable=C0103
    y_max = 1

    if not source_columns:
        source_columns = ['NewProcessName', 'EventID', 'CommandLine']
    if time_column not in source_columns:
        source_columns.append(time_column)

    if 'CommandLine' in source_columns:
        graph_df = data[source_columns].copy()
        graph_df[WRAP_CMDL] = graph_df.apply(
            lambda x: _wrap_text(x.CommandLine, WRAP), axis=1)
    else:
        graph_df = data[source_columns].copy()

    # if we have an overlay - add this data and shift the y co-ordinates to
    # show on two separate lines
    if overlay_data is not None:
        overlay_colums = (overlay_colums
                          if overlay_colums is not None else source_columns)
        if time_column not in overlay_colums:
            overlay_colums.append(time_column)
        if 'CommandLine' in overlay_colums:
            overlay_df = overlay_data[overlay_colums].copy()
            overlay_df[WRAP_CMDL] = overlay_df.apply(
                lambda x: _wrap_text(x.CommandLine, WRAP), axis=1)
        else:
            overlay_df = overlay_data[overlay_colums].copy()
        graph_df['y_index'] = 2
        overlay_df['y_index'] = 1
        y_max = 2
    else:
        graph_df['y_index'] = 1

    source = ColumnDataSource(graph_df)

    # build the tool tips from columns (excluding these)
    excl_cols = [time_column, 'CommandLine']
    tool_tip_items = [(f'{col}', f'@{col}') for col in source_columns
                      if col not in excl_cols]
    if WRAP_CMDL in graph_df:
        tool_tip_items.append(('CommandLine', f'@{WRAP_CMDL}'))
    hover = HoverTool(
        tooltips=tool_tip_items,
        formatters={'Tooltip': 'printf'}
        # display a tooltip whenever the cursor is vertically in line with a glyph
        # ,mode='vline'
    )

    if not title:
        title = 'Event Timeline'
    else:
        title = 'Timeline {}'.format(title)

    # tools = 'pan, box_zoom, wheel_zoom, reset, undo, redo, save, hover'
    plot = figure(min_border_left=50,
                  plot_height=height,
                  plot_width=900,
                  x_axis_label='Event Time',
                  x_axis_type='datetime',
                  x_minor_ticks=10,
                  tools=[hover, 'pan', 'xwheel_zoom', 'box_zoom', 'reset'],
                  title=title)
    plot.yaxis.visible = False

    # Tick formatting for different zoom levels
    # '%H:%M:%S.%3Nms
    tick_format = DatetimeTickFormatter()
    tick_format.days = ['%m-%d %H:%M']
    tick_format.hours = ['%H:%M:%S']
    tick_format.minutes = ['%H:%M:%S']
    tick_format.seconds = ['%H:%M:%S']
    tick_format.milliseconds = ['%H:%M:%S.%3N']

    plot.xaxis[0].formatter = tick_format
    plot.circle(x=time_column,
                y='y_index',
                color='navy',
                alpha=0.5,
                size=10,
                source=source)

    if overlay_data is not None:
        overlay_source = ColumnDataSource(overlay_df)
        plot.circle(x=time_column,
                    y='y_index',
                    color='green',
                    alpha=0.5,
                    size=10,
                    source=overlay_source)

    # Adding data labels stops everything working!
    # labels = LabelSet(x=time_column, y='y_index', y_offset=5,
    #                   text='NewProcessName', source=source,
    #                   angle='90deg', text_font_size='8pt')
    # p.add_layout(labels)

    # if we have an alert, plot the time as a line
    if alert is not None:
        x_alert_label = pd.Timestamp(alert['StartTimeUtc'])
        plot.line(x=[x_alert_label, x_alert_label], y=[0, y_max + 1])
        alert_label = Label(x=x_alert_label,
                            y=0,
                            y_offset=10,
                            x_units='data',
                            y_units='data',
                            text='< Alert time',
                            render_mode='css',
                            border_line_color='red',
                            border_line_alpha=1.0,
                            background_fill_color='white',
                            background_fill_alpha=1.0)

        plot.add_layout(alert_label)

        print('Alert start time = ', alert['StartTimeUtc'])

    show(plot)
Example #3
0
def display_timeline(
    data: pd.DataFrame,
    alert: SecurityAlert = None,
    overlay_data: pd.DataFrame = None,
    title: str = None,
    time_column: str = "TimeGenerated",
    source_columns: list = None,
    overlay_colums: list = None,
    height: int = 300,
):
    """
    Display a timeline of events.

    Parameters
    ----------
    data : pd.DataFrame
        Input DataFrame
    alert : SecurityAlert, optional
        Input alert (the default is None)
    overlay_data : pd.DataFrame, optional
        Second event stream to display as overlay
        (the default is None)
    title : str, optional
        Title to display (the default is None)
    time_column : str, optional
        Name of the timestamp column
        (the default is 'TimeGenerated')
    source_columns : list, optional
        List of source columns to use in tooltips
        (the default is None)
    overlay_colums : list, optional
        List of source columns to use in overlay data tooltips.
        (the default is None)
    height : int, optional
        the height of the plot figure (under 300 limits access
        to Bokeh tools)(the default is 300)

    """
    reset_output()
    output_notebook()

    y_max = 1

    if not source_columns:
        source_columns = ["NewProcessName", "EventID", "CommandLine"]
    if time_column not in source_columns:
        source_columns.append(time_column)

    if "CommandLine" in source_columns:
        graph_df = data[source_columns].copy()
        graph_df[_WRAP_CMDL] = graph_df.apply(
            lambda x: _wrap_text(x.CommandLine, _WRAP), axis=1)
    else:
        graph_df = data[source_columns].copy()

    # if we have an overlay - add this data and shift the y co-ordinates to
    # show on two separate lines
    if overlay_data is not None:
        overlay_colums = (overlay_colums
                          if overlay_colums is not None else source_columns)
        if time_column not in overlay_colums:
            overlay_colums.append(time_column)
        if "CommandLine" in overlay_colums:
            overlay_df = overlay_data[overlay_colums].copy()
            overlay_df[_WRAP_CMDL] = overlay_df.apply(
                lambda x: _wrap_text(x.CommandLine, _WRAP), axis=1)
        else:
            overlay_df = overlay_data[overlay_colums].copy()
        graph_df["y_index"] = 2
        overlay_df["y_index"] = 1
        y_max = 2
    else:
        graph_df["y_index"] = 1

    source = ColumnDataSource(graph_df)

    # build the tool tips from columns (excluding these)
    excl_cols = [time_column, "CommandLine"]
    tool_tip_items = [(f"{col}", f"@{col}") for col in source_columns
                      if col not in excl_cols]
    if _WRAP_CMDL in graph_df:
        tool_tip_items.append(("CommandLine", f"@{_WRAP_CMDL}"))
    hover = HoverTool(
        tooltips=tool_tip_items,
        formatters={"Tooltip": "printf"}
        # display a tooltip whenever the cursor is vertically in line with a glyph
        # ,mode='vline'
    )

    if not title:
        title = "Event Timeline"
    else:
        title = "Timeline {}".format(title)

    # tools = 'pan, box_zoom, wheel_zoom, reset, undo, redo, save, hover'
    plot = figure(
        min_border_left=50,
        plot_height=height,
        plot_width=900,
        x_axis_label="Event Time",
        x_axis_type="datetime",
        x_minor_ticks=10,
        tools=[hover, "pan", "xwheel_zoom", "box_zoom", "reset"],
        title=title,
    )
    plot.yaxis.visible = False

    # Tick formatting for different zoom levels
    # '%H:%M:%S.%3Nms
    tick_format = DatetimeTickFormatter()
    tick_format.days = ["%m-%d %H:%M"]
    tick_format.hours = ["%H:%M:%S"]
    tick_format.minutes = ["%H:%M:%S"]
    tick_format.seconds = ["%H:%M:%S"]
    tick_format.milliseconds = ["%H:%M:%S.%3N"]

    plot.xaxis[0].formatter = tick_format
    plot.circle(x=time_column,
                y="y_index",
                color="navy",
                alpha=0.5,
                size=10,
                source=source)

    if overlay_data is not None:
        overlay_source = ColumnDataSource(overlay_df)
        plot.circle(
            x=time_column,
            y="y_index",
            color="green",
            alpha=0.5,
            size=10,
            source=overlay_source,
        )

    # Adding data labels stops everything working!
    # labels = LabelSet(x=time_column, y='y_index', y_offset=5,
    #                   text='NewProcessName', source=source,
    #                   angle='90deg', text_font_size='8pt')
    # p.add_layout(labels)

    # if we have an alert, plot the time as a line
    if alert is not None:
        x_alert_label = pd.Timestamp(alert["StartTimeUtc"])
        plot.line(x=[x_alert_label, x_alert_label], y=[0, y_max + 1])
        alert_label = Label(
            x=x_alert_label,
            y=0,
            y_offset=10,
            x_units="data",
            y_units="data",
            text="< Alert time",
            render_mode="css",
            border_line_color="red",
            border_line_alpha=1.0,
            background_fill_color="white",
            background_fill_alpha=1.0,
        )

        plot.add_layout(alert_label)

        print("Alert start time = ", alert["StartTimeUtc"])

    show(plot)