def timesketch_get_sketches(
data: Optional[Text] = '') -> Dict[Text, api_sketch.Sketch]:
"""Returns a dict with available sketches.
Args:
data (str): not used.
Returns:
A dict with the available sketches, with keys as sketch name and values
as Sketch objects.
"""
connect(ignore_sketch=True)
state_obj = state.state()
client = state_obj.get_from_cache('timesketch_client')
return {x.name: x for x in client.list_sketches()}
def timesketch_get_searchindices(
data: Optional[Text] = '') -> Dict[Text, Text]:
"""Returns a dict with available searchindices.
Args:
data (str): not used.
Returns:
A dict with the available search indices, with keys as names and values as
search index names.
"""
connect(ignore_sketch=True)
state_obj = state.state()
client = state_obj.get_from_cache('timesketch_client')
return {x.name: x.index_name for x in client.list_searchindices()}
def timesketch_refresh_token(data: Optional[Text] = '') -> Text:
"""Refreshes OAUTH token and returns a string with information the token.
Args:
data (str): not used.
Returns:
A string with indication about OAUTH token expiration.
"""
connect()
state_obj = state.state()
client = state_obj.get_from_cache('timesketch_client')
if not client:
return 'Not connected to a Timesketch client.'
client.refresh_oauth_token()
return timesketch_get_token_status('')
def timesketch_upload_file(data: Text, name: Optional[Text] = ''):
"""Upload a file to Timesketch.
Args:
data (str): Path to the file that's about to be uploaded.
name (str): Name of the timeline.
"""
if not os.path.isfile(data):
print('File [{0:s}] does not exist.'.format(data))
return
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
result = None
if not name:
name = data.split(os.sep)[-1]
first, _, last = name.rpartition('.')
name = first or last
name = name.replace(' ', '_').replace('-', '_')
timeline = None
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
streamer.set_timeline_name(name)
# Set the file size to 20Mb before the file is split.
streamer.set_filesize_threshold(20971520)
streamer.add_file(data)
# Force a flush.
streamer.flush()
result = streamer.response
timeline = streamer.timeline
if not result:
print('Unable to upload data.')
return
if not timeline.name:
print('Unable to get a timeline.')
return
print(f'Timeline: {timeline.name}{timeline.description}\n'
f'Status: {timeline.status}')
def timesketch_create_sketch(data: Text,
description: Text,
set_active: Optional[bool] = True) -> Text:
"""Magic to create a new sketch in Timesketch.
Args:
data (str): name of the new sketch.
description (str): sketch description.
set_active (bool): whether to set the newly created sketch as the active
one. Defaults to True.
Returns:
str: response string with sketch IDs.
"""
connect(ignore_sketch=True)
state_obj = state.state()
client = state_obj.get_from_cache('timesketch_client')
name = data.strip()
sketch = client.create_sketch(name, description)
if not sketch:
return 'No response, sketch not created.'
try:
_ = sketch.name
except KeyError:
return 'It seems like the sketch was not created, verify on TS server.'
if set_active:
set_active_sketch(sketch.id)
return_string_list = []
return_string_list.append(f'Sketch: {sketch.id}')
return_string_list.append(f'Name: {sketch.name}')
data_objects = sketch.data.get('objects')
if data_objects:
data = data_objects[0]
status_dict = data.get('status', [{}])[0]
creation_time = status_dict.get('created_at', 'N/A')
return_string_list.append('Creation Time: {0:s}'.format(creation_time))
status = status_dict.get('status', 'N/A')
return_string_list.append('Status: {0:s}'.format(status))
return '\n'.join(return_string_list)
def get_sketch_details(sketch_id: Optional[int] = 0) -> Text:
"""Return back details about an existing sketch.
Args:
sketch_id (int): the sketch ID to check.
Returns:
string containing information about the sketch.
"""
sketch = None
state_obj = state.state()
if not sketch_id:
sketch = state_obj.get_from_cache('timesketch_sketch')
if not sketch:
return 'Need to provide a sketch id, no sketch provided.'
connect(ignore_sketch=True)
client = state_obj.get_from_cache('timesketch_client')
if not sketch:
sketch = client.get_sketch(sketch_id)
try:
_ = sketch.name
except KeyError:
return 'TS server returned no information back about the sketch'
return_string_list = []
return_string_list.append(f'Name: {sketch.name}')
return_string_list.append(f'Description: {sketch.description}')
return_string_list.append(f'ID: {sketch.id}')
return_string_list.append('')
return_string_list.append('Active Timelines:')
for timeline in sketch.list_timelines():
objects = timeline.data.get('objects', [])
if not objects:
continue
data = objects[0]
description = data.get('description', 'No description')
created_at = data.get('created_at', 'N/A')
return_string_list.append(
f'{timeline.name} [{description}] -> {created_at}')
return '\n'.join(return_string_list)
def timesketch_list_views(
data: Optional[Text] = '') -> Dict[str, api_view.View]:
"""List up all available views.
Args:
data (str): Not used.
Returns:
A dict with a list of available views.
"""
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
return_dict = {}
for view in sketch.list_views():
return_dict['{0:d}:{1:s}'.format(view.id, view.name)] = view
return return_dict
def timesketch_list_stories(
data: Optional[Text] = '') -> Dict[Text, api_story.Story]:
"""Returns a dict with all the stories that are saved to the sketch.
Args:
data (str): not used.
Returns:
A dict with keys as story titles and values as Story objects.
"""
connect(ignore_sketch=True)
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if not sketch:
return 'No data, not connected to a sketch.'
story_dict = {}
for story in sketch.list_stories():
story_dict[story.title] = story
return story_dict
def timesketch_list_saved_searches(
data: Optional[Text] = '') -> Dict[str, api_search.Search]:
"""List up all available saved searches.
Args:
data (str): Not used.
Returns:
A dict with a list of available saved searches.
"""
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
return_dict = {}
for search_obj in sketch.list_saved_searches():
key = f'{search_obj.id}:{search_obj.name}'
return_dict[key] = search_obj
return return_dict
def timesketch_upload_file(data: Text, name: Optional[Text] = ''):
"""Upload a file to Timesketch.
Args:
data (str): Path to the file that's about to be uploaded.
name (str): Name of the timeline.
"""
if not os.path.isfile(data):
print('File [{0:s}] does not exist.'.format(data))
return
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
result = None
if not name:
name = data.split(os.sep)[-1]
first, _, last = name.rpartition('.')
name = first or last
name = name.replace(' ', '_').replace('-', '_')
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
streamer.set_timeline_name(name)
# Set the file size to 20Mb before the file is split.
streamer.set_filesize_threshold(20971520)
streamer.add_file(data)
result = streamer.response
if not result:
print('Unable to upload data.')
return
for timesketch_object in result.get('objects', []):
if not timesketch_object:
continue
print('Timeline: {0:s}\nStatus: {1:s}'.format(
timesketch_object.get('description'), ','.join(
[x.get('status') for x in timesketch_object.get('status')])))
def timesketch_get_token_status(data: Optional[Text] = '') -> Text:
"""Returns a string with information about OAUTH token expiration.
Args:
data (str): not used.
Returns:
A string with indication about OAUTH token expiration.
"""
connect()
state_obj = state.state()
client = state_obj.get_from_cache('timesketch_client')
if not client:
return 'Not connected to a Timesketch client.'
status = client.get_oauth_token_status()
if status.get('expired', True):
return 'OAUTH token has expired.'
return 'OAUTH token has not expired, expires at: {0:s}'.format(
status.get('expiry_time', 'Unknown Time'))
def connect(
ignore_sketch: bool = False,
force_switch: bool = False):
"""Check if Timesketch has been set up and connect if it hasn't.
Args:
ignore_sketch (optional): if set to True sketch check is ignored.
force_switch (optional): if set to True then a new client will be created,
irrelevant if another client was stored.
Raises:
ValueError: if Timesketch is not properly configured.
"""
state_obj = state.state()
client = state_obj.get_from_cache('timesketch_client')
sketch = state_obj.get_from_cache('timesketch_sketch')
if client and force_switch:
state_obj.remove_from_cache('timesketch_client')
client = None
if client:
if ignore_sketch:
return
if sketch:
return
raise ValueError(
'No sketch configured, either create a new one using '
'%timesketch_create_sketch or assign an already existing '
'one using %timesketch_set_active_sketch <sketch_id>')
client = config.get_client()
if not client:
raise ValueError('Unable to connect to Timesketch')
state_obj.add_to_cache('timesketch_client', client)
def query_timesketch(query: Optional[Text] = None,
query_dsl: Optional[Text] = None,
query_filter: Optional[Dict[Text, Any]] = None,
view: Optional[api_view.View] = None,
return_fields: Optional[Text] = None,
start_date: Optional[Text] = None,
end_date: Optional[Text] = None,
max_entries: Optional[int] = None,
indices: Optional[List[Text]] = None) -> pd.DataFrame:
"""Return back a data frame from a Timesketch query.
Args:
query (str): the query string to send to Timesketch.
query_dsl (str): the query DSL to send to Timesketch.
view (api_view.View): View object.
return_fields (str): string with comma separated names of fields to
return back.
start_date (str): a timestamp in the form of
'YYYY-MM-DDTHH:MM:SS+00:00' of when to start the search query.
end_date (str): a timestamp in the form of
'YYYY-MM-DDTHH:MM:SS+00:00' of when to end the search query.
max_entries (int): Optional integer denoting a best effort to limit
the output size to the number of events. Events are read in,
10k at a time so there may be more events in the answer back
than this number denotes, this is a best effort. This value defaults
to 40k events. Setting max_entries to zero will return all entries
back (as in no limit).
indices (list): a list of indices to query, if not provided _all
will be used.
Returns:
A data frame with the results gathered from the query.
Raises:
KeyError: if the query is sent in without a query, query_dsl or a view.
"""
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if all([x is None for x in [query, query_dsl, view]]):
raise KeyError('Need to provide a query, query_dsl or a view')
if not query_filter:
query_filter = {'time_start': None, 'time_end': None, 'order': 'asc'}
if indices:
query_filter['indices'] = indices
else:
query_filter['indices'] = '_all'
date_string = ''
if start_date and end_date:
date_string = f'{start_date},{end_date}'
elif start_date:
date_string = f'{start_date},'
else:
date_string = f',{end_date}'
if date_string:
query_filter.setdefaults('chips', [])
query_filter['chips'].append({
'field': '',
'type': 'datetime_range',
'value': date_string
})
# If view is being sent in, view needs to be the only parameter to the search.
if view is not None:
query = None
query_dsl = None
return_fields = _fix_return_fields(return_fields)
return sketch.explore(query_string=query,
query_dsl=query_dsl,
query_filter=query_filter,
view=view,
return_fields=return_fields,
max_entries=max_entries,
as_pandas=True)
def timesketch_upload_data(data: pd.DataFrame,
name: Optional[Text] = '',
format_message_string: Optional[Text] = ''):
"""Upload a data frame to TimeSketch.
Args:
data (pandas.core.frame.DataFrame): the DataFrame to upload.
name (str): the name used for the timeline in Timesketch.
format_message_string (str): formatting string for the message column of
the data frame, eg: "{src_ip:s} to {dst_ip:s}, {bytes:d} bytes
transferred"'
Raises:
ValueError: if the dataframe cannot be uploaded to Timesketch or the
data is invalid.
"""
if not isinstance(data, pd.DataFrame):
raise ValueError(
('The data attribute is not a pandas DataFrame, please use curly '
'braces to expand variables.'))
if not name:
name = 'unknown_timeline'
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if not sketch:
raise ValueError('Unable to upload data frame, need to set sketch.')
result = None
import_helper = helper.ImportHelper()
timeline = None
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
if 'data_type' not in data:
data_type = utils.ask_question('What is the value of [data_type]?',
input_type=str)
if data_type:
streamer.set_data_type(data_type)
else:
data_types = data.data_type.unique()
data_type = data_types[0]
columns = list(data.columns)
streamer.set_config_helper(import_helper)
import_helper.configure_streamer(streamer,
data_type=data_type,
columns=columns)
if format_message_string:
streamer.set_message_format_string(format_message_string)
streamer.set_timeline_name(name)
streamer.add_data_frame(data)
streamer.flush()
result = streamer.response
timeline = streamer.timeline
if not result:
print('Unable to upload data.')
return
if not timeline.name:
print('Unable to import the timeline.')
return
print(
f'Timeline: [{timeline.id}]{timeline.name} - {timeline.description}\n'
f'Status: {timeline.status}')
def init():
"""Initialize the notebook."""
# Initialize the state object.
_ = state.state()
def query_timesketch(
query: Optional[Text] = None,
query_dsl: Optional[Text] = None,
query_filter: Optional[Dict[Text, Any]] = None,
return_fields: Optional[Text] = None,
start_date: Optional[Text] = '',
end_date: Optional[Text] = '',
max_entries: Optional[int] = None,
indices: Optional[List[Text]] = None) -> api_search.Search:
"""Return back a search object from a Timesketch query.
Args:
query (str): the query string to send to Timesketch.
query_dsl (str): the query DSL to send to Timesketch.
return_fields (str): string with comma separated names of fields to
return back.
start_date (str): a timestamp in the form of
'YYYY-MM-DDTHH:MM:SS+00:00' of when to start the search query.
end_date (str): a timestamp in the form of
'YYYY-MM-DDTHH:MM:SS+00:00' of when to end the search query.
max_entries (int): Optional integer denoting a best effort to limit
the output size to the number of events. Events are read in,
10k at a time so there may be more events in the answer back
than this number denotes, this is a best effort. This value defaults
to 40k events. Setting max_entries to zero will return all entries
back (as in no limit).
indices (list): a list of indices to query, if not provided _all
will be used.
Returns:
A search object (api_search.Search) that is pre-configured.
Raises:
KeyError: if the query is sent in without a query or query_dsl.
"""
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if all(x is None for x in [query, query_dsl]):
raise KeyError('Need to provide a query or query_dsl')
chip = None
if start_date or end_date:
if start_date and end_date:
chip = api_search.DateRangeChip()
chip.start_time = start_date
chip.end_time = end_date
else:
chip = api_search.DateIntervalChip()
if end_date:
chip.date = end_date
else:
chip.date = start_date
search_obj = api_search.Search(sketch)
search_obj.from_manual(query_string=query,
query_dsl=query_dsl,
query_filter=query_filter,
max_entries=max_entries)
return_fields = _fix_return_fields(return_fields)
if return_fields:
search_obj.return_fields = return_fields
if chip:
search_obj.add_chip(chip)
return search_obj
def wrapper(*args, **kwargs):
function_output = function(*args, **kwargs)
state_obj = state.state()
return state_obj.set_output(function_output, magic_name=name)
def timesketch_add_manual_event(
data: Text,
timestamp: Optional[int] = 0,
date_string: Optional[Text] = '',
timestamp_desc: Optional[Text] = '',
attributes: Optional[Dict[str, Any]] = None,
tags: Optional[List[str]] = None) -> Dict[str, str]:
"""Add a manually generated event to the sketch.
Args:
data (str): The message string for for the event to be generated.
timestamp (int): Optional timestamp in either seconds since Epoch or
microseconds since Epoch.
date_string (str): An optional date time as a human readable string. If
neither date_string nor timestamp is provided then the current timestamp
will be used as the time of the event.
timestamp_desc (str): Optional timestamp description field.
attributes (dict): Optional dict which contains extra attributes to add
to the manual event.
tags (list): Optional list of tags to add to the manual event.
Returns:
Dictionary with query results.
"""
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if not sketch:
print('Not able to connect to a sketch.')
return {}
# Default timestamp.
date_obj = datetime.datetime.now(datetime.timezone.utc)
date = date_obj.isoformat()
if timestamp:
try:
date_obj = datetime.datetime.fromtimestamp(timestamp,
datetime.timezone.utc)
except ValueError:
date_obj = datetime.datetime.fromtimestamp(timestamp / 1e6,
datetime.timezone.utc)
date = date_obj.isoformat()
elif date_string:
elements = time_elements.TimeElements()
if 'T' in date_string:
try:
elements.CopyFromStringISO8601(date_string)
except ValueError:
logger.error(
'Unable to convert date string, is it really in ISO 8601 format?'
)
return {}
try:
elements.CopyFromString(date_string)
except ValueError:
try:
elements.CopyFromStringRFC1123(date_string)
except ValueError:
logger.error(
'Unable to convert date string, needs to be in ISO 8601, 1123 or '
'in the format YYYY-MM-DD hh:mm:ss.######[+-]##:##')
return {}
date = elements.CopyToDateTimeStringISO8601()
if not timestamp_desc:
timestamp_desc = 'Event Logged'
if not isinstance(tags, (tuple, list)):
tags = []
if not isinstance(attributes, dict):
attributes = {}
if not date:
logger.error('Unable to convert date string, please check it.')
return {}
return sketch.add_event(data,
date,
timestamp_desc,
attributes=attributes,
tags=tags)
def __call__(self,
line: Text,
cell: Optional[Text] = None) -> Optional[Any]:
line_magic = cell is None
arguments = _parse_line_string(line)
try:
options = self.argument_parser.parse_args(arguments)
except KeyError as e:
print(('Unable to parse arguments, with error: {0!s}.\n'
'Correct usage is: {1:s}').format(
e, self.argument_parser.format_help()))
return
except error.ArgParserNonZeroStatus:
# When argparser ends execution but without an error this exception
# is raised, eg: when "-h" or help is used. In those cases we need
# to return without running the magic function.
return
if not line_magic:
variable = options.data
options.data = cell
bind_to = options.bindto
option_dict = options.__dict__
del option_dict['bindto']
# TODO: Change this so that there is no variable expansion
# done by the core and all variable expansion is only done here.
for key, value in iter(option_dict.items()):
if not value:
continue
if not isinstance(value, str):
continue
if value[0] == '{' and value[-1] == '}':
var_name = value[1:-1]
var_type = self.argument_parser.storage.get(key)
if '{' not in var_name and '}' not in var_name:
var_obj = utils.ipython_get_global(var_name)
if var_type and var_type != 'object':
type_string = type(var_obj).__name__
if (type_string !=
var_type) and not type_string.endswith(
var_type) and not var_type.endswith(
type_string):
raise KeyError((
'Variable [{0:s}] is not of the correct type [{1:s}] for '
'this magic. Type is: {2!s}').format(
var_name, var_type, type(var_obj)))
option_dict[key] = var_obj
return_value = self.fn(**option_dict)
state_obj = state.state()
if not line_magic and variable:
bind_to = variable
return state_obj.set_output(output=return_value,
magic_name=self.magic_name,
bind_to=bind_to)
