def timesketch_get_sketches( data: Optional[Text] = '') -> Dict[Text, api_sketch.Sketch]: """Returns a dict with available sketches. Args: data (str): not used. Returns: A dict with the available sketches, with keys as sketch name and values as Sketch objects. """ connect(ignore_sketch=True) state_obj = state.state() client = state_obj.get_from_cache('timesketch_client') return {x.name: x for x in client.list_sketches()}
def timesketch_get_searchindices( data: Optional[Text] = '') -> Dict[Text, Text]: """Returns a dict with available searchindices. Args: data (str): not used. Returns: A dict with the available search indices, with keys as names and values as search index names. """ connect(ignore_sketch=True) state_obj = state.state() client = state_obj.get_from_cache('timesketch_client') return {x.name: x.index_name for x in client.list_searchindices()}
def timesketch_refresh_token(data: Optional[Text] = '') -> Text: """Refreshes OAUTH token and returns a string with information the token. Args: data (str): not used. Returns: A string with indication about OAUTH token expiration. """ connect() state_obj = state.state() client = state_obj.get_from_cache('timesketch_client') if not client: return 'Not connected to a Timesketch client.' client.refresh_oauth_token() return timesketch_get_token_status('')
def timesketch_upload_file(data: Text, name: Optional[Text] = ''): """Upload a file to Timesketch. Args: data (str): Path to the file that's about to be uploaded. name (str): Name of the timeline. """ if not os.path.isfile(data): print('File [{0:s}] does not exist.'.format(data)) return connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') result = None if not name: name = data.split(os.sep)[-1] first, _, last = name.rpartition('.') name = first or last name = name.replace(' ', '_').replace('-', '_') timeline = None with importer.ImportStreamer() as streamer: streamer.set_sketch(sketch) streamer.set_timeline_name(name) # Set the file size to 20Mb before the file is split. streamer.set_filesize_threshold(20971520) streamer.add_file(data) # Force a flush. streamer.flush() result = streamer.response timeline = streamer.timeline if not result: print('Unable to upload data.') return if not timeline.name: print('Unable to get a timeline.') return print(f'Timeline: {timeline.name}{timeline.description}\n' f'Status: {timeline.status}')
def timesketch_create_sketch(data: Text, description: Text, set_active: Optional[bool] = True) -> Text: """Magic to create a new sketch in Timesketch. Args: data (str): name of the new sketch. description (str): sketch description. set_active (bool): whether to set the newly created sketch as the active one. Defaults to True. Returns: str: response string with sketch IDs. """ connect(ignore_sketch=True) state_obj = state.state() client = state_obj.get_from_cache('timesketch_client') name = data.strip() sketch = client.create_sketch(name, description) if not sketch: return 'No response, sketch not created.' try: _ = sketch.name except KeyError: return 'It seems like the sketch was not created, verify on TS server.' if set_active: set_active_sketch(sketch.id) return_string_list = [] return_string_list.append(f'Sketch: {sketch.id}') return_string_list.append(f'Name: {sketch.name}') data_objects = sketch.data.get('objects') if data_objects: data = data_objects[0] status_dict = data.get('status', [{}])[0] creation_time = status_dict.get('created_at', 'N/A') return_string_list.append('Creation Time: {0:s}'.format(creation_time)) status = status_dict.get('status', 'N/A') return_string_list.append('Status: {0:s}'.format(status)) return '\n'.join(return_string_list)
def get_sketch_details(sketch_id: Optional[int] = 0) -> Text: """Return back details about an existing sketch. Args: sketch_id (int): the sketch ID to check. Returns: string containing information about the sketch. """ sketch = None state_obj = state.state() if not sketch_id: sketch = state_obj.get_from_cache('timesketch_sketch') if not sketch: return 'Need to provide a sketch id, no sketch provided.' connect(ignore_sketch=True) client = state_obj.get_from_cache('timesketch_client') if not sketch: sketch = client.get_sketch(sketch_id) try: _ = sketch.name except KeyError: return 'TS server returned no information back about the sketch' return_string_list = [] return_string_list.append(f'Name: {sketch.name}') return_string_list.append(f'Description: {sketch.description}') return_string_list.append(f'ID: {sketch.id}') return_string_list.append('') return_string_list.append('Active Timelines:') for timeline in sketch.list_timelines(): objects = timeline.data.get('objects', []) if not objects: continue data = objects[0] description = data.get('description', 'No description') created_at = data.get('created_at', 'N/A') return_string_list.append( f'{timeline.name} [{description}] -> {created_at}') return '\n'.join(return_string_list)
def timesketch_list_views( data: Optional[Text] = '') -> Dict[str, api_view.View]: """List up all available views. Args: data (str): Not used. Returns: A dict with a list of available views. """ connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') return_dict = {} for view in sketch.list_views(): return_dict['{0:d}:{1:s}'.format(view.id, view.name)] = view return return_dict
def timesketch_list_stories( data: Optional[Text] = '') -> Dict[Text, api_story.Story]: """Returns a dict with all the stories that are saved to the sketch. Args: data (str): not used. Returns: A dict with keys as story titles and values as Story objects. """ connect(ignore_sketch=True) state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') if not sketch: return 'No data, not connected to a sketch.' story_dict = {} for story in sketch.list_stories(): story_dict[story.title] = story return story_dict
def timesketch_list_saved_searches( data: Optional[Text] = '') -> Dict[str, api_search.Search]: """List up all available saved searches. Args: data (str): Not used. Returns: A dict with a list of available saved searches. """ connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') return_dict = {} for search_obj in sketch.list_saved_searches(): key = f'{search_obj.id}:{search_obj.name}' return_dict[key] = search_obj return return_dict
def timesketch_upload_file(data: Text, name: Optional[Text] = ''): """Upload a file to Timesketch. Args: data (str): Path to the file that's about to be uploaded. name (str): Name of the timeline. """ if not os.path.isfile(data): print('File [{0:s}] does not exist.'.format(data)) return connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') result = None if not name: name = data.split(os.sep)[-1] first, _, last = name.rpartition('.') name = first or last name = name.replace(' ', '_').replace('-', '_') with importer.ImportStreamer() as streamer: streamer.set_sketch(sketch) streamer.set_timeline_name(name) # Set the file size to 20Mb before the file is split. streamer.set_filesize_threshold(20971520) streamer.add_file(data) result = streamer.response if not result: print('Unable to upload data.') return for timesketch_object in result.get('objects', []): if not timesketch_object: continue print('Timeline: {0:s}\nStatus: {1:s}'.format( timesketch_object.get('description'), ','.join( [x.get('status') for x in timesketch_object.get('status')])))
def timesketch_get_token_status(data: Optional[Text] = '') -> Text: """Returns a string with information about OAUTH token expiration. Args: data (str): not used. Returns: A string with indication about OAUTH token expiration. """ connect() state_obj = state.state() client = state_obj.get_from_cache('timesketch_client') if not client: return 'Not connected to a Timesketch client.' status = client.get_oauth_token_status() if status.get('expired', True): return 'OAUTH token has expired.' return 'OAUTH token has not expired, expires at: {0:s}'.format( status.get('expiry_time', 'Unknown Time'))
def connect( ignore_sketch: bool = False, force_switch: bool = False): """Check if Timesketch has been set up and connect if it hasn't. Args: ignore_sketch (optional): if set to True sketch check is ignored. force_switch (optional): if set to True then a new client will be created, irrelevant if another client was stored. Raises: ValueError: if Timesketch is not properly configured. """ state_obj = state.state() client = state_obj.get_from_cache('timesketch_client') sketch = state_obj.get_from_cache('timesketch_sketch') if client and force_switch: state_obj.remove_from_cache('timesketch_client') client = None if client: if ignore_sketch: return if sketch: return raise ValueError( 'No sketch configured, either create a new one using ' '%timesketch_create_sketch or assign an already existing ' 'one using %timesketch_set_active_sketch <sketch_id>') client = config.get_client() if not client: raise ValueError('Unable to connect to Timesketch') state_obj.add_to_cache('timesketch_client', client)
def query_timesketch(query: Optional[Text] = None, query_dsl: Optional[Text] = None, query_filter: Optional[Dict[Text, Any]] = None, view: Optional[api_view.View] = None, return_fields: Optional[Text] = None, start_date: Optional[Text] = None, end_date: Optional[Text] = None, max_entries: Optional[int] = None, indices: Optional[List[Text]] = None) -> pd.DataFrame: """Return back a data frame from a Timesketch query. Args: query (str): the query string to send to Timesketch. query_dsl (str): the query DSL to send to Timesketch. view (api_view.View): View object. return_fields (str): string with comma separated names of fields to return back. start_date (str): a timestamp in the form of 'YYYY-MM-DDTHH:MM:SS+00:00' of when to start the search query. end_date (str): a timestamp in the form of 'YYYY-MM-DDTHH:MM:SS+00:00' of when to end the search query. max_entries (int): Optional integer denoting a best effort to limit the output size to the number of events. Events are read in, 10k at a time so there may be more events in the answer back than this number denotes, this is a best effort. This value defaults to 40k events. Setting max_entries to zero will return all entries back (as in no limit). indices (list): a list of indices to query, if not provided _all will be used. Returns: A data frame with the results gathered from the query. Raises: KeyError: if the query is sent in without a query, query_dsl or a view. """ connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') if all([x is None for x in [query, query_dsl, view]]): raise KeyError('Need to provide a query, query_dsl or a view') if not query_filter: query_filter = {'time_start': None, 'time_end': None, 'order': 'asc'} if indices: query_filter['indices'] = indices else: query_filter['indices'] = '_all' date_string = '' if start_date and end_date: date_string = f'{start_date},{end_date}' elif start_date: date_string = f'{start_date},' else: date_string = f',{end_date}' if date_string: query_filter.setdefaults('chips', []) query_filter['chips'].append({ 'field': '', 'type': 'datetime_range', 'value': date_string }) # If view is being sent in, view needs to be the only parameter to the search. if view is not None: query = None query_dsl = None return_fields = _fix_return_fields(return_fields) return sketch.explore(query_string=query, query_dsl=query_dsl, query_filter=query_filter, view=view, return_fields=return_fields, max_entries=max_entries, as_pandas=True)
def timesketch_upload_data(data: pd.DataFrame, name: Optional[Text] = '', format_message_string: Optional[Text] = ''): """Upload a data frame to TimeSketch. Args: data (pandas.core.frame.DataFrame): the DataFrame to upload. name (str): the name used for the timeline in Timesketch. format_message_string (str): formatting string for the message column of the data frame, eg: "{src_ip:s} to {dst_ip:s}, {bytes:d} bytes transferred"' Raises: ValueError: if the dataframe cannot be uploaded to Timesketch or the data is invalid. """ if not isinstance(data, pd.DataFrame): raise ValueError( ('The data attribute is not a pandas DataFrame, please use curly ' 'braces to expand variables.')) if not name: name = 'unknown_timeline' connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') if not sketch: raise ValueError('Unable to upload data frame, need to set sketch.') result = None import_helper = helper.ImportHelper() timeline = None with importer.ImportStreamer() as streamer: streamer.set_sketch(sketch) if 'data_type' not in data: data_type = utils.ask_question('What is the value of [data_type]?', input_type=str) if data_type: streamer.set_data_type(data_type) else: data_types = data.data_type.unique() data_type = data_types[0] columns = list(data.columns) streamer.set_config_helper(import_helper) import_helper.configure_streamer(streamer, data_type=data_type, columns=columns) if format_message_string: streamer.set_message_format_string(format_message_string) streamer.set_timeline_name(name) streamer.add_data_frame(data) streamer.flush() result = streamer.response timeline = streamer.timeline if not result: print('Unable to upload data.') return if not timeline.name: print('Unable to import the timeline.') return print( f'Timeline: [{timeline.id}]{timeline.name} - {timeline.description}\n' f'Status: {timeline.status}')
def init(): """Initialize the notebook.""" # Initialize the state object. _ = state.state()
def query_timesketch( query: Optional[Text] = None, query_dsl: Optional[Text] = None, query_filter: Optional[Dict[Text, Any]] = None, return_fields: Optional[Text] = None, start_date: Optional[Text] = '', end_date: Optional[Text] = '', max_entries: Optional[int] = None, indices: Optional[List[Text]] = None) -> api_search.Search: """Return back a search object from a Timesketch query. Args: query (str): the query string to send to Timesketch. query_dsl (str): the query DSL to send to Timesketch. return_fields (str): string with comma separated names of fields to return back. start_date (str): a timestamp in the form of 'YYYY-MM-DDTHH:MM:SS+00:00' of when to start the search query. end_date (str): a timestamp in the form of 'YYYY-MM-DDTHH:MM:SS+00:00' of when to end the search query. max_entries (int): Optional integer denoting a best effort to limit the output size to the number of events. Events are read in, 10k at a time so there may be more events in the answer back than this number denotes, this is a best effort. This value defaults to 40k events. Setting max_entries to zero will return all entries back (as in no limit). indices (list): a list of indices to query, if not provided _all will be used. Returns: A search object (api_search.Search) that is pre-configured. Raises: KeyError: if the query is sent in without a query or query_dsl. """ connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') if all(x is None for x in [query, query_dsl]): raise KeyError('Need to provide a query or query_dsl') chip = None if start_date or end_date: if start_date and end_date: chip = api_search.DateRangeChip() chip.start_time = start_date chip.end_time = end_date else: chip = api_search.DateIntervalChip() if end_date: chip.date = end_date else: chip.date = start_date search_obj = api_search.Search(sketch) search_obj.from_manual(query_string=query, query_dsl=query_dsl, query_filter=query_filter, max_entries=max_entries) return_fields = _fix_return_fields(return_fields) if return_fields: search_obj.return_fields = return_fields if chip: search_obj.add_chip(chip) return search_obj
def wrapper(*args, **kwargs): function_output = function(*args, **kwargs) state_obj = state.state() return state_obj.set_output(function_output, magic_name=name)
def timesketch_add_manual_event( data: Text, timestamp: Optional[int] = 0, date_string: Optional[Text] = '', timestamp_desc: Optional[Text] = '', attributes: Optional[Dict[str, Any]] = None, tags: Optional[List[str]] = None) -> Dict[str, str]: """Add a manually generated event to the sketch. Args: data (str): The message string for for the event to be generated. timestamp (int): Optional timestamp in either seconds since Epoch or microseconds since Epoch. date_string (str): An optional date time as a human readable string. If neither date_string nor timestamp is provided then the current timestamp will be used as the time of the event. timestamp_desc (str): Optional timestamp description field. attributes (dict): Optional dict which contains extra attributes to add to the manual event. tags (list): Optional list of tags to add to the manual event. Returns: Dictionary with query results. """ connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') if not sketch: print('Not able to connect to a sketch.') return {} # Default timestamp. date_obj = datetime.datetime.now(datetime.timezone.utc) date = date_obj.isoformat() if timestamp: try: date_obj = datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc) except ValueError: date_obj = datetime.datetime.fromtimestamp(timestamp / 1e6, datetime.timezone.utc) date = date_obj.isoformat() elif date_string: elements = time_elements.TimeElements() if 'T' in date_string: try: elements.CopyFromStringISO8601(date_string) except ValueError: logger.error( 'Unable to convert date string, is it really in ISO 8601 format?' ) return {} try: elements.CopyFromString(date_string) except ValueError: try: elements.CopyFromStringRFC1123(date_string) except ValueError: logger.error( 'Unable to convert date string, needs to be in ISO 8601, 1123 or ' 'in the format YYYY-MM-DD hh:mm:ss.######[+-]##:##') return {} date = elements.CopyToDateTimeStringISO8601() if not timestamp_desc: timestamp_desc = 'Event Logged' if not isinstance(tags, (tuple, list)): tags = [] if not isinstance(attributes, dict): attributes = {} if not date: logger.error('Unable to convert date string, please check it.') return {} return sketch.add_event(data, date, timestamp_desc, attributes=attributes, tags=tags)
def __call__(self, line: Text, cell: Optional[Text] = None) -> Optional[Any]: line_magic = cell is None arguments = _parse_line_string(line) try: options = self.argument_parser.parse_args(arguments) except KeyError as e: print(('Unable to parse arguments, with error: {0!s}.\n' 'Correct usage is: {1:s}').format( e, self.argument_parser.format_help())) return except error.ArgParserNonZeroStatus: # When argparser ends execution but without an error this exception # is raised, eg: when "-h" or help is used. In those cases we need # to return without running the magic function. return if not line_magic: variable = options.data options.data = cell bind_to = options.bindto option_dict = options.__dict__ del option_dict['bindto'] # TODO: Change this so that there is no variable expansion # done by the core and all variable expansion is only done here. for key, value in iter(option_dict.items()): if not value: continue if not isinstance(value, str): continue if value[0] == '{' and value[-1] == '}': var_name = value[1:-1] var_type = self.argument_parser.storage.get(key) if '{' not in var_name and '}' not in var_name: var_obj = utils.ipython_get_global(var_name) if var_type and var_type != 'object': type_string = type(var_obj).__name__ if (type_string != var_type) and not type_string.endswith( var_type) and not var_type.endswith( type_string): raise KeyError(( 'Variable [{0:s}] is not of the correct type [{1:s}] for ' 'this magic. Type is: {2!s}').format( var_name, var_type, type(var_obj))) option_dict[key] = var_obj return_value = self.fn(**option_dict) state_obj = state.state() if not line_magic and variable: bind_to = variable return state_obj.set_output(output=return_value, magic_name=self.magic_name, bind_to=bind_to)