def to_timesketch(self): """ to_timesketch: push dataframe to timesketch """ import_helper = helper.ImportHelper() with importer.ImportStreamer() as streamer: streamer.set_sketch(self.sketch) streamer.set_provider("MansToEs") streamer.set_config_helper(import_helper) streamer.set_timeline_name(self.timeline_name) for file in glob(self.folder_path + "/tmp__*.json"): df = pd.read_json(file, orient="records", lines=True, dtype=False) filetype = file.split("tmp___")[-1].split(".")[0].split("_")[0] streamer.set_upload_context(filetype) streamer.add_data_frame(df, part_of_iter=True) if self.exd_alerts: for alert in self.exd_alerts: streamer.set_upload_context("EXD alerts") streamer.add_dict(alert) logging.debug("[MAIN] Bulk timesketch push [✔]")
def timesketch_upload_data(data: pd.DataFrame, name: Optional[Text] = '', format_message_string: Optional[Text] = ''): """Upload a data frame to TimeSketch. Args: data (pandas.core.frame.DataFrame): the DataFrame to upload. name (str): the name used for the timeline in Timesketch. format_message_string (str): formatting string for the message column of the data frame, eg: "{src_ip:s} to {dst_ip:s}, {bytes:d} bytes transferred"' Raises: ValueError: if the dataframe cannot be uploaded to Timesketch or the data is invalid. """ if not isinstance(data, pd.DataFrame): raise ValueError( ('The data attribute is not a pandas DataFrame, please use curly ' 'braces to expand variables.')) if not name: name = 'unknown_timeline' connect() state_obj = state.state() sketch = state_obj.get_from_cache('timesketch_sketch') if not sketch: raise ValueError('Unable to upload data frame, need to set sketch.') result = None import_helper = helper.ImportHelper() timeline = None with importer.ImportStreamer() as streamer: streamer.set_sketch(sketch) if 'data_type' not in data: data_type = utils.ask_question('What is the value of [data_type]?', input_type=str) if data_type: streamer.set_data_type(data_type) else: data_types = data.data_type.unique() data_type = data_types[0] columns = list(data.columns) streamer.set_config_helper(import_helper) import_helper.configure_streamer(streamer, data_type=data_type, columns=columns) if format_message_string: streamer.set_message_format_string(format_message_string) streamer.set_timeline_name(name) streamer.add_data_frame(data) streamer.flush() result = streamer.response timeline = streamer.timeline if not result: print('Unable to upload data.') return if not timeline.name: print('Unable to import the timeline.') return print( f'Timeline: [{timeline.id}]{timeline.name} - {timeline.description}\n' f'Status: {timeline.status}')
def upload_file(my_sketch: sketch.Sketch, config_dict: Dict[str, any], file_path: str) -> str: """Uploads a file to Timesketch. Args: my_sketch (sketch.Sketch): a sketch object to point to the sketch the data will be imported to. config_dict (dict): dict with settings for the importer. file_path (str): the path to the file to upload. Returns: A tuple with the timeline object (timeline.Timeline) or None if not able to upload the timeline as well as the celery task identification for the indexing. """ if not my_sketch or not hasattr(my_sketch, 'id'): return 'Sketch needs to be set' _, _, file_extension = file_path.rpartition('.') if file_extension.lower() not in ('plaso', 'csv', 'jsonl'): return ('File needs to have one of the following extensions: ' '.plaso, .csv, ' '.jsonl (not {0:s})').format(file_extension.lower()) import_helper = helper.ImportHelper() import_helper.add_config_dict(config_dict) log_config_file = config_dict.get('log_config_file', '') if log_config_file: import_helper.add_config(log_config_file) timeline = None task_id = '' logger.info('About to upload file.') with importer.ImportStreamer() as streamer: streamer.set_sketch(my_sketch) streamer.set_config_helper(import_helper) streamer.set_provider('CLI importer tool') format_string = config_dict.get('message_format_string') if format_string: streamer.set_message_format_string(format_string) timeline_name = config_dict.get('timeline_name') if timeline_name: streamer.set_timeline_name(timeline_name) index_name = config_dict.get('index_name') if index_name: streamer.set_index_name(index_name) time_desc = config_dict.get('timestamp_description') if time_desc: streamer.set_timestamp_description(time_desc) entry_threshold = config_dict.get('entry_threshold') if entry_threshold: streamer.set_entry_threshold(entry_threshold) size_threshold = config_dict.get('size_threshold') if size_threshold: streamer.set_filesize_threshold(size_threshold) data_label = config_dict.get('data_label') if data_label: streamer.set_data_label(data_label) context = config_dict.get('context') if context: streamer.set_upload_context(context) else: streamer.set_upload_context(' '.join(sys.argv)) streamer.add_file(file_path) # Force a flush. streamer.flush() timeline = streamer.timeline task_id = streamer.celery_task_id logger.info('File upload completed.') return timeline, task_id
def upload_file( my_sketch: sketch.Sketch, config_dict: Dict[str, any], file_path: str) -> str: """Uploads a file to Timesketch. Args: my_sketch (sketch.Sketch): a sketch object to point to the sketch the data will be imported to. config_dict (dict): dict with settings for the importer. file_path (str): the path to the file to upload. Returns: A string with results (whether successful or not). """ if not my_sketch or not hasattr(my_sketch, 'id'): return 'Sketch needs to be set' _, _, file_extension = file_path.rpartition('.') if file_extension.lower() not in ('plaso', 'csv', 'jsonl'): return ( 'File needs to have one of the following extensions: ' '.plaso, .csv, ' '.jsonl (not {0:s})').format(file_extension.lower()) import_helper = helper.ImportHelper() import_helper.add_config_dict(config_dict) log_config_file = config_dict.get('log_config_file', '') if log_config_file: import_helper.add_config(log_config_file) with importer.ImportStreamer() as streamer: streamer.set_sketch(my_sketch) streamer.set_config_helper(import_helper) streamer.set_provider('CLI importer tool') format_string = config_dict.get('message_format_string') if format_string: streamer.set_message_format_string(format_string) timeline_name = config_dict.get('timeline_name') if timeline_name: streamer.set_timeline_name(timeline_name) index_name = config_dict.get('index_name') if index_name: streamer.set_index_name(index_name) time_desc = config_dict.get('timestamp_description') if time_desc: streamer.set_timestamp_description(time_desc) entry_threshold = config_dict.get('entry_threshold') if entry_threshold: streamer.set_entry_threshold(entry_threshold) size_threshold = config_dict.get('size_threshold') if size_threshold: streamer.set_filesize_threshold(size_threshold) data_label = config_dict.get('data_label') if data_label: streamer.set_data_label(data_label) context = config_dict.get('context') if context: streamer.set_upload_context(context) else: streamer.set_upload_context(' '.join(sys.argv)) streamer.add_file(file_path) return 'File got successfully uploaded to sketch: {0:d}'.format( my_sketch.id)