def to_timesketch(self):
"""
to_timesketch: push dataframe to timesketch
"""
import_helper = helper.ImportHelper()
with importer.ImportStreamer() as streamer:
streamer.set_sketch(self.sketch)
streamer.set_provider("MansToEs")
streamer.set_config_helper(import_helper)
streamer.set_timeline_name(self.timeline_name)
for file in glob(self.folder_path + "/tmp__*.json"):
df = pd.read_json(file,
orient="records",
lines=True,
dtype=False)
filetype = file.split("tmp___")[-1].split(".")[0].split("_")[0]
streamer.set_upload_context(filetype)
streamer.add_data_frame(df, part_of_iter=True)
if self.exd_alerts:
for alert in self.exd_alerts:
streamer.set_upload_context("EXD alerts")
streamer.add_dict(alert)
logging.debug("[MAIN] Bulk timesketch push [✔]")
def timesketch_upload_data(data: pd.DataFrame,
name: Optional[Text] = '',
format_message_string: Optional[Text] = ''):
"""Upload a data frame to TimeSketch.
Args:
data (pandas.core.frame.DataFrame): the DataFrame to upload.
name (str): the name used for the timeline in Timesketch.
format_message_string (str): formatting string for the message column of
the data frame, eg: "{src_ip:s} to {dst_ip:s}, {bytes:d} bytes
transferred"'
Raises:
ValueError: if the dataframe cannot be uploaded to Timesketch or the
data is invalid.
"""
if not isinstance(data, pd.DataFrame):
raise ValueError(
('The data attribute is not a pandas DataFrame, please use curly '
'braces to expand variables.'))
if not name:
name = 'unknown_timeline'
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if not sketch:
raise ValueError('Unable to upload data frame, need to set sketch.')
result = None
import_helper = helper.ImportHelper()
timeline = None
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
if 'data_type' not in data:
data_type = utils.ask_question('What is the value of [data_type]?',
input_type=str)
if data_type:
streamer.set_data_type(data_type)
else:
data_types = data.data_type.unique()
data_type = data_types[0]
columns = list(data.columns)
streamer.set_config_helper(import_helper)
import_helper.configure_streamer(streamer,
data_type=data_type,
columns=columns)
if format_message_string:
streamer.set_message_format_string(format_message_string)
streamer.set_timeline_name(name)
streamer.add_data_frame(data)
streamer.flush()
result = streamer.response
timeline = streamer.timeline
if not result:
print('Unable to upload data.')
return
if not timeline.name:
print('Unable to import the timeline.')
return
print(
f'Timeline: [{timeline.id}]{timeline.name} - {timeline.description}\n'
f'Status: {timeline.status}')
def upload_file(my_sketch: sketch.Sketch, config_dict: Dict[str, any],
file_path: str) -> str:
"""Uploads a file to Timesketch.
Args:
my_sketch (sketch.Sketch): a sketch object to point to the sketch the
data will be imported to.
config_dict (dict): dict with settings for the importer.
file_path (str): the path to the file to upload.
Returns:
A tuple with the timeline object (timeline.Timeline) or None if not
able to upload the timeline as well as the celery task identification
for the indexing.
"""
if not my_sketch or not hasattr(my_sketch, 'id'):
return 'Sketch needs to be set'
_, _, file_extension = file_path.rpartition('.')
if file_extension.lower() not in ('plaso', 'csv', 'jsonl'):
return ('File needs to have one of the following extensions: '
'.plaso, .csv, '
'.jsonl (not {0:s})').format(file_extension.lower())
import_helper = helper.ImportHelper()
import_helper.add_config_dict(config_dict)
log_config_file = config_dict.get('log_config_file', '')
if log_config_file:
import_helper.add_config(log_config_file)
timeline = None
task_id = ''
logger.info('About to upload file.')
with importer.ImportStreamer() as streamer:
streamer.set_sketch(my_sketch)
streamer.set_config_helper(import_helper)
streamer.set_provider('CLI importer tool')
format_string = config_dict.get('message_format_string')
if format_string:
streamer.set_message_format_string(format_string)
timeline_name = config_dict.get('timeline_name')
if timeline_name:
streamer.set_timeline_name(timeline_name)
index_name = config_dict.get('index_name')
if index_name:
streamer.set_index_name(index_name)
time_desc = config_dict.get('timestamp_description')
if time_desc:
streamer.set_timestamp_description(time_desc)
entry_threshold = config_dict.get('entry_threshold')
if entry_threshold:
streamer.set_entry_threshold(entry_threshold)
size_threshold = config_dict.get('size_threshold')
if size_threshold:
streamer.set_filesize_threshold(size_threshold)
data_label = config_dict.get('data_label')
if data_label:
streamer.set_data_label(data_label)
context = config_dict.get('context')
if context:
streamer.set_upload_context(context)
else:
streamer.set_upload_context(' '.join(sys.argv))
streamer.add_file(file_path)
# Force a flush.
streamer.flush()
timeline = streamer.timeline
task_id = streamer.celery_task_id
logger.info('File upload completed.')
return timeline, task_id
def upload_file(
my_sketch: sketch.Sketch, config_dict: Dict[str, any],
file_path: str) -> str:
"""Uploads a file to Timesketch.
Args:
my_sketch (sketch.Sketch): a sketch object to point to the sketch the
data will be imported to.
config_dict (dict): dict with settings for the importer.
file_path (str): the path to the file to upload.
Returns:
A string with results (whether successful or not).
"""
if not my_sketch or not hasattr(my_sketch, 'id'):
return 'Sketch needs to be set'
_, _, file_extension = file_path.rpartition('.')
if file_extension.lower() not in ('plaso', 'csv', 'jsonl'):
return (
'File needs to have one of the following extensions: '
'.plaso, .csv, '
'.jsonl (not {0:s})').format(file_extension.lower())
import_helper = helper.ImportHelper()
import_helper.add_config_dict(config_dict)
log_config_file = config_dict.get('log_config_file', '')
if log_config_file:
import_helper.add_config(log_config_file)
with importer.ImportStreamer() as streamer:
streamer.set_sketch(my_sketch)
streamer.set_config_helper(import_helper)
streamer.set_provider('CLI importer tool')
format_string = config_dict.get('message_format_string')
if format_string:
streamer.set_message_format_string(format_string)
timeline_name = config_dict.get('timeline_name')
if timeline_name:
streamer.set_timeline_name(timeline_name)
index_name = config_dict.get('index_name')
if index_name:
streamer.set_index_name(index_name)
time_desc = config_dict.get('timestamp_description')
if time_desc:
streamer.set_timestamp_description(time_desc)
entry_threshold = config_dict.get('entry_threshold')
if entry_threshold:
streamer.set_entry_threshold(entry_threshold)
size_threshold = config_dict.get('size_threshold')
if size_threshold:
streamer.set_filesize_threshold(size_threshold)
data_label = config_dict.get('data_label')
if data_label:
streamer.set_data_label(data_label)
context = config_dict.get('context')
if context:
streamer.set_upload_context(context)
else:
streamer.set_upload_context(' '.join(sys.argv))
streamer.add_file(file_path)
return 'File got successfully uploaded to sketch: {0:d}'.format(
my_sketch.id)