def _fix_dict(self, my_dict):
"""Adjusts a dict with so that it can be uploaded to Timesketch.
This function will take a dictionary and modify it. Summary of the
changes are:
* If "message" is not a key and a format message string has been
defined, a message field is constructed.
* If "datetime" is not a key, an attempt to generate it is made.
* If "timestamp_desc" is not set but defined by the importer it
is added.
* All keys that start with an underscore ("_") are removed.
Args:
my_dict: a dictionary that may be missing few fields needed
for Timesketch.
"""
if 'message' not in my_dict:
format_string = (
self._format_string or utils.get_combined_message_string(
mydict=my_dict))
my_dict['message'] = format_string.format(**my_dict)
_ = my_dict.setdefault('timestamp_desc', self._timestamp_desc)
if self._data_type:
_ = my_dict.setdefault('data_type', self._data_type)
if 'datetime' not in my_dict:
date = ''
if self._datetime_field:
value = my_dict.get(self._datetime_field)
if value:
date = utils.get_datestring_from_value(value)
if not date:
for key in my_dict:
key_string = key.lower()
if 'time' not in key_string:
continue
if key_string == 'timestamp_desc':
continue
value = my_dict[key]
date = utils.get_datestring_from_value(value)
if date:
break
if date:
my_dict['datetime'] = date
else:
my_dict['datetime'] = utils.get_datestring_from_value(
my_dict['datetime'])
# We don't want to include any columns that start with an underscore.
underscore_columns = [x for x in my_dict if x.startswith('_')]
if underscore_columns:
for column in underscore_columns:
del my_dict[column]
def _fix_data_frame(self, data_frame):
"""Returns a data frame with added columns for Timesketch upload.
Args:
data_frame: a pandas data frame.
Returns:
A pandas data frame with added columns needed for Timesketch.
"""
if "message" not in data_frame:
format_string = self._format_string or utils.get_combined_message_string(
dataframe=data_frame
)
utils.format_data_frame(data_frame, format_string)
if "timestamp_desc" not in data_frame:
data_frame["timestamp_desc"] = self._timestamp_desc
if self._data_type and "data_type" not in data_frame:
data_frame["data_type"] = self._data_type
if "datetime" not in data_frame:
if self._datetime_field and self._datetime_field in data_frame:
try:
data_frame["timestamp"] = pandas.to_datetime(
data_frame[self._datetime_field], utc=True
)
except ValueError as e:
logger.info(
"Unable to convert timestamp in column: %s, error %s",
self._datetime_field,
e,
)
else:
for column in data_frame.columns[
data_frame.columns.str.contains("time", case=False)
]:
if column.lower() == "timestamp_desc":
continue
try:
data_frame["timestamp"] = pandas.to_datetime(
data_frame[column], utc=True
)
# We want the first successful timestamp value.
break
except ValueError as e:
logger.info(
"Unable to convert timestamp in column: " "%s, error %s",
column,
e,
)