def _fix_dict(self, my_dict): """Adjusts a dict with so that it can be uploaded to Timesketch. This function will take a dictionary and modify it. Summary of the changes are: * If "message" is not a key and a format message string has been defined, a message field is constructed. * If "datetime" is not a key, an attempt to generate it is made. * If "timestamp_desc" is not set but defined by the importer it is added. * All keys that start with an underscore ("_") are removed. Args: my_dict: a dictionary that may be missing few fields needed for Timesketch. """ if 'message' not in my_dict: format_string = ( self._format_string or utils.get_combined_message_string( mydict=my_dict)) my_dict['message'] = format_string.format(**my_dict) _ = my_dict.setdefault('timestamp_desc', self._timestamp_desc) if self._data_type: _ = my_dict.setdefault('data_type', self._data_type) if 'datetime' not in my_dict: date = '' if self._datetime_field: value = my_dict.get(self._datetime_field) if value: date = utils.get_datestring_from_value(value) if not date: for key in my_dict: key_string = key.lower() if 'time' not in key_string: continue if key_string == 'timestamp_desc': continue value = my_dict[key] date = utils.get_datestring_from_value(value) if date: break if date: my_dict['datetime'] = date else: my_dict['datetime'] = utils.get_datestring_from_value( my_dict['datetime']) # We don't want to include any columns that start with an underscore. underscore_columns = [x for x in my_dict if x.startswith('_')] if underscore_columns: for column in underscore_columns: del my_dict[column]
def _fix_data_frame(self, data_frame): """Returns a data frame with added columns for Timesketch upload. Args: data_frame: a pandas data frame. Returns: A pandas data frame with added columns needed for Timesketch. """ if "message" not in data_frame: format_string = self._format_string or utils.get_combined_message_string( dataframe=data_frame ) utils.format_data_frame(data_frame, format_string) if "timestamp_desc" not in data_frame: data_frame["timestamp_desc"] = self._timestamp_desc if self._data_type and "data_type" not in data_frame: data_frame["data_type"] = self._data_type if "datetime" not in data_frame: if self._datetime_field and self._datetime_field in data_frame: try: data_frame["timestamp"] = pandas.to_datetime( data_frame[self._datetime_field], utc=True ) except ValueError as e: logger.info( "Unable to convert timestamp in column: %s, error %s", self._datetime_field, e, ) else: for column in data_frame.columns[ data_frame.columns.str.contains("time", case=False) ]: if column.lower() == "timestamp_desc": continue try: data_frame["timestamp"] = pandas.to_datetime( data_frame[column], utc=True ) # We want the first successful timestamp value. break except ValueError as e: logger.info( "Unable to convert timestamp in column: " "%s, error %s", column, e, )