def GetEstimatedYear(self): """Retrieves an estimate of the year. This function determines the year in the following manner: * see if the user provided a preferred year; * see if knowledge base defines a year e.g. derived from preprocessing; * determine the year based on the file entry metadata; * default to the current year; Returns: int: estimated year. """ # TODO: improve this method to get a more reliable estimate. # Preserve the year-less date and sort this out in the psort phase. if self._preferred_year: return self._preferred_year if self._knowledge_base.year: return self._knowledge_base.year # TODO: Find a decent way to actually calculate the correct year # instead of relying on stats object. year = self._GetEarliestYearFromFileEntry() if not year: year = self._GetLatestYearFromFileEntry() if not year: year = timelib.GetCurrentYear() return year
def _ParseLogLine(self, parser_mediator, structure, key): """Parse a single log line and produce an event object. Args: parser_mediator: A parser mediator object (instance of ParserMediator). key: An identification string indicating the name of the parsed structure. structure: A pyparsing.ParseResults object from a line in the log file. """ # TODO: improving this to get a valid year. if not self._year_use: self._year_use = parser_mediator.year if not self._year_use: # Get from the creation time of the file. self._year_use = self._GetYear(self.file_entry.GetStat(), parser_mediator.timezone) # If fail, get from the current time. if not self._year_use: self._year_use = timelib.GetCurrentYear() # Gap detected between years. month = timelib.MONTH_DICT.get(structure.month.lower()) if not self._last_month: self._last_month = month if month < self._last_month: self._year_use += 1 timestamp = self._GetTimestamp(structure.day, month, self._year_use, structure.time) if not timestamp: logging.debug(u'Invalid timestamp {0:s}'.format( structure.timestamp)) return self._last_month = month if key == u'logline': self.previous_structure = structure message = structure.message else: times = structure.times structure = self.previous_structure message = u'Repeated {0:d} times: {1:s}'.format( times, structure.message) # It uses CarsNotIn structure which leaves whitespaces # at the beginning of the sender and the caller. sender = structure.sender.strip() caller = structure.caller.strip() if not caller: caller = u'unknown' if not structure.security_api: security_api = u'unknown' else: security_api = structure.security_api event_object = MacSecuritydLogEvent(timestamp, structure, sender, structure.sender_pid, security_api, caller, message) parser_mediator.ProduceEvent(event_object)
def _ParseLogLine(self, structure): """Parse a logline and store appropriate attributes.""" # TODO: improving this to get a valid year. if not self._year_use: # Get from the creation time of the file. self._year_use = self._GetYear(self.file_entry.GetStat(), self.local_zone) # If fail, get from the current time. if not self._year_use: self._year_use = timelib.GetCurrentYear() # Gap detected between years. month = timelib.MONTH_DICT.get(structure.month.lower()) if not self._last_month: self._last_month = month if month < self._last_month: self._year_use += 1 timestamp = self._GetTimestamp(structure.day, month, self._year_use, structure.time) if not timestamp: logging.debug(u'Invalid timestamp {}'.format(structure.timestamp)) return self._last_month = month text = structure.text # Due to the use of CharsNotIn pyparsing structure contains whitespaces # that need to be removed. function = structure.function.strip() event_object = MacWifiLogEvent( timestamp, structure.agent, function, text, self._GetAction(structure.agent, function, text)) return event_object
def _ParseLogLine(self, parser_mediator, structure, key): """Parse a logline and store appropriate attributes. Args: parser_mediator: A parser mediator object (instance of ParserMediator). structure: log line of structure. key: type of line log (normal or repeated). Returns: Return an object MacAppFirewallLogEvent. """ # TODO: improve this to get a valid year. if not self._year_use: self._year_use = parser_mediator.year if not self._year_use: # Get from the creation time of the file. self._year_use = self._GetYear(self.file_entry.GetStat(), parser_mediator.timezone) # If fail, get from the current time. if not self._year_use: self._year_use = timelib.GetCurrentYear() # Gap detected between years. month = timelib.MONTH_DICT.get(structure.month.lower()) if not self._last_month: self._last_month = month if month < self._last_month: self._year_use += 1 timestamp = self._GetTimestamp(structure.day, month, self._year_use, structure.time) if not timestamp: logging.debug(u'Invalid timestamp {0:s}'.format( structure.timestamp)) return self._last_month = month # If the actual entry is a repeated entry, we take the basic information # from the previous entry, but using the timestmap from the actual entry. if key == 'logline': self.previous_structure = structure else: structure = self.previous_structure # Pyparsing reads in RAW, but the text is in UTF8. try: action = structure.action.decode('utf-8') except UnicodeDecodeError: logging.warning( u'Decode UTF8 failed, the message string may be cut short.') action = structure.action.decode('utf-8', 'ignore') # Due to the use of CharsNotIn pyparsing structure contains whitespaces # that need to be removed. process_name = structure.process_name.strip() event_object = MacAppFirewallLogEvent(timestamp, structure, process_name, action) return event_object
def testGetLatestYear(self): """Tests the GetLatestYear function.""" session = sessions.Session() storage_writer = fake_writer.FakeStorageWriter(session) parsers_mediator = self._CreateParserMediator(storage_writer) expected_latest_year = timelib.GetCurrentYear() latest_year = parsers_mediator.GetLatestYear() self.assertEqual(latest_year, expected_latest_year)
def _ParseLogLine(self, structure, key): """Parse a logline and store appropriate attributes. Args: structure: log line of structure. key: type of line log (normal or repeated). Returns: Return an object log event. """ # TODO: improving this to get a valid year. if not self._year_use: # Get from the creation time of the file. self._year_use = self._GetYear(self.file_entry.GetStat(), self.local_zone) # If fail, get from the current time. if not self._year_use: self._year_use = timelib.GetCurrentYear() # Gap detected between years. month = timelib.MONTH_DICT.get(structure.month.lower()) if not self._last_month: self._last_month = month if month < self._last_month: self._year_use += 1 timestamp = self._GetTimestamp( structure.day, month, self._year_use, structure.time) if not timestamp: logging.debug(u'Invalid timestamp {}'.format(structure.timestamp)) return self._last_month = month if key == 'logline': self.previous_structure = structure message = structure.message else: times = structure.times structure = self.previous_structure message = u'Repeated {} times: {}'.format( times, structure.message) # It uses CarsNotIn structure which leaves whitespaces # at the beginning of the sender and the caller. sender = structure.sender.strip() caller = structure.caller.strip() if not caller: caller = 'unknown' if not structure.security_api: security_api = u'unknown' else: security_api = structure.security_api return MacSecuritydLogEvent( timestamp, structure, sender, structure.sender_pid, security_api, caller, message)
def _GetYear(self, stat, timezone): """Retrieves the year either from the input file or from the settings.""" time = getattr(stat, u'crtime', 0) if not time: time = getattr(stat, u'ctime', 0) if not time: logging.error( u'Unable to determine correct year of log file, defaulting to ' u'current year.') return timelib.GetCurrentYear() try: timestamp = datetime.datetime.fromtimestamp(time, timezone) except ValueError as exception: logging.error(( u'Unable to determine correct year of log file with error: {0:s}, ' u'defaulting to current year.').format(exception)) return timelib.GetCurrentYear() return timestamp.year
def _GetYear(self, stat, zone): """Retrieves the year either from the input file or from the settings.""" time = getattr(stat, 'crtime', 0) if not time: time = getattr(stat, 'ctime', 0) if not time: logging.error(( 'Unable to determine correct year of syslog file, using current ' 'year')) return timelib.GetCurrentYear() try: timestamp = datetime.datetime.fromtimestamp(time, zone) except ValueError as exception: logging.error(( u'Unable to determine correct year of syslog file, using current ' u'one, with error: {0:s}').format(exception)) return timelib.GetCurrentYear() return timestamp.year
def GetLatestYear(self): """Retrieves the latest (newest) year for an event from a file. This function tries to determine the year based on the file entry metadata, if that fails the current year is used. Returns: int: year of the file entry or the current year. """ year = self._GetLatestYearFromFileEntry() if not year: year = timelib.GetCurrentYear() return year
def _GetYear(self, stat, zone): """Retrieves the year either from the input file or from the settings.""" time = getattr(stat, 'crtime', 0) if not time: time = getattr(stat, 'ctime', 0) if not time: current_year = timelib.GetCurrentYear() logging.error(( u'Unable to determine year of log file.\nDefaulting to: ' u'{0:d}').format(current_year)) return current_year try: timestamp = datetime.datetime.fromtimestamp(time, zone) except ValueError: current_year = timelib.GetCurrentYear() logging.error(( u'Unable to determine year of log file.\nDefaulting to: ' u'{0:d}').format(current_year)) return current_year return timestamp.year
def _ParseLogLine(self, parser_mediator, structure): """Parse a logline and store appropriate attributes. Args: parser_mediator: A parser mediator object (instance of ParserMediator). structure: A pyparsing.ParseResults object from a line in the log file. Returns: An event object (instance of EventObject) or None. """ # TODO: improving this to get a valid year. if not self._year_use: self._year_use = parser_mediator.year if not self._year_use: # Get from the creation time of the file. self._year_use = self._GetYear(self.file_entry.GetStat(), parser_mediator.timezone) # If fail, get from the current time. if not self._year_use: self._year_use = timelib.GetCurrentYear() # Gap detected between years. month = timelib.MONTH_DICT.get(structure.month.lower()) if not self._last_month: self._last_month = month if month < self._last_month: self._year_use += 1 timestamp = self._GetTimestamp(structure.day, month, self._year_use, structure.time) if not timestamp: logging.debug(u'Invalid timestamp {0:s}'.format( structure.timestamp)) return self._last_month = month text = structure.text # Due to the use of CharsNotIn pyparsing structure contains whitespaces # that need to be removed. function = structure.function.strip() action = self._GetAction(structure.agent, function, text) return MacWifiLogEvent(timestamp, structure.agent, function, text, action)
def GetEstimatedYear(self): """Retrieves an estimate of the year. This function first looks to see if the knowledge base defines a year, if not it tries to determine the year based on the file entry metadata, if that fails the current year is used. Returns: An integer containing the year of the file entry or None. """ # TODO: improve this method to get a more reliable estimate. # Preserve the year-less date and sort this out in the psort phase. year = self._knowledge_base.year if not year: # TODO: Find a decent way to actually calculate the correct year # instead of relying on stats object. year = self._GetYearFromFileEntry() if not year: year = timelib.GetCurrentYear() return year