def GetEstimatedYear(self):
"""Retrieves an estimate of the year.
This function determines the year in the following manner:
* see if the user provided a preferred year;
* see if knowledge base defines a year e.g. derived from preprocessing;
* determine the year based on the file entry metadata;
* default to the current year;
Returns:
int: estimated year.
"""
# TODO: improve this method to get a more reliable estimate.
# Preserve the year-less date and sort this out in the psort phase.
if self._preferred_year:
return self._preferred_year
if self._knowledge_base.year:
return self._knowledge_base.year
# TODO: Find a decent way to actually calculate the correct year
# instead of relying on stats object.
year = self._GetEarliestYearFromFileEntry()
if not year:
year = self._GetLatestYearFromFileEntry()
if not year:
year = timelib.GetCurrentYear()
return year
def _ParseLogLine(self, parser_mediator, structure, key):
"""Parse a single log line and produce an event object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
key: An identification string indicating the name of the parsed
structure.
structure: A pyparsing.ParseResults object from a line in the
log file.
"""
# TODO: improving this to get a valid year.
if not self._year_use:
self._year_use = parser_mediator.year
if not self._year_use:
# Get from the creation time of the file.
self._year_use = self._GetYear(self.file_entry.GetStat(),
parser_mediator.timezone)
# If fail, get from the current time.
if not self._year_use:
self._year_use = timelib.GetCurrentYear()
# Gap detected between years.
month = timelib.MONTH_DICT.get(structure.month.lower())
if not self._last_month:
self._last_month = month
if month < self._last_month:
self._year_use += 1
timestamp = self._GetTimestamp(structure.day, month, self._year_use,
structure.time)
if not timestamp:
logging.debug(u'Invalid timestamp {0:s}'.format(
structure.timestamp))
return
self._last_month = month
if key == u'logline':
self.previous_structure = structure
message = structure.message
else:
times = structure.times
structure = self.previous_structure
message = u'Repeated {0:d} times: {1:s}'.format(
times, structure.message)
# It uses CarsNotIn structure which leaves whitespaces
# at the beginning of the sender and the caller.
sender = structure.sender.strip()
caller = structure.caller.strip()
if not caller:
caller = u'unknown'
if not structure.security_api:
security_api = u'unknown'
else:
security_api = structure.security_api
event_object = MacSecuritydLogEvent(timestamp, structure, sender,
structure.sender_pid, security_api,
caller, message)
parser_mediator.ProduceEvent(event_object)
def _ParseLogLine(self, structure):
"""Parse a logline and store appropriate attributes."""
# TODO: improving this to get a valid year.
if not self._year_use:
# Get from the creation time of the file.
self._year_use = self._GetYear(self.file_entry.GetStat(),
self.local_zone)
# If fail, get from the current time.
if not self._year_use:
self._year_use = timelib.GetCurrentYear()
# Gap detected between years.
month = timelib.MONTH_DICT.get(structure.month.lower())
if not self._last_month:
self._last_month = month
if month < self._last_month:
self._year_use += 1
timestamp = self._GetTimestamp(structure.day, month, self._year_use,
structure.time)
if not timestamp:
logging.debug(u'Invalid timestamp {}'.format(structure.timestamp))
return
self._last_month = month
text = structure.text
# Due to the use of CharsNotIn pyparsing structure contains whitespaces
# that need to be removed.
function = structure.function.strip()
event_object = MacWifiLogEvent(
timestamp, structure.agent, function, text,
self._GetAction(structure.agent, function, text))
return event_object
def _ParseLogLine(self, parser_mediator, structure, key):
"""Parse a logline and store appropriate attributes.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
structure: log line of structure.
key: type of line log (normal or repeated).
Returns:
Return an object MacAppFirewallLogEvent.
"""
# TODO: improve this to get a valid year.
if not self._year_use:
self._year_use = parser_mediator.year
if not self._year_use:
# Get from the creation time of the file.
self._year_use = self._GetYear(self.file_entry.GetStat(),
parser_mediator.timezone)
# If fail, get from the current time.
if not self._year_use:
self._year_use = timelib.GetCurrentYear()
# Gap detected between years.
month = timelib.MONTH_DICT.get(structure.month.lower())
if not self._last_month:
self._last_month = month
if month < self._last_month:
self._year_use += 1
timestamp = self._GetTimestamp(structure.day, month, self._year_use,
structure.time)
if not timestamp:
logging.debug(u'Invalid timestamp {0:s}'.format(
structure.timestamp))
return
self._last_month = month
# If the actual entry is a repeated entry, we take the basic information
# from the previous entry, but using the timestmap from the actual entry.
if key == 'logline':
self.previous_structure = structure
else:
structure = self.previous_structure
# Pyparsing reads in RAW, but the text is in UTF8.
try:
action = structure.action.decode('utf-8')
except UnicodeDecodeError:
logging.warning(
u'Decode UTF8 failed, the message string may be cut short.')
action = structure.action.decode('utf-8', 'ignore')
# Due to the use of CharsNotIn pyparsing structure contains whitespaces
# that need to be removed.
process_name = structure.process_name.strip()
event_object = MacAppFirewallLogEvent(timestamp, structure,
process_name, action)
return event_object
def testGetLatestYear(self):
"""Tests the GetLatestYear function."""
session = sessions.Session()
storage_writer = fake_writer.FakeStorageWriter(session)
parsers_mediator = self._CreateParserMediator(storage_writer)
expected_latest_year = timelib.GetCurrentYear()
latest_year = parsers_mediator.GetLatestYear()
self.assertEqual(latest_year, expected_latest_year)
def _ParseLogLine(self, structure, key):
"""Parse a logline and store appropriate attributes.
Args:
structure: log line of structure.
key: type of line log (normal or repeated).
Returns:
Return an object log event.
"""
# TODO: improving this to get a valid year.
if not self._year_use:
# Get from the creation time of the file.
self._year_use = self._GetYear(self.file_entry.GetStat(), self.local_zone)
# If fail, get from the current time.
if not self._year_use:
self._year_use = timelib.GetCurrentYear()
# Gap detected between years.
month = timelib.MONTH_DICT.get(structure.month.lower())
if not self._last_month:
self._last_month = month
if month < self._last_month:
self._year_use += 1
timestamp = self._GetTimestamp(
structure.day,
month,
self._year_use,
structure.time)
if not timestamp:
logging.debug(u'Invalid timestamp {}'.format(structure.timestamp))
return
self._last_month = month
if key == 'logline':
self.previous_structure = structure
message = structure.message
else:
times = structure.times
structure = self.previous_structure
message = u'Repeated {} times: {}'.format(
times, structure.message)
# It uses CarsNotIn structure which leaves whitespaces
# at the beginning of the sender and the caller.
sender = structure.sender.strip()
caller = structure.caller.strip()
if not caller:
caller = 'unknown'
if not structure.security_api:
security_api = u'unknown'
else:
security_api = structure.security_api
return MacSecuritydLogEvent(
timestamp, structure, sender, structure.sender_pid, security_api,
caller, message)
def _GetYear(self, stat, timezone):
"""Retrieves the year either from the input file or from the settings."""
time = getattr(stat, u'crtime', 0)
if not time:
time = getattr(stat, u'ctime', 0)
if not time:
logging.error(
u'Unable to determine correct year of log file, defaulting to '
u'current year.')
return timelib.GetCurrentYear()
try:
timestamp = datetime.datetime.fromtimestamp(time, timezone)
except ValueError as exception:
logging.error((
u'Unable to determine correct year of log file with error: {0:s}, '
u'defaulting to current year.').format(exception))
return timelib.GetCurrentYear()
return timestamp.year
def _GetYear(self, stat, zone):
"""Retrieves the year either from the input file or from the settings."""
time = getattr(stat, 'crtime', 0)
if not time:
time = getattr(stat, 'ctime', 0)
if not time:
logging.error((
'Unable to determine correct year of syslog file, using current '
'year'))
return timelib.GetCurrentYear()
try:
timestamp = datetime.datetime.fromtimestamp(time, zone)
except ValueError as exception:
logging.error((
u'Unable to determine correct year of syslog file, using current '
u'one, with error: {0:s}').format(exception))
return timelib.GetCurrentYear()
return timestamp.year
def GetLatestYear(self):
"""Retrieves the latest (newest) year for an event from a file.
This function tries to determine the year based on the file entry metadata,
if that fails the current year is used.
Returns:
int: year of the file entry or the current year.
"""
year = self._GetLatestYearFromFileEntry()
if not year:
year = timelib.GetCurrentYear()
return year
def _GetYear(self, stat, zone):
"""Retrieves the year either from the input file or from the settings."""
time = getattr(stat, 'crtime', 0)
if not time:
time = getattr(stat, 'ctime', 0)
if not time:
current_year = timelib.GetCurrentYear()
logging.error((
u'Unable to determine year of log file.\nDefaulting to: '
u'{0:d}').format(current_year))
return current_year
try:
timestamp = datetime.datetime.fromtimestamp(time, zone)
except ValueError:
current_year = timelib.GetCurrentYear()
logging.error((
u'Unable to determine year of log file.\nDefaulting to: '
u'{0:d}').format(current_year))
return current_year
return timestamp.year
def _ParseLogLine(self, parser_mediator, structure):
"""Parse a logline and store appropriate attributes.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
structure: A pyparsing.ParseResults object from a line in the
log file.
Returns:
An event object (instance of EventObject) or None.
"""
# TODO: improving this to get a valid year.
if not self._year_use:
self._year_use = parser_mediator.year
if not self._year_use:
# Get from the creation time of the file.
self._year_use = self._GetYear(self.file_entry.GetStat(),
parser_mediator.timezone)
# If fail, get from the current time.
if not self._year_use:
self._year_use = timelib.GetCurrentYear()
# Gap detected between years.
month = timelib.MONTH_DICT.get(structure.month.lower())
if not self._last_month:
self._last_month = month
if month < self._last_month:
self._year_use += 1
timestamp = self._GetTimestamp(structure.day, month, self._year_use,
structure.time)
if not timestamp:
logging.debug(u'Invalid timestamp {0:s}'.format(
structure.timestamp))
return
self._last_month = month
text = structure.text
# Due to the use of CharsNotIn pyparsing structure contains whitespaces
# that need to be removed.
function = structure.function.strip()
action = self._GetAction(structure.agent, function, text)
return MacWifiLogEvent(timestamp, structure.agent, function, text,
action)
def GetEstimatedYear(self):
"""Retrieves an estimate of the year.
This function first looks to see if the knowledge base defines a year, if
not it tries to determine the year based on the file entry metadata, if
that fails the current year is used.
Returns:
An integer containing the year of the file entry or None.
"""
# TODO: improve this method to get a more reliable estimate.
# Preserve the year-less date and sort this out in the psort phase.
year = self._knowledge_base.year
if not year:
# TODO: Find a decent way to actually calculate the correct year
# instead of relying on stats object.
year = self._GetYearFromFileEntry()
if not year:
year = timelib.GetCurrentYear()
return year
