def _ConvertToTimestamp(self, date, time, timezone):
"""Converts date and time values into a timestamp.
The date and time are made up of two strings, the date and the time,
separated by a tab. The time is in local time. The month and day can
be either 1 or 2 characters long, e.g.: 7/30/2013\\t10:22:48 AM
Args:
date (str): date.
time (str): time.
timezone (pytz.timezone): timezone of the date and time.
Returns:
int: a timestamp integer containing the number of micro seconds since
January 1, 1970, 00:00:00 UTC.
Raises:
TimestampError: if the timestamp is badly formed or unable to transfer
the supplied date and time into a timestamp.
"""
# TODO: check if this is correct, likely not date or not time
# is more accurate.
if not date and not time:
raise errors.TimestampError(
'Unable to extract timestamp from McAfee AV logline.')
# TODO: Figure out how McAfee sets Day First and use that here.
# The in-file time format is '07/30/2013\t10:22:48 AM'.
try:
time_string = '{0:s} {1:s}'.format(date, time)
except UnicodeDecodeError:
raise errors.TimestampError('Unable to form a timestamp string.')
return timelib.Timestamp.FromTimeString(time_string, timezone=timezone)
def ParseLine(self, parser_mediator):
"""Return an event object extracted from the current line.
Args:
parser_mediator: a parser mediator object (instance of ParserMediator).
Raises:
TimestampError: if time is not defined or invalid.
"""
if not self.attributes[u'time']:
raise errors.TimestampError(
u'Unable to parse log line: {0:s} - missing time.'.format(
self.PrintLine()))
if not self.attributes[u'iyear']:
raise errors.TimestampError(
u'Unable to parse log line: {0:s} - missing year.'.format(
self.PrintLine()))
times = self.attributes[u'time'].split(u':')
if self.local_zone:
timezone = parser_mediator.timezone
else:
timezone = pytz.UTC
if len(times) < 3:
raise errors.TimestampError(
u'Unable to parse log line - unsupported format: {0:s}'.format(
self.PrintLine()))
try:
secs = times[2].split('.')
if len(secs) == 2:
sec, us = secs
else:
sec = times[2]
us = 0
timestamp = timelib.Timestamp.FromTimeParts(
int(self.attributes[u'iyear']),
self.attributes[u'imonth'],
self.attributes[u'iday'],
int(times[0]),
int(times[1]),
int(sec),
microseconds=int(us),
timezone=timezone)
except ValueError as exception:
raise errors.TimestampError(
u'Unable to parse log line: {0:s} with error: {1:s}'.format(
self.PrintLine(), exception))
event_object = self.CreateEvent(timestamp,
getattr(self, u'entry_offset', 0),
self.attributes)
parser_mediator.ProduceEvent(event_object)
def ParseRecord(self, parser_mediator, key, structure):
"""Parse the record and return an SCCM log event object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
key: An identification string indicating the name of the parsed
structure.
structure: A pyparsing.ParseResults object from a line in the
log file.
Raises:
TimestampError: when a non-int value for microseconds is encountered.
"""
# Sometimes, SCCM logs will exhibit a seven-digit sub-second precision
# (100 nanosecond intervals). Using six-digit precision because
# timestamps are in microseconds.
if len(structure.microsecond) > 6:
structure.microsecond = structure.microsecond[0:6]
try:
microsecond = int(structure.microsecond, 10)
except ValueError:
raise errors.TimestampError(
u'Unable to read number of microseconds value.')
# 3-digit precision is milliseconds,
# so multiply by 1000 to convert to microseconds
if len(structure.microsecond) == 3:
microsecond = microsecond * 1000
timestamp = timelib.Timestamp.FromTimeParts(year=structure.year,
month=structure.month,
day=structure.day,
hour=structure.hour,
minutes=structure.minute,
seconds=structure.second,
microseconds=microsecond)
# If an offset is given for the event, apply the offset to convert to UTC.
if u'offset' in key:
try:
delta_microseconds = int(structure.utc_offset_minutes[1:], 10)
except (IndexError, ValueError):
raise errors.TimestampError(
u'Unable to parse minute offset from UTC.')
delta_microseconds *= self._MICRO_SECONDS_PER_MINUTE
if structure.utc_offset_minutes[0] == u'-':
delta_microseconds = -delta_microseconds
timestamp += delta_microseconds
event_object = SCCMLogEvent(timestamp, 0, structure)
parser_mediator.ProduceEvent(event_object)
def FromSystemtime(cls, systemtime):
"""Converts a SYSTEMTIME structure into a timestamp.
The SYSTEMTIME structure is a 128-bit struct containing 8 little endian
16-bit integers structured like so:
struct {
WORD year,
WORD month,
WORD day_of_week,
WORD day,
WORD hour,
WORD minute,
WORD second,
WORD millisecond
}
Args:
systemtime (bytes): 128-bit SYSTEMTIME timestamp value.
Returns:
int: timestamp, which contains the number of micro seconds since
January 1, 1970, 00:00:00 UTC or 0 on error.
"""
try:
timestamp = cls.SYSTEMTIME_STRUCT.parse(systemtime)
except construct.ConstructError as exception:
raise errors.TimestampError(
u'Unable to create timestamp from {0:s} with error: {1:s}'.format(
systemtime, exception))
return cls.FromTimeParts(
year=timestamp.year, month=timestamp.month,
day=timestamp.day, hour=timestamp.hour,
minutes=timestamp.minutes, seconds=timestamp.seconds,
microseconds=(
timestamp.milliseconds * cls.MILLI_SECONDS_TO_MICRO_SECONDS))
def _GetTimestampFromEntry(self, structure):
"""Parses a timestamp from a TIME entry structure.
Args:
structure: TIME entry structure:
year: String with the number of the year.
month: String with the number of the month.
day: String with the number of the day.
hour: String with the number of the month.
minute: String with the number of the minute.
second: String with the number of the second.
Returns:
The timestamp which is an integer containing the number of micro seconds
since January 1, 1970, 00:00:00 UTC.
Raises:
TimestampError: if the timestamp cannot be created from the date and
time values.
"""
try:
year = int(structure.year, 10)
month = int(structure.month, 10)
day = int(structure.day, 10)
hours = int(structure.hour, 10)
minutes = int(structure.minute, 10)
seconds = int(structure.second, 10)
except ValueError:
raise errors.TimestampError(
u'Invalid keychain time {0!s}'.format(structure))
return timelib.Timestamp.FromTimeParts(year, month, day, hours,
minutes, seconds)
def _ConvertToTimestamp(self, structure, timezone, year=0):
"""Converts date and time values into a timestamp.
Args:
structure: a log line structure (instance of pyparsing.ParseResults)
that contains the log header.
timezone: The timezone object.
year: Optional current year.
Returns:
The timestamp which is an integer containing the number of micro seconds
since January 1, 1970, 00:00:00 UTC.
Raises:
TimestampError: if the timestamp cannot be created from the date and
time values.
"""
month = timelib.MONTH_DICT.get(structure.month_name.lower(), None)
if not month:
raise errors.TimestampError(u'Unsupport month name: {0:s}'.format(
structure.month_name))
hour, minute, second = structure.time
if not year:
# This condition could happen when parsing the header line: if unable
# to get a valid year, returns a '0' timestamp, thus preventing any
# log line parsing (since xchat_year is unset to '0') until a new good
# (it means supported) header with a valid year information is found.
# TODO: reconsider this behaviour.
year = structure.get(u'year', 0)
if not year:
raise errors.TimestampError(u'Missing year.')
self._xchat_year = year
day = structure.get(u'day', 0)
return timelib.Timestamp.FromTimeParts(year,
month,
day,
hour,
minute,
second,
timezone=timezone)
def FromTimeString(cls,
time_string,
dayfirst=False,
gmt_as_timezone=True,
timezone=pytz.UTC):
"""Converts a string containing a date and time value into a timestamp.
Args:
time_string: String that contains a date and time value.
dayfirst: An optional boolean argument. If set to true then the
parser will change the precedence in which it parses timestamps
from MM-DD-YYYY to DD-MM-YYYY (and YYYY-MM-DD will be
YYYY-DD-MM, etc).
gmt_as_timezone: Sometimes the dateutil parser will interpret GMT and UTC
the same way, that is not make a distinction. By default
this is set to true, that is GMT can be interpreted
differently than UTC. If that is not the expected result
this attribute can be set to false.
timezone: Optional timezone object (instance of pytz.timezone) that
the data and time value in the string represents. This value
is used when the timezone cannot be determined from the string.
Returns:
The timestamp which is an integer containing the number of micro seconds
since January 1, 1970, 00:00:00 UTC or 0 on error.
Raises:
TimestampError: if the time string could not be parsed.
"""
if not gmt_as_timezone and time_string.endswith(' GMT'):
time_string = '{0:s}UTC'.format(time_string[:-3])
try:
# TODO: deprecate the use of dateutil parser.
datetime_object = dateutil.parser.parse(time_string,
dayfirst=dayfirst)
except (TypeError, ValueError) as exception:
raise errors.TimestampError((
'Unable to convert time string: {0:s} in to a datetime object '
'with error: {1!s}').format(time_string, exception))
if datetime_object.tzinfo:
datetime_object = datetime_object.astimezone(pytz.UTC)
else:
datetime_object = timezone.localize(datetime_object)
posix_time = int(calendar.timegm(datetime_object.utctimetuple()))
timestamp = posix_time * definitions.MICROSECONDS_PER_SECOND
return timestamp + datetime_object.microsecond
def _GetTimestamp(self, date, time, timezone):
"""Determines a timestamp from the time string.
The date and time are made up of two strings, the date and the time,
separated by a tab. The time is in local time. The month and day can
be either 1 or 2 characters long, e.g.: 7/30/2013\\t10:22:48 AM
Args:
date: the string representing the date.
time: the string representing the time.
timezone: timezone (instance of pytz.timezone) that the data and time
values represent.
Returns:
The timestamp time value. The timestamp contains the number of
microseconds since Jan 1, 1970 00:00:00 UTC or None if the time string
could not be parsed.
Raises:
TimestampError: if the timestamp is badly formed or unable to transfer
the supplied date and time into a timestamp.
"""
# TODO: check if this is correct, likely not date or not time
# is more accurate.
if not (date and time):
raise errors.TimestampError(
u'Unable to extract timestamp from McAfee AV logline.')
# TODO: Figure out how McAfee sets Day First and use that here.
# The in-file time format is '07/30/2013\t10:22:48 AM'.
try:
time_string = u'{0:s} {1:s}'.format(date, time)
except UnicodeDecodeError:
raise errors.TimestampError(u'Unable to form a timestamp string.')
return timelib.Timestamp.FromTimeString(time_string, timezone=timezone)
def FromTimeParts(cls,
year,
month,
day,
hour,
minutes,
seconds,
microseconds=0,
timezone=pytz.UTC):
"""Converts a list of time entries to a timestamp.
Args:
year: An integer representing the year.
month: An integer between 1 and 12.
day: An integer representing the number of day in the month.
hour: An integer representing the hour, 0 <= hour < 24.
minutes: An integer, 0 <= minute < 60.
seconds: An integer, 0 <= second < 60.
microseconds: Optional number of microseconds ranging from:
0 <= microsecond < 1000000.
timezone: Optional timezone (instance of pytz.timezone).
Returns:
The timestamp which is an integer containing the number of micro seconds
since January 1, 1970, 00:00:00 UTC or 0 on error.
Raises:
TimestampError: if the timestamp cannot be created from the time parts.
"""
try:
date = datetime.datetime(year, month, day, hour, minutes, seconds,
microseconds)
except ValueError as exception:
raise errors.TimestampError(
(u'Unable to create timestamp from {0:04d}-{1:02d}-{2:02d} '
u'{3:02d}:{4:02d}:{5:02d}.{6:06d} with error: {7:s}').format(
year, month, day, hour, minutes, seconds, microseconds,
exception))
if isinstance(timezone, py2to3.STRING_TYPES):
timezone = pytz.timezone(timezone)
date_use = timezone.localize(date)
posix_time = int(calendar.timegm(date_use.utctimetuple()))
return cls.FromPosixTime(posix_time) + microseconds
def ParseLine(self, parser_mediator):
"""Parse a single line from the SELinux audit file.
This method extends the one from TextParser slightly, creating a
SELinux event with the timestamp (UTC) taken from log entries.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
Raises:
TimestampError: if timestamp is not defined.
"""
if not self.timestamp:
raise errors.TimestampError(
u'Unable to parse log line - missing timestamp.')
offset = getattr(self, u'entry_offset', 0)
event_object = SELinuxLineEvent(self.timestamp, offset, self.attributes)
parser_mediator.ProduceEvent(event_object)
self.timestamp = 0
def _CreateDateTime(self, date_string, time_string):
"""Creates a date time value from the date time strings.
The format stores the date and time as 2 separate strings separated by
a tab. The time is in local time. The month and day can be either 1 or 2
characters long, for example: "7/30/2013\\t10:22:48 AM"
Args:
date_string (str): date string.
time_string (str): time string.
Returns:
dfdatetime.TimeElements: date time object.
Raises:
TimestampError: if the date or time string cannot be converted in
a date time object.
"""
if not date_string and not time_string:
raise errors.TimestampError('Missing date or time string.')
# TODO: Figure out how McAfee sets Day First and use that here.
# The in-file time format is '07/30/2013\t10:22:48 AM'.
try:
month_string, day_of_month_string, year_string = date_string.split('/')
year = int(year_string, 10)
month = int(month_string, 10)
day_of_month = int(day_of_month_string, 10)
except (AttributeError, ValueError):
raise errors.TimestampError('Unsupported date string: {0:s}'.format(
date_string))
try:
time_value, time_suffix = time_string.split(' ')
hours_string, minutes_string, seconds_string = time_value.split(':')
hours = int(hours_string, 10)
minutes = int(minutes_string, 10)
seconds = int(seconds_string, 10)
except (AttributeError, ValueError):
raise errors.TimestampError('Unsupported time string: {0:s}'.format(
time_string))
if time_suffix == 'PM':
hours += 12
elif time_suffix != 'AM':
raise errors.TimestampError('Unsupported time suffix: {0:s}.'.format(
time_suffix))
time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
raise errors.TimestampError(
'Unsupported date and time strings: {0:s} {1:s}'.format(
date_string, time_string))
date_time.is_local_time = True
return date_time
def ParseRecord(self, parser_mediator, key, structure):
"""Parse the record and return an SCCM log event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
Raises:
ParseError: when the structure type is unknown.
TimestampError: when a non-int value for microseconds is encountered.
"""
if key not in (u'log_entry', u'log_entry_at_end', u'log_entry_offset',
u'log_entry_offset_at_end'):
raise errors.ParseError(
u'Unable to parse record, unknown structure: {0:s}'.format(
key))
# Sometimes, SCCM logs will exhibit a seven-digit sub-second precision
# (100 nanosecond intervals). Using six-digit precision because
# timestamps are in microseconds.
if len(structure.microsecond) > 6:
structure.microsecond = structure.microsecond[0:6]
try:
microseconds = int(structure.microsecond, 10)
except ValueError as exception:
parser_mediator.ProduceExtractionError(
u'unable to determine microseconds with error: {0:s}'.format(
exception))
return
# 3-digit precision is milliseconds,
# so multiply by 1000 to convert to microseconds
if len(structure.microsecond) == 3:
microseconds *= 1000
try:
timestamp = timelib.Timestamp.FromTimeParts(
structure.year, structure.month, structure.day, structure.hour,
structure.minute, structure.second, microseconds)
except errors.TimestampError as exception:
timestamp = timelib.Timestamp.NONE_TIMESTAMP
parser_mediator.ProduceExtractionError(
u'unable to determine timestamp with error: {0:s}'.format(
exception))
# If an offset is given for the event, apply the offset to convert to UTC.
if timestamp and u'offset' in key:
try:
delta_microseconds = int(structure.utc_offset_minutes[1:], 10)
except (IndexError, ValueError) as exception:
raise errors.TimestampError(
u'Unable to parse minute offset from UTC with error: {0:s}.'
.format(exception))
delta_microseconds *= self._MICRO_SECONDS_PER_MINUTE
if structure.utc_offset_minutes[0] == u'-':
delta_microseconds = -delta_microseconds
timestamp += delta_microseconds
event_data = SCCMLogEventData()
event_data.component = structure.component
# TODO: pass line number to offset or remove.
event_data.offset = 0
event_data.text = structure.text
event = time_events.TimestampEvent(
timestamp, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)