def test_gif_quantization(name):
"""Encode unquantized image and compare result to reference."""
# load un-quantized PNG
png_path = get_test_gif_path(name + '.png')
rgba = run('convert {} rgba:-', png_path)
rgba_tuples = [tuple(col) for col in construct.Array(
lambda ctx: len(rgba) / 4,
construct.Array(4, construct.ULInt8('col')),
).parse(rgba)]
# load expected quantized gif
ref = load_reference_gif(get_test_gif_path(name + '.gif'))
# convert PNG to quantized GIF
gif = GIF()
gif.size = ref['size']
gif.images = [Image(rgba_tuples, ref['size'], ref['images'][0]['delay'])]
gif.loop_count = ref['loop']
gif.comment = ref['comment']
with tempfile.NamedTemporaryFile() as encoded_file:
gif.save(encoded_file)
encoded_file.flush()
# load resulting gif and compare to reference
reencoded_ref = load_reference_gif(encoded_file.name)
assert ref == reencoded_ref
def _get_data_subblocks(name):
"""Return Adapter to parse GIF data sub-blocks."""
return construct.ExprAdapter(
construct.Struct(
name,
construct.RepeatUntil(
lambda obj, ctx: obj.block_size == 0x00,
construct.Struct(
'blocks',
construct.ULInt8('block_size'),
construct.Bytes('data_values', lambda ctx: ctx.block_size),
),
),
),
# from comment string, build Containers
encoder=lambda obj, ctx: construct.Container(blocks=[
construct.Container(
block_size=len(chunk),
data_values=chunk,
) for chunk in [obj[i:i + 255] for i in xrange(0, len(obj), 255)]
] + [construct.Container(block_size=0, data_values='')], ),
# from Containers, build comment string
decoder=lambda obj, ctx: ''.join(dsb.data_values
for dsb in obj.blocks),
)
def __init__(self):
super(AudioHandler, self).__init__()
self.header_base = construct.BitStruct(
'ASTRMBaseHeader', construct.BitField('fmt', 3),
construct.Bit('channel'), construct.Flag('vibrate'),
construct.Bit('packet_type'), construct.BitField('seq_id', 10),
construct.BitField('payload_size', 16))
self.header_aud = construct.Struct('ASTRMAudioHeader',
construct.ULInt32('timestamp'))
self.header_msg = construct.Struct(
'ASTRMMsgHeader',
# This is kind of a hack, (there are two timestamp fields, which one is used
# depends on packet_type
construct.ULInt32('timestamp_audio'),
construct.ULInt32('timestamp'),
construct.Array(2, construct.ULInt32('freq_0')), # -> mc_video
construct.Array(2, construct.ULInt32('freq_1')), # -> mc_sync
construct.ULInt8('vid_format'),
construct.Padding(3))
self.header = construct.Struct(
'ASTRMHeader', construct.Embed(self.header_base),
construct.Switch('format_hdr',
lambda ctx: ctx.packet_type, {
0: construct.Embed(self.header_aud),
1: construct.Embed(self.header_msg),
},
default=construct.Pass))
def MakeConstantRow():
return construct.Struct('ConstantRow',
construct.ULInt8('Type'),
construct.Padding(1, strict = True),
MDTag.HasConstant.parse('Parent'),
MDTag.BlobHeapRef.parse('Value')
)
def load_reference_gif(filename):
"""Load a GIF that we can use as a reference for testing.
exiftool and ImageMagick are used here as reference decoders.
"""
# get comment, size, loop count from exiftool
exiftool_json = json.loads(run('exiftool -j {}', filename))[0]
comment = exiftool_json.get('Comment', None)
size = (exiftool_json['ImageWidth'], exiftool_json['ImageHeight'])
loop = exiftool_json.get('AnimationIterations', 0)
loop = 0 if loop == 'Infinite' else loop + 1
# get delay for each frame from ImageMagick
delays = run('identify -format %T, {}', filename)
delays = [int(d) * 10 for d in delays.split(',')[:-1]]
# Work around bugs in ImageMagick. Animations need to be coalesced so we
# can extract frames with disposal methods applied correctly. But for still
# images, do not coalesce because it can corrupt the image.
if exiftool_json.get('FrameCount', 1) > 1:
tmp_filename = '{}.coalesced.gif'.format(filename)
run('convert {} -coalesce {}', filename, tmp_filename)
else:
tmp_filename = filename
images = []
for i in xrange(len(delays)):
# get the coalesced frame data as RGBA using ImageMagick
rgba = run('convert {}[{}] rgba:-', tmp_filename, i)
rgba_tuples = [tuple(col) for col in construct.Array(
lambda ctx: len(rgba) / 4,
construct.Array(4, construct.ULInt8('col')),
).parse(rgba)]
images.append({
'data': rgba_tuples,
'delay': delays[i],
})
return {
'size': size,
'loop': loop,
'images': images,
'comment': comment,
}
def __init__(s):
super(ServiceASTRM, s).__init__()
s.header_base = construct.BitStruct('ASTRMBaseHeader',
construct.BitField('fmt', 3),
construct.Bit('channel'),
construct.Flag('vibrate'),
construct.Bit('packet_type'),
construct.BitField('seq_id', 10),
construct.BitField('payload_size', 16)
)
s.header_aud = construct.Struct('ASTRMAudioHeader',
construct.ULInt32('timestamp'),
# construct.Array(lambda ctx: ctx.payload_size, construct.UBInt8("data"))
)
s.header_msg = construct.Struct('ASTRMMsgHeader',
# This is kind of a hack, (there are two timestamp fields, which one is used depends on packet_type
construct.ULInt32('timestamp_audio'),
construct.ULInt32('timestamp'),
construct.Array(2, construct.ULInt32('freq_0')), # -> mc_video
construct.Array(2, construct.ULInt32('freq_1')), # -> mc_sync
construct.ULInt8('vid_format'),
construct.Padding(3)
)
s.header = construct.Struct('ASTRMHeader',
construct.Embed(s.header_base),
construct.Switch('format_hdr', lambda ctx: ctx.packet_type,
{
0 : construct.Embed(s.header_aud),
1 : construct.Embed(s.header_msg),
},
default = construct.Pass
)
)
s.is_streaming = False
s.p = pyaudio.PyAudio()
s.stream = None
s.pa_num_bufs = 15
s.pa_ring = [array.array('H', '\0' * 416 * 2)] * s.pa_num_bufs
s.pa_wpos = s.pa_rpos = 0
class SystemdJournalParser(interface.FileObjectParser):
"""Parses Systemd Journal files."""
NAME = 'systemd_journal'
DESCRIPTION = 'Parser for Systemd Journal files.'
_OBJECT_COMPRESSED_FLAG = 0x00000001
# Unfortunately this doesn't help us knowing about the "dirtiness" or
# "corrupted" file state.
# A file can be in any of these states and still be corrupted, for example, by
# an unexpected shut down. Once journald detects one of these, it will
# "rotate" the corrupted journal file, an store it away, and change the status
# to STATE_OFFLINE.
# STATE_ONLINE means the file wasn't closed, but the journal can still be in a
# clean state.
_JOURNAL_STATE = construct.Enum(
construct.ULInt8('state'),
STATE_OFFLINE=0,
STATE_ONLINE=1,
STATE_ARCHIVED=2
)
_OBJECT_HEADER_TYPE = construct.Enum(
construct.ULInt8('type'),
UNUSED=0,
DATA=1,
FIELD=2,
ENTRY=3,
DATA_HASH_TABLE=4,
FIELD_HASH_TABLE=5,
ENTRY_ARRAY=6,
TAG=7
)
_ULInt64 = construct.ULInt64('int')
_OBJECT_HEADER = construct.Struct(
'object_header',
_OBJECT_HEADER_TYPE,
construct.ULInt8('flags'),
construct.Bytes('reserved', 6),
construct.ULInt64('size')
)
_OBJECT_HEADER_SIZE = _OBJECT_HEADER.sizeof()
_DATA_OBJECT = construct.Struct(
'data_object',
construct.ULInt64('hash'),
construct.ULInt64('next_hash_offset'),
construct.ULInt64('next_field_offset'),
construct.ULInt64('entry_offset'),
construct.ULInt64('entry_array_offset'),
construct.ULInt64('n_entries')
)
_DATA_OBJECT_SIZE = _DATA_OBJECT.sizeof()
_ENTRY_ITEM = construct.Struct(
'entry_item',
construct.ULInt64('object_offset'),
construct.ULInt64('hash')
)
_ENTRY_OBJECT = construct.Struct(
'entry_object',
construct.ULInt64('seqnum'),
construct.ULInt64('realtime'),
construct.ULInt64('monotonic'),
construct.Struct(
'boot_id',
construct.Bytes('bytes', 16),
construct.ULInt64('qword1'),
construct.ULInt64('qword2')),
construct.ULInt64('xor_hash'),
construct.Rename('object_items', construct.GreedyRange(_ENTRY_ITEM))
)
_JOURNAL_HEADER = construct.Struct(
'journal_header',
construct.Const(construct.String('signature', 8), b'LPKSHHRH'),
construct.ULInt32('compatible_flags'),
construct.ULInt32('incompatible_flags'),
_JOURNAL_STATE,
construct.Bytes('reserved', 7),
construct.Bytes('file_id', 16),
construct.Bytes('machine_id', 16),
construct.Bytes('boot_id', 16),
construct.Bytes('seqnum_id', 16),
construct.ULInt64('header_size'),
construct.ULInt64('arena_size'),
construct.ULInt64('data_hash_table_offset'),
construct.ULInt64('data_hash_table_size'),
construct.ULInt64('field_hash_table_offset'),
construct.ULInt64('field_hash_table_size'),
construct.ULInt64('tail_object_offset'),
construct.ULInt64('n_objects'),
construct.ULInt64('n_entries'),
construct.ULInt64('tail_entry_seqnum'),
construct.ULInt64('head_entry_seqnum'),
construct.ULInt64('entry_array_offset'),
construct.ULInt64('head_entry_realtime'),
construct.ULInt64('tail_entry_realtime'),
construct.ULInt64('tail_entry_monotonic'),
# Added in format version 187
construct.ULInt64('n_data'),
construct.ULInt64('n_fields'),
# Added in format version 189
construct.ULInt64('n_tags'),
construct.ULInt64('n_entry_arrays')
)
def __init__(self):
"""Initializes a parser object."""
super(SystemdJournalParser, self).__init__()
self._max_journal_file_offset = 0
def _ParseObjectHeader(self, file_object, offset):
"""Parses a Systemd journal object header structure.
Args:
file_object (dfvfs.FileIO): a file-like object.
offset (int): offset to the object header.
Returns:
tuple[construct.Struct, int]: parsed object header and size of the
object payload (data) that follows.
"""
file_object.seek(offset, os.SEEK_SET)
object_header_data = file_object.read(self._OBJECT_HEADER_SIZE)
object_header = self._OBJECT_HEADER.parse(object_header_data)
payload_size = object_header.size - self._OBJECT_HEADER_SIZE
return (object_header, payload_size)
def _ParseItem(self, file_object, offset):
"""Parses a Systemd journal DATA object.
This method will read, and decompress if needed, the content of a DATA
object.
Args:
file_object (dfvfs.FileIO): a file-like object.
offset (int): offset to the DATA object.
Returns:
tuple[str, str]: key and value of this item.
Raises:
ParseError: When an unexpected object type is parsed.
"""
object_header, payload_size = self._ParseObjectHeader(file_object, offset)
file_object.read(self._DATA_OBJECT_SIZE)
if object_header.type != 'DATA':
raise errors.ParseError(
'Expected an object of type DATA, but got {0:s}'.format(
object_header.type))
event_data = file_object.read(payload_size - self._DATA_OBJECT_SIZE)
if object_header.flags & self._OBJECT_COMPRESSED_FLAG:
event_data = lzma.decompress(event_data)
event_string = event_data.decode('utf-8')
event_key, event_value = event_string.split('=', 1)
return (event_key, event_value)
def _ParseJournalEntry(self, parser_mediator, file_object, offset):
"""Parses a Systemd journal ENTRY object.
This method will generate an event per ENTRY object.
Args:
parser_mediator (ParserMediator): parser mediator.
file_object (dfvfs.FileIO): a file-like object.
offset (int): offset of the ENTRY object.
Raises:
ParseError: When an unexpected object type is parsed.
"""
object_header, payload_size = self._ParseObjectHeader(file_object, offset)
entry_object_data = file_object.read(payload_size)
entry_object = self._ENTRY_OBJECT.parse(entry_object_data)
if object_header.type != 'ENTRY':
raise errors.ParseError(
'Expected an object of type ENTRY, but got {0:s}'.format(
object_header.type))
fields = {}
for item in entry_object.object_items:
if item.object_offset < self._max_journal_file_offset:
raise errors.ParseError(
'object offset should be after hash tables ({0:d} < {1:d})'.format(
offset, self._max_journal_file_offset))
key, value = self._ParseItem(file_object, item.object_offset)
fields[key] = value
reporter = fields.get('SYSLOG_IDENTIFIER', None)
if reporter and reporter != 'kernel':
pid = fields.get('_PID', fields.get('SYSLOG_PID', None))
else:
pid = None
event_data = SystemdJournalEventData()
event_data.body = fields['MESSAGE']
event_data.hostname = fields['_HOSTNAME']
event_data.pid = pid
event_data.reporter = reporter
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=entry_object.realtime)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseEntries(self, file_object, offset):
"""Parses Systemd journal ENTRY_ARRAY objects.
Args:
file_object (dfvfs.FileIO): a file-like object.
offset (int): offset of the ENTRY_ARRAY object.
Returns:
list[dict]: every ENTRY objects offsets.
Raises:
ParseError: When an unexpected object type is parsed.
"""
entry_offsets = []
object_header, payload_size = self._ParseObjectHeader(file_object, offset)
if object_header.type != 'ENTRY_ARRAY':
raise errors.ParseError(
'Expected an object of type ENTRY_ARRAY, but got {0:s}'.format(
object_header.type))
next_array_offset = self._ULInt64.parse_stream(file_object)
entry_offests_numbers, _ = divmod((payload_size - 8), 8)
for entry_offset in range(entry_offests_numbers):
entry_offset = self._ULInt64.parse_stream(file_object)
if entry_offset != 0:
entry_offsets.append(entry_offset)
if next_array_offset != 0:
next_entry_offsets = self._ParseEntries(file_object, next_array_offset)
entry_offsets.extend(next_entry_offsets)
return entry_offsets
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Systemd journal file-like object.
Args:
parser_mediator (ParserMediator): parser mediator.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the header cannot be parsed.
"""
try:
journal_header = self._JOURNAL_HEADER.parse_stream(file_object)
except construct.ConstructError as exception:
raise errors.UnableToParseFile(
'Unable to parse journal header with error: {0!s}'.format(exception))
max_data_hash_table_offset = (
journal_header.data_hash_table_offset +
journal_header.data_hash_table_size)
max_field_hash_table_offset = (
journal_header.field_hash_table_offset +
journal_header.field_hash_table_size)
self._max_journal_file_offset = max(
max_data_hash_table_offset, max_field_hash_table_offset)
entries_offsets = self._ParseEntries(
file_object, journal_header.entry_array_offset)
for entry_offset in entries_offsets:
try:
self._ParseJournalEntry(parser_mediator, file_object, entry_offset)
except errors.ParseError as exception:
parser_mediator.ProduceExtractionError((
'Unable to complete parsing journal file: {0:s} at offset '
'0x{1:08x}').format(exception, entry_offset))
return
except construct.ConstructError as exception:
raise errors.UnableToParseFile((
'Unable to parse journal header at offset: 0x{0:08x} with '
'error: {1:s}').format(entry_offset, exception))
class SAMUsersWindowsRegistryPlugin(interface.WindowsRegistryPlugin):
"""Windows Registry plugin for SAM Users Account information."""
NAME = u'windows_sam_users'
DESCRIPTION = u'Parser for SAM Users and Names Registry keys.'
FILTERS = frozenset([
interface.WindowsRegistryKeyPathFilter(
u'HKEY_LOCAL_MACHINE\\SAM\\Domains\\Account\\Users')
])
_F_VALUE_STRUCT = construct.Struct(u'f_struct', construct.Padding(8),
construct.ULInt64(u'last_login'),
construct.Padding(8),
construct.ULInt64(u'password_reset'),
construct.Padding(16),
construct.ULInt16(u'rid'),
construct.Padding(16),
construct.ULInt8(u'login_count'))
_V_VALUE_HEADER = construct.Struct(
u'v_header', construct.Array(11, construct.ULInt32(u'values')))
_V_VALUE_STRINGS_OFFSET = 204
_SOURCE_APPEND = u': User Account Information'
def GetEntries(self, parser_mediator, registry_key, **kwargs):
"""Collect data from Users and Names and produce event objects.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
registry_key: A Windows Registry key (instance of
dfwinreg.WinRegistryKey).
"""
names_key = registry_key.GetSubkeyByName(u'Names')
if not names_key:
parser_mediator.ProduceParseError(u'missing subkey: "Names".')
return
values = [(v.name, v.last_written_time)
for v in names_key.GetSubkeys()]
usernames_dict = dict(values)
for subkey in registry_key.GetSubkeys():
if subkey.name == u'Names':
continue
f_value = subkey.GetValueByName(u'F')
if not f_value:
parser_mediator.ProduceParseError(
u'missing Registry value: "F" in subkey: {0:s}.'.format(
subkey.name))
continue
v_value = subkey.GetValueByName(u'V')
if not v_value:
parser_mediator.ProduceParseError(
u'missing Registry value: "V" in subkey: {0:s}.'.format(
subkey.name))
continue
try:
f_data_struct = self._F_VALUE_STRUCT.parse(f_value.data)
except construct.FieldError as exception:
parser_mediator.ProduceParseError(
(u'unable to parse Registry value: "F" in subkey: {0:s} '
u'with error: {1:s}.').format(subkey.name, exception))
continue
try:
v_data_struct = self._V_VALUE_HEADER.parse(v_value.data)
except construct.FieldError as exception:
parser_mediator.ProduceParseError(
(u'unable to parse Registry value: "V" in subkey: {0:s} '
u'with error: {1:s}.').format(subkey.name, exception))
continue
v_header_values = v_data_struct.values()[0]
data_start_offset = v_header_values[
3] + self._V_VALUE_STRINGS_OFFSET
data_end_offset = v_header_values[4] + data_start_offset
utf16_stream = v_value.data[data_start_offset:data_end_offset]
try:
username = utf16_stream.decode(u'utf-16-le')
except (UnicodeDecodeError, UnicodeEncodeError) as exception:
username = utf16_stream.decode(u'utf-16-le', errors=u'replace')
parser_mediator.ProduceParseError((
u'unable to decode username string with error: {0:s}. Characters '
u'that cannot be decoded will be replaced with "?" or '
u'"\\ufffd".').format(exception))
data_start_offset = v_header_values[
6] + self._V_VALUE_STRINGS_OFFSET
data_end_offset = v_header_values[7] + data_start_offset
utf16_stream = v_value.data[data_start_offset:data_end_offset]
try:
fullname = utf16_stream.decode(u'utf-16-le')
except (UnicodeDecodeError, UnicodeEncodeError) as exception:
fullname = utf16_stream.decode(u'utf-16-le', errors=u'replace')
parser_mediator.ProduceParseError((
u'unable to decode fullname string with error: {0:s}. Characters '
u'that cannot be decoded will be replaced with "?" or '
u'"\\ufffd".').format(exception))
data_start_offset = v_header_values[
9] + self._V_VALUE_STRINGS_OFFSET
data_end_offset = v_header_values[10] + data_start_offset
utf16_stream = v_value.data[data_start_offset:data_end_offset]
try:
comments = utf16_stream.decode(u'utf-16-le')
except (UnicodeDecodeError, UnicodeEncodeError) as exception:
comments = utf16_stream.decode(u'utf-16-le', errors=u'replace')
parser_mediator.ProduceParseError((
u'unable to decode comments string with error: {0:s}. Characters '
u'that cannot be decoded will be replaced with "?" or '
u'"\\ufffd".').format(exception))
filetime = None
if usernames_dict:
filetime = usernames_dict.get(username, None)
# TODO: check if subkey.name == f_data_struct.rid
if filetime:
values_dict = {
u'account_rid': f_data_struct.rid,
u'login_count': f_data_struct.login_count
}
if username:
values_dict[u'username'] = username
if fullname:
values_dict[u'full_name'] = fullname
if comments:
values_dict[u'comments'] = comments
event_object = windows_events.WindowsRegistryEvent(
filetime,
registry_key.path,
values_dict,
offset=registry_key.offset,
source_append=self._SOURCE_APPEND)
parser_mediator.ProduceEvent(event_object)
if f_data_struct.last_login > 0:
event_object = SAMUsersWindowsRegistryEvent(
f_data_struct.last_login,
eventdata.EventTimestamp.LAST_LOGIN_TIME,
registry_key.path, f_value.offset, f_data_struct.rid,
f_data_struct.login_count, username, fullname, comments)
parser_mediator.ProduceEvent(event_object)
if f_data_struct.password_reset > 0:
event_object = SAMUsersWindowsRegistryEvent(
f_data_struct.password_reset,
eventdata.EventTimestamp.LAST_PASSWORD_RESET,
registry_key.path, f_value.offset, f_data_struct.rid,
f_data_struct.login_count, username, fullname, comments)
parser_mediator.ProduceEvent(event_object)
def save(self, stream):
"""Encode GIF to a file-like object."""
# create one list of pixels and alpha mask for all images
alpha_mask = flatten([a != 255 for r, g, b, a in img.rgba_data]
for img in self.images)
rgb = flatten([(r, g, b) for r, g, b, a in img.rgba_data]
for img in self.images)
# if there is any alpha, need to reverse space for a transparent colour
use_transparency = any(alpha_mask)
max_colours = 255 if use_transparency else 256
# quantize to get colour table and map
colour_table, colour_map = quantize(rgb, max_colours)
# add transparent colour to the table if necessary
if use_transparency:
transparent_col_index = len(colour_table)
colour_table.append((0, 0, 0))
else:
transparent_col_index = 0
# pad colour table to nearest power of two length
# colour table length must also be at least 2
colour_table_len = max(2, int(pow(2, ceil(log(len(colour_table), 2)))))
colour_table += [(0, 0, 0)] * (colour_table_len - len(colour_table))
if self.comment is not None:
comment_containers = [
construct.Container(
block_type='comment',
block_start=0x21,
ext_label=0xFE,
comment=self.comment,
)
]
else:
comment_containers = []
lzw_min = max(2, int(log(len(colour_table), 2)))
image_containers = flatten([[
construct.Container(
block_type='gce',
block_start=0x21,
ext_label=0xF9,
block_size=4,
disposal_method=0,
user_input_flag=False,
transparent_colour_flag=use_transparency,
delay_time=int(image.delay_ms / 10),
transparent_colour_index=transparent_col_index,
terminator=0,
),
construct.Container(
block_type='image',
block_start=0x2C,
image_descriptor=construct.Container(
left=0,
top=0,
width=image.size[0],
height=image.size[1],
lct_flag=False,
interlace_flag=False,
sort_flag=False,
lct_size=0,
),
lct=None,
lzw_min=lzw_min,
compressed_indices=lzw.compress(
''.join(
chr(colour_map[(
r, g,
b)]) if a == 255 else chr(transparent_col_index)
for r, g, b, a in image.rgba_data), lzw_min),
),
] for image in self.images])
app_ext_containers = []
# if this gif loops, add the application extension for looping
if self.loop_count != 1:
if self.loop_count == 0:
real_count = 0
else:
real_count = self.loop_count - 1
data = construct.Struct(
'loop',
construct.ULInt8('id'),
construct.ULInt16('count'),
).build(construct.Container(id=1, count=real_count))
app_ext_containers.append(
construct.Container(
block_type='application_extension',
block_start=0x21,
ext_label=0xFF,
block_size=11,
app_id='NETSCAPE',
app_auth_code='2.0',
app_data=data,
))
trailer = [
construct.Container(block_start=0x3B, terminator='terminator')
]
gif = gifprime.parser.gif.build_stream(
construct.Container(
magic='GIF89a',
logical_screen_descriptor=construct.Container(
logical_width=self.size[0],
logical_height=self.size[1],
gct_flag=True,
colour_res=7,
sort_flag=True,
gct_size=int(log(len(colour_table), 2)) - 1,
bg_col_index=0,
pixel_aspect=0,
),
gct=colour_table,
body=(comment_containers + image_containers +
app_ext_containers + trailer),
), stream)
construct.ULInt32("nb_files"))
AXSString = construct.Struct("AXSString", construct.ULInt32("length"),
construct.Bytes("data", lambda ctx: ctx.length))
AXSKeyEntry = construct.Struct("AXSKeyEntry", construct.Array(2, AXSString),
construct.ULInt32("unk"))
AXSKeyMan = construct.Struct(
"AXSKeyMan",
construct.ULInt32("nb_entry"),
#construct.Array(lambda ctx: ctx.nb_entry, AXSKeyEntry))
construct.Array(1, AXSKeyEntry))
binInfo = construct.Struct(
"binInfo", construct.ULInt16("a"), construct.ULInt8("b"),
construct.Value("type", lambda ctx: (ctx["b"] << 16) | ctx["a"]),
construct.ULInt32("size"), construct.ULInt32("unk_dword_00"),
construct.ULInt8("unk_byte_00"), construct.ULInt32("type_compression"),
construct.ULInt32("length"), construct.Bytes("data",
lambda ctx: ctx.length))
binInfoNext = construct.Struct("binInfoNext", construct.ULInt16("type"),
construct.ULInt32("size"),
construct.ULInt32("length"),
construct.Bytes("data", lambda ctx: ctx.length))
class Buffer:
def __init__(self, buf):
self.buf = buf
def GrfID (name) :
return BinHex_Adapter(c.Bytes(name, 4))
def MD5 (name) :
return BinHex_Adapter(c.Bytes(name, 16))
MAX_INFO_LEN = 1024
MASTER_SERVER_PORT = 3978 # The default port of the master server (UDP)
MASTER_SERVER_HOST = "master.openttd.org"
MASTER_SERVER_MSG = "OpenTTDRegister"
DEFAULT_PORT = 3979 # The default port of the game server (TCP & UDP)
Header = c.Struct("header",
c.ULInt16("size"),
c.Enum(c.ULInt8("type"),
UDP_CLIENT_FIND_SERVER=0,
UDP_SERVER_RESPONSE=1,
UDP_CLIENT_DETAIL_INFO=2,
UDP_SERVER_DETAIL_INFO=3,
UDP_SERVER_REGISTER=4,
UDP_MASTER_ACK_REGISTER=5,
UDP_CLIENT_GET_LIST=6,
UDP_MASTER_RESPONSE_LIST=7,
UDP_SERVER_UNREGISTER=8,
UDP_CLIENT_GET_NEWGRFS=9,
UDP_SERVER_NEWGRFS=10,
UDP_END=11
)
)
IMAGE_DIRECTORY_ENTRY_ARCHITECTURE = 7 # Architecture Specific Data
IMAGE_DIRECTORY_ENTRY_GLOBALPTR = 8 # RVA of GP
IMAGE_DIRECTORY_ENTRY_TLS = 9 # TLS Directory
IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG = 10 # Load Configuration Directory
IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT = 11 # Bound Import Directory in headers
IMAGE_DIRECTORY_ENTRY_IAT = 12 # Import Address Table
IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT = 13 # Delay Load Import Descriptors
IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR = 14 # COM Runtime descriptor
IMAGE_NUMBEROF_DIRECTORY_ENTRIES = 16
ImageOptionalHeader = construct.Struct('ImageOptionalHeader',
construct.Enum(construct.ULInt16('Magic'),
IMAGE_NT_OPTIONAL_HDR32_MAGIC = 0x10b,
IMAGE_NT_OPTIONAL_HDR64_MAGIC = 0x20b
),
construct.ULInt8('MajorLinkerVersion'),
construct.ULInt8('MinorLinkerVersion'),
construct.ULInt32('SizeOfCode'),
construct.ULInt32('SizeOfInitializedData'),
construct.ULInt32('SizeOfUninitializedData'),
construct.ULInt32('AddressOfEntryPoint'),
construct.ULInt32('BaseOfCode'),
construct.If(lambda ctx: ctx.Magic == 'IMAGE_NT_OPTIONAL_HDR32_MAGIC',
construct.ULInt32('BaseOfData')
),
construct.Switch('ImageBase', lambda ctx: ctx.Magic, {
'IMAGE_NT_OPTIONAL_HDR32_MAGIC' : construct.ULInt32('ImageBase_'),
'IMAGE_NT_OPTIONAL_HDR64_MAGIC' : construct.ULInt64('ImageBase_')
}
),
construct.ULInt32('SectionAlignment'),
MYP_FileEntryHeader = construct.Struct(
"MYP_FileEntryHeader",
construct.ULInt64("offset"),
construct.ULInt32("size_header"),
construct.ULInt32("sizez"),
construct.ULInt32("size"),
construct.ULInt64("name"),
construct.ULInt32("crc"),
construct.ULInt16("flag"),
)
MYP_FILE_HEADER_SIZE = 64
MYP_FileHeader = construct.Struct("MYP_FileEntryHeader",
construct.ULInt16("unk_word_00"),
construct.ULInt8("unk_byte_00"),
construct.ULInt16("unk_word_01"),
construct.ULInt16("unk_word_02"),
construct.ULInt8("unk_byte_01"))
def hexdump(src, length=16):
FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.'
for x in range(256)])
lines = []
for c in xrange(0, len(src), length):
chars = src[c:c + length]
hex = ' '.join(["%02x" % ord(x) for x in chars])
printable = ''.join([
"%s" % ((ord(x) <= 127 and FILTER[ord(x)]) or '.') for x in chars
])
class UsersPlugin(interface.KeyPlugin):
"""SAM Windows Registry plugin for Users Account information."""
NAME = u'windows_sam_users'
DESCRIPTION = u'Parser for SAM Users and Names Registry keys.'
REG_KEYS = [u'\\SAM\\Domains\\Account\\Users']
REG_TYPE = u'SAM'
F_VALUE_STRUCT = construct.Struct(u'f_struct', construct.Padding(8),
construct.ULInt64(u'last_login'),
construct.Padding(8),
construct.ULInt64(u'password_reset'),
construct.Padding(16),
construct.ULInt16(u'rid'),
construct.Padding(16),
construct.ULInt8(u'login_count'))
V_VALUE_HEADER = construct.Struct(
u'v_header', construct.Array(11, construct.ULInt32(u'values')))
V_VALUE_HEADER_SIZE = 0xCC
def GetEntries(self,
parser_mediator,
key=None,
registry_file_type=None,
codepage=u'cp1252',
**unused_kwargs):
"""Collect data from Users and Names and produce event objects.
Args:
parser_mediator: A parser context object (instance of ParserContext).
key: Optional Registry key (instance of winreg.WinRegKey).
The default is None.
registry_file_type: Optional string containing the Windows Registry file
type, e.g. NTUSER, SOFTWARE. The default is None.
codepage: Optional extended ASCII string codepage. The default is cp1252.
"""
name_dict = {}
name_key = key.GetSubkey(u'Names')
if not name_key:
parser_mediator.ProduceParseError(u'Unable to locate Names key.')
return
values = [(v.name, v.last_written_timestamp)
for v in name_key.GetSubkeys()]
name_dict = dict(values)
for subkey in key.GetSubkeys():
text_dict = {}
if subkey.name == u'Names':
continue
text_dict[u'user_guid'] = subkey.name
parsed_v_value = self._ParseVValue(subkey)
if not parsed_v_value:
parser_mediator.ProduceParseError(
u'Unable to parse SAM key: {0:s} V value.'.format(subkey))
return
username = parsed_v_value[0]
full_name = parsed_v_value[1]
comments = parsed_v_value[2]
if username:
text_dict[u'username'] = username
if full_name:
text_dict[u'full_name'] = full_name
if comments:
text_dict[u'comments'] = comments
if name_dict:
account_create_time = name_dict.get(text_dict.get(u'username'),
0)
else:
account_create_time = 0
f_data = self._ParseFValue(subkey)
last_login_time = timelib.Timestamp.FromFiletime(f_data.last_login)
password_reset_time = timelib.Timestamp.FromFiletime(
f_data.password_reset)
text_dict[u'account_rid'] = f_data.rid
text_dict[u'login_count'] = f_data.login_count
if account_create_time > 0:
event_object = windows_events.WindowsRegistryEvent(
account_create_time,
key.path,
text_dict,
usage=eventdata.EventTimestamp.ACCOUNT_CREATED,
offset=key.offset,
registry_file_type=registry_file_type,
source_append=u'User Account Information')
parser_mediator.ProduceEvent(event_object)
if last_login_time > 0:
event_object = windows_events.WindowsRegistryEvent(
last_login_time,
key.path,
text_dict,
usage=eventdata.EventTimestamp.LAST_LOGIN_TIME,
offset=key.offset,
registry_file_type=registry_file_type,
source_append=u'User Account Information')
parser_mediator.ProduceEvent(event_object)
if password_reset_time > 0:
event_object = windows_events.WindowsRegistryEvent(
password_reset_time,
key.path,
text_dict,
usage=eventdata.EventTimestamp.LAST_PASSWORD_RESET,
offset=key.offset,
registry_file_type=registry_file_type,
source_append=u'User Account Information')
parser_mediator.ProduceEvent(event_object)
def _ParseVValue(self, key):
"""Parses V value and returns name, fullname, and comments data.
Args:
key: Registry key (instance of winreg.WinRegKey).
Returns:
name: Name data parsed with name start and length values.
fullname: Fullname data parsed with fullname start and length values.
comments: Comments data parsed with comments start and length values.
"""
v_value = key.GetValue(u'V')
if not v_value:
logging.error(u'Unable to locate V Value in key.')
return
try:
structure = self.V_VALUE_HEADER.parse(v_value.data)
except construct.FieldError as exception:
logging.error(
u'Unable to extract V value header data: {:s}'.format(
exception))
return
name_offset = structure.values()[0][3] + self.V_VALUE_HEADER_SIZE
full_name_offset = structure.values()[0][6] + self.V_VALUE_HEADER_SIZE
comments_offset = structure.values()[0][9] + self.V_VALUE_HEADER_SIZE
name_raw = v_value.data[name_offset:name_offset +
structure.values()[0][4]]
full_name_raw = v_value.data[full_name_offset:full_name_offset +
structure.values()[0][7]]
comments_raw = v_value.data[comments_offset:comments_offset +
structure.values()[0][10]]
name = binary.ReadUtf16(name_raw)
full_name = binary.ReadUtf16(full_name_raw)
comments = binary.ReadUtf16(comments_raw)
return name, full_name, comments
def _ParseFValue(self, key):
"""Parses F value and returns parsed F data construct object.
Args:
key: Registry key (instance of winreg.WinRegKey).
Returns:
f_data: Construct parsed F value containing rid, login count,
and timestamp information.
"""
f_value = key.GetValue(u'F')
if not f_value:
logging.error(u'Unable to locate F Value in key.')
return
try:
f_data = self.F_VALUE_STRUCT.parse(f_value.data)
except construct.FieldError as exception:
logging.error(
u'Unable to extract F value data: {:s}'.format(exception))
return
return f_data
class VolumeBootRecord(BootRecord):
_NTFS_VBR_STRUCT = construct.Struct(
'NTFS-VBR',
construct.Field('JumpOverBPB', 3),
construct.String("OemId", 8),
construct.Struct(
'BiosParameterBlock',
construct.ULInt16('SectorSize'),
construct.ULInt8('SectorsPerCluster'),
construct.Field('Reserved1', 2),
construct.Field('MustBeZero1', 3),
construct.Field('MustBeZero2', 2),
construct.ULInt8('MediaDescriptor'),
construct.Field('MustBeZero3', 2),
construct.ULInt16('SectorsPerTrack'),
construct.ULInt16('NumberOfHeads'),
construct.ULInt32('HiddenSectors'),
construct.Field('NotUsed1', 4),
construct.Const(construct.Field('DriveNumber', 1),
'80'.decode('hex')),
construct.Field('Reserved2', 3),
construct.ULInt64('TotalSectors'),
construct.ULInt64('MFTCluster'),
construct.ULInt64('MFTMirrCluster'),
construct.SLInt8('ClustersPerMFTRecord'),
construct.Field('NotUsed2', 3),
construct.SLInt8('ClustersPerIdxBuffer'),
construct.Field('NotUsed3', 3),
construct.ULInt64('VolumneSN'),
construct.Field('NotUsed4', 4),
),
construct.HexDumpAdapter(construct.Bytes("Code", 426)),
construct.Const(construct.Bytes("signature", 2), '55aa'.decode('hex')),
)
_BITLOCKER_VBR_STRUCT = construct.Struct(
'FVE-VBR',
construct.Field('JumpOverBPB', 3),
construct.Const(construct.String("OemId", 8),
'-FVE-FS-'.encode('utf8')),
construct.Struct(
'BiosParameterBlock',
construct.ULInt16('SectorSize'),
construct.ULInt8('SectorsPerCluster'),
construct.Field('Reserved1', 2),
construct.Field('MustBeZero1', 3),
construct.Field('MustBeZero2', 2),
construct.ULInt8('MediaDescriptor'),
construct.Field('MustBeZero3', 2),
construct.ULInt16('SectorsPerTrack'),
construct.ULInt16('NumberOfHeads'),
construct.ULInt32('HiddenSectors'),
construct.ULInt32('TotalSectors'),
construct.ULInt32('SectorsPerFAT'),
construct.ULInt16('FATFlags'),
construct.ULInt16('Version'),
construct.ULInt32('RootDirCluster'),
construct.ULInt16('FSInfoSector'),
construct.ULInt16('BackupSector'),
construct.Field('Reserved2', 12),
construct.Const(construct.Field('DriveNumber', 1),
'80'.decode('hex')),
construct.Field('Reserved3', 1),
construct.Field('ExtendedBootSignature', 1),
construct.ULInt32('VolumneSN'),
construct.Const(construct.String("VolumeLabel", 11),
'NO NAME '.encode('utf8')),
construct.Const(construct.String("SystemId", 8),
'FAT32 '.encode('utf8')),
),
construct.HexDumpAdapter(construct.Bytes("Code1", 70)),
construct.Field('BitlockerGUID', 16),
construct.ULInt64('FVEMetadataBlockOffset1'),
construct.ULInt64('FVEMetadataBlockOffset2'),
construct.ULInt64('FVEMetadataBlockOffset3'),
construct.HexDumpAdapter(construct.Bytes("Code2", 307)),
construct.ULInt8('FirstStrOffset'),
construct.ULInt8('SecondStrOffset'),
construct.ULInt8('ThirdStrOffset'),
construct.Const(construct.Bytes("signature", 2), '55aa'.decode('hex')),
)
def __init__(self, filePath, size, offset=None, whitelist=()):
self._type = 'VBR'
super(VolumeBootRecord, self).__init__(filePath, size, offset,
whitelist)
def _parse(self):
"""
Main method in charge of parsing the VBR.
It will try to parse the boot record according to known structures (NTFS and Bitlocker supported).
It will then try to narrow down invariant code, hash it and match the hash against a whitelist.
If no match was found, it will try some simple heuristics to detect malicious behaviours.
Finally it will compare the HiddenSectors value in BPB to that of the record's dump offset.
Returns: nothing
"""
try:
# This will parse both NTFS and Vista bitlocker volumes since they only differ by their OEM ID
vbr = self._NTFS_VBR_STRUCT.parse(self._raw)
expectedLoader, invariantCode = self._getInvariantCode('NTFS', vbr)
except construct.core.ConstructError as e1:
# Retry with Bitlocker (Win7+) volume header structure
try:
vbr = self._BITLOCKER_VBR_STRUCT.parse(self._raw)
expectedLoader, invariantCode = self._getInvariantCode(
'bitlocker', vbr)
except construct.core.ConstructError as e2:
raise InvalidVBRError(
'Invalid VBR structure: e1={0}, e2={1}\n{2}'.format(
e1, e2, hexdump(self._raw)))
self._oemId = vbr.OemId
self._bpb = vbr.BiosParameterBlock
codeHash = hashlib.sha256(invariantCode)
self._matchHash(codeHash, expectedLoader)
# If no whitelisted signature matched, try some simple heuristics to flag this VBR as malicious
# Note that the self._checkCode method is only given the "invariant" code section to help with the
# disassembling. This will obviously leads to broken offsets, but it doesn't matter since the heuristics don't
# use them.
if len(self._signature) == 0:
self._checkCode(invariantCode)
# At last, compare the offset at which this VBR was found with the value of the BPB HiddenSectors
if self._offset is not None \
and (vbr.BiosParameterBlock.HiddenSectors * vbr.BiosParameterBlock.SectorSize) != self._offset:
self._suspiciousBehaviour.append(
'Suspicious HiddenSectors value: {0} ({1} bytes)'.format(
vbr.BiosParameterBlock.HiddenSectors,
vbr.BiosParameterBlock.HiddenSectors *
vbr.BiosParameterBlock.SectorSize))
def _getInvariantCode(self, vbrType, vbrStruct):
"""
Helper method that finds all the sections of the boot code that can be hashed and compared to a whitelist.
This means that localized strings and other variable parameters (BPB, etc...) are excluded.
Currently, this method only supports NTFS and Bitlocker VBR.
Args:
vbrType: unicode string corresponding to the VBR type ('NTFS' or 'bitlocker')
vbrStruct: construct.container of the VBR
Returns: 2-tuple (unicode string of expected loader, concatenated strings of invariant sections of code)
"""
codeStart = 0
codeEnd = None
invariantCode = str()
expectedLoader = None
if vbrType == 'NTFS':
# The first three bytes are a jump over the NTFS BPB to where the code really starts (0x54) and a NOP
invariantCode += vbrStruct.JumpOverBPB
codeStart = 0x54
# NTFS VBR contains localized strings which must be excluded from the hash computation.
# Before Windows 8, these strings are located at 4 different offsets which can be calculated by adding
# 0x100 to the values respectively stored in bytes 0x1f8, 0x1f9, 0x1fa and 0x1fb.
# Starting from Windows 8, these strings are located at 3 different offsets which are directly stored in
# little endian words respectively at 0x1f6, 0x1f8 and 0x1fa
# Since there is no easy way to tell which version of Windows we are dealing with beforehand, we first
# assume it is a Windows < 8 by testing 0x1f8 against all the known first offset. If all tests fail, assume
# it is Windows >= 8 and check 0x1f6 against the only known first offset (to date)
firstStrOffset = construct.UBInt8('FirstStringOffset').parse(
self._raw[0x1f8])
# Windows NT5
if firstStrOffset == 0x83:
expectedLoader = 'NT5.1/NT5.2 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.0
elif firstStrOffset == 0x80:
expectedLoader = 'NT6.0 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.1
elif firstStrOffset == 0x8c:
expectedLoader = 'NT6.1 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.2+
else:
firstStrOffset = construct.ULInt16('FirstStringOffset').parse(
self._raw[0x1f6:0x1f8])
if firstStrOffset == 0x18a:
expectedLoader = 'NT6.2+ VBR'
codeEnd = firstStrOffset
if codeEnd is None:
self._suspiciousBehaviour.append(
'Invalid string offset: {0:#x}'.format(firstStrOffset))
self._logger.debug(
'First localized string offset is wrong for a NTFS VBR: {0:#x}. '
'It should be 0x83, 0x80, 0x8c or 0x18a.'.format(
firstStrOffset))
codeEnd = 0
elif vbrType == 'bitlocker':
expectedLoader = 'NT6.1+ Bitlocker VBR'
# The first three bytes are a jump over the NTFS BPB to where the code really starts (0x5A) and a NOP
invariantCode += vbrStruct.JumpOverBPB
# First section of code (_BITLOCKER_VBR_STRUCT.Code1)
invariantCode += vbrStruct.Code1
# In the second section of code, there are localized strings which must be excluded from hash computation.
# Their offsets are stored in the last 3 bytes before the VBR signature (0x55aa).
# For Windows 8, 8.1 and 10, the first string offset seems to always be 0x100 (ie. FirstStrOffset = 0x00)
if vbrStruct.FirstStrOffset != 0:
self._suspiciousBehaviour.append(
'Invalid string offset: {0:#x}'.format(
vbrStruct.FirstStrOffset))
self._logger.debug(
'First localized string offset is wrong for a Bitlocker VBR. '
'It should be 0x00) : {0:#x}'.format(
vbrStruct.FirstStrOffset))
codeStart = 0xc8 # Offset of Code2
codeEnd = 0x100 + vbrStruct.FirstStrOffset
else:
raise NotImplementedError(
'VBR type "{0}" is not implemented yet'.format(vbrType))
self._logger.debug(
'Expecting {0}. Code starts at {1:#x} and ends at {2:#x}'.format(
expectedLoader, codeStart, codeEnd))
invariantCode += self._raw[codeStart:codeEnd]
return expectedLoader, invariantCode
def _checkCode(self, code):
md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_16)
md.detail = True
for i in md.disasm(code, 0):
# Check for unknown interrupt
if i.mnemonic == 'int' and i.bytes[1] not in (0x10, 0x13, 0x18,
0x1a):
self._suspiciousBehaviour.append(
'Unknown Interrupt : {0:#x}'.format(i.bytes[1]))
construct.ULInt32("unk_dword_01"),
construct.ULInt32("AddressOfEntryPoint"),
construct.ULInt32("StartSection_Offset"),
construct.ULInt32("unk_dword_04"),
construct.ULInt32("OriginalEntryPoint"),
construct.ULInt32("unk_dword_06"),
construct.ULInt32("Size_Payload"),
construct.ULInt32("Offset_DLL"),
construct.ULInt32("Size_DLL"),
construct.ULInt32("unk_dword_10"),
construct.ULInt32("FLAG"), # DEBUG : 0x20
construct.ULInt32("unk_dword_12"),
construct.ULInt32("CRC_DLL"), # NOT THE WHOLE DLL
construct.ULInt32("unk_dword_14"),
construct.ULInt32("text_raw_size"),
construct.Array(0x20, construct.ULInt8("AES_KEY")),
construct.Array(0x10, construct.ULInt8("AES_IV")),
construct.Array(0x10, construct.ULInt8("BUF_PAD")),
construct.Array(0x4, construct.ULInt32("XTEA_KEY")),
construct.ULInt32("unk_dword_16"),
construct.ULInt32("unk_dword_17"),
construct.ULInt32("unk_dword_18"),
construct.ULInt32("unk_dword_19"),
construct.ULInt32("unk_dword_20"),
construct.ULInt32("unk_dword_21"),
construct.ULInt32("GetModuleHandleA_idata"),
construct.ULInt32("GetModuleHandleW_idata"),
construct.ULInt32("LoadLibraryA_idata"),
construct.ULInt32("LoadLibraryW_idata"),
construct.ULInt32("GetProcAddress_idata"),
construct.ULInt32("unk_dword_27"),
construct.ULInt16('top'),
construct.ULInt16('width'),
construct.ULInt16('height'),
construct.EmbeddedBitStruct(
construct.Flag('lct_flag'),
construct.Flag('interlace_flag'),
construct.Flag('sort_flag'),
construct.Padding(2), # reserved
construct.macros.BitField('lct_size', 3),
),
),
construct.If(
lambda ctx: ctx.image_descriptor.lct_flag,
construct.Array(
lambda ctx: pow(2, ctx.image_descriptor.lct_size + 1),
construct.Array(3, construct.ULInt8('lct')),
),
),
construct.ULInt8('lzw_min'),
_get_data_subblocks('compressed_indices'),
)
_application_extension = construct.Struct(
'application_extension',
construct.Value('block_type', lambda ctx: 'application'),
construct.Const(construct.ULInt8('block_size'), 11),
construct.String('app_id', 8),
construct.Bytes('app_auth_code', 3),
_get_data_subblocks('app_data'),
)
class SAMUsersWindowsRegistryPlugin(interface.WindowsRegistryPlugin):
"""Windows Registry plugin for SAM Users Account information."""
NAME = 'windows_sam_users'
DESCRIPTION = 'Parser for SAM Users and Names Registry keys.'
FILTERS = frozenset([
interface.WindowsRegistryKeyPathFilter(
'HKEY_LOCAL_MACHINE\\SAM\\Domains\\Account\\Users')
])
_F_VALUE_STRUCT = construct.Struct('f_struct', construct.Padding(8),
construct.ULInt64('last_login'),
construct.Padding(8),
construct.ULInt64('password_reset'),
construct.Padding(16),
construct.ULInt16('rid'),
construct.Padding(16),
construct.ULInt8('login_count'))
_V_VALUE_HEADER = construct.Struct(
'v_header', construct.Array(11, construct.ULInt32('values')))
_V_VALUE_STRINGS_OFFSET = 204
_SOURCE_APPEND = ': User Account Information'
def ExtractEvents(self, parser_mediator, registry_key, **kwargs):
"""Extracts events from a Windows Registry key.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
registry_key (dfwinreg.WinRegistryKey): Windows Registry key.
"""
names_key = registry_key.GetSubkeyByName('Names')
if not names_key:
parser_mediator.ProduceExtractionError('missing subkey: Names.')
return
last_written_time_per_username = {
registry_value.name: registry_value.last_written_time
for registry_value in names_key.GetSubkeys()
}
for subkey in registry_key.GetSubkeys():
if subkey.name == 'Names':
continue
f_value = subkey.GetValueByName('F')
if not f_value:
parser_mediator.ProduceExtractionError(
'missing Registry value: "F" in subkey: {0:s}.'.format(
subkey.name))
continue
v_value = subkey.GetValueByName('V')
if not v_value:
parser_mediator.ProduceExtractionError(
'missing Registry value: "V" in subkey: {0:s}.'.format(
subkey.name))
continue
try:
f_data_struct = self._F_VALUE_STRUCT.parse(f_value.data)
except construct.FieldError as exception:
parser_mediator.ProduceExtractionError(
('unable to parse Registry value: "F" in subkey: {0:s} '
'with error: {1!s}.').format(subkey.name, exception))
continue
try:
v_data_struct = self._V_VALUE_HEADER.parse(v_value.data)
except construct.FieldError as exception:
parser_mediator.ProduceExtractionError(
('unable to parse Registry value: "V" in subkey: {0:s} '
'with error: {1!s}.').format(subkey.name, exception))
continue
v_header_values = v_data_struct.values()[0]
data_start_offset = v_header_values[
3] + self._V_VALUE_STRINGS_OFFSET
data_end_offset = v_header_values[4] + data_start_offset
utf16_stream = v_value.data[data_start_offset:data_end_offset]
try:
username = utf16_stream.decode('utf-16-le')
except (UnicodeDecodeError, UnicodeEncodeError) as exception:
username = utf16_stream.decode('utf-16-le', errors='replace')
parser_mediator.ProduceExtractionError((
'unable to decode username string with error: {0!s}. Characters '
'that cannot be decoded will be replaced with "?" or '
'"\\ufffd".').format(exception))
data_start_offset = v_header_values[
6] + self._V_VALUE_STRINGS_OFFSET
data_end_offset = v_header_values[7] + data_start_offset
utf16_stream = v_value.data[data_start_offset:data_end_offset]
try:
fullname = utf16_stream.decode('utf-16-le')
except (UnicodeDecodeError, UnicodeEncodeError) as exception:
fullname = utf16_stream.decode('utf-16-le', errors='replace')
parser_mediator.ProduceExtractionError((
'unable to decode fullname string with error: {0!s}. Characters '
'that cannot be decoded will be replaced with "?" or '
'"\\ufffd".').format(exception))
data_start_offset = v_header_values[
9] + self._V_VALUE_STRINGS_OFFSET
data_end_offset = v_header_values[10] + data_start_offset
utf16_stream = v_value.data[data_start_offset:data_end_offset]
try:
comments = utf16_stream.decode('utf-16-le')
except (UnicodeDecodeError, UnicodeEncodeError) as exception:
comments = utf16_stream.decode('utf-16-le', errors='replace')
parser_mediator.ProduceExtractionError((
'unable to decode comments string with error: {0!s}. Characters '
'that cannot be decoded will be replaced with "?" or '
'"\\ufffd".').format(exception))
last_written_time = last_written_time_per_username.get(
username, None)
# TODO: check if subkey.name == f_data_struct.rid
if last_written_time:
values_dict = {
'account_rid': f_data_struct.rid,
'login_count': f_data_struct.login_count
}
if username:
values_dict['username'] = username
if fullname:
values_dict['full_name'] = fullname
if comments:
values_dict['comments'] = comments
event_data = windows_events.WindowsRegistryEventData()
event_data.key_path = registry_key.path
event_data.offset = registry_key.offset
event_data.regvalue = values_dict
event_data.source_append = self._SOURCE_APPEND
event = time_events.DateTimeValuesEvent(
last_written_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
event_data = SAMUsersWindowsRegistryEventData()
event_data.account_rid = f_data_struct.rid
event_data.comments = comments
event_data.fullname = fullname
event_data.key_path = registry_key.path
event_data.login_count = f_data_struct.login_count
event_data.offset = f_value.offset
event_data.username = username
if f_data_struct.last_login != 0:
date_time = dfdatetime_filetime.Filetime(
timestamp=f_data_struct.last_login)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_LAST_LOGIN)
parser_mediator.ProduceEventWithEventData(event, event_data)
if f_data_struct.password_reset != 0:
date_time = dfdatetime_filetime.Filetime(
timestamp=f_data_struct.password_reset)
event = time_events.DateTimeValuesEvent(
date_time,
definitions.TIME_DESCRIPTION_LAST_PASSWORD_RESET)
parser_mediator.ProduceEventWithEventData(event, event_data)
class SAMUsersWindowsRegistryPlugin(interface.WindowsRegistryPlugin):
"""Windows Registry plugin for SAM Users Account information."""
NAME = u'windows_sam_users'
DESCRIPTION = u'Parser for SAM Users and Names Registry keys.'
FILTERS = frozenset([
interface.WindowsRegistryKeyPathFilter(
u'HKEY_LOCAL_MACHINE\\SAM\\Domains\\Account\\Users')])
F_VALUE_STRUCT = construct.Struct(
u'f_struct',
construct.Padding(8),
construct.ULInt64(u'last_login'),
construct.Padding(8),
construct.ULInt64(u'password_reset'),
construct.Padding(16),
construct.ULInt16(u'rid'),
construct.Padding(16),
construct.ULInt8(u'login_count'))
V_VALUE_HEADER = construct.Struct(
u'v_header',
construct.Array(11, construct.ULInt32(u'values')))
V_VALUE_HEADER_SIZE = 0xCC
_SOURCE_APPEND = u'User Account Information'
def _ParseFValue(self, key):
"""Parses F value and returns parsed F data construct object.
Args:
key: Registry key (instance of dfwinreg.WinRegistryKey).
Returns:
f_data: Construct parsed F value containing rid, login count,
and timestamp information.
"""
f_value = key.GetValueByName(u'F')
if not f_value:
logging.error(u'Unable to locate F Value in key.')
return
try:
f_data = self.F_VALUE_STRUCT.parse(f_value.data)
except construct.FieldError as exception:
logging.error(
u'Unable to extract F value data: {:s}'.format(exception))
return
return f_data
def _ParseVValue(self, key):
"""Parses V value and returns name, fullname, and comments data.
Args:
key: Registry key (instance of dfwinreg.WinRegistryKey).
Returns:
name: Name data parsed with name start and length values.
fullname: Fullname data parsed with fullname start and length values.
comments: Comments data parsed with comments start and length values.
"""
v_value = key.GetValueByName(u'V')
if not v_value:
logging.error(u'Unable to locate V Value in key.')
return
try:
structure = self.V_VALUE_HEADER.parse(v_value.data)
except construct.FieldError as exception:
logging.error(
u'Unable to extract V value header data with error: {0:s}'.format(
exception))
return
name_offset = structure.values()[0][3] + self.V_VALUE_HEADER_SIZE
full_name_offset = structure.values()[0][6] + self.V_VALUE_HEADER_SIZE
comments_offset = structure.values()[0][9] + self.V_VALUE_HEADER_SIZE
name_raw = v_value.data[
name_offset:name_offset + structure.values()[0][4]]
full_name_raw = v_value.data[
full_name_offset:full_name_offset + structure.values()[0][7]]
comments_raw = v_value.data[
comments_offset:comments_offset + structure.values()[0][10]]
name = binary.ReadUTF16(name_raw)
full_name = binary.ReadUTF16(full_name_raw)
comments = binary.ReadUTF16(comments_raw)
return name, full_name, comments
def GetEntries(self, parser_mediator, registry_key, **kwargs):
"""Collect data from Users and Names and produce event objects.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
registry_key: A Windows Registry key (instance of
dfwinreg.WinRegistryKey).
"""
name_key = registry_key.GetSubkeyByName(u'Names')
if not name_key:
parser_mediator.ProduceParseError(u'Unable to locate Names key.')
return
values = [(v.name, v.last_written_time) for v in name_key.GetSubkeys()]
name_dict = dict(values)
for subkey in registry_key.GetSubkeys():
if subkey.name == u'Names':
continue
parsed_v_value = self._ParseVValue(subkey)
if not parsed_v_value:
parser_mediator.ProduceParseError(
u'Unable to parse SAM key: {0:s} V value.'.format(subkey))
return
username = parsed_v_value[0]
full_name = parsed_v_value[1]
comments = parsed_v_value[2]
values_dict = {u'user_guid': subkey.name}
if username:
values_dict[u'username'] = username
if full_name:
values_dict[u'full_name'] = full_name
if comments:
values_dict[u'comments'] = comments
if name_dict:
account_create_time = name_dict.get(username, 0)
else:
account_create_time = 0
f_data = self._ParseFValue(subkey)
values_dict[u'account_rid'] = f_data.rid
values_dict[u'login_count'] = f_data.login_count
if account_create_time > 0:
event_object = windows_events.WindowsRegistryEvent(
account_create_time, registry_key.path, values_dict,
usage=eventdata.EventTimestamp.ACCOUNT_CREATED,
offset=registry_key.offset, source_append=self._SOURCE_APPEND)
parser_mediator.ProduceEvent(event_object)
if f_data.last_login > 0:
event_object = windows_events.WindowsRegistryEvent(
f_data.last_login, registry_key.path, values_dict,
usage=eventdata.EventTimestamp.LAST_LOGIN_TIME,
offset=registry_key.offset, source_append=self._SOURCE_APPEND)
parser_mediator.ProduceEvent(event_object)
if f_data.password_reset > 0:
event_object = windows_events.WindowsRegistryEvent(
f_data.password_reset, registry_key.path, values_dict,
usage=eventdata.EventTimestamp.LAST_PASSWORD_RESET,
offset=registry_key.offset, source_append=self._SOURCE_APPEND)
parser_mediator.ProduceEvent(event_object)
def generate_images():
# the most recent GCE block since the last image block.
active_gce = None
# initialize the previous state
prev_state = [bg_colour] * (self.size[0] * self.size[1])
num_images = 0
logger.info('GIF<%s>: Started decoding image frames',
self.filename)
for block in parsed_data.body:
if 'block_type' not in block: # it's just the terminator
pass
elif block.block_type == 'image':
lct = (block.lct
if block.image_descriptor.lct_flag else None)
# Select the active colour table.
if lct is not None:
active_colour_table = lct
elif gct is not None:
active_colour_table = gct
else:
# TODO: Spec says we can use a default colour table
# in this case.
raise NotImplementedError('No colour table')
# set transparency index
if active_gce is not None:
if active_gce.transparent_colour_flag:
trans_index = (
active_gce.transparent_colour_index)
else:
trans_index = None
delay_ms = active_gce.delay_time * 10
disposal_method = active_gce.disposal_method
else:
trans_index = None
delay_ms = 0
disposal_method = 0
# If not specified, deinterlace the images only if
# necessary.
if force_deinterlace is None:
deinterlace = block.image_descriptor.interlace_flag
else:
deinterlace = force_deinterlace
# get the decompressed colour indices
indices_bytes = ''.join(
lzw.decompress(block.compressed_indices,
block.lzw_min))
indices = struct.unpack(
'{}B'.format(len(indices_bytes)), indices_bytes)
# de-interlace the colour indices if necessary
if deinterlace:
indices = self._de_interlace(
indices,
block.image_descriptor.height,
block.image_descriptor.width,
)
# interpret colour indices
rgba_data = [
tuple(active_colour_table[i]) +
((0, ) if i == trans_index else (255, ))
for i in indices
]
image_size = (block.image_descriptor.width,
block.image_descriptor.height)
image_pos = (block.image_descriptor.left,
block.image_descriptor.top)
new_state = blit_rgba(rgba_data, image_size, image_pos,
prev_state, self.size)
if disposal_method in [0, 1]:
# disposal method is unspecified or none
# do not restore the previous frame in any way
prev_state = new_state
elif disposal_method == 2:
# disposal method is background
# restore the used area to the background colour
fill_rgba = ([bg_colour] *
(image_size[0] * image_size[1]))
prev_state = blit_rgba(fill_rgba,
image_size,
image_pos,
new_state,
self.size,
transparency=False)
elif disposal_method == 3:
# disposal method is previous
# restore to previous frame after drawing on it
pass # prev_state is unchanged
else:
raise ValueError(
'Unknown disposal method: {}'.format(
disposal_method))
image = Image(new_state, image_size, delay_ms)
self.uncompressed_size += image_size[0] * image_size[1]
num_images += 1
logger.debug('GIF<%s>: Decoded frame %d',
self.filename, num_images)
yield image
# the GCE goes out of scope after being used once
active_gce = None
elif block.block_type == 'gce':
active_gce = block
elif block.block_type == 'comment':
# If there are multiple comment blocks, we ignore all
# but the last (this is unspecified behaviour).
self.comment = block.comment
elif block.block_type == 'application':
if (block.app_id == 'NETSCAPE'
and block.app_auth_code == '2.0'):
contents = construct.Struct(
'loop',
construct.ULInt8('id'),
construct.ULInt16('count'),
).parse(block.app_data)
if contents.id == 1:
self.loop_count = (contents.count + 1 if
contents.count != 0 else 0)
else:
logger.debug(
'Found unknown NETSCAPE extension id: %s',
contents.id,
)
else:
logger.debug('Found unknown app extension: %s',
(block.app_id, block.app_auth_code))
else:
logger.debug('Found unknown extension block: %s',
hex(block.ext_label))
self.is_loading = False
logger.info('GIF<%s>: Finished decoding image frames',
self.filename)
