def printout(self, stream, context):
obj = Container()
if self.show_stream:
obj.stream_position = stream.tell()
follows = stream.read(self.stream_lookahead)
if not follows:
obj.following_stream_data = "EOF reached"
else:
stream.seek(-len(follows), 1)
obj.following_stream_data = HexString(follows)
print("")
if self.show_context:
obj.context = context
if self.show_stack:
obj.stack = ListContainer()
frames = [s[0] for s in inspect.stack()][1:-1]
frames.reverse()
for f in frames:
a = Container()
a.__update__(f.f_locals)
obj.stack.append(a)
print("=" * 80)
print("Probe %s" % (self.printname, ))
print(obj)
print("=" * 80)
def _parse_file_references(self, stream=None):
'''
Args:
stream: TextIOWrapper|BytesIO => stream to read from
Returns:
List<Container<String, Any>>
List of file references (see: src.structures.prefetch.PrefetchFileReferences)
Preconditions:
stream is of type TextIOWrapper or BytesIO (assumed True)
'''
original_position = stream.tell()
try:
file_refs = list()
for volumes_info_entry in self.volumes_info:
try:
stream.seek(self.file_info.SectionDOffset +
volumes_info_entry.SectionEOffset)
file_refs_entry = pfstructs.PrefetchFileReferences.parse_stream(
stream)
file_refs_entry.References = list(
map(lambda ref: Container(**ref),
file_refs_entry.References))
file_refs.append(file_refs_entry)
except Exception as e:
Logger.error('Error parsing file_refs_entry (%s)' % str(e))
file_refs.append(None)
return file_refs
finally:
stream.seek(original_position)
def get_metadata(self, simple_hash=True):
'''
Args:
simple_hash: Boolean => whether to only collect SHA256 hash or
MD5 and SHA1 as well
Returns:
Container<String, Any>
Container of metadata about this prefetch file:
file_name: prefetch file name
file_path: full path on local system
file_size: size of file on local system
md5hash: MD5 hash of prefetch file
sha1hash: SHA1 hash of prefetch file
sha2hash: SHA256 hash of prefetch file
modify_time: last modification time of prefetch file on local system
access_time: last access time of prefetch file on local system
create_time: create time of prefetch file on local system
Preconditions:
simple_hash is of type Boolean
'''
assert isinstance(simple_hash, bool), 'Simple_hash is of type Boolean'
return Container(\
file_name=path.basename(self._filepath),
file_path=path.abspath(self._filepath),
file_size=path.getsize(self._filepath),
md5hash=self._hash_file('md5') if not simple_hash else None,
sha1hash=self._hash_file('sha1') if not simple_hash else None,
sha2hash=self._hash_file('sha256'),
modify_time=datetime.fromtimestamp(path.getmtime(self._filepath), tzlocal()).astimezone(tzutc()),
access_time=datetime.fromtimestamp(path.getatime(self._filepath), tzlocal()).astimezone(tzutc()),
create_time=datetime.fromtimestamp(path.getctime(self._filepath), tzlocal()).astimezone(tzutc())\
)
def _clean_transform(self, value, serialize=False):
'''
Args:
value: Any => value to be converted
Returns:
Any
Raw value if it is not of type Container, else recursively removes
any key beginning with 'Raw'
Preconditions:
N/A
'''
if issubclass(type(value), Container):
cleaned_value = Container(value)
if '_filepath' in cleaned_value:
del cleaned_value['_filepath']
for key in cleaned_value:
if key.startswith('Raw') or key.startswith('_'):
del cleaned_value[key]
else:
cleaned_value[key] = self._clean_transform(
cleaned_value[key], serialize)
return cleaned_value
elif isinstance(value, list):
return list(
map(lambda entry: self._clean_transform(entry, serialize),
value))
elif isinstance(value, datetime) and serialize:
return value.strftime('%Y-%m-%d %H:%M:%S.%f%z')
else:
return value
def _prepare_worker_pools(self):
'''
@ParseDirectiveMixin._prepare_worker_pools
'''
if self.pools is None:
self.pools = Container()
self.pools.progress = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
None,
daemonize=False,
worker_class=parallel.ProgressTrackerWorker,
worker_count=1,
worker_kwargs=dict(\
pcount=len(self.frontier),
pdesc='Total',
punit='files'\
)\
)
self.pools.parser = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
self._TASK_CLASS,
daemonize=False,
worker_count=self.args.threads,
worker_kwargs=self._get_worker_kwargs(),
task_kwargs=self._get_task_kwargs()\
)
def __init__(self, source, nodeidx, recordidx, **context):
super(BaseParseFileOutputTask, self).__init__(source)
self._nodeidx = nodeidx
self._recordidx = recordidx
if 'target' not in context:
raise KeyError('target was not provided as a keyword argument')
self._context = Container(**context)
def _prepare_worker_pools(self):
'''
@ParseDirectiveMixin._prepare_worker_pools
'''
if self.pools is None:
self.pools = Container()
self.pools.progress = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
tasks.ParseDBTaskStage2,
daemonize=False,
worker_class=parallel.DBProgressTrackerWorker,
worker_count=1,
worker_kwargs=dict(\
log_path=self.args.log_path,
pcount=len(self.frontier),
pdesc='Total',
punit='files',
manager=DBManager(conn_string=self.conn_string)\
)\
)
self.pools.parser = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
tasks.ParseDBTaskStage1,
daemonize=False,
worker_count=self.args.threads,
worker_kwargs=dict(\
result_queue=self.pools.progress.queue,
log_path=self.args.log_path\
)
)
def _clean_value(self, value, serialize=False):
'''
Args:
value: Any => value to be converted
serialize: Boolean => transform values recursively to be JSON-serializable
Returns:
Any
Raw value if it is not of type Container, else recursively removes
any key beginning with 'Raw'
Preconditions:
serialize is of type Boolean
'''
assert isinstance(serialize, bool)
cleaned = value
if issubclass(type(value), Container):
cleaned = Container()
for key in value.keys():
if not (key.startswith('Raw') or key.startswith('_')):
cleaned[key] = self._clean_value(value[key], serialize)
elif isinstance(value, list):
cleaned = list(
map(lambda entry: self._clean_value(entry, serialize), value))
elif isinstance(value, datetime) and serialize:
cleaned = value.strftime('%Y-%m-%d %H:%M:%S.%f%z')
return cleaned
def __str__(self):
return str(
Container(
**{
key: str(getattr(self, prop.name))
for key, prop in self._PROPERTIES.items()
}))
def _parse(self, stream, context):
if "<obj>" in context:
obj = context["<obj>"]
del context["<obj>"]
else:
obj = Container()
if self.nested:
context = Container(_ = context)
for sc in self.subcons:
if sc.conflags & self.FLAG_EMBED:
context["<obj>"] = obj
sc._parse(stream, context)
else:
subobj = sc._parse(stream, context)
if sc.name is not None:
obj[sc.name] = subobj
context[sc.name] = subobj
return obj
def parse_stream(self, stream):
"""
Parse a stream.
Files, pipes, sockets, and other streaming sources of data are handled
by this method.
"""
return self._parse(stream, Container())
def _parse(self, stream, context):
if "<obj>" in context:
obj = context["<obj>"]
del context["<obj>"]
else:
obj = Container()
if self.nested:
context = Container(_ = context)
for sc in self.subcons:
if sc.conflags & self.FLAG_EMBED:
context["<obj>"] = obj
sc._parse(stream, context)
else:
subobj = sc._parse(stream, context)
if sc.name is not None:
if sc.name in obj and not self.allow_overwrite:
raise OverwriteError("%r would be overwritten but allow_overwrite is False" % (sc.name,))
obj[sc.name] = subobj
context[sc.name] = subobj
return obj
def _build(self, obj, stream, context):
if "<unnested>" in context:
del context["<unnested>"]
elif self.nested:
context = Container(_ = context)
for sc in self.subcons:
if sc.conflags & self.FLAG_EMBED:
context["<unnested>"] = True
subobj = obj
elif sc.name is None:
subobj = None
else:
subobj = obj[sc.name]
context[sc.name] = subobj
sc._build(subobj, stream, context)
def string_to_guid(guid):
'''
Args:
guid: String => guid string to convert to NTFSGUID
Returns:
Container<String, Any>
Struct representation of String guid
Preconditions:
guid is of the format specified above (assumed True)
'''
groups = guid.strip().split('-')
return Container(Group1=int(groups[0], 16),
Group2=int(groups[1], 16),
Group3=int(groups[2], 16),
Group4=int(groups[3], 16),
Group5=int(groups[4], 16))
def string_to_sid(identifier):
'''
Args:
identifier: String => sid string to convert to NTFSSID
Returns:
Container<String, Any>
Struct representation of String SID identifier
Preconditions:
identifier is of the format specified above (assumed True)
'''
split_identifier = identifier.strip().split('-')
sub_authorities = [int(subauth) for subauth in split_identifier[3:]]
return Container(\
Revision=int(split_identifier[1]),
SubAuthoritiesCount=len(sub_authorities),
Authority=int(split_identifier[2]),
SubAuthorities=sub_authorities\
)
def sizeof(self, context=None):
"""
Calculate the size of this object, optionally using a context.
Some constructs have no fixed size and can only know their size for a
given hunk of data; these constructs will raise an error if they are
not passed a context.
:param context: contextual data
:returns: int of the length of this construct
:raises SizeofError: the size could not be determined
"""
if context is None:
context = Container()
try:
return self._sizeof(context)
except Exception:
raise SizeofError(sys.exc_info()[1])
def _parse_file_references(self,
original_position,
stream=None,
file_info=None,
volumes_info=None):
'''
Args:
original_position: Integer => position in stream before parsing this structure
stream: TextIOWrapper|BytesIO => stream to read from
file_info: Container => file information parsed from stream
volumes_info: List<Container<String, Any>> => volumes information parsed from stream
Returns:
List<Container<String, Any>>
List of file references (see: src.structures.prefetch.PrefetchFileReferences)
Preconditions:
original_position is of type Integer (assumed True)
stream is of type TextIOWrapper or BytesIO (assumed True)
file_info is of type Container (assumed True)
volume_info is of type List<Container<String, Any>> (assumed True)
'''
try:
file_refs = list()
for volumes_info_entry in volumes_info:
try:
stream.seek(file_info.SectionDOffset +
volumes_info_entry.SectionEOffset)
file_refs_entry = pfstructs.PrefetchFileReferences.parse_stream(
stream)
file_refs_entry.References = list(
map(lambda ref: Container(**ref),
file_refs_entry.References))
file_refs.append(file_refs_entry)
except Exception as e:
Logger.error('Error parsing file_refs_entry (%s)' % str(e))
file_refs.append(None)
return self._clean_transform(file_refs)
finally:
stream.seek(original_position)
def _prepare_worker_pools(self):
'''
@ParseDirectiveMixin._prepare_worker_pools
'''
if self.pools is None:
self.pools = Container()
self.pools.progress = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
tasks.ParseDBTaskStage2,
daemonize=False,
worker_class=parallel.DBProgressTrackerWorker,
worker_count=1\
)
self.pools.parser = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
tasks.ParseDBTaskStage1,
daemonize=False,
worker_count=self.args.threads,
worker_kwargs=dict(\
result_queue=self.pools.progress.queue,
log_path=self.args.log_path\
)
)
def build_stream(self, obj, stream):
"""
Build an object directly into a stream.
"""
self._build(obj, stream, Container())
def _sizeof(self, context):
if self.nested:
context = Container(_ = context)
return sum(sc._sizeof(context) for sc in self.subcons)
