def printout(self, stream, context): obj = Container() if self.show_stream: obj.stream_position = stream.tell() follows = stream.read(self.stream_lookahead) if not follows: obj.following_stream_data = "EOF reached" else: stream.seek(-len(follows), 1) obj.following_stream_data = HexString(follows) print("") if self.show_context: obj.context = context if self.show_stack: obj.stack = ListContainer() frames = [s[0] for s in inspect.stack()][1:-1] frames.reverse() for f in frames: a = Container() a.__update__(f.f_locals) obj.stack.append(a) print("=" * 80) print("Probe %s" % (self.printname, )) print(obj) print("=" * 80)
def _parse_file_references(self, stream=None): ''' Args: stream: TextIOWrapper|BytesIO => stream to read from Returns: List<Container<String, Any>> List of file references (see: src.structures.prefetch.PrefetchFileReferences) Preconditions: stream is of type TextIOWrapper or BytesIO (assumed True) ''' original_position = stream.tell() try: file_refs = list() for volumes_info_entry in self.volumes_info: try: stream.seek(self.file_info.SectionDOffset + volumes_info_entry.SectionEOffset) file_refs_entry = pfstructs.PrefetchFileReferences.parse_stream( stream) file_refs_entry.References = list( map(lambda ref: Container(**ref), file_refs_entry.References)) file_refs.append(file_refs_entry) except Exception as e: Logger.error('Error parsing file_refs_entry (%s)' % str(e)) file_refs.append(None) return file_refs finally: stream.seek(original_position)
def get_metadata(self, simple_hash=True): ''' Args: simple_hash: Boolean => whether to only collect SHA256 hash or MD5 and SHA1 as well Returns: Container<String, Any> Container of metadata about this prefetch file: file_name: prefetch file name file_path: full path on local system file_size: size of file on local system md5hash: MD5 hash of prefetch file sha1hash: SHA1 hash of prefetch file sha2hash: SHA256 hash of prefetch file modify_time: last modification time of prefetch file on local system access_time: last access time of prefetch file on local system create_time: create time of prefetch file on local system Preconditions: simple_hash is of type Boolean ''' assert isinstance(simple_hash, bool), 'Simple_hash is of type Boolean' return Container(\ file_name=path.basename(self._filepath), file_path=path.abspath(self._filepath), file_size=path.getsize(self._filepath), md5hash=self._hash_file('md5') if not simple_hash else None, sha1hash=self._hash_file('sha1') if not simple_hash else None, sha2hash=self._hash_file('sha256'), modify_time=datetime.fromtimestamp(path.getmtime(self._filepath), tzlocal()).astimezone(tzutc()), access_time=datetime.fromtimestamp(path.getatime(self._filepath), tzlocal()).astimezone(tzutc()), create_time=datetime.fromtimestamp(path.getctime(self._filepath), tzlocal()).astimezone(tzutc())\ )
def _clean_transform(self, value, serialize=False): ''' Args: value: Any => value to be converted Returns: Any Raw value if it is not of type Container, else recursively removes any key beginning with 'Raw' Preconditions: N/A ''' if issubclass(type(value), Container): cleaned_value = Container(value) if '_filepath' in cleaned_value: del cleaned_value['_filepath'] for key in cleaned_value: if key.startswith('Raw') or key.startswith('_'): del cleaned_value[key] else: cleaned_value[key] = self._clean_transform( cleaned_value[key], serialize) return cleaned_value elif isinstance(value, list): return list( map(lambda entry: self._clean_transform(entry, serialize), value)) elif isinstance(value, datetime) and serialize: return value.strftime('%Y-%m-%d %H:%M:%S.%f%z') else: return value
def _prepare_worker_pools(self): ''' @ParseDirectiveMixin._prepare_worker_pools ''' if self.pools is None: self.pools = Container() self.pools.progress = parallel.WorkerPool(\ parallel.JoinableQueue(-1), None, daemonize=False, worker_class=parallel.ProgressTrackerWorker, worker_count=1, worker_kwargs=dict(\ pcount=len(self.frontier), pdesc='Total', punit='files'\ )\ ) self.pools.parser = parallel.WorkerPool(\ parallel.JoinableQueue(-1), self._TASK_CLASS, daemonize=False, worker_count=self.args.threads, worker_kwargs=self._get_worker_kwargs(), task_kwargs=self._get_task_kwargs()\ )
def __init__(self, source, nodeidx, recordidx, **context): super(BaseParseFileOutputTask, self).__init__(source) self._nodeidx = nodeidx self._recordidx = recordidx if 'target' not in context: raise KeyError('target was not provided as a keyword argument') self._context = Container(**context)
def _prepare_worker_pools(self): ''' @ParseDirectiveMixin._prepare_worker_pools ''' if self.pools is None: self.pools = Container() self.pools.progress = parallel.WorkerPool(\ parallel.JoinableQueue(-1), tasks.ParseDBTaskStage2, daemonize=False, worker_class=parallel.DBProgressTrackerWorker, worker_count=1, worker_kwargs=dict(\ log_path=self.args.log_path, pcount=len(self.frontier), pdesc='Total', punit='files', manager=DBManager(conn_string=self.conn_string)\ )\ ) self.pools.parser = parallel.WorkerPool(\ parallel.JoinableQueue(-1), tasks.ParseDBTaskStage1, daemonize=False, worker_count=self.args.threads, worker_kwargs=dict(\ result_queue=self.pools.progress.queue, log_path=self.args.log_path\ ) )
def _clean_value(self, value, serialize=False): ''' Args: value: Any => value to be converted serialize: Boolean => transform values recursively to be JSON-serializable Returns: Any Raw value if it is not of type Container, else recursively removes any key beginning with 'Raw' Preconditions: serialize is of type Boolean ''' assert isinstance(serialize, bool) cleaned = value if issubclass(type(value), Container): cleaned = Container() for key in value.keys(): if not (key.startswith('Raw') or key.startswith('_')): cleaned[key] = self._clean_value(value[key], serialize) elif isinstance(value, list): cleaned = list( map(lambda entry: self._clean_value(entry, serialize), value)) elif isinstance(value, datetime) and serialize: cleaned = value.strftime('%Y-%m-%d %H:%M:%S.%f%z') return cleaned
def __str__(self): return str( Container( **{ key: str(getattr(self, prop.name)) for key, prop in self._PROPERTIES.items() }))
def _parse(self, stream, context): if "<obj>" in context: obj = context["<obj>"] del context["<obj>"] else: obj = Container() if self.nested: context = Container(_ = context) for sc in self.subcons: if sc.conflags & self.FLAG_EMBED: context["<obj>"] = obj sc._parse(stream, context) else: subobj = sc._parse(stream, context) if sc.name is not None: obj[sc.name] = subobj context[sc.name] = subobj return obj
def parse_stream(self, stream): """ Parse a stream. Files, pipes, sockets, and other streaming sources of data are handled by this method. """ return self._parse(stream, Container())
def _parse(self, stream, context): if "<obj>" in context: obj = context["<obj>"] del context["<obj>"] else: obj = Container() if self.nested: context = Container(_ = context) for sc in self.subcons: if sc.conflags & self.FLAG_EMBED: context["<obj>"] = obj sc._parse(stream, context) else: subobj = sc._parse(stream, context) if sc.name is not None: if sc.name in obj and not self.allow_overwrite: raise OverwriteError("%r would be overwritten but allow_overwrite is False" % (sc.name,)) obj[sc.name] = subobj context[sc.name] = subobj return obj
def _build(self, obj, stream, context): if "<unnested>" in context: del context["<unnested>"] elif self.nested: context = Container(_ = context) for sc in self.subcons: if sc.conflags & self.FLAG_EMBED: context["<unnested>"] = True subobj = obj elif sc.name is None: subobj = None else: subobj = obj[sc.name] context[sc.name] = subobj sc._build(subobj, stream, context)
def string_to_guid(guid): ''' Args: guid: String => guid string to convert to NTFSGUID Returns: Container<String, Any> Struct representation of String guid Preconditions: guid is of the format specified above (assumed True) ''' groups = guid.strip().split('-') return Container(Group1=int(groups[0], 16), Group2=int(groups[1], 16), Group3=int(groups[2], 16), Group4=int(groups[3], 16), Group5=int(groups[4], 16))
def string_to_sid(identifier): ''' Args: identifier: String => sid string to convert to NTFSSID Returns: Container<String, Any> Struct representation of String SID identifier Preconditions: identifier is of the format specified above (assumed True) ''' split_identifier = identifier.strip().split('-') sub_authorities = [int(subauth) for subauth in split_identifier[3:]] return Container(\ Revision=int(split_identifier[1]), SubAuthoritiesCount=len(sub_authorities), Authority=int(split_identifier[2]), SubAuthorities=sub_authorities\ )
def sizeof(self, context=None): """ Calculate the size of this object, optionally using a context. Some constructs have no fixed size and can only know their size for a given hunk of data; these constructs will raise an error if they are not passed a context. :param context: contextual data :returns: int of the length of this construct :raises SizeofError: the size could not be determined """ if context is None: context = Container() try: return self._sizeof(context) except Exception: raise SizeofError(sys.exc_info()[1])
def _parse_file_references(self, original_position, stream=None, file_info=None, volumes_info=None): ''' Args: original_position: Integer => position in stream before parsing this structure stream: TextIOWrapper|BytesIO => stream to read from file_info: Container => file information parsed from stream volumes_info: List<Container<String, Any>> => volumes information parsed from stream Returns: List<Container<String, Any>> List of file references (see: src.structures.prefetch.PrefetchFileReferences) Preconditions: original_position is of type Integer (assumed True) stream is of type TextIOWrapper or BytesIO (assumed True) file_info is of type Container (assumed True) volume_info is of type List<Container<String, Any>> (assumed True) ''' try: file_refs = list() for volumes_info_entry in volumes_info: try: stream.seek(file_info.SectionDOffset + volumes_info_entry.SectionEOffset) file_refs_entry = pfstructs.PrefetchFileReferences.parse_stream( stream) file_refs_entry.References = list( map(lambda ref: Container(**ref), file_refs_entry.References)) file_refs.append(file_refs_entry) except Exception as e: Logger.error('Error parsing file_refs_entry (%s)' % str(e)) file_refs.append(None) return self._clean_transform(file_refs) finally: stream.seek(original_position)
def _prepare_worker_pools(self): ''' @ParseDirectiveMixin._prepare_worker_pools ''' if self.pools is None: self.pools = Container() self.pools.progress = parallel.WorkerPool(\ parallel.JoinableQueue(-1), tasks.ParseDBTaskStage2, daemonize=False, worker_class=parallel.DBProgressTrackerWorker, worker_count=1\ ) self.pools.parser = parallel.WorkerPool(\ parallel.JoinableQueue(-1), tasks.ParseDBTaskStage1, daemonize=False, worker_count=self.args.threads, worker_kwargs=dict(\ result_queue=self.pools.progress.queue, log_path=self.args.log_path\ ) )
def build_stream(self, obj, stream): """ Build an object directly into a stream. """ self._build(obj, stream, Container())
def _sizeof(self, context): if self.nested: context = Container(_ = context) return sum(sc._sizeof(context) for sc in self.subcons)