def _get_headparts(self, head):
mw = mimewords()
mw = partial(mw.process.__wrapped__.__wrapped__, mw)
jh = defaultdict(list)
for key, value in head:
jh[key].append(mw(''.join(t.lstrip() for t in value.splitlines(False))))
jh = {k: v[0] if len(v) == 1 else [t for t in v if t] for k, v in jh.items()}
yield UnpackResult('headers.txt',
lambda h=head: '\n'.join(F'{k}: {v}' for k, v in h).encode(self.codec))
yield UnpackResult('headers.json',
lambda jsn=jh: json.dumps(jsn, indent=4).encode(self.codec))
def make_message(name, msg):
with NoLogging():
try:
htm = msg.htmlBody
except Exception:
htm = None
try:
txt = msg.body
except Exception:
txt = None
if txt:
yield UnpackResult(F'{name}.txt', ensure_bytes(txt))
if htm:
yield UnpackResult(F'{name}.htm', ensure_bytes(htm))
def _search(self, pe, directory, level=0, *parts):
if level >= 3:
self.log_warn(F'unexpected resource tree level {level + 1:d}')
for entry in directory.entries:
if entry.name:
identifier = str(entry.name)
elif level == 0 and entry.id in iter(RSRC):
identifier = RSRC(entry.id).name
elif entry.id is not None:
identifier = str(entry.id)
else:
self.log_warn(
F'resource entry has name {entry.name} and id {entry.id} at level {level + 1:d}'
)
continue
if entry.struct.DataIsDirectory:
yield from self._search(pe, entry.directory, level + 1, *parts,
identifier)
else:
def extracted(p=pe, e=entry):
return p.get_data(e.data.struct.OffsetToData,
e.data.struct.Size)
path = '/'.join((*parts, identifier))
yield UnpackResult(path, extracted)
def unpack(self, data):
parser = self._oletools.rtfobj.RtfObjParser(data)
parser.parse()
width = len(str(len(parser.objects)))
for k, item in enumerate(parser.objects):
item: RtfObject
path = item.filename or F'carve{k:0{width}}.bin'
data = item.rawdata
meta = {}
if item.is_ole:
if item.format_id == self._oletools.oleobj.OleObject.TYPE_EMBEDDED:
meta['ole_type'] = 'EMBEDDED'
elif item.format_id == self._oletools.oleobj.OleObject.TYPE_LINKED:
meta['ole_type'] = 'LINKED'
if item.is_package:
meta['src_path'] = item.src_path
meta['tmp_path'] = item.temp_path
if item.clsid is not None:
meta['ole_info'] = item.clsid_desc
meta['ole_guid'] = item.clsid
meta['ole_name'] = item.class_name
if item.oledata:
data = item.oledata
pos = item.rawdata.find(data)
if pos > 0:
meta['raw_header'] = item.rawdata[:pos]
if item.olepkgdata:
data = item.olepkgdata
pos = item.oledata.find(data)
if pos >= 0:
meta['ole_header'] = item.oledata[:pos]
yield UnpackResult(path, data, **meta)
def tree(root: HTMLNode, *path):
def outer(root: HTMLNode = root):
return root.recover(inner=False).encode(self.codec)
def inner(root: HTMLNode = root):
return root.recover().encode(self.codec)
tagpath = '/'.join(path)
if root.root:
yield UnpackResult(tagpath, inner)
elif self.args.outer:
yield UnpackResult(tagpath, outer)
else:
yield UnpackResult(tagpath, inner)
for k, node in enumerate((n for n in root.children if not n.textual)):
yield from tree(node, *path, F'{k}.{node.tag}')
def _walk(self, key, *path):
here = '/'.join(path)
if not self._check_reachable(here):
self.log_debug(F'pruning search at {here}')
return
for value in key.values():
vpath = F'{here}/{value.name()}'
yield UnpackResult(vpath, lambda v=value: v.raw_data())
for subkey in key.subkeys():
yield from self._walk(subkey, *path, subkey.name())
def unpack(self, data):
header = DotNetHeader(data)
if not header.resources:
if self.args.list:
return
raise ValueError('This file contains no resources.')
for resource in header.resources:
yield UnpackResult(resource.Name, resource.Data)
def unpack(self, data):
cursor = 0
mv = memoryview(data)
while True:
offset = data.find(B'MZ', cursor)
if offset < cursor: break
cursor = offset + 2
ntoffset = mv[offset + 0x3C:offset + 0x3E]
if len(ntoffset) < 2:
return
ntoffset, = unpack('H', ntoffset)
if mv[offset + ntoffset:offset + ntoffset + 2] != B'PE':
self.log_debug(
F'invalid NT header signature for candidate at 0x{offset:08X}'
)
continue
try:
pe = PE(data=data[offset:], fast_load=True)
except PEFormatError as err:
self.log_debug(
F'parsing of PE header at 0x{offset:08X} failed:', err)
continue
pesize = get_pe_size(pe, memdump=self.args.memdump)
pedata = mv[offset:offset + pesize]
info = {}
if self.args.fileinfo:
try:
info = pemeta().parse_version(pe) or {}
except Exception as error:
self.log_warn(
F'Unable to obtain file information: {error!s}')
try:
path = info['OriginalFilename']
except KeyError:
extension = 'exe' if pe.is_exe() else 'dll' if pe.is_dll(
) else 'sys'
path = F'carve-0x{offset:08X}.{extension}'
if offset > 0 or self.args.keep_root:
yield UnpackResult(path, pedata, offset=offset)
self.log_info(
F'extracted PE file of size 0x{pesize:08X} from 0x{offset:08X}'
)
else:
self.log_info(
F'ignored root file of size 0x{pesize:08X} from 0x{offset:08X}'
)
continue
if not offset or self.args.recursive:
cursor += pe.OPTIONAL_HEADER.SizeOfHeaders
else:
cursor += pesize
def _pack(
self,
path: str,
date: Optional[Union[datetime, str]],
data: Union[ByteString, Callable[[], ByteString]],
**meta
) -> UnpackResult:
if isinstance(date, datetime):
date = date.isoformat(' ', 'seconds')
if isinstance(date, str):
meta[self.args.date.decode(self.codec)] = date
return UnpackResult(path, data, **meta)
def _unpack_file(self, data: bytearray):
for codec in ('utf16', 'utf-16le', 'utf8'):
try:
reg = data.decode(codec).splitlines(keepends=True)
except UnicodeError:
continue
if reg[0].startswith('Windows Registry Editor'):
break
else:
raise ParseException
config = WinRegFileParser()
config.read_string(''.join(reg[1:]))
for key in config.sections():
self.log_debug(key)
for value in config[key]:
name = next(iter(shlex.split(value)))
path = Path(key) / Path(name)
data = config[key][value]
decoded = list(self._decode_registry_export(data))
if len(decoded) == 1:
yield UnpackResult(str(path), decoded[0])
continue
for k, d in enumerate(decoded):
yield UnpackResult(F'{path!s}.{k}', d)
def unpack(self, data):
sentinel = uuid4()
try:
parser = self._olevba.VBA_Parser(sentinel,
data=bytes(data),
relaxed=True)
except self._olevba.FileOpenError:
raise ValueError('Input data not recognized by VBA parser')
for p1, stream_path, p2, code in parser.extract_all_macros():
if not stream_path:
if p1 == sentinel:
continue
if p2 == sentinel:
continue
yield UnpackResult(stream_path, code.encode(self.codec))
def unpack(self, data):
jc = JvClassFile(data)
tt = ' '
opcw = self._OPC_STRLEN
for method in jc.methods:
for attribute in method.attributes:
if attribute.name == 'Code': break
else:
self.log_warn(F'no code found for method: {method.name}')
continue
code: JvCode = attribute.parse(JvCode)
with io.StringIO() as display:
args, retval = re.match(R'^\((.*?)\)(.*?)$',
method.descriptor).groups()
print(F'{jc.this!s}::{method!s}{method.descriptor}',
file=display)
for op in code.disassembly:
olen = len(op.raw)
if op.table is None:
args = ', '.join(repr(a) for a in op.arguments)
else:
ow = 4 if op.code is opc.tableswitch else 8
olen = olen - (len(op.table) - 1) * ow
args = F'defaultjmp => {op.table[None]:#010x}'
jmps = []
for k, (key, jmp) in enumerate(op.table.items()):
if key is None:
continue
raw = self._hex(
op.raw[olen + k * ow:olen + k * ow + ow], ' ')
jmps.append(
F'{tt}{raw!s:<{opcw+15}} {key:#010x} => {jmp:#010x}'
)
args = '\n'.join((args, *jmps))
opch = self._hex(op.raw[:olen], ' ')
if len(opch) > 14:
opch += F'\n{tt}{tt:<15}'
print(F'{tt}{opch:<15}{op.code!r:<{opcw}} {args}',
file=display)
name = method.name
if name.startswith('<'):
this = jc.this.value.split('/')
this = this[-1]
name = F'{this}${name[1:-1]}'
yield UnpackResult(F'{name}.jd',
display.getvalue().encode(self.codec))
def _get_parts_outlook(self, data):
def ensure_bytes(data):
return data if isinstance(data, bytes) else data.encode(self.codec)
def make_message(name, msg):
with NoLogging():
try:
htm = msg.htmlBody
except Exception:
htm = None
try:
txt = msg.body
except Exception:
txt = None
if txt:
yield UnpackResult(F'{name}.txt', ensure_bytes(txt))
if htm:
yield UnpackResult(F'{name}.htm', ensure_bytes(htm))
msgcount = 0
with NoLogging():
msg = self._extract_msg.Message(bytes(data))
yield from self._get_headparts(msg.header.items())
yield from make_message('body', msg)
def attachments(msg):
for attachment in getattr(msg, 'attachments', ()):
yield attachment
if attachment.type == 'data':
continue
yield from attachments(attachment.data)
for attachment in attachments(msg):
self.log_debug(attachment)
if attachment.type == 'msg':
msgcount += 1
yield from make_message(F'attachments/msg_{msgcount:d}', attachment.data)
continue
if not isbuffer(attachment.data):
self.log_warn(F'unknown attachment of type {attachment.type}, please report this!')
continue
path = attachment.longFilename or attachment.shortFilename
yield UnpackResult(F'attachments/{path}', attachment.data)
def walk(node: xml.XMLNode, *path: str):
def extract(node: xml.XMLNode = node):
if not node.children:
return node.content.encode(self.codec)
with MemoryFile() as stream:
node.write(stream)
return bytes(stream.getbuffer() | ppxml)
children_by_tag = defaultdict(list)
for child in node.children:
children_by_tag[child.tag].append(child)
yield UnpackResult('/'.join(path), extract, **node.attributes)
for tag, children in children_by_tag.items():
if len(children) == 1:
yield from walk(children[0], *path, tag)
continue
width = len(F'{len(children):X}')
for k, child in enumerate(children):
yield from walk(child, *path, F'{tag}[0x{k:0{width}X}]')
def unpack(self, data):
with MemoryFile(data) as stream:
try:
oledoc = self._olefile.OleFileIO(stream)
except OSError as error:
self.log_info(F'error, {error}, treating input as zip file')
yield from xtzip().unpack(data)
return
for item in oledoc.listdir():
if not item or not item[-1]:
continue
path = '/'.join(item)
olestream = oledoc.openstream(path)
c0 = ord(item[-1][:1])
if c0 < 20:
item[-1] = F'[{c0:d}]{item[-1][1:]}'
path = '/'.join(item)
self.log_debug('exploring:', path)
yield UnpackResult(path, olestream.read())
def unpack(self, data):
try:
managed = NetStructuredResources(data)
except NoManagedResource:
managed = None
if not managed:
raise RefineryPartialResult('no managed resources found',
partial=data)
for entry in managed:
if entry.Error:
self.log_warn(
F'entry {entry.Name} carried error message: {entry.Error}')
data = entry.Data
if not self.args.raw:
if isinstance(entry.Value, str):
data = entry.Value.encode('utf-16le')
elif isbuffer(entry.Value):
data = entry.Value
yield UnpackResult(entry.Name, data)
def _get_parts_regular(self, data):
if not re.match(BR'^[\s!-~]+$', data):
raise ValueError('This is not a plaintext email message.')
msg = BytesParser().parsebytes(data)
yield from self._get_headparts(msg.items())
for k, part in enumerate(msg.walk()):
path = part.get_filename()
elog = None
if path is None:
extension = file_extension(part.get_content_type(), 'txt')
path = F'body.{extension}'
else:
path = F'attachments/{path}'
try:
data = part.get_payload(decode=True)
except Exception as E:
try:
data = part.get_payload(decode=False)
except Exception as E:
elog = str(E)
data = None
else:
from refinery import carve
self.log_warn(F'manually decoding part {k}, data might be corrupted: {path}')
if isinstance(data, str):
data = data.encode('latin1')
if isbuffer(data):
data = next(data | carve('b64', stripspace=True, single=True, decode=True))
else:
elog = str(E)
data = None
if not data:
if elog is not None:
self.log_warn(F'could not get content of message part {k}: {elog!s}')
continue
yield UnpackResult(path, data)
def unpack(self, data):
def crawl(path, cursor):
if isinstance(cursor, (dict, list)) and path:
path = F'{path}/'
if isinstance(cursor, dict):
for key, value in cursor.items():
yield from crawl(F'{path}{key}', value)
elif isinstance(cursor, list):
width = len(F'{len(cursor)-1:d}')
for key, value in enumerate(cursor):
yield from crawl(F'{path}#{key:0{width}d}', value)
if path:
yield path, cursor, cursor.__class__.__name__
for path, item, typename in crawl('', json.loads(data)):
def extract(item=item):
if isinstance(item, (list, dict)):
dumped = json.dumps(item, indent=4)
else:
dumped = str(item)
return dumped.encode(self.codec)
yield UnpackResult(path, extract, type=typename)
def unpack(self, data):
mv = memoryview(data)
for name, start, size in exeroute(data, self._unpack_elf,
self._unpack_macho, self._unpack_pe):
end = start + size
yield UnpackResult(name, mv[start:end], offset=start)
def unpack(self, data):
header = DotNetHeader(data, parse_resources=False)
tables = header.meta.Streams.Tables
fields = tables.FieldRVA
if not fields:
return
iwidth = len(str(len(fields)))
rwidth = max(len(F'{field.RVA:X}') for field in fields)
rwidth = max(rwidth, 4)
remaining_field_indices = set(range(len(tables.Field)))
for k, rv in enumerate(fields):
_index = rv.Field.Index
field = tables.Field[_index - 1]
remaining_field_indices.discard(_index - 1)
fname = field.Name
ftype = None
if len(field.Signature) == 2:
# Crude signature parser for non-array case. Reference:
# https://www.codeproject.com/Articles/42649/NET-File-Format-Signatures-Under-the-Hood-Part-1
# https://www.codeproject.com/Articles/42655/NET-file-format-Signatures-under-the-hood-Part-2
guess = {
0x03: FieldInfo('Char', 1, 1, None), # noqa
0x04: FieldInfo('SByte', 1, 1, None), # noqa
0x05: FieldInfo('Byte', 1, 1, None), # noqa
0x06: FieldInfo('Int16', 1, 2, None), # noqa
0x07: FieldInfo('UInt16', 1, 2, None), # noqa
0x08: FieldInfo('Int32', 1, 4, None), # noqa
0x09: FieldInfo('UInt32', 1, 4, None), # noqa
0x0A: FieldInfo('Int64', 1, 8, None), # noqa
0x0B: FieldInfo('UInt64', 1, 8, None), # noqa
0x0C: FieldInfo('Single', 1, 4, None), # noqa
0x0D: FieldInfo('Double', 1, 8, None), # noqa
}.get(field.Signature[1], None)
else:
guess = self._guess_field_info(tables, data, _index)
if guess is None:
self.log_debug(
lambda:
F'field {k:0{iwidth}d} name {field.Signature}: unable to guess type information'
)
continue
totalsize = guess.count * guess.size
if guess.name is not None:
fname = guess.name
if not fname.isprintable():
fname = F'F{rv.RVA:0{rwidth}X}'
ext = ftype = guess.type.lower()
if guess.count > 1:
ftype += F'[{guess.count}]'
self.log_info(
lambda:
F'field {k:0{iwidth}d} at RVA 0x{rv.RVA:04X} of type {guess.type}, count: {guess.count}, name: {fname}'
)
offset = header.pe.get_offset_from_rva(rv.RVA)
yield UnpackResult(
F'{fname}.{ext}',
lambda t=offset, s=totalsize: data[t:t + s],
name=fname,
type=ftype,
)
for _index in remaining_field_indices:
field = tables.Field[_index]
index = _index + 1
name = field.Name
if field.Flags.HasFieldRVA:
self.log_warn(
F'field {name} has RVA flag set, but no RVA was found')
token = index.to_bytes(3, 'little')
values = set()
for match in re.finditer(
(
BR'\x72(?P<token>...)\x70' # ldstr
BR'(?:\x6F(?P<function>...)\x0A)?' # call GetBytes
BR'\x80%s\x04' # stsfld
) % re.escape(token),
data,
re.DOTALL):
md = match.groupdict()
fn_token = md.get('function')
fn_index = fn_token and int.from_bytes(fn_token,
'little') or None
if fn_index is not None:
fn_name = tables.MemberRef[fn_index].Name
if fn_name != 'GetBytes':
self.log_warn(
F'skipping string assignment passing through call to {fn_name}'
)
continue
k = int.from_bytes(md['token'], 'little')
values.add(header.meta.Streams.US[k].encode(self.codec))
if not values:
continue
if len(values) == 1:
yield UnpackResult(F'{name}.str',
next(iter(values)),
name=name,
type='string')
def _walk(self, blob, memo: Optional[Set[int]] = None, *path):
while isinstance(blob, self._pypdf2.generic.IndirectObject):
blob = blob.getObject()
if memo is None:
memo = {id(blob)}
elif id(blob) in memo:
return
else:
memo.add(id(blob))
try:
name = blob['/F']
blob = blob['/EF']['/F']
except Exception:
pass
else:
path = *path[:-1], F'/{name}'
try:
if TYPE_CHECKING:
blob: EncodedStreamObject = cast(EncodedStreamObject, blob)
extract = blob.getData
except AttributeError:
pass
else:
yield UnpackResult(''.join(path), extract, kind='object')
return
if isinstance(blob, self._pypdf2.generic.ByteStringObject):
yield UnpackResult(''.join(path), blob, kind='bytes')
return
if isinstance(blob, self._pypdf2.generic.TextStringObject):
yield UnpackResult(''.join(path),
blob.encode(self.codec),
kind='string')
return
if isinstance(blob, (
self._pypdf2.generic.BooleanObject,
self._pypdf2.generic.ByteStringObject,
self._pypdf2.generic.FloatObject,
self._pypdf2.generic.NameObject,
self._pypdf2.generic.NullObject,
self._pypdf2.generic.NumberObject,
self._pypdf2.generic.RectangleObject,
)):
# unhandled PDF objects
return
if isinstance(blob, self._pypdf2.generic.TreeObject):
blob = list(blob)
pdf = self._pypdf2.generic.PdfObject
if isinstance(blob, list):
if (len(blob) % 2 == 0 and all(
isinstance(key, str)
for key in islice(iter(blob), 0, None, 2)) and all(
isinstance(key, pdf)
for key in islice(iter(blob), 1, None, 2))):
blob = dict(zip(*([iter(blob)] * 2)))
else:
for key, value in enumerate(blob):
yield from self._walk(value, memo, *path, F'/{key}')
return
if isinstance(blob, dict):
for key, value in blob.items():
if not isinstance(key, str):
continue
if not key.startswith('/'):
key = F'/{key}'
yield from self._walk(value, memo, *path, key)