def _decompress_chunk(self, size=None):
bitcount = 0
bitstore = 0
decompressed = 1
def readbit():
nonlocal bitcount, bitstore
if not bitcount:
bitstore = int.from_bytes(self._src.read_exactly(2), 'little')
bitcount = 0xF
else:
bitcount = bitcount - 1
return (bitstore >> bitcount) & 1
def readint():
result = 2 + readbit()
while readbit():
result <<= 1
result += readbit()
return result
self._dst.write(self._src.read_exactly(1))
try:
while not size or decompressed < size:
if readbit():
length = readint() + 2
sector = readint() - 2
offset = self._src.read_byte() + 1
delta = offset + 0x100 * sector
available = self._dst.tell()
if delta not in range(available + 1):
raise RefineryPartialResult(
F'Requested rewind by 0x{delta:08X} bytes with only 0x{available:08X} bytes in output buffer.',
partial=self._dst.getvalue())
quotient, remainder = divmod(length, delta)
replay = memoryview(self._dst.getbuffer())
replay = bytes(replay[-delta:] if quotient else replay[-delta:length - delta])
replay = quotient * replay + replay[:remainder]
self._dst.write(replay)
decompressed += length
else:
self._dst.write(self._src.read_exactly(1))
decompressed += 1
except EOF as E:
raise RefineryPartialResult(str(E), partial=self._dst.getbuffer())
dst = self._dst.getbuffer()
if decompressed < size:
raise RefineryPartialResult(
F'Attempted to decompress {size} bytes, got only {len(dst)}.', dst)
if decompressed > size:
raise RuntimeError('Decompressed buffer contained more bytes than expected.')
return dst
def process(self, password):
from Crypto.Cipher import DES
from Crypto.Util.strxor import strxor
key = bytearray(8)
for i, j in enumerate(password):
if ((i % 16) < 8):
key[i % 8] ^= (j << 1) & 0xFF
else:
j = (((j << 4) & 0xf0) | ((j >> 4) & 0x0f))
j = (((j << 2) & 0xcc) | ((j >> 2) & 0x33))
j = (((j << 1) & 0xaa) | ((j >> 1) & 0x55))
key[7 - (i % 8)] ^= j
des_set_odd_parity(key)
if password:
n = len(password)
password = password.ljust(n + 7 - ((n - 1) % 8), b'\0')
des = DES.new(key, DES.MODE_ECB)
for k in range(0, n, 8):
key[:] = des.encrypt(strxor(password[k:k + 8], key))
des_set_odd_parity(key)
if self.args.size > 8:
raise RefineryPartialResult('DESDerive can provide at most 8 bytes.', partial=key)
return key[:self.args.size]
def process(self, data):
crypted = bytes(UnixCrypt(data, salt=self.args.salt))
if len(crypted) < self.args.size:
raise RefineryPartialResult(
F'unix crypt only provided {len(crypted)} bytes, but {self.args.size} '
F'were requested.',
partial=crypted)
return crypted[:self.args.size]
def process(self, data):
out = io.BytesIO()
offset = 0
while offset < len(data):
try:
header, = struct.unpack('<H', data[offset:offset + 2])
except struct.error as err:
raise RefineryPartialResult(str(err), partial=out.getvalue())
offset += 2
size = (header & 0xFFF) + 1
if size + 1 >= len(data):
raise RefineryPartialResult(
F'chunk header indicates size {size}, but only {len(data)} bytes remain.',
partial=out.getvalue())
chunk = data[offset:offset + size]
offset += size
if header & 0x8000:
chunk = self._decompress_chunk(chunk)
out.write(chunk)
return out.getvalue()
def process(self, data: ByteString) -> ByteString:
previous = crc32(data)
for _ in range(self.args.timeout):
try:
data = super().process(data)
except KeyboardInterrupt:
raise RefineryPartialResult('Returning partially deobfuscated data', partial=data)
checksum = crc32(data)
if checksum == previous:
break
previous = checksum
else:
raise AutoDeobfuscationTimeout(data)
return data
def _read_block(self, reader, output, ubound=None):
entry = reader.tell()
lastend = 0
def ubound_check():
if ubound is None:
return False
consumed = reader.tell() - entry
if consumed > ubound:
raise ValueError(
F'upper bound {ubound} exceeded by {consumed-ubound} in LZ4 block'
)
return consumed == ubound
while not reader.eof:
reflen = reader.read_nibble()
litlen = reader.read_nibble()
litlen = reader.read_size(litlen)
literal = reader.read(litlen)
output.write(literal)
if ubound_check(): break
try:
refpos = reader.u16()
except EOF:
break
if refpos - 1 not in range(output.tell()):
with StreamDetour(output, lastend):
if output.read(len(literal)) == literal:
# This literal could have been encoded in the last match, but it wasn't.
# Therefore, it is very likely that we have reached the end of the stream.
break
position = reader.tell()
remaining = len(literal) - position
raise RefineryPartialResult(
F'encountered invalid match offset value {refpos} at position {position} with {remaining} bytes remaining',
partial=output.getvalue())
reflen = reader.read_size(reflen)
if ubound_check():
raise ValueError('last sequence in block contained a match')
reflen += 4
available_bytes = min(refpos, reflen)
q, r = divmod(reflen, available_bytes)
with StreamDetour(output, -refpos, io.SEEK_CUR):
match = output.read(available_bytes)
match = q * match + match[:r]
assert len(match) == reflen
lastend = output.tell() - available_bytes + r
output.write(match)
def process(self, data: ByteString) -> ByteString:
from Crypto.Util.Padding import unpad
result = super().process(data)
for p in self.args.padding:
if p == 'raw':
return result
try:
unpadded = unpad(result, self.blocksize, p.lower())
except Exception:
pass
else:
self.log_info(F'unpadding worked using {p}')
return unpadded
raise RefineryPartialResult(
'None of these paddings worked: {}'.format(', '.join(self.args.padding)),
partial=result)
def process(self, data):
unpacker = mp.Unpacker(MemoryFile(data, read_as_bytes=True))
while True:
try:
item = unpacker.unpack()
except mp.exceptions.OutOfData:
position = unpacker.tell()
if position < len(data):
self.log_warn("oops")
break
except Exception as E:
position = unpacker.tell()
if not position:
raise
view = memoryview(data)
raise RefineryPartialResult(str(E), view[position:])
else:
yield json.dumps(item).encode(self.codec)
def unpack(self, data):
try:
managed = NetStructuredResources(data)
except NoManagedResource:
managed = None
if not managed:
raise RefineryPartialResult('no managed resources found',
partial=data)
for entry in managed:
if entry.Error:
self.log_warn(
F'entry {entry.Name} carried error message: {entry.Error}')
data = entry.Data
if not self.args.raw:
if isinstance(entry.Value, str):
data = entry.Value.encode('utf-16le')
elif isbuffer(entry.Value):
data = entry.Value
yield UnpackResult(entry.Name, data)
def process(self, data):
def digest(x):
return self.hash.new(x).digest()
if self.args.hash in (HASH.SHA224, HASH.SHA256, HASH.SHA384,
HASH.SHA512):
return digest(data)[:self.args.size]
max_size = 2 * self.hash.digest_size
value = digest(data)
del data
buffer1 = bytearray([0x36] * 64)
buffer2 = bytearray([0x5C] * 64)
for k, b in enumerate(value):
buffer1[k] ^= b
buffer2[k] ^= b
buffer = digest(buffer1) + digest(buffer2)
if self.args.size > max_size:
raise RefineryPartialResult(
F'too many bytes requested, can only provide {max_size}',
partial=buffer)
return buffer[:self.args.size]
def process(self, data: bytearray) -> bytearray:
pe = PE(data=data, fast_load=True)
pe.parse_data_directories(directories=[self._SECDIRID])
security = pe.OPTIONAL_HEADER.DATA_DIRECTORY[self._SECDIRID]
self.log_info(F'signature offset: 0x{security.VirtualAddress:08X}')
self.log_info(F'signature length: 0x{security.Size:08X}')
if security.VirtualAddress == 0 or security.Size == 0:
raise ValueError(
F'IMAGE_DIRECTORY_ENTRY_SECURITY ({self._SECDIRID}) is corrupt.'
)
sgnoff = security.VirtualAddress + 8
sgnend = sgnoff + security.Size
length, revision, certtype = unpack('<IHH', data[sgnoff - 8:sgnoff])
signature = data[sgnoff:sgnend]
if len(signature) + 8 != length:
raise RefineryPartialResult(
F'Found {len(signature) + 8} bytes of signature, but length should be {length}.',
partial=signature)
return signature
def process(self, data):
def shlexjoin():
import shlex
return ' '.join(shlex.quote(cmd) for cmd in commandline)
meta = metavars(data, ghost=True)
used = set()
commandline = [
meta.format(cmd, self.codec, [data], None, False, used=used)
for cmd in self.args.commandline
]
if 0 in used:
self.log_info(
'input used as command-line argument; sending no input to process stdin'
)
data = None
self.log_debug(shlexjoin)
posix = 'posix' in sys.builtin_module_names
process = Popen(commandline,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
shell=False,
close_fds=posix)
if self.args.buffer and not self.args.timeout:
out, err = process.communicate(data)
for line in err.splitlines():
self.log_info(line)
yield out
return
import io
from threading import Thread, Event
from queue import Queue, Empty
from time import process_time, sleep
start = 0
result = None
qerr = Queue()
qout = Queue()
done = Event()
def adapter(stream, queue: Queue, event: Event):
while not event.is_set():
out = stream.read1()
if out: queue.put(out)
else: break
stream.close()
recvout = Thread(target=adapter,
args=(process.stdout, qout, done),
daemon=True)
recverr = Thread(target=adapter,
args=(process.stderr, qerr, done),
daemon=True)
recvout.start()
recverr.start()
if data:
process.stdin.write(data)
process.stdin.close()
start = process_time()
if self.args.buffer or self.args.timeout:
result = io.BytesIO()
def queue_read(q: Queue):
try:
return q.get_nowait()
except Empty:
return None
errbuf = io.BytesIO()
while True:
out = queue_read(qout)
err = None
if self.args.noerror:
err = queue_read(qerr)
else:
out = out or queue_read(qerr)
if err and self.log_info():
errbuf.write(err)
errbuf.seek(0)
lines = errbuf.readlines()
errbuf.seek(0)
errbuf.truncate()
if lines:
if not (done.is_set() or lines[~0].endswith(B'\n')):
errbuf.write(lines.pop())
for line in lines:
msg = line.rstrip(B'\n')
if msg: self.log_info(msg)
if out:
if self.args.buffer or self.args.timeout:
result.write(out)
if not self.args.buffer:
yield out
if done.is_set():
if recverr.is_alive():
self.log_warn('stderr receiver thread zombied')
if recvout.is_alive():
self.log_warn('stdout receiver thread zombied')
break
elif not err and not out and process.poll() is not None:
recverr.join(self._JOIN_TIME)
recvout.join(self._JOIN_TIME)
done.set()
elif self.args.timeout:
if process_time() - start > self.args.timeout:
self.log_info('terminating process after timeout expired')
done.set()
process.terminate()
for wait in range(4):
if process.poll() is not None:
break
sleep(self._JOIN_TIME)
else:
self.log_warn('process termination may have failed')
recverr.join(self._JOIN_TIME)
recvout.join(self._JOIN_TIME)
if not len(result.getbuffer()):
result = RuntimeError(
'timeout reached, process had no output')
else:
result = RefineryPartialResult(
'timeout reached, returning all collected output',
partial=result.getvalue())
if isinstance(result, Exception):
raise result
elif self.args.buffer:
yield result.getvalue()
def process(self, data: bytearray):
if len(data) <= 1:
yield data
return
memview = memoryview(data)
weight = 1 + (self.args.weight / 10)
if self.args.chug:
patterns = self._get_patterns(memview)
else:
patterns = set()
chunksize = self.args.buffer
for k in range(0, len(memview), chunksize):
patterns |= self._get_patterns(memview[k:k + chunksize])
if not patterns:
raise RefineryPartialResult('no repeating sequences found', data)
self.log_debug('removing duplicate pattern detections')
duplicates = set()
maxlen = max(len(p) for p in patterns)
for pattern in sorted(patterns, key=len):
for k in range(2, maxlen // len(pattern) + 1):
repeated = pattern * k
if repeated in patterns:
duplicates.add(repeated)
patterns -= duplicates
self.log_debug(F'counting coverage of {len(patterns)} patterns')
pattern_count = {p: data.count(p) for p in patterns}
pattern_performance = dict(pattern_count)
for consecutive in (False, True):
if consecutive:
self.log_debug(F're-counting coverage of {len(patterns)} patterns')
patterns = {self._truncate_pattern(p) for p in patterns}
pattern_performance = {p: self._consecutive_count(data, p) for p in patterns}
self.log_debug('evaluating pattern performance')
for pattern, count in pattern_performance.items():
pattern_performance[pattern] = count * (len(pattern) ** weight)
best_performance = max(pattern_performance.values())
for pattern, performance in pattern_performance.items():
pattern_performance[pattern] = performance / best_performance
self.log_debug('removing patterns below performance threshold')
threshold = self.args.threshold
patterns = {p for p in patterns if pattern_performance[p] * 100 >= threshold}
pattern_count = {p: data.count(p) for p in patterns}
if not self.args.consecutive:
break
if self.args.all:
for pattern in sorted(patterns, key=pattern_performance.get, reverse=True):
yield self.labelled(pattern, count=pattern_count[pattern])
return
best_patterns = [p for p in patterns if pattern_performance[p] == 1.0]
if len(best_patterns) > 1:
self.log_warn('could not determine unique best repeating pattern, returning the first of these:')
for k, pattern in enumerate(best_patterns):
self.log_warn(F'{k:02d}.: {pattern.hex()}')
result = best_patterns[0]
if self.args.align:
def rotated(pattern):
for k in range(len(pattern)):
yield pattern[k:] + pattern[:k]
rotations = {k % len(result): r for k, r in (
(data.find(r), r) for r in rotated(result)) if k >= 0}
result = rotations[min(rotations)]
yield result