def consume(self):
key, msg = self.db.brpop('queue/new_parts')
group, filename = msg.split('/', 1)
filename = filename.decode('ascii', 'replace')
file_posts = self.db.hgetall('%s/posts/%s' % (group, filename))
if not file_posts:
return
total = int(file_posts['total'])
have_posts = [file_posts.get(str(i), None) for i in range(1, total + 1)]
if not all(have_posts):
# Still don't have all the parts for this file, skip it
log.info('Skipping %s, not enough parts (%i/%i)' % (filename, len(have_posts), total))
return
else:
log.info('Have %i/%i parts for %s, downloading' % (len(have_posts), total, filename))
tmpdir = self.get_tmpdir(group)
nzbdir = self.get_nzbdir(group)
tmpfile = tmpdir + filename
self.send('GROUP %s' % group)
groupinfo = self.readline()
for postid in have_posts:
self.download_post(postid, tmpfile)
if not os.path.exists(tmpfile):
log.error('Unable to download %s to %s, giving up', postid, tmpfile)
return
outfile = nzbdir + filename
if os.path.exists(outfile):
log.error('Throwing away file %s, already decoded' % filename)
unlink(tmpfile)
return
try:
log.debug('yenc.decode("%s", "%s")' % (tmpfile, outfile))
yenc.decode(open(tmpfile, 'rb'), open(outfile, 'wb'))
except:
log.error(format_exc())
log.error('yEnc decode failed, cleaning up and bailing out')
unlink(tmpfile)
return
log.info('Successfully processed %s, cleaning up' % filename)
t = self.db.pipeline()
t.delete('%s/posts/%s' % (group, filename))
t.srem('%s/incomplete_files', filename)
t.lpush('queue/index_nzb', '%s/%s' % (group, filename))
t.execute()
unlink(tmpfile)
def test_sanity():
bites = b''.join([b.to_bytes(1, 'little') for b in range(256)])
size, crc, outdata = encode(bites)
size2, crc2, outdata2 = decode(outdata)
assert (bites == outdata2)
assert (crc == crc2)
assert (size == size2)
def testCrcIn(self):
"""Exercise yenc.decode(crc_in=...) parameter"""
with open(self.FILE_E, 'rb') as plain, \
open(self.FILE_E + ".out", 'w+b') as encoded, \
open(os.devnull, 'wb') as null:
_, crc = yenc.encode(plain, encoded)
# Correct CRC
encoded.seek(0)
yenc.decode(encoded, null, crc_in=crc)
# Incorrect CRC
crc = format(int(crc, 16) ^ 0xffffffff, "08x")
encoded.seek(0)
with self.assertRaises(yenc.Error):
yenc.decode(encoded, null, crc_in=crc)
def decode_from_lines(lines):
info = {}
if re.match("=ybegin ", lines[0]):
m = re.match("name=([^\s]*)", lines[0])
if m:
info["filename"] = m.group(1)
info["uuid"] = uuid.uuid4().hex
m = re.match("part=(\d*)", lines[0])
if m:
info["part"] = int(m.group(1))
m = re.match("total=(\d*)", lines[0])
if m:
info["total"] = int(m.group(1))
else:
raise Exception("First line must be start with '=ybegin ' (%s)" % lines[0])
if re.match("=yend", lines[-1]):
m = re.match("crc32=([^\s]*)", lines[-1])
if m:
info["crc32"] = m.group(1)
if re.match("=ypart ", lines[1]):
data = lines[2:-1]
else:
data = lines[1:-1]
# fenc = open(info['uuid']+'.yenc', "w")
data = "\n".join(data)
# fenc.write(data)
# fenc.close()
file_encoded = tempfile.NamedTemporaryFile(delete=False)
file_encoded.write(data)
file_encoded.close()
file_decoded = tempfile.NamedTemporaryFile(delete=False)
file_decoded.close()
yenc.decode(str(file_encoded.name), str(file_decoded.name))
info["data"] = open(file_decoded.name, "r").read()
file_decoded.close()
# info['tmpfilename'] = info['uuid'] + '.tmp'
# yenc.decode(info['uuid']+'.yenc', info['tmpfilename'])
# os.remove(info['uuid']+'.yenc')
return info
def _testDecodeFile(self, filename):
data, crc = self._readFile(filename)
with open(filename, 'rb') as file_in, \
open(filename + ".out", 'wb') as file_out:
bytes_out, crc_out = yenc.encode(file_in, file_out, len(data))
with open(filename + ".out", 'rb') as file_in, \
open(filename + ".dec", 'wb') as file_out:
bytes_dec, crc_dec = yenc.decode(file_in, file_out)
self.assertEqual(crc, crc_dec)
def main():
os.system("dd if=/dev/urandom of=sampledata bs=1M count=1")
file_in = open("sampledata","r")
file_out = open("sampledata.out","w")
data = file_in.read()
file_in.seek(0,0)
size = len(data)
crc = hex(crc32(data))[2:]
print "initial file crc: %s dec: %d" % (crc, crc32(data))
del(data)
print "Starting encoding test:"
startime = time.time()
bytes_out, crc_out = yenc.encode(file_in, file_out, size)
secs = time.time() - startime
print "Bytes out:", bytes_out, "output crc:", crc_out
if crc != crc_out:
print "crc error, 1Mb block encoding"
sys.exit(1)
elif bytes_out != size:
print "size error, 1Mb block encoding"
sys.exit(1)
else:
print "crc and size, 1Mb block encoding ok. %f secs." % secs
file_in.close()
file_out.close()
os.unlink("sampledata")
file_in = open("sampledata.out","r")
file_out = open("sampledata","w")
print "Starting decoding test:"
startime = time.time()
bytes_out, crc_out = yenc.decode(file_in, file_out, size)
secs = time.time() - startime
print "Bytes out:", bytes_out, "output crc:", crc_out
if crc != crc_out:
print "crc error, 1Mb block decoding"
sys.exit(1)
elif bytes_out != size:
print "size error, 1Mb block decoding"
sys.exit(1)
else:
print "crc and size, 1Mb block decoding ok. %f secs." % secs
os.unlink("sampledata")
os.unlink("sampledata.out")
def test_decode():
bites = [
b'*', b'+', b',', b'-', b'=n', b'/', b'0', b'1', b'2', b'3', b'4',
b'5', b'6', b'7', b'8', b'9', b':', b';', b'<', b'=}', b'>', b'?',
b'@', b'A', b'B', b'C', b'D', b'E', b'F', b'G', b'H', b'I', b'J', b'K',
b'L', b'M', b'N', b'O', b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W',
b'X', b'Y', b'Z', b'[', b'\\', b']', b'^', b'_', b'`', b'a', b'b',
b'c', b'd', b'e', b'f', b'g', b'h', b'i', b'j', b'k', b'l', b'm', b'n',
b'o', b'p', b'q', b'r', b's', b't', b'u', b'v', b'w', b'x', b'y', b'z',
b'{', b'|', b'}', b'~', b'\x7f', b'\x80', b'\x81', b'\x82', b'\x83',
b'\x84', b'\x85', b'\x86', b'\x87', b'\x88', b'\x89', b'\x8a', b'\x8b',
b'\x8c', b'\x8d', b'\x8e', b'\x8f', b'\x90', b'\x91', b'\x92', b'\x93',
b'\x94', b'\x95', b'\x96', b'\x97', b'\x98', b'\x99', b'\x9a', b'\x9b',
b'\x9c', b'\x9d', b'\x9e', b'\x9f', b'\xa0', b'\xa1', b'\xa2', b'\xa3',
b'\xa4', b'\xa5', b'\xa6', b'\xa7', b'\xa8', b'\xa9', b'\xaa', b'\xab',
b'\xac', b'\xad', b'\xae', b'\xaf', b'\xb0', b'\xb1', b'\xb2', b'\xb3',
b'\xb4', b'\xb5', b'\xb6', b'\xb7', b'\xb8', b'\xb9', b'\xba', b'\xbb',
b'\xbc', b'\xbd', b'\xbe', b'\xbf', b'\xc0', b'\xc1', b'\xc2', b'\xc3',
b'\xc4', b'\xc5', b'\xc6', b'\xc7', b'\xc8', b'\xc9', b'\xca', b'\xcb',
b'\xcc', b'\xcd', b'\xce', b'\xcf', b'\xd0', b'\xd1', b'\xd2', b'\xd3',
b'\xd4', b'\xd5', b'\xd6', b'\xd7', b'\xd8', b'\xd9', b'\xda', b'\xdb',
b'\xdc', b'\xdd', b'\xde', b'\xdf', b'\xe0', b'\xe1', b'\xe2', b'\xe3',
b'\xe4', b'\xe5', b'\xe6', b'\xe7', b'\xe8', b'\xe9', b'\xea', b'\xeb',
b'\xec', b'\xed', b'\xee', b'\xef', b'\xf0', b'\xf1', b'\xf2', b'\xf3',
b'\xf4', b'\xf5', b'\xf6', b'\xf7', b'\xf8', b'\xf9', b'\xfa', b'\xfb',
b'\xfc', b'\xfd', b'\xfe', b'\xff', b'=@', b'\x01', b'\x02', b'\x03',
b'\x04', b'\x05', b'\x06', b'\x07', b'\x08', b'=I', b'=J', b'\x0b',
b'\x0c', b'=M', b'\x0e', b'\x0f', b'\x10', b'\x11', b'\x12', b'\x13',
b'\x14', b'\x15', b'\x16', b'\x17', b'\x18', b'\x19', b'\x1a', b'\x1b',
b'\x1c', b'\x1d', b'\x1e', b'\x1f', b'=`', b'!', b'"', b'#', b'$',
b'%', b'&', b"'", b'(', b')'
]
soln = [b.to_bytes(1, 'little') for b in range(256)]
for i, b in enumerate(bites):
size, crc32, data = decode(b)
assert (data == soln[i])
def main():
head_crc = trail_crc = None
if len(sys.argv) > 1:
file_in = open(sys.argv[1],"rb")
else:
try: # Python 3
file_in = sys.stdin.detach()
except AttributeError: # Python < 3
file_in = sys.stdin
sys.stdin = None
with file_in:
while True:
line = file_in.readline()
if line.startswith(b"=ybegin "):
try:
name, size = NAME_RE.match(line).group(1), int(SIZE_RE.match(line).group(1))
m_obj = CRC32_RE.match(line)
if m_obj:
head_crc = m_obj.group(1)
except re.error:
sys.stderr.write("err-critical: malformed =ybegin header\n")
sys.exit(1)
break
elif not line:
sys.stderr.write("err-critical: no valid =ybegin header found\n")
sys.exit(1)
with open(name,"wb") as file_out:
try:
dec, dec_crc = yenc.decode(file_in, file_out, size)
except yenc.Error as e:
sys.stderr.write(str(e) + '\n')
sys.exit(1)
garbage = False
for line in file_in.read().split(b"\r\n"):
if line.startswith(b"=yend "):
try:
size = int( SIZE_RE.match(line).group(1) )
m_obj = CRC32_RE.match(line)
if m_obj:
trail_crc = m_obj.group(1)
except re.error:
sys.stderr.write("err: malformed =yend trailer\n")
break
elif not line:
continue
else:
garbage = True
else:
sys.stderr.write("warning: couldn't find =yend trailer\n")
if garbage:
sys.stderr.write("warning: garbage before =yend trailer\n")
if head_crc:
tmp_crc = head_crc.decode("ascii").lower()
elif trail_crc:
tmp_crc = trail_crc.decode("ascii").lower()
else:
sys.exit(0)
if tmp_crc != dec_crc:
sys.stderr.write("err: header: %s dec: %s CRC32 mismatch\n" % (tmp_crc,dec_crc) )
sys.exit(1)
else:
sys.exit(0)
m_obj = CRC32_RE.match(line)
if m_obj:
head_crc = m_obj.group(1)
except re.error, e:
sys.stderr.write(
"err-critical: malformed =ybegin header\n")
return False
break
elif not line:
sys.stderr.write(
"err-critical: no valid =ybegin header found\n")
return False
file_out = open(outFilename, "w")
try:
dec, dec_crc = yenc.decode(file_in, file_out, size)
except yenc.Error, e:
sys.stderr.write(str(e) + '\n')
return False
head_crc = trail_crc = tmp_crc = ""
garbage = 0
for line in file_in.read().split("\r\n"):
if line.startswith("=yend "):
try:
size = int(SIZE_RE.match(line).group(1))
m_obj = CRC32_RE.match(line)
if m_obj:
trail_crc = m_obj.group(1)
except re.error, e:
sys.stderr.write("err: malformed =yend trailer\n")
try:
name, size = NAME_RE.match(line).group(1), int(SIZE_RE.match(line).group(1))
m_obj = CRC32_RE.match(line)
if m_obj:
head_crc = m_obj.group(1)
except re.error, e:
sys.stderr.write("err-critical: malformed =ybegin header\n")
return False
break
elif not line:
sys.stderr.write("err-critical: no valid =ybegin header found\n")
return False
file_out = open(outFilename, "w")
try:
dec, dec_crc = yenc.decode(file_in, file_out, size)
except yenc.Error, e:
sys.stderr.write(str(e) + '\n')
return False
head_crc = trail_crc = tmp_crc = ""
garbage = 0
for line in file_in.read().split("\r\n"):
if line.startswith("=yend "):
try:
size = int( SIZE_RE.match(line).group(1) )
m_obj = CRC32_RE.match(line)
if m_obj:
trail_crc = m_obj.group(1)
except re.error, e:
sys.stderr.write("err: malformed =yend trailer\n")
b'=ybegin line=128 size=173 name=smallfile.rar\r\n',
b'|\x8b\x9cKD1*\xf9\xba\x9d**7*******\xcb\xfb\x9eJ\xaa[*\x8b***\x8b***-MW\xe5\xb8\x8e\x92\x95s>Z;*\xde\xab**\x9e\x8f\x9d\x9eW[ZZ\x8c\xa3\x9e\x8f\x9dX\x93\x97\x91\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x94\r\n',
b'4\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x9444\xeeg\xa5*j1*\r\n',
b'=yend size=173 crc32=8828b45c\r\n'
]
print("\nINPUT:")
print("First Line", data[0].rstrip())
print("Last Line", data[-1].rstrip())
bytes0 = bytearray()
for d in data[1:-1]:
bytes0.extend(d)
t0_yenc = time.time()
for i in range(10000000):
_, decodedcrc32, decoded = yenc.decode(bytes0)
dt_yenc = time.time() - t0_yenc
print(decoded)
# Find size in laste line of yencoded message:
lastline = data[-1].decode(
) # For example: '=yend size=173 crc32=8828b45c\r\n' (and ... assuming it's in the last line)
m = re.search('size=(.\d+?) ', lastline)
if m:
size = int(m.group(1))
print("size of decoded info will be", size)
# Now do the yencode-decoding using sabyenc:
t0_sabyenc = time.time()
for i in range(10000000):
decoded_data, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks(
from scapy.all import *
import base64
from yenc import decode
def get_ICMP_data(filename):
capture = rdpcap(filename)
ping_data = ""
for packet in capture:
if packet[ICMP].type == 8: # Echo request
ping_data += str(packet[Raw])
return ping_data
encrypted_data = ""
encrypted_data += get_ICMP_data("f101.pcap")
open('encrypted_data', "w").write(encrypted_data)
decode('encrypted_data', 'decrypted.txt')
