def consume(self): key, msg = self.db.brpop('queue/new_parts') group, filename = msg.split('/', 1) filename = filename.decode('ascii', 'replace') file_posts = self.db.hgetall('%s/posts/%s' % (group, filename)) if not file_posts: return total = int(file_posts['total']) have_posts = [file_posts.get(str(i), None) for i in range(1, total + 1)] if not all(have_posts): # Still don't have all the parts for this file, skip it log.info('Skipping %s, not enough parts (%i/%i)' % (filename, len(have_posts), total)) return else: log.info('Have %i/%i parts for %s, downloading' % (len(have_posts), total, filename)) tmpdir = self.get_tmpdir(group) nzbdir = self.get_nzbdir(group) tmpfile = tmpdir + filename self.send('GROUP %s' % group) groupinfo = self.readline() for postid in have_posts: self.download_post(postid, tmpfile) if not os.path.exists(tmpfile): log.error('Unable to download %s to %s, giving up', postid, tmpfile) return outfile = nzbdir + filename if os.path.exists(outfile): log.error('Throwing away file %s, already decoded' % filename) unlink(tmpfile) return try: log.debug('yenc.decode("%s", "%s")' % (tmpfile, outfile)) yenc.decode(open(tmpfile, 'rb'), open(outfile, 'wb')) except: log.error(format_exc()) log.error('yEnc decode failed, cleaning up and bailing out') unlink(tmpfile) return log.info('Successfully processed %s, cleaning up' % filename) t = self.db.pipeline() t.delete('%s/posts/%s' % (group, filename)) t.srem('%s/incomplete_files', filename) t.lpush('queue/index_nzb', '%s/%s' % (group, filename)) t.execute() unlink(tmpfile)
def test_sanity(): bites = b''.join([b.to_bytes(1, 'little') for b in range(256)]) size, crc, outdata = encode(bites) size2, crc2, outdata2 = decode(outdata) assert (bites == outdata2) assert (crc == crc2) assert (size == size2)
def testCrcIn(self): """Exercise yenc.decode(crc_in=...) parameter""" with open(self.FILE_E, 'rb') as plain, \ open(self.FILE_E + ".out", 'w+b') as encoded, \ open(os.devnull, 'wb') as null: _, crc = yenc.encode(plain, encoded) # Correct CRC encoded.seek(0) yenc.decode(encoded, null, crc_in=crc) # Incorrect CRC crc = format(int(crc, 16) ^ 0xffffffff, "08x") encoded.seek(0) with self.assertRaises(yenc.Error): yenc.decode(encoded, null, crc_in=crc)
def decode_from_lines(lines): info = {} if re.match("=ybegin ", lines[0]): m = re.match("name=([^\s]*)", lines[0]) if m: info["filename"] = m.group(1) info["uuid"] = uuid.uuid4().hex m = re.match("part=(\d*)", lines[0]) if m: info["part"] = int(m.group(1)) m = re.match("total=(\d*)", lines[0]) if m: info["total"] = int(m.group(1)) else: raise Exception("First line must be start with '=ybegin ' (%s)" % lines[0]) if re.match("=yend", lines[-1]): m = re.match("crc32=([^\s]*)", lines[-1]) if m: info["crc32"] = m.group(1) if re.match("=ypart ", lines[1]): data = lines[2:-1] else: data = lines[1:-1] # fenc = open(info['uuid']+'.yenc', "w") data = "\n".join(data) # fenc.write(data) # fenc.close() file_encoded = tempfile.NamedTemporaryFile(delete=False) file_encoded.write(data) file_encoded.close() file_decoded = tempfile.NamedTemporaryFile(delete=False) file_decoded.close() yenc.decode(str(file_encoded.name), str(file_decoded.name)) info["data"] = open(file_decoded.name, "r").read() file_decoded.close() # info['tmpfilename'] = info['uuid'] + '.tmp' # yenc.decode(info['uuid']+'.yenc', info['tmpfilename']) # os.remove(info['uuid']+'.yenc') return info
def _testDecodeFile(self, filename): data, crc = self._readFile(filename) with open(filename, 'rb') as file_in, \ open(filename + ".out", 'wb') as file_out: bytes_out, crc_out = yenc.encode(file_in, file_out, len(data)) with open(filename + ".out", 'rb') as file_in, \ open(filename + ".dec", 'wb') as file_out: bytes_dec, crc_dec = yenc.decode(file_in, file_out) self.assertEqual(crc, crc_dec)
def main(): os.system("dd if=/dev/urandom of=sampledata bs=1M count=1") file_in = open("sampledata","r") file_out = open("sampledata.out","w") data = file_in.read() file_in.seek(0,0) size = len(data) crc = hex(crc32(data))[2:] print "initial file crc: %s dec: %d" % (crc, crc32(data)) del(data) print "Starting encoding test:" startime = time.time() bytes_out, crc_out = yenc.encode(file_in, file_out, size) secs = time.time() - startime print "Bytes out:", bytes_out, "output crc:", crc_out if crc != crc_out: print "crc error, 1Mb block encoding" sys.exit(1) elif bytes_out != size: print "size error, 1Mb block encoding" sys.exit(1) else: print "crc and size, 1Mb block encoding ok. %f secs." % secs file_in.close() file_out.close() os.unlink("sampledata") file_in = open("sampledata.out","r") file_out = open("sampledata","w") print "Starting decoding test:" startime = time.time() bytes_out, crc_out = yenc.decode(file_in, file_out, size) secs = time.time() - startime print "Bytes out:", bytes_out, "output crc:", crc_out if crc != crc_out: print "crc error, 1Mb block decoding" sys.exit(1) elif bytes_out != size: print "size error, 1Mb block decoding" sys.exit(1) else: print "crc and size, 1Mb block decoding ok. %f secs." % secs os.unlink("sampledata") os.unlink("sampledata.out")
def test_decode(): bites = [ b'*', b'+', b',', b'-', b'=n', b'/', b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b':', b';', b'<', b'=}', b'>', b'?', b'@', b'A', b'B', b'C', b'D', b'E', b'F', b'G', b'H', b'I', b'J', b'K', b'L', b'M', b'N', b'O', b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W', b'X', b'Y', b'Z', b'[', b'\\', b']', b'^', b'_', b'`', b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h', b'i', b'j', b'k', b'l', b'm', b'n', b'o', b'p', b'q', b'r', b's', b't', b'u', b'v', b'w', b'x', b'y', b'z', b'{', b'|', b'}', b'~', b'\x7f', b'\x80', b'\x81', b'\x82', b'\x83', b'\x84', b'\x85', b'\x86', b'\x87', b'\x88', b'\x89', b'\x8a', b'\x8b', b'\x8c', b'\x8d', b'\x8e', b'\x8f', b'\x90', b'\x91', b'\x92', b'\x93', b'\x94', b'\x95', b'\x96', b'\x97', b'\x98', b'\x99', b'\x9a', b'\x9b', b'\x9c', b'\x9d', b'\x9e', b'\x9f', b'\xa0', b'\xa1', b'\xa2', b'\xa3', b'\xa4', b'\xa5', b'\xa6', b'\xa7', b'\xa8', b'\xa9', b'\xaa', b'\xab', b'\xac', b'\xad', b'\xae', b'\xaf', b'\xb0', b'\xb1', b'\xb2', b'\xb3', b'\xb4', b'\xb5', b'\xb6', b'\xb7', b'\xb8', b'\xb9', b'\xba', b'\xbb', b'\xbc', b'\xbd', b'\xbe', b'\xbf', b'\xc0', b'\xc1', b'\xc2', b'\xc3', b'\xc4', b'\xc5', b'\xc6', b'\xc7', b'\xc8', b'\xc9', b'\xca', b'\xcb', b'\xcc', b'\xcd', b'\xce', b'\xcf', b'\xd0', b'\xd1', b'\xd2', b'\xd3', b'\xd4', b'\xd5', b'\xd6', b'\xd7', b'\xd8', b'\xd9', b'\xda', b'\xdb', b'\xdc', b'\xdd', b'\xde', b'\xdf', b'\xe0', b'\xe1', b'\xe2', b'\xe3', b'\xe4', b'\xe5', b'\xe6', b'\xe7', b'\xe8', b'\xe9', b'\xea', b'\xeb', b'\xec', b'\xed', b'\xee', b'\xef', b'\xf0', b'\xf1', b'\xf2', b'\xf3', b'\xf4', b'\xf5', b'\xf6', b'\xf7', b'\xf8', b'\xf9', b'\xfa', b'\xfb', b'\xfc', b'\xfd', b'\xfe', b'\xff', b'=@', b'\x01', b'\x02', b'\x03', b'\x04', b'\x05', b'\x06', b'\x07', b'\x08', b'=I', b'=J', b'\x0b', b'\x0c', b'=M', b'\x0e', b'\x0f', b'\x10', b'\x11', b'\x12', b'\x13', b'\x14', b'\x15', b'\x16', b'\x17', b'\x18', b'\x19', b'\x1a', b'\x1b', b'\x1c', b'\x1d', b'\x1e', b'\x1f', b'=`', b'!', b'"', b'#', b'$', b'%', b'&', b"'", b'(', b')' ] soln = [b.to_bytes(1, 'little') for b in range(256)] for i, b in enumerate(bites): size, crc32, data = decode(b) assert (data == soln[i])
def main(): head_crc = trail_crc = None if len(sys.argv) > 1: file_in = open(sys.argv[1],"rb") else: try: # Python 3 file_in = sys.stdin.detach() except AttributeError: # Python < 3 file_in = sys.stdin sys.stdin = None with file_in: while True: line = file_in.readline() if line.startswith(b"=ybegin "): try: name, size = NAME_RE.match(line).group(1), int(SIZE_RE.match(line).group(1)) m_obj = CRC32_RE.match(line) if m_obj: head_crc = m_obj.group(1) except re.error: sys.stderr.write("err-critical: malformed =ybegin header\n") sys.exit(1) break elif not line: sys.stderr.write("err-critical: no valid =ybegin header found\n") sys.exit(1) with open(name,"wb") as file_out: try: dec, dec_crc = yenc.decode(file_in, file_out, size) except yenc.Error as e: sys.stderr.write(str(e) + '\n') sys.exit(1) garbage = False for line in file_in.read().split(b"\r\n"): if line.startswith(b"=yend "): try: size = int( SIZE_RE.match(line).group(1) ) m_obj = CRC32_RE.match(line) if m_obj: trail_crc = m_obj.group(1) except re.error: sys.stderr.write("err: malformed =yend trailer\n") break elif not line: continue else: garbage = True else: sys.stderr.write("warning: couldn't find =yend trailer\n") if garbage: sys.stderr.write("warning: garbage before =yend trailer\n") if head_crc: tmp_crc = head_crc.decode("ascii").lower() elif trail_crc: tmp_crc = trail_crc.decode("ascii").lower() else: sys.exit(0) if tmp_crc != dec_crc: sys.stderr.write("err: header: %s dec: %s CRC32 mismatch\n" % (tmp_crc,dec_crc) ) sys.exit(1) else: sys.exit(0)
m_obj = CRC32_RE.match(line) if m_obj: head_crc = m_obj.group(1) except re.error, e: sys.stderr.write( "err-critical: malformed =ybegin header\n") return False break elif not line: sys.stderr.write( "err-critical: no valid =ybegin header found\n") return False file_out = open(outFilename, "w") try: dec, dec_crc = yenc.decode(file_in, file_out, size) except yenc.Error, e: sys.stderr.write(str(e) + '\n') return False head_crc = trail_crc = tmp_crc = "" garbage = 0 for line in file_in.read().split("\r\n"): if line.startswith("=yend "): try: size = int(SIZE_RE.match(line).group(1)) m_obj = CRC32_RE.match(line) if m_obj: trail_crc = m_obj.group(1) except re.error, e: sys.stderr.write("err: malformed =yend trailer\n")
try: name, size = NAME_RE.match(line).group(1), int(SIZE_RE.match(line).group(1)) m_obj = CRC32_RE.match(line) if m_obj: head_crc = m_obj.group(1) except re.error, e: sys.stderr.write("err-critical: malformed =ybegin header\n") return False break elif not line: sys.stderr.write("err-critical: no valid =ybegin header found\n") return False file_out = open(outFilename, "w") try: dec, dec_crc = yenc.decode(file_in, file_out, size) except yenc.Error, e: sys.stderr.write(str(e) + '\n') return False head_crc = trail_crc = tmp_crc = "" garbage = 0 for line in file_in.read().split("\r\n"): if line.startswith("=yend "): try: size = int( SIZE_RE.match(line).group(1) ) m_obj = CRC32_RE.match(line) if m_obj: trail_crc = m_obj.group(1) except re.error, e: sys.stderr.write("err: malformed =yend trailer\n")
b'=ybegin line=128 size=173 name=smallfile.rar\r\n', b'|\x8b\x9cKD1*\xf9\xba\x9d**7*******\xcb\xfb\x9eJ\xaa[*\x8b***\x8b***-MW\xe5\xb8\x8e\x92\x95s>Z;*\xde\xab**\x9e\x8f\x9d\x9eW[ZZ\x8c\xa3\x9e\x8f\x9dX\x93\x97\x91\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x94\r\n', b'4\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x944\x8b\x8c\x8d\x8e\x8f\x8e\x90\x91\x92\x93\x9444\xeeg\xa5*j1*\r\n', b'=yend size=173 crc32=8828b45c\r\n' ] print("\nINPUT:") print("First Line", data[0].rstrip()) print("Last Line", data[-1].rstrip()) bytes0 = bytearray() for d in data[1:-1]: bytes0.extend(d) t0_yenc = time.time() for i in range(10000000): _, decodedcrc32, decoded = yenc.decode(bytes0) dt_yenc = time.time() - t0_yenc print(decoded) # Find size in laste line of yencoded message: lastline = data[-1].decode( ) # For example: '=yend size=173 crc32=8828b45c\r\n' (and ... assuming it's in the last line) m = re.search('size=(.\d+?) ', lastline) if m: size = int(m.group(1)) print("size of decoded info will be", size) # Now do the yencode-decoding using sabyenc: t0_sabyenc = time.time() for i in range(10000000): decoded_data, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks(
from scapy.all import * import base64 from yenc import decode def get_ICMP_data(filename): capture = rdpcap(filename) ping_data = "" for packet in capture: if packet[ICMP].type == 8: # Echo request ping_data += str(packet[Raw]) return ping_data encrypted_data = "" encrypted_data += get_ICMP_data("f101.pcap") open('encrypted_data', "w").write(encrypted_data) decode('encrypted_data', 'decrypted.txt')