] print "\nINPUT:" print "First Line", data[0].rstrip() print "Last Line", data[-1].rstrip() # Find size in laste line of yencoded message: lastline = data[ -1] # For example: '=yend size=173 crc32=8828b45c\r\n' (and ... assuming it's in the last line) m = re.search('size=(.\d+?) ', lastline) if m: size = int(m.group(1)) print "size of decoded info will be", size # Now do the yencode-decoding using sabyenc: decoded_data, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks( data, size) print "\nOUTPUT:" #print "decoded_data, first 20 bytes (Warning: probably binary stuff!):\n", decoded_data[:20] print "decoded_data -> length:", len(decoded_data) print "output_filename:", output_filename print "crc:", crc print "crc_yenc:", crc_yenc print "crc_correct:", crc_correct print "\nWriting to:", output_filename file = open(output_filename, "w") file.write(decoded_data) file.close()
def decode(self, article, data, raw_data): # Do we have SABYenc? Let it do all the work if sabnzbd.decoder.SABYENC_ENABLED: decoded_data, output_filename, crc, crc_expected, crc_correct = sabyenc.decode_usenet_chunks( raw_data, article.bytes) # Assume it is yenc article.nzf.type = 'yenc' # Only set the name if it was found and not obfuscated self.verify_filename(article, decoded_data, output_filename) # CRC check if not crc_correct: raise CrcError(crc_expected, crc, decoded_data) return decoded_data # Continue for _yenc or Python-yEnc # Filter out empty ones data = filter(None, data) # No point in continuing if we don't have any data left if data: nzf = article.nzf yenc, data = yCheck(data) ybegin, ypart, yend = yenc # Deal with non-yencoded posts if not ybegin: found = False try: for i in xrange(min(40, len(data))): if data[i].startswith('begin '): nzf.type = 'uu' found = True # Pause the job and show warning if nzf.nzo.status != Status.PAUSED: nzf.nzo.pause() msg = T( 'UUencode detected, only yEnc encoding is supported [%s]' ) % nzf.nzo.final_name logging.warning(msg) break except IndexError: raise BadYenc() if found: decoded_data = '' else: raise BadYenc() # Deal with yenc encoded posts elif ybegin and yend: if 'name' in ybegin: output_filename = yenc_name_fixer(ybegin['name']) else: output_filename = None logging.debug( "Possible corrupt header detected => ybegin: %s", ybegin) nzf.type = 'yenc' # Decode data if HAVE_YENC: decoded_data, crc = _yenc.decode_string(''.join(data))[:2] partcrc = '%08X' % ((crc ^ -1) & 2**32L - 1) else: data = ''.join(data) for i in (0, 9, 10, 13, 27, 32, 46, 61): j = '=%c' % (i + 64) data = data.replace(j, chr(i)) decoded_data = data.translate(YDEC_TRANS) crc = binascii.crc32(decoded_data) partcrc = '%08X' % (crc & 2**32L - 1) if ypart: crcname = 'pcrc32' else: crcname = 'crc32' if crcname in yend: _partcrc = yenc_name_fixer('0' * (8 - len(yend[crcname])) + yend[crcname].upper()) else: _partcrc = None logging.debug("Corrupt header detected => yend: %s", yend) if not _partcrc == partcrc: raise CrcError(_partcrc, partcrc, decoded_data) else: raise BadYenc() # Parse filename if there was data if decoded_data: # Only set the name if it was found and not obfuscated self.verify_filename(article, decoded_data, output_filename) return decoded_data
for fname in all_crc_fails: # Open file print '\n\n ======================================= \n\n' print fname data_p = open(fname, "r") try: data_chunks, data_size, lines = pickle.load(data_p) except: data_p.seek(0) data_chunks, data_size = pickle.load(data_p) data_p.close() """ First we check SABYenc """ output_buffer, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks( data_chunks, data_size) print '\n---- SABYenc output ----\n' print 'Filename:', output_filename print 'Size:', len(output_buffer) print 'NZB size:', data_size print 'CRC Calc:', crc print 'CRC Yenc:', crc_yenc print 'CRC Bool:', crc_correct """ Validate using _yenc """ data = [] new_lines = ''.join(data_chunks).split('\r\n') for i in xrange(len(new_lines)): if new_lines[i][:2] == '..': new_lines[i] = new_lines[i][1:]
################### # YENC C - NEW ################### # Time it! time1_new = time.clock() timet_new = 0.0 for i in xrange(nr_runs): time2 = time.clock() timet_new += time.clock()-time2 # Different from homemade decoded_data_new, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks(data_chunks, data_bytes) print "---" time1_new_disp = 1000*(time.clock()-time1_new) timet_new_disp = 1000*timet_new print "%15s took %4d ms" % ("yEnc C New", time1_new_disp) print "%15s took %4d ms = %d%%" % ("Base Python", timet_new_disp, 100*timet_new_disp/time1_new_disp) print "---" ################### # YENC C - Old ################### # Time it!
def sabyenc_wrapper(data_chunks, data_bytes): """ CRC's are """ decoded_data, filename, crc_calc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks( data_chunks, data_bytes) return decoded_data, filename, crc_correct
def decode(article, data, raw_data): # Do we have SABYenc? Let it do all the work if sabnzbd.decoder.SABYENC_ENABLED: decoded_data, output_filename, crc, crc_expected, crc_correct = sabyenc.decode_usenet_chunks(raw_data, article.bytes) # Assume it is yenc article.nzf.type = 'yenc' # Only set the name if it was found if output_filename: article.nzf.filename = output_filename # CRC check if not crc_correct: raise CrcError(crc_expected, crc, decoded_data) return decoded_data # Continue for _yenc or Python-yEnc # Filter out empty ones data = filter(None, data) # No point in continuing if we don't have any data left if data: nzf = article.nzf yenc, data = yCheck(data) ybegin, ypart, yend = yenc decoded_data = None # Deal with non-yencoded posts if not ybegin: found = False try: for i in xrange(min(40, len(data))): if data[i].startswith('begin '): nzf.type = 'uu' found = True # Pause the job and show warning if nzf.nzo.status != Status.PAUSED: nzf.nzo.pause() msg = T('UUencode detected, only yEnc encoding is supported [%s]') % nzf.nzo.final_name logging.warning(msg) break except IndexError: raise BadYenc() if found: decoded_data = '' else: raise BadYenc() # Deal with yenc encoded posts elif ybegin and yend: if 'name' in ybegin: nzf.filename = yenc_name_fixer(ybegin['name']) else: logging.debug("Possible corrupt header detected => ybegin: %s", ybegin) nzf.type = 'yenc' # Decode data if HAVE_YENC: decoded_data, crc = _yenc.decode_string(''.join(data))[:2] partcrc = '%08X' % ((crc ^ -1) & 2 ** 32L - 1) else: data = ''.join(data) for i in (0, 9, 10, 13, 27, 32, 46, 61): j = '=%c' % (i + 64) data = data.replace(j, chr(i)) decoded_data = data.translate(YDEC_TRANS) crc = binascii.crc32(decoded_data) partcrc = '%08X' % (crc & 2 ** 32L - 1) if ypart: crcname = 'pcrc32' else: crcname = 'crc32' if crcname in yend: _partcrc = yenc_name_fixer('0' * (8 - len(yend[crcname])) + yend[crcname].upper()) else: _partcrc = None logging.debug("Corrupt header detected => yend: %s", yend) if not _partcrc == partcrc: raise CrcError(_partcrc, partcrc, decoded_data) else: raise BadYenc() return decoded_data
def test_list_none(): with pytest.raises(TypeError) as excinfo: sabyenc.decode_usenet_chunks(None, 1) assert 'Expected list' in str(excinfo.value)
def test_size_dict(): with pytest.raises(TypeError) as excinfo: sabyenc.decode_usenet_chunks(['yenc1', 'yenc2'], {1: 'yenc'}) assert 'integer' in str(excinfo.value)
"size": int(re.search('size=(.\d+?) ', line).group(1)) } if not headers['size']: raise Exception("could not parse size=") if has_offsets: # try to parse begin/end try: headers['begin'] = int(re.search('begin=(.\d+?) ', data[i+1]).group(1)) headers['end'] = int(re.search('end=(.\d+?)\r\n', data[i+1]).group(1)) except: print "warning: begin/end could not be correctly parsed" if not headers['begin']: # sometimes begin turns into None, reset to 0 headers['begin'] = 0 # yencode-decoding using sabyenc - input data includes the 1/2 header lines and the trailing '=yend' decoded_data, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks( data[i:], headers['size']) if not crc_correct: raise Exception("faulty checksum") print "output_filename:", output_filename print "size:", headers['size'] print "decoded_data length:", len(decoded_data) if output_filename not in files: files[output_filename] = [] files[output_filename].append({ "begin": headers['begin'], "end": headers['end'], "data": decoded_data }) print "sorting chunks and writing files."