]
print "\nINPUT:"
print "First Line", data[0].rstrip()
print "Last Line", data[-1].rstrip()
# Find size in laste line of yencoded message:
lastline = data[
-1] # For example: '=yend size=173 crc32=8828b45c\r\n' (and ... assuming it's in the last line)
m = re.search('size=(.\d+?) ', lastline)
if m:
size = int(m.group(1))
print "size of decoded info will be", size
# Now do the yencode-decoding using sabyenc:
decoded_data, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks(
data, size)
print "\nOUTPUT:"
#print "decoded_data, first 20 bytes (Warning: probably binary stuff!):\n", decoded_data[:20]
print "decoded_data -> length:", len(decoded_data)
print "output_filename:", output_filename
print "crc:", crc
print "crc_yenc:", crc_yenc
print "crc_correct:", crc_correct
print "\nWriting to:", output_filename
file = open(output_filename, "w")
file.write(decoded_data)
file.close()
def decode(self, article, data, raw_data):
# Do we have SABYenc? Let it do all the work
if sabnzbd.decoder.SABYENC_ENABLED:
decoded_data, output_filename, crc, crc_expected, crc_correct = sabyenc.decode_usenet_chunks(
raw_data, article.bytes)
# Assume it is yenc
article.nzf.type = 'yenc'
# Only set the name if it was found and not obfuscated
self.verify_filename(article, decoded_data, output_filename)
# CRC check
if not crc_correct:
raise CrcError(crc_expected, crc, decoded_data)
return decoded_data
# Continue for _yenc or Python-yEnc
# Filter out empty ones
data = filter(None, data)
# No point in continuing if we don't have any data left
if data:
nzf = article.nzf
yenc, data = yCheck(data)
ybegin, ypart, yend = yenc
# Deal with non-yencoded posts
if not ybegin:
found = False
try:
for i in xrange(min(40, len(data))):
if data[i].startswith('begin '):
nzf.type = 'uu'
found = True
# Pause the job and show warning
if nzf.nzo.status != Status.PAUSED:
nzf.nzo.pause()
msg = T(
'UUencode detected, only yEnc encoding is supported [%s]'
) % nzf.nzo.final_name
logging.warning(msg)
break
except IndexError:
raise BadYenc()
if found:
decoded_data = ''
else:
raise BadYenc()
# Deal with yenc encoded posts
elif ybegin and yend:
if 'name' in ybegin:
output_filename = yenc_name_fixer(ybegin['name'])
else:
output_filename = None
logging.debug(
"Possible corrupt header detected => ybegin: %s",
ybegin)
nzf.type = 'yenc'
# Decode data
if HAVE_YENC:
decoded_data, crc = _yenc.decode_string(''.join(data))[:2]
partcrc = '%08X' % ((crc ^ -1) & 2**32L - 1)
else:
data = ''.join(data)
for i in (0, 9, 10, 13, 27, 32, 46, 61):
j = '=%c' % (i + 64)
data = data.replace(j, chr(i))
decoded_data = data.translate(YDEC_TRANS)
crc = binascii.crc32(decoded_data)
partcrc = '%08X' % (crc & 2**32L - 1)
if ypart:
crcname = 'pcrc32'
else:
crcname = 'crc32'
if crcname in yend:
_partcrc = yenc_name_fixer('0' * (8 - len(yend[crcname])) +
yend[crcname].upper())
else:
_partcrc = None
logging.debug("Corrupt header detected => yend: %s", yend)
if not _partcrc == partcrc:
raise CrcError(_partcrc, partcrc, decoded_data)
else:
raise BadYenc()
# Parse filename if there was data
if decoded_data:
# Only set the name if it was found and not obfuscated
self.verify_filename(article, decoded_data, output_filename)
return decoded_data
for fname in all_crc_fails:
# Open file
print '\n\n ======================================= \n\n'
print fname
data_p = open(fname, "r")
try:
data_chunks, data_size, lines = pickle.load(data_p)
except:
data_p.seek(0)
data_chunks, data_size = pickle.load(data_p)
data_p.close()
"""
First we check SABYenc
"""
output_buffer, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks(
data_chunks, data_size)
print '\n---- SABYenc output ----\n'
print 'Filename:', output_filename
print 'Size:', len(output_buffer)
print 'NZB size:', data_size
print 'CRC Calc:', crc
print 'CRC Yenc:', crc_yenc
print 'CRC Bool:', crc_correct
"""
Validate using _yenc
"""
data = []
new_lines = ''.join(data_chunks).split('\r\n')
for i in xrange(len(new_lines)):
if new_lines[i][:2] == '..':
new_lines[i] = new_lines[i][1:]
###################
# YENC C - NEW
###################
# Time it!
time1_new = time.clock()
timet_new = 0.0
for i in xrange(nr_runs):
time2 = time.clock()
timet_new += time.clock()-time2
# Different from homemade
decoded_data_new, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks(data_chunks, data_bytes)
print "---"
time1_new_disp = 1000*(time.clock()-time1_new)
timet_new_disp = 1000*timet_new
print "%15s took %4d ms" % ("yEnc C New", time1_new_disp)
print "%15s took %4d ms = %d%%" % ("Base Python", timet_new_disp, 100*timet_new_disp/time1_new_disp)
print "---"
###################
# YENC C - Old
###################
# Time it!
def sabyenc_wrapper(data_chunks, data_bytes):
""" CRC's are """
decoded_data, filename, crc_calc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks(
data_chunks, data_bytes)
return decoded_data, filename, crc_correct
def decode(article, data, raw_data):
# Do we have SABYenc? Let it do all the work
if sabnzbd.decoder.SABYENC_ENABLED:
decoded_data, output_filename, crc, crc_expected, crc_correct = sabyenc.decode_usenet_chunks(raw_data, article.bytes)
# Assume it is yenc
article.nzf.type = 'yenc'
# Only set the name if it was found
if output_filename:
article.nzf.filename = output_filename
# CRC check
if not crc_correct:
raise CrcError(crc_expected, crc, decoded_data)
return decoded_data
# Continue for _yenc or Python-yEnc
# Filter out empty ones
data = filter(None, data)
# No point in continuing if we don't have any data left
if data:
nzf = article.nzf
yenc, data = yCheck(data)
ybegin, ypart, yend = yenc
decoded_data = None
# Deal with non-yencoded posts
if not ybegin:
found = False
try:
for i in xrange(min(40, len(data))):
if data[i].startswith('begin '):
nzf.type = 'uu'
found = True
# Pause the job and show warning
if nzf.nzo.status != Status.PAUSED:
nzf.nzo.pause()
msg = T('UUencode detected, only yEnc encoding is supported [%s]') % nzf.nzo.final_name
logging.warning(msg)
break
except IndexError:
raise BadYenc()
if found:
decoded_data = ''
else:
raise BadYenc()
# Deal with yenc encoded posts
elif ybegin and yend:
if 'name' in ybegin:
nzf.filename = yenc_name_fixer(ybegin['name'])
else:
logging.debug("Possible corrupt header detected => ybegin: %s", ybegin)
nzf.type = 'yenc'
# Decode data
if HAVE_YENC:
decoded_data, crc = _yenc.decode_string(''.join(data))[:2]
partcrc = '%08X' % ((crc ^ -1) & 2 ** 32L - 1)
else:
data = ''.join(data)
for i in (0, 9, 10, 13, 27, 32, 46, 61):
j = '=%c' % (i + 64)
data = data.replace(j, chr(i))
decoded_data = data.translate(YDEC_TRANS)
crc = binascii.crc32(decoded_data)
partcrc = '%08X' % (crc & 2 ** 32L - 1)
if ypart:
crcname = 'pcrc32'
else:
crcname = 'crc32'
if crcname in yend:
_partcrc = yenc_name_fixer('0' * (8 - len(yend[crcname])) + yend[crcname].upper())
else:
_partcrc = None
logging.debug("Corrupt header detected => yend: %s", yend)
if not _partcrc == partcrc:
raise CrcError(_partcrc, partcrc, decoded_data)
else:
raise BadYenc()
return decoded_data
def test_list_none():
with pytest.raises(TypeError) as excinfo:
sabyenc.decode_usenet_chunks(None, 1)
assert 'Expected list' in str(excinfo.value)
def test_size_dict():
with pytest.raises(TypeError) as excinfo:
sabyenc.decode_usenet_chunks(['yenc1', 'yenc2'], {1: 'yenc'})
assert 'integer' in str(excinfo.value)
"size": int(re.search('size=(.\d+?) ', line).group(1))
}
if not headers['size']:
raise Exception("could not parse size=")
if has_offsets: # try to parse begin/end
try:
headers['begin'] = int(re.search('begin=(.\d+?) ', data[i+1]).group(1))
headers['end'] = int(re.search('end=(.\d+?)\r\n', data[i+1]).group(1))
except:
print "warning: begin/end could not be correctly parsed"
if not headers['begin']: # sometimes begin turns into None, reset to 0
headers['begin'] = 0
# yencode-decoding using sabyenc - input data includes the 1/2 header lines and the trailing '=yend'
decoded_data, output_filename, crc, crc_yenc, crc_correct = sabyenc.decode_usenet_chunks(
data[i:], headers['size'])
if not crc_correct: raise Exception("faulty checksum")
print "output_filename:", output_filename
print "size:", headers['size']
print "decoded_data length:", len(decoded_data)
if output_filename not in files:
files[output_filename] = []
files[output_filename].append({
"begin": headers['begin'],
"end": headers['end'],
"data": decoded_data
})
print "sorting chunks and writing files."
