def decode(self, raw_segment_path, temp_dir, download_path):
#print 'decoding', raw_segment_path
try:
decoded_data, metadata = yenc_decode(raw_segment_path)
except IndexError:
# Empty/broken file; most likely a broken segment.
return
except ValueError:
# Something went wrong.
return
except IOError:
# No such file.
return
if not decoded_data or not metadata:
# Return for now; later do something about the file tracker count.
return
decoded_data = ''.join(decoded_data)
# Actual decoded data's checksum.
checksum = binascii.hexlify(struct.pack('!l', binascii.crc32(decoded_data)))
# Get key values here.
dest_filename = metadata.get('name')
segment_part_size = metadata.get('size')
total_parts = metadata.get('total')
segment_part = metadata.get('part', 1)
segment_end_size = metadata.get('end', 1)
if 'crc32' in metadata:
# Single-part files do not have pcrc32 keys.
crc_key = metadata.get('crc32')
else:
crc_key = metadata.get('pcrc32')
if len(crc_key) < 8:
crc_key = crc_key.rjust(8, '0')
if crc_key.lower() != checksum.lower():
print 'broken: expected: %s, got: %s' % (crc_key, checksum)
segment_filename = os.path.join(temp_dir, '%s.%s' % (dest_filename, segment_part))
with open(segment_filename, 'wb') as segment_fp:
segment_fp.write(decoded_data)
# Cleanup.
os.remove(raw_segment_path)
if dest_filename not in self.tracker:
self.tracker.setdefault(dest_filename, SegmentTracker())
if (total_parts and total_parts == segment_part) or (segment_part_size == segment_end_size):
# Found the last segment number.
self.tracker[dest_filename].expected = int(segment_part)
self.tracker[dest_filename].count += 1
if self.tracker[dest_filename].is_complete():
self.join_files(dest_filename, temp_dir, download_path)
def _get_nzb_file_size(self, nzb_file):
try:
if not self.aux_connection:
self.aux_connection = nntplib.NNTP(self.server, self.port, self.username, self.password)
self.aux_group = ""
# switch groups if necessary
if self.aux_group != nzb_file.groups[0]:
self.aux_group = nzb_file.groups[0]
response, count, first, last, name = connection.group(self.aux_group)
# download the segment
response, number, id, encoded_lines = connection.body('<' + nzb_file.segments[0].message_id + '>')
decode_result = yenc_decode(encoded_lines)
if not decode_result:
raise Exception("malformed yEnc file: " + encoded_lines)
decoded_segment, part_number, part_begin, part_end, file_size = decode_result
return file_size
except (nntplib.NNTPTemporaryError, nntplib.NNTPPermanentError), ex:
code = ex.response[:3]
# authorization rejected
if code == "452" or code == "502" or code == "480":
raise Exception("Sending newsgroup command", "NNTP authorization failed! Check username and password.")
# invalid article or article not found
elif code == "423" or code == "430":
raise Exception("Article " + nzb_file.segments[0].message_id + " not found.")
def _decode_loop(self):
try:
current_status = STATUS_DOWNLOADING
while 1:
try:
filename, nzb_segment, encoded_lines, duration = self.decode_queue.get(True, 5)
except Queue.Empty:
# if all download threads are dead, abort
all_threads_dead = True
for thread in self.threads:
if thread.isAlive():
all_threads_dead = False
if all_threads_dead:
return
# check for user cancellation
if self._update_progress("Waiting to decode a segment", current_status):
return
# continue to wait for a segment to decode
continue
downloaded_file = self.downloaded_files[filename]
try:
# decode the segment and capture part metadata
#print "Decoding " + filename + " : " + encoded_lines[0]
#print encoded_lines[1]
decode_result = yenc_decode(encoded_lines)
if not decode_result:
raise Exception("malformed yEnc file: " + encoded_lines[0])
decoded_segment, part_number, part_begin, part_end, file_size = decode_result
except Exception, ex:
current_status = STATUS_BROKEN
if self._update_progress("Part " + `nzb_segment.number` + " is missing or corrupt.", current_status, filename,
self.finished_files, len(self.sorted_filenames),
downloaded_file.finished_bytes, downloaded_file.total_bytes):
return
continue
if part_number != nzb_segment.number:
print "Warning: decoded part number " + `part_number` + " does not match requested segment number " + `nzb_segment.number`
# first part decoded for a file sets the total file size;
# each part is tested to make sure the file size is equal
if downloaded_file.total_bytes == 0:
downloaded_file.total_bytes = file_size
elif downloaded_file.total_bytes != file_size:
print "Warning: decoded part number " + `part_number` + " reports file size different from first part."
current_file = filename
if self._update_progress(downloaded_file.name + ":" + `nzb_segment.number`, current_status, current_file,
self.finished_files, len(self.sorted_filenames),
downloaded_file.finished_bytes, downloaded_file.total_bytes):
return
def _get_nzb_file_size(self, nzb_file):
try:
if not self.aux_connection:
self.aux_connection = nntplib.NNTP(self.server, self.port,
self.username,
self.password)
self.aux_group = ""
# switch groups if necessary
if self.aux_group != nzb_file.groups[0]:
self.aux_group = nzb_file.groups[0]
response, count, first, last, name = connection.group(
self.aux_group)
# download the segment
response, number, id, encoded_lines = connection.body(
'<' + nzb_file.segments[0].message_id + '>')
decode_result = yenc_decode(encoded_lines)
if not decode_result:
raise Exception("malformed yEnc file: " + encoded_lines)
decoded_segment, part_number, part_begin, part_end, file_size = decode_result
return file_size
except (nntplib.NNTPTemporaryError, nntplib.NNTPPermanentError), ex:
code = ex.response[:3]
# authorization rejected
if code == "452" or code == "502" or code == "480":
raise Exception(
"Sending newsgroup command",
"NNTP authorization failed! Check username and password.")
# invalid article or article not found
elif code == "423" or code == "430":
raise Exception("Article " + nzb_file.segments[0].message_id +
" not found.")
def _decode_loop(self):
try:
current_status = STATUS_DOWNLOADING
while 1:
try:
filename, nzb_segment, encoded_lines, duration = self.decode_queue.get(
True, 5)
except Queue.Empty:
# if all download threads are dead, abort
all_threads_dead = True
for thread in self.threads:
if thread.isAlive():
all_threads_dead = False
if all_threads_dead:
return
# check for user cancellation
if self._update_progress("Waiting to decode a segment",
current_status):
return
# continue to wait for a segment to decode
continue
downloaded_file = self.downloaded_files[filename]
try:
# decode the segment and capture part metadata
#print "Decoding " + filename + " : " + encoded_lines[0]
#print encoded_lines[1]
decode_result = yenc_decode(encoded_lines)
if not decode_result:
raise Exception("malformed yEnc file: " +
encoded_lines[0])
decoded_segment, part_number, part_begin, part_end, file_size = decode_result
except Exception, ex:
current_status = STATUS_BROKEN
if self._update_progress(
"Part " + ` nzb_segment.number ` +
" is missing or corrupt.", current_status,
filename, self.finished_files,
len(self.sorted_filenames),
downloaded_file.finished_bytes,
downloaded_file.total_bytes):
return
continue
if part_number != nzb_segment.number:
print "Warning: decoded part number " + ` part_number ` + " does not match requested segment number " + ` nzb_segment.number `
# first part decoded for a file sets the total file size;
# each part is tested to make sure the file size is equal
if downloaded_file.total_bytes == 0:
downloaded_file.total_bytes = file_size
elif downloaded_file.total_bytes != file_size:
print "Warning: decoded part number " + ` part_number ` + " reports file size different from first part."
current_file = filename
if self._update_progress(
downloaded_file.name + ":" + ` nzb_segment.number `,
current_status, current_file, self.finished_files,
len(self.sorted_filenames),
downloaded_file.finished_bytes,
downloaded_file.total_bytes):
return
def test_yenc_decode(self):
yenc_decode()