def run(self):
self.server = nntp.NNTP(**self.credentials)
try:
while True:
if self.stop:
self.server.quit()
return
# Try to grab a segment from queue
f, segment, bytes = self.segments.get(False)
# Check for the article on the server
try:
self.server.stat(segment)
# print "Found: %s" % segment
except nntplib.NNTPTemporaryError, e:
# Error code 430 is "No such article"
error = nntp.get_error_code(e)
if error == "430":
# Found missing segment
self.missing.put((f, segment, bytes))
self.missing.task_done()
# print "Missing: %s" % segment
else:
# Some other error, put the segment back in the
# queue to be checked again
print "Error: %s" % e
self.segments.put(segment)
except Exception, e:
print "Unknown error: %s" % e
return
def run(self):
self.server = nntp.NNTP(**self.credentials)
try:
while True:
if self.stop:
self.server.quit()
return
# Try to grab a segment from queue
f, segment, bytes = self.segments.get(False)
# Check for the article on the server
try:
self.server.stat(segment)
#print "Found: %s" % segment
except nntplib.NNTPTemporaryError, e:
# Error code 430 is "No such article"
error = nntp.get_error_code(e)
if error == '430':
# Found missing segment
self.missing.put((f, segment, bytes))
self.missing.task_done()
#print "Missing: %s" % segment
else:
# Some other error, put the segment back in the
# queue to be checked again
print "Error: %s" % e
self.segments.put(segment)
except Exception, e:
print "Unknown error: %s" % e
return
def check_segment(self, connection, segment, segments, missing):
"""
Check to see if this server has the requested segment.
Arguments:
connection - NNTP connection
segment - Tuple of format: (file name, message id, bytes)
segments - Global segments queue
missing - Global missing segments queue
If the NNTP connection has be closed (due to idle) will raise an
``ConnectionClosedException`` exception.
"""
if self.has_missing_segment(segment):
# We already know we don't have this segment, so send it to the
# next server
self.log(
log.info,
"Segment '%s' already marked as missing, passing off to next server"
% segment[1])
self.try_next_server(segment, missing)
return
f, msgid, bytes = segment
# Check for the article on the server
try:
# Found the segment
connection.stat(msgid)
if self.conf.get("backup"):
self.log(log.info,
"Found segment '%s' on backup server" % segment[1])
except nntplib.NNTPTemporaryError as e:
# Error code 430 is "No such article"
error = nntp.get_error_code(e)
if error == '430':
# Found missing segment. Mark it as missing from this server
# and pass it to the highest priority server, where it'll then
# trickle its way down through all the servers
self.add_missing_segment(segment)
self._servers[0].segments.put(segment)
return
elif error == "400":
# Code 400 means the connection is closed, usually do to a timeout
raise ConnectionClosedException(
"Connection closed - idle timeout?")
else:
# Some other error, put the segment back in the
# queue to be checked again
log.error("Error: %s" % e)
segments.put(segment)
return
except Exception as e:
log.error("Unknown error: %s" % e)
return
def check_segment(self, connection, segment, segments, missing):
"""
Check to see if this server has the requested segment.
Arguments:
connection - NNTP connection
segment - Tuple of format: (file name, message id, bytes)
segments - Global segments queue
missing - Global missing segments queue
If the NNTP connection has be closed (due to idle) will raise an
``ConnectionClosedException`` exception.
"""
if self.has_missing_segment(segment):
# We already know we don't have this segment, so send it to the
# next server
self.log(log.info, "Segment '%s' already marked as missing, passing off to next server" % segment[1])
self.try_next_server(segment, missing)
return
f, msgid, bytes = segment
# Check for the article on the server
try:
# Found the segment
connection.stat(msgid)
if self.conf.get("backup"):
self.log(log.info, "Found segment '%s' on backup server" % segment[1])
except nntplib.NNTPTemporaryError, e:
# Error code 430 is "No such article"
error = nntp.get_error_code(e)
if error == '430':
# Found missing segment. Mark it as missing from this server
# and pass it to the highest priority server, where it'll then
# trickle its way down through all the servers
self.add_missing_segment(segment)
self._servers[0].segments.put(segment)
return
elif error == "400":
# Code 400 means the connection is closed, usually do to a timeout
raise ConnectionClosedException("Connection closed - idle timeout?")
else:
# Some other error, put the segment back in the
# queue to be checked again
log.error("Error: %s" % e)
segments.put(segment)
return
def stat_all(self, msg):
lastgrp = ''
#~ for info in infodata['detail']:
#~ if(info[2] == 0 or info[2] == 1 or info[2] == 4):
for m in msg['detail']:
if(m[2] == 0 or m[2] == 1 or m[2] == 4):
#~ simple semaphor
if(m[5] != -3):
continue
with lock:
global completion_counter
completion_counter += 1
m[5] = -1
repeat = MAXREPEAT
while (repeat):
try:
if(lastgrp != m[4][0]):
lastgrp = m[4][0]
self.tn.group(m[4][0])
#~ print 'change grp'
self.tn.stat("<"+m[3]+">")
m[5] = 1
repeat = 0
except Exception as e:
error = nntp.get_error_code(e)
if error == '430':
#~ print "Missing article " + m[3] + " : %s" % e
print "*",
m[5] = 0
repeat = 0
else:
#~ print "Error ("+ str(repeat) +" more attempts, trying again: " + m[3] + " %s" % e
print "e",
repeat -= 1
def stat_all(self, msg):
lastgrp = ''
#~ for info in infodata['detail']:
#~ if(info[2] == 0 or info[2] == 1 or info[2] == 4):
for m in msg['detail']:
if (m[2] == 0 or m[2] == 1 or m[2] == 4):
#~ simple semaphor
if (m[5] != -3):
continue
with lock:
global completion_counter
completion_counter += 1
m[5] = -1
repeat = MAXREPEAT
while (repeat):
try:
if (lastgrp != m[4][0]):
lastgrp = m[4][0]
self.tn.group(m[4][0])
#~ print 'change grp'
self.tn.stat("<" + m[3] + ">")
m[5] = 1
repeat = 0
except Exception as e:
error = nntp.get_error_code(e)
if error == '430':
#~ print "Missing article " + m[3] + " : %s" % e
print "*",
m[5] = 0
repeat = 0
else:
#~ print "Error ("+ str(repeat) +" more attempts, trying again: " + m[3] + " %s" % e
print "e",
repeat -= 1