def __init__(self, jsit, nthreads = 0):
log(DEBUG)
self._jsit = jsit
# Our session
self._session = requests.Session()
retries = Retry(total=2, backoff_factor=2)
self._session.mount('http://', HTTPAdapter(max_retries=retries))
self._session.mount('https://', HTTPAdapter(max_retries=retries))
# Currently running downloads
self._downloads = []
# Keep track of server failures and give them some time to recover
self._serverfails = {}
# Parallel? Create queues, start threads
self._nthreads = nthreads
if nthreads:
self._pieceQ = DPQueue(maxsize = maxQueueSize, prio = lambda e: e[0] == None or -e[0].piecePriority(e[1]))
self._writeQ = Queue.PriorityQueue(maxsize = maxQueueSize)
self._quitting = False
self._writeThread = threading.Thread(target=self.writePieceThread, name="PieceWriter")
self._writeThread.start()
self._pieceThreads = []
for t in xrange(0, nthreads):
t = threading.Thread(target=self.pieceFinishedThread, name="PieceDown-%d" % t)
##t.daemon = True
t.start()
self._pieceThreads.append(t)
class PieceDownloader(object):
def __init__(self, jsit, nthreads = 0):
log(DEBUG)
self._jsit = jsit
# Our session
self._session = requests.Session()
retries = Retry(total=2, backoff_factor=2)
self._session.mount('http://', HTTPAdapter(max_retries=retries))
self._session.mount('https://', HTTPAdapter(max_retries=retries))
# Currently running downloads
self._downloads = []
# Keep track of server failures and give them some time to recover
self._serverfails = {}
# Parallel? Create queues, start threads
self._nthreads = nthreads
if nthreads:
self._pieceQ = DPQueue(maxsize = maxQueueSize, prio = lambda e: e[0] == None or -e[0].piecePriority(e[1]))
self._writeQ = Queue.PriorityQueue(maxsize = maxQueueSize)
self._quitting = False
self._writeThread = threading.Thread(target=self.writePieceThread, name="PieceWriter")
self._writeThread.start()
self._pieceThreads = []
for t in xrange(0, nthreads):
t = threading.Thread(target=self.pieceFinishedThread, name="PieceDown-%d" % t)
##t.daemon = True
t.start()
self._pieceThreads.append(t)
def __repr__(self):
return "PieceDownloader(0x%x)"% id(self)
# Cleanup methods...
def __del__(self):
self.release()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.release()
def release(self):
if self._nthreads:
log(DEBUG, "Send suicide signals...")
self._quitting = True
self._writeQ.put((0, None, -1, ""))
for t in xrange(0, self._nthreads):
self._pieceQ.put((None, -1))
log(DEBUG, "Wait for threads to finish...")
self._writeThread.join()
log(DEBUG, "WriteThread done.")
for t in xrange(0, self._nthreads):
self._pieceThreads[t].join(5)
if self._pieceThreads[t].isAlive():
log(DEBUG, "PieceThread %d not responding!" % t)
else:
log(DEBUG, "PieceThread %d done." % t)
log(DEBUG, "All threads done!")
self._nthreads = 0
# Analyze stats
l = []
nc = 0
tc = 0
for k,v in apiStats.iteritems():
l.append([k, v[0], v[1]])
nc += v[0]
tc += v[1]
log(DEBUG, "API stats: called API %d times, taking %.02f secs" % (nc,tc))
log(DEBUG, "Most used:")
l.sort(key = lambda e: -e[1])
for i in xrange(0, min(len(l), 20)):
c = l[i]
log(DEBUG, " %s:\t%d calls" % (c[0], c[1]))
log(DEBUG, "Most time:")
l.sort(key = lambda e: -e[2])
for i in xrange(0, min(len(l), 20)):
c = l[i]
log(DEBUG, " %s:\t%.02f s in %d calls" % (c[0], c[2], c[1]))
log(DEBUG, "Most expensive:")
l.sort(key = lambda e: -e[2]/e[1])
for i in xrange(0, min(len(l), 20)):
c = l[i]
log(DEBUG, " %s:\t%.02f s/c in %d calls" % (c[0], c[2]/c[1], c[1]))
# (Parallel) Piece handling
def pieceFinished(self, tor, piece, abort_on_wait = False):
log(DEBUG)
if self._nthreads:
if abort_on_wait and self._pieceQ.full():
log(ERROR, "Put piece %s:%s: queue full and abort_on_wait set. Failing torrent to prevent deadlock!" % (tor, piece))
tor._nFailures = maxFailures
else:
log(DEBUG, "Put piece %s:%s (prio=%d)" % (tor, piece, tor.piecePriority(piece)))
self._pieceQ.put((tor, piece))
cont = None
else:
cont = tor.pieceFinished(piece)
return cont
def writePiece(self, tor, piece, cont):
log(DEBUG)
if self._nthreads:
prio = tor.piecePriority(piece)
log(DEBUG, "Put piece %s:%s (prio=%d)" % (tor, piece, prio))
self._writeQ.put((-prio, tor, piece, cont))
else:
tor.writePiece(piece, cont)
def pieceFinishedThread(self):
log(DEBUG)
try:
while not self._quitting:
piece = -2
while piece == -2:
try:
tor,piece = self._pieceQ.get(True, 300)
except Queue.Empty:
log(DEBUG3, "Heartbeat...")
log(DEBUG, "Got piece %s:%s" % (tor, piece))
if piece == -1:
log(DEBUG, "Got suicide signal, returning.")
return
tor.pieceFinished(piece)
self._pieceQ.task_done()
except Exception, e:
log(ERROR, "Caught %s" % e)
log(ERROR, traceback.format_exc())
log(DEBUG, "Ending (%s)" % self._quitting)