def _round_robin(self):
self.schedule(self._round_robin, 5)
if self.super_seed:
cons = range(len(self.connections))
to_close = []
count = self.config["min_uploads"] - self.last_preferred
if count > 0: # optimization
shuffle(cons)
for c in cons:
i = self.picker.next_have(self.connections[c], count > 0)
if i is None:
continue
if i < 0:
to_close.append(self.connections[c])
continue
self.connections[c].send_have(i)
count -= 1
for c in to_close:
c.close()
if self.last_round_robin + self.config["round_robin_period"] < clock():
self.last_round_robin = clock()
for i in xrange(1, len(self.connections)):
c = self.connections[i]
u = c.get_upload()
if u.is_choked() and u.is_interested():
self.connections = self.connections[i:] + self.connections[:i]
break
self._rechoke()
def is_snubbed(self):
if (self.interested and not self.choked and
clock() - self.last2 > self.downloader.snub_time):
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke() # treat it just like a choke
return clock() - self.last > self.downloader.snub_time
def _round_robin(self):
self.schedule(self._round_robin, 5)
if self.super_seed:
cons = range(len(self.connections))
to_close = []
count = self.config['min_uploads'] - self.last_preferred
if count > 0: # optimization
random.shuffle(cons)
for c in cons:
i = self.picker.next_have(self.connections[c], count > 0)
if i is None:
continue
if i < 0:
to_close.append(self.connections[c])
continue
self.connections[c].send_have(i)
count -= 1
for c in to_close:
c.close()
if self.last_round_robin + self.round_robin_period < clock():
self.last_round_robin = clock()
for i in xrange(1, len(self.connections)):
c = self.connections[i]
u = c.get_upload()
if u.is_choked() and u.is_interested():
self.connections = self.connections[i:] + \
self.connections[:i]
break
self._rechoke()
def next_have(self, connection, looser_upload):
if self.seed_time is None:
self.seed_time = clock()
return None
if clock() < self.seed_time+10: # wait 10 seconds after seeing the first peers
return None # to give time to grab have lists
if not connection.upload.super_seeding:
return None
if connection in self.seed_connections:
if looser_upload:
num = 1 # send a new have even if it hasn't spread that piece elsewhere
else:
num = 2
if self.seed_got_haves[self.seed_connections[connection]] < num:
return None
if not connection.upload.was_ever_interested: # it never downloaded it?
connection.upload.skipped_count += 1
if connection.upload.skipped_count >= 3: # probably another stealthed seed
return -1 # signal to close it
for tier in self.interests:
for piece in tier:
if not connection.download.have[piece]:
seedint = self.level_in_interests[piece]
self.level_in_interests[piece] += 1 # tweak it up one, so you don't duplicate effort
if seedint == len(self.interests) - 1:
self.interests.append([])
self._shift_over(piece,
self.interests[seedint], self.interests[seedint + 1])
self.seed_got_haves[piece] = 0 # reset this
self.seed_connections[connection] = piece
connection.upload.seed_have_list.append(piece)
return piece
return -1 # something screwy; terminate connection
def display(self, dpflag = Event(), fractionDone = None, timeEst = None,
downRate = None, upRate = None, activity = None,
statistics = None, **kws):
if self.last_update_time + 0.1 > clock() and fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
if timeEst is not None:
self.timeEst = hours(timeEst)
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
if statistics is not None:
if (statistics.shareRating < 0) or (statistics.shareRating > 100):
self.shareRating = 'oo (%.1f MB up / %.1f MB down)' % (float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
else:
self.shareRating = '%.3f (%.1f MB up / %.1f MB down)' % (statistics.shareRating, float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
if not self.done:
self.seedStatus = '%d seen now, plus %.3f distributed copies' % (statistics.numSeeds,0.001*int(1000*statistics.numCopies))
else:
self.seedStatus = '%d seen recently, plus %.3f distributed copies' % (statistics.numOldSeeds,0.001*int(1000*statistics.numCopies))
self.peerStatus = '%d seen now, %.1f%% done at %.1f kB/s' % (statistics.numPeers,statistics.percentDone,float(statistics.torrentRate) / (1 << 10))
for err in self.errors:
print 'ERROR:\n' + err + '\n'
dpflag.set()
def display(self, dpflag = Event(), fractionDone = None, timeEst = None,
downRate = None, upRate = None, activity = None,
statistics = None, **kws):
if self.last_update_time + 0.1 > clock() and fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
if timeEst is not None:
self.timeEst = hours(timeEst)
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
if statistics is not None:
if (statistics.shareRating < 0) or (statistics.shareRating > 100):
self.shareRating = 'oo (%.1f MB up / %.1f MB down)' % (float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
else:
self.shareRating = '%.3f (%.1f MB up / %.1f MB down)' % (statistics.shareRating, float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
if not self.done:
self.seedStatus = '%d seen now, plus %.3f distributed copies' % (statistics.numSeeds,0.001*int(1000*statistics.numCopies))
else:
self.seedStatus = '%d seen recently, plus %.3f distributed copies' % (statistics.numOldSeeds,0.001*int(1000*statistics.numCopies))
self.peerStatus = '%d seen now, %.1f%% done at %.1f kB/s' % (statistics.numPeers,statistics.percentDone,float(statistics.torrentRate) / (1 << 10))
for err in self.errors:
print 'ERROR:\n' + err + '\n'
dpflag.set()
def is_snubbed(self):
if ( self.interested and not self.choked
and clock() - self.last2 > self.downloader.snub_time ):
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke() # treat it just like a choke
return clock() - self.last > self.downloader.snub_time
def display(self,
dpflag=threading.Event(),
fractionDone=None,
timeEst=None,
downRate=None,
upRate=None,
activity=None,
statistics=None,
**kws):
if self.last_update_time + 0.1 > clock() and \
fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
if timeEst is not None:
self.timeEst = formatIntText(timeEst) or 'complete!'
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
if statistics is not None:
if statistics.shareRating < 0 or statistics.shareRating > 100:
self.shareRating = 'oo ({:.1f} MB up / {:.1f} MB down)' \
''.format(float(statistics.upTotal) / (1 << 20),
float(statistics.downTotal) / (1 << 20))
else:
self.shareRating = '{:.3f} ({:.1f} MB up / {:.1f} MB down)' \
''.format(statistics.shareRating,
float(statistics.upTotal) / (1 << 20),
float(statistics.downTotal) / (1 << 20))
if not self.done:
self.seedStatus = '{:d} seen now, plus {:.3f} distributed ' \
'copies'.format(statistics.numSeeds,
statistics.numCopies)
else:
self.seedStatus = '{:d} seen recently, plus {:.3f} ' \
'distributed copies'.format(statistics.numOldSeeds,
statistics.numCopies)
self.peerStatus = '{:d} seen now, {:.1%} done at {:.1f} kB/s' \
''.format(statistics.numPeers,
statistics.percentDone / 100,
float(statistics.torrentRate) / (1 << 10))
print '\n\n\n\n'
for err in self.errors:
print 'ERROR:\n' + err + '\n'
print 'saving: ', self.file
print 'percent done: ', self.percentDone
print 'time left: ', self.timeEst
print 'download to: ', self.downloadTo
print 'download rate: ', self.downRate
print 'upload rate: ', self.upRate
print 'share rating: ', self.shareRating
print 'seed status: ', self.seedStatus
print 'peer status: ', self.peerStatus
sys.stdout.flush()
dpflag.set()
def _get_map(self):
if self.last_got_map + EXPIRE_CACHE < clock():
try:
dispatcher = win32com.client.Dispatch("HNetCfg.NATUPnP")
self.map = dispatcher.StaticPortMappingCollection
self.last_got_map = clock()
except Exception:
self.map = None
return self.map
def _get_map(self):
if self.last_got_map + EXPIRE_CACHE < clock():
try:
dispatcher = win32com.client.Dispatch("HNetCfg.NATUPnP")
self.map = dispatcher.StaticPortMappingCollection
self.last_got_map = clock()
except Exception:
self.map = None
return self.map
def listen_forever(self, handler):
self.sockethandler.set_handler(handler)
try:
while not self.doneflag.isSet():
try:
self.pop_external()
self._kill_tasks()
if self.funcs:
period = self.funcs[0][0] + 0.001 - clock()
else:
period = 2 ** 30
if period < 0:
period = 0
events = self.sockethandler.do_poll(period)
if self.doneflag.isSet():
return
while self.funcs and self.funcs[0][0] <= clock():
_, func, id = self.funcs.pop(0)
if id in self.tasks_to_kill:
pass
try:
# print func.func_name
func()
except (SystemError, MemoryError) as e:
self.failfunc(str(e))
return
except KeyboardInterrupt:
# self.exception(True)
return
except Exception:
if self.noisy:
self.exception()
self.sockethandler.close_dead()
self.sockethandler.handle_events(events)
if self.doneflag.isSet():
return
self.sockethandler.close_dead()
except (SystemError, MemoryError) as e:
self.failfunc(str(e))
return
except select.error:
if self.doneflag.isSet():
return
except KeyboardInterrupt:
# self.exception(True)
return
except Exception:
self.exception()
if self.exccount > 10:
return
finally:
# self.sockethandler.shutdown()
self.finished.set()
def display(self, dpflag=threading.Event(), fractionDone=None,
timeEst=None, downRate=None, upRate=None, activity=None,
statistics=None, **kws):
if self.last_update_time + 0.1 > clock() and \
fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
if timeEst is not None:
self.timeEst = hours(timeEst)
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
if statistics is not None:
if (statistics.shareRating < 0) or (statistics.shareRating > 100):
self.shareRating = 'oo ({:.1f} MB up / {:.1f} MB down)' \
''.format(float(statistics.upTotal) / (1 << 20),
float(statistics.downTotal) / (1 << 20))
else:
self.shareRating = '{:.3f} ({:.1f} MB up / {:.1f} MB down)' \
''.format(statistics.shareRating,
float(statistics.upTotal) / (1 << 20),
float(statistics.downTotal) / (1 << 20))
if not self.done:
self.seedStatus = '{:d} seen now, plus {:.3f} distributed ' \
'copies'.format(statistics.numSeeds,
statistics.numCopies)
else:
self.seedStatus = '{:d} seen recently, plus {:.3f} ' \
'distributed copies'.format(statistics.numOldSeeds,
statistics.numCopies)
self.peerStatus = '{:d} seen now, {:.1%} done at {:.1f} kB/s' \
''.format(statistics.numPeers,
statistics.percentDone / 100,
float(statistics.torrentRate) / (1 << 10))
print '\n\n\n\n'
for err in self.errors:
print 'ERROR:\n' + err + '\n'
print 'saving: ', self.file
print 'percent done: ', self.percentDone
print 'time left: ', self.timeEst
print 'download to: ', self.downloadTo
print 'download rate: ', self.downRate
print 'upload rate: ', self.upRate
print 'share rating: ', self.shareRating
print 'seed status: ', self.seedStatus
print 'peer status: ', self.peerStatus
sys.stdout.flush()
dpflag.set()
def display(self, dpflag = Event(), fractionDone = None, timeEst = None,
downRate = None, upRate = None, activity = None,
statistics = None, **kws):
if self.last_update_time + 0.1 > clock() and fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
if timeEst is not None:
self.timeEst = hours(timeEst)
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
if statistics is not None:
if (statistics.shareRating < 0) or (statistics.shareRating > 100):
self.shareRating = 'oo (%.1f MB up / %.1f MB down)' % (float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
else:
self.shareRating = '%.3f (%.1f KB up / %.1f KB down)' % (statistics.shareRating, float(statistics.upTotal) / (1<<10), float(statistics.downTotal) / (1<<10))
if not self.done:
self.seedStatus = '%d seen now, plus %.3f distributed copies' % (statistics.numSeeds,0.001*int(1000*statistics.numCopies))
else:
self.seedStatus = '%d seen recently, plus %.3f distributed copies' % (statistics.numOldSeeds,0.001*int(1000*statistics.numCopies))
self.peerStatus = '%d seen now, %.1f%% done at %.1f kB/s' % (statistics.numPeers,statistics.percentDone,float(statistics.torrentRate) / (1 << 10))
self.timeTot = hours(self.last_update_time - self.first_update_time)
print '\n\n\n\n'
self.numOfVODPeers = statistics.numOfVODPeers
self.numOfVODSeeds = statistics.numOfVODSeeds
for err in self.errors:
print 'ERROR:\n' + err + '\n'
if self.verbose:
print '==================================================================================\r'
print 'total time: ', self.timeTot,'\r'
print 'share rating: ', self.shareRating,'\r'
print 'saving: ', self.file,'\r'
print 'percent done: ', self.percentDone,'\r'
print 'time left: ', self.timeEst,'\r'
print 'download to: ', self.downloadTo,'\r'
print 'download rate: ', self.downRate,'\r'
print 'upload rate: ', self.upRate,'\r'
print 'seed status: ', self.seedStatus,'\r'
print 'peer status: ', self.peerStatus,'\r'
#### P2PVODEX start ####
print 'VOD Peers: ', self.numOfVODPeers,'\r'
print 'VOD Seeds: ', self.numOfVODSeeds,'\r'
#### P2PVODEX end ####
stdout.flush()
dpflag.set()
def listen_forever(self, handler):
self.sockethandler.set_handler(handler)
try:
while not self.doneflag.isSet():
try:
self.pop_external()
self._kill_tasks()
if self.funcs:
period = max(0, self.funcs[0][0] + 0.001 - clock())
else:
period = 2 ** 30
events = self.sockethandler.do_poll(period)
if self.doneflag.isSet():
return
while self.funcs and self.funcs[0][0] <= clock():
_, func, tid = self.funcs.pop(0)
if tid in self.tasks_to_kill:
pass
try:
# print func.func_name
func()
except (SystemError, MemoryError) as e:
self.failfunc(str(e))
return
except KeyboardInterrupt:
# self.exception(True)
return
except Exception:
if self.noisy:
self.exception()
self.sockethandler.close_dead()
self.sockethandler.handle_events(events)
if self.doneflag.isSet():
return
self.sockethandler.close_dead()
except (SystemError, MemoryError) as e:
self.failfunc(str(e))
return
except select.error:
if self.doneflag.isSet():
return
except KeyboardInterrupt:
# self.exception(True)
return
except Exception:
self.exception()
if self.exccount > 10:
return
finally:
# self.sockethandler.shutdown()
self.finished.set()
def _get_services(self):
if not self.services or \
self.last_got_services + EXPIRE_CACHE < clock():
self.services = []
try:
f = win32com.client.Dispatch("UPnP.UPnPDeviceFinder")
for t in ("urn:schemas-upnp-org:service:WANIPConnection:1",
"urn:schemas-upnp-org:service:WANPPPConnection:1"):
for conn in f.FindByType(t, 0):
self.services.extend(conn.Services)
except Exception:
pass
self.last_got_services = clock()
return self.services
def _get_services(self):
if not self.services or \
self.last_got_services + EXPIRE_CACHE < clock():
self.services = []
try:
f = win32com.client.Dispatch("UPnP.UPnPDeviceFinder")
for t in ("urn:schemas-upnp-org:service:WANIPConnection:1",
"urn:schemas-upnp-org:service:WANPPPConnection:1"):
for conn in f.FindByType(t, 0):
self.services.extend(conn.Services)
except Exception:
pass
self.last_got_services = clock()
return self.services
def display(self, dpflag = Event(), fractionDone = None, timeEst = None,
downRate = None, upRate = None, activity = None,
statistics = None, **kws):
if self.last_update_time + 0.1 > clock() and fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
self.callbackCount +=1
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
if timeEst is not None:
self.timeEst = hours(timeEst)
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
self.averageDownSpeed += (float(downRate) - self.averageDownSpeed)/self.callbackCount
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
self.averageUpSpeed += (float(upRate)-self.averageUpSpeed)/self.callbackCount
if statistics is not None:
if (statistics.shareRating < 0) or (statistics.shareRating > 100):
self.shareRating = 'oo (%.1f MB up / %.1f MB down)' % (float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
else:
self.shareRating = '%.3f (%.1f MB up / %.1f MB down)' % (statistics.shareRating, float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
if not self.done:
self.seedStatus = '%d seen now, plus %.3f distributed copies' % (statistics.numSeeds,0.001*int(1000*statistics.numCopies))
else:
self.seedStatus = '%d seen recently, plus %.3f distributed copies' % (statistics.numOldSeeds,0.001*int(1000*statistics.numCopies))
self.peerStatus = '%d seen now, %.1f%% done at %.1f kB/s' % (statistics.numPeers,statistics.percentDone,float(statistics.torrentRate) / (1 << 10))
self.downloads = ' '.join(statistics.downloads)
self.logger.debug( '\n\n\n\n')
for err in self.errors:
self.logger.error( 'ERROR:\n' + err + '\n')
self.logger.debug( 'saving: '+ self.file)
self.logger.debug( 'percent done: '+ self.percentDone)
self.logger.debug( 'time left: '+ self.timeEst)
self.logger.debug( 'download to: '+ self.downloadTo)
self.logger.debug( 'download rate: '+ self.downRate)
self.logger.debug( 'upload rate: '+ self.upRate)
self.logger.debug( 'share rating: '+ self.shareRating)
self.logger.debug( 'seed status: '+ self.seedStatus)
self.logger.debug( 'peer status: '+ self.peerStatus)
self.logger.debug( 'all downloads: '+ self.downloads)
self.logger.debug( 'average downRate: {0:.1f} kB/s'.format(self.averageDownSpeed/(1<<10)))
self.logger.debug( 'average upRate: {0:.1f} kB/s'.format(self.averageUpSpeed/(1<<10)))
self.logger.debug( 'call count: {0}'.format(self.callbackCount))
dpflag.set()
def display(self, dpflag = Event(), fractionDone = None, timeEst = None,
downRate = None, upRate = None, activity = None,
statistics = None, **kws):
if self.last_update_time + 0.1 > clock() and fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
print "fraction done is ", fractionDone
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
percenty = float(int(fractionDone * 1000)) / 10
if(percenty >= 99.7): # there's some bug in there that some of them sit at 99.8 forever. Avoid it thus!
print "\nEXITING WAY EARLY WITH SUCCESS!!\n\n\n"
print "\nEXITING WAY EARLY WITH SUCCESS!!\n\n\n"
sys.stdout.flush()
os._exit(1)
if timeEst is not None:
self.timeEst = hours(timeEst)
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
if statistics is not None:
if (statistics.shareRating < 0) or (statistics.shareRating > 100):
self.shareRating = 'oo (%.1f MB up / %.1f MB down)' % (float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
else:
self.shareRating = '%.3f (%.1f MB up / %.1f MB down)' % (statistics.shareRating, float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
if not self.done:
self.seedStatus = '%d seen now, plus %.3f distributed copies' % (statistics.numSeeds,0.001*int(1000*statistics.numCopies))
else:
self.seedStatus = '%d seen recently, plus %.3f distributed copies' % (statistics.numOldSeeds,0.001*int(1000*statistics.numCopies))
self.peerStatus = '%d seen now, %.1f%% done at %.1f kB/s' % (statistics.numPeers,statistics.percentDone,float(statistics.torrentRate) / (1 << 10))
print '\n\n\n\n'
for err in self.errors:
print 'ERROR:\n' + err + '\n'
print 'saving: ', self.file
print 'percent done: ', self.percentDone
print 'time left: ', self.timeEst
print 'download to: ', self.downloadTo
print 'download rate: ', self.downRate
print 'upload rate: ', self.upRate
print 'share rating: ', self.shareRating
print 'seed status: ', self.seedStatus
print 'peer status: ', self.peerStatus
stdout.flush()
dpflag.set()
def insertNode(self, n, contacted = True):
"""
Insert a node in our local table, pinging oldest contact in bucket, if necessary
If all you have is a host/port, then use addContact, which calls this method after
receiving the PONG from the remote node. The reason for the seperation is we can't insert
a node into the table without it's peer-ID. That means of course the node passed into this
method needs to be a properly formed Node object with a valid ID.
"""
old = self.table.insertNode(n, contacted = contacted)
if old and (clock() - old.lastSeen) > MIN_PING_INTERVAL and old.id != self.node.id:
# the bucket is full, check to see if old node is still around and if so, replace it
## these are the callbacks used when we ping the oldest node in a bucket
def _staleNodeHandler(oldnode=old, newnode = n):
""" called if the pinged node never responds """
self.table.replaceStaleNode(old, newnode)
def _notStaleNodeHandler(dict, old=old):
""" called when we get a pong from the old node """
dict = dict['rsp']
if dict['id'] == old.id:
self.table.justSeenNode(old.id)
try:
df = old.ping(self.node.id)
except KrpcGenericError:
_staleNodeHandler()
else:
df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler)
def pop_external(self):
"""Prepare tasks queued with add_task to be run in the listen_forever
loop."""
to_add, self.externally_added = self.externally_added, []
for (func, delay, tid) in to_add:
if tid not in self.tasks_to_kill:
bisect.insort(self.funcs, (clock() + delay, func, tid))
def onPause(self):
self.whenpaused = clock()
if not self.downloader:
return
self.downloader.pause(True)
self.encoder.pause(True)
self.choker.pause(True)
def update_rate(self, amount):
self.total += amount
t = clock()
self.rate = (self.rate * (self.last - self.ratesince) + amount) / (t - self.ratesince + 0.0001)
self.last = t
if self.ratesince < t - self.max_rate_period:
self.ratesince = t - self.max_rate_period
def _connect(self):
"""Send a connect request, and record the connection ID
Return True/False on success/failure
"""
# No cost to using a new UDP socket
self.sock = socket.socket(type=socket.SOCK_DGRAM)
self.sock.settimeout(15)
self.sock.connect(self.sig[1:])
self.tx_id = random.randint(0, 0xffffffff).to_bytes(4, 'big')
self.sock.send(self.INITIAL + self.CONNECT + self.tx_id)
response = self.sock.recv(1024)
if len(response) < 8:
return False
if response[:4] in (self.ERROR, self.ERROR_ALT) and \
response[4:8] == self.tx_id:
raise IOError(response[8:-1].decode())
elif response[:8] == self.CONNECT + self.tx_id:
self.cnxn_id = response[8:16]
self.response_time = clock()
return True
else:
return False
def request(self, msg):
"""Make an announce request to a UDP tracker
Requests will have connection ID, announce code, and transaction ID
prepended. Response headers are validated and stripped, and the
remaining bytestring returned.
None will be returned on a timed-out or malformed response. If an
error message is received, an exception will be raised with its
contents.
"""
with self.lock:
if clock() >= self.response_time + 60:
if not self._connect():
return None
packet = b''.join((self.cnxn_id, self.ANNOUNCE, self.tx_id, msg))
self.sock.send(packet)
response = self.sock.recv(2048)
if len(response) < 8:
return None
if response[:4] in (self.ERROR, self.ERROR_ALT) and \
response[4:8] == self.tx_id:
if response.endswith(b'\x00'):
response = response[:-1]
raise IOError(response[8:].decode())
elif response[:8] == self.ANNOUNCE + self.tx_id:
return response[8:]
else:
return None
def get_time_left(self, left):
t = clock()
if not self.got_anything:
return None
if t - self.last > 15:
self.update(0)
try:
remaining = left / self.rate
if not self.lastten and remaining <= 10:
self.lastten = True
if self.lastten:
return remaining
delta = max(remaining / 20, 2)
if self.remaining is None:
self.remaining = remaining
elif abs(self.remaining - remaining) > delta:
self.remaining = remaining
else:
self.remaining -= t - self.last_checked
except ZeroDivisionError:
self.remaining = None
if self.remaining is not None and self.remaining < 0.1:
self.remaining = 0.1
self.last_checked = t
return self.remaining
def get_time_left(self, left):
t = clock()
if not self.got_anything:
return None
if t - self.last > 15:
self.update(0)
try:
remaining = left / self.rate
if not self.lastten and remaining <= 10:
self.lastten = True
if self.lastten:
return remaining
delta = max(remaining / 20, 2)
if self.remaining is None:
self.remaining = remaining
elif abs(self.remaining - remaining) > delta:
self.remaining = remaining
else:
self.remaining -= t - self.last_checked
except ZeroDivisionError:
self.remaining = None
if self.remaining is not None and self.remaining < 0.1:
self.remaining = 0.1
self.last_checked = t
return self.remaining
def pop_external(self):
"""Prepare tasks queued with add_task to be run in the listen_forever
loop."""
to_add, self.externally_added = self.externally_added, []
for (func, delay, tid) in to_add:
if tid not in self.tasks_to_kill:
bisect.insort(self.funcs, (clock() + delay, func, tid))
def try_send(self, check_time=False):
t = clock()
self.bytes_sent -= (t - self.lasttime) * self.upload_rate
self.lasttime = t
if check_time:
self.bytes_sent = max(self.bytes_sent, 0)
cur = self.last.next_upload
while self.bytes_sent <= 0:
bytes = cur.send_partial(self.unitsize)
self.bytes_sent += bytes
self.measure.update_rate(bytes)
if bytes == 0 or cur.backlogged():
if self.last is cur:
self.last = None
cur.next_upload = None
break
else:
self.last.next_upload = cur.next_upload
cur.next_upload = None
cur = self.last.next_upload
else:
self.last = cur
cur = cur.next_upload
else:
self.sched(self.try_send, self.bytes_sent / self.upload_rate)
def __init__(self, storage, picker, backlog, max_rate_period,
numpieces, chunksize, measurefunc, snub_time,
kickbans_ok, kickfunc, banfunc):
self.storage = storage
self.picker = picker
self.backlog = backlog
self.max_rate_period = max_rate_period
self.measurefunc = measurefunc
self.totalmeasure = Measure(max_rate_period*storage.piece_length/storage.request_size)
self.numpieces = numpieces
self.chunksize = chunksize
self.snub_time = snub_time
self.kickfunc = kickfunc
self.banfunc = banfunc
self.disconnectedseeds = {}
self.downloads = []
self.perip = {}
self.gotbaddata = {}
self.kicked = {}
self.banned = {}
self.kickbans_ok = kickbans_ok
self.kickbans_halted = False
self.super_seeding = False
self.endgamemode = False
self.endgame_queued_pieces = []
self.all_requests = []
self.discarded = 0L
self.download_rate = 0
self.bytes_requested = 0
self.last_time = clock()
self.queued_out = {}
self.requeueing = False
self.paused = False
def onPause(self):
self.whenpaused = clock()
if not self.downloader:
return
self.downloader.pause(True)
self.encoder.pause(True)
self.choker.pause(True)
def __init__(self, storage, picker, backlog, max_rate_period,
numpieces, chunksize, measurefunc, snub_time,
kickbans_ok, kickfunc, banfunc):
self.storage = storage
self.picker = picker
self.backlog = backlog
self.max_rate_period = max_rate_period
self.measurefunc = measurefunc
self.totalmeasure = Measure(max_rate_period * storage.piece_length /
storage.request_size)
self.numpieces = numpieces
self.chunksize = chunksize
self.snub_time = snub_time
self.kickfunc = kickfunc
self.banfunc = banfunc
self.disconnectedseeds = {}
self.downloads = []
self.perip = {}
self.gotbaddata = set()
self.kicked = {}
self.banned = {}
self.kickbans_ok = kickbans_ok
self.kickbans_halted = False
self.super_seeding = False
self.endgamemode = False
self.endgame_queued_pieces = []
self.all_requests = []
self.discarded = 0
# self.download_rate = 25000 # 25K/s test rate
self.download_rate = 0
self.bytes_requested = 0
self.last_time = clock()
self.queued_out = set()
self.requeueing = False
self.paused = False
def request(self, msg):
"""Make an announce request to a UDP tracker
Requests will have connection ID, announce code, and transaction ID
prepended. Response headers are validated and stripped, and the
remaining bytestring returned.
None will be returned on a timed-out or malformed response. If an
error message is received, an exception will be raised with its
contents.
"""
with self.lock:
if clock() >= self.response_time + 60:
if not self._connect():
return None
packet = b"".join((self.cnxn_id, self.ANNOUNCE, self.tx_id, msg))
self.sock.send(packet)
response = self.sock.recv(2048)
if len(response) < 8:
return None
if response[:4] in (self.ERROR, self.ERROR_ALT) and response[4:8] == self.tx_id:
if response.endswith(b"\x00"):
response = response[:-1]
raise IOError(response[8:].decode())
elif response[:8] == self.ANNOUNCE + self.tx_id:
return response[8:]
else:
return None
def update_rate(self, amount):
self.total += amount
t = clock()
self.rate = (self.rate * (self.last - self.ratesince) +
amount) / (t - self.ratesince + 0.0001)
self.last = t
if self.ratesince < t - self.max_rate_period:
self.ratesince = t - self.max_rate_period
def display(self, dpflag = Event(), fractionDone = None, timeEst = None,
downRate = None, upRate = None, activity = None,
statistics = None, **kws):
if __debug__: traceMsg('display - begin')
if self.last_update_time + 0.1 > clock() and fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
if timeEst is not None:
self.timeEst = hours(timeEst)
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
if statistics is not None:
if (statistics.shareRating < 0) or (statistics.shareRating > 100):
self.shareRating = 'oo (%.1f MB up / %.1f MB down)' % (float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
self.downTotal = statistics.downTotal
self.upTotal = statistics.upTotal
else:
self.shareRating = '%.3f (%.1f MB up / %.1f MB down)' % (statistics.shareRating, float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20))
self.downTotal = statistics.downTotal
self.upTotal = statistics.upTotal
if not self.done:
self.seedStatus = '%d seen now, plus %.3f distributed copies' % (statistics.numSeeds,0.001*int(1000*statistics.numCopies))
self.seeds = (str(statistics.numSeeds))
else:
self.seedStatus = '%d seen recently, plus %.3f distributed copies' % (statistics.numOldSeeds,0.001*int(1000*statistics.numCopies))
self.seeds = (str(statistics.numOldSeeds))
self.peers = '%d' % (statistics.numPeers)
self.distcopy = '%.3f' % (0.001*int(1000*statistics.numCopies))
self.peerStatus = '%d seen now, %.1f%% done at %.1f kB/s' % (statistics.numPeers,statistics.percentDone,float(statistics.torrentRate) / (1 << 10))
dpflag.set()
if __debug__: traceMsg('display - prior to self.write')
if self.stoppedAt == '':
self.writeStatus()
if __debug__: traceMsg('display - end')
def old_style_init(self):
while self.initialize_tasks:
msg, done, init, next = self.initialize_tasks.pop(0)
if init():
self.statusfunc(activity=msg, fractionDone=done)
t = clock() + STATS_INTERVAL
x = 0
while x is not None:
if t < clock():
t = clock() + STATS_INTERVAL
self.statusfunc(fractionDone=x)
self.unpauseflag.wait()
if self.flag.is_set():
return False
x = next()
self.statusfunc(fractionDone=0)
return True
def old_style_init(self):
while self.initialize_tasks:
msg, done, init, next = self.initialize_tasks.pop(0)
if init():
self.statusfunc(activity=msg, fractionDone=done)
t = clock() + STATS_INTERVAL
x = 0
while x is not None:
if t < clock():
t = clock() + STATS_INTERVAL
self.statusfunc(fractionDone=x)
self.unpauseflag.wait()
if self.flag.isSet():
return False
x = next()
self.statusfunc(fractionDone=0)
return True
def num_disconnected_seeds(self):
# first expire old ones
expired = []
for id, t in self.disconnectedseeds.items():
if clock() - t > EXPIRE_TIME: #Expire old seeds after so long
expired.append(id)
for id in expired:
# self.picker.seed_disappeared()
del self.disconnectedseeds[id]
return len(self.disconnectedseeds)
def onUnpause(self):
if not self.downloader:
return
self.downloader.pause(False)
self.encoder.pause(False)
self.choker.pause(False)
# rerequest automatically if paused for >60 seconds
if self.rerequest and self.whenpaused and \
clock() - self.whenpaused > 60:
self.rerequest.announce(3)
def log(self, ip, ident, username, header, responsecode, length, referrer,
useragent):
year, month, day, hour, minute, second = time.localtime()[:6]
print '%s %s %s [%02d/%3s/%04d:%02d:%02d:%02d] "%s" %i %i "%s" "%s"' \
'' % (ip, ident, username, day, months[month], year, hour, minute,
second, header, responsecode, length, referrer, useragent)
t = clock()
if t - self.lastflush > self.minflush:
self.lastflush = t
sys.stdout.flush()
def onUnpause(self):
if not self.downloader:
return
self.downloader.pause(False)
self.encoder.pause(False)
self.choker.pause(False)
# rerequest automatically if paused for >60 seconds
if self.rerequest and self.whenpaused and \
clock() - self.whenpaused > 60:
self.rerequest.announce(3)
def num_disconnected_seeds(self):
# first expire old ones
expired = []
for id,t in self.disconnectedseeds.items():
if clock() - t > EXPIRE_TIME: #Expire old seeds after so long
expired.append(id)
for id in expired:
# self.picker.seed_disappeared()
del self.disconnectedseeds[id]
return len(self.disconnectedseeds)
def log(self, ip, ident, username, header, responsecode, length, referrer,
useragent):
year, month, day, hour, minute, second = time.localtime()[:6]
print '%s %s %s [%02d/%3s/%04d:%02d:%02d:%02d] "%s" %i %i "%s" "%s"' \
'' % (ip, ident, username, day, months[month], year, hour, minute,
second, header, responsecode, length, referrer, useragent)
t = clock()
if t - self.lastflush > self.minflush:
self.lastflush = t
sys.stdout.flush()
def __init__(self, config, schedule, picker, done=lambda: False):
self.config = config
self.schedule = schedule
self.picker = picker
self.connections = []
self.last_preferred = 0
self.last_round_robin = clock()
self.done = done
self.super_seed = False
schedule(self._round_robin, 5)
def __init__(self, config, schedule, picker, done=lambda: False):
self.config = config
self.round_robin_period = config['round_robin_period']
self.schedule = schedule
self.picker = picker
self.connections = []
self.last_preferred = 0
self.last_round_robin = clock()
self.done = done
self.super_seed = False
schedule(self._round_robin, 5)
def cleanTokens(self):
"""
Remove old tokens
"""
for id in self.tokens.keys():
try:
if clock() - self.tokens[id][1] > TOKEN_UPDATE:
del self.tokens[id]
except KeyError:
pass
self.factory.rawserver.add_task(self.cleanTokens, TOKEN_UPDATE)
def refreshTable(self, force = False, callback=lambda a: None):
"""
Refresh the table
force=True will refresh table regardless of last bucket access time
"""
if DEBUG:
print("Debug: DHT - refreshTable")
for bucket in self.table.buckets:
if force or (clock() - bucket.lastAccessed >= BUCKET_STALENESS):
id = newIDInRange(bucket.min, bucket.max)
self.findNode(id, callback)
def send_unchoke(self):
if self.send_choke_queued:
self.send_choke_queued = False
if DEBUG1:
print (self.ccount,'CHOKE SUPPRESSED')
else:
self._send_message(UNCHOKE)
if ( self.partial_message or self.just_unchoked is None
or not self.upload.interested or self.download.active_requests ):
self.just_unchoked = 0
else:
self.just_unchoked = clock()
def send_unchoke(self):
if self.send_choke_queued:
self.send_choke_queued = False
if DEBUG:
print 'CHOKE SUPPRESSED'
else:
self._send_message(UNCHOKE)
if (self.partial_message or self.just_unchoked is None
or not self.upload.interested or self.download.active_requests):
self.just_unchoked = 0
else:
self.just_unchoked = clock()
def ping(self, delay):
if DEBUG:
print(delay)
if not self.autoadjust:
return
self.pings.append(delay > PING_BOUNDARY)
if len(self.pings) < PING_SAMPLES + PING_DISCARDS:
return
if DEBUG:
print('cycle')
pings = sum(self.pings[PING_DISCARDS:])
del self.pings[:]
if pings >= PING_THRESHHOLD: # assume flooded
if self.upload_rate == MAX_RATE:
self.upload_rate = self.measure.get_rate() * ADJUST_DOWN
else:
self.upload_rate = min(self.upload_rate,
self.measure.get_rate() * 1.1)
self.upload_rate = max(int(self.upload_rate * ADJUST_DOWN), 2)
self.slots = int(math.sqrt(self.upload_rate * SLOTS_FACTOR))
self.slotsfunc(self.slots)
if DEBUG:
print('adjust down to ', self.upload_rate)
self.lasttime = clock()
self.bytes_sent = 0
self.autoadjustup = UP_DELAY_FIRST
else: # not flooded
if self.upload_rate == MAX_RATE:
return
self.autoadjustup -= 1
if self.autoadjustup:
return
self.upload_rate = int(self.upload_rate * ADJUST_UP)
self.slots = int(math.sqrt(self.upload_rate * SLOTS_FACTOR))
self.slotsfunc(self.slots)
if DEBUG:
print('adjust up to ', self.upload_rate)
self.lasttime = clock()
self.bytes_sent = 0
self.autoadjustup = UP_DELAY_NEXT
def _CyclicalTasks(self):
self._getDownUpConnections()
self._updateTrayAndStatusBar()
self.ratemanager.RunTasks()
try:
# Run postponed deleting events
while utility.window.postponedevents:
ev = utility.window.postponedevents.pop(0)
#print "POSTPONED EVENT : ", ev[0]
# Handle multiple arguments
funcname = ev.pop(0)
funcname(*ev)
utility.window.list.Enable()
except wx.PyDeadObjectError:
pass
# Run Directory Scanner
freq = utility.config.Read('scandirfreq', "int")
if utility.config.Read('scandiractive', "boolean") and (self.lastDirScan == 0 or clock() - self.lastDirScan > freq):
self.invokeLater(self._ScanDir)
self.lastDirScan = clock()
# Scrape Queued Torrents
if utility.config.Read('scrapequeued', "boolean") and (self.lastScrape == 0 or clock() - self.lastScrape > 300):
self.invokeLater(self._ScrapeQueuedTorrents)
self.lastScrape = clock()
# Try invoking the scheduler
# (just in case we need to start more stuff:
# should return almost immediately otherwise)
self.invokeLater(self._Scheduler)
# Start Timer
##########################################
self.timers['frequent'] = Timer(2, self._CyclicalTasks)
self.timers['frequent'].start()
def get_ip(self):
if self.last_got_ip + EXPIRE_CACHE < clock():
local_ips = AddrList()
local_ips.set_intranet_addresses()
try:
for info in socket.getaddrinfo(socket.gethostname(), 0,
socket.AF_INET):
# exception if socket library isn't recent
self.local_ip = info[4][0]
if self.local_ip in local_ips:
self.last_got_ip = clock()
if DEBUG:
print 'Local IP found: ' + self.local_ip
break
else:
raise ValueError('couldn\'t find intranet IP')
except (ValueError, socket.error):
self.local_ip = None
if DEBUG:
print 'Error finding local IP'
print_exc()
return self.local_ip
def got_piece(self, index, begin, piece):
length = len(piece)
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, piece,
self.guard):
self.downloader.piece_flunked(index)
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
assert not d.active_requests
d.fix_download_endgame()
else:
try:
d.active_requests.remove((index, begin,
length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
else:
assert not d.active_requests
self._request_more()
self.downloader.check_complete(index)
return self.downloader.storage.do_I_have(index)
def __init__(self, config, schedule, picker, done=lambda: False):
self.config = config
self.round_robin_period = config['round_robin_period']
self.schedule = schedule
self.picker = picker
self.connections = []
self.last_preferred = 0
self.last_round_robin = clock()
self.done = done
self.super_seed = False
self.paused = False
schedule(self._round_robin, 5)
#### P2PVODEX start ####
self.passedEndOfMovieFlag = False
def update(self, amount):
t = clock()
exp = int(t) - int(self.last)
self.time *= (FACTOR**exp)
self.got *= (FACTOR**exp)
self.got += amount
if t - self.last < 20:
self.time += t - self.last
self.last = t
try:
self.rate = self.got / self.time
except ZeroDivisionError:
pass
def __init__(self, socket_handler, sock, handler, ip=None):
self.socket_handler = socket_handler
self.socket = sock
self.handler = handler
self.buffer = []
self.last_hit = clock()
self.fileno = sock.fileno()
self.connected = False
self.skipped = 0
try:
self.ip = self.socket.getpeername()[0]
except socket.error:
if ip is None:
self.ip = 'unknown'
else:
self.ip = ip
def queue_limit(self):
if not self.download_rate:
return 10e10 # that's a big queue!
t = clock()
self.bytes_requested -= (t - self.last_time) * self.download_rate
self.last_time = t
if not self.requeueing and self.queued_out and self.bytes_requested < 0:
self.requeueing = True
q = self.queued_out.keys()
shuffle(q)
self.queued_out = {}
for d in q:
d._request_more()
self.requeueing = False
if -self.bytes_requested > 5 * self.download_rate:
self.bytes_requested = -5 * self.download_rate
return max(int(-self.bytes_requested / self.chunksize), 0)
def set_upload_rate(self, rate):
# rate = -1 # test automatic
if rate < 0:
if self.autoadjust:
return
self.autoadjust = True
self.autoadjustup = 0
self.pings = []
rate = MAX_RATE
self.slots = SLOTS_STARTING
self.slotsfunc(self.slots)
else:
self.autoadjust = False
if not rate:
rate = MAX_RATE
self.upload_rate = rate * 1000
self.lasttime = clock()
self.bytes_sent = 0
