def _round_robin(self):
self.schedule(self._round_robin, 5)
if self.super_seed:
cons = range(len(self.connections))
to_close = []
count = self.config['min_uploads'] - self.last_preferred
if count > 0: # optimization
random.shuffle(cons)
for c in cons:
i = self.picker.next_have(self.connections[c], count > 0)
if i is None:
continue
if i < 0:
to_close.append(self.connections[c])
continue
self.connections[c].send_have(i)
count -= 1
for c in to_close:
c.close()
if self.last_round_robin + self.round_robin_period < clock():
self.last_round_robin = clock()
for i in xrange(1, len(self.connections)):
c = self.connections[i]
u = c.get_upload()
if u.is_choked() and u.is_interested():
self.connections = self.connections[i:] + \
self.connections[:i]
break
self._rechoke()
def is_snubbed(self):
if (self.interested and not self.choked
and clock() - self.last2 > self.downloader.snub_time):
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke() # treat it just like a choke
return clock() - self.last > self.downloader.snub_time
def display(self,
dpflag=threading.Event(),
fractionDone=None,
timeEst=None,
downRate=None,
upRate=None,
activity=None,
statistics=None,
**kws):
if self.last_update_time + 0.1 > clock() and \
fractionDone not in (0.0, 1.0) and activity is not None:
return
self.last_update_time = clock()
if fractionDone is not None:
self.percentDone = str(float(int(fractionDone * 1000)) / 10)
if timeEst is not None:
self.timeEst = formatIntText(timeEst) or 'complete!'
if activity is not None and not self.done:
self.timeEst = activity
if downRate is not None:
self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10))
if upRate is not None:
self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10))
if statistics is not None:
self.shareRating = '{} ({:.1f} MB up / {:.1f} MB down)'.format(
'{:.3f}'.format(statistics.shareRating)
if 0 <= statistics.shareRating <= 100 else 'oo',
float(statistics.upTotal) / (1 << 20),
float(statistics.downTotal) / (1 << 20))
if not self.done:
self.seedStatus = '{:d} seen now, plus {:.3f} distributed ' \
'copies'.format(statistics.numSeeds,
statistics.numCopies)
else:
self.seedStatus = '{:d} seen recently, plus {:.3f} ' \
'distributed copies'.format(statistics.numOldSeeds,
statistics.numCopies)
self.peerStatus = '{:d} seen now, {:.1%} done at {:.1f} kB/s' \
''.format(statistics.numPeers,
statistics.percentDone / 100,
float(statistics.torrentRate) / (1 << 10))
print '\n\n\n\n'
for err in self.errors:
print 'ERROR:\n' + err + '\n'
print 'saving: ', self.file
print 'percent done: ', self.percentDone
print 'time left: ', self.timeEst
print 'download to: ', self.downloadTo
print 'download rate: ', self.downRate
print 'upload rate: ', self.upRate
print 'share rating: ', self.shareRating
print 'seed status: ', self.seedStatus
print 'peer status: ', self.peerStatus
if self.timeEst == "Download Succeeded!":
exit()
sys.stdout.flush()
dpflag.set()
def _get_map(self):
if self.last_got_map + EXPIRE_CACHE < clock():
try:
dispatcher = win32com.client.Dispatch("HNetCfg.NATUPnP")
self.map = dispatcher.StaticPortMappingCollection
self.last_got_map = clock()
except Exception:
self.map = None
return self.map
def listen_forever(self, handler):
self.sockethandler.set_handler(handler)
try:
while not self.doneflag.isSet():
try:
self.pop_external()
self._kill_tasks()
if self.funcs:
period = max(0, self.funcs[0][0] + 0.001 - clock())
else:
period = 2**30
events = self.sockethandler.do_poll(period)
if self.doneflag.isSet():
return
while self.funcs and self.funcs[0][0] <= clock():
_, func, tid = self.funcs.pop(0)
if tid in self.tasks_to_kill:
pass
try:
# print func.func_name
func()
except (SystemError, MemoryError) as e:
self.failfunc(str(e))
return
except KeyboardInterrupt:
# self.exception(True)
return
except Exception:
if self.noisy:
self.exception()
self.sockethandler.close_dead()
self.sockethandler.handle_events(events)
if self.doneflag.isSet():
return
self.sockethandler.close_dead()
except (SystemError, MemoryError) as e:
self.failfunc(str(e))
return
except select.error:
if self.doneflag.isSet():
return
except KeyboardInterrupt:
# self.exception(True)
return
except Exception:
self.exception()
if self.exccount > 10:
return
finally:
# self.sockethandler.shutdown()
self.finished.set()
def _get_services(self):
if not self.services or \
self.last_got_services + EXPIRE_CACHE < clock():
self.services = []
try:
f = win32com.client.Dispatch("UPnP.UPnPDeviceFinder")
for t in ("urn:schemas-upnp-org:service:WANIPConnection:1",
"urn:schemas-upnp-org:service:WANPPPConnection:1"):
for conn in f.FindByType(t, 0):
self.services.extend(conn.Services)
except Exception:
pass
self.last_got_services = clock()
return self.services
def __init__(self, storage, picker, backlog, max_rate_period, numpieces,
chunksize, measurefunc, snub_time, kickbans_ok, kickfunc,
banfunc):
self.storage = storage
self.picker = picker
self.backlog = backlog
self.max_rate_period = max_rate_period
self.measurefunc = measurefunc
self.totalmeasure = Measure(max_rate_period * storage.piece_length /
storage.request_size)
self.numpieces = numpieces
self.chunksize = chunksize
self.snub_time = snub_time
self.kickfunc = kickfunc
self.banfunc = banfunc
self.disconnectedseeds = {}
self.downloads = []
self.perip = {}
self.gotbaddata = set()
self.kicked = {}
self.banned = {}
self.kickbans_ok = kickbans_ok
self.kickbans_halted = False
self.super_seeding = False
self.endgamemode = False
self.endgame_queued_pieces = []
self.all_requests = []
self.discarded = 0L
# self.download_rate = 25000 # 25K/s test rate
self.download_rate = 0
self.bytes_requested = 0
self.last_time = clock()
self.queued_out = set()
self.requeueing = False
self.paused = False
def try_send(self, check_time=False):
t = clock()
self.bytes_sent -= (t - self.lasttime) * self.upload_rate
self.lasttime = t
if check_time:
self.bytes_sent = max(self.bytes_sent, 0)
cur = self.last.next_upload
while self.bytes_sent <= 0:
bytes = cur.send_partial(self.unitsize)
self.bytes_sent += bytes
self.measure.update_rate(bytes)
if bytes == 0 or cur.backlogged():
if self.last is cur:
self.last = None
cur.next_upload = None
break
else:
self.last.next_upload = cur.next_upload
cur.next_upload = None
cur = self.last.next_upload
else:
self.last = cur
cur = cur.next_upload
else:
self.sched(self.try_send, self.bytes_sent / self.upload_rate)
def get_time_left(self, left):
t = clock()
if not self.got_anything:
return None
if t - self.last > 15:
self.update(0)
try:
remaining = left / self.rate
if not self.lastten and remaining <= 10:
self.lastten = True
if self.lastten:
return remaining
delta = max(remaining / 20, 2)
if self.remaining is None:
self.remaining = remaining
elif abs(self.remaining - remaining) > delta:
self.remaining = remaining
else:
self.remaining -= t - self.last_checked
except ZeroDivisionError:
self.remaining = None
if self.remaining is not None and self.remaining < 0.1:
self.remaining = 0.1
self.last_checked = t
return self.remaining
def pop_external(self):
"""Prepare tasks queued with add_task to be run in the listen_forever
loop."""
to_add, self.externally_added = self.externally_added, []
for (func, delay, tid) in to_add:
if tid not in self.tasks_to_kill:
bisect.insort(self.funcs, (clock() + delay, func, tid))
def onPause(self):
self.whenpaused = clock()
if not self.downloader:
return
self.downloader.pause(True)
self.encoder.pause(True)
self.choker.pause(True)
def next_have(self, connection, looser_upload):
if self.seed_time is None:
self.seed_time = clock()
return None
# wait 10 seconds after seeing the first peers to give time to grab
# have lists
if clock() < self.seed_time + 10:
return None
if not connection.upload.super_seeding:
return None
olddl = self.seed_connections.get(connection)
if olddl is None:
ip = connection.get_ip()
olddl = self.past_ips.get(ip)
if olddl is not None: # peer reconnected
self.seed_connections[connection] = olddl
if not looser_upload:
self.seed_got_haves[olddl] -= 1 # penalize
if olddl is not None:
# send a new have even if it hasn't spread that piece elsewhere
if looser_upload:
num = 1
else:
num = 2
if self.seed_got_haves[olddl] < num:
return None
# it never downloaded it?
if not connection.upload.was_ever_interested:
connection.upload.skipped_count += 1
# probably another stealthed seed
if connection.upload.skipped_count >= 3:
return -1 # signal to close it
for tier in self.interests:
for piece in tier:
if not connection.download.have[piece]:
seedint = self.level_in_interests[piece]
# tweak it up one, so you don't duplicate effort
self.level_in_interests[piece] += 1
if seedint == len(self.interests) - 1:
self.interests.append([])
self._shift_over(piece, self.interests[seedint],
self.interests[seedint + 1])
self.seed_got_haves[piece] = 0 # reset this
self.seed_connections[connection] = piece
connection.upload.seed_have_list.append(piece)
return piece
return -1 # something screwy; terminate connection
def update_rate(self, amount):
self.total += amount
t = clock()
self.rate = (self.rate * (self.last - self.ratesince) +
amount) / (t - self.ratesince + 0.0001)
self.last = t
if self.ratesince < t - self.max_rate_period:
self.ratesince = t - self.max_rate_period
def old_style_init(self):
while self.initialize_tasks:
msg, done, init, next = self.initialize_tasks.pop(0)
if init():
self.statusfunc(activity=msg, fractionDone=done)
t = clock() + STATS_INTERVAL
x = 0
while x is not None:
if t < clock():
t = clock() + STATS_INTERVAL
self.statusfunc(fractionDone=x)
self.unpauseflag.wait()
if self.flag.isSet():
return False
x = next()
self.statusfunc(fractionDone=0)
return True
def onUnpause(self):
if not self.downloader:
return
self.downloader.pause(False)
self.encoder.pause(False)
self.choker.pause(False)
# rerequest automatically if paused for >60 seconds
if self.rerequest and self.whenpaused and \
clock() - self.whenpaused > 60:
self.rerequest.announce(3)
def num_disconnected_seeds(self):
# first expire old ones
expired = []
for id, t in self.disconnectedseeds.iteritems():
if clock() - t > EXPIRE_TIME: # Expire old seeds after so long
expired.append(id)
for id in expired:
# self.picker.seed_disappeared()
del self.disconnectedseeds[id]
return len(self.disconnectedseeds)
def log(self, ip, ident, username, header, responsecode, length, referrer,
useragent):
year, month, day, hour, minute, second = time.localtime()[:6]
print '%s %s %s [%02d/%3s/%04d:%02d:%02d:%02d] "%s" %i %i "%s" "%s"' \
'' % (ip, ident, username, day, months[month], year, hour, minute,
second, header, responsecode, length, referrer, useragent)
t = clock()
if t - self.lastflush > self.minflush:
self.lastflush = t
sys.stdout.flush()
def __init__(self, config, schedule, picker, done=lambda: False):
self.config = config
self.round_robin_period = config['round_robin_period']
self.schedule = schedule
self.picker = picker
self.connections = []
self.last_preferred = 0
self.last_round_robin = clock()
self.done = done
self.super_seed = False
self.paused = False
schedule(self._round_robin, 5)
def ping(self, delay):
if DEBUG:
print delay
if not self.autoadjust:
return
self.pings.append(delay > PING_BOUNDARY)
if len(self.pings) < PING_SAMPLES + PING_DISCARDS:
return
if DEBUG:
print 'cycle'
pings = sum(self.pings[PING_DISCARDS:])
del self.pings[:]
if pings >= PING_THRESHHOLD: # assume flooded
if self.upload_rate == MAX_RATE:
self.upload_rate = self.measure.get_rate() * ADJUST_DOWN
else:
self.upload_rate = min(self.upload_rate,
self.measure.get_rate() * 1.1)
self.upload_rate = max(int(self.upload_rate * ADJUST_DOWN), 2)
self.slots = int(math.sqrt(self.upload_rate * SLOTS_FACTOR))
self.slotsfunc(self.slots)
if DEBUG:
print 'adjust down to ' + str(self.upload_rate)
self.lasttime = clock()
self.bytes_sent = 0
self.autoadjustup = UP_DELAY_FIRST
else: # not flooded
if self.upload_rate == MAX_RATE:
return
self.autoadjustup -= 1
if self.autoadjustup:
return
self.upload_rate = int(self.upload_rate * ADJUST_UP)
self.slots = int(math.sqrt(self.upload_rate * SLOTS_FACTOR))
self.slotsfunc(self.slots)
if DEBUG:
print 'adjust up to ' + str(self.upload_rate)
self.lasttime = clock()
self.bytes_sent = 0
self.autoadjustup = UP_DELAY_NEXT
def send_unchoke(self):
if self.send_choke_queued:
self.send_choke_queued = False
if DEBUG1:
print (self.ccount, 'CHOKE SUPPRESSED')
else:
self._send_message(UNCHOKE)
if (self.partial_message or self.just_unchoked is None or
not self.upload.interested or
self.download.active_requests):
self.just_unchoked = 0
else:
self.just_unchoked = clock()
def get_ip(self):
if self.last_got_ip + EXPIRE_CACHE < clock():
local_ips = AddrList()
local_ips.set_intranet_addresses()
try:
for info in socket.getaddrinfo(socket.gethostname(), 0,
socket.AF_INET):
# exception if socket library isn't recent
self.local_ip = info[4][0]
if self.local_ip in local_ips:
self.last_got_ip = clock()
if DEBUG:
print 'Local IP found: ' + self.local_ip
break
else:
raise ValueError('couldn\'t find intranet IP')
except (ValueError, socket.error):
self.local_ip = None
if DEBUG:
print 'Error finding local IP'
print_exc()
return self.local_ip
def got_piece(self, index, begin, piece):
length = len(piece)
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, piece,
self.guard):
self.downloader.piece_flunked(index)
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
assert not d.active_requests
d.fix_download_endgame()
else:
try:
d.active_requests.remove(
(index, begin, length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
else:
assert not d.active_requests
self._request_more()
self.downloader.check_complete(index)
return self.downloader.storage.do_I_have(index)
def update(self, amount):
t = clock()
exp = int(t) - int(self.last)
self.time *= (FACTOR ** exp)
self.got *= (FACTOR ** exp)
self.got += amount
if t - self.last < 20:
self.time += t - self.last
self.last = t
try:
self.rate = self.got / self.time
except ZeroDivisionError:
pass
def __init__(self, socket_handler, sock, handler, ip=None):
self.socket_handler = socket_handler
self.socket = sock
self.handler = handler
self.buffer = []
self.last_hit = clock()
self.fileno = sock.fileno()
self.connected = False
self.skipped = 0
# self.check = StreamCheck()
try:
self.ip = self.socket.getpeername()[0]
except socket.error:
if ip is None:
self.ip = 'unknown'
else:
self.ip = ip
def set_upload_rate(self, rate):
# rate = -1 # test automatic
if rate < 0:
if self.autoadjust:
return
self.autoadjust = True
self.autoadjustup = 0
self.pings = []
rate = MAX_RATE
self.slots = SLOTS_STARTING
self.slotsfunc(self.slots)
else:
self.autoadjust = False
if not rate:
rate = MAX_RATE
self.upload_rate = rate * 1000
self.lasttime = clock()
self.bytes_sent = 0
def queue_limit(self):
if not self.download_rate:
return 10e10 # that's a big queue!
t = clock()
self.bytes_requested -= (t - self.last_time) * self.download_rate
self.last_time = t
if not self.requeueing and self.queued_out and \
self.bytes_requested < 0:
self.requeueing = True
q = list(self.queued_out)
random.shuffle(q)
self.queued_out = set()
for d in q:
d._request_more()
self.requeueing = False
if -self.bytes_requested > 5 * self.download_rate:
self.bytes_requested = -5 * self.download_rate
return max(int(-self.bytes_requested / self.chunksize), 0)
def handle_events(self, events):
for sock, event in events:
s = self.servers.get(sock)
if s:
if event & (POLLHUP | POLLERR) != 0:
self.poll.unregister(s)
s.close()
del self.servers[sock]
print "lost server socket"
elif len(self.single_sockets) < self.max_connects:
try:
newsock, _ = s.accept()
newsock.setblocking(0)
nss = SingleSocket(self, newsock, self.handler)
self.single_sockets[newsock.fileno()] = nss
self.poll.register(newsock, POLLIN)
self.handler.external_connection_made(nss)
except socket.error:
time.sleep(1)
else:
s = self.single_sockets.get(sock)
if not s:
continue
s.connected = True
if event & (POLLHUP | POLLERR):
self._close_socket(s)
continue
if event & POLLIN:
try:
s.last_hit = clock()
data = s.socket.recv(self.readsize)
if not data:
self._close_socket(s)
else:
s.handler.data_came_in(s, data)
except socket.error as e:
if e[0] != EWOULDBLOCK:
self._close_socket(s)
continue
if event & POLLOUT and s.socket and not s.is_flushed():
s.try_write()
if s.is_flushed():
s.handler.connection_flushed(s)
def scan_for_timeouts(self):
t = clock() - self.timeout
for s in self.single_sockets.values():
if s.last_hit < t and s.socket is not None:
self._close_socket(s)
def __init__(self, max_rate_period, fudge=1):
self.max_rate_period = max_rate_period
self.ratesince = clock() - fudge
self.last = self.ratesince
self.rate = 0.0
self.total = 0l
def time_until_rate(self, newrate):
if self.rate <= newrate:
return 0
t = clock() - self.ratesince
return ((self.rate * t) / newrate) - t
