def __init__(self, infohash, storage, picker, backlog, max_rate_period,
numpieces, chunksize, measurefunc, snub_time,
kickbans_ok, kickfunc, banfunc, scheduler = None):
self.infohash = infohash
self.b64_infohash = b64encode(infohash)
self.storage = storage
self.picker = picker
self.backlog = backlog
self.max_rate_period = max_rate_period
self.measurefunc = measurefunc
self.totalmeasure = Measure(max_rate_period*storage.piece_length/storage.request_size)
self.numpieces = numpieces
self.chunksize = chunksize
self.snub_time = snub_time
self.kickfunc = kickfunc
self.banfunc = banfunc
self.disconnectedseeds = {}
self.downloads = []
self.perip = {}
self.gotbaddata = {}
self.kicked = {}
self.banned = {}
self.kickbans_ok = kickbans_ok
self.kickbans_halted = False
self.super_seeding = False
self.endgamemode = False
self.endgame_queued_pieces = []
self.all_requests = []
self.discarded = 0L
self.download_rate = 0
# self.download_rate = 25000 # 25K/s test rate
self.bytes_requested = 0
self.last_time = clock()
self.queued_out = {}
self.requeueing = False
self.paused = False
self.scheduler = scheduler
# hack: we should not import this since it is not part of the
# core nor should we import here, but otherwise we will get
# import errors
#
# _event_reporter stores events that are logged somewhere...
# from BaseLib.Core.Statistics.StatusReporter import get_reporter_instance
# self._event_reporter = get_reporter_instance()
self._event_reporter = get_status_holder("LivingLab")
# check periodicaly
self.scheduler(self.dlr_periodic_check, 1)
def get_upload_chunk(self):
if self.choked or not self.buffer:
return None
index, begin, length = self.buffer.pop(0)
if self.config['buffer_reads']:
if index != self.piecedl:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = index
# Merkle
[ self.piecebuf, self.hashlist ] = self.storage.get_piece(index, 0, -1)
try:
piece = self.piecebuf[begin:begin+length]
assert len(piece) == length
except: # fails if storage.get_piece returns None or if out of range
self.connection.close()
return None
if begin == 0:
hashlist = self.hashlist
else:
hashlist = []
else:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = None
[piece, hashlist] = self.storage.get_piece(index, begin, length)
if piece is None:
self.connection.close()
return None
self.measure.update_rate(len(piece))
self.totalup.update_rate(len(piece))
status = get_status_holder("LivingLab")
s_upload = status.get_or_create_status_element("uploaded",0)
s_upload.inc(len(piece))
# BarterCast counter
self.connection.total_uploaded += length
return (index, begin, hashlist, piece)
def __init__(self, connecter, raw_server, my_id, max_len,
schedulefunc, keepalive_delay, download_id,
measurefunc, config):
self.raw_server = raw_server
self.connecter = connecter
self.my_id = my_id
self.max_len = max_len
self.schedulefunc = schedulefunc
self.keepalive_delay = keepalive_delay
self.download_id = download_id
self.measurefunc = measurefunc
self.config = config
self.connections = {}
self.banned = {}
self.to_connect = Set()
self.trackertime = 0
self.paused = False
if self.config['max_connections'] == 0:
self.max_connections = 2 ** 30
else:
self.max_connections = self.config['max_connections']
"""
In r529 there was a problem when a single Windows client
would connect to our text-based seeder (i.e. btlaunchmany)
with no other clients present. Apparently both the seeder
and client would connect to eachother simultaneously, but
not end up with a good connection, halting the client.
Arno, 2006-03-10: Reappears in ~r890, fixed in r892. It
appears to be a problem of writing to a nonblocking socket
before it signalled it is ready for writing, although the
evidence is inconclusive.
Arno: 2006-12-15: Reappears in r2319. There is some weird
socket problem here. Using Python 2.4.4 doesn't solve it.
The problem I see here is that as soon as we register
at the tracker, the single seeder tries to connect to
us. He succeeds, but after a short while the connection
appears to be closed by him. We then wind up with no
connection at all and have to wait until we recontact
the tracker.
My workaround is to refuse these initial connections from
the seeder and wait until I've started connecting to peers
based on the info I got from the tracker before accepting
remote connections.
Arno: 2007-02-16: I think I finally found it. The Tribler
tracker (BitTornado/BT1/track.py) will do a NAT check
(BitTornado/BT1/NATCheck) by default, which consists of
initiating a connection and then closing it after a good
BT handshake was received.
The solution now is to make sure we check IP and port to
identify existing connections. I already added that 2006-12-15,
so I just removed the restriction on initial connections,
which are superfluous.
"""
self.rerequest = None
# 2fastbt_
self.toofast_banned = {}
self.coordinator_ip = None
# _2fastbt
# hack: we should not import this since it is not part of the
# core nor should we import here, but otherwise we will get
# import errors
#
# _event_reporter stores events that are logged somewhere...
# from BaseLib.Core.Statistics.StatusReporter import get_reporter_instance
self._event_reporter = get_status_holder("LivingLab")
# the addresses that have already been reported
self._known_addresses = {}
schedulefunc(self.send_keepalives, keepalive_delay)
# RePEX: added repexer field.
# Note: perhaps call it observer in the future and make the
# download engine more observable?
self.repexer = None
def __init__(self,infohash,metainfo,kvconfig,multihandler,get_extip_func,listenport,videoanalyserpath,vodfileindex,set_error_func,pstate,lmvodeventcallback,lmhashcheckcompletecallback):
self.dow = None
self.set_error_func = set_error_func
self.videoinfo = None
self.videostatus = None
self.lmvodeventcallback = lmvodeventcallback
self.lmhashcheckcompletecallback = lmhashcheckcompletecallback
self.logmsgs = []
self._hashcheckfunc = None
self._getstatsfunc = None
self.infohash = infohash
self.b64_infohash = b64encode(infohash)
self.repexer = None
try:
self.dldoneflag = Event()
self.dlrawserver = multihandler.newRawServer(infohash,self.dldoneflag)
self.lmvodeventcallback = lmvodeventcallback
if pstate is not None:
self.hashcheckfrac = pstate['dlstate']['progress']
else:
self.hashcheckfrac = 0.0
self.peerid = createPeerID()
# LOGGING
event_reporter = get_status_holder("LivingLab")
event_reporter.create_and_add_event("peerid", [self.b64_infohash, b64encode(self.peerid)])
#print >>sys.stderr,time.asctime(),'-', "SingleDownload: __init__: My peer ID is",`peerid`
self.dow = BT1Download(self.hashcheckprogressfunc,
self.finishedfunc,
self.fatalerrorfunc,
self.nonfatalerrorfunc,
self.logerrorfunc,
self.dldoneflag,
kvconfig,
metainfo,
infohash,
self.peerid,
self.dlrawserver,
get_extip_func,
listenport,
videoanalyserpath
)
file = self.dow.saveAs(self.save_as)
#if DEBUG:
# print >>sys.stderr,time.asctime(),'-', "SingleDownload: dow.saveAs returned",file
# Set local filename in vodfileindex
if vodfileindex is not None:
# Ric: for SVC the index is a list of indexes
index = vodfileindex['index']
if type(index) == ListType:
svc = len(index) > 1
else:
svc = False
if svc:
outpathindex = self.dow.get_dest(index[0])
else:
if index == -1:
index = 0
outpathindex = self.dow.get_dest(index)
vodfileindex['outpath'] = outpathindex
self.videoinfo = vodfileindex
if 'live' in metainfo['info']:
authparams = metainfo['info']['live']
else:
authparams = None
if svc:
self.videostatus = SVCVideoStatus(metainfo['info']['piece length'],self.dow.files,vodfileindex,authparams)
else:
self.videostatus = VideoStatus(metainfo['info']['piece length'],self.dow.files,vodfileindex,authparams)
self.videoinfo['status'] = self.videostatus
self.dow.set_videoinfo(vodfileindex,self.videostatus)
#if DEBUG:
# print >>sys.stderr,time.asctime(),'-', "SingleDownload: setting vodfileindex",vodfileindex
# RePEX: Start in RePEX mode
if kvconfig['initialdlstatus'] == DLSTATUS_REPEXING:
if pstate is not None and pstate.has_key('dlstate'):
swarmcache = pstate['dlstate'].get('swarmcache',{})
else:
swarmcache = {}
self.repexer = RePEXer(self.infohash, swarmcache)
else:
self.repexer = None
if pstate is None:
resumedata = None
else:
# Restarting download
resumedata=pstate['engineresumedata']
self._hashcheckfunc = self.dow.initFiles(resumedata=resumedata)
except Exception,e:
self.fatalerrorfunc(e)
def _reporting_thread(self):
"""
Send the report on a separate thread
We choose not to use a lock object to protect access to
self._enable_reporting, self._retry_delay, and
self._report_deadline because only a single thread will write
and the other threads will only read there variables. Python
doesn't cause problems in this case.
"""
# minimum retry delay. this value will grow exponentially with
# every failure
retry_delay = 15
# the amount of time to sleep before the next report (or until
# the _thread_event is set)
timeout = retry_delay
# a list containing all urlencoded reports that have yet been
# send (most of the time this list will be empty, except when
# reports could not be delivered)
reports = []
# local copy of the self._event when it is being reported
event = None
if USE_LIVING_LAB_REPORTING:
# the m18 trial statistics are gathered at the 'living lab'
session = Session.get_instance()
living_lab_reporter = LivingLabOnChangeReporter(
"vod-stats-reporter")
living_lab_reporter.set_permid(session.get_permid())
status_holder = get_status_holder("vod-stats")
status_holder.add_reporter(living_lab_reporter)
status_element = status_holder.create_status_element(
"action-list",
"A list containing timestamped VOD playback events",
initial_value=[])
else:
# there are several urls available where reports can be
# send. one should be picked randomly each time.
#
# when a report is successfull it will stay with the same
# reporter. when a report is unsuccessfull (could not
# connect) it will cycle through reporters.
report_urls = [
[0, 0, "http://reporter1.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter2.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter3.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter4.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter5.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter6.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter7.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter8.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter9.tribler.org/swarmplayer.py"]
]
shuffle(report_urls)
while True:
# sleep in between reports. will send a report immediately
# when the flush event is set
self._thread_flush.wait(timeout)
self._thread_flush.clear()
# create report
self._thread_lock.acquire()
try:
if self._event:
# copy between threads while locked
event = self._event
self._event = []
else:
# we have nothing to report... sleep
timeout = retry_delay
event = None
finally:
self._thread_lock.release()
if event:
# prepend the session-key
event.insert(
0, {
"key": "session-key",
"timestamp": time(),
"event": self._session_key
})
event.insert(
0, {
"key": "sequence-number",
"timestamp": time(),
"event": self._sequence_number
})
self._sequence_number += 1
if USE_LIVING_LAB_REPORTING:
if event:
try:
if status_element.set_value(event):
# Living lab's doesn't support dynamic reporting.
# We use 60 seconds by default
timeout = 60
else:
# something went wrong...
retry_delay *= 2
timeout = retry_delay
except:
# error contacting server
print_exc(file=sys.stderr)
retry_delay *= 2
timeout = retry_delay
else:
# add new report
if event:
if len(event) < 10:
# uncompressed
report = {
"version": "3",
"created": time(),
"event": event
}
else:
# compress
report = {
"version": "4",
"created": time(),
"event":
urllib.quote(zlib.compress(repr(event), 9))
}
reports.append(urllib.urlencode(report))
if not reports:
timeout = retry_delay
continue
reporter = report_urls[0]
if DEBUG:
print >> sys.stderr, "EventStatusReporter: attempting to report,", len(
reports[0]), "bytes to", reporter[2]
try:
sock = urllib.urlopen(reporter[2], reports[0])
result = sock.read()
sock.close()
# all ok? then remove the report
del reports[0]
# increase the 'good-report' counter, no need to re-order
reporter[1] += 1
except:
# error contacting server
print_exc(file=sys.stderr)
retry_delay *= 2
# increase the 'bad-report' counter and order by failures
reporter[0] += 1
report_urls.sort(lambda x, y: cmp(x[0], y[0]))
continue
if result.isdigit():
result = int(result)
if result == 0:
# remote server is not recording, so don't bother
# sending events
if DEBUG:
print >> sys.stderr, "EventStatusReporter: received -zero- from the HTTP server. Reporting disabled"
self._thread_lock.acquire()
self._enable_reporting = False
self._thread_lock.release()
# close thread
return
else:
# I choose not to reset the retry_delay because
# swarmplayer sessions tend to be short. And the
# if there are connection failures I want as few
# retries as possible
if DEBUG:
print >> sys.stderr, "EventStatusReporter: report successfull. Next report in", result, "seconds"
timeout = result
else:
self._thread_lock.acquire()
self._enable_reporting = False
self._thread_lock.release()
# close thread
return
def _reporting_thread(self):
"""
Send the report on a separate thread
We choose not to use a lock object to protect access to
self._enable_reporting, self._retry_delay, and
self._report_deadline because only a single thread will write
and the other threads will only read there variables. Python
doesn't cause problems in this case.
"""
# minimum retry delay. this value will grow exponentially with
# every failure
retry_delay = 15
# the amount of time to sleep before the next report (or until
# the _thread_event is set)
timeout = retry_delay
# a list containing all urlencoded reports that have yet been
# send (most of the time this list will be empty, except when
# reports could not be delivered)
reports = []
# local copy of the self._event when it is being reported
event = None
if USE_LIVING_LAB_REPORTING:
# the m18 trial statistics are gathered at the 'living lab'
session = Session.get_instance()
living_lab_reporter = LivingLabOnChangeReporter("vod-stats-reporter")
living_lab_reporter.set_permid(session.get_permid())
status_holder = get_status_holder("vod-stats")
status_holder.add_reporter(living_lab_reporter)
status_element = status_holder.create_status_element("action-list", "A list containing timestamped VOD playback events", initial_value=[])
else:
# there are several urls available where reports can be
# send. one should be picked randomly each time.
#
# when a report is successfull it will stay with the same
# reporter. when a report is unsuccessfull (could not
# connect) it will cycle through reporters.
report_urls = [[0, 0, "http://reporter1.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter2.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter3.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter4.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter5.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter6.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter7.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter8.tribler.org/swarmplayer.py"],
[0, 0, "http://reporter9.tribler.org/swarmplayer.py"]]
shuffle(report_urls)
while True:
# sleep in between reports. will send a report immediately
# when the flush event is set
self._thread_flush.wait(timeout)
self._thread_flush.clear()
# create report
self._thread_lock.acquire()
try:
if self._event:
# copy between threads while locked
event = self._event
self._event = []
else:
# we have nothing to report... sleep
timeout = retry_delay
event = None
finally:
self._thread_lock.release()
if event:
# prepend the session-key
event.insert(0, {"key":"session-key", "timestamp":time(), "event":self._session_key})
event.insert(0, {"key":"sequence-number", "timestamp":time(), "event":self._sequence_number})
self._sequence_number += 1
if USE_LIVING_LAB_REPORTING:
if event:
try:
if status_element.set_value(event):
# Living lab's doesn't support dynamic reporting.
# We use 60 seconds by default
timeout = 60
else:
# something went wrong...
retry_delay *= 2
timeout = retry_delay
except:
# error contacting server
print_exc(file=sys.stderr)
retry_delay *= 2
timeout = retry_delay
else:
# add new report
if event:
if len(event) < 10:
# uncompressed
report = {"version":"3",
"created":time(),
"event":event}
else:
# compress
report = {"version":"4",
"created":time(),
"event":urllib.quote(zlib.compress(repr(event), 9))}
reports.append(urllib.urlencode(report))
if not reports:
timeout = retry_delay
continue
reporter = report_urls[0]
if DEBUG: print >> sys.stderr, "EventStatusReporter: attempting to report,", len(reports[0]), "bytes to", reporter[2]
try:
sock = urllib.urlopen(reporter[2], reports[0])
result = sock.read()
sock.close()
# all ok? then remove the report
del reports[0]
# increase the 'good-report' counter, no need to re-order
reporter[1] += 1
except:
# error contacting server
print_exc(file=sys.stderr)
retry_delay *= 2
# increase the 'bad-report' counter and order by failures
reporter[0] += 1
report_urls.sort(lambda x, y:cmp(x[0], y[0]))
continue
if result.isdigit():
result = int(result)
if result == 0:
# remote server is not recording, so don't bother
# sending events
if DEBUG: print >> sys.stderr, "EventStatusReporter: received -zero- from the HTTP server. Reporting disabled"
self._thread_lock.acquire()
self._enable_reporting = False
self._thread_lock.release()
# close thread
return
else:
# I choose not to reset the retry_delay because
# swarmplayer sessions tend to be short. And the
# if there are connection failures I want as few
# retries as possible
if DEBUG: print >> sys.stderr, "EventStatusReporter: report successfull. Next report in", result, "seconds"
timeout = result
else:
self._thread_lock.acquire()
self._enable_reporting = False
self._thread_lock.release()
# close thread
return
def got_piece(self, index, begin, hashlist, piece):
"""
Returns True if the piece is complete.
Note that in this case a -piece- means a chunk!
"""
if self.bad_performance_counter:
self.bad_performance_counter -= 1
if DEBUG: print >>sys.stderr, "decreased bad_performance_counter to", self.bad_performance_counter
length = len(piece)
#if DEBUG:
# print >> sys.stderr, 'Downloader: got piece of length %d' % length
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
if DEBUG: print >>sys.stderr, "Downloader: got_piece: removed one request from all_requests", len(self.downloader.all_requests), "remaining"
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
# Update statistic gatherer
status = get_status_holder("LivingLab")
s_download = status.get_or_create_status_element("downloaded",0)
s_download.inc(length)
self.short_term_measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard):
self.downloader.piece_flunked(index)
return False
# boudewijn: we need more accurate (if possibly invalid)
# measurements on current download speed
self.downloader.picker.got_piece(index, begin, length)
# print "Got piece=", index, "begin=", begin, "len=", length
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
assert not d.active_requests
d.fix_download_endgame()
else:
try:
d.active_requests.remove((index, begin, length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
else:
assert not d.active_requests
self._request_more()
self.downloader.check_complete(index)
# BarterCast counter
self.connection.total_downloaded += length
return self.downloader.storage.do_I_have(index)
