def unwait(self, code):
if code == self.code and self.pause.locked():
if DEBUG_LOCK:
log(self.log_prefix + 'unwait: release pause: code', code, 'self.code', self.code, 'thread', currentThread().name)
self.pause.release()
elif DEBUG_LOCK:
log(self.log_prefix + 'unwait: do not release pause: code', code, 'self.code', self.code, 'thread', currentThread().name)
def got_piece(self, index):
self.piecelock.acquire()
try:
if index in self.have:
return
start_new_file = False
if DEBUG:
log('vs::got_piece: index', index, 'thread', currentThread().getName())
if self.in_download_range(index):
self.have.append(index)
self.numhave += 1
self.completed = self.numhave / float(self.movie_numpieces)
if not self.can_be_downloaded and self.completed >= self.min_download_percent:
self.can_be_downloaded = True
start_new_file = True
if index in self.prebuf_high_priority_pieces:
self.prebuf_high_priority_pieces.remove(index)
if len(self.prebuf_missing_pieces):
try:
if index in self.prebuf_missing_pieces:
self.prebuf_missing_pieces.remove(index)
if len(self.prebuf_missing_pieces) == 0 and self.is_extra:
self.prebuffering = False
except:
pass
elif DEBUG:
log('vs::got_piece: piece not in download range: index', index)
return start_new_file
finally:
self.piecelock.release()
def isfinished(self):
self.lock.acquire()
x = self.finished
self.lock.release()
if DEBUG_LOCK:
log(self.log_prefix + 'isfinished: x', x, 'thread', currentThread().name)
return x
def got_invalidate(self, index):
if DEBUG:
log('downloader::got_invalidate: index', index)
if not self.have[index]:
return
self.have[index] = False
self.downloader.picker.lost_have(index)
def send_torrent(
self, torrent_data, developer_id=None, affiliate_id=None, zone_id=None, protected=False, infohash=None
):
params = {}
if developer_id is not None:
params["d"] = str(developer_id)
if affiliate_id is not None:
params["a"] = str(affiliate_id)
if zone_id is not None:
params["z"] = str(zone_id)
if protected:
params["protected"] = "1"
if infohash is not None:
params["infohash"] = binascii.hexlify(infohash)
response = self.send_request(
"puttorrent", params, data=torrent_data, content_type="application/octet-stream", use_random=True
)
if response is None:
if DEBUG:
log("tsservice::send_torrent: request failed")
return
try:
player_id = domutils.get_tag_value(response, "id")
except Exception as e:
if DEBUG:
log("tsservice::send_torrent: failed to parse response: " + str(e))
return
return player_id
def read_type(self, data):
self.header = data.strip()
words = data.split()
if DEBUG2:
log('HTTPConnection::read_type: data', data, 'words', words)
if len(words) == 3:
self.command, self.path, garbage = words
self.pre1 = False
elif len(words) == 2:
self.command, self.path = words
self.pre1 = True
if self.command != 'GET':
if DEBUG2:
log('HTTPConnection::read_type: return none')
return None
else:
if DEBUG2:
log('HTTPConnection::read_type: return none')
return None
if self.command not in ('HEAD', 'GET'):
if DEBUG2:
log('HTTPConnection::read_type: return none')
return None
self.headers = {}
if DEBUG2:
log('HTTPConnection::read_type: command', self.command, 'path', self.path, 'next read_header()')
return self.read_header
def check_content(self, stat_type, tdef):
if not self.settings.has_key(stat_type):
return False
options = self.settings[stat_type]
if options['mode'] == RemoteStatisticsSettings.MODE_DENY_ALL:
return False
if options['mode'] == RemoteStatisticsSettings.MODE_ALLOW_ALL:
return True
if options['mode'] == RemoteStatisticsSettings.MODE_WHITELIST:
if options['whitelist_infohash'] is not None:
infohash = binascii.hexlify(tdef.get_infohash())
return infohash in options['whitelist_infohash']
if options['whitelist_content'] is not None:
provider_key = tdef.get_provider()
content_id = tdef.get_content_id()
if provider_key is None or content_id is None:
return False
k = provider_key + ':' + content_id
return k in options['whitelist_content']
elif options['mode'] == RemoteStatisticsSettings.MODE_BLACKLIST:
if options['blacklist_infohash'] is not None:
infohash = binascii.hexlify(tdef.get_infohash())
return infohash not in options['blacklist_infohash']
if options['blacklist_content'] is not None:
provider_key = tdef.get_provider()
content_id = tdef.get_content_id()
if provider_key is None or content_id is None:
return False
k = provider_key + ':' + content_id
return k not in options['blacklist_content']
if DEBUG:
log('RemoteStatisticsSettings::check_content: should not be here: settings', self.settings)
return False
def give_up(self):
self.lock.acquire()
self.success = False
self.finished = True
if DEBUG_LOCK:
log(self.log_prefix + 'give_up: self.success', self.success, 'self.finished', self.finished, 'thread', currentThread().name)
self.lock.release()
def connection_flushed(self, connection):
if self.connections[connection].done:
if DEBUG2:
log('HTTPHandler::connection_flushed: connection is done, call connection.shutdown')
connection.shutdown(1)
elif DEBUG2:
log('HTTPHandler::connection_flushed: connection is not done')
def _next(self, haves, wantfunc, complete_first, helper_con, willrequest = True, connection = None, shuffle = True):
cutoff = self.numgot < self.rarest_first_cutoff
complete_first = (complete_first or cutoff) and not haves.complete()
best = None
bestnum = 1073741824
for i in self.started:
if haves[i] and wantfunc(i) and (self.helper is None or helper_con or not self.helper.is_ignored(i)):
if self.level_in_interests[i] < bestnum:
best = i
bestnum = self.level_in_interests[i]
if best is not None:
if complete_first or cutoff and len(self.interests) > self.cutoff:
return best
if DEBUG:
log('pp:_next: haves.complete()', haves.complete())
if haves.complete():
r = [(0, min(bestnum, len(self.interests)))]
elif cutoff and len(self.interests) > self.cutoff:
r = [(self.cutoff, min(bestnum, len(self.interests))), (0, self.cutoff)]
else:
r = [(0, min(bestnum, len(self.interests)))]
if DEBUG:
log('pp:_next: r', r, 'interests', self.interests)
for lo, hi in r:
for i in xrange(lo, hi):
for j in self.interests[i]:
if haves[j] and wantfunc(j) and (self.helper is None or helper_con or not self.helper.is_ignored(j)):
return j
if best is not None:
return best
def next_have(self, connection, looser_upload):
if DEBUG:
log('pp::next_have: ---')
if self.seed_time is None:
self.seed_time = clock()
return
if clock() < self.seed_time + 10:
return
if not connection.upload.super_seeding:
return
if connection in self.seed_connections:
if looser_upload:
num = 1
else:
num = 2
if self.seed_got_haves[self.seed_connections[connection]] < num:
return
if not connection.upload.was_ever_interested:
connection.upload.skipped_count += 1
if connection.upload.skipped_count >= 3:
return -1
for tier in self.interests:
for piece in tier:
if not connection.download.have[piece]:
seedint = self.level_in_interests[piece]
self.level_in_interests[piece] += 1
if seedint == len(self.interests) - 1:
self.interests.append([])
self._shift_over(piece, self.interests[seedint], self.interests[seedint + 1])
self.seed_got_haves[piece] = 0
self.seed_connections[connection] = piece
connection.upload.seed_have_list.append(piece)
return piece
return -1
def __init__(self, i2iport, connhandler, timeout = 300.0, port_file = None):
Thread.__init__(self)
self.setDaemon(True)
self.setName('Instance2Instance' + self.getName())
self.i2iport = i2iport
self.port_file = port_file
self.connhandler = connhandler
self.i2idoneflag = Event()
self.rawserver = RawServer(self.i2idoneflag, timeout / 5.0, timeout, ipv6_enable=False, failfunc=self.rawserver_fatalerrorfunc, errorfunc=self.rawserver_nonfatalerrorfunc)
self.rawserver.add_task(self.rawserver_keepalive, 1)
if globalConfig.get_value('allow-non-local-client-connection'):
interfaces = self.rawserver.bind(self.i2iport, reuse=True)
else:
interfaces = self.rawserver.bind(self.i2iport, bind=['127.0.0.1'], reuse=True)
if DEBUG:
log('i2is::init: bound on interfaces', interfaces)
if i2iport == 0 and len(interfaces):
host, port = interfaces[0]
self.i2iport = port
if port_file is not None:
f = None
try:
f = open(port_file, 'w')
f.write(str(port))
except:
if DEBUG:
log('i2is::init: cannot save port to file', port_file)
raise Exception, 'Cannot save port'
finally:
if f:
f.close()
def complete(self, piece):
if DEBUG:
log('pp::complete: piece', piece)
self.pplock.acquire()
try:
if self.has[piece]:
return False
self.has[piece] = 1
self.numgot += 1
if DEBUG:
print >> sys.stderr, 'PiecePicker::complete: piece:', piece, 'self.numgot:', self.numgot, 'self.numpieces', self.numpieces
if self.numgot == self.numpieces:
if DEBUG:
print >> sys.stderr, 'PiecePicker::complete: self.done=True'
self.done = True
self.crosscount2 = self.crosscount
else:
numhaves = self.numhaves[piece]
self.crosscount2[numhaves] -= 1
if numhaves + 1 == len(self.crosscount2):
self.crosscount2.append(0)
self.crosscount2[numhaves + 1] += 1
self._remove_from_interests(piece)
return True
finally:
self.pplock.release()
def save_torrent_local(self, tdef, checksum):
save_name = binascii.hexlify(tdef.get_infohash()) + ".torrent"
torrent_dir = self.get_torrent_collecting_dir()
save_path = os.path.join(torrent_dir, save_name)
if DEBUG:
log(
"session::save_torrent_local: save torrent: save_path",
save_path,
"checksum",
binascii.hexlify(checksum),
)
torrent_data = tdef.save(save_path)
extra_info = {"status": "good"}
extra_info["filename"] = save_name
extra_info["checksum"] = checksum
db = self.open_dbhandler(NTFY_TORRENTS)
if db is None:
return
try:
db.addExternalTorrent(tdef, source="", extra_info=extra_info)
except:
if DEBUG:
log_exc()
finally:
self.close_dbhandler(db)
def acquire_inputstream(self, urlpath):
global DEBUG
if urlpath is None:
return
streaminfo = None
for mapper in self.mappers:
streaminfo = mapper.get(urlpath)
if streaminfo is not None and (streaminfo['statuscode'] == 200 or streaminfo['statuscode'] == 301):
return streaminfo
self.lock.acquire()
if DEBUGLOCK:
log('VideoServer::acquire_inputstream: lock done', urlpath, currentThread().getName())
try:
streaminfo = self.urlpath2streaminfo.get(urlpath, None)
if DEBUG:
log('videoserver::acquire_inputstream: got streaminfo: urlpath', urlpath, 'streaminfo', streaminfo)
finally:
if DEBUGLOCK:
log('VideoServer::acquire_inputstream: unlock', urlpath, currentThread().getName())
self.lock.release()
if streaminfo is not None and 'lock' in streaminfo:
if DEBUGLOCK:
log('VideoServer::acquire_inputstream: lock stream: urlpath', urlpath, 'streaminfo', streaminfo, 'thread', currentThread().getName())
streaminfo['lock'].acquire()
if DEBUGLOCK:
log('VideoServer::acquire_inputstream: lock stream done: urlpath', urlpath, 'thread', currentThread().getName())
return streaminfo
def __init__(self, main_url, download_url, dlhash, config, multihandler,
fileinfo, resumedata, vodeventcallback, set_error_func,
finished_func, failed_func):
self.main_url = main_url
self.download_url = download_url
self.dlhash = dlhash
self.config = config
self.dlmode = config['mode']
self.fileinfo = fileinfo
self.vodeventcallback = vodeventcallback
self.set_error_func = set_error_func
self.finished_func = finished_func
self.failed_func = failed_func
self.download_id = binascii.hexlify(self.dlhash) + '-' + str(
long(time.time())) + '-' + str(random.randint(0, 100000))
self.dldoneflag = Event()
self.rawserver = multihandler.newRawServer(dlhash, self.dldoneflag)
if download_url is not None:
url = download_url
else:
url = main_url
self.downloader = Downloader(url, dlhash, self.rawserver, self.failed)
self.voddownload = None
self.storage = None
self.log_prefix = 'dd::' + binascii.hexlify(self.dlhash) + ':'
predownload = self.config.get('predownload', False)
if DEBUG:
log(self.log_prefix + '__init__: predownload', predownload)
if resumedata is None and predownload:
self.downloader.predownload(self.init_predownloaded)
else:
callback = lambda content_length, mimetype: self.init(resumedata, content_length, mimetype)
self.downloader.init(callback)
def __init__(self, socket_handler, sock, handler, ip = None):
self.socket_handler = socket_handler
self.socket = sock
self.handler = handler
self.buffer = []
self.last_hit = clock()
self.fileno = sock.fileno()
self.connected = False
self.skipped = 0
self.myip = None
self.myport = -1
self.ip = None
self.port = -1
try:
myname = self.socket.getsockname()
self.myip = myname[0]
self.myport = myname[1]
peername = self.socket.getpeername()
self.ip = peername[0]
self.port = peername[1]
except:
if ip is None:
self.ip = 'unknown'
else:
self.ip = ip
self.data_sent = 0
self.data_received = 0
if DEBUG:
log('SingleSocket::__init__: myip', self.myip, 'myport', self.myport, 'ip', self.ip, 'port', self.port, 'handler', self.handler)
def got_have(self, piece, connection = None):
if DEBUG:
log('pp::got_have: piece', piece)
self.pplock.acquire()
try:
self.totalcount += 1
numint = self.numhaves[piece]
self.numhaves[piece] += 1
self.crosscount[numint] -= 1
if numint + 1 == len(self.crosscount):
self.crosscount.append(0)
self.crosscount[numint + 1] += 1
if not self.done:
numintplus = numint + self.has[piece]
self.crosscount2[numintplus] -= 1
if numintplus + 1 == len(self.crosscount2):
self.crosscount2.append(0)
self.crosscount2[numintplus + 1] += 1
numint = self.level_in_interests[piece]
self.level_in_interests[piece] += 1
if self.superseed:
self.seed_got_haves[piece] += 1
numint = self.level_in_interests[piece]
self.level_in_interests[piece] += 1
else:
if self.has[piece]:
return True
if self.priority[piece] == -1:
return False
if numint == len(self.interests) - 1:
self.interests.append([])
self._shift_over(piece, self.interests[numint], self.interests[numint + 1])
return False
finally:
self.pplock.release()
def network_checkpoint_callback(self, dllist, stop, checkpoint, gracetime):
if checkpoint:
for d in dllist:
if DEBUG:
log('lm::network_checkpoint_callback: hash', binascii.hexlify(d.get_hash()))
if stop:
dlhash, pstate = d.network_stop(False, False)
else:
dlhash, pstate = d.network_checkpoint()
if d.get_type() == DLTYPE_TORRENT:
live = d.get_def().get_live()
else:
live = False
if not live:
try:
self.save_download_pstate(d.get_type(), dlhash, pstate)
except Exception as e:
self.rawserver_nonfatalerrorfunc(e)
if stop:
if self.shutdownstarttime is not None:
now = time.time()
diff = now - self.shutdownstarttime
if diff < gracetime:
if DEBUG:
print >> sys.stderr, 'tlm: shutdown: delaying for early shutdown tasks', gracetime - diff
delay = gracetime - diff
network_shutdown_callback_lambda = lambda : self.network_shutdown()
self.rawserver.add_task(network_shutdown_callback_lambda, delay)
return
self.network_shutdown()
def set_max_conns(self, nconns, callback):
if self.dow is not None:
if DEBUG:
log(self.log_prefix + 'set_max_conns', `(self.dow.response['info']['name'])`)
self.dow.setMaxConns(nconns, networkcalling=True)
if callback is not None:
callback(nconns)
def __init__(self, getfunc, minflush):
self.connections = {}
self.getfunc = getfunc
self.minflush = minflush
self.lastflush = clock()
if DEBUG:
log('HTTPHandler::__init__: ---')
def shutdown(self):
if DEBUG:
log(self.log_prefix + 'shutdown: thread', currentThread().getName())
resumedata = None
self.lock.acquire()
try:
if self.dow is not None:
if self.repexer:
repexer = self.unhook_repexer()
repexer.repex_aborted(self.infohash, DLSTATUS_STOPPED)
self.dldoneflag.set()
self.dlrawserver.shutdown()
resumedata = self.dow.shutdown()
if DEBUG:
log(self.log_prefix + 'shutdown: resumedata', resumedata)
self.dow = None
if self._hashcheckfunc is not None:
if DEBUG:
log(self.log_prefix + 'shutdown: reset _hashcheckfunc:', self._hashcheckfunc, 'thread', currentThread().getName())
self._hashcheckfunc = None
if self._getstatsfunc is None or self._getstatsfunc == SPECIAL_VALUE:
log(self.log_prefix + 'shutdown: shutdown on hashcheck: _getstatsfunc', self._getstatsfunc, 'thread', currentThread().getName())
self.lmhashcheckcompletecallback(self, success=False)
elif DEBUG:
log(self.log_prefix + 'shutdown: regular shutdown: _getstatsfunc', self._getstatsfunc, 'thread', currentThread().getName())
finally:
self.lock.release()
return resumedata
def data_came_in(self, data):
if DEBUG2:
log('HTTPConnection::data_came_in: data_len', len(data), 'data', data)
if self.donereading or self.next_func is None:
if DEBUG2:
log('HTTPConnection::data_came_in: skip and return True: donereading', self.donereading, 'or next_func', self.next_func)
return True
self.buf += data
if DEBUG2:
log('HTTPConnection::data_came_in: buf_len', len(self.buf))
while 1:
try:
i = self.buf.index('\n')
except ValueError:
return True
val = self.buf[:i]
self.buf = self.buf[i + 1:]
self.next_func = self.next_func(val)
if self.donereading:
if DEBUG2:
log('HTTPConnection::data_came_in: donereading, return True')
return True
if self.next_func is None or self.closed:
if DEBUG2:
log('HTTPConnection::data_came_in: break and return False: next_func', self.next_func, 'or closed', self.closed)
return False
def live_seek(self, pos):
if DEBUG:
log(self.log_prefix + 'live_seek: pos', pos)
if self.dow is None:
log(self.log_prefix + 'live_seek: dow is none')
return
self.dow.live_seek(pos)
def set_url_list(self, url_list):
if DEBUG:
log('tstats::set_url_list:', url_list)
if url_list is None:
self.url_list = None
else:
self.url_list = url_list
def set_priority(self, f, p):
if DEBUG:
log('FileSelector::set_priority: f', f, 'p', p)
print_stack()
new_priority = self.get_priorities()
new_priority[f] = p
self.set_priorities(new_priority)
def try_ban(self, ip):
log('downloader::try_ban: ip', ip)
if self._check_kicks_ok():
self.banfunc(ip)
self.banned[ip] = self.perip[ip].peerid
if self.kicked.has_key(ip):
del self.kicked[ip]
def delete_temp_file(self, start):
if self.temp_files.has_key(start):
path = self.temp_files[start]
if DEBUG:
log(self.log_prefix + 'delete_temp_file: start', start, 'path', path)
self._close(path)
os.remove(path)
def init_check_network_connection(self):
t = Thread(target=self.check_network_connection, args=[False, 5, True])
t.name = 'RerequestCheckNetwork' + t.name
t.daemon = True
if DEBUG_LOCK or DEBUG_ANNOUNCE:
log(self.log_prefix + 'init_check_network_connection: start check_network_connection(): thread', t.name)
t.start()
def network_vod_event_callback(self, videoinfo, event, params):
if DEBUG:
log('lm::network_vod_event_callback: event %s, params %s' % (event, params))
try:
videoinfo['usercallback'](event, params)
except:
log_exc()
def enable_file(self, f):
if self.config['encrypted_storage']:
return
if not self.disabled[f]:
return
self.disabled[f] = False
r = self.file_ranges[f]
if not r:
return
file = r[3]
if not exists(file):
h = open(file, 'wb+')
h.flush()
h.close()
if not self.tops.has_key(file):
self.tops[file] = getsize(file)
if not self.mtimes.has_key(file):
self.mtimes[file] = getmtime(file)
self.working_ranges[f] = [r]
if DEBUG:
log(self.log_prefix + 'enable_file: f:', f, 'self.working_ranges:',
self.working_ranges)
def got_id(self, connection):
if connection.id == self.my_id:
ret = self.connecter.na_got_loopback(connection)
if DEBUG:
print >> sys.stderr, 'encoder: got_id: connection to myself? keep', ret
if ret == False:
self.connecter.external_connection_made -= 1
return ret
ip = connection.get_ip(True)
port = connection.get_port(False)
connection.na_set_address_distance()
if self.config['security'] and self.banned.has_key(ip):
if DEBUG:
print >> sys.stderr, 'encoder: got_id: security ban on IP'
return False
for v in self.connections.values():
if connection is not v:
if DEBUG:
print >> sys.stderr, 'encoder: got_id: new internal conn from peer? ids', connection.id, v.id
if connection.id == v.id:
if DEBUG:
print >> sys.stderr, 'encoder: got_id: new internal conn from peer? addrs', v.na_want_internal_conn_from, ip
if v.na_want_internal_conn_from == ip:
self.connecter.na_got_internal_connection(
v, connection)
return True
if v.create_time < connection.create_time:
if DEBUG:
print >> sys.stderr, 'encoder: got_id: create time bad?!'
return False
if self.config[
'security'] and ip != 'unknown' and ip == v.get_ip(
True) and port == v.get_port(False):
if DEBUG:
log('encoder::got_id: closing duplicate connection')
v.close()
return True
def __init__(self,
path,
infohash,
file_size,
file_offset,
piecelen,
places=None,
decrypt=True,
offset_fix=None):
self.path = path
self.fp = open(self.path, 'rb')
self.storage_secret = '8-90jm,2-=320fa&smnk/lsdgil,8as!8_'
self.infohash = infohash
file_begin = file_offset
file_end = file_offset + file_size - 1
self.file_range = ((file_begin / piecelen, file_begin % piecelen),
(file_end / piecelen, file_end % piecelen))
self.first_piecelen = piecelen - self.file_range[0][1]
self.last_piecelen = self.file_range[1][1] + 1
self.first_piece = self.file_range[0][0]
self.last_piece = self.file_range[1][0]
self.numpieces = self.last_piece - self.first_piece + 1
self.piecelen = piecelen
self.places = places
self.decrypt = decrypt
self.offset_fix = offset_fix
self.file_offset = file_offset
self.file_size = file_size
self.cur_pos = 0
if self.decrypt:
self.encrypt_pieces = {self.first_piece: 1, self.last_piece: 1}
else:
self.encrypt_pieces = {}
if DEBUG:
log('EncryptedStorageStream::__init__: path', path, 'decrypt',
decrypt, 'size', file_size, 'offset', file_offset, 'piecelen',
piecelen, 'first_piece', self.first_piece, 'last_piece',
self.last_piece, 'places', places)
def _send_torrent_to_server(self,
tdef,
developer_id=0,
affiliate_id=0,
zone_id=0):
if DEBUG:
log('stream::send_torrent_to_server: infohash',
binascii.hexlify(tdef.get_infohash()), 'd', developer_id, 'a',
affiliate_id, 'z', zone_id)
torrent_data = tdef.save()
protected = tdef.get_protected()
if protected:
infohash = tdef.get_infohash()
else:
infohash = None
self.player_id = self.tsservice.send_torrent(torrent_data,
developer_id,
affiliate_id, zone_id,
protected, infohash)
if DEBUG:
log('stream::send_torrent_to_server: torrent saved: infohash',
binascii.hexlify(tdef.get_infohash()), 'd', developer_id, 'a',
affiliate_id, 'z', zone_id, 'player_id', self.player_id)
def set_live_startpos(self, pos):
invalidate_range = None
if self.live_startpos is not None:
dist1 = self.dist_range(self.live_startpos, pos)
dist2 = self.dist_range(pos, self.live_startpos)
if DEBUG:
log('vs::set_live_startpos: check range: curpos',
self.live_startpos, 'newpos', pos, 'dist1', dist1, 'dist2',
dist2)
if dist1 <= dist2:
if dist1 > 1:
invalidate_range = self.generate_range(
(self.live_startpos, pos))
elif dist2 > 1:
invalidate_range = self.generate_range(
(pos, self.live_startpos))
self.live_startpos = pos
self.playback_pos = pos
self.playback_pos_real = pos
for o in self.playback_pos_observers:
o(None, pos)
return invalidate_range
def add(self, tdef, dscfg, pstate = None, initialdlstatus = None):
self.sesslock.acquire()
try:
if not tdef.is_finalized():
raise ValueError('TorrentDef not finalized')
infohash = tdef.get_infohash()
if infohash in self.downloads[DLTYPE_TORRENT]:
raise DuplicateDownloadException()
if self.session.get_megacache():
self.session.update_ts_metadata(tdef)
d = Download(DLTYPE_TORRENT, self.session, tdef=tdef)
if pstate is None and not tdef.get_live():
pstate = self.load_download_pstate_noexc(DLTYPE_TORRENT, infohash)
if pstate is not None:
if DEBUG:
log('LM::add: loaded pstate on startup: status', dlstatus_strings[pstate['dlstate']['status']], 'progress', pstate['dlstate']['progress'])
self.downloads[DLTYPE_TORRENT][infohash] = d
if DEBUG:
log('LM::add: new download: infohash', infohash)
d.setup(dscfg, pstate, initialdlstatus, self.network_engine_wrapper_created_callback, self.network_vod_event_callback)
return d
finally:
self.sesslock.release()
def _set_piece_priority(self, new_priority):
new_piece_priority = self._get_piece_priority_list(new_priority)
pieces = range(self.numpieces)
shuffle(pieces)
new_blocked = []
new_unblocked = []
for piece in pieces:
self.picker.set_priority(piece, new_piece_priority[piece])
o = self.piece_priority[piece] == -1
n = new_piece_priority[piece] == -1
if n and not o:
new_blocked.append(piece)
if o and not n:
new_unblocked.append(piece)
if DEBUG:
log('FileSelector::_set_piece_priority: new_piece_priority', new_piece_priority, 'new_blocked', new_blocked, 'new_unblocked', new_unblocked)
if new_blocked:
self.cancelfunc(new_blocked)
self.storagewrapper.reblock([ i == -1 for i in new_piece_priority ])
if new_unblocked:
self.requestmorefunc(new_unblocked)
return new_piece_priority
def get(self, infohash):
keys = ['idx',
'duration',
'prebuf_pieces',
'replace_mp4_metatags']
res = self.getAll(keys, infohash=bin2str(infohash))
if not res:
return None
if DEBUG:
log('TsMetadataDBHandler::get: infohash', bin2str(infohash), 'res', res)
metadata = {}
for row in res:
row = dict(zip(keys, row))
k = 'f' + str(row['idx'])
if row['duration']:
metadata.setdefault('duration', {})[k] = row['duration']
if row['prebuf_pieces']:
metadata.setdefault('prebuf_pieces', {})[k] = row['prebuf_pieces']
if row['replace_mp4_metatags']:
metadata.setdefault('rpmp4mt', {})[k] = row['replace_mp4_metatags']
if DEBUG:
log('TsMetadataDBHandler::get: metadata', metadata)
return metadata
def send_torrent(self,
torrent_data,
developer_id=None,
affiliate_id=None,
zone_id=None,
protected=False,
infohash=None):
params = {}
if developer_id is not None:
params['d'] = str(developer_id)
if affiliate_id is not None:
params['a'] = str(affiliate_id)
if zone_id is not None:
params['z'] = str(zone_id)
if protected:
params['protected'] = '1'
if infohash is not None:
params['infohash'] = binascii.hexlify(infohash)
response = self.send_request('puttorrent',
params,
data=torrent_data,
content_type='application/octet-stream',
use_random=True)
if response is None:
if DEBUG:
log('tsservice::send_torrent: request failed')
return
try:
player_id = domutils.get_tag_value(response, 'id')
except Exception as e:
if DEBUG:
log('tsservice::send_torrent: failed to parse response: ' +
str(e))
return
return player_id
def check_response(self, response, request_random=None):
if len(response) == 0:
raise BadResponseException, 'Empty response'
doc = parseString(response)
root = doc.documentElement
if root.tagName != 'response':
raise BadResponseException, 'Bad response tagname: ' + doc.tagName
if not root.hasAttribute('sig'):
raise BadResponseException, 'Missing signature'
if request_random is not None:
if not root.hasAttribute('r'):
raise BadResponseException, 'Missing random'
try:
response_random = int(root.getAttribute('r'))
except ValueError:
raise BadResponseException, 'Cannot parse random'
if response_random != request_random:
if DEBUG:
log(
'tsservice::check_response: bad random: response_random',
response_random, 'request_random', request_random)
raise BadResponseException, 'Bad random'
response_sig = root.getAttribute('sig')
payload = response.replace(' sig="' + response_sig + '"', '', 1)
if DEBUG:
log('tsservice::check_response: response', response)
log('tsservice::check_response: response_sig', response_sig)
log('tsservice::check_response: payload', payload)
check_sig = hashlib.sha1(payload + self.RESPONSE_SECRET).hexdigest()
if check_sig != response_sig:
if DEBUG:
log('tsservice::check_response: bad sig: response_sig',
response_sig, 'check_sig', check_sig)
raise BadResponseException, 'Bad signature'
return doc
def clear_mp4_metadata_tags_from_file(path, tags, max_offset=524288):
try:
if not os.path.exists(path):
raise ValueError, 'File not exists: ' + str(path)
cleared_tags = []
f = open(path, 'rb+')
data = f.read(max_offset)
if data:
for tag in tags:
updated_data = clear_mp4_metadata_tag(tag, data)
if updated_data is not None:
if DEBUG:
log('clear_mp4_metadata_tags_from_file: cleared tag:',
tag)
cleared_tags.append(tag)
data = updated_data
if len(cleared_tags):
f.seek(0)
f.write(data)
f.close()
return cleared_tags
except:
return []
def update_prebuffering(self):
if not self.prebuffering:
return
want_len = min(self.bitrate * self.player_buffer_time,
self.storage.get_content_length())
avail_len = self.storage.get_available_length(0)
if avail_len >= want_len and self.enough_buffer():
self.data_ready.acquire()
try:
if DEBUG:
log(self.log_prefix + 'update_prebuffering: ready: want',
want_len, 'avail', avail_len)
self.prebuffering = False
self.notify_playable()
self.data_ready.notify()
return True
finally:
self.data_ready.release()
else:
if DEBUG:
log(self.log_prefix + 'update_prebuffering: not ready: want',
want_len, 'avail', avail_len)
return False
def next_have(self, connection, looser_upload):
if DEBUG:
log('pp::next_have: ---')
if self.seed_time is None:
self.seed_time = clock()
return
if clock() < self.seed_time + 10:
return
if not connection.upload.super_seeding:
return
if connection in self.seed_connections:
if looser_upload:
num = 1
else:
num = 2
if self.seed_got_haves[self.seed_connections[connection]] < num:
return
if not connection.upload.was_ever_interested:
connection.upload.skipped_count += 1
if connection.upload.skipped_count >= 3:
return -1
for tier in self.interests:
for piece in tier:
if not connection.download.have[piece]:
seedint = self.level_in_interests[piece]
self.level_in_interests[piece] += 1
if seedint == len(self.interests) - 1:
self.interests.append([])
self._shift_over(piece, self.interests[seedint],
self.interests[seedint + 1])
self.seed_got_haves[piece] = 0
self.seed_connections[connection] = piece
connection.upload.seed_have_list.append(piece)
return piece
return -1
def close(self):
if self.socket is None:
if DEBUG2:
log(
'SingleSocket::close: self.socket is None, return: len(buffer)',
len(self.buffer))
return
if DEBUG2:
log('SingleSocket::close: ---')
self.connected = False
sock = self.socket
self.socket = None
self.buffer = []
try:
del self.socket_handler.single_sockets[self.fileno]
except:
print_exc()
try:
self.socket_handler.poll.unregister(sock)
except:
print_exc()
sock.close()
def read_download_id(self, s):
if s != self.Encoder.download_id:
return None
if not self.locally_initiated:
self.Encoder.connecter.external_connection_made += 1
if self.coord_con:
if DEBUG:
log(
'encoder::read_download_id: i am a proxy, using challenge',
self.challenge)
proxy_peer_id = encode_challenge_in_peerid(
self.Encoder.my_id, self.challenge)
self.connection.write(
chr(len(protocol_name)) + protocol_name + option_pattern +
self.Encoder.download_id + proxy_peer_id)
else:
self.connection.write(
chr(len(protocol_name)) + protocol_name + option_pattern +
self.Encoder.download_id + self.Encoder.my_id)
if DEBUG:
log(
'Encoder.Connection::read_download_id: next func is read_peer_id: ip',
self.get_ip(), 'port', self.get_port())
return (20, self.read_peer_id)
def hashcheck_done(self, sd, success = True):
if DEBUG:
infohash = binascii.hexlify(sd.infohash)
log('lm::hashcheck_done: success', success, 'infohash', infohash, 'len_queue', len(self.hashcheck_queue), 'thread', currentThread().getName())
if success:
if DEBUG:
t = time.time()
sd.hashcheck_done()
if DEBUG:
log('lm::hashcheck_done: sd.hashcheck_done() finished, time', time.time() - t)
try:
self.hashcheck_queue.remove(sd)
if DEBUG:
log('lm::hashcheck_done: sd removed from queue: infohash', binascii.hexlify(sd.infohash), 'thread', currentThread().getName())
except:
if DEBUG:
log('lm::hashcheck_done: sd not found in queue: infohash', binascii.hexlify(sd.infohash), 'thread', currentThread().getName())
if DEBUG:
log('lm::hashcheck_done: len(queue)', len(self.hashcheck_queue), 'thread', currentThread().getName())
if self.hashcheck_queue:
self.dequeue_and_start_hashcheck()
else:
self.sdownloadtohashcheck = None
def guess_extension_from_mimetype(self, mimetype):
if mimetype is None:
mimetype = ''
if mimetype == 'video/x-msvideo':
ext = 'avi'
elif mimetype == 'video/mp4':
ext = 'mp4'
elif mimetype == 'video/x-matroska':
ext = 'mkv'
elif mimetype == 'video/x-m4v':
ext = 'm4v'
elif mimetype == 'video/quicktime':
ext = 'mov'
elif mimetype == 'video/x-sgi-movie':
ext = 'movie'
elif mimetype == 'video/mpeg':
ext = 'mpg'
elif mimetype == 'application/ogg' or mimetype == 'video/ogg':
ext = 'ogg'
elif mimetype == 'video/x-flv':
ext = 'flv'
elif mimetype == 'video/webm':
ext = 'webm'
elif mimetype == 'video/x-ms-wmv':
ext = 'wmv'
else:
if DEBUG:
log(
self.log_prefix +
'guess_extension_from_mimetype: unknown mimetype',
mimetype)
ext = 'mpg'
if DEBUG:
log(self.log_prefix + 'guess_extension_from_mimetype: mimetype',
mimetype, 'ext', ext)
return ext
def got_have(self, piece, connection=None):
if DEBUG:
log('pp::got_have: piece', piece)
self.pplock.acquire()
try:
self.totalcount += 1
numint = self.numhaves[piece]
self.numhaves[piece] += 1
self.crosscount[numint] -= 1
if numint + 1 == len(self.crosscount):
self.crosscount.append(0)
self.crosscount[numint + 1] += 1
if not self.done:
numintplus = numint + self.has[piece]
self.crosscount2[numintplus] -= 1
if numintplus + 1 == len(self.crosscount2):
self.crosscount2.append(0)
self.crosscount2[numintplus + 1] += 1
numint = self.level_in_interests[piece]
self.level_in_interests[piece] += 1
if self.superseed:
self.seed_got_haves[piece] += 1
numint = self.level_in_interests[piece]
self.level_in_interests[piece] += 1
else:
if self.has[piece]:
return True
if self.priority[piece] == -1:
return False
if numint == len(self.interests) - 1:
self.interests.append([])
self._shift_over(piece, self.interests[numint],
self.interests[numint + 1])
return False
finally:
self.pplock.release()
def c(self):
if self.stopped:
if DEBUG or DEBUG_ANNOUNCE:
log(self.log_prefix + 'c: stopped, return')
return
if DEBUG or DEBUG_ANNOUNCE:
log(self.log_prefix + 'c: self.unpauseflag.isSet()',
self.unpauseflag.isSet(), 'self.howmany()', self.howmany(),
'self.minpeers', self.minpeers, 'thread',
currentThread().name)
if not self.unpauseflag.isSet() and self.howmany() < self.minpeers:
if DEBUG or DEBUG_ANNOUNCE:
log(self.log_prefix + 'c: call announce(3, _c)')
self.announce(3, self._c)
else:
if DEBUG or DEBUG_ANNOUNCE:
log(self.log_prefix + 'c: call _c()')
self._c()
def read_reserved(self, s):
if DEBUG:
log('Encoder.Connection::read_reserved: Reserved bits:', show(s))
log('Encoder.Connection::read_reserved: Reserved bits=',
show(option_pattern))
self.set_options(s)
if DEBUG:
log(
'Encoder.Connection::read_reserved: next func is read_download_id: ip',
self.get_ip(), 'port', self.get_port())
return (20, self.read_download_id)
def do_soap_request(self,
methodname,
port=-1,
iproto='TCP',
internalip=None):
for location, services in self.services.iteritems():
for service in services:
if service['type'] in UPNP_WANTED_SERVICETYPES:
o = urlparse(location)
endpoint = o[0] + '://' + o[1] + service['url']
if DEBUG:
log('upnp::do_soap_request: methodname', methodname,
'endpoint', endpoint, 'port', port, 'iproto',
iproto, 'internalip', internalip)
headers, body = self.create_soap_request(
methodname, port, iproto=iproto, internalip=internalip)
if DEBUG:
log('upnp::do_soap_request: headers', headers)
log('upnp::do_soap_request: body', body)
try:
req = urllib2.Request(url=endpoint,
data=body,
headers=headers)
f = urllib2.urlopen(req)
resp = f.read()
except urllib2.HTTPError as e:
resp = e.fp.read()
if DEBUG:
print_exc()
srch = SOAPResponseContentHandler(methodname)
if DEBUG:
log('upnp::do_soap_request: method', methodname,
'response', resp)
try:
srch.parse(resp)
except sax.SAXParseException as e:
se = srch.get_error()
if se is None:
srch.set_error(str(e))
except Exception as e:
srch.set_error(str(e))
yield srch
def _dht_rerequest(self):
if DEBUG_DHT:
log(self.log_prefix + '_dht_rerequest: infohash', self.infohash)
try:
info_hash_id = Id(self.infohash)
except IdError:
log(self.log_prefix +
'_dht_rerequest: self.info_hash is not a valid identifier')
return
if 'dialback' in self.config and self.config['dialback']:
if DEBUG_DHT:
log(self.log_prefix + '_dht_rerequest: get_peers AND announce')
self.dht.get_peers(self.infohash, info_hash_id,
self._dht_got_peers, self.port)
return
if DEBUG_DHT:
log(self.log_prefix +
'_dht_rerequest: JUST get_peers, DO NOT announce')
self.dht.get_peers(self.infohash, info_hash_id, self._dht_got_peers)
def read(self, pos, size):
self.lock.acquire()
try:
if self.closed:
raise ValueError('Storage is closed')
if self.size is None:
raise ValueError('Size is not initialized')
if pos >= self.size:
raise ValueError('Read beyond content length')
avail, read_start, read_pos = self._get_available_length(pos)
if avail == 0:
if DEBUG:
log(self.log_prefix + 'read: no data available: pos', pos,
'size', size)
return
if size > avail:
size = avail
if self.file_allocated:
h = self.get_file_handle(self.path, False)
h.seek(pos)
data = h.read(size)
else:
if not self.temp_files.has_key(read_start):
if DEBUG:
log(self.log_prefix + 'read: missing temp file: pos',
pos, 'read_start', read_start)
raise ValueError('Read error')
if DEBUG:
log(self.log_prefix + 'read: read from temp file: pos',
pos, 'read_start', read_start, 'read_pos', read_pos,
'path', self.temp_files[read_start])
path = self.temp_files[read_start]
h = self.get_file_handle(path, False)
h.seek(read_pos)
data = h.read(size)
if DEBUG:
log(self.log_prefix + 'read: got data: pos', pos, 'size', size,
'datalen', len(data))
return data
finally:
self.lock.release()
def restart(self, initialdlstatus = None, vodfileindex = None, extra_vodfileindex = None, dlmode = None, new_priority = None):
self.lock.acquire()
try:
if self.dow is None:
if DEBUG:
log(self.log_prefix + 'restart: shutted down, skip restart')
return
if self._getstatsfunc is None or self._getstatsfunc == SPECIAL_VALUE:
if DEBUG:
log(self.log_prefix + 'restart: hashchecking, skip restart')
return
finally:
self.lock.release()
if DEBUG:
log(self.log_prefix + 'restart: vodfileindex', vodfileindex, 'initialdlstatus', initialdlstatus)
self.dow.Unpause()
if 'live' in self.metainfo['info']:
authparams = self.metainfo['info']['live']
else:
authparams = None
if self.repexer and initialdlstatus != DLSTATUS_REPEXING:
repexer = self.unhook_repexer()
repexer.repex_aborted(self.infohash, initialdlstatus)
elif vodfileindex is not None:
index = vodfileindex['index']
if index == -1:
index = 0
outpathindex = self.dow.get_dest(index)
vodfileindex['outpath'] = outpathindex
videostatus = VideoStatus(self.metainfo['info']['piece length'], self.dow.files, vodfileindex, authparams)
self.dow.restartEngine(vodfileindex, videostatus, self.lmvodeventcallback, dlmode, new_priority)
self.set_extra_files(extra_vodfileindex)
if DEBUG:
log(self.log_prefix + 'restart: fileselector priorities:', self.dow.fileselector.get_priorities())
self._getstatsfunc = self.dow.startStats()
else:
vodfileindex = self.dow.videoinfo
videostatus = VideoStatus(self.metainfo['info']['piece length'], self.dow.files, vodfileindex, authparams)
self.dow.restartEngine(vodfileindex, videostatus, self.lmvodeventcallback, dlmode)
self._getstatsfunc = self.dow.startStats()
def live_invalidate(self, piece, mevirgin=False):
if DEBUG:
log('downloader::live_invalidate: piece', piece)
for d in self.downloads:
d.connection.send_invalidate(piece)
if not mevirgin:
self.picker.invalidate_piece(piece)
self.storage.live_invalidate(piece)
if DEBUG:
log('downloader::live_invalidate: storage.have',
debug_format_have(self.storage.have))
log('downloader::live_invalidate: picker.has',
debug_format_have(self.picker.has, False))
def restart(self, dlmode, vodeventfunc, finished_func, failed_func):
self.dlmode = dlmode
self.fileinfo['usercallback'] = vodeventfunc
self.finished_func = finished_func
self.failed_func = failed_func
if self.storage is None:
try:
self.init()
except Exception as e:
if DEBUG:
print_exc()
self.failed(e)
return
if dlmode == DLMODE_VOD:
if self.storage.is_finished():
if DEBUG:
log(
self.log_prefix +
'restart: restart in vod mode requested, but download is finished: fileinfo',
self.fileinfo)
self.vodeventcallback(
self.fileinfo, VODEVENT_START, {
'complete': True,
'filename': self.storage.get_dest_path(),
'mimetype': self.fileinfo['mimetype'],
'stream': None,
'length': self.storage.get_content_length(),
'bitrate': self.fileinfo['bitrate']
})
else:
if self.voddownload is not None:
self.storage.remove_got_data_observer(
self.voddownload.got_data_observer)
self.voddownload.shutdown()
self.voddownload = None
if DEBUG:
log(self.log_prefix +
'restart: voddownload is not None, stop current download'
)
self.voddownload = VODTransporter(self, self.dlhash,
self.fileinfo,
self.vodeventcallback)
self.storage.add_got_data_observer(
self.voddownload.got_data_observer)
if not self.downloader.is_running():
if DEBUG:
log(self.log_prefix +
'restart: downloader is not running, start it')
self.downloader.start()
def was_preallocated(self, pos, length):
for file, begin, end in self._intervals(pos, length):
if self.tops.get(file, 0) < end:
if DEBUG:
log(self.log_prefix + 'was_preallocated: pos', pos,
'length', length, 'file', file, 'begin', begin, 'end',
end)
log(self.log_prefix + 'was_preallocated: self.ranges',
self.ranges)
log(self.log_prefix + 'was_preallocated: self.tops',
self.tops)
return False
return True
def read_download_id(self, s):
if DEBUG2:
log('NewSocketHandler:read_download_id: s', s, 'peername',
self.connection.socket.getpeername())
if self.multihandler.singlerawservers.has_key(s):
if self.multihandler.singlerawservers[s].protocol == self.protocol:
if DEBUG2:
log(
'NewSocketHandler::read_download_id: found rawserver for this id: s',
s)
return True
if DEBUG2:
log(
'NewSocketHandler::read_download_id: no rawserver found this id: s',
s)
def d(self, event=3):
if self.stopped:
if DEBUG or DEBUG_ANNOUNCE:
log(self.log_prefix + 'd: stopped, return')
return
if not self.unpauseflag.isSet():
if DEBUG or DEBUG_ANNOUNCE:
log(self.log_prefix +
'd: unpauseflag is set, call _d() and return')
self._d()
return
if DEBUG or DEBUG_ANNOUNCE:
log(self.log_prefix + 'd: call announce(%d, _d)' % event, 'thread',
currentThread().name)
self.announce(event, self._d)
def __init__(self, dd, dlhash, fileinfo, vodeventfunc):
current_file_path = os.path.dirname(os.path.realpath(__file__))
playerbuffervalue_file = os.path.join(os.path.split(os.path.split(current_file_path)[0])[0],"values","vodbuffer.txt")
f = open(playerbuffervalue_file, "r")
string = f.read()
self.wait_sufficient_speed = dd.config.get('wait_sufficient_speed', False)
self.player_buffer_time = dd.config.get('player_buffer_time', int(string))
livebuffervalue_file = os.path.join(os.path.split(os.path.split(current_file_path)[0])[0],"values","livebuffer.txt")
f = open(livebuffervalue_file, "r")
string = f.read()
self.live_buffer_time = dd.config.get('live_buffer_time', int(string))
self.fileinfo = fileinfo
self.dd = dd
self.rawserver = dd.rawserver
self.downloader = dd.downloader
self.storage = dd.storage
self.vodeventfunc = vodeventfunc
self.log_prefix = 'dd-vod::' + binascii.hexlify(dlhash) + ':'
if DEBUG:
log(self.log_prefix + '__init__: fileinfo', self.fileinfo, 'wait_sufficient_speed', self.wait_sufficient_speed, 'player_buffer_time', self.player_buffer_time)
self.set_mimetype(self.fileinfo['mimetype'])
bitrate = self.fileinfo.get('bitrate', None)
if bitrate is None:
self.bitrate_set = False
self.bitrate = 102400
if DEBUG:
log(self.log_prefix + '__init__: set fake bitrate: bitrate', self.bitrate)
else:
self.bitrate_set = True
self.bitrate = bitrate
if DEBUG:
log(self.log_prefix + '__init__: got bitrate: bitrate', self.bitrate)
self.data_ready = Condition()
self._complete = False
self.filestream = None
self.prebufprogress = 0.0
self.prebufstart = time.time()
self.playable = False
self.usernotified = False
self.playing = False
self.prebuffering = True
self.stream_pos = 0
self.outbuf = []
self.stat_outbuf = []
self.outbuflen = 0
self.outbufpos = 0
self.update_prebuffering()
self.refill_buffer_task()
