def doRequest(self):
try:
while True:
infohash = self.queue.get_nowait()
if infohash in self.sources:
break
else:
self.queue.task_done()
try:
permid = choice(list(self.sources[infohash]))
self.sources[infohash].remove(permid)
if len(self.sources[infohash]) < 1:
del self.sources[infohash]
self.nr_times_requested[infohash] = self.nr_times_requested.get(infohash, 0) + 1
if DEBUG:
print >> sys.stderr, 'rtorrent: requesting', bin2str(infohash), bin2str(permid)
self.metadatahandler.send_metadata_request(permid, infohash, caller='rquery')
if self.prio <= 1 or infohash not in self.sources and infohash in self.nr_times_requested and self.nr_times_requested[infohash] > self.MAGNET_THRESHOLD:
self.overlay_bridge.add_task(lambda : self.magnet_requester.add_request(self.prio, infohash), self.MAGNET_TIMEOUT * (self.prio + 1), infohash)
except:
if DEBUG:
print_exc()
self.queue.task_done()
self.overlay_bridge.add_task(self.doRequest, self.REQUEST_INTERVAL * self.prio, self)
except Queue.Empty:
pass
def put(self, infohash, metadata, commit=True):
if DEBUG:
log('TsMetadataDBHandler::put: infohash', bin2str(infohash),
'metadata', metadata)
data = {}
if metadata.has_key('duration'):
for idx, duration in metadata['duration'].iteritems():
data.setdefault(idx, {})['duration'] = duration
if metadata.has_key('prebuf_pieces'):
for idx, prebuf_pieces in metadata['prebuf_pieces'].iteritems():
data.setdefault(idx, {})['prebuf_pieces'] = prebuf_pieces
if metadata.has_key('rpmp4mt'):
for idx, replace_mp4_metatags in metadata['rpmp4mt'].iteritems():
data.setdefault(
idx, {})['replace_mp4_metatags'] = replace_mp4_metatags
if DEBUG:
log('TsMetadataDBHandler::put: formatted data:', data)
for idx, meta in data.iteritems():
idx = int(idx.replace('f', ''))
self._db.insert_or_replace(self.table_name,
commit=commit,
infohash=bin2str(infohash),
idx=idx,
**meta)
def process_response(self, permid, d):
mypermid = self.session.get_permid()
self.friendshipStatistics_db.updateFriendshipResponseTime(bin2str(mypermid), bin2str(permid), int(time()))
fs = self.frienddb.getFriendState(permid)
if d['response'] == 1:
if fs == FS_I_INVITED:
self.frienddb.setFriendState(permid, commit=True, state=FS_MUTUAL)
elif fs != FS_MUTUAL:
self.frienddb.setFriendState(permid, commit=True, state=FS_HE_INVITED)
else:
self.frienddb.setFriendState(permid, commit=True, state=FS_HE_DENIED)
def hasTorrent(self, infohash, checksum = None):
if (infohash, checksum) in self.existed_torrents:
return True
else:
kw = {'infohash': bin2str(infohash)}
if checksum is not None:
kw['checksum'] = bin2str(checksum)
existed = self._db.getOne('CollectedTorrent', 'torrent_id', **kw)
if existed is None:
return False
self.existed_torrents.add((infohash, checksum))
return True
def hasTorrent(self, infohash, checksum=None):
if (infohash, checksum) in self.existed_torrents:
return True
else:
kw = {'infohash': bin2str(infohash)}
if checksum is not None:
kw['checksum'] = bin2str(checksum)
existed = self._db.getOne('CollectedTorrent', 'torrent_id', **kw)
if existed is None:
return False
self.existed_torrents.add((infohash, checksum))
return True
def fmsg_connect_callback(self, exc, dns, permid, selversion, type = None):
if exc is None:
if selversion < OLPROTO_VER_SEVENTH:
self.remove_msgs_for_ltv7_peer(permid)
return
sendlist = self.get_msgs_as_sendlist(targetpermid=permid)
if DEBUG:
print >> sys.stderr, 'friendship: fmsg_connect_callback: sendlist len', len(sendlist)
for i in range(0, len(sendlist)):
tuple = sendlist[i]
permid, msgid, msg = tuple
send_callback = lambda exc, permid: self.fmsg_send_callback(exc, permid, msgid)
if DEBUG:
print >> sys.stderr, 'friendship: fmsg_connect_callback: Sending', `msg`, msgid
mypermid = self.session.get_permid()
commit = i == len(sendlist) - 1
isForwarder = 0
no_of_helpers = 0
if type == F_FORWARD_MSG:
isForwarder = 1
no_of_helpers = 10
no_of_attempts = 0
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
if msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics(bin2str(mypermid), bin2str(permid), int(time()), isForwarder, no_of_attempts, no_of_helpers, commit=commit)
self.overlay_bridge.send(permid, FRIENDSHIP + bencode(msg), send_callback)
else:
if DEBUG:
peer = self.peerdb.getPeer(permid)
if peer is None:
print >> sys.stderr, 'friendship: Could not connect to peer', show_permid_short(permid), peer
else:
print >> sys.stderr, 'friendship: Could not connect to peer', show_permid_short(permid), peer['name']
print >> sys.stderr, exc
mypermid = self.session.get_permid()
isForwarder = 0
no_of_helpers = 0
if type == F_FORWARD_MSG:
isForwarder = 1
no_of_helpers = 10
no_of_attempts = 0
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
for msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics(bin2str(mypermid), bin2str(permid), int(time()), isForwarder, no_of_attempts, no_of_helpers)
def olthread_download_torrent_callback(self, permid, infohash, usercallback, prio = 1):
self.callbacks[infohash] = usercallback
requester = None
for i in range(prio):
if i in self.requesters and self.requesters[i].is_being_requested(infohash):
requester = self.requesters[i]
break
if not requester:
if prio not in self.requesters:
self.requesters[prio] = TorrentRequester(self, self.metadatahandler, self.overlay_bridge, self.session, prio)
requester = self.requesters[prio]
requester.add_source(infohash, permid)
if DEBUG:
print >> sys.stderr, 'rtorrent: adding request:', bin2str(infohash), bin2str(permid), prio
def _updateChannelcastDB(self, query_permid, query, hits, listOfAdditions):
if DEBUG:
print >> sys.stderr, 'channelcast: updating channelcastdb', query, len(
hits)
channel_ids = Set()
infohashes = Set()
for hit in listOfAdditions:
channel_ids.add(hit[0])
infohashes.add(hit[2])
my_favorites = self.votecastdb.getChannelsWithPosVote(
bin2str(self.my_permid))
for channel_id in my_favorites:
if channel_id in channel_ids:
self.updateAChannel(channel_id, [query_permid])
self.channelcastdb.on_torrents_from_channelcast(listOfAdditions)
missing_infohashes = {}
for channel_id in channel_ids:
for infohash in self.channelcastdb.selectTorrentsToCollect(
channel_id):
missing_infohashes[infohash] = channel_id
def notify(channel_id):
self.notifier.notify(NTFY_CHANNELCAST, NTFY_UPDATE, channel_id)
for infohash, channel_id in missing_infohashes.iteritems():
if infohash in infohashes:
self.rtorrent_handler.download_torrent(
query_permid, infohash,
lambda infohash, metadata, filename: notify(channel_id), 2)
else:
self.rtorrent_handler.download_torrent(
query_permid, infohash,
lambda infohash, metadata, filename: notify(channel_id), 3)
def put(self, url, infohash, commit=True):
urlhash = hashlib.sha1(url).hexdigest()
if self.getOne('infohash', urlhash=urlhash) is NULL:
self._db.insert(self.table_name,
commit=commit,
urlhash=urlhash,
url=url,
infohash=bin2str(infohash),
updated=long(time()))
else:
where = 'urlhash=' + repr(urlhash)
self._db.update(self.table_name,
where,
commit=commit,
infohash=bin2str(infohash),
updated=long(time()))
def _updateChannelcastDB(self, query_permid, query, hits, listOfAdditions):
if DEBUG:
print >> sys.stderr, 'channelcast: updating channelcastdb', query, len(hits)
channel_ids = Set()
infohashes = Set()
for hit in listOfAdditions:
channel_ids.add(hit[0])
infohashes.add(hit[2])
my_favorites = self.votecastdb.getChannelsWithPosVote(bin2str(self.my_permid))
for channel_id in my_favorites:
if channel_id in channel_ids:
self.updateAChannel(channel_id, [query_permid])
self.channelcastdb.on_torrents_from_channelcast(listOfAdditions)
missing_infohashes = {}
for channel_id in channel_ids:
for infohash in self.channelcastdb.selectTorrentsToCollect(channel_id):
missing_infohashes[infohash] = channel_id
def notify(channel_id):
self.notifier.notify(NTFY_CHANNELCAST, NTFY_UPDATE, channel_id)
for infohash, channel_id in missing_infohashes.iteritems():
if infohash in infohashes:
self.rtorrent_handler.download_torrent(query_permid, infohash, lambda infohash, metadata, filename: notify(channel_id), 2)
else:
self.rtorrent_handler.download_torrent(query_permid, infohash, lambda infohash, metadata, filename: notify(channel_id), 3)
def put(self, url, infohash, commit = True):
urlhash = hashlib.sha1(url).hexdigest()
if self.getOne('infohash', urlhash=urlhash) is NULL:
self._db.insert(self.table_name, commit=commit, urlhash=urlhash, url=url, infohash=bin2str(infohash), updated=long(time()))
else:
where = 'urlhash=' + repr(urlhash)
self._db.update(self.table_name, where, commit=commit, infohash=bin2str(infohash), updated=long(time()))
def getPlayerId(self, checksum, infohash, developer_id, affiliate_id,
zone_id):
if checksum is None:
player_id = self.getOne('player_id',
infohash=bin2str(infohash),
developer_id=developer_id,
affiliate_id=affiliate_id,
zone_id=zone_id)
else:
player_id = self.getOne('player_id',
checksum=bin2str(checksum),
infohash=bin2str(infohash),
developer_id=developer_id,
affiliate_id=affiliate_id,
zone_id=zone_id)
return player_id
def selectTorrentToCheck(self, policy = 'random', infohash = None, return_value = None):
if infohash is None:
sql = 'select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check \n from CollectedTorrent T, TorrentTracker TT\n where TT.torrent_id=T.torrent_id and announce_tier=1 '
if policy.lower() == 'random':
ntorrents = self.getNumberCollectedTorrents()
if ntorrents == 0:
rand_pos = 0
else:
rand_pos = randint(0, ntorrents - 1)
last_check_threshold = int(time()) - 300
sql += 'and last_check < %d \n limit 1 offset %d ' % (last_check_threshold, rand_pos)
elif policy.lower() == 'oldest':
last_check_threshold = int(time()) - 300
sql += ' and last_check < %d and status_id <> 2\n order by last_check\n limit 1 ' % last_check_threshold
elif policy.lower() == 'popular':
last_check_threshold = int(time()) - 14400
sql += ' and last_check < %d and status_id <> 2 \n order by 3*num_seeders+num_leechers desc\n limit 1 ' % last_check_threshold
res = self._db.fetchone(sql)
else:
sql = 'select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check \n from CollectedTorrent T, TorrentTracker TT\n where TT.torrent_id=T.torrent_id and announce_tier=1\n and infohash=? \n '
infohash_str = bin2str(infohash)
res = self._db.fetchone(sql, (infohash_str,))
if res:
torrent_file_name = res[3]
torrent_dir = self.getTorrentDir()
torrent_path = os.path.join(torrent_dir, torrent_file_name)
if res is not None:
res = {'torrent_id': res[0],
'ignored_times': res[1],
'retried_times': res[2],
'torrent_path': torrent_path,
'infohash': str2bin(res[4])}
return_value['torrent'] = res
return_value['event'].set()
def put(self,
player_id,
checksum,
infohash,
developer_id,
affiliate_id,
zone_id,
commit=True):
if self.getOne('infohash', player_id=player_id) is NULL:
self._db.insert(self.table_name,
commit=commit,
player_id=player_id,
checksum=bin2str(checksum),
infohash=bin2str(infohash),
developer_id=developer_id,
affiliate_id=affiliate_id,
zone_id=zone_id)
def getTorrent(self,
checksum=None,
infohash=None,
keys=None,
include_mypref=True):
if checksum is None and infohash is None:
return
if infohash is not None:
pass
if keys is None:
keys = deepcopy(self.value_name)
else:
keys = list(keys)
keys.append('infohash')
keys.append('checksum')
where = 'C.torrent_id = T.torrent_id and announce_tier=1 '
if checksum is not None:
res = self._db.getOne('CollectedTorrent C, TorrentTracker T',
keys,
where=where,
checksum=bin2str(checksum))
else:
res = self._db.getOne('CollectedTorrent C, TorrentTracker T',
keys,
where=where,
infohash=bin2str(infohash))
if not res:
return
if not isinstance(res, (tuple, list)):
res = (res, )
torrent = dict(zip(keys, res))
if 'source_id' in torrent:
torrent['source'] = self.id2src[torrent['source_id']]
del torrent['source_id']
if 'category_id' in torrent:
torrent['category'] = [self.id2category[torrent['category_id']]]
del torrent['category_id']
if 'status_id' in torrent:
torrent['status'] = self.id2status[torrent['status_id']]
del torrent['status_id']
torrent['checksum'] = str2bin(torrent['checksum'])
torrent['infohash'] = str2bin(torrent['infohash'])
if 'last_check' in torrent:
torrent['last_check_time'] = torrent['last_check']
del torrent['last_check']
return torrent
def addOrGetTorrentID(self, infohash):
torrent_id = self._db.getTorrentID(infohash)
if torrent_id is None:
self._db.insert('Torrent',
commit=True,
infohash=bin2str(infohash),
status_id=self._getStatusID('good'))
torrent_id = self._db.getTorrentID(infohash)
return torrent_id
def saveFriendshipStatistics(self, permid, currentTime, stats):
if stats:
for stat in stats:
if len(stat) == 7:
stat.append(0)
if len(stat) == 7 or len(stat) == 8:
stat.append(bin2str(permid))
self.friendshipStatistics_db.saveFriendshipStatisticData(stats)
def process_response(self, permid, d):
mypermid = self.session.get_permid()
self.friendshipStatistics_db.updateFriendshipResponseTime(
bin2str(mypermid), bin2str(permid), int(time()))
fs = self.frienddb.getFriendState(permid)
if d['response'] == 1:
if fs == FS_I_INVITED:
self.frienddb.setFriendState(permid,
commit=True,
state=FS_MUTUAL)
elif fs != FS_MUTUAL:
self.frienddb.setFriendState(permid,
commit=True,
state=FS_HE_INVITED)
else:
self.frienddb.setFriendState(permid,
commit=True,
state=FS_HE_DENIED)
def _get_database_dict(self, torrentdef, source='TS', extra_info={}):
mime, thumb = torrentdef.get_thumbnail()
checksum = extra_info.get('checksum', None)
if checksum is not None:
checksum = bin2str(checksum)
return {
'infohash':
bin2str(torrentdef.get_infohash()),
'checksum':
checksum,
'name':
torrentdef.get_name_as_unicode(),
'torrent_file_name':
extra_info.get('filename', None),
'length':
torrentdef.get_length(),
'creation_date':
torrentdef.get_creation_date(),
'num_files':
len(torrentdef.get_files()),
'thumbnail':
bool(thumb),
'insert_time':
long(time()),
'secret':
0,
'relevance':
0.0,
'source_id':
self._getSourceID(source),
'category_id':
self._getCategoryID(
self.category.calculateCategory(
torrentdef.metainfo, torrentdef.get_name_as_unicode())),
'status_id':
self._getStatusID(extra_info.get('status', 'unknown')),
'num_seeders':
extra_info.get('seeder', -1),
'num_leechers':
extra_info.get('leecher', -1),
'comment':
torrentdef.get_comment_as_unicode()
}
def log(self, permid, decoded_message):
lt = T.localtime(T.time())
timestamp = '%04d-%02d-%02d %02d:%02d:%02d' % (lt[0], lt[1], lt[2],
lt[3], lt[4], lt[5])
ip = self.peer_db.getPeer(permid, 'ip')
s = '%s\t%s\t%s\t%s\n' % (timestamp, bin2str(permid), ip,
decoded_message)
print dunno2unicode(s)
self.logfile.write(dunno2unicode(s))
self.logfile.flush()
def get_dns_from_peerdb(self, permid, use_cache = True):
if currentThread().getName().startswith('NetworkThread'):
print >> sys.stderr, 'secover: get_dns_from_peerdb: called by NetworkThread!'
print_stack()
dns = self.dns.get(permid, None)
if not dns:
values = ('ip', 'port')
peer = self.peer_db.getOne(values, permid=bin2str(permid))
if peer and peer[0] and peer[1]:
ip = hostname_or_ip2ip(peer[0])
dns = (ip, int(peer[1]))
return dns
def put(self, infohash, metadata, commit = True):
if DEBUG:
log('TsMetadataDBHandler::put: infohash', bin2str(infohash), 'metadata', metadata)
data = {}
if metadata.has_key('duration'):
for idx, duration in metadata['duration'].iteritems():
data.setdefault(idx, {})['duration'] = duration
if metadata.has_key('prebuf_pieces'):
for idx, prebuf_pieces in metadata['prebuf_pieces'].iteritems():
data.setdefault(idx, {})['prebuf_pieces'] = prebuf_pieces
if metadata.has_key('rpmp4mt'):
for idx, replace_mp4_metatags in metadata['rpmp4mt'].iteritems():
data.setdefault(idx, {})['replace_mp4_metatags'] = replace_mp4_metatags
if DEBUG:
log('TsMetadataDBHandler::put: formatted data:', data)
for idx, meta in data.iteritems():
idx = int(idx.replace('f', ''))
self._db.insert_or_replace(self.table_name, commit=commit, infohash=bin2str(infohash), idx=idx, **meta)
def put(self, adid, infohash, commit=True):
if self.getOne('infohash', adid=adid) is NULL:
self._db.insert(self.table_name,
commit=commit,
adid=adid,
infohash=bin2str(infohash),
last_seen=long(time()))
else:
where = 'adid=' + repr(str(adid))
self._db.update(self.table_name,
where,
commit=commit,
last_seen=long(time()))
def fmsg_send_callback(self, exc, permid, msgid):
if exc is None:
self.delete_msg(permid, msgid)
elif DEBUG:
print >> sys.stderr, 'friendship: Could not send to ', show_permid_short(
permid)
print_exc()
mypermid = self.session.get_permid()
no_of_attempts = 0
no_of_helpers = 10
isForwarder = False
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
for msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
if msgrec['forwarded'] == True:
isForwarder = 1
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics(
bin2str(mypermid), bin2str(permid), int(time()), isForwarder,
no_of_attempts, no_of_helpers)
def _get_database_dict(self, torrentdef, source = 'TS', extra_info = {}):
mime, thumb = torrentdef.get_thumbnail()
checksum = extra_info.get('checksum', None)
if checksum is not None:
checksum = bin2str(checksum)
return {'infohash': bin2str(torrentdef.get_infohash()),
'checksum': checksum,
'name': torrentdef.get_name_as_unicode(),
'torrent_file_name': extra_info.get('filename', None),
'length': torrentdef.get_length(),
'creation_date': torrentdef.get_creation_date(),
'num_files': len(torrentdef.get_files()),
'thumbnail': bool(thumb),
'insert_time': long(time()),
'secret': 0,
'relevance': 0.0,
'source_id': self._getSourceID(source),
'category_id': self._getCategoryID(self.category.calculateCategory(torrentdef.metainfo, torrentdef.get_name_as_unicode())),
'status_id': self._getStatusID(extra_info.get('status', 'unknown')),
'num_seeders': extra_info.get('seeder', -1),
'num_leechers': extra_info.get('leecher', -1),
'comment': torrentdef.get_comment_as_unicode()}
def addOrGetTorrentIDS(self, infohashes):
if len(infohashes) == 1:
return [self.addOrGetTorrentID(infohashes[0])]
to_be_inserted = []
torrent_ids = self._db.getTorrentIDS(infohashes)
for i in range(len(torrent_ids)):
torrent_id = torrent_ids[i]
if torrent_id is None:
to_be_inserted.append(infohashes[i])
status_id = self._getStatusID('good')
sql = 'INSERT OR IGNORE INTO Torrent (infohash, status_id) VALUES (?, ?)'
self._db.executemany(sql, [ (bin2str(infohash), status_id) for infohash in to_be_inserted ])
return self._db.getTorrentIDS(infohashes)
def addOrGetTorrentIDS(self, infohashes):
if len(infohashes) == 1:
return [self.addOrGetTorrentID(infohashes[0])]
to_be_inserted = []
torrent_ids = self._db.getTorrentIDS(infohashes)
for i in range(len(torrent_ids)):
torrent_id = torrent_ids[i]
if torrent_id is None:
to_be_inserted.append(infohashes[i])
status_id = self._getStatusID('good')
sql = 'INSERT OR IGNORE INTO Torrent (infohash, status_id) VALUES (?, ?)'
self._db.executemany(sql, [(bin2str(infohash), status_id)
for infohash in to_be_inserted])
return self._db.getTorrentIDS(infohashes)
def get(self, infohash):
keys = ['idx',
'duration',
'prebuf_pieces',
'replace_mp4_metatags']
res = self.getAll(keys, infohash=bin2str(infohash))
if not res:
return None
if DEBUG:
log('TsMetadataDBHandler::get: infohash', bin2str(infohash), 'res', res)
metadata = {}
for row in res:
row = dict(zip(keys, row))
k = 'f' + str(row['idx'])
if row['duration']:
metadata.setdefault('duration', {})[k] = row['duration']
if row['prebuf_pieces']:
metadata.setdefault('prebuf_pieces', {})[k] = row['prebuf_pieces']
if row['replace_mp4_metatags']:
metadata.setdefault('rpmp4mt', {})[k] = row['replace_mp4_metatags']
if DEBUG:
log('TsMetadataDBHandler::get: metadata', metadata)
return metadata
def __torrentdef_retrieved(self, tdef):
infohash = tdef.get_infohash()
if DEBUG:
print >> sys.stderr, 'magnetrequester: received torrent', bin2str(infohash)
if infohash in self.requestedInfohashes:
self.requestedInfohashes.remove(infohash)
torrent = self.torrent_db.getTorrent(infohash, ['torrent_file_name'], include_mypref=False)
if torrent.get('torrent_file_name', False):
torrent_filename = os.path.join(self.metadatahandler.torrent_dir, torrent['torrent_file_name'])
else:
torrent_filename = os.path.join(self.metadatahandler.torrent_dir, get_collected_torrent_filename(infohash))
tdef.save(torrent_filename)
self.torrent_db.addExternalTorrent(tdef)
self.remoteTorrentHandler.metadatahandler_got_torrent(infohash, tdef, torrent_filename)
self.overlay_bridge.add_task(self.__requestMagnet, self.REQUEST_INTERVAL)
def get(self, infohash):
keys = ['idx', 'duration', 'prebuf_pieces', 'replace_mp4_metatags']
res = self.getAll(keys, infohash=bin2str(infohash))
if not res:
return None
if DEBUG:
log('TsMetadataDBHandler::get: infohash', bin2str(infohash), 'res',
res)
metadata = {}
for row in res:
row = dict(zip(keys, row))
k = 'f' + str(row['idx'])
if row['duration']:
metadata.setdefault('duration', {})[k] = row['duration']
if row['prebuf_pieces']:
metadata.setdefault('prebuf_pieces',
{})[k] = row['prebuf_pieces']
if row['replace_mp4_metatags']:
metadata.setdefault('rpmp4mt',
{})[k] = row['replace_mp4_metatags']
if DEBUG:
log('TsMetadataDBHandler::get: metadata', metadata)
return metadata
def log(self, permid, decoded_message):
lt = T.localtime(T.time())
timestamp = '%04d-%02d-%02d %02d:%02d:%02d' % (lt[0],
lt[1],
lt[2],
lt[3],
lt[4],
lt[5])
ip = self.peer_db.getPeer(permid, 'ip')
s = '%s\t%s\t%s\t%s\n' % (timestamp,
bin2str(permid),
ip,
decoded_message)
print dunno2unicode(s)
self.logfile.write(dunno2unicode(s))
self.logfile.flush()
def fmsg_send_callback(self, exc, permid, msgid):
if exc is None:
self.delete_msg(permid, msgid)
elif DEBUG:
print >> sys.stderr, 'friendship: Could not send to ', show_permid_short(permid)
print_exc()
mypermid = self.session.get_permid()
no_of_attempts = 0
no_of_helpers = 10
isForwarder = False
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
for msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
if msgrec['forwarded'] == True:
isForwarder = 1
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics(bin2str(mypermid), bin2str(permid), int(time()), isForwarder, no_of_attempts, no_of_helpers)
def selectTorrentToCheck(self,
policy='random',
infohash=None,
return_value=None):
if infohash is None:
sql = 'select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check \n from CollectedTorrent T, TorrentTracker TT\n where TT.torrent_id=T.torrent_id and announce_tier=1 '
if policy.lower() == 'random':
ntorrents = self.getNumberCollectedTorrents()
if ntorrents == 0:
rand_pos = 0
else:
rand_pos = randint(0, ntorrents - 1)
last_check_threshold = int(time()) - 300
sql += 'and last_check < %d \n limit 1 offset %d ' % (
last_check_threshold, rand_pos)
elif policy.lower() == 'oldest':
last_check_threshold = int(time()) - 300
sql += ' and last_check < %d and status_id <> 2\n order by last_check\n limit 1 ' % last_check_threshold
elif policy.lower() == 'popular':
last_check_threshold = int(time()) - 14400
sql += ' and last_check < %d and status_id <> 2 \n order by 3*num_seeders+num_leechers desc\n limit 1 ' % last_check_threshold
res = self._db.fetchone(sql)
else:
sql = 'select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check \n from CollectedTorrent T, TorrentTracker TT\n where TT.torrent_id=T.torrent_id and announce_tier=1\n and infohash=? \n '
infohash_str = bin2str(infohash)
res = self._db.fetchone(sql, (infohash_str, ))
if res:
torrent_file_name = res[3]
torrent_dir = self.getTorrentDir()
torrent_path = os.path.join(torrent_dir, torrent_file_name)
if res is not None:
res = {
'torrent_id': res[0],
'ignored_times': res[1],
'retried_times': res[2],
'torrent_path': torrent_path,
'infohash': str2bin(res[4])
}
return_value['torrent'] = res
return_value['event'].set()
def __requestMagnet(self):
try:
if len(self.requestedInfohashes) < self.MAX_CONCURRENT:
while True:
if len(self.list) == 0:
return
prio, infohash = self.list.pop(0)
if infohash in self.requestedInfohashes:
if DEBUG:
print >> sys.stderr, 'magnetrequester: magnet already requested', bin2str(infohash)
continue
torrent = self.torrent_db.getTorrent(infohash, ['torrent_file_name'], include_mypref=False)
if torrent.get('torrent_file_name', False):
torrent_filename = os.path.join(self.metadatahandler.torrent_dir, torrent['torrent_file_name'])
else:
torrent_filename = None
torrent_alt_filename = os.path.join(self.metadatahandler.torrent_dir, get_collected_torrent_filename(infohash))
if os.path.isfile(torrent_filename) or os.path.isfile(torrent_alt_filename):
if DEBUG:
print >> sys.stderr, 'magnetrequester: magnet already on disk', bin2str(infohash)
else:
break
else:
return
except:
print_exc()
magnetlink = 'magnet:?xt=urn:btih:' + hexlify(infohash)
if DEBUG:
print >> sys.stderr, 'magnetrequester: requesting magnet', bin2str(infohash), prio, magnetlink
self.requestedInfohashes.add(infohash)
TorrentDef.retrieve_from_magnet(magnetlink, self.__torrentdef_retrieved, self.MAGNET_RETRIEVE_TIMEOUT)
self.overlay_bridge.add_task(lambda : self.__torrentdef_failed(infohash), self.MAGNET_RETRIEVE_TIMEOUT, infohash)
if len(self.requestedInfohashes) < self.MAX_CONCURRENT:
self.overlay_bridge.add_task(self.__requestMagnet, self.REQUEST_INTERVAL)
def updateTorrent(self, infohash, commit=True, **kw):
if 'category' in kw:
cat_id = self._getCategoryID(kw.pop('category'))
kw['category_id'] = cat_id
if 'status' in kw:
status_id = self._getStatusID(kw.pop('status'))
kw['status_id'] = status_id
if 'seeder' in kw:
kw['num_seeders'] = kw.pop('seeder')
if 'leecher' in kw:
kw['num_leechers'] = kw.pop('leecher')
if 'last_check_time' in kw or 'ignore_number' in kw or 'retry_number' in kw or 'retried_times' in kw or 'ignored_times' in kw:
self.updateTracker(infohash, kw, commit=False)
for key in kw.keys():
if key not in self.keys:
kw.pop(key)
if len(kw) > 0:
infohash_str = bin2str(infohash)
where = "infohash='%s'" % infohash_str
self._db.update(self.table_name, where, commit=False, **kw)
if commit:
self.commit()
self.notifier.notify(NTFY_TORRENTS, NTFY_UPDATE, infohash)
def updateTorrent(self, infohash, commit = True, **kw):
if 'category' in kw:
cat_id = self._getCategoryID(kw.pop('category'))
kw['category_id'] = cat_id
if 'status' in kw:
status_id = self._getStatusID(kw.pop('status'))
kw['status_id'] = status_id
if 'seeder' in kw:
kw['num_seeders'] = kw.pop('seeder')
if 'leecher' in kw:
kw['num_leechers'] = kw.pop('leecher')
if 'last_check_time' in kw or 'ignore_number' in kw or 'retry_number' in kw or 'retried_times' in kw or 'ignored_times' in kw:
self.updateTracker(infohash, kw, commit=False)
for key in kw.keys():
if key not in self.keys:
kw.pop(key)
if len(kw) > 0:
infohash_str = bin2str(infohash)
where = "infohash='%s'" % infohash_str
self._db.update(self.table_name, where, commit=False, **kw)
if commit:
self.commit()
self.notifier.notify(NTFY_TORRENTS, NTFY_UPDATE, infohash)
def put(self, adid, infohash, commit = True):
if self.getOne('infohash', adid=adid) is NULL:
self._db.insert(self.table_name, commit=commit, adid=adid, infohash=bin2str(infohash), last_seen=long(time()))
else:
where = 'adid=' + repr(str(adid))
self._db.update(self.table_name, where, commit=commit, last_seen=long(time()))
def addOrGetTorrentID(self, infohash):
torrent_id = self._db.getTorrentID(infohash)
if torrent_id is None:
self._db.insert('Torrent', commit=True, infohash=bin2str(infohash), status_id=self._getStatusID('good'))
torrent_id = self._db.getTorrentID(infohash)
return torrent_id
def updateChannel(self, query_permid, query, hits):
if DEBUG:
print >> sys.stderr, 'channelcast: sending message to', bin2str(query_permid), query, len(hits)
return self._updateChannelInternal(query_permid, query, hits)
def updateChannel(self, query_permid, query, hits):
if DEBUG:
print >> sys.stderr, 'channelcast: sending message to', bin2str(
query_permid), query, len(hits)
return self._updateChannelInternal(query_permid, query, hits)
def getTorrent(self, checksum = None, infohash = None, keys = None, include_mypref = True):
if checksum is None and infohash is None:
return
if infohash is not None:
pass
if keys is None:
keys = deepcopy(self.value_name)
else:
keys = list(keys)
keys.append('infohash')
keys.append('checksum')
where = 'C.torrent_id = T.torrent_id and announce_tier=1 '
if checksum is not None:
res = self._db.getOne('CollectedTorrent C, TorrentTracker T', keys, where=where, checksum=bin2str(checksum))
else:
res = self._db.getOne('CollectedTorrent C, TorrentTracker T', keys, where=where, infohash=bin2str(infohash))
if not res:
return
if not isinstance(res, (tuple, list)):
res = (res,)
torrent = dict(zip(keys, res))
if 'source_id' in torrent:
torrent['source'] = self.id2src[torrent['source_id']]
del torrent['source_id']
if 'category_id' in torrent:
torrent['category'] = [self.id2category[torrent['category_id']]]
del torrent['category_id']
if 'status_id' in torrent:
torrent['status'] = self.id2status[torrent['status_id']]
del torrent['status_id']
torrent['checksum'] = str2bin(torrent['checksum'])
torrent['infohash'] = str2bin(torrent['infohash'])
if 'last_check' in torrent:
torrent['last_check_time'] = torrent['last_check']
del torrent['last_check']
return torrent
def getPlayerId(self, checksum, infohash, developer_id, affiliate_id, zone_id):
if checksum is None:
player_id = self.getOne('player_id', infohash=bin2str(infohash), developer_id=developer_id, affiliate_id=affiliate_id, zone_id=zone_id)
else:
player_id = self.getOne('player_id', checksum=bin2str(checksum), infohash=bin2str(infohash), developer_id=developer_id, affiliate_id=affiliate_id, zone_id=zone_id)
return player_id
def get_last_seen(self, infohash):
last_seen = self.getOne('last_seen', infohash=bin2str(infohash))
return last_seen
def get_last_seen(self, infohash):
last_seen = self.getOne('last_seen', infohash=bin2str(infohash))
return last_seen
def addInfohash(self, infohash, commit=True):
if self._db.getTorrentID(infohash) is None:
self._db.insert('Torrent',
commit=commit,
infohash=bin2str(infohash))
def fmsg_connect_callback(self, exc, dns, permid, selversion, type=None):
if exc is None:
if selversion < OLPROTO_VER_SEVENTH:
self.remove_msgs_for_ltv7_peer(permid)
return
sendlist = self.get_msgs_as_sendlist(targetpermid=permid)
if DEBUG:
print >> sys.stderr, 'friendship: fmsg_connect_callback: sendlist len', len(
sendlist)
for i in range(0, len(sendlist)):
tuple = sendlist[i]
permid, msgid, msg = tuple
send_callback = lambda exc, permid: self.fmsg_send_callback(
exc, permid, msgid)
if DEBUG:
print >> sys.stderr, 'friendship: fmsg_connect_callback: Sending', ` msg `, msgid
mypermid = self.session.get_permid()
commit = i == len(sendlist) - 1
isForwarder = 0
no_of_helpers = 0
if type == F_FORWARD_MSG:
isForwarder = 1
no_of_helpers = 10
no_of_attempts = 0
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
if msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics(
bin2str(mypermid),
bin2str(permid),
int(time()),
isForwarder,
no_of_attempts,
no_of_helpers,
commit=commit)
self.overlay_bridge.send(permid, FRIENDSHIP + bencode(msg),
send_callback)
else:
if DEBUG:
peer = self.peerdb.getPeer(permid)
if peer is None:
print >> sys.stderr, 'friendship: Could not connect to peer', show_permid_short(
permid), peer
else:
print >> sys.stderr, 'friendship: Could not connect to peer', show_permid_short(
permid), peer['name']
print >> sys.stderr, exc
mypermid = self.session.get_permid()
isForwarder = 0
no_of_helpers = 0
if type == F_FORWARD_MSG:
isForwarder = 1
no_of_helpers = 10
no_of_attempts = 0
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
for msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics(
bin2str(mypermid), bin2str(permid), int(time()), isForwarder,
no_of_attempts, no_of_helpers)
def put(self, player_id, checksum, infohash, developer_id, affiliate_id, zone_id, commit = True):
if self.getOne('infohash', player_id=player_id) is NULL:
self._db.insert(self.table_name, commit=commit, player_id=player_id, checksum=bin2str(checksum), infohash=bin2str(infohash), developer_id=developer_id, affiliate_id=affiliate_id, zone_id=zone_id)
def addInfohash(self, infohash, commit = True):
if self._db.getTorrentID(infohash) is None:
self._db.insert('Torrent', commit=commit, infohash=bin2str(infohash))
