def get(self, player_id):
keys = [
'checksum', 'infohash', 'developer_id', 'affiliate_id', 'zone_id'
]
res = self.getOne(keys, player_id=player_id)
if res is None:
return
res = dict(zip(keys, res))
res['infohash'] = str2bin(res['infohash'])
res['checksum'] = str2bin(res['checksum'])
return res
def get(self, player_id):
keys = ['checksum',
'infohash',
'developer_id',
'affiliate_id',
'zone_id']
res = self.getOne(keys, player_id=player_id)
if res is None:
return
res = dict(zip(keys, res))
res['infohash'] = str2bin(res['infohash'])
res['checksum'] = str2bin(res['checksum'])
return res
def freeSpace(self, torrents2del):
sql = '\n select torrent_file_name, torrent_id, infohash, relevance,\n min(relevance,2500) + min(500,num_leechers) + 4*min(500,num_seeders) - (max(0,min(500,(%d-creation_date)/86400)) ) as weight\n from CollectedTorrent\n where torrent_id not in (select torrent_id from MyPreference)\n order by weight \n limit %d \n ' % (
int(time()), torrents2del)
res_list = self._db.fetchall(sql)
if len(res_list) == 0:
return False
sql_del_torrent = 'delete from Torrent where torrent_id=?'
sql_del_tracker = 'delete from TorrentTracker where torrent_id=?'
tids = [(torrent_id, ) for torrent_file_name, torrent_id, infohash,
relevance, weight in res_list]
self._db.executemany(sql_del_torrent, tids, commit=False)
self._db.executemany(sql_del_tracker, tids, commit=False)
self._db.commit()
torrent_dir = self.getTorrentDir()
deleted = 0
for torrent_file_name, torrent_id, infohash, relevance, weight in res_list:
torrent_path = os.path.join(torrent_dir, torrent_file_name)
try:
os.remove(torrent_path)
print >> sys.stderr, 'Erase torrent:', os.path.basename(
torrent_path)
deleted += 1
except Exception as msg:
pass
self.notifier.notify(NTFY_TORRENTS, NTFY_DELETE, str2bin(infohash))
return deleted
def valuelist2torrentlist(self, value_name, res_list, ranks, mypref_stats):
torrent_list = []
for item in res_list:
value_name[0] = 'torrent_id'
torrent = dict(zip(value_name, item))
try:
torrent['source'] = self.id2src[torrent['source_id']]
except:
print_exc()
torrent['source'] = 'http://some/RSS/feed'
torrent['category'] = [self.id2category[torrent['category_id']]]
torrent['status'] = self.id2status[torrent['status_id']]
torrent['simRank'] = ranksfind(ranks, torrent['infohash'])
torrent['infohash'] = str2bin(torrent['infohash'])
torrent['last_check_time'] = torrent['last_check']
del torrent['last_check']
del torrent['source_id']
del torrent['category_id']
del torrent['status_id']
torrent_id = torrent['torrent_id']
if mypref_stats is not None and torrent_id in mypref_stats:
torrent['myDownloadHistory'] = True
data = mypref_stats[torrent_id]
torrent['download_started'] = data[0]
torrent['progress'] = data[1]
torrent['destdir'] = data[2]
torrent_list.append(torrent)
return torrent_list
def selectTorrentToCheck(self, policy = 'random', infohash = None, return_value = None):
if infohash is None:
sql = 'select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check \n from CollectedTorrent T, TorrentTracker TT\n where TT.torrent_id=T.torrent_id and announce_tier=1 '
if policy.lower() == 'random':
ntorrents = self.getNumberCollectedTorrents()
if ntorrents == 0:
rand_pos = 0
else:
rand_pos = randint(0, ntorrents - 1)
last_check_threshold = int(time()) - 300
sql += 'and last_check < %d \n limit 1 offset %d ' % (last_check_threshold, rand_pos)
elif policy.lower() == 'oldest':
last_check_threshold = int(time()) - 300
sql += ' and last_check < %d and status_id <> 2\n order by last_check\n limit 1 ' % last_check_threshold
elif policy.lower() == 'popular':
last_check_threshold = int(time()) - 14400
sql += ' and last_check < %d and status_id <> 2 \n order by 3*num_seeders+num_leechers desc\n limit 1 ' % last_check_threshold
res = self._db.fetchone(sql)
else:
sql = 'select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check \n from CollectedTorrent T, TorrentTracker TT\n where TT.torrent_id=T.torrent_id and announce_tier=1\n and infohash=? \n '
infohash_str = bin2str(infohash)
res = self._db.fetchone(sql, (infohash_str,))
if res:
torrent_file_name = res[3]
torrent_dir = self.getTorrentDir()
torrent_path = os.path.join(torrent_dir, torrent_file_name)
if res is not None:
res = {'torrent_id': res[0],
'ignored_times': res[1],
'retried_times': res[2],
'torrent_path': torrent_path,
'infohash': str2bin(res[4])}
return_value['torrent'] = res
return_value['event'].set()
def valuelist2torrentlist(self, value_name, res_list, ranks, mypref_stats):
torrent_list = []
for item in res_list:
value_name[0] = 'torrent_id'
torrent = dict(zip(value_name, item))
try:
torrent['source'] = self.id2src[torrent['source_id']]
except:
print_exc()
torrent['source'] = 'http://some/RSS/feed'
torrent['category'] = [self.id2category[torrent['category_id']]]
torrent['status'] = self.id2status[torrent['status_id']]
torrent['simRank'] = ranksfind(ranks, torrent['infohash'])
torrent['infohash'] = str2bin(torrent['infohash'])
torrent['last_check_time'] = torrent['last_check']
del torrent['last_check']
del torrent['source_id']
del torrent['category_id']
del torrent['status_id']
torrent_id = torrent['torrent_id']
if mypref_stats is not None and torrent_id in mypref_stats:
torrent['myDownloadHistory'] = True
data = mypref_stats[torrent_id]
torrent['download_started'] = data[0]
torrent['progress'] = data[1]
torrent['destdir'] = data[2]
torrent_list.append(torrent)
return torrent_list
def getTorrent(self,
checksum=None,
infohash=None,
keys=None,
include_mypref=True):
if checksum is None and infohash is None:
return
if infohash is not None:
pass
if keys is None:
keys = deepcopy(self.value_name)
else:
keys = list(keys)
keys.append('infohash')
keys.append('checksum')
where = 'C.torrent_id = T.torrent_id and announce_tier=1 '
if checksum is not None:
res = self._db.getOne('CollectedTorrent C, TorrentTracker T',
keys,
where=where,
checksum=bin2str(checksum))
else:
res = self._db.getOne('CollectedTorrent C, TorrentTracker T',
keys,
where=where,
infohash=bin2str(infohash))
if not res:
return
if not isinstance(res, (tuple, list)):
res = (res, )
torrent = dict(zip(keys, res))
if 'source_id' in torrent:
torrent['source'] = self.id2src[torrent['source_id']]
del torrent['source_id']
if 'category_id' in torrent:
torrent['category'] = [self.id2category[torrent['category_id']]]
del torrent['category_id']
if 'status_id' in torrent:
torrent['status'] = self.id2status[torrent['status_id']]
del torrent['status_id']
torrent['checksum'] = str2bin(torrent['checksum'])
torrent['infohash'] = str2bin(torrent['infohash'])
if 'last_check' in torrent:
torrent['last_check_time'] = torrent['last_check']
del torrent['last_check']
return torrent
def getTorrent(self, checksum = None, infohash = None, keys = None, include_mypref = True):
if checksum is None and infohash is None:
return
if infohash is not None:
pass
if keys is None:
keys = deepcopy(self.value_name)
else:
keys = list(keys)
keys.append('infohash')
keys.append('checksum')
where = 'C.torrent_id = T.torrent_id and announce_tier=1 '
if checksum is not None:
res = self._db.getOne('CollectedTorrent C, TorrentTracker T', keys, where=where, checksum=bin2str(checksum))
else:
res = self._db.getOne('CollectedTorrent C, TorrentTracker T', keys, where=where, infohash=bin2str(infohash))
if not res:
return
if not isinstance(res, (tuple, list)):
res = (res,)
torrent = dict(zip(keys, res))
if 'source_id' in torrent:
torrent['source'] = self.id2src[torrent['source_id']]
del torrent['source_id']
if 'category_id' in torrent:
torrent['category'] = [self.id2category[torrent['category_id']]]
del torrent['category_id']
if 'status_id' in torrent:
torrent['status'] = self.id2status[torrent['status_id']]
del torrent['status_id']
torrent['checksum'] = str2bin(torrent['checksum'])
torrent['infohash'] = str2bin(torrent['infohash'])
if 'last_check' in torrent:
torrent['last_check_time'] = torrent['last_check']
del torrent['last_check']
return torrent
def selectTorrentToCheck(self,
policy='random',
infohash=None,
return_value=None):
if infohash is None:
sql = 'select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check \n from CollectedTorrent T, TorrentTracker TT\n where TT.torrent_id=T.torrent_id and announce_tier=1 '
if policy.lower() == 'random':
ntorrents = self.getNumberCollectedTorrents()
if ntorrents == 0:
rand_pos = 0
else:
rand_pos = randint(0, ntorrents - 1)
last_check_threshold = int(time()) - 300
sql += 'and last_check < %d \n limit 1 offset %d ' % (
last_check_threshold, rand_pos)
elif policy.lower() == 'oldest':
last_check_threshold = int(time()) - 300
sql += ' and last_check < %d and status_id <> 2\n order by last_check\n limit 1 ' % last_check_threshold
elif policy.lower() == 'popular':
last_check_threshold = int(time()) - 14400
sql += ' and last_check < %d and status_id <> 2 \n order by 3*num_seeders+num_leechers desc\n limit 1 ' % last_check_threshold
res = self._db.fetchone(sql)
else:
sql = 'select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check \n from CollectedTorrent T, TorrentTracker TT\n where TT.torrent_id=T.torrent_id and announce_tier=1\n and infohash=? \n '
infohash_str = bin2str(infohash)
res = self._db.fetchone(sql, (infohash_str, ))
if res:
torrent_file_name = res[3]
torrent_dir = self.getTorrentDir()
torrent_path = os.path.join(torrent_dir, torrent_file_name)
if res is not None:
res = {
'torrent_id': res[0],
'ignored_times': res[1],
'retried_times': res[2],
'torrent_path': torrent_path,
'infohash': str2bin(res[4])
}
return_value['torrent'] = res
return_value['event'].set()
def freeSpace(self, torrents2del):
sql = '\n select torrent_file_name, torrent_id, infohash, relevance,\n min(relevance,2500) + min(500,num_leechers) + 4*min(500,num_seeders) - (max(0,min(500,(%d-creation_date)/86400)) ) as weight\n from CollectedTorrent\n where torrent_id not in (select torrent_id from MyPreference)\n order by weight \n limit %d \n ' % (int(time()), torrents2del)
res_list = self._db.fetchall(sql)
if len(res_list) == 0:
return False
sql_del_torrent = 'delete from Torrent where torrent_id=?'
sql_del_tracker = 'delete from TorrentTracker where torrent_id=?'
tids = [ (torrent_id,) for torrent_file_name, torrent_id, infohash, relevance, weight in res_list ]
self._db.executemany(sql_del_torrent, tids, commit=False)
self._db.executemany(sql_del_tracker, tids, commit=False)
self._db.commit()
torrent_dir = self.getTorrentDir()
deleted = 0
for torrent_file_name, torrent_id, infohash, relevance, weight in res_list:
torrent_path = os.path.join(torrent_dir, torrent_file_name)
try:
os.remove(torrent_path)
print >> sys.stderr, 'Erase torrent:', os.path.basename(torrent_path)
deleted += 1
except Exception as msg:
pass
self.notifier.notify(NTFY_TORRENTS, NTFY_DELETE, str2bin(infohash))
return deleted
def get(self, adid):
infohash = self.getOne('infohash', adid=adid)
if infohash is None:
return
return str2bin(infohash)
def get(self, url):
urlhash = hashlib.sha1(url).hexdigest()
infohash = self.getOne('infohash', urlhash=urlhash)
if infohash is None:
return
return str2bin(infohash)
def getInfohashFromTorrentName(self, name):
sql = "select infohash from Torrent where name='" + str2bin(name) + "'"
infohash = self._db.fetchone(sql)
return infohash
def get(self, adid):
infohash = self.getOne('infohash', adid=adid)
if infohash is None:
return
return str2bin(infohash)
def get(self, url):
urlhash = hashlib.sha1(url).hexdigest()
infohash = self.getOne('infohash', urlhash=urlhash)
if infohash is None:
return
return str2bin(infohash)
def getInfohashFromTorrentName(self, name):
sql = "select infohash from Torrent where name='" + str2bin(name) + "'"
infohash = self._db.fetchone(sql)
return infohash
else:
torrent['retried_times'] += 1
torrent['ignored_times'] = torrent['retried_times']
elif torrent['status'] == 'dead':
if torrent['retried_times'] < self.retryThreshold:
torrent['retried_times'] += 1
def tooMuchRetry(self, torrent):
if torrent['retried_times'] > self.retryThreshold:
return True
return False
if __name__ == '__main__':
from freestream.Core.CacheDB.sqlitecachedb import init as init_db, str2bin
configure_dir = sys.argv[1]
config = {}
config['state_dir'] = configure_dir
config['install_dir'] = '.'
config['peer_icon_path'] = '.'
init_db(config)
t = TorrentChecking()
t.start()
t.join()
infohash_str = 'TkFX5S4qd2DPW63La/VObgOH/Nc='
infohash = str2bin(infohash_str)
del t
t = TorrentChecking(infohash)
t.start()
t.join()