def __init__(self, install_dir='.'):
if Category.__single:
raise RuntimeError, 'Category is singleton'
filename = os.path.join(install_dir, 'data', 'category', category_file)
Category.__single = self
self.utility = None
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >> sys.stderr, 'category: Categories defined by user', self.getCategoryNames(
)
def __init__(self, install_dir='.'):
if Category.__single:
raise RuntimeError, "Category is singleton"
filename = os.path.join(install_dir,LIBRARYNAME, 'Category', category_file)
Category.__single = self
self.utility = None
#self.torrent_db = TorrentDBHandler.getInstance() # Arno, 2009-01-30: apparently unused
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >>sys.stderr,"category: Categories defined by user",self.getCategoryNames()
def __init__(self, install_dir='.', ffEnabled=False):
if Category.__single:
raise RuntimeError("Category is singleton")
filename = os.path.join(install_dir, LIBRARYNAME, 'Category',
category_file)
Category.__single = self
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >> sys.stderr, "category: Categories defined by user", self.getCategoryNames(
)
self.ffEnabled = ffEnabled
self.set_family_filter(None)
def __init__(self, install_dir = '.'):
if Category.__single:
raise RuntimeError, 'Category is singleton'
filename = os.path.join(install_dir, 'data', 'category', category_file)
Category.__single = self
self.utility = None
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >> sys.stderr, 'category: Categories defined by user', self.getCategoryNames()
def __init__(self, install_dir='.'):
if Category.__single:
raise RuntimeError, "Category is singleton"
filename = os.path.join(install_dir,LIBRARYNAME, 'Category', category_file)
Category.__single = self
self.utility = None
#self.torrent_db = TorrentDBHandler.getInstance() # Arno, 2009-01-30: apparently unused
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >>sys.stderr,"category: Categories defined by user",self.getCategoryNames()
def __init__(self, install_dir='.', ffEnabled=False):
if Category.__single:
raise RuntimeError("Category is singleton")
filename = os.path.join(install_dir, LIBRARYNAME, 'Category', category_file)
Category.__single = self
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >> sys.stderr, "category: Categories defined by user", self.getCategoryNames()
self.ffEnabled = ffEnabled
self.set_family_filter(None)
class Category:
# Code to make this a singleton
__single = None
__size_change = 1024 * 1024
def __init__(self, install_dir='.'):
if Category.__single:
raise RuntimeError, "Category is singleton"
filename = os.path.join(install_dir,LIBRARYNAME, 'Category', category_file)
Category.__single = self
self.utility = None
#self.torrent_db = TorrentDBHandler.getInstance() # Arno, 2009-01-30: apparently unused
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >>sys.stderr,"category: Categories defined by user",self.getCategoryNames()
# return Category instance
def getInstance(*args, **kw):
if Category.__single is None:
Category(*args, **kw)
return Category.__single
getInstance = staticmethod(getInstance)
def register(self,metadata_handler):
self.metadata_handler = metadata_handler
def init_from_main(self, utility):
self.utility = utility
self.set_family_filter(None) # init family filter to saved state
"""
# check to see whether need to resort torrent file
# return bool
def checkResort(self, data_manager):
data = data_manager.data
#===============================================================================
# if not data:
# data = data_manager.torrent_db.getRecommendedTorrents(all = True)
#===============================================================================
if not data:
return False
# data = data_manager.torrent_db.getRecommendedTorrents(all = True)
# self.reSortAll(data)
# return True
torrent = data[0]
if torrent["category"] == ["?"]:
#data = data_manager.torrent_db.getRecommendedTorrents(all = True)
self.reSortAll(data)
# del data
return True
begin = time()
for item in data:
if len(item['category']) > 1:
#data = data_manager.torrent_db.getRecommendedTorrents(all = True)
self.reSortAll(data)
# del data
return True
if DEBUG:
print >>sys.stderr,'torrcoll: Checking of %d torrents costs: %f s' % (len(data), time() - begin)
return False
# recalculate category of all torrents, remove torrents from db if not existed
def reSortAll(self, data, parent = None):
max = len(data)
if max == 0:
return
import wx
dlgHolder = []
event = Event()
def makeDialog():
dlg = wx.ProgressDialog("Upgrading Database",
"Upgrading Old Database to New Database",
maximum = max,
parent = None,
style = wx.PD_AUTO_HIDE
| wx.PD_ELAPSED_TIME
| wx.PD_REMAINING_TIME
)
dlgHolder.append(dlg)
event.set()
wx.CallAfter(makeDialog)
# Wait for dialog to be ready
event.wait()
dlg = dlgHolder[0]
count = 0
step = int(float(max) / 20) + 1
# sort each torrent file
for i in xrange(len(data)):
count += 1
if count % step == 0:
wx.CallAfter(dlg.Update, [count])
try:
# try alternative dir if bsddb doesnt match with current Tribler install
rec = data[i]
(torrent_dir,torrent_name) = self.metadata_handler.get_std_torrent_dir_name(rec)
# read the torrent file
filesrc = os.path.join(torrent_dir,torrent_name)
# print filesrc
f = open(filesrc, "rb")
torrentdata = f.read() # torrent decoded string
f.close()
except IOError: # torrent file not found
# delete the info from db
self.torrent_db.deleteTorrent(data[i]['infohash'])
continue
# decode the data
torrent_dict = bencode.bdecode(torrentdata)
content_name = dunno2unicode(torrent_dict["info"].get('name', '?'))
category_belong = []
category_belong = self.calculateCategory(torrent_dict, content_name)
if (category_belong == []):
category_belong = ['other']
data[i]['category'] = category_belong # should have updated self.data
self.torrent_db.updateTorrent(data[i]['infohash'], updateFlag=False, category=category_belong)
self.torrent_db.sync()
wx.CallAfter(dlg.Destroy)
"""
def getCategoryKeys(self):
if self.category_info is None:
return []
keys = []
keys.append("All")
keys.append("other")
for category in self.category_info:
keys.append(category['name'])
keys.sort()
return keys
def getCategoryNames(self):
if self.category_info is None:
return []
keys = []
for category in self.category_info:
rank = category['rank']
if rank == -1:
break
keys.append((category['name'],category['displayname']))
return keys
def hasActiveCategory(self, torrent):
try:
name = torrent['category'][0]
except:
print >> sys.stderr, 'Torrent: %s has no valid category' % `torrent['content_name']`
return False
for category in [{'name':'other', 'rank':1}]+self.category_info:
rank = category['rank']
if rank == -1:
break
if name.lower() == category['name'].lower():
return True
#print >> sys.stderr, 'Category: %s was not in %s' % (name.lower(), [a['name'].lower() for a in self.category_info if a['rank'] != -1])
return False
def getCategoryRank(self,cat):
for category in self.category_info:
if category['name'] == cat:
return category['rank']
return None
# calculate the category for a given torrent_dict of a torrent file
# return list
def calculateCategory(self, torrent_dict, display_name):
# torrent_dict is the dict of
# a torrent file
# return value: list of category the torrent belongs to
torrent_category = None
files_list = []
try:
# the multi-files mode
for ifiles in torrent_dict['info']["files"]:
files_list.append((ifiles['path'][-1], ifiles['length'] / float(self.__size_change)))
except KeyError:
# single mode
files_list.append((torrent_dict['info']["name"],torrent_dict['info']['length'] / float(self.__size_change)))
# Check xxx
try:
tracker = torrent_dict.get('announce')
if not tracker:
tracker = torrent_dict.get('announce-list',[['']])[0][0]
if self.xxx_filter.isXXXTorrent(files_list, display_name, torrent_dict.get('announce'), torrent_dict.get('comment')):
return ['xxx']
except:
print >> sys.stderr, 'Category: Exception in explicit terms filter in torrent: %s' % torrent_dict
print_exc()
# filename_list ready
strongest_cat = 0.0
for category in self.category_info: # for each category
(decision, strength) = self.judge(category, files_list, display_name)
if decision and (strength > strongest_cat):
torrent_category = [category['name']]
strongest_cat = strength
if torrent_category == None:
torrent_category = ['other']
return torrent_category
# judge whether a torrent file belongs to a certain category
# return bool
def judge(self, category, files_list, display_name = ''):
# judge file keywords
display_name = display_name.lower()
factor = 1.0
fileKeywords = self._getWords(display_name)
for ikeywords in category['keywords'].keys():
try:
fileKeywords.index(ikeywords)
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if (1 - factor) > 0.5:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, (1- factor))
# judge each file
matchSize = 0
totalSize = 1e-19
for name, length in files_list:
totalSize += length
# judge file size
if ( length < category['minfilesize'] ) or \
(category['maxfilesize'] > 0 and length > category['maxfilesize'] ):
continue
# judge file suffix
OK = False
for isuffix in category['suffix']:
if name.lower().endswith( isuffix ):
OK = True
break
if OK:
matchSize += length
continue
# judge file keywords
factor = 1.0
fileKeywords = self._getWords(name.lower())
for ikeywords in category['keywords'].keys():
# pass
try:
fileKeywords.index(ikeywords)
#print ikeywords
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if factor < 0.5:
# print filename_list[index] + '#######################'
matchSize += length
# match file
if (matchSize / totalSize) >= category['matchpercentage']:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, (matchSize/ totalSize))
return (False, 0)
WORDS_REGEXP = re.compile('[a-zA-Z0-9]+')
def _getWords(self, string):
return self.WORDS_REGEXP.findall(string)
def family_filter_enabled(self):
"""
Return is xxx filtering is enabled in this client
"""
if self.utility is None:
return False
state = self.utility.config.Read('family_filter')
if state in ('1', '0'):
return state == '1'
else:
self.utility.config.Write('family_filter', '1')
self.utility.config.Flush()
return True
def set_family_filter(self, b=None):
assert b in (True, False, None)
old = self.family_filter_enabled()
if b != old or b is None: # update category data if initial call, or if state changes
if b is None:
b=old
if self.utility is None:
return
#print >> sys.stderr , b
if b:
self.utility.config.Write('family_filter', '1')
else:
self.utility.config.Write('family_filter', '0')
self.utility.config.Flush()
# change category data
for category in self.category_info:
if category['name'] == 'xxx':
if b:
category['old-rank'] = category['rank']
category['rank'] = -1
elif category['rank'] == -1:
category['rank'] = category['old-rank']
break
def get_family_filter_sql(self, _getCategoryID, table_name=''):
if self.family_filter_enabled():
forbiddencats = [cat['name'] for cat in self.category_info if cat['rank'] == -1]
if table_name:
table_name+='.'
if forbiddencats:
return " and %scategory_id not in (%s)" % (table_name, ','.join([str(_getCategoryID([cat])) for cat in forbiddencats]))
return ''
class Category():
__single = None
__size_change = 1048576
def __init__(self, install_dir='.'):
if Category.__single:
raise RuntimeError, 'Category is singleton'
filename = os.path.join(install_dir, 'data', 'category', category_file)
Category.__single = self
self.utility = None
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >> sys.stderr, 'category: Categories defined by user', self.getCategoryNames(
)
def getInstance(*args, **kw):
if Category.__single is None:
Category(*args, **kw)
return Category.__single
getInstance = staticmethod(getInstance)
def register(self, metadata_handler):
self.metadata_handler = metadata_handler
def init_from_main(self, utility):
self.utility = utility
self.set_family_filter(None)
def getCategoryKeys(self):
if self.category_info is None:
return []
keys = []
keys.append('All')
keys.append('other')
for category in self.category_info:
keys.append(category['name'])
keys.sort()
return keys
def getCategoryNames(self):
if self.category_info is None:
return []
keys = []
for category in self.category_info:
rank = category['rank']
if rank == -1:
break
keys.append((category['name'], category['displayname']))
return keys
def hasActiveCategory(self, torrent):
try:
name = torrent['category'][0]
except:
print >> sys.stderr, 'Torrent: %s has no valid category' % ` (
torrent['content_name']) `
return False
for category in [{'name': 'other', 'rank': 1}] + self.category_info:
rank = category['rank']
if rank == -1:
break
if name.lower() == category['name'].lower():
return True
return False
def getCategoryRank(self, cat):
for category in self.category_info:
if category['name'] == cat:
return category['rank']
def calculateCategory(self, torrent_dict, display_name):
torrent_category = None
files_list = []
try:
for ifiles in torrent_dict['info']['files']:
files_list.append(
(ifiles['path'][-1],
ifiles['length'] / float(self.__size_change)))
except KeyError:
files_list.append(
(torrent_dict['info']['name'],
torrent_dict['info']['length'] / float(self.__size_change)))
try:
tracker = torrent_dict.get('announce')
if not tracker:
tracker = torrent_dict.get('announce-list', [['']])[0][0]
if self.xxx_filter.isXXXTorrent(files_list, display_name,
torrent_dict.get('announce'),
torrent_dict.get('comment')):
return ['xxx']
except:
print >> sys.stderr, 'Category: Exception in explicit terms filter in torrent: %s' % torrent_dict
print_exc()
strongest_cat = 0.0
for category in self.category_info:
decision, strength = self.judge(category, files_list, display_name)
if decision and strength > strongest_cat:
torrent_category = [category['name']]
strongest_cat = strength
if torrent_category == None:
torrent_category = ['other']
return torrent_category
def judge(self, category, files_list, display_name=''):
display_name = display_name.lower()
factor = 1.0
fileKeywords = self._getWords(display_name)
for ikeywords in category['keywords'].keys():
try:
fileKeywords.index(ikeywords)
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if 1 - factor > 0.5:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, 1 - factor)
matchSize = 0
totalSize = 1e-19
for name, length in files_list:
totalSize += length
if length < category['minfilesize'] or category[
'maxfilesize'] > 0 and length > category['maxfilesize']:
continue
OK = False
for isuffix in category['suffix']:
if name.lower().endswith(isuffix):
OK = True
break
if OK:
matchSize += length
continue
factor = 1.0
fileKeywords = self._getWords(name.lower())
for ikeywords in category['keywords'].keys():
try:
fileKeywords.index(ikeywords)
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if factor < 0.5:
matchSize += length
if matchSize / totalSize >= category['matchpercentage']:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, matchSize / totalSize)
return (False, 0)
WORDS_REGEXP = re.compile('[a-zA-Z0-9]+')
def _getWords(self, string):
return self.WORDS_REGEXP.findall(string)
def family_filter_enabled(self):
if self.utility is None:
return False
else:
state = self.utility.config.Read('family_filter')
if state in ('1', '0'):
return state == '1'
self.utility.config.Write('family_filter', '1')
self.utility.config.Flush()
return True
def set_family_filter(self, b=None):
old = self.family_filter_enabled()
if b != old or b is None:
if b is None:
b = old
if self.utility is None:
return
if b:
self.utility.config.Write('family_filter', '1')
else:
self.utility.config.Write('family_filter', '0')
self.utility.config.Flush()
for category in self.category_info:
if category['name'] == 'xxx':
if b:
category['old-rank'] = category['rank']
category['rank'] = -1
elif category['rank'] == -1:
category['rank'] = category['old-rank']
break
def get_family_filter_sql(self, _getCategoryID, table_name=''):
if self.family_filter_enabled():
forbiddencats = [
cat['name'] for cat in self.category_info if cat['rank'] == -1
]
if table_name:
table_name += '.'
if forbiddencats:
return ' and %scategory_id not in (%s)' % (
table_name, ','.join(
[str(_getCategoryID([cat])) for cat in forbiddencats]))
return ''
class Category:
# Code to make this a singleton
__single = None
__size_change = 1024 * 1024
def __init__(self, install_dir='.', ffEnabled=False):
if Category.__single:
raise RuntimeError("Category is singleton")
filename = os.path.join(install_dir, LIBRARYNAME, 'Category', category_file)
Category.__single = self
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >> sys.stderr, "category: Categories defined by user", self.getCategoryNames()
self.ffEnabled = ffEnabled
self.set_family_filter(None)
# return Category instance
def getInstance(*args, **kw):
if Category.__single is None:
Category(*args, **kw)
return Category.__single
getInstance = staticmethod(getInstance)
def delInstance(*args, **kw):
Category.__single = None
delInstance = staticmethod(delInstance)
def getCategoryKeys(self):
if self.category_info is None:
return []
keys = []
keys.append("All")
keys.append("other")
for category in self.category_info:
keys.append(category['name'])
keys.sort()
return keys
def getCategoryNames(self, filter=True):
if self.category_info is None:
return []
keys = []
for category in self.category_info:
rank = category['rank']
if rank == -1 and filter:
break
keys.append((category['name'], category['displayname']))
return keys
def hasActiveCategory(self, torrent):
try:
name = torrent['category'][0]
except:
print >> sys.stderr, 'Torrent: %s has no valid category' % repr(torrent['content_name'])
return False
for category in [{'name': 'other', 'rank': 1}] + self.category_info:
rank = category['rank']
if rank == -1:
break
if name.lower() == category['name'].lower():
return True
# print >> sys.stderr, 'Category: %s was not in %s' % (name.lower(), [a['name'].lower() for a in self.category_info if a['rank'] != -1])
return False
def getCategoryRank(self, cat):
for category in self.category_info:
if category['name'] == cat:
return category['rank']
return None
# calculate the category for a given torrent_dict of a torrent file
# return list
def calculateCategory(self, torrent_dict, display_name):
# torrent_dict is the dict of
# a torrent file
# return value: list of category the torrent belongs to
files_list = []
try:
# the multi-files mode
for ifiles in torrent_dict['info']["files"]:
files_list.append((ifiles['path'][-1], ifiles['length'] / float(self.__size_change)))
except KeyError:
# single mode
files_list.append((torrent_dict['info']["name"], torrent_dict['info']['length'] / float(self.__size_change)))
tracker = torrent_dict.get('announce')
if not tracker:
tracker = torrent_dict.get('announce-list', [['']])[0][0]
comment = torrent_dict.get('comment')
return self.calculateCategoryNonDict(files_list, display_name, tracker, comment)
def calculateCategoryNonDict(self, files_list, display_name, tracker, comment):
# Check xxx
try:
if self.xxx_filter.isXXXTorrent(files_list, display_name, tracker, comment):
return ['xxx']
except:
print >> sys.stderr, 'Category: Exception in explicit terms filter in torrent: %s' % display_name
print_exc()
torrent_category = None
# filename_list ready
strongest_cat = 0.0
for category in self.category_info: # for each category
(decision, strength) = self.judge(category, files_list, display_name)
if decision and (strength > strongest_cat):
torrent_category = [category['name']]
strongest_cat = strength
if torrent_category == None:
torrent_category = ['other']
return torrent_category
# judge whether a torrent file belongs to a certain category
# return bool
def judge(self, category, files_list, display_name=''):
# judge file keywords
display_name = display_name.lower()
factor = 1.0
fileKeywords = self._getWords(display_name)
for ikeywords in category['keywords'].keys():
try:
fileKeywords.index(ikeywords)
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if (1 - factor) > 0.5:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, (1 - factor))
# judge each file
matchSize = 0
totalSize = 1e-19
for name, length in files_list:
totalSize += length
# judge file size
if (length < category['minfilesize']) or \
(category['maxfilesize'] > 0 and length > category['maxfilesize']):
continue
# judge file suffix
OK = False
for isuffix in category['suffix']:
if name.lower().endswith(isuffix):
OK = True
break
if OK:
matchSize += length
continue
# judge file keywords
factor = 1.0
fileKeywords = self._getWords(name.lower())
for ikeywords in category['keywords'].keys():
# pass
try:
fileKeywords.index(ikeywords)
# print ikeywords
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if factor < 0.5:
# print filename_list[index] + '#######################'
matchSize += length
# match file
if (matchSize / totalSize) >= category['matchpercentage']:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, (matchSize / totalSize))
return (False, 0)
WORDS_REGEXP = re.compile('[a-zA-Z0-9]+')
def _getWords(self, string):
return self.WORDS_REGEXP.findall(string)
def family_filter_enabled(self):
"""
Return is xxx filtering is enabled in this client
"""
return self.ffEnabled
def set_family_filter(self, b=None):
assert b in (True, False, None)
old = self.family_filter_enabled()
if b != old or b is None: # update category data if initial call, or if state changes
if b is None:
b = old
self.ffEnabled = b
# change category data
for category in self.category_info:
if category['name'] == 'xxx':
if b:
category['old-rank'] = category['rank']
category['rank'] = -1
elif category['rank'] == -1:
category['rank'] = category['old-rank']
break
def get_family_filter_sql(self, _getCategoryID, table_name=''):
if self.family_filter_enabled():
forbiddencats = [cat['name'] for cat in self.category_info if cat['rank'] == -1]
if table_name:
table_name += '.'
if forbiddencats:
return " and %scategory_id not in (%s)" % (table_name, ','.join([str(_getCategoryID([cat])) for cat in forbiddencats]))
return ''
class Category():
__single = None
__size_change = 1048576
def __init__(self, install_dir = '.'):
if Category.__single:
raise RuntimeError, 'Category is singleton'
filename = os.path.join(install_dir, 'data', 'category', category_file)
Category.__single = self
self.utility = None
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >> sys.stderr, 'category: Categories defined by user', self.getCategoryNames()
def getInstance(*args, **kw):
if Category.__single is None:
Category(*args, **kw)
return Category.__single
getInstance = staticmethod(getInstance)
def register(self, metadata_handler):
self.metadata_handler = metadata_handler
def init_from_main(self, utility):
self.utility = utility
self.set_family_filter(None)
def getCategoryKeys(self):
if self.category_info is None:
return []
keys = []
keys.append('All')
keys.append('other')
for category in self.category_info:
keys.append(category['name'])
keys.sort()
return keys
def getCategoryNames(self):
if self.category_info is None:
return []
keys = []
for category in self.category_info:
rank = category['rank']
if rank == -1:
break
keys.append((category['name'], category['displayname']))
return keys
def hasActiveCategory(self, torrent):
try:
name = torrent['category'][0]
except:
print >> sys.stderr, 'Torrent: %s has no valid category' % `(torrent['content_name'])`
return False
for category in [{'name': 'other',
'rank': 1}] + self.category_info:
rank = category['rank']
if rank == -1:
break
if name.lower() == category['name'].lower():
return True
return False
def getCategoryRank(self, cat):
for category in self.category_info:
if category['name'] == cat:
return category['rank']
def calculateCategory(self, torrent_dict, display_name):
torrent_category = None
files_list = []
try:
for ifiles in torrent_dict['info']['files']:
files_list.append((ifiles['path'][-1], ifiles['length'] / float(self.__size_change)))
except KeyError:
files_list.append((torrent_dict['info']['name'], torrent_dict['info']['length'] / float(self.__size_change)))
try:
tracker = torrent_dict.get('announce')
if not tracker:
tracker = torrent_dict.get('announce-list', [['']])[0][0]
if self.xxx_filter.isXXXTorrent(files_list, display_name, torrent_dict.get('announce'), torrent_dict.get('comment')):
return ['xxx']
except:
print >> sys.stderr, 'Category: Exception in explicit terms filter in torrent: %s' % torrent_dict
print_exc()
strongest_cat = 0.0
for category in self.category_info:
decision, strength = self.judge(category, files_list, display_name)
if decision and strength > strongest_cat:
torrent_category = [category['name']]
strongest_cat = strength
if torrent_category == None:
torrent_category = ['other']
return torrent_category
def judge(self, category, files_list, display_name = ''):
display_name = display_name.lower()
factor = 1.0
fileKeywords = self._getWords(display_name)
for ikeywords in category['keywords'].keys():
try:
fileKeywords.index(ikeywords)
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if 1 - factor > 0.5:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, 1 - factor)
matchSize = 0
totalSize = 1e-19
for name, length in files_list:
totalSize += length
if length < category['minfilesize'] or category['maxfilesize'] > 0 and length > category['maxfilesize']:
continue
OK = False
for isuffix in category['suffix']:
if name.lower().endswith(isuffix):
OK = True
break
if OK:
matchSize += length
continue
factor = 1.0
fileKeywords = self._getWords(name.lower())
for ikeywords in category['keywords'].keys():
try:
fileKeywords.index(ikeywords)
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if factor < 0.5:
matchSize += length
if matchSize / totalSize >= category['matchpercentage']:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, matchSize / totalSize)
return (False, 0)
WORDS_REGEXP = re.compile('[a-zA-Z0-9]+')
def _getWords(self, string):
return self.WORDS_REGEXP.findall(string)
def family_filter_enabled(self):
if self.utility is None:
return False
else:
state = self.utility.config.Read('family_filter')
if state in ('1', '0'):
return state == '1'
self.utility.config.Write('family_filter', '1')
self.utility.config.Flush()
return True
def set_family_filter(self, b = None):
old = self.family_filter_enabled()
if b != old or b is None:
if b is None:
b = old
if self.utility is None:
return
if b:
self.utility.config.Write('family_filter', '1')
else:
self.utility.config.Write('family_filter', '0')
self.utility.config.Flush()
for category in self.category_info:
if category['name'] == 'xxx':
if b:
category['old-rank'] = category['rank']
category['rank'] = -1
elif category['rank'] == -1:
category['rank'] = category['old-rank']
break
def get_family_filter_sql(self, _getCategoryID, table_name = ''):
if self.family_filter_enabled():
forbiddencats = [ cat['name'] for cat in self.category_info if cat['rank'] == -1 ]
if table_name:
table_name += '.'
if forbiddencats:
return ' and %scategory_id not in (%s)' % (table_name, ','.join([ str(_getCategoryID([cat])) for cat in forbiddencats ]))
return ''
class Category:
# Code to make this a singleton
__single = None
__size_change = 1024 * 1024
def __init__(self, install_dir='.'):
if Category.__single:
raise RuntimeError, "Category is singleton"
filename = os.path.join(install_dir,LIBRARYNAME, 'Category', category_file)
Category.__single = self
self.utility = None
#self.torrent_db = TorrentDBHandler.getInstance() # Arno, 2009-01-30: apparently unused
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >>sys.stderr,"category: Categories defined by user",self.getCategoryNames()
# return Category instance
def getInstance(*args, **kw):
if Category.__single is None:
Category(*args, **kw)
return Category.__single
getInstance = staticmethod(getInstance)
def init_from_main(self, utility):
self.utility = utility
self.set_family_filter(None) # init family filter to saved state
def getCategoryKeys(self):
if self.category_info is None:
return []
keys = []
keys.append("All")
keys.append("other")
for category in self.category_info:
keys.append(category['name'])
keys.sort()
return keys
def getCategoryNames(self, filter = True):
if self.category_info is None:
return []
keys = []
for category in self.category_info:
rank = category['rank']
if rank == -1 and filter:
break
keys.append((category['name'],category['displayname']))
return keys
def hasActiveCategory(self, torrent):
try:
name = torrent['category'][0]
except:
print >> sys.stderr, 'Torrent: %s has no valid category' % `torrent['content_name']`
return False
for category in [{'name':'other', 'rank':1}]+self.category_info:
rank = category['rank']
if rank == -1:
break
if name.lower() == category['name'].lower():
return True
#print >> sys.stderr, 'Category: %s was not in %s' % (name.lower(), [a['name'].lower() for a in self.category_info if a['rank'] != -1])
return False
def getCategoryRank(self,cat):
for category in self.category_info:
if category['name'] == cat:
return category['rank']
return None
# calculate the category for a given torrent_dict of a torrent file
# return list
def calculateCategory(self, torrent_dict, display_name):
# torrent_dict is the dict of
# a torrent file
# return value: list of category the torrent belongs to
files_list = []
try:
# the multi-files mode
for ifiles in torrent_dict['info']["files"]:
files_list.append((ifiles['path'][-1], ifiles['length'] / float(self.__size_change)))
except KeyError:
# single mode
files_list.append((torrent_dict['info']["name"],torrent_dict['info']['length'] / float(self.__size_change)))
tracker = torrent_dict.get('announce')
if not tracker:
tracker = torrent_dict.get('announce-list',[['']])[0][0]
comment = torrent_dict.get('comment')
return self.calculateCategoryNonDict(files_list, display_name, tracker, comment)
def calculateCategoryNonDict(self, files_list, display_name, tracker, comment):
# Check xxx
try:
if self.xxx_filter.isXXXTorrent(files_list, display_name, tracker, comment):
return ['xxx']
except:
print >> sys.stderr, 'Category: Exception in explicit terms filter in torrent: %s' % display_name
print_exc()
torrent_category = None
# filename_list ready
strongest_cat = 0.0
for category in self.category_info: # for each category
(decision, strength) = self.judge(category, files_list, display_name)
if decision and (strength > strongest_cat):
torrent_category = [category['name']]
strongest_cat = strength
if torrent_category == None:
torrent_category = ['other']
return torrent_category
# judge whether a torrent file belongs to a certain category
# return bool
def judge(self, category, files_list, display_name = ''):
# judge file keywords
display_name = display_name.lower()
factor = 1.0
fileKeywords = self._getWords(display_name)
for ikeywords in category['keywords'].keys():
try:
fileKeywords.index(ikeywords)
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if (1 - factor) > 0.5:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, (1- factor))
# judge each file
matchSize = 0
totalSize = 1e-19
for name, length in files_list:
totalSize += length
# judge file size
if ( length < category['minfilesize'] ) or \
(category['maxfilesize'] > 0 and length > category['maxfilesize'] ):
continue
# judge file suffix
OK = False
for isuffix in category['suffix']:
if name.lower().endswith( isuffix ):
OK = True
break
if OK:
matchSize += length
continue
# judge file keywords
factor = 1.0
fileKeywords = self._getWords(name.lower())
for ikeywords in category['keywords'].keys():
# pass
try:
fileKeywords.index(ikeywords)
#print ikeywords
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if factor < 0.5:
# print filename_list[index] + '#######################'
matchSize += length
# match file
if (matchSize / totalSize) >= category['matchpercentage']:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, (matchSize/ totalSize))
return (False, 0)
WORDS_REGEXP = re.compile('[a-zA-Z0-9]+')
def _getWords(self, string):
return self.WORDS_REGEXP.findall(string)
def family_filter_enabled(self):
"""
Return is xxx filtering is enabled in this client
"""
if self.utility is None:
return False
state = self.utility.config.Read('family_filter')
if state in ('1', '0'):
return state == '1'
else:
self.utility.config.Write('family_filter', '1')
self.utility.config.Flush()
return True
def set_family_filter(self, b=None):
assert b in (True, False, None)
old = self.family_filter_enabled()
if b != old or b is None: # update category data if initial call, or if state changes
if b is None:
b=old
if self.utility is None:
return
#print >> sys.stderr , b
if b:
self.utility.config.Write('family_filter', '1')
else:
self.utility.config.Write('family_filter', '0')
self.utility.config.Flush()
# change category data
for category in self.category_info:
if category['name'] == 'xxx':
if b:
category['old-rank'] = category['rank']
category['rank'] = -1
elif category['rank'] == -1:
category['rank'] = category['old-rank']
break
def get_family_filter_sql(self, _getCategoryID, table_name=''):
if self.family_filter_enabled():
forbiddencats = [cat['name'] for cat in self.category_info if cat['rank'] == -1]
if table_name:
table_name+='.'
if forbiddencats:
return " and %scategory_id not in (%s)" % (table_name, ','.join([str(_getCategoryID([cat])) for cat in forbiddencats]))
return ''
class Category:
# Code to make this a singleton
__single = None
__size_change = 1024 * 1024
def __init__(self, install_dir='.'):
if Category.__single:
raise RuntimeError, "Category is singleton"
filename = os.path.join(install_dir, LIBRARYNAME, 'Category',
category_file)
Category.__single = self
self.utility = None
#self.torrent_db = TorrentDBHandler.getInstance() # Arno, 2009-01-30: apparently unused
try:
self.category_info = getCategoryInfo(filename)
self.category_info.sort(rankcmp)
except:
self.category_info = []
if DEBUG:
print_exc()
self.xxx_filter = XXXFilter(install_dir)
if DEBUG:
print >> sys.stderr, "category: Categories defined by user", self.getCategoryNames(
)
# return Category instance
def getInstance(*args, **kw):
if Category.__single is None:
Category(*args, **kw)
return Category.__single
getInstance = staticmethod(getInstance)
def register(self, metadata_handler):
self.metadata_handler = metadata_handler
def init_from_main(self, utility):
self.utility = utility
self.set_family_filter(None) # init family filter to saved state
# # check to see whether need to resort torrent file
# # return bool
# def checkResort(self, data_manager):
# data = data_manager.data
# #===============================================================================
# # if not data:
# # data = data_manager.torrent_db.getRecommendedTorrents(all = True)
# #===============================================================================
# if not data:
# return False
# # data = data_manager.torrent_db.getRecommendedTorrents(all = True)
# # self.reSortAll(data)
# # return True
# torrent = data[0]
# if torrent["category"] == ["?"]:
# #data = data_manager.torrent_db.getRecommendedTorrents(all = True)
# self.reSortAll(data)
# # del data
# return True
# begin = time()
# for item in data:
# if len(item['category']) > 1:
# #data = data_manager.torrent_db.getRecommendedTorrents(all = True)
# self.reSortAll(data)
# # del data
# return True
# if DEBUG:
# print >>sys.stderr,'torrcoll: Checking of %d torrents costs: %f s' % (len(data), time() - begin)
# return False
# # recalculate category of all torrents, remove torrents from db if not existed
# def reSortAll(self, data, parent = None):
# max = len(data)
# if max == 0:
# return
# import wx
# dlgHolder = []
# event = Event()
# def makeDialog():
# dlg = wx.ProgressDialog("Upgrading Database",
# "Upgrading Old Database to New Database",
# maximum = max,
# parent = None,
# style = wx.PD_AUTO_HIDE
# | wx.PD_ELAPSED_TIME
# | wx.PD_REMAINING_TIME
# )
# dlgHolder.append(dlg)
# event.set()
# wx.CallAfter(makeDialog)
# # Wait for dialog to be ready
# event.wait()
# dlg = dlgHolder[0]
# count = 0
# step = int(float(max) / 20) + 1
# # sort each torrent file
# for i in xrange(len(data)):
# count += 1
# if count % step == 0:
# wx.CallAfter(dlg.Update, [count])
# try:
# # try alternative dir if bsddb doesnt match with current Tribler install
# rec = data[i]
# (torrent_dir,torrent_name) = self.metadata_handler.get_std_torrent_dir_name(rec)
# # read the torrent file
# filesrc = os.path.join(torrent_dir,torrent_name)
# # print filesrc
# f = open(filesrc, "rb")
# torrentdata = f.read() # torrent decoded string
# f.close()
# except IOError: # torrent file not found
# # delete the info from db
# self.torrent_db.deleteTorrent(data[i]['infohash'])
# continue
# # decode the data
# torrent_dict = bencode.bdecode(torrentdata)
# content_name = dunno2unicode(torrent_dict["info"].get('name', '?'))
# category_belong = []
# category_belong = self.calculateCategory(torrent_dict, content_name)
# if (category_belong == []):
# category_belong = ['other']
# data[i]['category'] = category_belong # should have updated self.data
# self.torrent_db.updateTorrent(data[i]['infohash'], updateFlag=False, category=category_belong)
# self.torrent_db.sync()
# wx.CallAfter(dlg.Destroy)
def getCategoryKeys(self):
if self.category_info is None:
return []
keys = []
keys.append("All")
keys.append("other")
for category in self.category_info:
keys.append(category['name'])
keys.sort()
return keys
def getCategoryNames(self):
if self.category_info is None:
return []
keys = []
for category in self.category_info:
rank = category['rank']
if rank == -1:
break
keys.append((category['name'], category['displayname']))
return keys
def hasActiveCategory(self, torrent):
try:
name = torrent['category'][0]
except:
print >> sys.stderr, 'Torrent: %s has no valid category' % ` torrent[
'content_name'] `
return False
for category in [{'name': 'other', 'rank': 1}] + self.category_info:
rank = category['rank']
if rank == -1:
break
if name.lower() == category['name'].lower():
return True
#print >> sys.stderr, 'Category: %s was not in %s' % (name.lower(), [a['name'].lower() for a in self.category_info if a['rank'] != -1])
return False
def getCategoryRank(self, cat):
for category in self.category_info:
if category['name'] == cat:
return category['rank']
return None
# calculate the category for a given torrent_dict of a torrent file
# return list
def calculateCategory(self, torrent_dict, display_name):
# torrent_dict is the dict of
# a torrent file
# return value: list of category the torrent belongs to
files_list = []
try:
# the multi-files mode
for ifiles in torrent_dict['info']["files"]:
files_list.append(
(ifiles['path'][-1],
ifiles['length'] / float(self.__size_change)))
except KeyError:
# single mode
files_list.append(
(torrent_dict['info']["name"],
torrent_dict['info']['length'] / float(self.__size_change)))
tracker = torrent_dict.get('announce')
if not tracker:
tracker = torrent_dict.get('announce-list', [['']])[0][0]
comment = torrent_dict.get('comment')
return self.calculateCategoryNonDict(files_list, display_name, tracker,
comment)
def calculateCategoryNonDict(self, files_list, display_name, tracker,
comment):
# Check xxx
try:
if self.xxx_filter.isXXXTorrent(files_list, display_name, tracker,
comment):
return ['xxx']
except:
print >> sys.stderr, 'Category: Exception in explicit terms filter in torrent: %s' % display_name
print_exc()
torrent_category = None
# filename_list ready
strongest_cat = 0.0
for category in self.category_info: # for each category
(decision, strength) = self.judge(category, files_list,
display_name)
if decision and (strength > strongest_cat):
torrent_category = [category['name']]
strongest_cat = strength
if torrent_category == None:
torrent_category = ['other']
return torrent_category
# judge whether a torrent file belongs to a certain category
# return bool
def judge(self, category, files_list, display_name=''):
# judge file keywords
display_name = display_name.lower()
factor = 1.0
fileKeywords = self._getWords(display_name)
for ikeywords in category['keywords'].keys():
try:
fileKeywords.index(ikeywords)
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if (1 - factor) > 0.5:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, (1 - factor))
# judge each file
matchSize = 0
totalSize = 1e-19
for name, length in files_list:
totalSize += length
# judge file size
if ( length < category['minfilesize'] ) or \
(category['maxfilesize'] > 0 and length > category['maxfilesize'] ):
continue
# judge file suffix
OK = False
for isuffix in category['suffix']:
if name.lower().endswith(isuffix):
OK = True
break
if OK:
matchSize += length
continue
# judge file keywords
factor = 1.0
fileKeywords = self._getWords(name.lower())
for ikeywords in category['keywords'].keys():
# pass
try:
fileKeywords.index(ikeywords)
#print ikeywords
factor *= 1 - category['keywords'][ikeywords]
except:
pass
if factor < 0.5:
# print filename_list[index] + '#######################'
matchSize += length
# match file
if (matchSize / totalSize) >= category['matchpercentage']:
if 'strength' in category:
return (True, category['strength'])
else:
return (True, (matchSize / totalSize))
return (False, 0)
WORDS_REGEXP = re.compile('[a-zA-Z0-9]+')
def _getWords(self, string):
return self.WORDS_REGEXP.findall(string)
def family_filter_enabled(self):
"""
Return is xxx filtering is enabled in this client
"""
if self.utility is None:
return False
state = self.utility.config.Read('family_filter')
if state in ('1', '0'):
return state == '1'
else:
self.utility.config.Write('family_filter', '1')
self.utility.config.Flush()
return True
def set_family_filter(self, b=None):
assert b in (True, False, None)
old = self.family_filter_enabled()
if b != old or b is None: # update category data if initial call, or if state changes
if b is None:
b = old
if self.utility is None:
return
#print >> sys.stderr , b
if b:
self.utility.config.Write('family_filter', '1')
else:
self.utility.config.Write('family_filter', '0')
self.utility.config.Flush()
# change category data
for category in self.category_info:
if category['name'] == 'xxx':
if b:
category['old-rank'] = category['rank']
category['rank'] = -1
elif category['rank'] == -1:
category['rank'] = category['old-rank']
break
def get_family_filter_sql(self, _getCategoryID, table_name=''):
if self.family_filter_enabled():
forbiddencats = [
cat['name'] for cat in self.category_info if cat['rank'] == -1
]
if table_name:
table_name += '.'
if forbiddencats:
return " and %scategory_id not in (%s)" % (
table_name, ','.join(
[str(_getCategoryID([cat])) for cat in forbiddencats]))
return ''
