def __init__(self):
self._sqlite_cache_db = SQLiteCacheDB.getInstance()
crawler = Crawler.get_instance()
if crawler.am_crawler():
self._file = open('databasecrawler.txt', 'a')
self._file.write(''.join(
('# ', '*' * 80, '\n# ', strftime('%Y/%m/%d %H:%M:%S'),
' Crawler started\n')))
self._file.flush()
else:
self._file = None
def __init__(self):
self._sqlite_cache_db = SQLiteCacheDB.getInstance()
crawler = Crawler.get_instance()
if crawler.am_crawler():
self._file = open('channelcrawler.txt', 'a')
self._file.write(''.join(('# ',
'*' * 80,
'\n# ',
strftime('%Y/%m/%d %H:%M:%S'),
' Crawler started\n')))
self._file.flush()
else:
self._file = None
def __init__(self, db = SQLiteCacheDB.getInstance()):
try:
MetadataDBHandler._lock.acquire()
MetadataDBHandler.__single = self
finally:
MetadataDBHandler._lock.release()
try:
self._db = db
print >> sys.stderr, 'Metadata: DB made'
except:
print >> sys.stderr, "Metadata: couldn't make the tables"
print >> sys.stderr, 'Metadata DB Handler initialized'
def network_shutdown(self):
try:
if self.peer_db is not None:
db = SQLiteCacheDB.getInstance()
db.commit()
mainlineDHT.deinit()
if DEBUG:
ts = enumerate()
log('LM::network_shutdown: number of threads still running', len(ts))
for t in ts:
log('LM::network_shutdown: thread still running', t.name, 'daemon', t.daemon, 'instance', t)
except:
log_exc()
self.sessdoneflag.set()
self.session.uch.shutdown()
def __init__(self):
if TorrentDBHandler.__single is not None:
raise RuntimeError, 'TorrentDBHandler is singleton'
TorrentDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'Torrent')
self.status_table = {'good': 1, 'unknown': 0, 'dead': 2}
self.status_table.update(self._db.getTorrentStatusTable())
self.id2status = dict([(x, y) for y, x in self.status_table.items()])
self.torrent_dir = None
self.category_table = {
'Video': 1,
'VideoClips': 2,
'Audio': 3,
'Compressed': 4,
'Document': 5,
'Picture': 6,
'xxx': 7,
'other': 8
}
self.category_table.update(self._db.getTorrentCategoryTable())
self.category_table['unknown'] = 0
self.id2category = dict([(x, y)
for y, x in self.category_table.items()])
self.src_table = self._db.getTorrentSourceTable()
self.id2src = dict([(x, y) for y, x in self.src_table.items()])
self.keys = [
'torrent_id', 'checksum', 'name', 'torrent_file_name', 'length',
'creation_date', 'num_files', 'thumbnail', 'insert_time', 'secret',
'relevance', 'source_id', 'category_id', 'status_id',
'num_seeders', 'num_leechers', 'comment'
]
self.existed_torrents = set()
self.value_name = [
'C.torrent_id', 'category_id', 'status_id', 'name',
'creation_date', 'num_files', 'num_leechers', 'num_seeders',
'length', 'secret', 'insert_time', 'source_id',
'torrent_file_name', 'relevance', 'infohash', 'tracker',
'last_check'
]
self.value_name_for_channel = [
'C.torrent_id', 'infohash', 'name', 'torrent_file_name', 'length',
'creation_date', 'num_files', 'thumbnail', 'insert_time', 'secret',
'relevance', 'source_id', 'category_id', 'status_id',
'num_seeders', 'num_leechers', 'comment'
]
def __init__(self):
if MyDBHandler.__single is not None:
raise RuntimeError, 'MyDBHandler is singleton'
MyDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'MyInfo')
def __init__(self):
if UserProfileDBHandler.__single is not None:
raise RuntimeError, 'UserProfileDBHandler is singleton'
UserProfileDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'user_profiles')
def __init__(self):
if Url2TorrentDBHandler.__single is not None:
raise RuntimeError, 'Url2TorrentDBHandler is singleton'
Url2TorrentDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'url2torrent')
def __init__(self):
if AdID2InfohashDBHandler.__single is not None:
raise RuntimeError, 'AdID2Infohash is singleton'
AdID2InfohashDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'adid2infohash')
def __init__(self):
if TsMetadataDBHandler.__single is not None:
raise RuntimeError, 'TsMetadataDBHandler is singleton'
TsMetadataDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'ts_metadata')
def __init__(self):
if UserProfileDBHandler.__single is not None:
raise RuntimeError, 'UserProfileDBHandler is singleton'
UserProfileDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'user_profiles')
def __init__(self):
if Url2TorrentDBHandler.__single is not None:
raise RuntimeError, 'Url2TorrentDBHandler is singleton'
Url2TorrentDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'url2torrent')
def __init__(self):
self._sqlite_cache_db = SQLiteCacheDB.getInstance()
crawler = Crawler.get_instance()
if crawler.am_crawler():
msg = '# Crawler started'
self.__log(msg)
def __init__(self):
if MyDBHandler.__single is not None:
raise RuntimeError, 'MyDBHandler is singleton'
MyDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'MyInfo')
def __init__(self):
if TorrentDBHandler.__single is not None:
raise RuntimeError, 'TorrentDBHandler is singleton'
TorrentDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'Torrent')
self.status_table = {'good': 1,
'unknown': 0,
'dead': 2}
self.status_table.update(self._db.getTorrentStatusTable())
self.id2status = dict([ (x, y) for y, x in self.status_table.items() ])
self.torrent_dir = None
self.category_table = {'Video': 1,
'VideoClips': 2,
'Audio': 3,
'Compressed': 4,
'Document': 5,
'Picture': 6,
'xxx': 7,
'other': 8}
self.category_table.update(self._db.getTorrentCategoryTable())
self.category_table['unknown'] = 0
self.id2category = dict([ (x, y) for y, x in self.category_table.items() ])
self.src_table = self._db.getTorrentSourceTable()
self.id2src = dict([ (x, y) for y, x in self.src_table.items() ])
self.keys = ['torrent_id',
'checksum',
'name',
'torrent_file_name',
'length',
'creation_date',
'num_files',
'thumbnail',
'insert_time',
'secret',
'relevance',
'source_id',
'category_id',
'status_id',
'num_seeders',
'num_leechers',
'comment']
self.existed_torrents = set()
self.value_name = ['C.torrent_id',
'category_id',
'status_id',
'name',
'creation_date',
'num_files',
'num_leechers',
'num_seeders',
'length',
'secret',
'insert_time',
'source_id',
'torrent_file_name',
'relevance',
'infohash',
'tracker',
'last_check']
self.value_name_for_channel = ['C.torrent_id',
'infohash',
'name',
'torrent_file_name',
'length',
'creation_date',
'num_files',
'thumbnail',
'insert_time',
'secret',
'relevance',
'source_id',
'category_id',
'status_id',
'num_seeders',
'num_leechers',
'comment']
def __init__(self):
if TsMetadataDBHandler.__single is not None:
raise RuntimeError, 'TsMetadataDBHandler is singleton'
TsMetadataDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'ts_metadata')
def __init__(self):
if AdID2InfohashDBHandler.__single is not None:
raise RuntimeError, 'AdID2Infohash is singleton'
AdID2InfohashDBHandler.__single = self
db = SQLiteCacheDB.getInstance()
BasicDBHandler.__init__(self, db, 'adid2infohash')