Example #1
0
 def __init__(self):
     self._sqlite_cache_db = SQLiteCacheDB.getInstance()
     crawler = Crawler.get_instance()
     if crawler.am_crawler():
         self._file = open('databasecrawler.txt', 'a')
         self._file.write(''.join(
             ('# ', '*' * 80, '\n# ', strftime('%Y/%m/%d %H:%M:%S'),
              ' Crawler started\n')))
         self._file.flush()
     else:
         self._file = None
Example #2
0
 def __init__(self):
     self._sqlite_cache_db = SQLiteCacheDB.getInstance()
     crawler = Crawler.get_instance()
     if crawler.am_crawler():
         self._file = open('channelcrawler.txt', 'a')
         self._file.write(''.join(('# ',
          '*' * 80,
          '\n# ',
          strftime('%Y/%m/%d %H:%M:%S'),
          ' Crawler started\n')))
         self._file.flush()
     else:
         self._file = None
    def __init__(self, db = SQLiteCacheDB.getInstance()):
        try:
            MetadataDBHandler._lock.acquire()
            MetadataDBHandler.__single = self
        finally:
            MetadataDBHandler._lock.release()

        try:
            self._db = db
            print >> sys.stderr, 'Metadata: DB made'
        except:
            print >> sys.stderr, "Metadata: couldn't make the tables"

        print >> sys.stderr, 'Metadata DB Handler initialized'
Example #4
0
    def network_shutdown(self):
        try:
            if self.peer_db is not None:
                db = SQLiteCacheDB.getInstance()
                db.commit()
            mainlineDHT.deinit()
            if DEBUG:
                ts = enumerate()
                log('LM::network_shutdown: number of threads still running', len(ts))
                for t in ts:
                    log('LM::network_shutdown: thread still running', t.name, 'daemon', t.daemon, 'instance', t)

        except:
            log_exc()

        self.sessdoneflag.set()
        self.session.uch.shutdown()
 def __init__(self):
     if TorrentDBHandler.__single is not None:
         raise RuntimeError, 'TorrentDBHandler is singleton'
     TorrentDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'Torrent')
     self.status_table = {'good': 1, 'unknown': 0, 'dead': 2}
     self.status_table.update(self._db.getTorrentStatusTable())
     self.id2status = dict([(x, y) for y, x in self.status_table.items()])
     self.torrent_dir = None
     self.category_table = {
         'Video': 1,
         'VideoClips': 2,
         'Audio': 3,
         'Compressed': 4,
         'Document': 5,
         'Picture': 6,
         'xxx': 7,
         'other': 8
     }
     self.category_table.update(self._db.getTorrentCategoryTable())
     self.category_table['unknown'] = 0
     self.id2category = dict([(x, y)
                              for y, x in self.category_table.items()])
     self.src_table = self._db.getTorrentSourceTable()
     self.id2src = dict([(x, y) for y, x in self.src_table.items()])
     self.keys = [
         'torrent_id', 'checksum', 'name', 'torrent_file_name', 'length',
         'creation_date', 'num_files', 'thumbnail', 'insert_time', 'secret',
         'relevance', 'source_id', 'category_id', 'status_id',
         'num_seeders', 'num_leechers', 'comment'
     ]
     self.existed_torrents = set()
     self.value_name = [
         'C.torrent_id', 'category_id', 'status_id', 'name',
         'creation_date', 'num_files', 'num_leechers', 'num_seeders',
         'length', 'secret', 'insert_time', 'source_id',
         'torrent_file_name', 'relevance', 'infohash', 'tracker',
         'last_check'
     ]
     self.value_name_for_channel = [
         'C.torrent_id', 'infohash', 'name', 'torrent_file_name', 'length',
         'creation_date', 'num_files', 'thumbnail', 'insert_time', 'secret',
         'relevance', 'source_id', 'category_id', 'status_id',
         'num_seeders', 'num_leechers', 'comment'
     ]
 def __init__(self):
     if MyDBHandler.__single is not None:
         raise RuntimeError, 'MyDBHandler is singleton'
     MyDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'MyInfo')
 def __init__(self):
     if UserProfileDBHandler.__single is not None:
         raise RuntimeError, 'UserProfileDBHandler is singleton'
     UserProfileDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'user_profiles')
 def __init__(self):
     if Url2TorrentDBHandler.__single is not None:
         raise RuntimeError, 'Url2TorrentDBHandler is singleton'
     Url2TorrentDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'url2torrent')
 def __init__(self):
     if AdID2InfohashDBHandler.__single is not None:
         raise RuntimeError, 'AdID2Infohash is singleton'
     AdID2InfohashDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'adid2infohash')
 def __init__(self):
     if TsMetadataDBHandler.__single is not None:
         raise RuntimeError, 'TsMetadataDBHandler is singleton'
     TsMetadataDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'ts_metadata')
 def __init__(self):
     if UserProfileDBHandler.__single is not None:
         raise RuntimeError, 'UserProfileDBHandler is singleton'
     UserProfileDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'user_profiles')
 def __init__(self):
     if Url2TorrentDBHandler.__single is not None:
         raise RuntimeError, 'Url2TorrentDBHandler is singleton'
     Url2TorrentDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'url2torrent')
Example #13
0
 def __init__(self):
     self._sqlite_cache_db = SQLiteCacheDB.getInstance()
     crawler = Crawler.get_instance()
     if crawler.am_crawler():
         msg = '# Crawler started'
         self.__log(msg)
 def __init__(self):
     if MyDBHandler.__single is not None:
         raise RuntimeError, 'MyDBHandler is singleton'
     MyDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'MyInfo')
 def __init__(self):
     if TorrentDBHandler.__single is not None:
         raise RuntimeError, 'TorrentDBHandler is singleton'
     TorrentDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'Torrent')
     self.status_table = {'good': 1,
      'unknown': 0,
      'dead': 2}
     self.status_table.update(self._db.getTorrentStatusTable())
     self.id2status = dict([ (x, y) for y, x in self.status_table.items() ])
     self.torrent_dir = None
     self.category_table = {'Video': 1,
      'VideoClips': 2,
      'Audio': 3,
      'Compressed': 4,
      'Document': 5,
      'Picture': 6,
      'xxx': 7,
      'other': 8}
     self.category_table.update(self._db.getTorrentCategoryTable())
     self.category_table['unknown'] = 0
     self.id2category = dict([ (x, y) for y, x in self.category_table.items() ])
     self.src_table = self._db.getTorrentSourceTable()
     self.id2src = dict([ (x, y) for y, x in self.src_table.items() ])
     self.keys = ['torrent_id',
      'checksum',
      'name',
      'torrent_file_name',
      'length',
      'creation_date',
      'num_files',
      'thumbnail',
      'insert_time',
      'secret',
      'relevance',
      'source_id',
      'category_id',
      'status_id',
      'num_seeders',
      'num_leechers',
      'comment']
     self.existed_torrents = set()
     self.value_name = ['C.torrent_id',
      'category_id',
      'status_id',
      'name',
      'creation_date',
      'num_files',
      'num_leechers',
      'num_seeders',
      'length',
      'secret',
      'insert_time',
      'source_id',
      'torrent_file_name',
      'relevance',
      'infohash',
      'tracker',
      'last_check']
     self.value_name_for_channel = ['C.torrent_id',
      'infohash',
      'name',
      'torrent_file_name',
      'length',
      'creation_date',
      'num_files',
      'thumbnail',
      'insert_time',
      'secret',
      'relevance',
      'source_id',
      'category_id',
      'status_id',
      'num_seeders',
      'num_leechers',
      'comment']
 def __init__(self):
     if TsMetadataDBHandler.__single is not None:
         raise RuntimeError, 'TsMetadataDBHandler is singleton'
     TsMetadataDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'ts_metadata')
 def __init__(self):
     if AdID2InfohashDBHandler.__single is not None:
         raise RuntimeError, 'AdID2Infohash is singleton'
     AdID2InfohashDBHandler.__single = self
     db = SQLiteCacheDB.getInstance()
     BasicDBHandler.__init__(self, db, 'adid2infohash')