Exemple #1
0
    def __init__(self, links, start_now=False, deep_traverse=False):
        """ Init a new crawler. """
        # If a user is blocking a given user, then do
        # not show their messages or replies.
        self.block_list = []
        self.ignore_user = False

        FeedCrawler.__init__(self, links, start_now=start_now, deep_traverse=deep_traverse)
 def __init__(self, links, start_now=False, deep_traverse=False, cache_location=None):
     """ Init a new crawler. """
     # If a user is blocking a given user, then do
     # not show their messages or replies.
     self.block_list = []
     self.ignore_user = False
     self.user_info = []
     # Instantiate the cache manager at the location given. Now we can use the cache.
     if cache_location is None:
         raise ValueError('Cache location is required for multiprocessed crawling.')
     CacheManager(cache_location)
     # Call the superclass init.
     FeedCrawler.__init__(self, links, start_now=start_now, deep_traverse=deep_traverse)
Exemple #3
0
 def __init__(self, links, start_now=False):
     self.total_time = 0
     self.count = 0
     FeedCrawler.__init__(self, links, start_now=start_now)
 def __init__(self):
     self._data = {}
     FeedCrawler.__init__(self, [], start_now=False, deep_traverse=False)
 def __init__(self, links, start_now=False):
     FeedCrawler.__init__(self, links, start_now=start_now)