示例#1
0
 def __init__(self, s_file = 'visit_history'):
     """
     Loads last state.
     """
     settings = globalcrawersettings.load_settings('visit_history')
     self._visited_sites = settings.get('_visited_sites', {})
示例#2
0
 def __init__(self, s_file = 'results'):
     self._settings_file = s_file
     self._settings = globalcrawersettings.load_settings(self._settings_file)
     self.sites = self._settings.get('sites', set())
     self.books = self._settings.get('books', set())
示例#3
0
 def _load_state(self):
     settings = load_settings('PageLoader')
     future_tasks = settings.get('future_tasks', [])
     for url in future_tasks:
         self._scheduled_tasks.append(PageLoaderTask(url))