def got_non_redirect(self, task): self._page_archive[task.url] = task.archive_info() # update dependencies status, headers, body, parsed = task.archive_info() resource = make_resource_key(task.url, task.environ) if parsed is not None: dep_list = self.find_dependencies(parsed, task.url) else: dep_list = [] resource = make_resource_key(task.url, self._environ) self.deptracker.set_direct_deps(resource, dep_list) if self._state == PMState.check_modification: self._init_speculative_gets() self._speculative_dep_info[task.url] = dep_list for dep in dep_list: self.add_get(dep) if task.url in self._needed: self._got_needed(task.url) self.fetchlist.completed(task) self.notify()
def __init__(self, request_url, environ, deptracker, find_dependencies, tasklist, request_func): self.deptracker = deptracker self.tasklist = tasklist self.fetchlist = FetchList(tasklist) self.find_dependencies = find_dependencies self.request = request_func self._request_url = request_url self._environ = environ.copy() self._root_resource = make_resource_key(self._request_url, self._environ) self._page_archive = {} self._speculative_dep_info = {} self._needed = Set() self._actual_deps = Set() self._lock = RLock() self.cv = Condition(self._lock) self._state = PMState.initial