def got_non_redirect(self, task):
self._page_archive[task.url] = task.archive_info()
# update dependencies
status, headers, body, parsed = task.archive_info()
resource = make_resource_key(task.url, task.environ)
if parsed is not None:
dep_list = self.find_dependencies(parsed, task.url)
else:
dep_list = []
resource = make_resource_key(task.url, self._environ)
self.deptracker.set_direct_deps(resource, dep_list)
if self._state == PMState.check_modification:
self._init_speculative_gets()
self._speculative_dep_info[task.url] = dep_list
for dep in dep_list:
self.add_get(dep)
if task.url in self._needed:
self._got_needed(task.url)
self.fetchlist.completed(task)
self.notify()
def __init__(self, request_url, environ, deptracker, find_dependencies, tasklist, request_func):
self.deptracker = deptracker
self.tasklist = tasklist
self.fetchlist = FetchList(tasklist)
self.find_dependencies = find_dependencies
self.request = request_func
self._request_url = request_url
self._environ = environ.copy()
self._root_resource = make_resource_key(self._request_url, self._environ)
self._page_archive = {}
self._speculative_dep_info = {}
self._needed = Set()
self._actual_deps = Set()
self._lock = RLock()
self.cv = Condition(self._lock)
self._state = PMState.initial
