def store_response(self, spider, request, response): """Store the given response in the cache.""" rpath = self._get_request_path(spider, request) if not exists(rpath): os.makedirs(rpath) metadata = { 'url': request.url, 'method': request.method, 'status': response.status, 'timestamp': time(), } with open(join(rpath, 'meta'), 'wb') as f: f.write(repr(metadata)) with open(join(rpath, 'pickled_meta'), 'wb') as f: pickle.dump(metadata, f, protocol=2) with open(join(rpath, 'response_headers'), 'wb') as f: f.write(headers_dict_to_raw(response.headers)) with open(join(rpath, 'response_body'), 'wb') as f: f.write(response.body) with open(join(rpath, 'request_headers'), 'wb') as f: f.write(headers_dict_to_raw(request.headers)) with open(join(rpath, 'request_body'), 'wb') as f: f.write(request.body)
def store_response(self, spider, request, response): """Store the given response in the cache.""" rpath = self._get_request_path(spider, request) if not exists(rpath): os.makedirs(rpath) metadata = { 'url': request.url, 'method': request.method, 'status': response.status, 'response_url': response.url, 'timestamp': time(), } with open(join(rpath, 'meta'), 'wb') as f: f.write(repr(metadata)) with open(join(rpath, 'pickled_meta'), 'wb') as f: pickle.dump(metadata, f, protocol=2) with open(join(rpath, 'response_headers'), 'wb') as f: f.write(headers_dict_to_raw(response.headers)) with open(join(rpath, 'response_body'), 'wb') as f: f.write(response.body) with open(join(rpath, 'request_headers'), 'wb') as f: f.write(headers_dict_to_raw(request.headers)) with open(join(rpath, 'request_body'), 'wb') as f: f.write(request.body)
def to_string(self): return headers_dict_to_raw(self)