def exists_in_cache(req): reqid = gen_hash(req) exists = os.path.exists cache_loc = DiskCachedResponse._get_cache_location() reqfname = os.path.join(cache_loc, reqid) return exists(reqfname + ".headers") and exists(reqfname + ".body") \ and exists(reqfname + ".code") and exists(reqfname + ".msg")
def store_in_cache(request, response): reqid = gen_hash(request) cache_loc = DiskCachedResponse._get_cache_location() fname = os.path.join(cache_loc, reqid) try: f = open(fname + ".headers", "w") headers = str(response.info()) f.write(headers) f.close() except Exception, e: msg = 'cache.py: Could not save headers file. Exception: "%s".' raise FileException(msg % e)
def __init__(self, request): self._hash_id = gen_hash(request) self.from_cache = True self.url = request.get_full_url() self._code = None self._msg = None self._headers = None self._encoding = None # Call parent's __init__ self._body = self._get_from_response(CachedResponse.PART_BODY) StringIO.StringIO.__init__(self, self._body) # This kludge is necessary, do not touch! class PlaceHolder: sock = None self._connection = PlaceHolder()
def store_in_cache(request, response): reqid = gen_hash(request) cache_loc = DiskCachedResponse._get_cache_location() fname = os.path.join(cache_loc, reqid) try: f = open(fname + ".headers", "w") headers = str(response.info()) f.write(headers) f.close() except Exception as e: msg = 'cache.py: Could not save headers file. Exception: "%s".' raise FileException(msg % e) try: body = response.read() except Exception as e: om.out.error('cache.py: Timeout while fetching page body.') else: try: f = open(fname + ".body", "w") f.write(body) f.close() except Exception as e: msg = 'cache.py: Could not save body file. Exception: "%s".' raise FileException(msg % e) try: f = open(fname + ".code", "w") # minimal validation before storing the data to disk int(response.code) # store data to disk f.write(str(response.code)) f.close() except Exception as e: msg = 'cache.py: Could not save code file. Exception: "%s".' raise FileException(msg % e) try: f = open(fname + ".msg", "w") f.write(str(response.msg)) f.close() except Exception as e: msg = 'cache.py: Could not save msg file. Exception: "%s".' raise FileException(msg % e)
def __init__(self, request): self._hash_id = gen_hash(request) self.from_cache = True self.url = request.get_full_url() self._code = None self._msg = None self._headers = None self._encoding = None self._time = None # Call parent's __init__ self._body = self._get_from_response(CachedResponse.PART_BODY) StringIO.StringIO.__init__(self, self._body) # This kludge is necessary, do not touch! class PlaceHolder: sock = None self._connection = PlaceHolder()
def store_in_cache(request, response): # Create the http response object resp = HTTPResponse.from_httplib_resp(response, original_url=request.url_object) resp.set_id(response.id) resp.set_alias(gen_hash(request)) hi = HistoryItem() hi.request = request hi.response = resp # Now save them try: hi.save() except sqlite3.Error, e: msg = 'A sqlite3 error was raised: "%s".' % e if 'disk' in str(e).lower(): msg += ' Please check if your disk is full.' raise ScanMustStopException(msg)