def test_05_recheck(self): # make the request, and then immediately look up the id in the cache resp = requests.post(lookup_url + "10.1371/journal.pone.0035089") queued = True waited = 0 while queued: time.sleep(0.2) waited += 0.2 cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089") queued = cached_result.get("queued", False) assert waited < 10 # only give it 10 seconds, max, that should be easily enough archive_result = models.Record.check_archive("doi:10.1371/journal.pone.0035089") assert archive_result["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089" assert archive_result["license"][0]["title"] == "UK Open Government Licence (OGL)" cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089") assert cached_result["identifier"]["canonical"] == "doi:10.1371/journal.pone.0035089" assert "queued" not in cached_result assert cached_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089" assert cached_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)" # now we know it is done, re-request the result set resp = requests.post(lookup_url + "10.1371/journal.pone.0035089") obj = json.loads(resp.text) assert len(obj["results"]) == 1 assert len(obj["processing"]) == 0 assert len(obj["errors"]) == 0 assert obj["requested"] == 1
def cache_is_empty(gone, there): for i in gone: id = str(i) * 3 assert cache.check_cache(id) is None, id for i in there: id = str(i) * 3 assert cache.check_cache(id) is not None, id
def test_03_back_end(self): # kick off a known good doi lookup record = {"identifier" : {"id" : "http://dx.doi.org/10.1371/journal.pone.0035089", "type" : "doi", "canonical" : "doi:10.1371/journal.pone.0035089"}, "queued" : True} record = models.MessageObject(record=record) a = workflow._start_back_end(record) # wait for the process to finish waited = 0 while not a.ready(): time.sleep(0.2) waited += 0.2 assert waited < 10 # only give it 10 seconds, max, that should be easily enough # now we need to wait until the buffer itself gets flushed time.sleep(config.BUFFER_FLUSH_PERIOD) # we should find the record in the async result, the archive, and the cache async_result = a.result assert async_result['identifier']['canonical'] == "doi:10.1371/journal.pone.0035089" assert not async_result.get("queued", False) #assert "queued" not in async_result assert async_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089" assert async_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)" archive_result = models.Record.check_archive("doi:10.1371/journal.pone.0035089") assert archive_result["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089" assert archive_result["license"][0]["title"] == "UK Open Government Licence (OGL)" cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089") cached_result = cached_result.record assert cached_result['identifier']['canonical'] == "doi:10.1371/journal.pone.0035089" assert not cached_result.get("queued", False) #assert "queued" not in cached_result assert cached_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089" assert cached_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)"
def test_03_back_end(self): # kick off a known good doi lookup record = { "identifier": { "id": "http://dx.doi.org/10.1371/journal.pone.0035089", "type": "doi", "canonical": "doi:10.1371/journal.pone.0035089", }, "queued": True, } a = workflow._start_back_end(record) # wait for the process to finish waited = 0 while not a.ready(): time.sleep(0.2) waited += 0.2 assert waited < 10 # only give it 10 seconds, max, that should be easily enough # we should find the record in the async result, the archive, and the cache async_result = a.result assert async_result["identifier"]["canonical"] == "doi:10.1371/journal.pone.0035089" assert "queued" not in async_result assert async_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089" assert async_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)" archive_result = models.Record.check_archive("doi:10.1371/journal.pone.0035089") assert archive_result["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089" assert archive_result["license"][0]["title"] == "UK Open Government Licence (OGL)" cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089") assert cached_result["identifier"]["canonical"] == "doi:10.1371/journal.pone.0035089" assert "queued" not in cached_result assert cached_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089" assert cached_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)"
def test_04_check_cache_corrupt(self): client = redis.StrictRedis(host=test_host, port=test_port, db=test_db) client.setex("corrupt", 2, "{askjdfafds}") result = cache.check_cache("corrupt") assert result is None corrupt = client.get("corrupt") assert corrupt is None
def test_03_check_cache_exists(self): client = redis.StrictRedis(host=test_host, port=test_port, db=test_db) obj = {"key" : "value"} client.set("exists", json.dumps(obj)) result = cache.check_cache("exists") assert result.has_key("key") assert result["key"] == "value" client.delete("exists")
def test_04_processing(self): # make the request, and then immediately look up the id in the cache resp = requests.post(lookup_url + "10.1371/journal.pone.0035089") cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089") # make some assertions about the response and then the cached record (that it is queued) obj = json.loads(resp.text) assert len(obj["processing"]) == 1 assert cached_result["identifier"]["canonical"] == "doi:10.1371/journal.pone.0035089" assert "queued" in cached_result
def _check_cache(record): """ check the live local cache for a copy of the object. Whatever we find, return it (a record of a queued item, a full item, or None) arguments: record -- an OAG record object, see the module documentation for details returns: - None if nothing in the cache or the cached record is found to be stale - OAG record object if one is found """ if record.canonical is None: raise models.LookupException("can't look anything up in the cache without a canonical id") log.debug("checking cache for key: " + record.canonical) cached_copy = cache.check_cache(record.canonical) # if it's not in the cache, then return if cached_copy is None: log.debug(record.canonical + " not found in cache") return None # if the cached copy exists ... # first check to see if the cached copy is already on the queue if cached_copy.queued: log.debug(record.canonical + " is in the cache and is queued for processing") return cached_copy # next check to see if this is a record of an error if cached_copy.has_error(): log.debug(record.canonical + " is in the cache, and has a permanent error attached to it") return cached_copy # next check to see if the cached copy has a bibjson record in it if cached_copy.has_bibjson(): # if it does, we need to see if the record is stale. If so, we remember that fact, # and we'll deal with updating stale items later (once we've checked bibserver) if _is_stale(cached_copy): log.debug(record.canonical + " is in the cache but is a stale record") _invalidate_cache(record) return None # otherwise, just return the cached copy # log.debug(record['identifier']['canonical'] + " is in the cache") log.debug(record.canonical + " is in the cache") return cached_copy
def _check_cache(record): """ check the live local cache for a copy of the object. Whatever we find, return it (a record of a queued item, a full item, or None) """ if not record.has_key('identifier'): raise model_exceptions.LookupException("no identifier in record object") if not record['identifier'].has_key('canonical'): raise model_exceptions.LookupException("can't look anything up in the cache without a canonical id") log.debug("checking cache for key: " + record['identifier']['canonical']) cached_copy = cache.check_cache(record['identifier']['canonical']) # if it's not in the cache, then return if cached_copy is None: log.debug(record['identifier']['canonical'] + " not found in cache") return None # if the cached copy exists ... # first check to see if the cached copy is already on the queue if cached_copy.get('queued', False): log.debug(record['identifier']['canonical'] + " is in the cache and is queued for processing") return cached_copy # next check to see if the cached copy has a bibjson record in it if cached_copy.has_key('bibjson'): # if it does, we need to see if the record is stale. If so, we remember that fact, # and we'll deal with updating stale items later (once we've checked bibserver) if _is_stale(cached_copy['bibjson']): log.debug(record['identifier']['canonical'] + " is in the cache but is a stale record") _invalidate_cache(record) return None # otherwise, just return the cached copy log.debug(record['identifier']['canonical'] + " is in the cache") return cached_copy
def test_02_check_cache_not_exists(self): # check the cache for a value which doesn't exist result = cache.check_cache("not_exists") assert result is None