def test_05_recheck(self):
# make the request, and then immediately look up the id in the cache
resp = requests.post(lookup_url + "10.1371/journal.pone.0035089")
queued = True
waited = 0
while queued:
time.sleep(0.2)
waited += 0.2
cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089")
queued = cached_result.get("queued", False)
assert waited < 10 # only give it 10 seconds, max, that should be easily enough
archive_result = models.Record.check_archive("doi:10.1371/journal.pone.0035089")
assert archive_result["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert archive_result["license"][0]["title"] == "UK Open Government Licence (OGL)"
cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089")
assert cached_result["identifier"]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert "queued" not in cached_result
assert cached_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert cached_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)"
# now we know it is done, re-request the result set
resp = requests.post(lookup_url + "10.1371/journal.pone.0035089")
obj = json.loads(resp.text)
assert len(obj["results"]) == 1
assert len(obj["processing"]) == 0
assert len(obj["errors"]) == 0
assert obj["requested"] == 1
def cache_is_empty(gone, there):
for i in gone:
id = str(i) * 3
assert cache.check_cache(id) is None, id
for i in there:
id = str(i) * 3
assert cache.check_cache(id) is not None, id
def test_03_back_end(self):
# kick off a known good doi lookup
record = {"identifier" : {"id" : "http://dx.doi.org/10.1371/journal.pone.0035089", "type" : "doi", "canonical" : "doi:10.1371/journal.pone.0035089"}, "queued" : True}
record = models.MessageObject(record=record)
a = workflow._start_back_end(record)
# wait for the process to finish
waited = 0
while not a.ready():
time.sleep(0.2)
waited += 0.2
assert waited < 10 # only give it 10 seconds, max, that should be easily enough
# now we need to wait until the buffer itself gets flushed
time.sleep(config.BUFFER_FLUSH_PERIOD)
# we should find the record in the async result, the archive, and the cache
async_result = a.result
assert async_result['identifier']['canonical'] == "doi:10.1371/journal.pone.0035089"
assert not async_result.get("queued", False)
#assert "queued" not in async_result
assert async_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert async_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)"
archive_result = models.Record.check_archive("doi:10.1371/journal.pone.0035089")
assert archive_result["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert archive_result["license"][0]["title"] == "UK Open Government Licence (OGL)"
cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089")
cached_result = cached_result.record
assert cached_result['identifier']['canonical'] == "doi:10.1371/journal.pone.0035089"
assert not cached_result.get("queued", False)
#assert "queued" not in cached_result
assert cached_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert cached_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)"
def test_03_back_end(self):
# kick off a known good doi lookup
record = {
"identifier": {
"id": "http://dx.doi.org/10.1371/journal.pone.0035089",
"type": "doi",
"canonical": "doi:10.1371/journal.pone.0035089",
},
"queued": True,
}
a = workflow._start_back_end(record)
# wait for the process to finish
waited = 0
while not a.ready():
time.sleep(0.2)
waited += 0.2
assert waited < 10 # only give it 10 seconds, max, that should be easily enough
# we should find the record in the async result, the archive, and the cache
async_result = a.result
assert async_result["identifier"]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert "queued" not in async_result
assert async_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert async_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)"
archive_result = models.Record.check_archive("doi:10.1371/journal.pone.0035089")
assert archive_result["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert archive_result["license"][0]["title"] == "UK Open Government Licence (OGL)"
cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089")
assert cached_result["identifier"]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert "queued" not in cached_result
assert cached_result["bibjson"]["identifier"][0]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert cached_result["bibjson"]["license"][0]["title"] == "UK Open Government Licence (OGL)"
def test_04_check_cache_corrupt(self):
client = redis.StrictRedis(host=test_host, port=test_port, db=test_db)
client.setex("corrupt", 2, "{askjdfafds}")
result = cache.check_cache("corrupt")
assert result is None
corrupt = client.get("corrupt")
assert corrupt is None
def test_03_check_cache_exists(self):
client = redis.StrictRedis(host=test_host, port=test_port, db=test_db)
obj = {"key" : "value"}
client.set("exists", json.dumps(obj))
result = cache.check_cache("exists")
assert result.has_key("key")
assert result["key"] == "value"
client.delete("exists")
def test_04_processing(self):
# make the request, and then immediately look up the id in the cache
resp = requests.post(lookup_url + "10.1371/journal.pone.0035089")
cached_result = cache.check_cache("doi:10.1371/journal.pone.0035089")
# make some assertions about the response and then the cached record (that it is queued)
obj = json.loads(resp.text)
assert len(obj["processing"]) == 1
assert cached_result["identifier"]["canonical"] == "doi:10.1371/journal.pone.0035089"
assert "queued" in cached_result
def _check_cache(record):
"""
check the live local cache for a copy of the object. Whatever we find,
return it (a record of a queued item, a full item, or None)
arguments:
record -- an OAG record object, see the module documentation for details
returns:
- None if nothing in the cache or the cached record is found to be stale
- OAG record object if one is found
"""
if record.canonical is None:
raise models.LookupException("can't look anything up in the cache without a canonical id")
log.debug("checking cache for key: " + record.canonical)
cached_copy = cache.check_cache(record.canonical)
# if it's not in the cache, then return
if cached_copy is None:
log.debug(record.canonical + " not found in cache")
return None
# if the cached copy exists ...
# first check to see if the cached copy is already on the queue
if cached_copy.queued:
log.debug(record.canonical + " is in the cache and is queued for processing")
return cached_copy
# next check to see if this is a record of an error
if cached_copy.has_error():
log.debug(record.canonical + " is in the cache, and has a permanent error attached to it")
return cached_copy
# next check to see if the cached copy has a bibjson record in it
if cached_copy.has_bibjson():
# if it does, we need to see if the record is stale. If so, we remember that fact,
# and we'll deal with updating stale items later (once we've checked bibserver)
if _is_stale(cached_copy):
log.debug(record.canonical + " is in the cache but is a stale record")
_invalidate_cache(record)
return None
# otherwise, just return the cached copy
# log.debug(record['identifier']['canonical'] + " is in the cache")
log.debug(record.canonical + " is in the cache")
return cached_copy
def _check_cache(record):
"""
check the live local cache for a copy of the object. Whatever we find,
return it (a record of a queued item, a full item, or None)
"""
if not record.has_key('identifier'):
raise model_exceptions.LookupException("no identifier in record object")
if not record['identifier'].has_key('canonical'):
raise model_exceptions.LookupException("can't look anything up in the cache without a canonical id")
log.debug("checking cache for key: " + record['identifier']['canonical'])
cached_copy = cache.check_cache(record['identifier']['canonical'])
# if it's not in the cache, then return
if cached_copy is None:
log.debug(record['identifier']['canonical'] + " not found in cache")
return None
# if the cached copy exists ...
# first check to see if the cached copy is already on the queue
if cached_copy.get('queued', False):
log.debug(record['identifier']['canonical'] + " is in the cache and is queued for processing")
return cached_copy
# next check to see if the cached copy has a bibjson record in it
if cached_copy.has_key('bibjson'):
# if it does, we need to see if the record is stale. If so, we remember that fact,
# and we'll deal with updating stale items later (once we've checked bibserver)
if _is_stale(cached_copy['bibjson']):
log.debug(record['identifier']['canonical'] + " is in the cache but is a stale record")
_invalidate_cache(record)
return None
# otherwise, just return the cached copy
log.debug(record['identifier']['canonical'] + " is in the cache")
return cached_copy
def test_02_check_cache_not_exists(self):
# check the cache for a value which doesn't exist
result = cache.check_cache("not_exists")
assert result is None
