def _lookup_score_in_map(result_id, model_name):
logger.info("Looking up {0} in {1}".format(model_name, result_id))
score_map_result = self._process_score_map.AsyncResult(result_id)
try:
score_map = score_map_result.get(timeout=self.timeout)
except celery.exceptions.TimeoutError:
raise errors.TimeoutError(
"Timed out after {0} seconds.".format(self.timeout))
logger.info("Found {0} in {1}!".format(model_name, result_id))
return score_map[model_name]
def _process_missing_scores(self,
request,
missing_model_set_revs,
root_caches,
inprogress_results=None):
logger.debug("Processing missing scores {0}:{1}.".format(
request.context_name, missing_model_set_revs))
context = self[request.context_name]
inprogress_results = inprogress_results or {}
# Generate score results
results = {}
for missing_models, rev_ids in missing_model_set_revs.items():
for rev_id in rev_ids:
injection_cache = request.injection_caches.get(rev_id)
if rev_id not in root_caches:
for model_name in missing_models:
task_id = context.format_id_string(
model_name,
rev_id,
request,
injection_cache=injection_cache)
self.application.backend.mark_as_failure(
task_id, RuntimeError("Never started"))
continue
root_cache = {
str(k): v
for k, v in root_caches[rev_id].items()
}
result = self._process_score_map.delay(request.to_json(),
list(missing_models),
rev_id, root_cache)
self._lock_process(missing_models, rev_id, request,
injection_cache, result.id)
for model_name in missing_models:
if rev_id in results:
results[rev_id][model_name] = result
else:
results[rev_id] = {model_name: result}
# Read results
rev_scores = {}
score_errors = {}
combined_results = chain(inprogress_results.items(), results.items())
for rev_id, model_results in combined_results:
injection_cache = request.injection_caches.get(rev_id)
if rev_id not in rev_scores:
rev_scores[rev_id] = {}
for model_name, score_result in model_results.items():
try:
task_result = score_result.get(timeout=self.timeout)
except celery.exceptions.TimeoutError:
timeout_error = errors.TimeoutError(
"Timed out after {0} seconds.".format(self.timeout))
score_errors[rev_id] = timeout_error
self.application.backend.mark_as_failure(
score_result.id, timeout_error)
except Exception as error:
score_errors[rev_id] = error
else:
if model_name in task_result:
rev_scores[rev_id][model_name] = task_result[
model_name]
else:
raise RuntimeError('Model is not in the task but '
'the task locked the model')
key = context.format_id_string(
model_name,
rev_id,
request,
injection_cache=injection_cache)
self.task_tracker.release(key)
return rev_scores, score_errors
def _process_missing_scores(self,
request,
missing_model_set_revs,
root_caches,
inprogress_results=None):
logger.debug("Processing missing scores {0}:{1}.".format(
request.context_name, missing_model_set_revs))
context = self[request.context_name]
inprogress_results = inprogress_results or {}
# Generate score results
results = {}
for missing_models, rev_ids in missing_model_set_revs.items():
for rev_id in rev_ids:
injection_cache = request.injection_caches.get(rev_id)
if rev_id not in root_caches:
for model_name in missing_models:
task_id = context.format_id_string(
model_name,
rev_id,
request,
injection_cache=injection_cache)
self.application.backend.mark_as_failure(
task_id, RuntimeError("Never started"))
continue
root_cache = {
str(k): v
for k, v in root_caches[rev_id].items()
}
result = self._process_score_map.delay(request, missing_models,
rev_id, root_cache)
for model_name in missing_models:
task_id = context.format_id_string(
model_name,
rev_id,
request,
injection_cache=injection_cache)
score_result = self._lookup_score_in_map.apply_async(
args=(result.id, model_name),
task_id=task_id,
expires=self.timeout)
if rev_id in results:
results[rev_id][model_name] = score_result
else:
results[rev_id] = {model_name: score_result}
# Read results
rev_scores = {}
score_errors = {}
combined_results = chain(inprogress_results.items(), results.items())
for rev_id, model_results in combined_results:
if rev_id not in rev_scores:
rev_scores[rev_id] = {}
for model_name, score_result in model_results.items():
try:
rev_scores[rev_id][model_name] = \
score_result.get(timeout=self.timeout)
except celery.exceptions.TimeoutError:
timeout_error = errors.TimeoutError(
"Timed out after {0} seconds.".format(self.timeout))
score_errors[rev_id] = timeout_error
self.application.backend.mark_as_failure(
score_result.id, timeout_error)
except Exception as error:
score_errors[rev_id] = error
return rev_scores, score_errors