def release_query_id_and_return_results(self, engine, query_id):
"""
Instructs the backend to return the results of the query
and then to release the query ID.
Arguments:
engine: backend engine to contact
query_id: ID of the query
Results:
A List with all the results.
It raises a ResultReadError if the results cannot be read.
"""
backend_port = self.visor_opts.engines_dict[engine]['backend_port']
ses = backend_client.Session(backend_port)
# get ranking from backend
rlist = ses.get_ranking(query_id)
# Get all query results (as above) or just a subset.
# Note that the number of results can also be restricted
# in the backend. This last option seems better as it can
# be adjusted PER engine
#rsubset = ses.get_ranking_subset(query_id, 0, 5000)
#rlist = rsubset[0]
# release query id in backend
ses.release_query_id(query_id)
# check if something went wrong
if isinstance(rlist, bool) and not rlist:
raise errors.ResultReadError(
'Could not read in results from backend')
return rlist
def _compute_feat(out_dict):
"""
Performs the computation of features for one file.
Arguments:
out_dicts: Dictionary with at least the following entries:
'clean_fn': Path to input file for processing.
'feat_fn': Path to file where features are stored.
'backend_port': Communication port with the backend
'anno': 1 if the image is a positive training image, -1
if it is a negative training image, 0 otherwise.
'query_id': id of the query being executed.
'from_dataset': Boolean indicating whether the training image
is part of the dataset or not.
'extra_params': Dictionary containing any other parameter that
can be useful to the backend.
Returns:
It raises FeatureCompError is the backend reports something went
wrong with the feature computation.
"""
impath = out_dict['clean_fn']
featpath = out_dict['feat_fn']
ses = backend_client.Session(out_dict['backend_port'])
is_symlink = os.path.islink(impath)
if is_symlink:
canonical_impath = os.path.realpath(impath)
else:
canonical_impath = impath
is_symlink_feat = os.path.islink(featpath)
if is_symlink_feat:
canonical_featpath = os.path.realpath(featpath)
else:
canonical_featpath = featpath
if out_dict['anno'] == 1:
call_succeeded = ses.add_pos_trs(out_dict['query_id'],
canonical_impath, canonical_featpath,
out_dict['from_dataset'],
out_dict['extra_params'])
elif out_dict['anno'] == -1:
call_succeeded = ses.add_neg_trs(out_dict['query_id'],
canonical_impath, canonical_featpath,
out_dict['from_dataset'],
out_dict['extra_params'])
else:
call_succeeded = True
if not call_succeeded:
raise models.errors.FeatureCompError('Failed computing features of ' + canonical_impath)
if is_symlink:
sys.stdout.write('computed features for: ' + impath +
' (=>' + canonical_impath + ')\n')
else:
sys.stdout.write('computed features for: ' + impath + '\n')
def is_backend_available(self):
"""
Checks whether the backend engines are running or not.
Returns:
True, only if ALL configured backend engines are reachable.
It returns False otherwise.
"""
backends_available = len(self.opts.engines_dict) > 0
if backends_available:
for engine in self.opts.engines_dict:
ses = backend_client.Session(
self.opts.engines_dict[engine]['backend_port'])
backends_available = backends_available and ses.self_test()
return backends_available
def _save_annotations(self, query, fname, query_id):
"""
Instructs the backend to save the annotations
of a query.
Arguments:
query: query in dictionary form.
query_id: id of the query.
fname: Full path to the annotations file.
Results:
It raises a AnnoSaveLoadError in case of error.
"""
backend_port = self.visor_opts.engines_dict[
query['engine']]['backend_port']
ses = backend_client.Session(backend_port)
if not ses.save_annotations(query_id, fname):
raise errors.AnnoSaveLoadError('Could not save annotations to %s' %
fname)
def _load_annotations_and_trs(self, query, fname, query_id=None):
"""
Instructs the backend to load the annotations
of a query.
Arguments:
query: query in dictionary form.
query_id: id of the query.
fname: Full path to the annotations file.
Results:
True on success, False otherwise.
"""
if not os.path.isfile(fname):
return False
backend_port = self.visor_opts.engines_dict[
query['engine']]['backend_port']
ses = backend_client.Session(backend_port)
loaded = ses.load_annotations_and_trs(query_id, fname)
#if not loaded:
# raise errors.AnnoSaveLoadError('Could not get annotations from %s' % fname)
return loaded
def get_query_id(self, engine, dsetname):
"""
Contacts the backend requesting a new Query ID.
Returns a new unique query id which can be used with a
subsequent call to process.
Arguments:
engine: backend engine to contact
dsetname: dataset used for the query
Returns:
A positive integer number corresponding to the new query ID.
It raises a QueryIdError if the ID is negative or 0.
"""
ses = backend_client.Session(
self.visor_opts.engines_dict[engine]['backend_port'])
query_id = ses.get_query_id(dsetname)
if query_id <= 0:
raise errors.QueryIdError(
'Could not get a Query ID from VISOR backend')
return query_id
def _load_classifier(self, query, fname, query_id):
"""
Instructs the backend to load the classifier
for the query.
Arguments:
query: query in dictionary form.
query_id: id of the query.
fname: Full path to the classifier file.
Results:
True on success.
It raises a ClassifierSaveLoadError in case of error.
"""
if not os.path.isfile(fname):
return False
backend_port = self.visor_opts.engines_dict[
query['engine']]['backend_port']
ses = backend_client.Session(backend_port)
if not ses.load_classifier(query_id, fname):
raise errors.ClassifierSaveLoadError(
'Could not load classifier from %s' % fname)
return True
def _get_annotations(self, query, fname, query_id=None):
"""
Retrieved the annotations of a query from the backend.
Arguments:
query: query in dictionary form.
query_id: id of the query.
fname: Full path to the annotations file.
Results:
On success, it returns a list of dictionaries with the
annotations and paths to the training images.
It raises a AnnoSaveLoadError in case of error.
"""
if not os.path.isfile(fname):
return []
backend_port = self.visor_opts.engines_dict[
query['engine']]['backend_port']
ses = backend_client.Session(backend_port)
annos = ses.get_annotations(query_id, fname)
if not annos:
raise errors.AnnoSaveLoadError(
'Could not get annotations from %s' % fname)
return annos
def process(self, query, query_id, shared_vars, opts, user_ses_id=None):
"""
Executes a query using the VISOR backend
Arguments:
query: query in dictionary form.
query_id: ID of the query
shared_vars: holder of global shared variables
opts: current configuration of options for the visor engine
user_ses_id: user session id
Returns:
'None' in case of success, as the results are saved to the caches.
It can raise several exceptions, depending on the error:
ValueError: In the presence of incorrect options
ClassifierTrainError: In case the training fails
Exception: In case of any other error
"""
if not isinstance(opts, param_sets.VisorEngineProcessOpts):
raise ValueError(
'opts must be of type param_sets.VisorEngineProcessOpts')
backend_port = self.visor_opts.engines_dict[
query['engine']]['backend_port']
ses = backend_client.Session(backend_port)
try:
# check if classifier has been trained and saved to file or not
if not self.compdata_cache.load_classifier(
query, query_id=query_id, user_ses_id=user_ses_id):
# if not try loading in precomputed annotations and features
if not self.compdata_cache.load_annotations_and_trs(
query, query_id=query_id, user_ses_id=user_ses_id):
# if both classifier and annotations could not be loaded, start
# computation from scratch now...
print('Getting query handler for Query ID: %d' % query_id)
query_handler = factory.get_query_handler(
query, query_id, backend_port, self.compdata_cache,
opts)
print('Computing features for Query ID: %d' % query_id)
shared_vars.exectime_processing = query_handler.compute_feats(
shared_vars)
self.compdata_cache.save_annotations(
query, query_id=query_id, user_ses_id=user_ses_id)
# train classifier now if it hasn't been already
print('Training classifier for Query ID: %d' % query_id)
shared_vars.state = States.training
with timing.TimerBlock() as timer:
anno_path = None
# if the query is in the exclude list, specify the annotation path as it can be used in the backend
if self.result_cache[
query['engine']].query_in_exclude_list(
query, ses_id=user_ses_id):
anno_path = self.compdata_cache._get_annotations_fname(
query)
error = ses.train(query_id, anno_path)
if error:
if isinstance(error, str):
error = error.encode('ascii')
raise errors.ClassifierTrainError(
'Could not train classifier. Backend response: ' +
str(error))
shared_vars.exectime_training = timer.interval
# and save it to file
self.compdata_cache.save_classifier(query,
query_id=query_id,
user_ses_id=user_ses_id)
else:
print('Loaded classifier from file for Query ID: %d' %
query_id)
# compute ranking
do_regular_rank = False
if opts.rf_rank_type == RfRankTypes.full:
# set flag to do regular ranking
do_regular_rank = True
elif opts.rf_rank_type == RfRankTypes.topn:
if 'prev_qsid' in query:
# reranking of top n results from previous query
prev_qsid = query['prev_qsid']
topn = opts.rf_rank_topn
print(
'Computing reranking of top %d results from previous query (%s) for Query ID %d'
% (topn, prev_qsid, query_id))
# Look up ranking list for previous query (by query session ID)
prev_rlist = self.result_cache[
query['engine']].get_results_from_query_ses_id(
prev_qsid, user_ses_id)
# extract dataset string IDs for top N results from the ranking list
subset_ids = [
ritem['path'] for ritem in prev_rlist[0:topn]
]
with timing.TimerBlock() as timer:
ses.rank(query_id, subset_ids)
shared_vars.exectime_ranking = timer.interval
else:
# do regular rank if no previous query session id was specified
do_regular_rank = True
else:
# unrecognised ranking type
raise ValueError(
'Unrecognised value for rf_rank_type parameter')
if do_regular_rank:
# regular ranking
print('Computing ranking for Query ID: %d' % query_id)
shared_vars.state = States.ranking
with timing.TimerBlock() as timer:
ses.rank(query_id)
shared_vars.exectime_ranking = timer.interval
# mark results as ready
shared_vars.state = States.results_ready
except Exception as e:
# determine if on cache exclude list
excl_query = self.result_cache[
query['engine']].query_in_exclude_list(query,
ses_id=user_ses_id)
# do the clean up below only if the query is not in the excluded list, to avoid removing previous valid results by mistake
if not excl_query:
# clear cache before leaving the method
self.compdata_cache.delete_compdata(query)
self.result_cache[query['engine']].delete_results(
query, for_all_datasets=True)
# logging
print(
'Unexpected error occurred while processing Query ID %d: %s' %
(query_id, str(e)))
traceback.print_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2],
limit=2)
shared_vars.state = States.fatal_error_or_socket_timeout
shared_vars.err_msg = str(e)
print(traceback.format_exc())
raise e