def __init__(self, query, visor_engine, on_cache_exclude_list):
"""
Initializes the worker.
Arguments:
query: query in dictionary form
visor_engine: visor engine object, which will execute the
query
on_cache_exclude_list: boolean indicating if the query in
on the list of excluded cached text
queries.
"""
# get a new query ID
# print ('Generating a new query ID...')
self.qid = visor_engine.get_query_id(query['engine'],
query['dsetname'])
self.query = query
self.qindex = query_translations.get_qhash(query)
# print ('Initializing query namespace...')
self.shared_vars = manager.Namespace()
# configure initial shared memory namespace values
self.shared_vars.state = opts.States.processing
self.shared_vars.postrainimg_paths = []
self.shared_vars.curatedtrainimgs_paths = []
self.shared_vars.negtrainimg_count = 0
self.shared_vars.exectime_processing = 0.0
self.shared_vars.exectime_training = 0.0
self.shared_vars.exectime_ranking = 0.0
self.shared_vars.err_msg = ''
def add_results(self, rlist, query):
"""
Stores the results of a query into the cache.
Arguments:
query: query in dictionary form.
rlist: the list of results.
"""
qhash = query_translations.get_qhash(query)
self._max_size_cache.add_data(
rlist, (qhash, query['qtype'], query['engine'], query['dsetname']))
def get_results(self, query):
"""
Retrieves the cached results of a query.
Arguments:
query: query in dictionary form.
Returns:
the cached list of results.
"""
qhash = query_translations.get_qhash(query)
return self._max_size_cache.get_data(
(qhash, query['qtype'], query['engine'], query['dsetname']))
def add_results(self, rlist, ses_id, query):
"""
Adds the results of a query to a session cache
Arguments:
rlist: List of results associated to the query.
query: query in dictionary form.
ses_id: ID of the session.
"""
qhash = query_translations.get_qhash(query, include_qtype=True)
self._session_cache.add_data(
rlist, ses_id, (qhash, query['engine'], query['dsetname']))
def start_query(self, query, user_ses_id=None, force_new_worker=False):
"""
Starts a new worker and return its status (or if the worker
already exists, return its status directly).
Arguments:
query: query in dictionary form.
user_ses_id: user session id.
force_new_worker: Boolean instructing this function to
mandatorily create a new worker for the
query.
Returns:
A QueryStatus object.
"""
# start a query only if an existing query of the same name does not exist
worker = None
qindex = query_translations.get_qhash(query)
for key, value in self._workers.items():
if value.qindex == qindex:
if not force_new_worker:
# return the already created worker
worker = value
else:
# if force_new_worker is True, create a new worker
# for the same qhash and remove the previous one
del self._workers[key]
break
if not worker:
# determine if on cache exclude list
excl_query = self.result_cache[
query['engine']].query_in_exclude_list(query,
ses_id=user_ses_id)
# print ('Initializing query worker process...')
try:
worker = QueryWorker(query, self._engine, excl_query)
self._workers[worker.qid] = worker
# start the query
# print ('Launching query process...')
self.process_pool.apply_async(
func=call_it,
args=(self._engine, 'process',
(query, worker.qid, worker.shared_vars,
self._proc_opts, user_ses_id)))
except errors.QueryIdError:
# this exception is triggered by the constructor of QueryWorker
# if the backend cannot be contacted
return query_data.QueryStatus(
state=opts.States.fatal_error_or_socket_timeout)
# return query status (including qid as a field)
return worker.get_status()
def _get_worker_from_definition(self, query):
"""
Gets the worker instance assigned to a query.
Arguments:
query: query in dictionary form.
Returns:
A QueryWorker object
"""
qindex = query_translations.get_qhash(query)
for key, value in self._workers.items():
if value.qindex == qindex:
return value
return None
def get_results(self, ses_id, query=None):
"""
Gets the results of a query from a session.
Arguments:
query: query in dictionary form.
ses_id: ID of the session.
Returns:
The list of results associated to the query.
"""
if query:
qhash = query_translations.get_qhash(query, include_qtype=True)
return self._session_cache.get_data(
ses_id, (qhash, query['engine'], query['dsetname']))
# returns data dictionary directly if no query key was specified
return self._session_cache.get_data(ses_id)
def add_query_to_exclude_list(self, query, ses_id=None):
"""
Adds a query to the query exclude list.
Arguments:
query: query in dictionary form.
ses_id: ID of the session.
"""
# if no session ID specified, allow storage under a 'universal' tag
if not ses_id:
ses_id = 'universal'
qhash = query_translations.get_qhash(query,
include_engine=True,
include_qtype=True)
# print 'Adding query to exclude list: (%s), ses_id: %s, class: %s' %
# (qhash, ses_id, self.__class__.__name__)
# add dummy data as only the keys of excluded queries are important
self._session_cache.add_data('excluded query', ses_id, qhash)
def __init__(self, query, visor_engine, on_cache_exclude_list):
"""
Initializes the worker.
Arguments:
query: query in dictionary form
visor_engine: visor engine object, which will execute the
query
on_cache_exclude_list: boolean indicating if the query in
on the list of excluded cached text
queries.
"""
# get a new query ID
# print 'Generating a new query ID...'
self.qid = visor_engine.get_query_id(query['engine'],
query['dsetname'])
self.query = query
self.qindex = query_translations.get_qhash(query)
# print 'Initializing query namespace...'
self.shared_vars = manager.Namespace()
# configure initial shared memory namespace values
self.shared_vars.state = models.opts.States.processing
self.shared_vars.postrainimg_paths = []
self.shared_vars.curatedtrainimgs_paths = []
self.shared_vars.negtrainimg_count = 0
self.shared_vars.exectime_processing = 0.0
self.shared_vars.exectime_training = 0.0
self.shared_vars.exectime_ranking = 0.0
self.shared_vars.err_msg = ''
# define pickleable visor_engine function for processing
global visor_engine_process_func
copy_reg.pickle(types.MethodType, pickle_class_methods._pickle_method,
pickle_class_methods._unpickle_method)
# NOTE: The pickling fails if the query has been excluded (unselected) from the
# list of cached text queries, probably because its session seems to have
# been removed from memory
try:
if not on_cache_exclude_list:
visor_engine_process_func = pickle.loads(
pickle.dumps(visor_engine.process))
except Exception as e:
print e
pass
def delete_results(self, query, for_all_datasets=False):
"""
Deletes the data associated with the specified query.
Arguments:
query: query in dictionary form.
for_all_datasets: Boolean indicating whether to delete the results for
all datasets.
"""
qhash = query_translations.get_qhash(query)
if not for_all_datasets:
self._max_size_cache.delete_data(
(qhash, query['qtype'], query['engine'], query['dsetname']))
else:
self._max_size_cache.delete_data_partial_tuple((
qhash,
query['qtype'],
query['engine'],
))
def delete_results(self, ses_id, query, for_all_datasets=False):
"""
Deletes the results of the specified query from a session cache.
Arguments:
ses_id: ID of the session cache.
query: query in dictionary form.
for_all_datasets: Boolean indicating whether to delete the results for
all datasets.
"""
qhash = query_translations.get_qhash(query, include_qtype=True)
if not for_all_datasets:
self._session_cache.delete_data(
ses_id, (qhash, query['engine'], query['dsetname']))
else:
self._session_cache.delete_data_partial_tuple(
ses_id, (
qhash,
query['engine'],
))
def query_in_exclude_list(self, query, ses_id=None):
"""
Searches for a query in the query exclude list.
Arguments:
query: query in dictionary form.
ses_id: ID of the session.
Returns:
True if the query is found in the list, False otherwise.
"""
# if no session ID specified, allow storage under a 'universal' tag
if not ses_id:
ses_id = 'universal'
qhash = query_translations.get_qhash(query,
include_engine=True,
include_qtype=True)
# get the data itself
data = self._session_cache.get_data(ses_id, qhash)
# print 'Looking for query in exclude list: (%s), ses_id: %s, class: %s - %s' %
# (qhash, ses_id, self.__class__.__name__, data != None)
# return only whether the key was present or not (and had non-None data)
return data != None