def __init__(self, query, visor_engine, on_cache_exclude_list): """ Initializes the worker. Arguments: query: query in dictionary form visor_engine: visor engine object, which will execute the query on_cache_exclude_list: boolean indicating if the query in on the list of excluded cached text queries. """ # get a new query ID # print ('Generating a new query ID...') self.qid = visor_engine.get_query_id(query['engine'], query['dsetname']) self.query = query self.qindex = query_translations.get_qhash(query) # print ('Initializing query namespace...') self.shared_vars = manager.Namespace() # configure initial shared memory namespace values self.shared_vars.state = opts.States.processing self.shared_vars.postrainimg_paths = [] self.shared_vars.curatedtrainimgs_paths = [] self.shared_vars.negtrainimg_count = 0 self.shared_vars.exectime_processing = 0.0 self.shared_vars.exectime_training = 0.0 self.shared_vars.exectime_ranking = 0.0 self.shared_vars.err_msg = ''
def add_results(self, rlist, query): """ Stores the results of a query into the cache. Arguments: query: query in dictionary form. rlist: the list of results. """ qhash = query_translations.get_qhash(query) self._max_size_cache.add_data( rlist, (qhash, query['qtype'], query['engine'], query['dsetname']))
def get_results(self, query): """ Retrieves the cached results of a query. Arguments: query: query in dictionary form. Returns: the cached list of results. """ qhash = query_translations.get_qhash(query) return self._max_size_cache.get_data( (qhash, query['qtype'], query['engine'], query['dsetname']))
def add_results(self, rlist, ses_id, query): """ Adds the results of a query to a session cache Arguments: rlist: List of results associated to the query. query: query in dictionary form. ses_id: ID of the session. """ qhash = query_translations.get_qhash(query, include_qtype=True) self._session_cache.add_data( rlist, ses_id, (qhash, query['engine'], query['dsetname']))
def start_query(self, query, user_ses_id=None, force_new_worker=False): """ Starts a new worker and return its status (or if the worker already exists, return its status directly). Arguments: query: query in dictionary form. user_ses_id: user session id. force_new_worker: Boolean instructing this function to mandatorily create a new worker for the query. Returns: A QueryStatus object. """ # start a query only if an existing query of the same name does not exist worker = None qindex = query_translations.get_qhash(query) for key, value in self._workers.items(): if value.qindex == qindex: if not force_new_worker: # return the already created worker worker = value else: # if force_new_worker is True, create a new worker # for the same qhash and remove the previous one del self._workers[key] break if not worker: # determine if on cache exclude list excl_query = self.result_cache[ query['engine']].query_in_exclude_list(query, ses_id=user_ses_id) # print ('Initializing query worker process...') try: worker = QueryWorker(query, self._engine, excl_query) self._workers[worker.qid] = worker # start the query # print ('Launching query process...') self.process_pool.apply_async( func=call_it, args=(self._engine, 'process', (query, worker.qid, worker.shared_vars, self._proc_opts, user_ses_id))) except errors.QueryIdError: # this exception is triggered by the constructor of QueryWorker # if the backend cannot be contacted return query_data.QueryStatus( state=opts.States.fatal_error_or_socket_timeout) # return query status (including qid as a field) return worker.get_status()
def _get_worker_from_definition(self, query): """ Gets the worker instance assigned to a query. Arguments: query: query in dictionary form. Returns: A QueryWorker object """ qindex = query_translations.get_qhash(query) for key, value in self._workers.items(): if value.qindex == qindex: return value return None
def get_results(self, ses_id, query=None): """ Gets the results of a query from a session. Arguments: query: query in dictionary form. ses_id: ID of the session. Returns: The list of results associated to the query. """ if query: qhash = query_translations.get_qhash(query, include_qtype=True) return self._session_cache.get_data( ses_id, (qhash, query['engine'], query['dsetname'])) # returns data dictionary directly if no query key was specified return self._session_cache.get_data(ses_id)
def add_query_to_exclude_list(self, query, ses_id=None): """ Adds a query to the query exclude list. Arguments: query: query in dictionary form. ses_id: ID of the session. """ # if no session ID specified, allow storage under a 'universal' tag if not ses_id: ses_id = 'universal' qhash = query_translations.get_qhash(query, include_engine=True, include_qtype=True) # print 'Adding query to exclude list: (%s), ses_id: %s, class: %s' % # (qhash, ses_id, self.__class__.__name__) # add dummy data as only the keys of excluded queries are important self._session_cache.add_data('excluded query', ses_id, qhash)
def __init__(self, query, visor_engine, on_cache_exclude_list): """ Initializes the worker. Arguments: query: query in dictionary form visor_engine: visor engine object, which will execute the query on_cache_exclude_list: boolean indicating if the query in on the list of excluded cached text queries. """ # get a new query ID # print 'Generating a new query ID...' self.qid = visor_engine.get_query_id(query['engine'], query['dsetname']) self.query = query self.qindex = query_translations.get_qhash(query) # print 'Initializing query namespace...' self.shared_vars = manager.Namespace() # configure initial shared memory namespace values self.shared_vars.state = models.opts.States.processing self.shared_vars.postrainimg_paths = [] self.shared_vars.curatedtrainimgs_paths = [] self.shared_vars.negtrainimg_count = 0 self.shared_vars.exectime_processing = 0.0 self.shared_vars.exectime_training = 0.0 self.shared_vars.exectime_ranking = 0.0 self.shared_vars.err_msg = '' # define pickleable visor_engine function for processing global visor_engine_process_func copy_reg.pickle(types.MethodType, pickle_class_methods._pickle_method, pickle_class_methods._unpickle_method) # NOTE: The pickling fails if the query has been excluded (unselected) from the # list of cached text queries, probably because its session seems to have # been removed from memory try: if not on_cache_exclude_list: visor_engine_process_func = pickle.loads( pickle.dumps(visor_engine.process)) except Exception as e: print e pass
def delete_results(self, query, for_all_datasets=False): """ Deletes the data associated with the specified query. Arguments: query: query in dictionary form. for_all_datasets: Boolean indicating whether to delete the results for all datasets. """ qhash = query_translations.get_qhash(query) if not for_all_datasets: self._max_size_cache.delete_data( (qhash, query['qtype'], query['engine'], query['dsetname'])) else: self._max_size_cache.delete_data_partial_tuple(( qhash, query['qtype'], query['engine'], ))
def delete_results(self, ses_id, query, for_all_datasets=False): """ Deletes the results of the specified query from a session cache. Arguments: ses_id: ID of the session cache. query: query in dictionary form. for_all_datasets: Boolean indicating whether to delete the results for all datasets. """ qhash = query_translations.get_qhash(query, include_qtype=True) if not for_all_datasets: self._session_cache.delete_data( ses_id, (qhash, query['engine'], query['dsetname'])) else: self._session_cache.delete_data_partial_tuple( ses_id, ( qhash, query['engine'], ))
def query_in_exclude_list(self, query, ses_id=None): """ Searches for a query in the query exclude list. Arguments: query: query in dictionary form. ses_id: ID of the session. Returns: True if the query is found in the list, False otherwise. """ # if no session ID specified, allow storage under a 'universal' tag if not ses_id: ses_id = 'universal' qhash = query_translations.get_qhash(query, include_engine=True, include_qtype=True) # get the data itself data = self._session_cache.get_data(ses_id, qhash) # print 'Looking for query in exclude list: (%s), ses_id: %s, class: %s - %s' % # (qhash, ses_id, self.__class__.__name__, data != None) # return only whether the key was present or not (and had non-None data) return data != None