def get_all(search_id): """ retrieves results from Redis worker cache """ key = 'searchrequest/%s' % search_id R = get_cache() RW = get_worker_cache() data = RW.hgetall(key) log.info('worker items found %s' % len(data)) reference_ids = {} reference_data = {} results = [] for search_id, search_result_obj in data.iteritems(): r = msgpack.unpackb(search_result_obj) r['status'] = 'NO_MATCH' if len(r['matches']) == 1: r['status'] = 'EXACT_MATCH' elif len(r['matches']) > 1: r['status'] = 'MULTIPLE_MATCH' for m in r['matches']: individual_id = m['individual_id'] # # we just need the ids # reference_ids[m['individual_id']] = None firm_id = None if 'firm' in m: firm_id = m['firm']['id'] m['firms'] = get_firms(R, individual_id, firm_id ) results.append(r) if len(reference_ids) > 0: reference_data = get_reference_data( reference_ids.keys() ) return results, reference_data
def process_search(search_request_id): logger.info('searching %s' % search_request_id) # purge any recent work key = 'searchrequest/%s' % search_request_id RW = get_worker_cache() RW.delete(key) db = get_database() try: find_search_items_sql = """ select s.id, COALESCE( reg.register_code, 'All'), s.namelist_id, si.id as search_item_id, si.search_name, si.details from core.search_request s left outer join core.register reg on ( reg.id = s.register_id ) join core.searchitem si on ( si.namelist_id = s.namelist_id ) where s.id = %s """ logger.info('searching database') cursor = db.cursor('searchCursor') cursor.itersize = 1000 cursor.execute(find_search_items_sql, [search_request_id]) logger.info('start document ') count = 0 subtasks = [] for (search_request_id, source, namelist_id, search_item_id, search_name, details) in cursor: subtasks.append( search_name_task.subtask( (search_request_id, source, namelist_id, search_item_id, search_name, details)).set(queue='searchNameQueue')) except Exception, e: update_status(search_request_id, 'F', dict(message='failed', current=0, total=0)) return False
def process_search(search_request_id): logger.info( 'searching %s' % search_request_id ) # purge any recent work key = 'searchrequest/%s' % search_request_id RW = get_worker_cache() RW.delete(key) db = get_database() try: find_search_items_sql = """ select s.id, COALESCE( reg.register_code, 'All'), s.namelist_id, si.id as search_item_id, si.search_name, si.details from core.search_request s left outer join core.register reg on ( reg.id = s.register_id ) join core.searchitem si on ( si.namelist_id = s.namelist_id ) where s.id = %s """ logger.info('searching database') cursor = db.cursor('searchCursor') cursor.itersize = 1000 cursor.execute(find_search_items_sql, [search_request_id] ) logger.info('start document ') count = 0 subtasks = [] for (search_request_id, source, namelist_id, search_item_id, search_name, details ) in cursor: subtasks.append( search_name_task.subtask( (search_request_id, source, namelist_id, search_item_id, search_name, details )).set(queue='searchNameQueue') ) except Exception, e: update_status(search_request_id, 'F', dict(message='failed', current=0, total=0)) return False
""" @author: Weijian """ import logging as log import msgpack from worker.common.connection import get_cache, get_worker_cache from worker.search.fuzzy import get_fuzz from worker.search.name_util import get_search_key, get_partials R = get_cache() RW = get_worker_cache() def do_search_name(search_request_id, source, namelist_id, search_item_id, search_name, details): """ executes the worker against individuals in database. all results are stored in cache. """ key = 'searchrequest/%s' % search_request_id log.info([details]) search_firm = '' if 'firm' in details: search_firm = details['firm'].upper() search_name = search_name.strip().upper() search_key = get_search_key(search_name) search_first_name, search_last_name, search_middle = get_partials(
""" @author: Weijian """ import logging as log import msgpack from worker.common.connection import get_cache, get_worker_cache from worker.search.fuzzy import get_fuzz from worker.search.name_util import get_search_key, get_partials R = get_cache() RW = get_worker_cache() def do_search_name(search_request_id, source, namelist_id, search_item_id, search_name, details): """ executes the worker against individuals in database. all results are stored in cache. """ key = 'searchrequest/%s' % search_request_id log.info([details]) search_firm = '' if 'firm' in details: search_firm = details['firm'].upper() search_name = search_name.strip().upper() search_key = get_search_key(search_name) search_first_name, search_last_name, search_middle = get_partials(search_name, remove_punc=True)