def get_all(search_id):
"""
retrieves results from Redis worker cache
"""
key = 'searchrequest/%s' % search_id
R = get_cache()
RW = get_worker_cache()
data = RW.hgetall(key)
log.info('worker items found %s' % len(data))
reference_ids = {}
reference_data = {}
results = []
for search_id, search_result_obj in data.iteritems():
r = msgpack.unpackb(search_result_obj)
r['status'] = 'NO_MATCH'
if len(r['matches']) == 1:
r['status'] = 'EXACT_MATCH'
elif len(r['matches']) > 1:
r['status'] = 'MULTIPLE_MATCH'
for m in r['matches']:
individual_id = m['individual_id']
#
# we just need the ids
#
reference_ids[m['individual_id']] = None
firm_id = None
if 'firm' in m:
firm_id = m['firm']['id']
m['firms'] = get_firms(R, individual_id, firm_id )
results.append(r)
if len(reference_ids) > 0:
reference_data = get_reference_data( reference_ids.keys() )
return results, reference_data
def get_all(search_id):
"""
retrieves results from Redis worker cache
"""
key = 'searchrequest/%s' % search_id
R = get_cache()
RW = get_worker_cache()
data = RW.hgetall(key)
log.info('worker items found %s' % len(data))
reference_ids = {}
reference_data = {}
results = []
for search_id, search_result_obj in data.iteritems():
r = msgpack.unpackb(search_result_obj)
r['status'] = 'NO_MATCH'
if len(r['matches']) == 1:
r['status'] = 'EXACT_MATCH'
elif len(r['matches']) > 1:
r['status'] = 'MULTIPLE_MATCH'
for m in r['matches']:
individual_id = m['individual_id']
#
# we just need the ids
#
reference_ids[m['individual_id']] = None
firm_id = None
if 'firm' in m:
firm_id = m['firm']['id']
m['firms'] = get_firms(R, individual_id, firm_id )
results.append(r)
if len(reference_ids) > 0:
reference_data = get_reference_data( reference_ids.keys() )
return results, reference_data
def process_search(search_request_id):
logger.info('searching %s' % search_request_id)
# purge any recent work
key = 'searchrequest/%s' % search_request_id
RW = get_worker_cache()
RW.delete(key)
db = get_database()
try:
find_search_items_sql = """
select s.id,
COALESCE( reg.register_code, 'All'),
s.namelist_id,
si.id as search_item_id,
si.search_name,
si.details
from core.search_request s
left outer join core.register reg on ( reg.id = s.register_id )
join core.searchitem si on ( si.namelist_id = s.namelist_id )
where s.id = %s
"""
logger.info('searching database')
cursor = db.cursor('searchCursor')
cursor.itersize = 1000
cursor.execute(find_search_items_sql, [search_request_id])
logger.info('start document ')
count = 0
subtasks = []
for (search_request_id, source, namelist_id, search_item_id,
search_name, details) in cursor:
subtasks.append(
search_name_task.subtask(
(search_request_id, source, namelist_id, search_item_id,
search_name, details)).set(queue='searchNameQueue'))
except Exception, e:
update_status(search_request_id, 'F',
dict(message='failed', current=0, total=0))
return False
def process_search(search_request_id):
logger.info( 'searching %s' % search_request_id )
# purge any recent work
key = 'searchrequest/%s' % search_request_id
RW = get_worker_cache()
RW.delete(key)
db = get_database()
try:
find_search_items_sql = """
select s.id,
COALESCE( reg.register_code, 'All'),
s.namelist_id,
si.id as search_item_id,
si.search_name,
si.details
from core.search_request s
left outer join core.register reg on ( reg.id = s.register_id )
join core.searchitem si on ( si.namelist_id = s.namelist_id )
where s.id = %s
"""
logger.info('searching database')
cursor = db.cursor('searchCursor')
cursor.itersize = 1000
cursor.execute(find_search_items_sql, [search_request_id] )
logger.info('start document ')
count = 0
subtasks = []
for (search_request_id, source, namelist_id, search_item_id, search_name, details ) in cursor:
subtasks.append( search_name_task.subtask( (search_request_id, source, namelist_id, search_item_id, search_name, details )).set(queue='searchNameQueue') )
except Exception, e:
update_status(search_request_id, 'F', dict(message='failed', current=0, total=0))
return False
"""
@author: Weijian
"""
import logging as log
import msgpack
from worker.common.connection import get_cache, get_worker_cache
from worker.search.fuzzy import get_fuzz
from worker.search.name_util import get_search_key, get_partials
R = get_cache()
RW = get_worker_cache()
def do_search_name(search_request_id, source, namelist_id, search_item_id,
search_name, details):
"""
executes the worker against individuals in database. all results are stored in cache.
"""
key = 'searchrequest/%s' % search_request_id
log.info([details])
search_firm = ''
if 'firm' in details:
search_firm = details['firm'].upper()
search_name = search_name.strip().upper()
search_key = get_search_key(search_name)
search_first_name, search_last_name, search_middle = get_partials(
"""
@author: Weijian
"""
import logging as log
import msgpack
from worker.common.connection import get_cache, get_worker_cache
from worker.search.fuzzy import get_fuzz
from worker.search.name_util import get_search_key, get_partials
R = get_cache()
RW = get_worker_cache()
def do_search_name(search_request_id, source, namelist_id, search_item_id, search_name, details):
"""
executes the worker against individuals in database. all results are stored in cache.
"""
key = 'searchrequest/%s' % search_request_id
log.info([details])
search_firm = ''
if 'firm' in details:
search_firm = details['firm'].upper()
search_name = search_name.strip().upper()
search_key = get_search_key(search_name)
search_first_name, search_last_name, search_middle = get_partials(search_name, remove_punc=True)
