def mocked_requests(*args, **kwargs):
class MockResponse:
def __init__(self, status_code, text):
self.status_code = status_code
self.text = text
url = kwargs['url']
data = str(kwargs['data'])
if url == f'{tr_url}/calais':
print(data)
if get_correct_keyword(found_keyword) in data:
return MockResponse(200,
'{"http://d.opencalais.com/comphash-1/520f1c53": {"_type": "Company",'
'"confidence": {"aggregate": "1.0", "resolution": "1.0"},'
'"resolutions": [{"name": "MICROSOFT CORPORATION",'
'"permid": "4295907168", "score": 0.9946112}],'
'"instances": [{"exact": "Microsoft"}]}}')
elif get_correct_keyword(not_found_keyword) in data:
return MockResponse(200, '{}')
elif get_correct_keyword(unsupported_language_keyword) in data:
return MockResponse(400, '{which is not currently supported}')
elif get_correct_keyword(api_error_keyword) in data:
return MockResponse(429, '{}')
return MockResponse(None, 404)
def test_entity_match_when_not_found(self, mock_post):
result = issue_entity_match_request(not_found_keyword)
expected_result = default_not_found_result(resolution_algorithm=ResolutionAlgo.ENTITY_MATCH.value,
keyword=get_correct_keyword(not_found_keyword))
self.assertEqual(result, expected_result)
pass
def test_open_calais_when_unsupported_language(self, mock_post):
response = issue_open_calais_request(keyword_arg=unsupported_language_keyword)
expected_result = default_not_found_result(resolution_algorithm=ResolutionAlgo.OPEN_CALAIS,
keyword=get_correct_keyword(unsupported_language_keyword))
self.assertEqual(response, expected_result)
pass
def mocked_requests(**kwargs):
class MockResponse:
def __init__(self, status_code, text):
self.status_code = status_code
self.text = text
url = kwargs['url']
data = str(kwargs['data'])
if url == f'{tr_url}/match':
if get_correct_keyword(found_keyword) in data:
return MockResponse(200,
'{"outputContentResponse": [{"Match OpenPermID":"' + found_perm_id_url + '", '
'"Match OrgName": "' + found_org_name + '","Match Score": "' + found_match_score + '",'
'"Match Level": "' + found_match_level + '", "Input_Name": "' + get_correct_keyword(found_keyword) + '"}]}')
elif get_correct_keyword(not_found_keyword) in data:
return MockResponse(200,
'{"outputContentResponse": [{"Match Level": "' + not_found_match_level +
'", "Input_Name": "' + get_correct_keyword(not_found_keyword) + '"}]}')
elif get_correct_keyword(api_error_keyword) in data:
return MockResponse(429, '{}')
return MockResponse(None, 404)
def process_redo_data():
global CORRECTED_COUNT
dynamo_client = boto3.resource('dynamodb', region_name=AWS_REGION)
table = dynamo_client.Table(RESOLVED_ENTITY_TABLE_NAME)
local_list = []
with open(FILE_NAME) as fp:
keyword_orig = fp.readline()
batch_count = 0
local_count = 0
while keyword_orig:
keyword_orig = chomp(keyword_orig)
local_count = local_count + 1
if keyword_orig is not None and keyword_orig is not '':
keyword = get_correct_keyword(keyword_orig)
local_list.append(f'{local_count},{keyword}')
ORIG_KEYWORD_DICT[f'{local_count}'] = chomp(keyword_orig)
EM_KEYWORD_DICT[f'{local_count}'] = keyword
if local_count % BATCH_SIZE is 0:
batch_count = batch_count + 1
if batch_count >= SKIP_BATCHES_UNTIL:
__issue_em_request(local_list, table)
print(
f'Finished processing {batch_count} batches of {BATCH_SIZE} entities each'
)
local_list.clear()
keyword_orig = fp.readline()
__issue_em_request(local_list, table)
print(f'Corrected {CORRECTED_COUNT} entries')
def issue_entity_match_request(keyword_arg, max_retry=5):
keyword = get_correct_keyword(keyword=keyword_arg)
log.info(f'Issuing request to TR for {keyword}')
url = f'{get_tr_url()}/match'
data = f"LocalID,Name\n1,{keyword}"
retry_count = 0
result = requests.post(url=url,
headers=__get_headers(),
data=data.encode('utf-8'))
while result.status_code == requests.codes.too_many_requests:
log.error(f'{result.status_code}, {result.text}')
retry_count += 1
if retry_count > max_retry:
log.error(
f'Max Retry(s) reached for {keyword}, unable to resolve via Entity Match- EM issue!'
)
return None
else:
log.info(f'Retry {retry_count}/{max_retry}...')
time.sleep(1)
result = requests.post(url=url, headers=__get_headers(), data=data)
log.info(
f'Got a result for {data} from Entity Match Code : {result.status_code}, Result : {result.text}'
)
if result.status_code != requests.codes.ok:
log.error(f'{result.status_code}, {result.text}')
return None
json_result = json.loads(result.text)
relevant_result = get_val(keyword, json_result['outputContentResponse'][0])
return relevant_result
def issue_open_calais_request(keyword_arg,
max_retry=3,
do_smart_phrasing=True):
keyword = get_correct_keyword(keyword=keyword_arg)
log.info(f'Issuing request to TR (Open Calais) for {keyword}')
smart_keyword = smart_phrasing(keyword=keyword,
do_smart_phrasing=do_smart_phrasing)
try:
url = f'{get_tr_url()}/calais'
retry_count = 0
result = requests.post(url=url,
headers=__get_headers(),
data=smart_keyword.encode('utf-8'))
while result.status_code != requests.codes.ok:
log.error(f'{result.status_code}, {result.text}')
retry_count += 1
if retry_count > max_retry:
log.error(
f'Max Retry(s) reached for {smart_keyword}, unable to resolve via Open Calais- OC issue!'
)
return None
elif 'which is not currently supported' in result.text or 'Unrecognized-Language' in result.text:
log.error(
f'Unsupported language for {keyword}, unable to resolve via Open Calais- OC Language !'
)
return default_not_found_result(
resolution_algorithm=ResolutionAlgo.OPEN_CALAIS,
keyword=keyword)
else:
log.info(f'Retry {retry_count}/{max_retry}...')
time.sleep(1)
result = requests.post(url,
headers=__get_headers(),
data=smart_keyword.encode('utf-8'))
result = json.loads(result.text)
companies = [
v for v in result.values()
if v.get('_type') == 'Company' and v.get('resolutions') is not None
]
vals = [
_get_val(keyword, r) for r in companies for i in r['instances']
]
log.info(f'Got a result from Open Calais {vals}')
if vals is None or len(vals) <= 0:
log.error(
f'No result in Open Calais for {smart_keyword}- No confident match'
)
return default_not_found_result(
resolution_algorithm=ResolutionAlgo.OPEN_CALAIS,
keyword=keyword)
return vals
except:
log.error(traceback.format_exc())
log.error(
f'OPEN-CALAIS: Unable to successfully issue request to OC for entity resolution for {smart_keyword}'
)
return None