def swap_aliases(client, index):
client.indices.refresh(index)
# add null values to contracts alias
if client.indices.get_alias(index, '*'):
printf({
'msg': 'Removing old aliases for index "{}"'.format(index),
'job': None,
'f': 'ES Alias Drop'
})
client.indices.delete_alias(index, '_all')
indices_to_award_types['contracts'] += ('NULL', )
alias_patterns = settings.TRANSACTIONS_INDEX_ROOT + '*'
try:
old_indices = client.indices.get_alias('*', alias_patterns).keys()
for old_index in old_indices:
client.indices.delete_alias(old_index, '_all')
client.indices.close(old_index)
printf({
'msg': 'Removing aliases & closing "{}"'.format(old_index),
'job': None,
'f': 'ES Alias Drop'
})
except Exception as e:
printf({
'msg': 'ERROR: no aliases found for {}'.format(alias_patterns),
'f': 'ES Alias Drop'
})
for award_type, award_type_codes in indices_to_award_types.items():
alias_name = '{}-{}'.format(settings.TRANSACTIONS_INDEX_ROOT,
award_type)
printf({
'msg':
'Putting alias "{}" with award codes {}'.format(
alias_name, award_type_codes),
'job':
'',
'f':
'ES Alias Put'
})
put_alias(client, index, alias_name, award_type_codes)
es_settingsfile = os.path.join(settings.BASE_DIR,
'usaspending_api/etl/es_settings.json')
with open(es_settingsfile) as f:
settings_dict = json.load(f)
index_settings = settings_dict['settings']['index']
current_settings = client.indices.get(index)[index]['settings']['index']
client.indices.put_settings(index_settings, index)
client.indices.refresh(index)
for setting_, value_ in index_settings.items():
message = 'Changing "{}" from {} to {}'.format(
setting_, current_settings.get(setting_), value_)
printf({'msg': message, 'job': None, 'f': 'ES Settings Put'})
def search_transactions(request_data, lower_limit, limit):
"""
filters: dictionary
fields: list
sort: string
order: string
lower_limit: integer
limit: integer
if transaction_type_code not found, return results for contracts
"""
keyword = request_data['filters']['keywords']
query_fields = [TRANSACTIONS_LOOKUP[i] for i in request_data['fields']]
query_fields.extend(['award_id'])
query_sort = TRANSACTIONS_LOOKUP[request_data['sort']]
query = {
'_source': query_fields,
'from': lower_limit,
'size': limit,
'query': base_query(keyword),
'sort': [{
query_sort: {
'order': request_data['order']
}
}]
}
for index, award_types in indices_to_award_types.items():
if sorted(award_types) == sorted(
request_data['filters']['award_type_codes']):
index_name = '{}-{}*'.format(TRANSACTIONS_INDEX_ROOT, index)
break
else:
logger.exception(
'Bad/Missing Award Types. Did not meet 100% of a category\'s types'
)
return False, 'Bad/Missing Award Types requested', None
response = es_client_query(index=index_name, body=query, retries=10)
if response:
total = response['hits']['total']
results = format_for_frontend(response['hits']['hits'])
return True, results, total
else:
return False, 'There was an error connecting to the ElasticSearch cluster', None
def search_transactions(request_data, lower_limit, limit):
"""
request_data: dictionary
lower_limit: integer
limit: integer
if transaction_type_code not found, return results for contracts
"""
keyword = request_data["filters"]["keywords"]
query_fields = [TRANSACTIONS_LOOKUP[i] for i in request_data["fields"]]
query_fields.extend(["award_id"])
query_sort = TRANSACTIONS_LOOKUP[request_data["sort"]]
query = {
"_source": query_fields,
"from": lower_limit,
"size": limit,
"query": base_query(keyword),
"sort": [{
query_sort: {
"order": request_data["order"]
}
}],
}
for index, award_types in indices_to_award_types.items():
if sorted(award_types) == sorted(
request_data["filters"]["award_type_codes"]):
index_name = "{}-{}*".format(TRANSACTIONS_INDEX_ROOT, index)
break
else:
logger.exception(
"Bad/Missing Award Types. Did not meet 100% of a category's types")
return False, "Bad/Missing Award Types requested", None
response = es_client_query(index=index_name, body=query, retries=10)
if response:
total = response["hits"]["total"]
results = format_for_frontend(response["hits"]["hits"])
return True, results, total
else:
return False, "There was an error connecting to the ElasticSearch cluster", None
def swap_aliases(client, index):
client.indices.refresh(index)
# add null values to contracts alias
if client.indices.get_alias(index, '*'):
printf({'msg': 'Removing old aliases for index "{}"'.format(index),
'job': None, 'f': 'ES Alias Drop'})
client.indices.delete_alias(index, '_all')
indices_to_award_types['contracts'] += ('NULL',)
alias_patterns = settings.TRANSACTIONS_INDEX_ROOT + '*'
try:
old_indices = client.indices.get_alias('*', alias_patterns).keys()
for old_index in old_indices:
client.indices.delete_alias(old_index, '_all')
client.indices.close(old_index)
printf({'msg': 'Removing aliases & closing "{}"'.format(old_index),
'job': None, 'f': 'ES Alias Drop'})
except Exception:
printf({'msg': 'ERROR: no aliases found for {}'.format(alias_patterns),
'f': 'ES Alias Drop'})
for award_type, award_type_codes in indices_to_award_types.items():
alias_name = '{}-{}'.format(settings.TRANSACTIONS_INDEX_ROOT, award_type)
printf({'msg': 'Putting alias "{}" with award codes {}'.format(alias_name, award_type_codes),
'job': '', 'f': 'ES Alias Put'})
put_alias(client, index, alias_name, award_type_codes)
es_settingsfile = os.path.join(settings.BASE_DIR, 'usaspending_api/etl/es_settings.json')
with open(es_settingsfile) as f:
settings_dict = json.load(f)
index_settings = settings_dict['settings']['index']
current_settings = client.indices.get(index)[index]['settings']['index']
client.indices.put_settings(index_settings, index)
client.indices.refresh(index)
for setting_, value_ in index_settings.items():
message = 'Changing "{}" from {} to {}'.format(setting_,
current_settings.get(setting_),
value_)
printf({'msg': message, 'job': None, 'f': 'ES Settings Put'})