def swap_aliases(client, index): client.indices.refresh(index) # add null values to contracts alias if client.indices.get_alias(index, '*'): printf({ 'msg': 'Removing old aliases for index "{}"'.format(index), 'job': None, 'f': 'ES Alias Drop' }) client.indices.delete_alias(index, '_all') indices_to_award_types['contracts'] += ('NULL', ) alias_patterns = settings.TRANSACTIONS_INDEX_ROOT + '*' try: old_indices = client.indices.get_alias('*', alias_patterns).keys() for old_index in old_indices: client.indices.delete_alias(old_index, '_all') client.indices.close(old_index) printf({ 'msg': 'Removing aliases & closing "{}"'.format(old_index), 'job': None, 'f': 'ES Alias Drop' }) except Exception as e: printf({ 'msg': 'ERROR: no aliases found for {}'.format(alias_patterns), 'f': 'ES Alias Drop' }) for award_type, award_type_codes in indices_to_award_types.items(): alias_name = '{}-{}'.format(settings.TRANSACTIONS_INDEX_ROOT, award_type) printf({ 'msg': 'Putting alias "{}" with award codes {}'.format( alias_name, award_type_codes), 'job': '', 'f': 'ES Alias Put' }) put_alias(client, index, alias_name, award_type_codes) es_settingsfile = os.path.join(settings.BASE_DIR, 'usaspending_api/etl/es_settings.json') with open(es_settingsfile) as f: settings_dict = json.load(f) index_settings = settings_dict['settings']['index'] current_settings = client.indices.get(index)[index]['settings']['index'] client.indices.put_settings(index_settings, index) client.indices.refresh(index) for setting_, value_ in index_settings.items(): message = 'Changing "{}" from {} to {}'.format( setting_, current_settings.get(setting_), value_) printf({'msg': message, 'job': None, 'f': 'ES Settings Put'})
def search_transactions(request_data, lower_limit, limit): """ filters: dictionary fields: list sort: string order: string lower_limit: integer limit: integer if transaction_type_code not found, return results for contracts """ keyword = request_data['filters']['keywords'] query_fields = [TRANSACTIONS_LOOKUP[i] for i in request_data['fields']] query_fields.extend(['award_id']) query_sort = TRANSACTIONS_LOOKUP[request_data['sort']] query = { '_source': query_fields, 'from': lower_limit, 'size': limit, 'query': base_query(keyword), 'sort': [{ query_sort: { 'order': request_data['order'] } }] } for index, award_types in indices_to_award_types.items(): if sorted(award_types) == sorted( request_data['filters']['award_type_codes']): index_name = '{}-{}*'.format(TRANSACTIONS_INDEX_ROOT, index) break else: logger.exception( 'Bad/Missing Award Types. Did not meet 100% of a category\'s types' ) return False, 'Bad/Missing Award Types requested', None response = es_client_query(index=index_name, body=query, retries=10) if response: total = response['hits']['total'] results = format_for_frontend(response['hits']['hits']) return True, results, total else: return False, 'There was an error connecting to the ElasticSearch cluster', None
def search_transactions(request_data, lower_limit, limit): """ request_data: dictionary lower_limit: integer limit: integer if transaction_type_code not found, return results for contracts """ keyword = request_data["filters"]["keywords"] query_fields = [TRANSACTIONS_LOOKUP[i] for i in request_data["fields"]] query_fields.extend(["award_id"]) query_sort = TRANSACTIONS_LOOKUP[request_data["sort"]] query = { "_source": query_fields, "from": lower_limit, "size": limit, "query": base_query(keyword), "sort": [{ query_sort: { "order": request_data["order"] } }], } for index, award_types in indices_to_award_types.items(): if sorted(award_types) == sorted( request_data["filters"]["award_type_codes"]): index_name = "{}-{}*".format(TRANSACTIONS_INDEX_ROOT, index) break else: logger.exception( "Bad/Missing Award Types. Did not meet 100% of a category's types") return False, "Bad/Missing Award Types requested", None response = es_client_query(index=index_name, body=query, retries=10) if response: total = response["hits"]["total"] results = format_for_frontend(response["hits"]["hits"]) return True, results, total else: return False, "There was an error connecting to the ElasticSearch cluster", None
def swap_aliases(client, index): client.indices.refresh(index) # add null values to contracts alias if client.indices.get_alias(index, '*'): printf({'msg': 'Removing old aliases for index "{}"'.format(index), 'job': None, 'f': 'ES Alias Drop'}) client.indices.delete_alias(index, '_all') indices_to_award_types['contracts'] += ('NULL',) alias_patterns = settings.TRANSACTIONS_INDEX_ROOT + '*' try: old_indices = client.indices.get_alias('*', alias_patterns).keys() for old_index in old_indices: client.indices.delete_alias(old_index, '_all') client.indices.close(old_index) printf({'msg': 'Removing aliases & closing "{}"'.format(old_index), 'job': None, 'f': 'ES Alias Drop'}) except Exception: printf({'msg': 'ERROR: no aliases found for {}'.format(alias_patterns), 'f': 'ES Alias Drop'}) for award_type, award_type_codes in indices_to_award_types.items(): alias_name = '{}-{}'.format(settings.TRANSACTIONS_INDEX_ROOT, award_type) printf({'msg': 'Putting alias "{}" with award codes {}'.format(alias_name, award_type_codes), 'job': '', 'f': 'ES Alias Put'}) put_alias(client, index, alias_name, award_type_codes) es_settingsfile = os.path.join(settings.BASE_DIR, 'usaspending_api/etl/es_settings.json') with open(es_settingsfile) as f: settings_dict = json.load(f) index_settings = settings_dict['settings']['index'] current_settings = client.indices.get(index)[index]['settings']['index'] client.indices.put_settings(index_settings, index) client.indices.refresh(index) for setting_, value_ in index_settings.items(): message = 'Changing "{}" from {} to {}'.format(setting_, current_settings.get(setting_), value_) printf({'msg': message, 'job': None, 'f': 'ES Settings Put'})