def sanity_check_new_index(self, attempt, document, new_index_name, previous_record_count): """ Ensure that we do not point to an index that looks like it has missing data. """ current_record_count = self.get_record_count(document) percentage_change = self.percentage_change(current_record_count, previous_record_count) # Verify there was not a big shift in record count record_count_is_sane = percentage_change < settings.INDEX_SIZE_CHANGE_THRESHOLD # Spot check a known-flaky field type to detect VAN-391 aggregation_type = Mapping.from_es(new_index_name)['aggregation_key'].name record_count_is_sane = record_count_is_sane and aggregation_type == 'keyword' if not record_count_is_sane: conn = get_connection() alternate_current_record_count = conn.search({"query": {"match_all": {}}}, index=new_index_name).get( 'hits', {}).get('total', {}).get('value', 0) message = ''' Sanity check failed for attempt #{0}. Index name: {1} Percentage change: {2} Previous record count: {3} Base record count: {4} Search record count: {5} Aggregation key type: {6} '''.format( attempt, new_index_name, str(int(round(percentage_change * 100, 0))) + '%', previous_record_count, current_record_count, alternate_current_record_count, aggregation_type, ) logger.info(message) logger.info('...sleeping for 5 seconds...') time.sleep(5) else: message = ''' Sanity check passed for attempt #{0}. Index name: {1} Percentage change: {2} Previous record count: {3} Current record count: {4} '''.format( attempt, new_index_name, str(int(round(percentage_change * 100, 0))) + '%', previous_record_count, current_record_count ) logger.info(message) index_info_string = ( 'The previous index contained [{}] records. ' 'The new index contains [{}] records, a [{:.2f}%] change.'.format( previous_record_count, current_record_count, percentage_change * 100 ) ) return record_count_is_sane, index_info_string
def view_mappings(index='buddyupevents', doc_type='event'): """Return a Mapping of mappings. Usage: explore.view_mappings().to_dict() """ m = Mapping.from_es(index, doc_type) return m
def mapping(self, index, document): """ This method looks for the mapping in an index for a given document type :param index: Elasticsearch index :param document: type of document :returns: dictionary with the mapping """ mapping = Mapping.from_es(index, document, using=self.es) return mapping.to_dict()[document]['properties']
def mapping(self, index, document): """ This method looks for the mapping in an index for a given document type :param index: Elasticsearch index :param document: type of document :returns: dictionary with the mapping """ mapping = Mapping.from_es(index, document, using=self.es) return mapping.to_dict()[document]['properties']
def _get_es_facets(): """Returns a dict from UI facet name to Elasticsearch facet object.""" using = Elasticsearch(app.app.config['ELASTICSEARCH_URL']) try: mapping = Mapping.from_es(app.app.config['INDEX_NAME'], 'type', using=using).to_dict() except TransportError as e: if 'index_not_found_exception' in e.error: app.app.logger.error('Index %s not found at %s' % (app.app.config['INDEX_NAME'], app.app.config['ELASTICSEARCH_URL'])) raise e config_path = os.path.join(app.app.config['DATASET_CONFIG_DIR'], 'ui.json') facets_config = _parse_json_file(config_path)['facets'] # Preserve order, so facets are returned in same order as the config file. facets = OrderedDict() for facet_config in facets_config: field_name = facet_config['elasticsearch_field_name'] if not field_name in mapping['type']['properties']: raise ValueError( 'elasticsearch_field_name %s not found in Elasticsearch index %s' % (field_name, app.app.config['INDEX_NAME'])) field_type = mapping['type']['properties'][field_name]['type'] ui_facet_name = facet_config['ui_facet_name'] if field_type == 'text': # Use ".keyword" because we want aggregation on keyword field, not # term field. See # https://www.elastic.co/guide/en/elasticsearch/reference/6.2/fielddata.html#before-enabling-fielddata facets[ui_facet_name] = TermsFacet(field=field_name + '.keyword', size=20) else: # Assume numeric type. # TODO: Handle other types. # TODO: Automatically figure out bucket intervals. Unfortunately # Elasticsearch won't do this for us # (https://github.com/elastic/elasticsearch/issues/9572). Make the # ranges easy to read (10-19,20-29 instead of 10-17,18-25). facets[ui_facet_name] = HistogramFacet(field=field_name, interval=10) app.app.logger.info('Elasticsearch facets: %s' % facets) return facets
def ensure_vocabulary_mappings(term_info): """ Ensure the mapping is properly set in Elasticsearch to always do exact matches on taxonomy terms. Accepts the output of get_resource_terms. Calling this function during indexing means that vocabularies do not need to be added to the mapping in advance. This deals with the fact that vocabularies can be added on-the-fly without having to play around with extra signals. Args: term_info (dict): Details of terms for a group of LearningResources. """ if len(term_info) == 0: return get_conn() # We don't need the return value; just for it to exist. # Retrieve current mapping from Elasticsearch. mapping = Mapping.from_es(index=INDEX_NAME, doc_type=DOC_TYPE) # Get the field names from the mapping. existing_vocabs = set(mapping.to_dict()["learningresource"]["properties"]) # Get all the taxonomy names from the data. vocab_ids = set() for vocab_terms in term_info.values(): for vocab_id in vocab_terms.keys(): vocab_ids.add(vocab_id) updated = False # Add vocabulary to mapping if necessary. for vocab_id in vocab_ids: vocab_key = make_vocab_key(vocab_id) if vocab_key in existing_vocabs: continue mapping.field(vocab_key, "string", index="not_analyzed") updated = True if updated: mapping.save(INDEX_NAME) refresh_index()
def ensure_vocabulary_mappings(term_info): """ Ensure the mapping is properly set in Elasticsearch to always do exact matches on taxonomy terms. Accepts the output of get_resource_terms. Calling this function during indexing means that vocabularies do not need to be added to the mapping in advance. This deals with the fact that vocabularies can be added on-the-fly without having to play around with extra signals. Args: term_info (dict): Details of terms for a group of LearningResources. """ if len(term_info) == 0: return get_conn() # We don't need the return value; just for it to exist. # Retrieve current mapping from Elasticsearch. mapping = Mapping.from_es(index=INDEX_NAME, doc_type=DOC_TYPE) # Get the field names from the mapping. existing_vocabs = set(mapping.to_dict()["learningresource"]["properties"]) # Get all the taxonomy names from the data. vocab_ids = set() for vocab_terms in term_info.values(): for vocab_id in vocab_terms.keys(): vocab_ids.add(vocab_id) updated = False # Add vocabulary to mapping if necessary. for vocab_id in vocab_ids: vocab_key = make_vocab_key(vocab_id) if vocab_key in existing_vocabs: continue mapping.field(vocab_key, "string", index="not_analyzed") updated = True if updated: mapping.save(INDEX_NAME) refresh_index()
def add_mapping_to_index(self, lang_code, lang_analyzer, delete_old_index=False, kuromoji_synonyms=None): """ Add or update mail/irc-mapping to EL-index, create/update required analyzers and add fields. :param lang_code: ``str`` Language of index e.g. 'ja' :param lang_analyzer: ``str`` Name of analyzer for language e.g. 'kuromoji', 'standard' etc. :param delete_old_index: ``bool`` Delete index if existing? Default: False = Update existing index (Close, Update, Open) :param kuromoji_synonyms: ``dict`` Synonyms for kuromoji Japanese analyzer. Keep old synonyms if synonyms list empty and index not deleted :return: None """ if kuromoji_synonyms is None: kuromoji_synonyms = [] analyzer_lang = helpers.get_analyzer(lang_analyzer, delete_old_index=delete_old_index, user_dictionary_file=self._user_dictionary_file, synonyms=kuromoji_synonyms) analyzer_case_insensitive_sort = analysis.analyzer('case_insensitive_sort', tokenizer=analysis.tokenizer('keyword'), filter=['lowercase']) mapping = Mapping(self._type_name) reopen_index = False index_name = self._index_prefix.format(lang_code) if self._es.indices.exists(index=index_name): if delete_old_index: self._es.indices.delete(index=index_name, ignore=[400, 404]) else: self._es.indices.close(index=index_name) reopen_index = True mapping = Mapping.from_es(index_name, self._type_name, using=self._es) # Get existing index from server self.add_mapping_fields(mapping, analyzer_lang, analyzer_case_insensitive_sort) mapping.save(index_name, using=self._es) # Insert or update if reopen_index: self._es.indices.open(index=index_name)
def test_mapping(setup_es): """Test the ES mapping for a contact.""" mapping = Mapping.from_es( ContactSearchApp.es_model.get_write_index(), ContactSearchApp.name, ) assert mapping.to_dict() == { 'contact': { 'properties': { 'accepts_dit_email_marketing': { 'type': 'boolean' }, 'address_1': { 'type': 'text' }, 'address_2': { 'type': 'text' }, 'address_country': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'address_county': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'address_postcode': { 'type': 'text' }, 'address_same_as_company': { 'type': 'boolean' }, 'address_town': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'adviser': { 'properties': { 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'archived': { 'type': 'boolean' }, 'archived_by': { 'properties': { 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'archived_on': { 'type': 'date' }, 'archived_reason': { 'type': 'text' }, 'company': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'trading_names': { 'type': 'text', 'fields': { 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'company_sector': { 'properties': { 'ancestors': { 'properties': { 'id': { 'type': 'keyword' } }, 'type': 'object', }, 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'company_uk_region': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'created_by': { 'properties': { 'dit_team': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'created_on': { 'type': 'date' }, 'email': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'email_alternative': { 'type': 'text' }, 'first_name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, }, 'id': { 'type': 'keyword' }, 'job_title': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'last_name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, }, 'modified_on': { 'type': 'date' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'notes': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'primary': { 'type': 'boolean' }, 'telephone_alternative': { 'type': 'text' }, 'telephone_countrycode': { 'type': 'keyword' }, 'telephone_number': { 'type': 'keyword' }, 'title': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, }, 'dynamic': 'false', }, }
def test_mapping(setup_es): """Test the ES mapping for a companies house company.""" mapping = Mapping.from_es( CompaniesHouseCompanySearchApp.es_model.get_target_index_name(), CompaniesHouseCompanySearchApp.name, ) assert mapping.to_dict() == { 'companieshousecompany': { 'dynamic': 'false', 'properties': { 'company_category': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'company_number': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'company_status': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword', }, 'incorporation_date': { 'type': 'date', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'registered_address_1': { 'type': 'text', }, 'registered_address_2': { 'type': 'text', }, 'registered_address_country': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'registered_address_county': { 'type': 'text', }, 'registered_address_postcode': { 'copy_to': ['registered_address_postcode_trigram'], 'type': 'text', }, 'registered_address_postcode_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, 'registered_address_town': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'registered_address': { 'type': 'object', 'properties': { 'line_1': { 'index': False, 'type': 'text' }, 'line_2': { 'index': False, 'type': 'text' }, 'town': { 'index': False, 'type': 'text' }, 'county': { 'index': False, 'type': 'text' }, 'postcode': { 'type': 'text', 'fields': { 'trigram': { 'type': 'text', 'analyzer': 'trigram_analyzer', }, }, }, 'country': { 'type': 'object', 'properties': { 'id': { 'index': False, 'type': 'keyword' }, 'name': { 'index': False, 'type': 'text' }, }, }, }, }, 'sic_code_1': { 'type': 'text', }, 'sic_code_2': { 'type': 'text', }, 'sic_code_3': { 'type': 'text', }, 'sic_code_4': { 'type': 'text', }, 'uri': { 'type': 'text', }, }, }, }
def test_mapping(es): """Test the ES mapping for a company.""" mapping = Mapping.from_es( CompanySearchApp.es_model.get_write_index(), CompanySearchApp.name, ) assert mapping.to_dict() == { 'company': { 'dynamic': 'false', 'properties': { 'archived': { 'type': 'boolean' }, 'archived_by': { 'properties': { 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'archived_on': { 'type': 'date' }, 'archived_reason': { 'type': 'text' }, 'business_type': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'company_number': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'created_on': { 'type': 'date' }, 'description': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'duns_number': { 'type': 'keyword' }, 'employee_range': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'export_experience_category': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'export_to_countries': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'future_interest_countries': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'global_headquarters': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'headquarter_type': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'id': { 'type': 'keyword' }, 'modified_on': { 'type': 'date' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'reference_code': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'address': { 'type': 'object', 'properties': { 'line_1': { 'index': False, 'type': 'text' }, 'line_2': { 'index': False, 'type': 'text' }, 'town': { 'index': False, 'type': 'text' }, 'county': { 'index': False, 'type': 'text' }, 'postcode': { 'type': 'text', 'fields': { 'trigram': { 'type': 'text', 'analyzer': 'trigram_analyzer', }, }, }, 'country': { 'type': 'object', 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'trigram': { 'type': 'text', 'analyzer': 'trigram_analyzer', }, }, }, }, }, }, }, 'registered_address': { 'type': 'object', 'properties': { 'line_1': { 'index': False, 'type': 'text' }, 'line_2': { 'index': False, 'type': 'text' }, 'town': { 'index': False, 'type': 'text' }, 'county': { 'index': False, 'type': 'text' }, 'postcode': { 'type': 'text', 'fields': { 'trigram': { 'type': 'text', 'analyzer': 'trigram_analyzer', }, }, }, 'country': { 'type': 'object', 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'trigram': { 'type': 'text', 'analyzer': 'trigram_analyzer', }, }, }, }, }, }, }, 'uk_address_postcode': { 'analyzer': 'postcode_analyzer_v2', 'search_analyzer': 'postcode_search_analyzer_v2', 'type': 'text', }, 'uk_registered_address_postcode': { 'analyzer': 'postcode_analyzer_v2', 'search_analyzer': 'postcode_search_analyzer_v2', 'type': 'text', }, 'one_list_group_global_account_manager': { 'properties': { 'first_name': { 'index': False, 'type': 'text', }, 'id': { 'type': 'keyword', }, 'last_name': { 'index': False, 'type': 'text', }, 'name': { 'index': False, 'type': 'text', }, }, 'type': 'object', }, 'sector': { 'properties': { 'ancestors': { 'properties': { 'id': { 'type': 'keyword' } }, 'type': 'object', }, 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'suggest': { 'analyzer': 'simple', 'max_input_length': 50, 'preserve_position_increments': True, 'preserve_separators': True, 'type': 'completion', 'contexts': [ { 'name': 'country', 'type': 'CATEGORY' }, ], }, 'trading_names': { 'type': 'text', 'fields': { 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'turnover_range': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'uk_based': { 'type': 'boolean' }, 'uk_region': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'vat_number': { 'index': False, 'type': 'keyword', }, 'website': { 'type': 'text' }, 'latest_interaction_date': { 'type': 'date' }, }, }, }
def search(index, **kwargs): output = {'success': False} mapping = Mapping.from_es(index, 'doc') srch = Metadata.search(index=index) logger.info('search in index {}'.format(index)) source_fields = '' sort_field = None sort_order = 'asc' from_date = None to_date = None date_type = None filters = [] relation = None coords = None match = None start = 1 size = 100 q_list = [] for key in kwargs: if key == 'fields': source_fields = kwargs[key] continue elif key == 'start': start = kwargs[key] continue elif key == 'size': size = kwargs[key] continue elif key == 'sort': sort_field = kwargs[key] continue elif key == 'sortorder': sort_order = kwargs[key].lower() == 'desc' and 'desc' or 'asc' continue elif key == 'from': from_date = kwargs[key] continue elif key == 'to': to_date = kwargs[key] continue elif key == 'dates.dateType': date_type = kwargs[key].lower() continue elif key == 'encloses': relation = key coords = kwargs[key] continue elif key == 'includes': relation = key coords = kwargs[key] continue elif key == 'overlaps': relation = key coords = kwargs[key] continue elif key == 'excludes': relation = key coords = kwargs[key] continue elif key == 'match': match = kwargs[key] if kwargs[key].lower() not in \ ['must', 'filter', 'should', 'must_not']: msg = 'Unknown match value: {}'.format(kwargs[key]) output['error'] = msg return output continue elif key == 'and': params = kwargs[key].split(',') queries = [] for param in params: k, v = param.split('=') qry, msg = _create_query(k, v, mapping) if msg == 'geo-ok': filters.extend(qry) continue elif msg != 'ok': output['error'] = msg return output if not isinstance(qry, list): qry = [qry] queries.extend(qry) q_list.append(Q({'bool': {'must': queries}})) continue elif key == 'or': params = kwargs[key].split(',') queries = [] for param in params: k, v = param.split('=') qry, msg = _create_query(k, v, mapping) if msg != 'ok': output['error'] = msg return output if not isinstance(qry, list): qry = [qry] queries.extend(qry) q_list.append(Q({'bool': {'should': queries}})) continue elif key == 'not': params = kwargs[key].split(',') queries = [] for param in params: k, v = param.split('=') qry, msg = _create_query(k, v, mapping) if msg != 'ok': output['error'] = msg return output if not isinstance(qry, list): qry = [qry] queries.extend(qry) q_list.append(Q({'bool': {'must_not': queries}})) continue # Otherwise add to query qry, msg = _create_query(key, kwargs[key], mapping) if msg not in ['geo-ok', 'ok']: output['error'] = msg return output if not isinstance(qry, list): qry = [qry] q_list.extend(qry) if sort_field: logger.debug('Sort on {}'.format(sort_field)) sort_field_name = sort_field if sort_field.startswith('-'): sort_order = 'desc' sort_field = sort_field_name = sort_field[1:] sort_field = {sort_field: {'order': sort_order}} field_type = mapping.resolve_field(sort_field_name) if field_type is None: msg = 'Unknown sort field: {}'.format(sort_field_name) output['error'] = msg return output if type(field_type).name in [ 'object', ]: msg = 'Cannot sort on field: {}'.format(sort_field_name) output['error'] = msg return output srch = srch.sort(sort_field) if source_fields: new_fields = [] if isinstance(source_fields, str): source_fields = source_fields.split(',') for field in source_fields: field = field.strip() logger.debug('limit output field: {}'.format(field)) field_name = field if mapping.resolve_field(field_name) is None: msg = 'Unknown source field: {}'.format(field) output['error'] = msg return output new_fields.append(field_name) srch = srch.source(new_fields) if from_date: try: from_date = datetime.strptime(from_date, '%Y-%m-%d').date() except Exception as e: msg = 'from date {} format should be YYYY-MM-DD'.format(from_date) output['error'] = msg return output if to_date: try: to_date = datetime.strptime(to_date, '%Y-%m-%d').date() except Exception as e: msg = 'to date {} format should be YYYY-MM-DD'.format(to_date) output['error'] = msg return output if from_date or to_date: dates = [] if from_date: dates.append({ 'range': { 'metadata_json.dates.date.lte': { 'gte': from_date } } }) if to_date: dates.append( {'range': { 'metadata_json.dates.date.gte': { 'lte': to_date } }}) if date_type: dates.append({'term': {'metadata_json.dates.dateType': date_type}}) filters.append({'bool': {'must': dates}}) elif date_type: q_list.append(Q({"match": { 'metadata_json.dates.dateType': date_type }})) if relation: afilter, msg = _create_geo_query(relation, coords) if msg != 'geo-ok': output['error'] = msg return output filters.extend(afilter) # Add Query if len(q_list) == 0: q_list = {'match_all': {}} if match: qry = {match: q_list} else: qry = {'must': q_list} # Add Filter if filters: if len(filters) == 1: qry['filter'] = filters[0] else: qry['filter'] = {'bool': {'should': filters}} qry = {'bool': qry} logger.info('Search Query {}'.format(qry)) srch.query = qry try: start = int(start) except Exception as e: msg = 'start {} must be an integer'.format(start) output['error'] = msg return output if start <= 0: msg = 'start must be greater that zero' output['error'] = msg return output start -= 1 try: size = int(size) except Exception as e: msg = 'size {} must be an integer'.format(size) output['error'] = msg return output if size < 0: msg = 'size must be greater that zero' output['error'] = msg return output size = size + start logger.debug('page {} - {}'.format(start, size)) srch = srch[start:size] try: output['result'] = srch.execute() output['count'] = srch.count() logger.debug('count {}'.format(srch.count())) output['success'] = True except TransportError as e: if e.error == 'search_phase_execution_exception': msg = 'Search phrase error {}'.format(e) else: msg = 'Search Engine Transport error: {}'.format(e) output['error'] = msg except Exception as e: msg = 'Search Engine unknown error: {}'.format(e) output['error'] = msg return output
class Meta: mapping = Mapping.from_es("blog") index = "blog"
def test_mapping(es): """Test the ES mapping for an investment project.""" mapping = Mapping.from_es( ESInvestmentProject.get_write_index(), DEFAULT_MAPPING_TYPE, ) assert mapping.to_dict() == { DEFAULT_MAPPING_TYPE: { 'dynamic': 'false', 'properties': { '_document_type': { 'type': 'keyword', }, 'actual_land_date': { 'type': 'date' }, 'actual_uk_regions': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'address_1': { 'type': 'text' }, 'address_2': { 'type': 'text' }, 'address_postcode': { 'type': 'text' }, 'address_town': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'allow_blank_estimated_land_date': { 'index': False, 'type': 'boolean', }, 'allow_blank_possible_uk_regions': { 'index': False, 'type': 'boolean', }, 'anonymous_description': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'approved_commitment_to_invest': { 'type': 'boolean' }, 'approved_fdi': { 'type': 'boolean' }, 'approved_good_value': { 'type': 'boolean' }, 'approved_high_value': { 'type': 'boolean' }, 'approved_landed': { 'type': 'boolean' }, 'approved_non_fdi': { 'type': 'boolean' }, 'archived': { 'type': 'boolean' }, 'archived_by': { 'properties': { 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'archived_on': { 'type': 'date' }, 'archived_reason': { 'type': 'text' }, 'associated_non_fdi_r_and_d_project': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'project_code': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'average_salary': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'business_activities': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'client_cannot_provide_foreign_investment': { 'type': 'boolean' }, 'client_cannot_provide_total_investment': { 'type': 'boolean' }, 'client_contacts': { 'properties': { 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'client_relationship_manager': { 'properties': { 'dit_team': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'client_requirements': { 'type': 'text', 'index': False, }, 'comments': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'country_investment_originates_from': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'country_lost_to': { 'properties': { 'id': { 'type': 'keyword', 'index': False, }, 'name': { 'type': 'text', 'index': False, }, }, 'type': 'object', }, 'created_by': { 'properties': { 'dit_team': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'created_on': { 'type': 'date' }, 'date_abandoned': { 'type': 'date' }, 'date_lost': { 'type': 'date' }, 'delivery_partners': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'description': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'estimated_land_date': { 'type': 'date' }, 'export_revenue': { 'type': 'boolean' }, 'fdi_type': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'fdi_value': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'foreign_equity_investment': { 'type': 'double' }, 'government_assistance': { 'type': 'boolean' }, 'gross_value_added': { 'type': 'double' }, 'id': { 'type': 'keyword' }, 'intermediate_company': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'investment_type': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'investor_company': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'investor_company_country': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'investor_type': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'level_of_involvement': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'level_of_involvement_simplified': { 'type': 'keyword' }, 'likelihood_to_land': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'modified_on': { 'type': 'date' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'new_tech_to_uk': { 'type': 'boolean' }, 'non_fdi_r_and_d_budget': { 'type': 'boolean' }, 'number_new_jobs': { 'type': 'integer' }, 'number_safeguarded_jobs': { 'type': 'long' }, 'other_business_activity': { 'type': 'text', 'index': False, }, 'project_arrived_in_triage_on': { 'type': 'date' }, 'project_assurance_adviser': { 'properties': { 'dit_team': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'project_code': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', 'fields': { 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'project_manager': { 'properties': { 'dit_team': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'proposal_deadline': { 'type': 'date' }, 'quotable_as_public_case_study': { 'type': 'boolean' }, 'r_and_d_budget': { 'type': 'boolean' }, 'reason_abandoned': { 'type': 'text', 'index': False, }, 'reason_delayed': { 'type': 'text', 'index': False, }, 'reason_lost': { 'type': 'text', 'index': False, }, 'referral_source_activity': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'referral_source_activity_event': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'referral_source_activity_marketing': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'referral_source_activity_website': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'referral_source_adviser': { 'properties': { 'first_name': { 'type': 'text', 'index': False, }, 'id': { 'type': 'keyword', 'index': False, }, 'last_name': { 'type': 'text', 'index': False, }, 'name': { 'type': 'text', 'index': False, }, }, 'type': 'object', }, 'sector': { 'properties': { 'ancestors': { 'properties': { 'id': { 'type': 'keyword' } }, 'type': 'object', }, 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'site_decided': { 'type': 'boolean' }, 'some_new_jobs': { 'type': 'boolean' }, 'specific_programme': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'stage': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'status': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'team_members': { 'properties': { 'dit_team': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'total_investment': { 'type': 'double' }, 'uk_company': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'uk_company_decided': { 'type': 'boolean' }, 'uk_region_locations': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'will_new_jobs_last_two_years': { 'type': 'boolean' }, }, }, }
def test_mapping(es): """Test the ES mapping for an event.""" mapping = Mapping.from_es( EventSearchApp.es_model.get_write_index(), DEFAULT_MAPPING_TYPE, ) assert mapping.to_dict() == { DEFAULT_MAPPING_TYPE: { 'dynamic': 'false', 'properties': { '_document_type': { 'type': 'keyword', }, 'address_1': { 'type': 'text' }, 'address_2': { 'type': 'text' }, 'address_country': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'address_county': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'address_postcode': { 'type': 'text', 'fields': { 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'address_town': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'created_on': { 'type': 'date' }, 'disabled_on': { 'type': 'date' }, 'end_date': { 'type': 'date' }, 'event_type': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'id': { 'type': 'keyword' }, 'lead_team': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'location_type': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'modified_on': { 'type': 'date' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, 'notes': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'organiser': { 'properties': { 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword' }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'related_programmes': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'service': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'start_date': { 'type': 'date' }, 'teams': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'uk_region': { 'properties': { 'id': { 'type': 'keyword' }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, }, }, }
from elasticsearch_dsl import Keyword, Mapping, Nested, Text m = Mapping('my-type') m.field('title', 'text') m.field('category', 'text', fields={'raw': Keyword()}) comment = Nested() comment.field('author', Text()) comment.field('created_at', Date()) m.field('comments', comment) m.meta('_all', enabled=False) m.save('my-index') # We can also get the mapping from our production cluster m = Mapping.from_es('my-index', 'my-type', using='prod') m.update_from_es('my-index', using='qa') m.save('my-index', using='prod') ################################################# # DOCTYPE from datetime import datetime from elasticsearch_dsl import DocType, Date, Nested, Boolean, \ analyzer, InnerObjectWrapper, Completion, Keyword, Text html_strip = analyzer('html_strip', tokenizer='standard',
class Meta: mapping = Mapping.from_es("user") index = "user"
def test_mapping(es): """Test the ES mapping for a large capital investor profile.""" mapping = Mapping.from_es( LargeInvestorProfileSearchApp.es_model.get_write_index(), LargeInvestorProfileSearchApp.name, ) assert mapping.to_dict() == { 'large-investor-profile': { 'properties': { 'asset_classes_of_interest': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'construction_risks': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'country_of_origin': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'fields': { 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'text', }, }, 'type': 'object', }, 'created_by': { 'properties': { 'dit_team': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'object', }, 'first_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'id': { 'type': 'keyword', }, 'last_name': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'name': { 'type': 'text', 'fields': { 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, }, }, 'type': 'object', }, 'created_on': { 'type': 'date', }, 'deal_ticket_sizes': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'desired_deal_roles': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'global_assets_under_management': { 'type': 'long', }, 'id': { 'type': 'keyword', }, 'investable_capital': { 'type': 'long', }, 'investment_types': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'investor_company': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'fields': { 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, 'keyword': { 'normalizer': 'lowercase_asciifolding_normalizer', 'type': 'keyword', }, }, 'type': 'text', }, 'trading_names': { 'fields': { 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'text', }, }, 'type': 'object', }, 'investor_description': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'investor_type': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'minimum_equity_percentage': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'minimum_return_rate': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'modified_on': { 'type': 'date', }, 'notes_on_locations': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'other_countries_being_considered': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'fields': { 'trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'text', }, }, 'type': 'object', }, 'required_checks_conducted': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'restrictions': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'time_horizons': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, 'uk_region_locations': { 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'index': False, 'type': 'keyword', }, }, 'type': 'object', }, }, 'dynamic': 'false', }, }
def test_mapping(setup_es): """Test the ES mapping for an order.""" mapping = Mapping.from_es(OrderSearchApp.es_model.get_write_index(), OrderSearchApp.name) assert mapping.to_dict() == { 'order': { 'dynamic': 'false', 'properties': { 'assignees': { 'include_in_parent': True, 'properties': { 'dit_team': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, }, 'type': 'nested', }, 'first_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'id': { 'type': 'keyword', }, 'last_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name': { 'copy_to': ['assignees.name_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'nested', }, 'billing_address_1': { 'type': 'text', }, 'billing_address_2': { 'type': 'text', }, 'billing_address_country': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, }, 'type': 'nested', }, 'billing_address_county': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'billing_address_postcode': { 'type': 'text', }, 'billing_address_town': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'billing_contact_name': { 'type': 'text', }, 'billing_company_name': { 'type': 'text', }, 'billing_email': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'billing_phone': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'cancellation_reason': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, }, 'type': 'nested', }, 'cancelled_by': { 'include_in_parent': True, 'properties': { 'first_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'id': { 'type': 'keyword', }, 'last_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name': { 'copy_to': ['cancelled_by.name_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'nested', }, 'cancelled_on': { 'type': 'date', }, 'company': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'copy_to': ['company.name_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, 'trading_name': { 'copy_to': ['company.trading_name_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'trading_name_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'nested', }, 'completed_by': { 'include_in_parent': True, 'properties': { 'first_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'id': { 'type': 'keyword', }, 'last_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name': { 'copy_to': ['completed_by.name_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'nested', }, 'completed_on': { 'type': 'date', }, 'contact': { 'include_in_parent': True, 'properties': { 'first_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'id': { 'type': 'keyword', }, 'last_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name': { 'copy_to': ['contact.name_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'nested', }, 'contact_email': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'contact_phone': { 'type': 'keyword', }, 'contacts_not_to_approach': { 'type': 'text', }, 'created_by': { 'include_in_parent': True, 'properties': { 'first_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'id': { 'type': 'keyword', }, 'last_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name': { 'copy_to': ['created_by.name_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, 'dit_team': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, }, 'type': 'nested', }, }, 'type': 'nested', }, 'created_on': { 'type': 'date', }, 'delivery_date': { 'type': 'date', }, 'description': { 'analyzer': 'english_analyzer', 'type': 'text', }, 'discount_value': { 'index': False, 'type': 'integer', }, 'existing_agents': { 'index': False, 'type': 'text', }, 'further_info': { 'type': 'text', }, 'id': { 'type': 'keyword', }, 'modified_on': { 'type': 'date', }, 'net_cost': { 'index': False, 'type': 'integer', }, 'paid_on': { 'type': 'date', }, 'payment_due_date': { 'type': 'date', }, 'po_number': { 'index': False, 'type': 'keyword', }, 'primary_market': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, }, 'type': 'nested', }, 'reference': { 'copy_to': ['reference_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'reference_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, 'sector': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'ancestors': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, }, 'type': 'nested', }, }, 'type': 'nested', }, 'uk_region': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, }, 'type': 'nested', }, 'service_types': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, }, 'type': 'nested', }, 'status': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'subscribers': { 'include_in_parent': True, 'properties': { 'dit_team': { 'include_in_parent': True, 'properties': { 'id': { 'type': 'keyword', }, 'name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, }, 'type': 'nested', }, 'first_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'id': { 'type': 'keyword', }, 'last_name': { 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name': { 'copy_to': ['subscribers.name_trigram'], 'analyzer': 'lowercase_keyword_analyzer', 'fielddata': True, 'type': 'text', }, 'name_trigram': { 'analyzer': 'trigram_analyzer', 'type': 'text', }, }, 'type': 'nested', }, 'subtotal_cost': { 'copy_to': ['subtotal_cost_string'], 'type': 'integer', }, 'subtotal_cost_string': { 'type': 'keyword', }, 'total_cost': { 'copy_to': ['total_cost_string'], 'type': 'integer', }, 'total_cost_string': { 'type': 'keyword', }, 'vat_cost': { 'index': False, 'type': 'integer', }, 'vat_number': { 'index': False, 'type': 'keyword', }, 'vat_status': { 'index': False, 'type': 'keyword', }, 'vat_verified': { 'index': False, 'type': 'boolean', }, }, }, }
def test_create_mapping(self): @my_index_alias.doc_type class MyDocType2(DocType): value2 = Float() class Meta: doc_type = 'my_doc_type_2' @register_indexer('my_alias') class MyIndexer(BaseIndexer): doc_type_klass = MyDocType index_alias = my_index_alias @register_indexer('my_alias') class MyIndexer2(BaseIndexer): doc_type_klass = MyDocType2 index_alias = my_index_alias class MyPartialIndexer(PartialIndexer, MyIndexer): batch_size = 2 def get_batch_queryset(self, keys): return Mock(count=Mock(return_value=len(keys)), __iter__=Mock(return_value=iter( [Mock(id=key, value=key + 10) for key in keys]))) def get_batch_update_docs_queries(self, keys): return self.doc_type_klass.search().query('terms', id=keys) def extract_datum(self, datum): return {'id': datum.id, 'value': datum.value} for value in [1, 2, 3]: MyDocType(id=value, value=value).save() MyDocType2(id=value, value2=value).save() my_index_alias.read_index.refresh() my_indexer = MyPartialIndexer(updating_keys=[1, 2]) with my_indexer.index_alias.indexing(): my_indexer.reindex() expect(MyDocType.search().count()).to.equal(3) expect(MyDocType.search().query( 'term', id=1).execute()[0].to_dict()['value']).to.equal(11) expect(MyDocType.search().query( 'term', id=2).execute()[0].to_dict()['value']).to.equal(12) expect(MyDocType.search().query( 'term', id=3).execute()[0].to_dict()['value']).to.equal(3) expect(MyDocType2.search().count()).to.equal(3) expect(MyDocType2.search().query( 'term', id=1).execute()[0].to_dict()['value2']).to.equal(1) expect(MyDocType2.search().query( 'term', id=2).execute()[0].to_dict()['value2']).to.equal(2) expect(MyDocType2.search().query( 'term', id=3).execute()[0].to_dict()['value2']).to.equal(3) expect(Mapping.from_es('test_my_alias', 'my_doc_type').to_dict()).to.eq({ 'my_doc_type': { 'properties': { 'id': { 'type': 'keyword' }, 'value': { 'type': 'float' } } } }) expect(Mapping.from_es('test_my_alias', 'my_doc_type_2').to_dict()).to.eq({ 'my_doc_type_2': { 'properties': { 'id': { 'type': 'keyword' }, 'value2': { 'type': 'float' } } } })