async def test_get_mappings_nofails(es_requester): try: # Two content types define SAME mapping for same field -> # everything is ok get_mappings(schemas=[IB, IC]) except Exception: pytest.fail("get_mappings() shouldn't fail " "if 'field_mapping' are the same")
async def test_calculate_mapping_diff(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) version, new_index_name = await migrator.create_next_index() migrator.work_index_name = new_index_name mappings = get_mappings() index_settings = DEFAULT_SETTINGS.copy() index_settings.update(app_settings.get('index', {})) # tweak mappings so we can get the diff... for key, value in mappings.items(): # need to modify on *all* or it won't work with ES.. if 'creators' in value['properties']: value['properties']['creators']['type'] = 'text' mappings['Item']['properties']['foobar'] = { 'type': 'keyword', 'index': True } await search.conn.indices.close(new_index_name) await search.conn.indices.put_settings(index_settings, new_index_name) for key, value in mappings.items(): await search.conn.indices.put_mapping(new_index_name, key, value) await search.conn.indices.open(new_index_name) diff = await migrator.calculate_mapping_diff() assert len(diff['Folder']) == 1 assert len(diff['Item']) == 2
async def test_calculate_mapping_diff(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) version, new_index_name = await migrator.create_next_index() migrator.work_index_name = new_index_name mappings = get_mappings() index_settings = DEFAULT_SETTINGS.copy() index_settings.update(app_settings.get('index', {})) # tweak mappings so we can get the diff... if 'creators' in mappings['properties']: mappings['properties']['creators']['type'] = 'text' mappings['properties']['foobar'] = {'type': 'keyword', 'index': True} await search.conn.indices.close(new_index_name) await search.conn.indices.put_settings(body=index_settings, index=new_index_name) await search.conn.indices.put_mapping(index=new_index_name, doc_type=DOC_TYPE, body=mappings) await search.conn.indices.open(new_index_name) diff = await migrator.calculate_mapping_diff() assert len(diff[DOC_TYPE]) == 2
async def install_mappings_on_index(self, index_name): mappings = get_mappings() index_settings = DEFAULT_SETTINGS.copy() index_settings.update(app_settings.get('index', {})) await self.conn.indices.close(index_name) await self.conn.indices.put_settings(index_settings, index_name) for key, value in mappings.items(): await self.conn.indices.put_mapping(index_name, key, value) await self.conn.indices.open(index_name)
async def install_mappings_on_index(self, index_name): mappings = get_mappings() index_settings = DEFAULT_SETTINGS.copy() index_settings.update(app_settings.get('index', {})) await self.conn.indices.close(index_name) await self.conn.indices.put_settings( body=index_settings, index=index_name) await self.conn.indices.put_mapping( doc_type=DOC_TYPE, body=mappings, index=index_name) await self.conn.indices.open(index_name)
def summary(self): for field in get_mappings(self.selected_schemas, schema_info=True)['properties'].values(): self._count_field(field) pprint({ 'total': self.total, 'stored': self.stored, 'type_counts': sorted( self.type_counts.items(), key=operator.itemgetter(1), reverse=True), 'schema_counts': sorted( self.schema_counts.items(), key=operator.itemgetter(1), reverse=True) })
async def run(self, arguments, settings, app): if arguments.schema: self.selected_schemas = [ resolve_dotted_name(s) for s in arguments.schema] if arguments.type: if self.selected_schemas is None: self.selected_schemas = [] for type_name in arguments.type: for schema in get_all_possible_schemas_for_type(type_name): self.selected_schemas.append(schema) if self.arguments.summary: self.summary() else: fields = get_mappings(self.selected_schemas, schema_info=True)['properties'] pprint(fields)
def summary(self): for field in get_mappings( self.selected_schemas, schema_info=True)["properties"].values(): # noqa self._count_field(field) pprint({ "total": self.total, "stored": self.stored, "type_counts": sorted(self.type_counts.items(), key=operator.itemgetter(1), reverse=True), "schema_counts": sorted(self.schema_counts.items(), key=operator.itemgetter(1), reverse=True), })
async def get_mappings(self): return get_mappings()
async def get_mappings(self): schemas = await self.get_schemas() if schemas is not None: return get_mappings(schemas) return get_mappings()
async def test_get_mappings_fails_on_conflict(es_requester): with pytest.raises(Exception): # Two content types define DIFFERENT mapping for same field -> # conflict! get_mappings(schemas=[IA, IB])