async def test_get_mappings_nofails(es_requester):
try:
# Two content types define SAME mapping for same field ->
# everything is ok
get_mappings(schemas=[IB, IC])
except Exception:
pytest.fail("get_mappings() shouldn't fail "
"if 'field_mapping' are the same")
async def test_calculate_mapping_diff(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
migrator = Migrator(search, container, force=True, request=request)
version, new_index_name = await migrator.create_next_index()
migrator.work_index_name = new_index_name
mappings = get_mappings()
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
# tweak mappings so we can get the diff...
for key, value in mappings.items():
# need to modify on *all* or it won't work with ES..
if 'creators' in value['properties']:
value['properties']['creators']['type'] = 'text'
mappings['Item']['properties']['foobar'] = {
'type': 'keyword',
'index': True
}
await search.conn.indices.close(new_index_name)
await search.conn.indices.put_settings(index_settings, new_index_name)
for key, value in mappings.items():
await search.conn.indices.put_mapping(new_index_name, key, value)
await search.conn.indices.open(new_index_name)
diff = await migrator.calculate_mapping_diff()
assert len(diff['Folder']) == 1
assert len(diff['Item']) == 2
async def test_calculate_mapping_diff(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
migrator = Migrator(search, container, force=True, request=request)
version, new_index_name = await migrator.create_next_index()
migrator.work_index_name = new_index_name
mappings = get_mappings()
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
# tweak mappings so we can get the diff...
if 'creators' in mappings['properties']:
mappings['properties']['creators']['type'] = 'text'
mappings['properties']['foobar'] = {'type': 'keyword', 'index': True}
await search.conn.indices.close(new_index_name)
await search.conn.indices.put_settings(body=index_settings,
index=new_index_name)
await search.conn.indices.put_mapping(index=new_index_name,
doc_type=DOC_TYPE,
body=mappings)
await search.conn.indices.open(new_index_name)
diff = await migrator.calculate_mapping_diff()
assert len(diff[DOC_TYPE]) == 2
async def install_mappings_on_index(self, index_name):
mappings = get_mappings()
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
await self.conn.indices.close(index_name)
await self.conn.indices.put_settings(index_settings, index_name)
for key, value in mappings.items():
await self.conn.indices.put_mapping(index_name, key, value)
await self.conn.indices.open(index_name)
async def install_mappings_on_index(self, index_name):
mappings = get_mappings()
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
await self.conn.indices.close(index_name)
await self.conn.indices.put_settings(
body=index_settings, index=index_name)
await self.conn.indices.put_mapping(
doc_type=DOC_TYPE, body=mappings, index=index_name)
await self.conn.indices.open(index_name)
def summary(self):
for field in get_mappings(self.selected_schemas, schema_info=True)['properties'].values():
self._count_field(field)
pprint({
'total': self.total,
'stored': self.stored,
'type_counts': sorted(
self.type_counts.items(), key=operator.itemgetter(1), reverse=True),
'schema_counts': sorted(
self.schema_counts.items(), key=operator.itemgetter(1), reverse=True)
})
async def run(self, arguments, settings, app):
if arguments.schema:
self.selected_schemas = [
resolve_dotted_name(s) for s in arguments.schema]
if arguments.type:
if self.selected_schemas is None:
self.selected_schemas = []
for type_name in arguments.type:
for schema in get_all_possible_schemas_for_type(type_name):
self.selected_schemas.append(schema)
if self.arguments.summary:
self.summary()
else:
fields = get_mappings(self.selected_schemas, schema_info=True)['properties']
pprint(fields)
def summary(self):
for field in get_mappings(
self.selected_schemas,
schema_info=True)["properties"].values(): # noqa
self._count_field(field)
pprint({
"total":
self.total,
"stored":
self.stored,
"type_counts":
sorted(self.type_counts.items(),
key=operator.itemgetter(1),
reverse=True),
"schema_counts":
sorted(self.schema_counts.items(),
key=operator.itemgetter(1),
reverse=True),
})
async def get_mappings(self):
return get_mappings()
async def get_mappings(self):
schemas = await self.get_schemas()
if schemas is not None:
return get_mappings(schemas)
return get_mappings()
async def test_get_mappings_fails_on_conflict(es_requester):
with pytest.raises(Exception):
# Two content types define DIFFERENT mapping for same field ->
# conflict!
get_mappings(schemas=[IA, IB])