def test_dict_access(self):
response = {
"foo": "bar",
}
response_object = IndexingResponse({}, [response])
self.assertEqual(response_object[0]["foo"], "bar")
response_object = MultipleResponse([response])
self.assertEqual(response_object[0]["foo"], "bar")
response_object = MultipleResponse([IndexingResponse({}, [response])])
self.assertEqual(response_object[0][0]["foo"], "bar")
response_object = AddApiKeyResponse({}, response)
self.assertEqual(response_object["foo"], "bar")
response_object = UpdateApiKeyResponse({}, response, {})
self.assertEqual(response_object["foo"], "bar")
response_object = DeleteApiKeyResponse({}, response, "")
self.assertEqual(response_object["foo"], "bar")
response_object = MultipleIndexBatchIndexingResponse({}, response)
self.assertEqual(response_object["foo"], "bar")
def test_dict_access(self):
response = {
'foo': 'bar',
}
response_object = IndexingResponse({}, [response])
self.assertEqual(response_object[0]['foo'], 'bar')
response_object = MultipleResponse([response])
self.assertEqual(response_object[0]['foo'], 'bar')
response_object = MultipleResponse([IndexingResponse({}, [response])])
self.assertEqual(response_object[0][0]['foo'], 'bar')
response_object = AddApiKeyResponse({}, response)
self.assertEqual(response_object['foo'], 'bar')
response_object = UpdateApiKeyResponse({}, response, {})
self.assertEqual(response_object['foo'], 'bar')
response_object = DeleteApiKeyResponse({}, response, '')
self.assertEqual(response_object['foo'], 'bar')
response_object = MultipleIndexBatchIndexingResponse({}, response)
self.assertEqual(response_object['foo'], 'bar')
def test_cross_app_copy_index(self):
rule = F.rule(object_id='one')
synonym = F.synonym(object_id='one')
responses = [
self.index.save_object({'objectID': 'one'}),
self.index.save_rule(rule),
self.index.save_synonym(synonym),
self.index.set_settings({'searchableAttributes': ['objectID']})
]
MultipleResponse(responses).wait()
AccountClient.copy_index(self.index, self.index2).wait()
# Assert objects got copied
res = self.index2.search('')
self.assertEqual(len(res['hits']), 1)
self.assertEqual(res['hits'][0], {'objectID': 'one'})
# Assert settings got copied
settings = self.index2.get_settings()
self.assertEqual(settings['searchableAttributes'], ['objectID'])
# Assert synonyms got copied
self.assertEqual(self.index2.get_rule('one'), rule)
# Assert synonyms got copied
self.assertEqual(self.index2.get_synonym('one'), synonym)
# Assert that copying again fails because index already exists
with self.assertRaises(AlgoliaException) as _:
AccountClient.copy_index(self.index, self.index2)
def test_batching(self):
responses = MultipleResponse()
responses.push(self.index.save_objects([
{"objectID": "one", "key": "value"},
{"objectID": "two", "key": "value"},
{"objectID": "three", "key": "value"},
{"objectID": "four", "key": "value"},
{"objectID": "five", "key": "value"},
]))
responses.push(self.index.batch([
{
"action": "addObject",
"body": {"objectID": "zero", "key": "value"}
},
{
"action": "updateObject",
"body": {"objectID": "one", "k": "v"}
},
{
"action": "partialUpdateObject",
"body": {"objectID": "two", "k": "v"}
},
{
"action": "partialUpdateObject",
"body": {"objectID": "two_bis", "key": "value"}
},
{
"action": "partialUpdateObjectNoCreate",
"body": {"objectID": "three", "k": "v"}
},
{
"action": "deleteObject",
"body": {"objectID": "four"}
}
]))
responses.wait()
objects = [
{"objectID": "zero", "key": "value"},
{"objectID": "one", "k": "v"},
{"objectID": "two", "key": "value", "k": "v"},
{"objectID": "two_bis", "key": "value"},
{"objectID": "three", "key": "value", "k": "v"},
{"objectID": "five", "key": "value"},
]
results = [obj for obj in self.index.browse_objects()]
for obj in objects:
self.assertIn(obj, results)
def test_batching(self):
responses = MultipleResponse()
responses.push(self.index.save_objects([
{"objectID": "one", "key": "value"},
{"objectID": "two", "key": "value"},
{"objectID": "three", "key": "value"},
{"objectID": "four", "key": "value"},
{"objectID": "five", "key": "value"},
]))
responses.push(self.index.batch([
{
"action": "addObject",
"body": {"objectID": "zero", "key": "value"}
},
{
"action": "updateObject",
"body": {"objectID": "one", "k": "v"}
},
{
"action": "partialUpdateObject",
"body": {"objectID": "two", "k": "v"}
},
{
"action": "partialUpdateObject",
"body": {"objectID": "two_bis", "key": "value"}
},
{
"action": "partialUpdateObjectNoCreate",
"body": {"objectID": "three", "k": "v"}
},
{
"action": "deleteObject",
"body": {"objectID": "four"}
}
]))
responses.wait()
objects = [
{"objectID": "zero", "key": "value"},
{"objectID": "one", "k": "v"},
{"objectID": "two", "key": "value", "k": "v"},
{"objectID": "two_bis", "key": "value"},
{"objectID": "three", "key": "value", "k": "v"},
{"objectID": "five", "key": "value"},
]
results = [obj for obj in self.index.browse_objects()]
for obj in objects:
self.assertIn(obj, results)
def copy_index(source_index, destination_index, request_options=None):
# type: (SearchIndex, SearchIndex, Optional[Union[Dict[str, Any], RequestOptions]]) -> MultipleResponse # noqa: E501
if source_index.app_id == destination_index.app_id:
raise AlgoliaException("The indices are on the same application. "
"Use client.copy_index instead.")
try:
destination_index.get_settings()
except RequestException:
pass
else:
raise AlgoliaException(
"Destination index already exists. Please "
"delete it before copying index across applications.")
responses = MultipleResponse()
# Copy settings
settings = source_index.get_settings()
responses.push(
destination_index.set_settings(settings, request_options))
# Copy synonyms
synonyms = source_index.browse_synonyms()
responses.push(
destination_index.save_synonyms(synonyms, request_options))
# Copy rules
rules = source_index.browse_rules()
responses.push(destination_index.save_rules(rules, request_options))
# Copy objects
objects = source_index.browse_objects()
responses.push(destination_index.save_objects(objects,
request_options))
return responses
def copy_index(source_index, destination_index, request_options=None):
# type: (SearchIndex, SearchIndex, Optional[Union[Dict[str, Any], RequestOptions]]) -> MultipleResponse # noqa: E501
if source_index.app_id == destination_index.app_id:
raise AlgoliaException(
'The indices are on the same application. '
'Use client.copy_index instead.')
try:
destination_index.get_settings()
except RequestException:
pass
else:
raise AlgoliaException(
'Destination index already exists. Please '
'delete it before copying index across applications.')
responses = MultipleResponse()
# Copy settings
settings = source_index.get_settings()
responses.push(
destination_index.set_settings(settings, request_options)
)
# Copy synonyms
synonyms = source_index.browse_synonyms()
responses.push(
destination_index.save_synonyms(synonyms, request_options)
)
# Copy rules
rules = source_index.browse_rules()
responses.push(destination_index.save_rules(rules, request_options))
# Copy objects
objects = source_index.browse_objects()
responses.push(
destination_index.save_objects(objects, request_options)
)
return responses
async def main():
async with SearchClient.create(app_id, api_key) as client:
index = client.init_index('articles')
try:
(await index.clear_objects_async()).wait()
except AlgoliaException: # Index not found
pass
results = await asyncio.gather(
index.save_object_async({
'objectID': 1,
'foo': 'bar'
}), index.save_object_async({
'objectID': 2,
'foo': 'foo'
}))
MultipleResponse(results).wait()
print(await index.search_async(''))
def test_cross_app_copy_index(self):
rule = F.rule(object_id="one")
synonym = F.synonym(object_id="one")
responses = [
self.index.save_object({"objectID": "one"}),
self.index.save_rule(rule),
self.index.save_synonym(synonym),
self.index.set_settings({"searchableAttributes": ["objectID"]}),
]
MultipleResponse(responses).wait()
AccountClient.copy_index(self.index, self.index2).wait()
# Assert objects got copied
res = self.index2.search("")
self.assertEqual(len(res["hits"]), 1)
self.assertEqual(res["hits"][0], {"objectID": "one"})
# Assert settings got copied
settings = self.index2.get_settings()
self.assertEqual(settings["searchableAttributes"], ["objectID"])
# Assert rules got copied
self.assertEqual(rule_without_metadata(self.index2.get_rule("one")),
rule)
# Assert synonyms got copied
list_synonyms1 = [synonym for synonym in self.index.browse_synonyms()]
list_synonyms2 = [synonym for synonym in self.index2.browse_synonyms()]
self.assertEqual(list_synonyms1, list_synonyms2)
# Assert synomys are the same
self.assertEqual(self.index2.get_synonym("one"), synonym)
# Assert that copying again fails because index already exists
with self.assertRaises(AlgoliaException) as _:
AccountClient.copy_index(self.index, self.index2)
def replace_all_objects_async(
self,
objects, # type: ignore
request_options=None):
# type: (Union[List[dict], Iterator[dict]], Optional[Union[dict, RequestOptions]]) -> MultipleResponse # noqa: E501
safe = False
if isinstance(request_options, dict) \
and 'safe' in request_options:
safe = request_options.pop('safe')
tmp_index_name = self._create_temporary_name()
responses = MultipleResponse()
response = yield from self.copy_to_async( # type: ignore
tmp_index_name, {'scope': ['settings', 'synonyms', 'rules']})
responses.push(response)
if safe:
responses.wait()
tmp_index = SearchIndexAsync(self._search_index,
self._transporter_async, self._config,
tmp_index_name)
response = yield from tmp_index.save_objects_async( # type: ignore
objects, request_options)
responses.push(response)
if safe:
responses.wait()
response = yield from tmp_index.move_to_async( # type: ignore
self._name)
responses.push(response)
if safe:
responses.wait()
return responses
def test_rules(self):
responses = MultipleResponse()
responses.push(self.index.save_objects([
{"objectID": "iphone_7", "brand": "Apple", "model": "7"},
{"objectID": "iphone_8", "brand": "Apple", "model": "8"},
{"objectID": "iphone_x", "brand": "Apple", "model": "X"},
{"objectID": "one_plus_one", "brand": "OnePlus",
"model": "One"},
{"objectID": "one_plus_two", "brand": "OnePlus",
"model": "Two"},
]))
responses.push(self.index.set_settings({
'attributesForFaceting': ['brand']
}))
rule1 = {
"objectID": "brand_automatic_faceting",
"enabled": False,
"condition": {"anchoring": "is", "pattern": "{facet:brand}"},
"consequence": {
"params": {
"automaticFacetFilters": [
{"facet": "brand", "disjunctive": True, "score": 42},
]
}
},
"validity": [
{
"from": 1532439300,
"until": 1532525700
},
{
"from": 1532612100,
"until": 1532698500
}
],
"description": "Automatic apply the faceting on `brand` if a"
}
responses.push(self.index.save_rule(rule1))
rule2 = {
"objectID": "query_edits",
"condition": {"anchoring": "is", "pattern": "mobile phone"},
"consequence": {
"params": {
"query": {
"edits": [
{"type": "remove", "delete": "mobile"},
{"type": "replace", "delete": "phone",
"insert": "iphone"},
]
}
}
}
}
responses.push(self.index.save_rules([rule2]))
responses.wait()
self.assertEqual(self.index.get_rule(rule1['objectID']),
rule1)
self.assertEqual(self.index.get_rule(rule2['objectID']),
rule2)
self.assertEqual(self.index.search_rules('')['nbHits'], 2)
# Browse all records with browse_rules
results = []
for obj in self.index.browse_rules():
results.append(obj)
rules = [
rule1,
rule2,
]
for rule in rules:
self.assertIn(rule, results)
self.index.delete_rule(rule1['objectID']).wait()
# Try to get the first rule with get_rule and check
# that the rule does not exist anymore
with self.assertRaises(RequestException) as _:
self.index.get_rule(rule1['objectID'])
# Clear all the rules using clear_rules
self.index.clear_rules().wait()
# Perform a rule search using search_rule with an empty query
# and check that the number of returned nbHits is equal to 0
self.assertEqual(self.index.search_rules('')['nbHits'], 0)
def replace_all_objects(self, objects, request_options=None):
# type: (Union[List[dict], Iterator[dict]], Optional[Union[dict, RequestOptions]]) -> MultipleResponse # noqa: E501
safe = False
if isinstance(request_options, dict) \
and 'safe' in request_options:
safe = request_options.pop('safe')
tmp_index_name = self._create_temporary_name()
responses = MultipleResponse()
responses.push(self.copy_to(tmp_index_name, {
'scope': ['settings', 'synonyms', 'rules']
}))
if safe:
responses.wait()
tmp_index = copy.copy(self)
tmp_index._name = tmp_index_name
responses.push(tmp_index.save_objects(objects, request_options))
if safe:
responses.wait()
responses.push(tmp_index.move_to(self._name))
if safe:
responses.wait()
return responses
def replace_all_objects(self, objects, request_options=None):
# type: (Union[List[dict], Iterator[dict]], Optional[Union[dict, RequestOptions]]) -> MultipleResponse # noqa: E501
safe = False
if isinstance(request_options, dict) \
and 'safe' in request_options:
safe = request_options.pop('safe')
tmp_index_name = self._create_temporary_name()
responses = MultipleResponse()
responses.push(self.copy_to(tmp_index_name, {
'scope': ['settings', 'synonyms', 'rules']
}))
if safe:
responses.wait()
try:
from algoliasearch.search_client import SearchClient
except ImportError: # Already imported.
pass
tmp_client = SearchClient(self._transporter, self._config)
tmp_index = tmp_client.init_index(tmp_index_name)
responses.push(tmp_index.save_objects(objects, request_options))
if safe:
responses.wait()
responses.push(tmp_index.move_to(self._name))
if safe:
responses.wait()
return responses
def test_replacing(self):
responses = MultipleResponse()
responses.push(self.index.save_object({"objectID": "one"}))
responses.push(self.index.save_rule(F.rule(object_id="one")))
responses.push(self.index.save_synonym(
{"objectID": "one", "type": "synonym", "synonyms": ["one", "two"]}
))
responses.wait()
responses.push(self.index.replace_all_objects([{"objectID": "two"}]))
responses.push(self.index.replace_all_rules([{
"objectID": "two",
"condition": {"anchoring": "is", "pattern": "pattern"},
"consequence": {
"params": {
"query": {
"edits": [
{"type": "remove", "delete": "pattern"}
]
}
}
}
}
]))
responses.push(self.index.replace_all_synonyms([
{"objectID": "two", "type": "synonym", "synonyms": ["one", "two"]}
]))
responses.wait()
# Check that record with objectID=`one` does not exist
with self.assertRaises(RequestException) as _:
self.index.get_object('one')
# Check that record with objectID=`two` does exist
self.assertEqual(self.index.get_object('two')['objectID'], 'two')
# Check that rule with objectID=`one` does not exist
with self.assertRaises(RequestException) as _:
self.index.get_rule('one')
# Check that rule with objectID=`two` does exist
self.assertEqual(self.index.get_rule('two')['objectID'], 'two')
# Check that synonym with objectID=`one` does not exist
with self.assertRaises(RequestException) as _:
self.index.get_synonym('one')
# Check that synonym with objectID="two" does exist using getSynonym
self.assertEqual(self.index.get_synonym('two')['objectID'], 'two')
def test_copy_move_index(self):
objects = [
{'objectID': 'one', 'company': 'apple'},
{'objectID': 'two', 'company': 'algolia'}
]
responses = MultipleResponse([
self.index.save_objects(objects),
self.index.set_settings({'attributesForFaceting': ['company']}),
self.index.save_synonym({
'objectID': 'google_placeholder',
'type': 'placeholder',
'placeholder': '<GOOG>',
'replacements': ['Google', 'GOOG']
}),
self.index.save_rule({
"objectID": "company_auto_faceting",
"condition": {
"anchoring": "contains",
"pattern": "{facet:company}",
},
"consequence": {
"params": {"automaticFacetFilters": ["company"]}
}
})
]).wait()
self.index2 = F.index('{}_settings'.format(self._testMethodName))
responses.push(self.client.copy_settings(
self.index.name, self.index2.name
))
self.index3 = F.index('{}_rules'.format(self._testMethodName))
responses.push(self.client.copy_rules(
self.index.name, self.index3.name
))
self.index4 = F.index('{}_synonyms'.format(self._testMethodName))
responses.push(self.client.copy_synonyms(
self.index.name, self.index4.name
))
self.index5 = F.index('{}_full_copy'.format(self._testMethodName))
responses.push(self.client.copy_index(
self.index.name, self.index5.name
))
responses.wait()
self.assertEqual(
self.index2.get_settings()['attributesForFaceting'], ['company']
)
self.index3.get_rule('company_auto_faceting')
with self.assertRaises(RequestException) as cm:
self.index3.get_synonym('google_placeholder')
self.index4.get_synonym('google_placeholder')
with self.assertRaises(RequestException) as cm:
self.index4.get_rule('company_auto_faceting')
self.index5.get_synonym('google_placeholder')
self.index5.get_rule('company_auto_faceting')
self.assertEqual(
self.index5.get_settings()['attributesForFaceting'], ['company']
)
for obj in self.index5.browse_objects():
self.assertIn(obj, objects)
self.index6 = F.index('{}_after_move'.format(self._testMethodName))
self.client.move_index(
self.index.name,
self.index6.name
).wait()
self.index6.get_synonym('google_placeholder')
self.index6.get_rule('company_auto_faceting')
self.assertEqual(
self.index6.get_settings()['attributesForFaceting'], ['company']
)
for obj in self.index6.browse_objects():
self.assertIn(obj, objects)
with self.assertRaises(RequestException) as cm:
self.client.init_index(self.index.name).search('')
self.assertEqual(cm.exception.status_code, 404)
def test_synonyms(self):
responses = MultipleResponse()
responses.push(self.index.save_objects([
{"console": "Sony PlayStation <PLAYSTATIONVERSION>"},
{"console": "Nintendo Switch"},
{"console": "Nintendo Wii U"},
{"console": "Nintendo Game Boy Advance"},
{"console": "Microsoft Xbox"},
{"console": "Microsoft Xbox 360"},
{"console": "Microsoft Xbox One"}
], {'autoGenerateObjectIDIfNotExist': True}))
responses.push(self.index.save_synonym(F.synonym({
'synonyms': [
"gba",
"gameboy advance",
"game boy advance"
]
}, 'gba')))
synonym1 = {
'objectID': 'wii_to_wii_u',
'type': 'onewaysynonym',
'input': 'wii',
'synonyms': ['wii U']
}
synonym2 = {
'objectID': 'playstation_version_placeholder',
'type': 'placeholder',
'placeholder': '<PLAYSTATIONVERSION>',
'replacements': [
"1",
"One",
"2",
"3",
"4",
"4 Pro",
]
}
synonym3 = {
'objectID': 'ps4',
'type': 'altcorrection1',
'word': 'ps4',
'corrections': ['playstation4']
}
synonym4 = {
'objectID': 'psone',
'type': 'altcorrection2',
'word': 'psone',
'corrections': ['playstationone']
}
responses.push(self.index.save_synonyms([
synonym1,
synonym2,
synonym3,
synonym4
]))
responses.wait()
self.assertEqual(self.index.get_synonym(synonym1['objectID']),
synonym1)
self.assertEqual(self.index.get_synonym(synonym2['objectID']),
synonym2)
self.assertEqual(self.index.get_synonym(synonym3['objectID']),
synonym3)
self.assertEqual(self.index.get_synonym(synonym4['objectID']),
synonym4)
self.assertEqual(self.index.search_synonyms('')['nbHits'], 5)
# Browse all records with browse_objects
results = []
for obj in self.index.browse_synonyms():
results.append(obj)
synonyms = [
synonym1,
synonym2,
synonym3,
synonym4
]
for synonym in synonyms:
self.assertIn(synonym, results)
self.index.delete_synonym('gba').wait()
# Try to get the synonym with getSynonym with objectID `gba and c
# heck that the synonym does not exist anymore
with self.assertRaises(RequestException) as _:
self.index.get_synonym('gba')
# Clear all the synonyms using clear_synonyms
self.index.clear_synonyms().wait()
# Perform a synonym search using searchSynonyms with an empty query
# and check that the number of returned synonyms is equal to 0
self.assertEqual(self.index.search_synonyms('')['nbHits'], 0)
def replace_all_objects(self, objects, request_options=None):
# type: (Union[List[dict], Iterator[dict]], Optional[Union[dict, RequestOptions]]) -> MultipleResponse # noqa: E501
safe = False
if isinstance(request_options, dict) \
and 'safe' in request_options:
safe = request_options.pop('safe')
tmp_index_name = self._create_temporary_name()
responses = MultipleResponse()
responses.push(self.copy_to(tmp_index_name, {
'scope': ['settings', 'synonyms', 'rules']
}))
if safe:
responses.wait()
tmp_index = copy.copy(self)
tmp_index._name = tmp_index_name
responses.push(tmp_index.save_objects(objects, request_options))
if safe:
responses.wait()
responses.push(tmp_index.move_to(self._name))
if safe:
responses.wait()
return responses
def test_search(self):
responses = MultipleResponse()
responses.push(
self.index.save_objects(
[
{
"company": "Algolia",
"name": "Julien Lemoine",
"objectID": "julien-lemoine",
}, # noqa: E501
{
"company": "Algolia",
"name": "Nicolas Dessaigne",
"objectID": "nicolas-dessaigne",
}, # noqa: E501
{
"company": "Amazon",
"name": "Jeff Bezos"
},
{
"company": "Apple",
"name": "Steve Jobs"
},
{
"company": "Apple",
"name": "Steve Wozniak"
},
{
"company": "Arista Networks",
"name": "Jayshree Ullal"
},
{
"company": "Google",
"name": "Larry Page"
},
{
"company": "Google",
"name": "Rob Pike"
},
{
"company": "Google",
"name": "Serguey Brin"
},
{
"company": "Microsoft",
"name": "Bill Gates"
},
{
"company": "SpaceX",
"name": "Elon Musk"
},
{
"company": "Tesla",
"name": "Elon Musk"
},
{
"company": "Yahoo",
"name": "Marissa Mayer"
},
],
{"autoGenerateObjectIDIfNotExist": True},
))
responses.push(
self.index.set_settings(
{"attributesForFaceting": ["searchable(company)"]}))
responses.wait()
# Perform a search query using search with the query `algolia` and no
# parameter and check that the number of returned hits is equal to 2
result = self.index.search("algolia")
self.assertEqual(result["nbHits"], 2)
self.assertEqual(
SearchIndex.get_object_position(result, "nicolas-dessaigne"), 0)
self.assertEqual(
SearchIndex.get_object_position(result, "julien-lemoine"), 1)
self.assertEqual(SearchIndex.get_object_position(result, ""), -1)
# Call find_object with the following parameters and check that
# no object is found
with self.assertRaises(ObjectNotFoundException):
self.index.find_object(lambda _: False)
# Call find_object with the following parameters and check that
# the first object is returned with a `position=0` and `page=0`
found = self.index.find_object(lambda _: True)
self.assertEqual(found["position"], 0)
self.assertEqual(found["page"], 0)
def callback(obj):
# type: (dict) -> bool
return obj.get("company") == "Apple"
# Call find_object with the following parameters and check that
# no object is found
with self.assertRaises(ObjectNotFoundException):
self.index.find_object(callback, {"query": "algolia"})
# Call find_object with the following parameters and check that
# no object is found
with self.assertRaises(ObjectNotFoundException):
self.index.find_object(callback, {
"query": "",
"paginate": False,
"hitsPerPage": 5
})
# Call find_object with the following parameters and check that
# the first object is returned with a `position=0` and `page=2`
found = self.index.find_object(callback, {
"query": "",
"paginate": True,
"hitsPerPage": 5
})
self.assertEqual(found["position"], 0)
self.assertEqual(found["page"], 2)
# Perform a search using search with the query `elon` and the
# following parameter and check that the queryID field from
# the response is not empty
result = self.index.search("elon", {"clickAnalytics": True})
self.assertIn("queryID", result)
# Perform a faceted search using search with the query `elon` and the
# following parameters and check that the number of returned hits is
# equal to 1
result = self.index.search("elon", {
"facets": "*",
"facetFilters": "company:tesla"
})
self.assertEqual(result["nbHits"], 1)
# Perform a filtered search using search with the query `elon` and the
# following parameters and check that the number of returned hits is
# equal to 2
result = self.index.search(
"elon", {
"facets": "*",
"filters": "(company:tesla OR company:spacex)"
})
self.assertEqual(result["nbHits"], 2)
result = self.index.search_for_facet_values("company",
"a")["facetHits"]
values = list(map(lambda facet: facet["value"], result))
self.assertIn("Algolia", values)
self.assertIn("Amazon", values)
self.assertIn("Apple", values)
self.assertIn("Arista Networks", values)
def test_synonyms(self):
responses = MultipleResponse()
responses.push(self.index.save_objects([
{"console": "Sony PlayStation <PLAYSTATIONVERSION>"},
{"console": "Nintendo Switch"},
{"console": "Nintendo Wii U"},
{"console": "Nintendo Game Boy Advance"},
{"console": "Microsoft Xbox"},
{"console": "Microsoft Xbox 360"},
{"console": "Microsoft Xbox One"}
], {'autoGenerateObjectIDIfNotExist': True}))
responses.push(self.index.save_synonym(F.synonym({
'synonyms': [
"gba",
"gameboy advance",
"game boy advance"
]
}, 'gba')))
synonym1 = {
'objectID': 'wii_to_wii_u',
'type': 'onewaysynonym',
'input': 'wii',
'synonyms': ['wii U']
}
synonym2 = {
'objectID': 'playstation_version_placeholder',
'type': 'placeholder',
'placeholder': '<PLAYSTATIONVERSION>',
'replacements': [
"1",
"One",
"2",
"3",
"4",
"4 Pro",
]
}
synonym3 = {
'objectID': 'ps4',
'type': 'altcorrection1',
'word': 'ps4',
'corrections': ['playstation4']
}
synonym4 = {
'objectID': 'psone',
'type': 'altcorrection2',
'word': 'psone',
'corrections': ['playstationone']
}
responses.push(self.index.save_synonyms([
synonym1,
synonym2,
synonym3,
synonym4
]))
responses.wait()
self.assertEqual(self.index.get_synonym(synonym1['objectID']),
synonym1)
self.assertEqual(self.index.get_synonym(synonym2['objectID']),
synonym2)
self.assertEqual(self.index.get_synonym(synonym3['objectID']),
synonym3)
self.assertEqual(self.index.get_synonym(synonym4['objectID']),
synonym4)
self.assertEqual(self.index.search_synonyms('')['nbHits'], 5)
# Browse all records with browse_objects
results = []
for obj in self.index.browse_synonyms():
results.append(obj)
synonyms = [
synonym1,
synonym2,
synonym3,
synonym4
]
for synonym in synonyms:
self.assertIn(synonym, results)
self.index.delete_synonym('gba').wait()
# Try to get the synonym with getSynonym with objectID `gba and c
# heck that the synonym does not exist anymore
with self.assertRaises(RequestException) as _:
self.index.get_synonym('gba')
# Clear all the synonyms using clear_synonyms
self.index.clear_synonyms().wait()
# Perform a synonym search using searchSynonyms with an empty query
# and check that the number of returned synonyms is equal to 0
self.assertEqual(self.index.search_synonyms('')['nbHits'], 0)
def test_search(self):
responses = MultipleResponse()
responses.push(self.index.save_objects([
{"company": "Algolia", "name": "Julien Lemoine",
"objectID": "julien-lemoine"}, # noqa: E501
{"company": "Algolia", "name": "Nicolas Dessaigne",
"objectID": "nicolas-dessaigne"}, # noqa: E501
{"company": "Amazon", "name": "Jeff Bezos"},
{"company": "Apple", "name": "Steve Jobs"},
{"company": "Apple", "name": "Steve Wozniak"},
{"company": "Arista Networks", "name": "Jayshree Ullal"},
{"company": "Google", "name": "Larry Page"},
{"company": "Google", "name": "Rob Pike"},
{"company": "Google", "name": "Serguey Brin"},
{"company": "Microsoft", "name": "Bill Gates"},
{"company": "SpaceX", "name": "Elon Musk"},
{"company": "Tesla", "name": "Elon Musk"},
{"company": "Yahoo", "name": "Marissa Mayer"}
], {'autoGenerateObjectIDIfNotExist': True}))
responses.push(self.index.set_settings({
'attributesForFaceting': ["searchable(company)"]
}))
responses.wait()
# Perform a search query using search with the query `algolia` and no
# parameter and check that the number of returned hits is equal to 2
result = self.index.search('algolia')
self.assertEqual(result['nbHits'], 2)
self.assertEqual(SearchIndex.get_object_position(
result, 'nicolas-dessaigne'), 0
)
self.assertEqual(SearchIndex.get_object_position(
result, 'julien-lemoine'), 1
)
self.assertEqual(SearchIndex.get_object_position(result, ''), -1)
# Call find_object with the following parameters and check that
# no object is found
with self.assertRaises(ObjectNotFoundException):
self.index.find_object(lambda _: False)
# Call find_object with the following parameters and check that
# the first object is returned with a `position=0` and `page=0`
found = self.index.find_object(lambda _: True)
self.assertEqual(found['position'], 0)
self.assertEqual(found['page'], 0)
def callback(obj):
# type: (dict) -> bool
return obj.get('company') == 'Apple'
# Call find_object with the following parameters and check that
# no object is found
with self.assertRaises(ObjectNotFoundException):
self.index.find_object(callback, {
'query': 'algolia'
})
# Call find_object with the following parameters and check that
# no object is found
with self.assertRaises(ObjectNotFoundException):
self.index.find_object(callback, {
'query': '',
'paginate': False,
'hitsPerPage': 5
})
# Call find_object with the following parameters and check that
# the first object is returned with a `position=0` and `page=2`
found = self.index.find_object(callback, {
'query': '',
'paginate': True,
'hitsPerPage': 5
})
self.assertEqual(found['position'], 0)
self.assertEqual(found['page'], 2)
# Perform a search using search with the query `elon` and the
# following parameter and check that the queryID field from
# the response is not empty
result = self.index.search('elon', {'clickAnalytics': True})
self.assertIn('queryID', result)
# Perform a faceted search using search with the query `elon` and the
# following parameters and check that the number of returned hits is
# equal to 1
result = self.index.search('elon', {
'facets': '*',
'facetFilters': 'company:tesla'
})
self.assertEqual(result['nbHits'], 1)
# Perform a filtered search using search with the query `elon` and the
# following parameters and check that the number of returned hits is
# equal to 2
result = self.index.search('elon', {
'facets': '*',
'filters': '(company:tesla OR company:spacex)'
})
self.assertEqual(result['nbHits'], 2)
result = self.index.search_for_facet_values('company', 'a')[
'facetHits']
values = list(
map(lambda facet: facet['value'], result))
self.assertIn('Algolia', values)
self.assertIn('Amazon', values)
self.assertIn('Apple', values)
self.assertIn('Arista Networks', values)
def test_synonyms(self):
responses = MultipleResponse()
responses.push(
self.index.save_objects(
[
{
"console": "Sony PlayStation <PLAYSTATIONVERSION>"
},
{
"console": "Nintendo Switch"
},
{
"console": "Nintendo Wii U"
},
{
"console": "Nintendo Game Boy Advance"
},
{
"console": "Microsoft Xbox"
},
{
"console": "Microsoft Xbox 360"
},
{
"console": "Microsoft Xbox One"
},
],
{"autoGenerateObjectIDIfNotExist": True},
))
responses.push(
self.index.save_synonym(
F.synonym(
{
"synonyms":
["gba", "gameboy advance", "game boy advance"]
}, "gba")))
synonym1 = {
"objectID": "wii_to_wii_u",
"type": "onewaysynonym",
"input": "wii",
"synonyms": ["wii U"],
}
synonym2 = {
"objectID": "playstation_version_placeholder",
"type": "placeholder",
"placeholder": "<PLAYSTATIONVERSION>",
"replacements": ["1", "One", "2", "3", "4", "4 Pro"],
}
synonym3 = {
"objectID": "ps4",
"type": "altcorrection1",
"word": "ps4",
"corrections": ["playstation4"],
}
synonym4 = {
"objectID": "psone",
"type": "altcorrection2",
"word": "psone",
"corrections": ["playstationone"],
}
responses.push(
self.index.save_synonyms([synonym1, synonym2, synonym3, synonym4]))
responses.wait()
self.assertEqual(self.index.get_synonym(synonym1["objectID"]),
synonym1)
self.assertEqual(self.index.get_synonym(synonym2["objectID"]),
synonym2)
self.assertEqual(self.index.get_synonym(synonym3["objectID"]),
synonym3)
self.assertEqual(self.index.get_synonym(synonym4["objectID"]),
synonym4)
self.assertEqual(self.index.search_synonyms("")["nbHits"], 5)
# Browse all records with browse_objects
results = []
for obj in self.index.browse_synonyms():
results.append(obj)
synonyms = [synonym1, synonym2, synonym3, synonym4]
for synonym in synonyms:
self.assertIn(synonym, results)
self.index.delete_synonym("gba").wait()
# Try to get the synonym with getSynonym with objectID `gba and c
# heck that the synonym does not exist anymore
with self.assertRaises(RequestException) as _:
self.index.get_synonym("gba")
# Clear all the synonyms using clear_synonyms
self.index.clear_synonyms().wait()
# Perform a synonym search using searchSynonyms with an empty query
# and check that the number of returned synonyms is equal to 0
self.assertEqual(self.index.search_synonyms("")["nbHits"], 0)
def replace_all_objects_async( # type: ignore
self,
objects,
request_options=None,
):
# type: (Union[List[dict], Iterator[dict]], Optional[Union[dict, RequestOptions]]) -> MultipleResponse # noqa: E501
safe = False
if isinstance(request_options, dict) and "safe" in request_options:
safe = request_options.pop("safe")
tmp_index_name = self._create_temporary_name()
responses = MultipleResponse()
response = yield from self.copy_to_async( # type: ignore
tmp_index_name, {"scope": ["settings", "synonyms", "rules"]})
responses.push(response)
if safe:
responses.wait()
tmp_client = SearchClient(self._transporter, self._config)
tmp_index = SearchIndexAsync(
tmp_client.init_index(tmp_index_name),
self._transporter_async,
self._config,
tmp_index_name,
)
response = yield from tmp_index.save_objects_async( # type: ignore
objects, request_options)
responses.push(response)
if safe:
responses.wait()
response = yield from tmp_index.move_to_async( # type: ignore
self._name)
responses.push(response)
if safe:
responses.wait()
return responses
def replace_all_objects_async(self, objects, # type: ignore
request_options=None):
# type: (Union[List[dict], Iterator[dict]], Optional[Union[dict, RequestOptions]]) -> MultipleResponse # noqa: E501
safe = False
if isinstance(request_options, dict) \
and 'safe' in request_options:
safe = request_options.pop('safe')
tmp_index_name = self._create_temporary_name()
responses = MultipleResponse()
response = yield from self.copy_to_async( # type: ignore
tmp_index_name,
{
'scope': ['settings',
'synonyms',
'rules']
})
responses.push(response)
if safe:
responses.wait()
tmp_index = SearchIndexAsync(
self._search_index,
self._transporter_async,
self._config,
tmp_index_name
)
response = yield from tmp_index.save_objects_async( # type: ignore
objects,
request_options
)
responses.push(response)
if safe:
responses.wait()
response = yield from tmp_index.move_to_async( # type: ignore
self._name)
responses.push(response)
if safe:
responses.wait()
return responses
def test_indexing(self):
responses = []
# adding a object with object id
obj1 = F.obj()
responses.append(self.index.save_object(obj1))
# adding a object w/o object id
obj2 = F.obj(object_id=False)
opts = {'autoGenerateObjectIDIfNotExist': True}
responses.append(self.index.save_object(obj2, opts))
# adding two objects with object id
obj3 = F.obj({'_tags': ['algolia']})
obj4 = F.obj({'_tags': ['algolia']})
responses.append(self.index.save_objects([obj3, obj4]))
# adding two objects w/o object id
obj5 = F.obj(object_id=False)
obj6 = F.obj(object_id=False)
opts = {'autoGenerateObjectIDIfNotExist': True}
responses.append(self.index.save_objects([obj5, obj6], opts))
object1_id = self.get_object_id(responses[0])
object2_id = self.get_object_id(responses[1])
object3_id = self.get_object_id(responses[2])
object4_id = self.get_object_id(responses[2], 1)
object5_id = self.get_object_id(responses[3])
object6_id = self.get_object_id(responses[3], 1)
# adding 1000 objects with object id
objects = []
for i in range(1000):
object_id = i
objects.append({
'objectID': str(object_id),
'name': object_id,
})
self.index._config.batch_size = 100
responses.append(self.index.save_objects(objects))
# waiting for all responses
MultipleResponse(responses).wait()
# Check 6 first records with get_object
self.assertEqual(obj1['name'],
self.index.get_object(object1_id)['name'])
self.assertEqual(obj2['name'],
self.index.get_object(object2_id)['name'])
self.assertEqual(obj3['name'],
self.index.get_object(object3_id)['name'])
self.assertEqual(obj4['name'],
self.index.get_object(object4_id)['name'])
self.assertEqual(obj5['name'],
self.index.get_object(object5_id)['name'])
self.assertEqual(obj6['name'],
self.index.get_object(object6_id)['name'])
# Check 1000 remaining records with get_objects
results = self.index.get_objects(range(1000))['results']
for obj in results:
self.assertIn(obj, objects)
self.assertEqual(len(results), len(objects))
# Browse all records with browse_objects
results = []
for obj in self.index.browse_objects():
results.append(obj)
for obj in objects:
self.assertIn(obj, results)
for obj in [obj1, obj3, obj4]:
self.assertIn(obj, results)
self.assertEqual(len(results), 1006)
responses = []
# Alter 1 record with partial_update_object
obj1['name'] = 'This is an altered name'
responses.append(self.index.partial_update_object(obj1))
# Alter 2 records with partial_update_objects
obj3['bar'] = 40
obj4['foo'] = 30
responses.append(self.index.partial_update_objects([obj3, obj4]))
MultipleResponse(responses).wait()
self.assertEqual(self.index.get_object(object1_id), obj1)
self.assertEqual(self.index.get_object(object3_id), obj3)
self.assertEqual(self.index.get_object(object4_id), obj4)
responses = []
# Delete the 6 first records with delete_object
responses.append(self.index.delete_object(object1_id))
responses.append(self.index.delete_object(object2_id))
responses.append(self.index.delete_by({'tagFilters': ['algolia']}))
responses.append(self.index.delete_objects([
object5_id, object6_id
]))
# Delete the 1000 remaining records with delete_objects
responses.append(self.index.clear_objects())
MultipleResponse(responses).wait()
objects = [obj for obj in self.index.browse_objects()]
self.assertEqual(len(objects), 0)
def test_search(self):
responses = MultipleResponse()
responses.push(
self.index.save_objects([{
"company": "Algolia",
"name": "Julien Lemoine"
}, {
"company": "Algolia",
"name": "Nicolas Dessaigne"
}, {
"company": "Amazon",
"name": "Jeff Bezos"
}, {
"company": "Apple",
"name": "Steve Jobs"
}, {
"company": "Apple",
"name": "Steve Wozniak"
}, {
"company": "Arista Networks",
"name": "Jayshree Ullal"
}, {
"company": "Google",
"name": "Larry Page"
}, {
"company": "Google",
"name": "Rob Pike"
}, {
"company": "Google",
"name": "Serguey Brin"
}, {
"company": "Microsoft",
"name": "Bill Gates"
}, {
"company": "SpaceX",
"name": "Elon Musk"
}, {
"company": "Tesla",
"name": "Elon Musk"
}, {
"company": "Yahoo",
"name": "Marissa Mayer"
}], {'autoGenerateObjectIDIfNotExist': True}))
responses.push(
self.index.set_settings(
{'attributesForFaceting': ["searchable(company)"]}))
responses.wait()
# Perform a search query using search with the query `algolia` and no
# parameter and check that the number of returned hits is equal to 2
result = self.index.search('algolia')
self.assertEqual(result['nbHits'], 2)
# Perform a search using search with the query `elon` and the
# following parameter and check that the queryID field from
# the response is not empty
result = self.index.search('elon', {'clickAnalytics': True})
self.assertIn('queryID', result)
# Perform a faceted search using search with the query `elon` and the
# following parameters and check that the number of returned hits is
# equal to 1
result = self.index.search('elon', {
'facets': '*',
'facetFilters': 'company:tesla'
})
self.assertEqual(result['nbHits'], 1)
# Perform a filtered search using search with the query `elon` and the
# following parameters and check that the number of returned hits is
# equal to 2
result = self.index.search(
'elon', {
'facets': '*',
'filters': '(company:tesla OR company:spacex)'
})
self.assertEqual(result['nbHits'], 2)
result = self.index.search_for_facet_values('company',
'a')['facetHits']
values = list(map(lambda facet: facet['value'], result))
self.assertIn('Algolia', values)
self.assertIn('Amazon', values)
self.assertIn('Apple', values)
self.assertIn('Arista Networks', values)
def test_rules(self):
responses = MultipleResponse()
responses.push(self.index.save_objects([
{"objectID": "iphone_7", "brand": "Apple", "model": "7"},
{"objectID": "iphone_8", "brand": "Apple", "model": "8"},
{"objectID": "iphone_x", "brand": "Apple", "model": "X"},
{"objectID": "one_plus_one", "brand": "OnePlus",
"model": "One"},
{"objectID": "one_plus_two", "brand": "OnePlus",
"model": "Two"},
]))
responses.push(self.index.set_settings({
'attributesForFaceting': ['brand', 'model']
}))
rule1 = {
"objectID": "brand_automatic_faceting",
"enabled": False,
"condition": {"anchoring": "is", "pattern": "{facet:brand}"},
"consequence": {
"params": {
"automaticFacetFilters": [
{"facet": "brand", "disjunctive": True, "score": 42},
]
}
},
"validity": [
{
"from": 1532439300,
"until": 1532525700
},
{
"from": 1532612100,
"until": 1532698500
}
],
"description": "Automatic apply the faceting on `brand` if a"
}
responses.push(self.index.save_rule(rule1))
rule2 = {
"objectID": "query_edits",
"condition": {
"anchoring": "is",
"pattern": "mobile phone",
"alternatives": True
},
"consequence": {
"params": {
"query": {
"edits": [
{"type": "remove", "delete": "mobile"},
{"type": "replace", "delete": "phone",
"insert": "iphone"},
]
}
}
}
}
rule3 = {
"objectID": "query_promo",
"consequence": {
"params": {
"filters": "brand:OnePlus"
}
}
}
rule4 = {
"objectID": "query_promo_only_summer",
"condition": {
"context": "summer"
},
"consequence": {
"params": {
"filters": "model:One"
}
}
}
responses.push(self.index.save_rules([rule2, rule3, rule4]))
responses.wait()
# Should be only the One Plus model One
self.assertEqual(self.index.search('', {
'ruleContexts': ['summer']
})['nbHits'], 1);
self.assertEqual(self.index.get_rule(rule1['objectID']),
rule1)
self.assertEqual(self.index.get_rule(rule2['objectID']),
rule2)
self.assertEqual(self.index.get_rule(rule3['objectID']),
rule3)
self.assertEqual(self.index.get_rule(rule4['objectID']),
rule4)
self.assertEqual(self.index.search_rules('')['nbHits'], 4)
# Browse all records with browse_rules
results = []
for obj in self.index.browse_rules():
results.append(obj)
rules = [
rule1,
rule2,
rule3,
rule4
]
for rule in rules:
self.assertIn(rule, results)
self.index.delete_rule(rule1['objectID']).wait()
# Try to get the first rule with get_rule and check
# that the rule does not exist anymore
with self.assertRaises(RequestException) as _:
self.index.get_rule(rule1['objectID'])
# Clear all the rules using clear_rules
self.index.clear_rules().wait()
# Perform a rule search using search_rule with an empty query
# and check that the number of returned nbHits is equal to 0
self.assertEqual(self.index.search_rules('')['nbHits'], 0)
def test_copy_move_index(self):
objects = [
{"objectID": "one", "company": "apple"},
{"objectID": "two", "company": "algolia"},
]
responses = MultipleResponse(
[
self.index.save_objects(objects),
self.index.set_settings({"attributesForFaceting": ["company"]}),
self.index.save_synonym(
{
"objectID": "google_placeholder",
"type": "placeholder",
"placeholder": "<GOOG>",
"replacements": ["Google", "GOOG"],
}
),
self.index.save_rule(
{
"objectID": "company_auto_faceting",
"condition": {
"anchoring": "contains",
"pattern": "{facet:company}",
},
"consequence": {
"params": {"automaticFacetFilters": ["company"]}
},
}
),
]
).wait()
index2 = F.index(
self.client, "{}_settings".format(self._testMethodName)
) # noqa: E501
responses.push(self.client.copy_settings(self.index.name, index2.name))
index3 = F.index(
self.client, "{}_rules".format(self._testMethodName)
) # noqa: E501
responses.push(self.client.copy_rules(self.index.name, index3.name))
index4 = F.index(
self.client, "{}_synonyms".format(self._testMethodName)
) # noqa: E501
responses.push(self.client.copy_synonyms(self.index.name, index4.name))
index5 = F.index(
self.client, "{}_full_copy".format(self._testMethodName)
) # noqa: E501
responses.push(self.client.copy_index(self.index.name, index5.name))
responses.wait()
self.assertEqual(index2.get_settings()["attributesForFaceting"], ["company"])
index3.get_rule("company_auto_faceting")
with self.assertRaises(RequestException) as cm:
index3.get_synonym("google_placeholder")
index4.get_synonym("google_placeholder")
with self.assertRaises(RequestException) as cm:
index4.get_rule("company_auto_faceting")
index5.get_synonym("google_placeholder")
index5.get_rule("company_auto_faceting")
self.assertEqual(index5.get_settings()["attributesForFaceting"], ["company"])
for obj in index5.browse_objects():
self.assertIn(obj, objects)
index6 = F.index(
self.client, "{}_after_move".format(self._testMethodName)
) # noqa: E501
self.client.move_index(self.index.name, index6.name).wait()
index6.get_synonym("google_placeholder")
index6.get_rule("company_auto_faceting")
self.assertEqual(index6.get_settings()["attributesForFaceting"], ["company"])
for obj in index6.browse_objects():
self.assertIn(obj, objects)
with self.assertRaises(RequestException) as cm:
self.client.init_index(self.index.name).search("")
self.assertEqual(cm.exception.status_code, 404)
def test_replacing(self):
responses = MultipleResponse()
responses.push(self.index.save_object({"objectID": "one"}))
responses.push(self.index.save_rule(F.rule(object_id="one")))
responses.push(self.index.save_synonym(
{"objectID": "one", "type": "synonym", "synonyms": ["one", "two"]}
))
responses.wait()
responses.push(self.index.replace_all_objects([{"objectID": "two"}]))
responses.push(self.index.replace_all_rules([{
"objectID": "two",
"condition": {"anchoring": "is", "pattern": "pattern"},
"consequence": {
"params": {
"query": {
"edits": [
{"type": "remove", "delete": "pattern"}
]
}
}
}
}
]))
responses.push(self.index.replace_all_synonyms([
{"objectID": "two", "type": "synonym", "synonyms": ["one", "two"]}
]))
responses.wait()
# Check that record with objectID=`one` does not exist
with self.assertRaises(RequestException) as _:
self.index.get_object('one')
# Check that record with objectID=`two` does exist
self.assertEqual(self.index.get_object('two')['objectID'], 'two')
# Check that rule with objectID=`one` does not exist
with self.assertRaises(RequestException) as _:
self.index.get_rule('one')
# Check that rule with objectID=`two` does exist
self.assertEqual(self.index.get_rule('two')['objectID'], 'two')
# Check that synonym with objectID=`one` does not exist
with self.assertRaises(RequestException) as _:
self.index.get_synonym('one')
# Check that synonym with objectID="two" does exist using getSynonym
self.assertEqual(self.index.get_synonym('two')['objectID'], 'two')
async def ingest_ltd_lander_jsonld_document(
*,
app: web.Application,
logger: BoundLogger,
url_ingest_message: Dict[str, Any],
) -> None:
"""Run the Algolia ingest of a LTD_LANDER_JSONLD content type.
Parameters
----------
app : `aiohttp.web.Application`
The app.
logger
A structlog logger that is bound with context about the Kafka message.
url_ingest_message : `dict`
The deserialized value of the Kafka message.
"""
logger = logger.bind(
content_url=url_ingest_message["url"],
content_type=url_ingest_message["content_type"],
)
logger.info("Starting LTD_LANDER_JSONLD ingest")
http_session = app["safir/http_session"]
edition_data = await get_json_data(
url=url_ingest_message["edition"]["url"],
logger=logger,
http_session=http_session,
)
published_url = edition_data["published_url"]
jsonld_name = "metadata.jsonld"
if published_url.endswith("/"):
jsonld_url = f"{published_url}{jsonld_name}"
else:
jsonld_url = f"{published_url}/{jsonld_name}"
try:
metadata = await get_json_data(
url=jsonld_url,
logger=logger,
http_session=http_session,
# by-pass aiohttp's encoding check; the jsonld files do not have
# correct CONTENT-TYPE headers.
encoding="utf-8",
content_type=None,
)
except Exception:
logger.exception("Failure getting metadata.jsonld",
jsonld_url=jsonld_url)
raise
try:
reduced_document = ReducedLtdLanderDocument(url=published_url,
metadata=metadata,
logger=logger)
except Exception:
logger.exception("Failed to build record")
raise
surrogate_key = generate_surrogate_key()
logger.debug("Reduced LTD Lander Document",
chunks=len(reduced_document.chunks))
try:
records = [
create_record(
chunk=s,
document=reduced_document,
surrogate_key=surrogate_key,
) for s in reduced_document.chunks
]
description_chunk = ContentChunk(
headers=[reduced_document.h1],
content=reduced_document.description,
)
records.append(
create_record(
chunk=description_chunk,
document=reduced_document,
surrogate_key=surrogate_key,
))
except Exception:
logger.exception("Failed to build records")
raise
logger.info("Finished building records")
if app["ook/algolia_search"] is not None:
try:
client = app["ook/algolia_search"]
index = client.init_index(
app["safir/config"].algolia_document_index_name)
except Exception:
logger.exception(
"Error initializing Algolia index",
index_name=app["safir/config"].algolia_document_index_name,
)
raise
tasks = [index.save_object_async(record) for record in records]
try:
results = await asyncio.gather(*tasks)
MultipleResponse(results).wait()
except Exception:
logger.error("Got algoliasearch request error")
for record in records:
logger.debug(json.dumps(record, indent=2, sort_keys=True))
logger.info("Finished uploading to Algolia")
await delete_old_records(
index=index,
base_url=records[0]["baseUrl"],
surrogate_key=surrogate_key,
logger=logger,
)
def test_copy_move_index(self):
objects = [{
'objectID': 'one',
'company': 'apple'
}, {
'objectID': 'two',
'company': 'algolia'
}]
responses = MultipleResponse([
self.index.save_objects(objects),
self.index.set_settings({'attributesForFaceting': ['company']}),
self.index.save_synonym({
'objectID': 'google_placeholder',
'type': 'placeholder',
'placeholder': '<GOOG>',
'replacements': ['Google', 'GOOG']
}),
self.index.save_rule({
"objectID": "company_auto_faceting",
"condition": {
"anchoring": "contains",
"pattern": "{facet:company}",
},
"consequence": {
"params": {
"automaticFacetFilters": ["company"]
}
}
})
]).wait()
self.index2 = F.index('{}_settings'.format(self._testMethodName))
responses.push(
self.client.copy_settings(self.index.name, self.index2.name))
self.index3 = F.index('{}_rules'.format(self._testMethodName))
responses.push(
self.client.copy_rules(self.index.name, self.index3.name))
self.index4 = F.index('{}_synonyms'.format(self._testMethodName))
responses.push(
self.client.copy_synonyms(self.index.name, self.index4.name))
self.index5 = F.index('{}_full_copy'.format(self._testMethodName))
responses.push(
self.client.copy_index(self.index.name, self.index5.name))
responses.wait()
self.assertEqual(self.index2.get_settings()['attributesForFaceting'],
['company'])
self.index3.get_rule('company_auto_faceting')
with self.assertRaises(RequestException) as cm:
self.index3.get_synonym('google_placeholder')
self.index4.get_synonym('google_placeholder')
with self.assertRaises(RequestException) as cm:
self.index4.get_rule('company_auto_faceting')
self.index5.get_synonym('google_placeholder')
self.index5.get_rule('company_auto_faceting')
self.assertEqual(self.index5.get_settings()['attributesForFaceting'],
['company'])
for obj in self.index5.browse_objects():
self.assertIn(obj, objects)
self.index6 = F.index('{}_after_move'.format(self._testMethodName))
self.client.move_index(self.index.name, self.index6.name).wait()
self.index6.get_synonym('google_placeholder')
self.index6.get_rule('company_auto_faceting')
self.assertEqual(self.index6.get_settings()['attributesForFaceting'],
['company'])
for obj in self.index6.browse_objects():
self.assertIn(obj, objects)
with self.assertRaises(RequestException) as cm:
self.client.init_index(self.index.name).search('')
self.assertEqual(cm.exception.status_code, 404)
def test_search(self):
responses = MultipleResponse()
responses.push(self.index.save_objects([
{"company": "Algolia", "name": "Julien Lemoine"},
{"company": "Algolia", "name": "Nicolas Dessaigne"},
{"company": "Amazon", "name": "Jeff Bezos"},
{"company": "Apple", "name": "Steve Jobs"},
{"company": "Apple", "name": "Steve Wozniak"},
{"company": "Arista Networks", "name": "Jayshree Ullal"},
{"company": "Google", "name": "Larry Page"},
{"company": "Google", "name": "Rob Pike"},
{"company": "Google", "name": "Serguey Brin"},
{"company": "Microsoft", "name": "Bill Gates"},
{"company": "SpaceX", "name": "Elon Musk"},
{"company": "Tesla", "name": "Elon Musk"},
{"company": "Yahoo", "name": "Marissa Mayer"}
], {'autoGenerateObjectIDIfNotExist': True}))
responses.push(self.index.set_settings({
'attributesForFaceting': ["searchable(company)"]
}))
responses.wait()
# Perform a search query using search with the query `algolia` and no
# parameter and check that the number of returned hits is equal to 2
result = self.index.search('algolia')
self.assertEqual(result['nbHits'], 2)
# Perform a search using search with the query `elon` and the
# following parameter and check that the queryID field from
# the response is not empty
result = self.index.search('elon', {'clickAnalytics': True})
self.assertIn('queryID', result)
# Perform a faceted search using search with the query `elon` and the
# following parameters and check that the number of returned hits is
# equal to 1
result = self.index.search('elon', {
'facets': '*',
'facetFilters': 'company:tesla'
})
self.assertEqual(result['nbHits'], 1)
# Perform a filtered search using search with the query `elon` and the
# following parameters and check that the number of returned hits is
# equal to 2
result = self.index.search('elon', {
'facets': '*',
'filters': '(company:tesla OR company:spacex)'
})
self.assertEqual(result['nbHits'], 2)
result = self.index.search_for_facet_values('company', 'a')[
'facetHits']
values = list(
map(lambda facet: facet['value'], result))
self.assertIn('Algolia', values)
self.assertIn('Amazon', values)
self.assertIn('Apple', values)
self.assertIn('Arista Networks', values)
