def test_indexer_endpoint_https(self):
credential = AzureKeyCredential(key="old_api_key")
client = SearchIndexerClient("endpoint", credential)
assert client._endpoint.startswith('https')
client = SearchIndexerClient("https://endpoint", credential)
assert client._endpoint.startswith('https')
with pytest.raises(ValueError):
client = SearchIndexerClient("http://endpoint", credential)
with pytest.raises(ValueError):
client = SearchIndexerClient(12345, credential)
async def test_delete_skillset_if_unchanged(self, api_key, endpoint,
index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
s = EntityRecognitionSkill(inputs=[
InputFieldMappingEntry(name="text", source="/document/content")
],
outputs=[
OutputFieldMappingEntry(
name="organizations",
target_name="organizations")
])
skillset = SearchIndexerSkillset(name='test-ss',
skills=list([s]),
description="desc")
result = await client.create_skillset(skillset)
etag = result.e_tag
skillset1 = SearchIndexerSkillset(name='test-ss',
skills=list([s]),
description="updated")
updated = await client.create_or_update_skillset(skillset1)
updated.e_tag = etag
with pytest.raises(HttpResponseError):
await client.delete_skillset(
updated, match_condition=MatchConditions.IfNotModified)
async def create_indexer():
# create an index
index_name = "hotels"
fields = [
SimpleField(name="hotelId", type=SearchFieldDataType.String, key=True),
SimpleField(name="baseRate", type=SearchFieldDataType.Double)
]
index = SearchIndex(name=index_name, fields=fields)
ind_client = SearchIndexerClient(service_endpoint, AzureKeyCredential(key))
async with ind_client:
await ind_client.create_index(index)
# [START create_indexer_async]
# create a datasource
container = SearchIndexerDataContainer(name='searchcontainer')
data_source_connection = SearchIndexerDataSourceConnection(
name="indexer-datasource",
type="azureblob",
connection_string=connection_string,
container=container)
async with ind_client:
data_source = await ind_client.create_data_source_connection(
data_source_connection)
# create an indexer
indexer = SearchIndexer(name="async-sample-indexer",
data_source_name="async-indexer-datasource",
target_index_name="indexer-hotels")
async with indexers_client:
result = await indexers_client.create_indexer(indexer)
print("Create new Indexer - async-sample-indexer")
async def _prepare_indexer(self, endpoint, api_key, name="sample-indexer", ds_name="sample-datasource", id_name="hotels"):
con_str = self.settings.AZURE_STORAGE_CONNECTION_STRING
self.scrubber.register_name_pair(con_str, 'connection_string')
container = SearchIndexerDataContainer(name='searchcontainer')
data_source_connection = SearchIndexerDataSourceConnection(
name=ds_name,
type="azureblob",
connection_string=con_str,
container=container
)
ds_client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
ds = await ds_client.create_data_source_connection(data_source_connection)
index_name = id_name
fields = [
{
"name": "hotelId",
"type": "Edm.String",
"key": True,
"searchable": False
}]
index = SearchIndex(name=index_name, fields=fields)
ind_client = SearchIndexClient(endpoint, AzureKeyCredential(api_key))
ind = await ind_client.create_index(index)
return SearchIndexer(name=name, data_source_name=ds.name, target_index_name=ind.name)
async def test_get_datasource_async(self, api_key, endpoint, index_name,
**kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
data_source = self._create_datasource()
created = await client.create_datasource(data_source)
result = await client.get_datasource("sample-datasource")
assert result.name == "sample-datasource"
async def test_search_indexers(self, endpoint, api_key, **kwargs):
storage_cs = kwargs.get("search_storage_connection_string")
container_name = kwargs.get("search_storage_container_name")
client = SearchIndexerClient(endpoint, api_key)
index_client = SearchIndexClient(endpoint, api_key)
async with client:
async with index_client:
await self._test_create_indexer(client, index_client,
storage_cs, container_name)
await self._test_delete_indexer(client, index_client,
storage_cs, container_name)
await self._test_get_indexer(client, index_client, storage_cs,
container_name)
await self._test_list_indexer(client, index_client, storage_cs,
container_name)
await self._test_create_or_update_indexer(
client, index_client, storage_cs, container_name)
await self._test_reset_indexer(client, index_client,
storage_cs, container_name)
await self._test_run_indexer(client, index_client, storage_cs,
container_name)
await self._test_get_indexer_status(client, index_client,
storage_cs, container_name)
await self._test_create_or_update_indexer_if_unchanged(
client, index_client, storage_cs, container_name)
await self._test_delete_indexer_if_unchanged(
client, index_client, storage_cs, container_name)
async def test_create_indexer(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
indexer = await self._prepare_indexer(endpoint, api_key)
result = await client.create_indexer(indexer)
assert result.name == "sample-indexer"
assert result.target_index_name == "hotels"
assert result.data_source_name == "sample-datasource"
async def test_create_or_update_skillset_if_unchanged(
self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
s = EntityRecognitionSkill(inputs=[
InputFieldMappingEntry(name="text", source="/document/content")
],
outputs=[
OutputFieldMappingEntry(
name="organizations",
target_name="organizations")
])
ss = await client.create_or_update_skillset(name='test-ss',
skills=[s],
description="desc1")
etag = ss.e_tag
await client.create_or_update_skillset(name='test-ss',
skills=[s],
description="desc2",
skillset=ss)
assert len(await client.get_skillsets()) == 1
ss.e_tag = etag
with pytest.raises(HttpResponseError):
await client.create_or_update_skillset(
name='test-ss',
skills=[s],
skillset=ss,
match_condition=MatchConditions.IfNotModified)
async def test_get_indexer_status(self, api_key, endpoint, index_name,
**kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
indexer = await self._prepare_indexer(endpoint, api_key)
result = await client.create_indexer(indexer)
status = await client.get_indexer_status("sample-indexer")
assert status.status is not None
async def test_delete_datasource_async(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
data_source_connection = self._create_data_source_connection()
result = await client.create_data_source_connection(data_source_connection)
assert len(await client.get_data_source_connections()) == 1
await client.delete_data_source_connection("sample-datasource")
assert len(await client.get_data_source_connections()) == 0
async def test_delete_indexer(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
indexer = await self._prepare_indexer(endpoint, api_key)
result = await client.create_indexer(indexer)
assert len(await client.get_indexers()) == 1
await client.delete_indexer("sample-indexer")
assert len(await client.get_indexers()) == 0
async def test_reset_indexer(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
indexer = await self._prepare_indexer(endpoint, api_key)
result = await client.create_indexer(indexer)
assert len(await client.get_indexers()) == 1
await client.reset_indexer("sample-indexer")
assert (await client.get_indexer_status("sample-indexer")).last_result.status.lower() in ('inprogress', 'reset')
async def test_create_or_update_skillset_inplace(self, api_key, endpoint,
index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
s = EntityRecognitionSkill(inputs=[
InputFieldMappingEntry(name="text", source="/document/content")
],
outputs=[
OutputFieldMappingEntry(
name="organizations",
target_name="organizations")
])
skillset1 = SearchIndexerSkillset(name='test-ss',
skills=list([s]),
description="desc1")
ss = await client.create_or_update_skillset(skillset1)
skillset2 = SearchIndexerSkillset(name='test-ss',
skills=[s],
description="desc2",
skillset=ss)
await client.create_or_update_skillset(skillset2)
assert len(await client.get_skillsets()) == 1
result = await client.get_skillset("test-ss")
assert isinstance(result, SearchIndexerSkillset)
assert result.name == "test-ss"
assert result.description == "desc2"
async def test_run_indexer(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
indexer = await self._prepare_indexer(endpoint, api_key)
result = await client.create_indexer(indexer)
assert len(await client.get_indexers()) == 1
start = time.time()
await client.run_indexer("sample-indexer")
assert (await client.get_indexer_status("sample-indexer")).status == 'running'
async def test_create_datasource_async(self, api_key, endpoint, index_name,
**kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
data_source_connection = self._create_data_source_connection()
result = await client.create_data_source_connection(
data_source_connection)
assert result.name == "sample-datasource"
assert result.type == "azureblob"
async def test_list_indexer(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
indexer1 = await self._prepare_indexer(endpoint, api_key)
indexer2 = await self._prepare_indexer(endpoint, api_key, name="another-indexer", ds_name="another-datasource", id_name="another-index")
created1 = await client.create_indexer(indexer1)
created2 = await client.create_indexer(indexer2)
result = await client.get_indexers()
assert isinstance(result, list)
assert set(x.name for x in result) == {"sample-indexer", "another-indexer"}
async def test_list_datasource_async(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
data_source_connection1 = self._create_data_source_connection()
data_source_connection2 = self._create_data_source_connection(name="another-sample")
created1 = await client.create_data_source_connection(data_source_connection1)
created2 = await client.create_data_source_connection(data_source_connection2)
result = await client.get_data_source_connections()
assert isinstance(result, list)
assert set(x.name for x in result) == {"sample-datasource", "another-sample"}
async def test_create_or_update_datasource_async(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
data_source_connection = self._create_data_source_connection()
created = await client.create_data_source_connection(data_source_connection)
assert len(await client.get_data_source_connections()) == 1
data_source_connection.description = "updated"
await client.create_or_update_data_source_connection(data_source_connection)
assert len(await client.get_data_source_connections()) == 1
result = await client.get_data_source_connection("sample-datasource")
assert result.name == "sample-datasource"
assert result.description == "updated"
async def test_create_or_update_indexer(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
indexer = await self._prepare_indexer(endpoint, api_key)
created = await client.create_indexer(indexer)
assert len(await client.get_indexers()) == 1
indexer.description = "updated"
await client.create_or_update_indexer(indexer)
assert len(await client.get_indexers()) == 1
result = await client.get_indexer("sample-indexer")
assert result.name == "sample-indexer"
assert result.description == "updated"
async def test_get_skillsets(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
s = EntityRecognitionSkill(inputs=[InputFieldMappingEntry(name="text", source="/document/content")],
outputs=[OutputFieldMappingEntry(name="organizations", target_name="organizations")])
await client.create_skillset(name='test-ss-1', skills=[s], description="desc1")
await client.create_skillset(name='test-ss-2', skills=[s], description="desc2")
result = await client.get_skillsets()
assert isinstance(result, list)
assert all(isinstance(x, SearchIndexerSkillset) for x in result)
assert set(x.name for x in result) == {"test-ss-1", "test-ss-2"}
async def test_skillset_crud(self, api_key, endpoint):
client = SearchIndexerClient(endpoint, api_key)
async with client:
await self._test_create_skillset(client)
await self._test_get_skillset(client)
await self._test_get_skillsets(client)
await self._test_create_or_update_skillset(client)
await self._test_create_or_update_skillset_if_unchanged(client)
await self._test_create_or_update_skillset_inplace(client)
await self._test_delete_skillset_if_unchanged(client)
await self._test_delete_skillset(client)
async def test_delete_indexer_if_unchanged(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
indexer = await self._prepare_indexer(endpoint, api_key)
result = await client.create_indexer(indexer)
etag = result.e_tag
indexer.description = "updated"
await client.create_or_update_indexer(indexer)
indexer.e_tag = etag
with pytest.raises(HttpResponseError):
await client.delete_indexer(indexer, match_condition=MatchConditions.IfNotModified)
async def test_data_source(self, endpoint, api_key, **kwargs):
storage_cs = kwargs.get("search_storage_connection_string")
client = SearchIndexerClient(endpoint, api_key)
async with client:
await self._test_create_datasource(client, storage_cs)
await self._test_delete_datasource(client, storage_cs)
await self._test_get_datasource(client, storage_cs)
await self._test_list_datasources(client, storage_cs)
await self._test_create_or_update_datasource(client, storage_cs)
await self._test_create_or_update_datasource_if_unchanged(
client, storage_cs)
await self._test_delete_datasource_if_unchanged(client, storage_cs)
def test_indexer_credential_roll(self):
credential = AzureKeyCredential(key="old_api_key")
client = SearchIndexerClient("endpoint", credential)
assert client._headers == {
"api-key": "old_api_key",
"Accept": "application/json;odata.metadata=minimal",
}
credential.update("new_api_key")
assert client._headers == {
"api-key": "new_api_key",
"Accept": "application/json;odata.metadata=minimal",
}
async def test_delete_skillset(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
s = EntityRecognitionSkill(inputs=[InputFieldMappingEntry(name="text", source="/document/content")],
outputs=[OutputFieldMappingEntry(name="organizations", target_name="organizations")])
result = await client.create_skillset(name='test-ss', skills=[s], description="desc")
assert len(await client.get_skillsets()) == 1
await client.delete_skillset("test-ss")
if self.is_live:
time.sleep(TIME_TO_SLEEP)
assert len(await client.get_skillsets()) == 0
async def test_create_skillset(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
name = "test-ss"
s1 = EntityRecognitionSkill(inputs=[InputFieldMappingEntry(name="text", source="/document/content")],
outputs=[OutputFieldMappingEntry(name="organizations", target_name="organizationsS1")],
description="Skill Version 1",
model_version="1",
include_typeless_entities=True)
s2 = EntityRecognitionSkill(inputs=[InputFieldMappingEntry(name="text", source="/document/content")],
outputs=[OutputFieldMappingEntry(name="organizations", target_name="organizationsS2")],
skill_version=EntityRecognitionSkillVersion.LATEST,
description="Skill Version 3",
model_version="3",
include_typeless_entities=True)
s3 = SentimentSkill(inputs=[InputFieldMappingEntry(name="text", source="/document/content")],
outputs=[OutputFieldMappingEntry(name="score", target_name="scoreS3")],
skill_version=SentimentSkillVersion.V1,
description="Sentiment V1",
include_opinion_mining=True)
s4 = SentimentSkill(inputs=[InputFieldMappingEntry(name="text", source="/document/content")],
outputs=[OutputFieldMappingEntry(name="confidenceScores", target_name="scoreS4")],
skill_version=SentimentSkillVersion.V3,
description="Sentiment V3",
include_opinion_mining=True)
s5 = EntityLinkingSkill(inputs=[InputFieldMappingEntry(name="text", source="/document/content")],
outputs=[OutputFieldMappingEntry(name="entities", target_name="entitiesS5")],
minimum_precision=0.5)
skillset = SearchIndexerSkillset(name=name, skills=list([s1, s2, s3, s4, s5]), description="desc")
result = await client.create_skillset(skillset)
assert isinstance(result, SearchIndexerSkillset)
assert result.name == "test-ss"
assert result.description == "desc"
assert result.e_tag
assert len(result.skills) == 5
assert isinstance(result.skills[0], EntityRecognitionSkill)
assert result.skills[0].skill_version == EntityRecognitionSkillVersion.V1
assert isinstance(result.skills[1], EntityRecognitionSkill)
assert result.skills[1].skill_version == EntityRecognitionSkillVersion.V3
assert isinstance(result.skills[2], SentimentSkill)
assert result.skills[2].skill_version == SentimentSkillVersion.V1
assert isinstance(result.skills[3], SentimentSkill)
assert result.skills[3].skill_version == SentimentSkillVersion.V3
assert isinstance(result.skills[4], EntityLinkingSkill)
assert result.skills[4].minimum_precision == 0.5
assert len(await client.get_skillsets()) == 1
async def test_skillset_crud(self, api_key, endpoint):
client = SearchIndexerClient(endpoint, api_key)
async with client:
await self._test_create_skillset(client)
await self._test_get_skillset(client)
await self._test_get_skillsets(client)
# TODO: Disabled due to service regression. See #22769
#await self._test_create_or_update_skillset(client)
await self._test_create_or_update_skillset_if_unchanged(client)
# TODO: Disabled due to service regression. See #22769
#await self._test_create_or_update_skillset_inplace(client)
# TODO: Disabled due to service regression. See #22769
#await self._test_delete_skillset_if_unchanged(client)
await self._test_delete_skillset(client)
async def test_create_skillset(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
s = EntityRecognitionSkill(inputs=[InputFieldMappingEntry(name="text", source="/document/content")],
outputs=[OutputFieldMappingEntry(name="organizations", target_name="organizations")])
result = await client.create_skillset(name='test-ss', skills=[s], description="desc")
assert isinstance(result, SearchIndexerSkillset)
assert result.name == "test-ss"
assert result.description == "desc"
assert result.e_tag
assert len(result.skills) == 1
assert isinstance(result.skills[0], EntityRecognitionSkill)
assert len(await client.get_skillsets()) == 1
async def test_delete_datasource_if_unchanged(self, api_key, endpoint, index_name, **kwargs):
client = SearchIndexerClient(endpoint, AzureKeyCredential(api_key))
data_source_connection = self._create_data_source_connection()
created = await client.create_data_source_connection(data_source_connection)
etag = created.e_tag
# Now update the data source connection
data_source_connection.description = "updated"
await client.create_or_update_data_source_connection(data_source_connection)
# prepare data source connection
data_source_connection.e_tag = etag # reset to the original data source connection
with pytest.raises(HttpResponseError):
await client.delete_data_source_connection(data_source_connection, match_condition=MatchConditions.IfNotModified)
assert len(await client.get_data_source_connections()) == 1
"""
import asyncio
import os
service_endpoint = os.getenv("AZURE_SEARCH_SERVICE_ENDPOINT")
key = os.getenv("AZURE_SEARCH_API_KEY")
connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
from azure.core.credentials import AzureKeyCredential
from azure.search.documents.indexes.models import (
SearchIndexerDataContainer, SearchIndexerDataSourceConnection, SearchIndex,
SearchIndexer, SimpleField, SearchFieldDataType)
from azure.search.documents.indexes.aio import SearchIndexerClient, SearchIndexClient
indexers_client = SearchIndexerClient(service_endpoint,
AzureKeyCredential(key))
async def create_indexer():
# create an index
index_name = "hotels"
fields = [
SimpleField(name="hotelId", type=SearchFieldDataType.String, key=True),
SimpleField(name="baseRate", type=SearchFieldDataType.Double)
]
index = SearchIndex(name=index_name, fields=fields)
ind_client = SearchIndexerClient(service_endpoint, AzureKeyCredential(key))
async with ind_client:
await ind_client.create_index(index)
# [START create_indexer_async]