def test_translation_with_glossary(self, client):
doc = Document(data=b'testing')
source_container_sas_url = self.create_source_container(data=[doc])
target_container_sas_url = self.create_target_container()
container_client = ContainerClient(self.storage_endpoint,
self.source_container_name,
self.storage_key)
with open(GLOSSARY_FILE_NAME, "rb") as fd:
container_client.upload_blob(name=GLOSSARY_FILE_NAME,
data=fd.read())
prefix, suffix = source_container_sas_url.split("?")
glossary_file_sas_url = prefix + "/" + GLOSSARY_FILE_NAME + "?" + suffix
poller = client.begin_translation(
source_container_sas_url,
target_container_sas_url,
"es",
glossaries=[
TranslationGlossary(glossary_url=glossary_file_sas_url,
file_format="csv")
])
result = poller.result()
container_client = ContainerClient(self.storage_endpoint,
self.target_container_name,
self.storage_key)
# download translated file and assert that translation reflects glossary changes
document = doc.name + doc.suffix
with open(document, "wb") as my_blob:
download_stream = container_client.download_blob(document)
my_blob.write(download_stream.readall())
with open(document, "rb") as fd:
translated = fd.readline()
assert b'essai' in translated # glossary worked
os.remove(document)
async def _begin_multiple_translations_async(self, async_client, operations_count, **kwargs):
wait_for_operation = kwargs.pop('wait', True)
language_code = kwargs.pop('language_code', "es")
docs_per_operation = kwargs.pop('docs_per_operation', 2)
result_ids = []
for i in range(operations_count):
# prepare containers and test data
'''
# note
since we're only testing the client library
we can use sync container calls in here
no need for async container clients!
'''
blob_data = Document.create_dummy_docs(docs_per_operation)
source_container_sas_url = self.create_source_container(data=blob_data)
target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(
target_url=target_container_sas_url,
language_code=language_code
)
]
)
]
# submit multiple operations
poller = await async_client.begin_translation(translation_inputs)
self.assertIsNotNone(poller.id)
if wait_for_operation:
await poller.result()
else:
await poller.wait()
result_ids.append(poller.id)
return result_ids
def test_single_source_single_target_with_suffix(self, client):
# prepare containers and test data
blob_data = b'This is some text'
suffix = "txt"
source_container_sas_url = self.create_source_container(data=Document(
data=blob_data))
target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code="es")
],
suffix=suffix)
]
# submit job and test
self._submit_and_validate_translation_job(client, translation_inputs,
1)
async def _create_and_submit_sample_translation_jobs_async(
self, async_client, jobs_count):
result_job_ids = []
for i in range(jobs_count):
# prepare containers and test data
'''
# WARNING!!
TOTAL_DOC_COUNT_IN_JOB = 1
if you plan to create more docs in the job,
please update this variable TOTAL_DOC_COUNT_IN_JOB in respective test
# note
since we're only testing the client library
we can use sync container calls in here
no need for async container clients!
'''
blob_data = b'This is some text'
source_container_sas_url = self.create_source_container(
data=Document(data=blob_data))
target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code="es")
])
]
# submit multiple jobs
job_details = await async_client.create_translation_job(
translation_inputs)
self.assertIsNotNone(job_details.id)
await async_client.wait_until_done(job_details.id)
result_job_ids.append(job_details.id)
return result_job_ids
async def test_single_input_with_kwargs(self, client):
# prepare containers and test data
source_container_sas_url = self.create_source_container(data=Document(data=b'hello world'))
target_container_sas_url = self.create_target_container()
def callback(request):
req = _StartTranslationDetails.deserialize(json.loads(request.http_request.body))
input = req.inputs[0]
assert input.source.source_url == source_container_sas_url
assert input.source.language == "en"
assert input.source.filter.prefix == ""
assert input.source.filter.suffix == ".txt"
assert input.storage_type == "File"
assert input.targets[0].category == "fake"
assert input.targets[0].glossaries[0].format == "txt"
assert input.targets[0].glossaries[0].glossary_url == "https://glossaryfile.txt"
assert input.targets[0].language == "es"
assert input.targets[0].target_url == target_container_sas_url
async with client:
try:
poller = await client.begin_translation(
source_container_sas_url,
target_container_sas_url,
"es",
storage_type="File",
source_language_code="en",
prefix="",
suffix=".txt",
category_id="fake",
glossaries=[TranslationGlossary(
glossary_url="https://glossaryfile.txt",
file_format="txt"
)],
raw_response_hook=callback
)
await poller.result()
except HttpResponseError as e:
pass
def test_single_source_single_target_with_prefix(self, client):
# prepare containers and test data
blob_data = b'This is some text'
prefix = "xyz"
source_container_sas_url = self.create_source_container(
data=Document(data=blob_data, prefix=prefix))
target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code="es")
],
prefix=prefix)
]
# submit translation and test
self._begin_and_validate_translation(client, translation_inputs, 1,
"es")
def test_bad_input_target(self, client):
# prepare containers and test data
blob_data = b'This is some text'
source_container_sas_url = self.create_source_container(data=Document(data=blob_data))
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(
target_url="https://idont.ex.ist",
language_code="es"
)
]
)
]
with pytest.raises(HttpResponseError) as e:
poller = client.begin_translation(translation_inputs)
result = poller.result()
assert e.value.error.code == "InvalidTargetDocumentAccessLevel"
async def test_bad_input_target(self, client):
# prepare containers and test data
blob_data = b'This is some text'
source_container_sas_url = self.create_source_container(data=Document(data=blob_data))
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(
target_url="https://idont.ex.ist",
language_code="es"
)
]
)
]
with pytest.raises(HttpResponseError) as e:
job = await client.create_translation_job(translation_inputs)
job = await client.wait_until_done(job.id)
assert e.value.error.code == "InvalidDocumentAccessLevel"
def test_overloaded_inputs(self, **kwargs):
client = kwargs.pop("client")
variables = kwargs.pop("variables", {})
# prepare containers and test data
source_container_sas_url = self.create_source_container(
data=Document(data=b'hello world'), variables=variables)
target_container_sas_url = self.create_target_container(
variables=variables)
target_container_sas_url_2 = self.create_target_container(
variables=variables, container_suffix="2")
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code="es")
])
]
# positional
poller = client.begin_translation(translation_inputs)
result = poller.result()
self._validate_translation_metadata(poller,
status="Succeeded",
total=1,
succeeded=1)
# keyword
translation_inputs[0].targets[
0].target_url = target_container_sas_url_2
poller = client.begin_translation(inputs=translation_inputs)
result = poller.result()
self._validate_translation_metadata(poller,
status="Succeeded",
total=1,
succeeded=1)
return variables
def test_list_statuses(self, **kwargs):
client = kwargs.pop("client")
variables = kwargs.pop("variables", {})
# prepare containers and test data
blob_data = [Document(data=b'This is some text')]
source_container_sas_url = self.create_source_container(
data=blob_data, variables=variables)
target_container_sas_url = self.create_target_container(
variables=variables)
target_language = "es"
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code=target_language)
])
]
# submit and validate translation operation
translation_id = self._begin_and_validate_translation(
client, translation_inputs, len(blob_data), target_language)
# get doc statuses
doc_statuses = client.list_document_statuses(translation_id)
assert doc_statuses is not None
# get first doc
first_doc = next(doc_statuses)
assert first_doc.id is not None
# get doc details
doc_status = client.get_document_status(translation_id=translation_id,
document_id=first_doc.id)
self._validate_doc_status(doc_status, target_language)
return variables
async def _create_and_submit_sample_translation_jobs_async(self, async_client, jobs_count, **kwargs):
wait_for_job = kwargs.pop('wait', True)
language_code = kwargs.pop('language_code', "es")
docs_per_job = kwargs.pop('docs_per_job', 2)
result_job_ids = []
for i in range(jobs_count):
# prepare containers and test data
'''
# note
since we're only testing the client library
we can use sync container calls in here
no need for async container clients!
'''
blob_data = Document.create_dummy_docs(docs_per_job)
source_container_sas_url = self.create_source_container(data=blob_data)
target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(
target_url=target_container_sas_url,
language_code=language_code
)
]
)
]
# submit multiple jobs
job_details = await async_client.create_translation_job(translation_inputs)
self.assertIsNotNone(job_details.id)
if wait_for_job:
await async_client.wait_until_done(job_details.id)
result_job_ids.append(job_details.id)
return result_job_ids
def test_active_directory_auth(self):
token = self.generate_oauth_token()
endpoint = self.get_oauth_endpoint()
client = DocumentTranslationClient(endpoint, token)
# prepare containers and test data
blob_data = b'This is some text'
source_container_sas_url = self.create_source_container(data=Document(
data=blob_data))
target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code="fr")
])
]
# submit translation and test
self._begin_and_validate_translation(client, translation_inputs, 1,
"fr")
def test_single_source_two_targets(self, client):
# prepare containers and test data
blob_data = b'This is some text'
source_container_sas_url = self.create_source_container(data=Document(
data=blob_data))
target_container_sas_url = self.create_target_container()
additional_target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code="es"),
TranslationTarget(
target_url=additional_target_container_sas_url,
language_code="fr")
])
]
# submit translation and test
self._begin_and_validate_translation(client, translation_inputs, 2)
async def _create_translation_job_with_dummy_docs_async(
self, async_client, docs_count, **kwargs):
'''
appropriated this method from another PR! #18302
please resolve conflict before merge
keep in mind it's the exact same method
'''
# get input parms
wait_for_job = kwargs.pop('wait', False)
language_code = kwargs.pop('language_code', "es")
# prepare containers and test data
blob_data = Document.create_dummy_docs(docs_count=docs_count)
source_container_sas_url = self.create_source_container(data=blob_data)
target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code=language_code)
])
]
# submit job
job_details = await async_client.create_translation_job(
translation_inputs)
self.assertIsNotNone(job_details.id)
# wait for result
if wait_for_job:
await async_client.wait_until_done(job_details.id)
# validate
self._validate_translation_job(job_details=job_details)
return job_details.id
async def test_single_source_single_target(self, **kwargs):
client = kwargs.pop("client")
variables = kwargs.pop("variables", {})
# prepare containers and test data
blob_data = b'This is some text'
source_container_sas_url = self.create_source_container(data=Document(data=blob_data), variables=variables)
target_container_sas_url = self.create_target_container(variables=variables)
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(
target_url=target_container_sas_url,
language_code="es"
)
]
)
]
# submit translation and test
await self._begin_and_validate_translation_async(client, translation_inputs, 1, "es")
return variables
def test_empty_document(self, client):
# prepare containers and test data
source_container_sas_url = self.create_source_container(
Document(data=b''))
target_container_sas_url = self.create_target_container()
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code="es")
])
]
job = client.create_translation_job(translation_inputs)
job = client.wait_until_done(job.id)
self._validate_translation_job(job, status="Failed", total=1, failed=1)
doc_status = client.list_all_document_statuses(job.id)
doc = next(doc_status)
assert doc.status == "Failed"
assert doc.error.code == "WrongDocumentEncoding"
def test_list_statuses_with_pagination(self, client):
# prepare containers and test data
blob_text = b'blob text'
blob_data = [
Document(data=blob_text),
Document(data=blob_text),
Document(data=blob_text),
Document(data=blob_text),
Document(data=blob_text),
Document(data=blob_text)
]
source_container_sas_url = self.create_source_container(data=blob_data)
target_container_sas_url = self.create_target_container()
result_per_page = 2
no_of_pages = len(blob_data) // result_per_page
target_language = "es"
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code=target_language)
])
]
# submit and validate job
job_id = self._submit_and_validate_translation_job(
client, translation_inputs, len(blob_data))
# check doc statuses
doc_statuses_pages = list(
client.list_all_document_statuses(
job_id=job_id, results_per_page=result_per_page))
self.assertEqual(len(doc_statuses_pages), no_of_pages)
# iterate by page
for page in doc_statuses_pages:
page_items = list(page)
self.assertEqual(len(page_items), result_per_page)
for document in page:
self._validate_doc_status(document, target_language)
def test_list_statuses_with_skip(self, client):
# prepare containers and test data
blob_text = b'blob text'
blob_data = [
Document(data=blob_text),
Document(data=blob_text),
Document(data=blob_text),
Document(data=blob_text),
Document(data=blob_text),
Document(data=blob_text)
]
source_container_sas_url = self.create_source_container(data=blob_data)
target_container_sas_url = self.create_target_container()
docs_len = len(blob_data)
skip = 2
target_language = "es"
# prepare translation inputs
translation_inputs = [
DocumentTranslationInput(
source_url=source_container_sas_url,
targets=[
TranslationTarget(target_url=target_container_sas_url,
language_code=target_language)
])
]
# submit and validate job
job_id = self._submit_and_validate_translation_job(
client, translation_inputs, len(blob_data))
# check doc statuses
doc_statuses = list(
client.list_all_document_statuses(job_id=job_id, skip=skip))
self.assertEqual(len(doc_statuses), docs_len - skip)
# iterate over docs
for document in doc_statuses:
self._validate_doc_status(document, target_language)
