def test_analyse_texts_with_files(client, pipeline_analyse_text_mock):
pipeline = Pipeline(Project(client, "LoadTesting"), "discharge")
with open("tests/resources/texts/text1.txt", "rb") as file1, open(
"tests/resources/texts/text2.txt", "rb"
) as file2:
results = pipeline.analyse_texts([file1, file2])
sources = [result.source.replace(os.sep, "/") for result in results]
assert sources == ["tests/resources/texts/text1.txt", "tests/resources/texts/text2.txt"]
def test_export_text_analysis_export_v5(client_version_5):
process = Process(
project=Project(client_version_5, PROJECT_NAME),
name="my-process",
pipeline_name="my-pipeline",
document_source_name="my-collection",
)
with pytest.raises(OperationNotSupported):
process.export_text_analysis()
def test_delete_pipeline_v6(client_version_6, requests_mock):
pipeline = Pipeline(Project(client_version_6, PROJECT_NAME), "discharge")
requests_mock.delete(
f"{URL_BASE_ID}/rest/experimental/textanalysis/projects/{pipeline.project.name}/pipelines/{pipeline.name}",
headers={"Content-Type": "application/json"},
json={
"payload": None,
"errorMessages": []
},
)
response = pipeline.delete()
assert response is None
def test_delete_resources(client, requests_mock):
pipeline = Pipeline(Project(client, PROJECT_NAME), "discharge")
requests_mock.delete(
f"{API_EXPERIMENTAL}/textanalysis"
f"/projects/{pipeline.project.name}"
f"/pipelines/{pipeline.name}/resources",
headers={"Content-Type": "application/json"},
json={
"payload": None,
"errorMessages": []
},
)
pipeline.delete_resources()
def test_analyse_texts(client, pipeline_analyse_text_mock):
pipeline = Pipeline(Project(client, PROJECT_NAME), "discharge")
file1_path = os.path.join(TEST_DIRECTORY, "resources/texts/text1.txt")
file2_path = os.path.join(TEST_DIRECTORY, "resources/texts/text2.txt")
with open(file1_path, "rb") as file1, open(file2_path, "rb") as file2:
results = list(pipeline.analyse_texts([file1, file2]))
sources = [result.source.replace(os.sep, "/") for result in results]
cases = [result.data for result in results]
exceptions = [result.exception for result in results]
assert not exceptions[0]
assert not exceptions[1]
assert sources[0].endswith("tests/resources/texts/text1.txt")
assert sources[1].endswith("tests/resources/texts/text2.txt")
def test_collection_process_complete(client_version_6, requests_mock):
pipeline = Pipeline(Project(client_version_6, PROJECT_NAME), "discharge")
requests_mock.post(
f"{URL_BASE_ID}/rest/experimental/textanalysis/projects/"
f"{PROJECT_NAME}/pipelines/discharge/collectionProcessComplete",
headers={"Content-Type": "application/json"},
json={
"payload": None,
"errorMessages": []
},
status_code=200,
)
response = pipeline.collection_process_complete()
assert response is None
def test_analyse_texts_with_paths(client, pipeline_analyse_text_mock):
pipeline = Pipeline(Project(client, "LoadTesting"), "discharge")
results = pipeline.analyse_texts(Path("tests/resources/texts").glob("*.txt"))
expected_results = []
for input_file in Path("tests/resources/texts").glob("*.txt"):
with open(input_file, "r", encoding="UTF-8") as input_io:
expected_results.append(
{"source": str(input_file).replace(os.sep, "/"), "text": input_io.read()}
)
assert [
{"source": result.source.replace(os.sep, "/"), "text": result.data[0]["coveredText"]}
for result in sorted(results, key=lambda x: x.source)
] == sorted(expected_results, key=lambda x: x["source"])
def test_analyse_texts_with_some_working_and_some_failing(
client_version_5, requests_mock):
requests_mock.get(
f"{API_BASE}/textanalysis/projects/{PROJECT_NAME}/pipelines/discharge/configuration",
headers={"Content-Type": "application/json"},
json={
"payload": {
"analysisEnginePoolSize": 4
},
"errorMessages": [],
},
)
def callback(request, context):
doc_text = request.text.read().decode("utf-8")
if doc_text == "works":
context.status_code = 200
return {
"payload": [
{
"begin": 0,
"end": len(doc_text),
"type": "uima.tcas.DocumentAnnotation",
"coveredText": doc_text
# ... truncated ...
},
],
"errorMessages": [],
}
else:
context.status_code = 500
return {
"payload": [],
"errorMessages": ["Kaputt!"],
}
requests_mock.post(
f"{API_BASE}/textanalysis/projects/{PROJECT_NAME}/pipelines/discharge/analyseText",
headers={"Content-Type": "application/json"},
json=callback,
)
pipeline = Pipeline(Project(client_version_5, PROJECT_NAME), "discharge")
results = list(pipeline.analyse_texts(["works", "fails"]))
assert results[0].successful() is True
assert results[1].successful() is False
def test_upload_resources(client_version_6, requests_mock):
pipeline = Pipeline(Project(client_version_6, PROJECT_NAME), "discharge")
requests_mock.post(
f"{API_EXPERIMENTAL}/textanalysis"
f"/projects/{pipeline.project.name}"
f"/pipelines/{pipeline.name}/resources",
headers={"Content-Type": "application/json"},
status_code=200,
json={
"payload": {
"files": [
"text1.txt",
]
},
"errorMessages": [],
},
)
resources = pipeline.upload_resources(TEST_DIRECTORY + "/" +
"resources/zip_test/text1.txt")
assert len(resources) == 1
def test_download_resources(client, requests_mock):
pipeline = Pipeline(Project(client, PROJECT_NAME), "discharge")
target_path = Path(TEST_DIRECTORY) / "resources/download/zip_test.zip"
try:
os.remove(target_path)
except OSError:
pass
example_text = "some text"
requests_mock.get(
f"{API_EXPERIMENTAL}/textanalysis/projects/{pipeline.project.name}/pipelines/{pipeline.name}/resources",
headers={"Content-Type": "application/zip"},
text=example_text,
)
pipeline.download_resources(target_path)
assert os.path.exists(target_path)
assert example_text == target_path.read_text()
os.remove(target_path)
def test_list_resources(client, requests_mock):
pipeline = Pipeline(Project(client, PROJECT_NAME), "discharge")
expected_resources_list = [
"test1.txt",
"test2.txt",
"test3.txt",
]
requests_mock.get(
f"{API_EXPERIMENTAL}/textanalysis/projects/{pipeline.project.name}/pipelines/{pipeline.name}/resources",
headers={"Content-Type": "application/json"},
json={
"payload": {
"files": expected_resources_list
},
"errorMessages": []
},
)
actual_resources_list = pipeline.list_resources()
assert actual_resources_list == expected_resources_list
def test_delete_pipeline_v5(client_version_5):
pipeline = Pipeline(Project(client_version_5, PROJECT_NAME), "discharge")
with pytest.raises(OperationNotSupported):
pipeline.delete()
def test_export_text_analysis_export_v6(client_version_6, requests_mock):
project = Project(client_version_6, PROJECT_NAME)
collection = project.get_document_collection("my-collection")
process_name = "my-process"
requests_mock.get(
f"{API_EXPERIMENTAL}/textanalysis/projects/{project.name}/"
f"documentSources/{collection.name}/processes/{process_name}",
headers={"Content-Type": "application/json"},
json={
"payload": {
"processName": process_name,
"pipelineName": "my-pipeline",
"documentSourceName": collection.name,
"state": "IDLE",
"numberOfTotalDocuments": 6871,
"numberOfSuccessfulDocuments": 6871,
"numberOfUnsuccessfulDocuments": 0,
"errorMessages": [],
"precedingProcessName": "precedingProcessName",
},
"errorMessages": [],
},
)
process = collection.get_process(process_name)
requests_mock.get(
f"{API_BASE}/textanalysis/projects/{project.name}/"
f"documentSources/{process.document_source_name}/processes/{process.name}/export",
headers={"Content-Type": "application/json"},
json={
"payload": {
"projectName":
project.name,
"documentSourceName":
collection.name,
"textAnalysisResultSetName":
process.name,
"pipelineName":
"discharge",
"textAnalysisResultDtos": [{
"documentName":
"abcdef.txt",
"annotationDtos": [{
"begin": 0,
"end": 12,
"type": "uima.tcas.DocumentAnnotation",
"coveredText": "Hello World",
"id": 66753,
}]
# truncated #
}
# truncated #
],
},
"errorMessages": [],
},
)
export = process.export_text_analysis()
assert export["documentSourceName"] == collection.name