def test_source_api_call(spark_session: SparkSession) -> None:
test_path: Path = Path(__file__).parent.joinpath("./")
test_name = "test_source_api_call"
mock_server_url = "http://mock-server:1080"
mock_client = MockServerFriendlyClient(mock_server_url)
mock_client.clear(test_name)
input_file = input_types.FileInput()
request = input_types.ApiJsonResponse(
response_data_folder="api_json_response")
logger = get_logger(__name__)
SparkPipelineFrameworkTestRunnerV2(
spark_session=spark_session,
test_path=test_path,
test_name=test_name,
test_validators=None,
logger=logger,
auto_find_helix_transformer=False,
helix_transformers=[FeaturesComplexFeature],
mock_client=mock_client,
test_inputs=[input_file, request],
temp_folder="output/temp",
).run_test2()
with open(test_path.joinpath(
"api_json_response/getProviderApptTypes.json")) as f:
content = json.load(f)
response: Response = requests.get(
f"{mock_server_url}/{test_name}/getProviderApptTypes")
assert response.json() == content
def load_mock_source_api_responses_from_folder(
folder: Path, mock_client: MockServerFriendlyClient, url_prefix: Optional[str]
) -> List[str]:
"""
Mock responses for all files from the folder and its sub-folders
from https://pypi.org/project/mockserver-friendly-client/
:param folder: where to look for files (recursively)
:param mock_client:
:param url_prefix:
"""
file_path: str
files: List[str] = sorted(
glob.glob(str(folder.joinpath("**/*")), recursive=True)
)
for file_path in files:
with open(file_path, "r") as file:
content = json.load(file)
path = f"{('/' + url_prefix) if url_prefix else ''}/{os.path.splitext(os.path.basename(file_path))[0]}"
mock_client.expect(
request(
method="GET",
path=path,
),
response(body=json.dumps(content)),
timing=times(1),
)
print(f"Mocking: GET {mock_client.base_url}{path}")
return files
def load_mock_fhir_everything_requests_from_folder(
folder: Path,
mock_client: MockServerFriendlyClient,
resourceType: str,
url_prefix: Optional[str] = None,
) -> List[str]:
"""
Loads all .json files from the folder and its sub-folders
from https://pypi.org/project/mockserver-friendly-client/
:param folder: where to look for .json files (recursively)
:param mock_client:
:param resourceType:
:param url_prefix:
"""
file_name: str
files: List[str] = glob(str(folder.joinpath("**/*.json")), recursive=True)
for file_name in files:
# load file as json
with open(file_name, "r") as file:
fhir_request: Dict[str, Any] = json.loads(file.read())
# find id and resourceType
id_: str = fhir_request["id"]
path = f"{('/' + url_prefix) if url_prefix else ''}/4_0_0/{resourceType}/{id_}/$everything"
mock_client.expect(
request(
method="GET",
path=path,
),
response(body=json.dumps(fhir_request)),
timing=times(1),
)
print(f"Mocking: GET {mock_client.base_url}{path}")
return files
def load_mock_http_json_request(
folder: Path,
mock_client: MockServerFriendlyClient,
url_prefix: Optional[str],
add_file_name: bool = False,
) -> List[str]:
"""
Mock responses for all files from the folder and its sub-folders
:param folder: where to look for files (recursively)
:param mock_client:
:param url_prefix: http://{mock_server_url}/{url_prefix}...
:param add_file_name: http://{mock_server_url}/{url_prefix}/{add_file_name}...
"""
file_path: str
files: List[str] = sorted(
glob.glob(str(folder.joinpath("**/*.json")), recursive=True)
)
for file_path in files:
file_name = os.path.basename(file_path)
with open(file_path, "r") as file:
content = json.loads(file.read())
try:
request_parameters = content["request_parameters"]
except ValueError:
raise Exception(
"`request_parameters` key not found! It is supposed to contain parameters of the request function."
)
path = f"{('/' + url_prefix) if url_prefix else ''}"
path = (
f"{path}/{os.path.splitext(file_name)[0]}"
if add_file_name
else path
)
try:
request_result = content["request_result"]
except ValueError:
raise Exception(
"`request_result` key not found. It is supposed to contain the expected result of the requst function."
)
mock_client.expect(
request(path=path, **request_parameters),
response(body=json.dumps(request_result)),
timing=times(1),
)
print(f"Mocking {mock_client.base_url}{path}: {request_parameters}")
return files
def load_mock_fhir_everything_batch_requests_from_folder(
folder: Path,
mock_client: MockServerFriendlyClient,
resourceType: str,
ids: List[str],
url_prefix: Optional[str] = None,
) -> List[str]:
"""
Loads all .json files from the folder and its sub-folders
from https://pypi.org/project/mockserver-friendly-client/
:param folder: where to look for .json files (recursively)
:param mock_client:
:param resourceType:
:param url_prefix:
:param ids: id of resources for this batch to load
"""
file_name: str
files: List[str] = glob(str(folder.joinpath("**/*.json")), recursive=True)
result_bundle = {
"resourceType": "Bundle",
"id": "bundle-example",
"type": "collection",
"entry": [],
}
print(f"mock fhir batch request for {ids}")
for file_name in files:
with open(file_name, "r") as file:
fhir_bundle: Dict[str, Any] = json.loads(file.read())
if "entry" not in fhir_bundle:
print(f"{file_name} has no entry property!")
continue
for entry in fhir_bundle["entry"]:
id = entry.get("resource", {}).get("id", "")
if id in ids:
result_bundle["entry"].append(entry) # type: ignore
# find id and resourceType
path = (
f"{('/' + url_prefix) if url_prefix else ''}/4_0_0/{resourceType}/$everything"
)
mock_client.expect(
request(method="GET", path=path, querystring={"id": ",".join(ids)}),
response(body=json.dumps(result_bundle)),
timing=times(1),
)
print(f"Mocking: GET {mock_client.base_url}{path}")
return files
def test_mock_fhir_graph_request(spark_session: SparkSession) -> None:
"""
expect 2 $graph calls to be made to get Organization data
"""
test_path: Path = Path(__file__).parent.joinpath("./")
test_name = "test_mock_fhir_graph_request"
mock_server_url = "http://mock-server:1080"
mock_client = MockServerFriendlyClient(mock_server_url)
mock_client.clear(f"/{test_name}/")
fhir_calls = MockRequestResponseCalls(
fhir_calls_folder=f"{test_path}/mock_request_responses/Organization",
mock_url_prefix=f"{test_name}/4_0_0/Organization",
url_suffix="$graph",
)
test_validators = MockCallValidator(related_inputs=fhir_calls)
params = {
"test_name":
test_name,
"mock_server_url":
mock_server_url,
"url_fhir_segments":
"Organization/$graph",
"files_path": [
test_path.joinpath(
"mock_request_responses/Organization/1023011178.json"),
test_path.joinpath(
"mock_request_responses/Organization/1841293990.json"),
],
}
logger = get_logger(__name__)
SparkPipelineFrameworkTestRunnerV2(
spark_session=spark_session,
test_path=test_path,
test_name=test_name,
logger=logger,
auto_find_helix_transformer=False,
helix_transformers=[PipelineMockRequestResponsesV1],
mock_client=mock_client,
test_validators=[test_validators],
test_inputs=[fhir_calls],
temp_folder="temp",
helix_pipeline_parameters=params,
).run_test2()
def test_fhir_mock(spark_session: SparkSession) -> None:
test_path: Path = Path(__file__).parent.joinpath("./")
test_name = "test_fhir_mock"
mock_server_url = "http://mock-server:1080"
mock_client = MockServerFriendlyClient(mock_server_url)
mock_client.clear(f"/{test_name}/")
fhir_calls = FhirCalls()
test_validators = MockCallValidator(related_inputs=fhir_calls)
params = {
"test_name":
test_name,
"mock_server_url":
mock_server_url,
"files_path": [
test_path.joinpath(
"fhir_calls/healthcare_service/1629334859-TT3-GPPC.json"),
test_path.joinpath(
"fhir_calls/healthcare_service/1790914448-TT4-GPPC.json"),
test_path.joinpath(
"fhir_calls/location/Medstar-Alias-TT3-GPPC.json"),
test_path.joinpath(
"fhir_calls/location/Medstar-Alias-TT4-GPPC.json"),
],
}
logger = get_logger(__name__)
SparkPipelineFrameworkTestRunnerV2(
spark_session=spark_session,
test_path=test_path,
test_name=test_name,
test_validators=[test_validators],
logger=logger,
auto_find_helix_transformer=False,
helix_transformers=[PipelineFhirCallsFhirMockV1],
mock_client=mock_client,
test_inputs=[fhir_calls],
temp_folder="temp",
helix_pipeline_parameters=params,
).run_test2()
def mock_single_request(
fhir_request: Dict[str, Any],
method: str,
mock_client: MockServerFriendlyClient,
relative_path: Optional[str],
query_string: Optional[Dict[str, str]],
url_prefix: Optional[str],
response_body: Optional[str],
) -> None:
# find id and resourceType
if method == "POST":
# id_ = fhir_request["id"]
# noinspection PyPep8Naming
resourceType = fhir_request["resourceType"]
id_ = fhir_request["id"]
path = (
f"{('/' + url_prefix) if url_prefix else ''}/4_0_0/{resourceType}/1/$merge"
)
payload: str = json.dumps([{
"id": id_,
"updated": False,
"created": True
}])
mock_client.expect(
request(
method="POST",
path=path,
body=json_equals([fhir_request]),
),
response(body=payload),
timing=times_any(),
)
print(
f"Mocking: POST {mock_client.base_url}{path}: {json.dumps(fhir_request)}"
)
else:
if not relative_path:
id_ = fhir_request["id"]
# noinspection PyPep8Naming
resourceType = fhir_request["resourceType"]
path = (
f"{('/' + url_prefix) if url_prefix else ''}/4_0_0/{resourceType}/{id_}"
)
mock_client.expect(
request(method="GET", path=path, querystring=query_string),
response(body=json.dumps(fhir_request)),
timing=times(1),
)
else:
path = f"{('/' + url_prefix) if url_prefix else ''}/4_0_0/{relative_path}"
mock_client.expect(
request(method="GET", path=path, querystring=query_string),
response(body=json.dumps(fhir_request)),
timing=times(1),
)
print(f"Mocking: GET {mock_client.base_url}{path}{query_string or ''}")
def test_practitioner_fail_on_fhir_validation(spark_session: SparkSession) -> None:
test_path: Path = Path(__file__).parent.joinpath("./")
# setup servers
test_name = "practitioner_fail_on_fhir_validation"
test_input = FileInput()
test_fhir = FhirCalls()
logger = get_logger(__name__)
mock_server_url = "http://mock-server:1080"
mock_client = MockServerFriendlyClient(mock_server_url)
mock_client.clear(f"/{test_name}/")
mock_client.expect_default()
params = {
"test_name": test_name,
"mock_server_url": mock_server_url,
}
test_validator = FhirValidator(
related_inputs=test_fhir,
related_file_inputs=test_input,
mock_server_url=mock_server_url,
test_name=test_name,
fhir_validation_url="http://fhir:3000/4_0_0",
)
with pytest.raises(AssertionError, match=r"Failed validation for resource*"):
SparkPipelineFrameworkTestRunnerV2(
spark_session=spark_session,
test_path=test_path,
test_name=test_name,
test_validators=[test_validator],
logger=logger,
auto_find_helix_transformer=False,
helix_transformers=[
FeaturesDoctorFeaturePractitionerFailOnFhirValidationV1
],
test_inputs=[test_input],
temp_folder="output/temp",
helix_pipeline_parameters=params,
).run_test2()
def test_doctor_feature_practitioner(spark_session: SparkSession) -> None:
data_dir: Path = Path(__file__).parent.joinpath("./")
test_name = "test_doctor_feature_practitioner"
test_input = input_types.FileInput()
test_fhir = input_types.FhirCalls()
logger = get_logger(__name__)
mock_server_url = "http://mock-server:1080"
mock_client = MockServerFriendlyClient(mock_server_url)
mock_client.clear(f"/{test_name}/")
mock_client.expect_default()
params = {
"test_name": test_name,
"mock_server_url": mock_server_url,
}
test_validator = FhirValidator(
related_inputs=test_fhir,
related_file_inputs=test_input,
mock_server_url=mock_server_url,
test_name=test_name,
fhir_validation_url="http://fhir:3000/4_0_0",
)
SparkPipelineFrameworkTestRunnerV2(
spark_session=spark_session,
test_path=data_dir,
test_name=test_name,
test_validators=[
test_validator,
# OutputFileValidator(related_inputs=test_input, sort_output_by=["id"]),
],
logger=logger,
test_inputs=[test_input],
temp_folder="output/temp",
mock_client=mock_client,
helix_pipeline_parameters=params,
).run_test2()
def load_mock_elasticsearch_requests_from_folder(
folder: Path, mock_client: MockServerFriendlyClient,
index: str) -> List[str]:
"""
Loads all .json files from the folder and its sub-folders
from https://pypi.org/project/mockserver-friendly-client/
:param folder: where to look for .json files (recursively)
:param mock_client:
:param index:
"""
file_name: str
files: List[str] = glob(str(folder.joinpath("**/*.json")), recursive=True)
for file_name in files:
# load file as json
with open(file_name, "r") as file:
lines: List[str] = file.readlines()
http_request: str = "\n".join([
(json.dumps(json.loads(line))) if line != "\n" else ""
for line in lines
])
# noinspection PyPep8Naming
path = f"/{index}/_bulk"
# noinspection SpellCheckingInspection
mock_client.expect(
request(
method="POST",
path=path,
body=text_equals(http_request),
),
response(
headers={"Content-Type": "application/json"},
body=f"""
{{
"took": 194,
"errors": false,
"items": [
{{
"index": {{
"_index": "{index}",
"_type": "_doc",
"_id": "TESQ93YBW4SQ_M9deEJw",
"_version": 1,
"result": "created"
}}
}},
{{
"index": {{
"_index": "{index}",
"_type": "_doc",
"_id": "TUSQ93YBW4SQ_M9deEJw",
"_version": 1,
"result": "created"
}}
}}
]
}}""",
),
timing=times(1),
)
print(f"Mocking: POST {mock_client.base_url}{path}")
return files