async def test_end_to_end_send_only_dataframe_data():
response = mock.Mock()
response.status_code = 200
post_mock = mock.AsyncMock(return_value=response)
with mock.patch(
"hetdesrun.adapters.generic_rest.send_framelike.get_generic_rest_adapter_base_url",
return_value="https://hetida.de",
):
with mock.patch(
"hetdesrun.adapters.generic_rest.send_dataframe.AsyncClient.post",
new=post_mock,
):
# one frame
await send_data(
{
"inp_1": FilteredSink(
ref_id="sink_id_1", type="dataframe", filters={}
)
},
{"inp_1": pd.DataFrame({"a": [1.2, 3.4, 5.9], "b": [2.9, 8.7, 2.2]})},
adapter_key="test_end_to_end_send_only_dataframe_data_adapter_key",
)
assert post_mock.called # we got through to actually posting!
func_name, args, kwargs = post_mock.mock_calls[0]
assert kwargs["params"] == [("id", "sink_id_1")]
assert kwargs["json"] == [
{"a": 1.2, "b": 2.9},
{"a": 3.4, "b": 8.7},
{"a": 5.9, "b": 2.2},
]
# more than one frames
await send_data(
{
"inp_1": FilteredSink(
ref_id="sink_id_1", type="dataframe", filters={}
),
"inp_2": FilteredSink(
ref_id="sink_id_2",
type=ExternalType.DATAFRAME,
filters={},
),
},
{
"inp_1": pd.DataFrame({"a": [1.2, 3.4, 5.9], "b": [2.9, 8.7, 2.2]}),
"inp_2": pd.DataFrame({"c": [1.9, np.nan]}),
},
adapter_key="test_end_to_end_send_only_dataframe_data_adapter_key",
)
# note: can be async!
func_name_1, args_1, kwargs_1 = post_mock.mock_calls[1]
func_name_2, args_2, kwargs_2 = post_mock.mock_calls[2]
assert (len(kwargs_1["json"]) == 3) or (len(kwargs_2["json"]) == 3)
assert (len(kwargs_1["json"]) == 2) or (len(kwargs_2["json"]) == 2)
async def resolve_and_send_data_from_wiring(
workflow_wiring: WorkflowWiring,
result_data: Dict[str, Any]) -> Dict[str, Any]:
"""Sends data to sinks
Data that is not send to a sink by the workflow wiring is returned.
"""
wirings_by_adapter = defaultdict(list)
for output_wiring in workflow_wiring.output_wirings:
wirings_by_adapter[output_wiring.adapter_id].append(output_wiring)
all_data_not_send_by_adapter = {}
# data is loaded adapter-wise:
for adapter_key, output_wirings_of_adapter in wirings_by_adapter.items():
# call adapter with these wirings / sources
data_not_send_by_adapter: Optional[Dict[
str, Any]] = await send_data_with_adapter(
adapter_key,
{
output_wiring.workflow_output_name: FilteredSink(
sink_id=output_wiring.sink_id, filters={})
for output_wiring in output_wirings_of_adapter
},
result_data,
)
if (data_not_send_by_adapter
is not None): # and adapter_key in [1, "direct_provisioning"]?
all_data_not_send_by_adapter.update(data_not_send_by_adapter)
return all_data_not_send_by_adapter
async def test_end_to_end_send_only_timeseries_data():
response = mock.Mock()
response.status_code = 200
post_mock = mock.AsyncMock(return_value=response)
with mock.patch(
"hetdesrun.adapters.generic_rest.send_framelike.get_generic_rest_adapter_base_url",
return_value="https://hetida.de",
):
with mock.patch(
"hetdesrun.adapters.generic_rest.send_ts_data.AsyncClient.post",
new=post_mock,
):
ts_1 = pd.Series(
[1.2, 3.4, 5.9],
index=pd.to_datetime([
"2020-01-15T00:00:00.000Z",
"2020-01-15T01:00:00.000Z",
"2020-01-15T02:00:00.000Z",
]),
)
# one timeseries
await send_data(
{
"inp_1":
FilteredSink(ref_id="sink_id_1",
type="timeseries(float)",
filters={})
},
{"inp_1": ts_1},
adapter_key=
"test_end_to_end_send_only_timeseries_data_adapter_key",
)
assert post_mock.called # we got through to actually posting!
func_name, args, kwargs = post_mock.mock_calls[0]
assert kwargs["params"] == [("timeseriesId", "sink_id_1")]
assert kwargs["json"] == [
{
"timestamp": "2020-01-15T00:00:00.000000000Z",
"value": 1.2
},
{
"timestamp": "2020-01-15T01:00:00.000000000Z",
"value": 3.4
},
{
"timestamp": "2020-01-15T02:00:00.000000000Z",
"value": 5.9
},
]
# more than one timeseries
ts_2 = pd.Series(
["first", "second"],
index=pd.to_datetime([
"2020-01-15T00:00:00.000Z",
"2020-01-15T01:00:00.000Z",
]),
)
await send_data(
{
"inp_1":
FilteredSink(ref_id="sink_id_1",
type="timeseries(float)",
filters={}),
"inp_2":
FilteredSink(
ref_id="sink_id_2",
type=ExternalType.TIMESERIES_STR,
filters={},
),
},
{
"inp_1": ts_1,
"inp_2": ts_2,
},
adapter_key=
"test_end_to_end_send_only_timeseries_data_adapter_key",
)
# note: can be async!
func_name_1, args_1, kwargs_1 = post_mock.mock_calls[1]
func_name_2, args_2, kwargs_2 = post_mock.mock_calls[2]
assert (len(kwargs_1["json"]) == 3) or (len(kwargs_2["json"]) == 3)
assert (len(kwargs_1["json"]) == 2) or (len(kwargs_2["json"]) == 2)
# a timeseries with attributes
ts = pd.Series(
[1.2, 3.4, np.nan],
index=pd.to_datetime([
"2020-01-15T00:00:00.000Z",
"2020-01-15T01:00:00.000Z",
"2020-01-15T02:00:00.000Z",
]),
)
ts_1_attrs = {"a": 1}
ts_1.attrs = ts_1_attrs
await send_data(
{
"inp_1":
FilteredSink(ref_id="sink_id_1",
type="timeseries(float)",
filters={}),
},
{"inp_1": ts_1},
adapter_key=
"test_end_to_end_send_only_timeseries_data_adapter_key",
)
# note: can be async!
func_name_3, args_3, kwargs_3 = post_mock.mock_calls[3]
assert "Data-Attributes" in kwargs_3["headers"]
received_attrs = decode_attributes(
kwargs_3["headers"]["Data-Attributes"])
for key, value in ts_1_attrs.items():
key in received_attrs
assert received_attrs[key] == value
async def test_end_to_end_send_only_metadata_data():
response = mock.Mock()
response.status_code = 200
post_mock = mock.AsyncMock(return_value=response)
with mock.patch(
"hetdesrun.adapters.generic_rest.send_metadata.get_generic_rest_adapter_base_url",
return_value="https://hetida.de",
):
with mock.patch(
"hetdesrun.adapters.generic_rest.send_metadata.httpx.AsyncClient.post",
new=post_mock,
):
# more than one
await send_data(
{
"outp_1":
FilteredSink(
ref_id="th_node_id",
type="metadata(string)",
ref_id_type="THINGNODE",
ref_key="description",
filters={},
),
"outp_2":
FilteredSink(
ref_id="sink_id",
type="metadata(float)",
ref_id_type="SINK",
ref_key="upper_lim",
filters={},
),
},
{
"outp_1": "some description",
"outp_2": 47.8
},
adapter_key=
"test_end_to_end_send_only_metadata_data_adapter_key",
)
func_name_1, args_1, kwargs_1 = post_mock.mock_calls[0]
func_name_2, args_2, kwargs_2 = post_mock.mock_calls[1]
assert ({
"key": "description",
"value": "some description",
"dataType": "string",
}) in [kwargs_1["json"], kwargs_2["json"]]
assert ({
"key": "upper_lim",
"value": 47.8,
"dataType": "float",
}) in [kwargs_1["json"], kwargs_2["json"]]
assert "https://hetida.de/thingNodes/th_node_id/metadata/description" in [
args_1[0],
args_2[0],
]
assert "https://hetida.de/sinks/sink_id/metadata/upper_lim" in [
args_1[0],
args_2[0],
]
async def test_end_to_end_send_only_single_metadata_data():
response = mock.Mock()
response.status_code = 200
post_mock = mock.AsyncMock(return_value=response)
with mock.patch(
"hetdesrun.adapters.generic_rest.send_metadata.get_generic_rest_adapter_base_url",
return_value="https://hetida.de",
):
with mock.patch(
"hetdesrun.adapters.generic_rest.send_metadata.httpx.AsyncClient.post",
new=post_mock,
):
# one frame
await send_data(
{
"inp_1":
FilteredSink(
ref_id="sink_id_1",
type="metadata(int)",
ref_id_type="SOURCE",
ref_key="number",
filters={},
)
},
{"inp_1": 55},
adapter_key=
"test_end_to_end_send_only_metadata_data_adapter_key",
)
assert post_mock.called # we got through to actually posting!
func_name, args, kwargs = post_mock.mock_calls[0]
assert kwargs["json"] == {
"key": "number",
"value": 55,
"dataType": "int"
}
assert args[
0] == "https://hetida.de/sources/sink_id_1/metadata/number"
response.status_code = 400
with pytest.raises(AdapterConnectionError):
await send_data(
{
"inp_1":
FilteredSink(
ref_id="sink_id_1",
type="metadata(int)",
ref_id_type="SOURCE",
ref_key="number",
filters={},
)
},
{"inp_1": 55},
adapter_key=
"test_end_to_end_send_only_metadata_data_adapter_key",
)
async def test_resources_offered_from_structure_hierarchy(async_test_client):
"""Walks through the hierarchy provided by structure endpoint and gets/posts offered resources"""
async with async_test_client as client:
response_obj = (await client.get("/adapters/localfile/structure")).json()
assert len(response_obj["sources"]) == 0
assert len(response_obj["sinks"]) == 0
roots = response_obj["thingNodes"]
assert len(roots) == 1
root = roots[0]
all_tns = []
all_srcs = []
all_snks = []
tn_attached_metadata_dict = {}
src_attached_metadata_dict = {}
snk_attached_metadata_dict = {}
await walk_thing_nodes(
root["id"],
tn_append_list=all_tns,
src_append_list=all_srcs,
snk_append_list=all_snks,
src_attached_metadata_dict=src_attached_metadata_dict,
snk_attached_metadata_dict=snk_attached_metadata_dict,
tn_attached_metadata_dict=tn_attached_metadata_dict,
open_async_test_client=client,
)
assert len(all_tns) == 4
assert len(all_srcs) == 7
assert len(all_snks) == 2
assert len(src_attached_metadata_dict) == 0
assert len(snk_attached_metadata_dict) == 0
assert len(tn_attached_metadata_dict) == 0
for src in all_srcs:
response_obj = (
await client.get(f'/adapters/localfile/sources/{src["id"]}')
).json()
for key in src.keys():
assert response_obj[key] == src[key]
for snk in all_snks:
response_obj = (
await client.get(f'/adapters/localfile/sinks/{snk["id"]}')
).json()
for key in snk.keys():
print(response_obj)
assert response_obj[key] == snk[key]
for tn in all_tns:
response_obj = (
await client.get(f'/adapters/localfile/thingNodes/{tn["id"]}')
).json()
for key in tn.keys():
print(response_obj)
assert response_obj[key] == tn[key]
# we actually get all metadata that is available as attached to something:
for ((src_id, key), md) in src_attached_metadata_dict.items():
response_obj = (
await client.get(f"/adapters/localfile/sources/{src_id}/metadata/{key}")
).json()
print(response_obj, "versus", md)
assert response_obj["key"] == key
assert response_obj["value"] == md["value"]
assert response_obj["dataType"] == md["dataType"]
if md.get("isSink", False):
assert response_obj["isSink"]
resp = await client.post(
f"/adapters/localfile/sources/{src_id}/metadata/{key}", json=md
)
assert resp.status_code == 200
for ((snk_id, key), md) in snk_attached_metadata_dict.items():
response_obj = (
await client.get(f"/adapters/localfile/sinks/{snk_id}/metadata/{key}")
).json()
print(response_obj, "versus", md)
assert response_obj["key"] == key
assert response_obj["value"] == md["value"]
assert response_obj["dataType"] == md["dataType"]
if md.get("isSink", False):
assert response_obj["isSink"]
resp = await client.post(
f"/adapters/localfile/sinks/{snk_id}/metadata/{key}", json=md
)
assert resp.status_code == 200
for ((tn_id, key), md) in tn_attached_metadata_dict.items():
response_obj = (
await client.get(
f"/adapters/localfile/thingNodes/{tn_id}/metadata/{key}"
)
).json()
print(response_obj, "versus", md)
assert response_obj["key"] == key
assert response_obj["value"] == md["value"]
assert response_obj["dataType"] == md["dataType"]
if md.get("isSink", False):
assert response_obj["isSink"]
resp = await client.post(
f"/adapters/localfile/thingNodes/{snk_id}/metadata/{key}", json=md
)
assert resp.status_code == 200
# all metadata that is a source in the tree is also found
for src in all_srcs:
if src["type"].startswith("metadata"):
response_obj = (
await client.get(
f'/adapters/localfile/thingNodes/{src["thingNodeId"]}/metadata/{src["metadataKey"]}'
)
).json()
print(response_obj, "versus", src)
assert response_obj["key"] == src["metadataKey"]
assert response_obj["dataType"] == (
ExternalType(src["type"]).value_datatype.value
)
if src["type"].startswith("dataframe"):
loaded_df = (
await load_data(
{
"wf_input": FilteredSource(
ref_id=src["id"],
ref_id_type="SOURCE",
ref_key=None,
type="dataframe",
),
},
adapter_key="local-file-adapter",
)
)["wf_input"]
assert isinstance(loaded_df, pd.DataFrame)
if src["type"].startswith("timeseries"):
raise AssertionError(
"No timeseries type expected in local file adapter sources"
)
# metadata that is a sink in the tree is also always obtainable
for snk in all_snks:
if snk["type"].startswith("metadata"):
response_obj = (
await client.get(
f'/adapters/localfile/thingNodes/{snk["thingNodeId"]}/metadata/{snk["metadataKey"]}'
)
).json()
print(response_obj, "versus", snk)
assert response_obj["key"] == snk["metadataKey"]
assert response_obj["dataType"] == (
ExternalType(snk["type"]).value_datatype.value
)
resp = await client.post(
f'/adapters/localfile/thingNodes/{snk["thingNodeId"]}/metadata/{snk["metadataKey"]}',
json=response_obj,
)
assert resp.status_code == 200
if snk["type"].startswith("dataframe"):
with mock.patch(
"hetdesrun.adapters.local_file.write_file.pd.DataFrame.to_csv"
) as to_csv_mock:
with mock.patch(
"hetdesrun.adapters.local_file.write_file.pd.DataFrame.to_excel"
) as to_excel_mock:
await send_data(
{
"wf_output": FilteredSink(
ref_id=snk["id"],
ref_id_type="SINK",
ref_key=None,
type="dataframe",
),
},
{
"wf_output": pd.DataFrame(
{"a": [1, 2, 3], "b": [12.2, 13.3, 14.4]}
)
},
adapter_key="local-file-adapter",
)
if snk["type"].startswith("timeseries"):
raise AssertionError(
"No timeseries type expected in local file adapter sinks"
)
assert to_csv_mock.called_once
func_name, args, kwargs = to_csv_mock.mock_calls[0]
if snk["id"].endswith(".csv"):
assert (
kwargs["sep"] == ";"
) # option from the settings file of the only test sink
assert to_excel_mock.called_once
async def test_end_to_end_send_only_dataframe_data():
response = mock.Mock()
response.status_code = 200
post_mock = mock.AsyncMock(return_value=response)
with mock.patch(
"hetdesrun.adapters.generic_rest.send_framelike.get_generic_rest_adapter_base_url",
return_value="https://hetida.de",
):
with mock.patch(
"hetdesrun.adapters.generic_rest.send_dataframe.AsyncClient.post",
new=post_mock,
):
# one frame
await send_data(
{
"inp_1":
FilteredSink(
ref_id="sink_id_1", type="dataframe", filters={})
},
{
"inp_1":
pd.DataFrame({
"a": [1.2, 3.4, 5.9],
"b": [2.9, 8.7, 2.2]
})
},
adapter_key=
"test_end_to_end_send_only_dataframe_data_adapter_key",
)
assert post_mock.called # we got through to actually posting!
func_name, args, kwargs = post_mock.mock_calls[0]
assert kwargs["params"] == [("id", "sink_id_1")]
assert kwargs["json"] == [
{
"a": 1.2,
"b": 2.9
},
{
"a": 3.4,
"b": 8.7
},
{
"a": 5.9,
"b": 2.2
},
]
# more than one frame
await send_data(
{
"inp_1":
FilteredSink(
ref_id="sink_id_1", type="dataframe", filters={}),
"inp_2":
FilteredSink(
ref_id="sink_id_2",
type=ExternalType.DATAFRAME,
filters={},
),
},
{
"inp_1":
pd.DataFrame({
"a": [1.2, 3.4, 5.9],
"b": [2.9, 8.7, 2.2]
}),
"inp_2":
pd.DataFrame({"c": [1.9, np.nan]}),
},
adapter_key=
"test_end_to_end_send_only_dataframe_data_adapter_key",
)
# note: can be async!
func_name_1, args_1, kwargs_1 = post_mock.mock_calls[1]
func_name_2, args_2, kwargs_2 = post_mock.mock_calls[2]
assert (len(kwargs_1["json"]) == 3) or (len(kwargs_2["json"]) == 3)
assert (len(kwargs_1["json"]) == 2) or (len(kwargs_2["json"]) == 2)
# one dataframe frame with timestamps and attributes
df = pd.DataFrame({
"a": [1.2, 3.4, 5.9],
"b": [2.9, 8.7, 2.2],
"timestamp": [
pd.Timestamp("2020-08-03 15:30:00+0000", tz="UTC"),
pd.Timestamp("2020-12-01 07:15:00+0000", tz="UTC"),
pd.Timestamp("2021-01-05 09:20:00+0000", tz="UTC"),
],
})
df_attrs = {"c": "test"}
df.attrs = df_attrs
await send_data(
{
"inp_1":
FilteredSink(
ref_id="sink_id_1", type="dataframe", filters={})
},
{"inp_1": df},
adapter_key=
"test_end_to_end_send_only_dataframe_data_adapter_key",
)
assert post_mock.called # we got through to actually posting!
func_name, args, kwargs = post_mock.mock_calls[3]
assert kwargs["params"] == [("id", "sink_id_1")]
assert kwargs["json"] == [
{
"a": 1.2,
"b": 2.9,
"timestamp": "2020-08-03T15:30:00+00:00"
},
{
"a": 3.4,
"b": 8.7,
"timestamp": "2020-12-01T07:15:00+00:00"
},
{
"a": 5.9,
"b": 2.2,
"timestamp": "2021-01-05T09:20:00+00:00"
},
]
assert "Data-Attributes" in kwargs["headers"]