async def test_end_to_end_send_only_dataframe_data(): response = mock.Mock() response.status_code = 200 post_mock = mock.AsyncMock(return_value=response) with mock.patch( "hetdesrun.adapters.generic_rest.send_framelike.get_generic_rest_adapter_base_url", return_value="https://hetida.de", ): with mock.patch( "hetdesrun.adapters.generic_rest.send_dataframe.AsyncClient.post", new=post_mock, ): # one frame await send_data( { "inp_1": FilteredSink( ref_id="sink_id_1", type="dataframe", filters={} ) }, {"inp_1": pd.DataFrame({"a": [1.2, 3.4, 5.9], "b": [2.9, 8.7, 2.2]})}, adapter_key="test_end_to_end_send_only_dataframe_data_adapter_key", ) assert post_mock.called # we got through to actually posting! func_name, args, kwargs = post_mock.mock_calls[0] assert kwargs["params"] == [("id", "sink_id_1")] assert kwargs["json"] == [ {"a": 1.2, "b": 2.9}, {"a": 3.4, "b": 8.7}, {"a": 5.9, "b": 2.2}, ] # more than one frames await send_data( { "inp_1": FilteredSink( ref_id="sink_id_1", type="dataframe", filters={} ), "inp_2": FilteredSink( ref_id="sink_id_2", type=ExternalType.DATAFRAME, filters={}, ), }, { "inp_1": pd.DataFrame({"a": [1.2, 3.4, 5.9], "b": [2.9, 8.7, 2.2]}), "inp_2": pd.DataFrame({"c": [1.9, np.nan]}), }, adapter_key="test_end_to_end_send_only_dataframe_data_adapter_key", ) # note: can be async! func_name_1, args_1, kwargs_1 = post_mock.mock_calls[1] func_name_2, args_2, kwargs_2 = post_mock.mock_calls[2] assert (len(kwargs_1["json"]) == 3) or (len(kwargs_2["json"]) == 3) assert (len(kwargs_1["json"]) == 2) or (len(kwargs_2["json"]) == 2)
async def resolve_and_send_data_from_wiring( workflow_wiring: WorkflowWiring, result_data: Dict[str, Any]) -> Dict[str, Any]: """Sends data to sinks Data that is not send to a sink by the workflow wiring is returned. """ wirings_by_adapter = defaultdict(list) for output_wiring in workflow_wiring.output_wirings: wirings_by_adapter[output_wiring.adapter_id].append(output_wiring) all_data_not_send_by_adapter = {} # data is loaded adapter-wise: for adapter_key, output_wirings_of_adapter in wirings_by_adapter.items(): # call adapter with these wirings / sources data_not_send_by_adapter: Optional[Dict[ str, Any]] = await send_data_with_adapter( adapter_key, { output_wiring.workflow_output_name: FilteredSink( sink_id=output_wiring.sink_id, filters={}) for output_wiring in output_wirings_of_adapter }, result_data, ) if (data_not_send_by_adapter is not None): # and adapter_key in [1, "direct_provisioning"]? all_data_not_send_by_adapter.update(data_not_send_by_adapter) return all_data_not_send_by_adapter
async def test_end_to_end_send_only_timeseries_data(): response = mock.Mock() response.status_code = 200 post_mock = mock.AsyncMock(return_value=response) with mock.patch( "hetdesrun.adapters.generic_rest.send_framelike.get_generic_rest_adapter_base_url", return_value="https://hetida.de", ): with mock.patch( "hetdesrun.adapters.generic_rest.send_ts_data.AsyncClient.post", new=post_mock, ): ts_1 = pd.Series( [1.2, 3.4, 5.9], index=pd.to_datetime([ "2020-01-15T00:00:00.000Z", "2020-01-15T01:00:00.000Z", "2020-01-15T02:00:00.000Z", ]), ) # one timeseries await send_data( { "inp_1": FilteredSink(ref_id="sink_id_1", type="timeseries(float)", filters={}) }, {"inp_1": ts_1}, adapter_key= "test_end_to_end_send_only_timeseries_data_adapter_key", ) assert post_mock.called # we got through to actually posting! func_name, args, kwargs = post_mock.mock_calls[0] assert kwargs["params"] == [("timeseriesId", "sink_id_1")] assert kwargs["json"] == [ { "timestamp": "2020-01-15T00:00:00.000000000Z", "value": 1.2 }, { "timestamp": "2020-01-15T01:00:00.000000000Z", "value": 3.4 }, { "timestamp": "2020-01-15T02:00:00.000000000Z", "value": 5.9 }, ] # more than one timeseries ts_2 = pd.Series( ["first", "second"], index=pd.to_datetime([ "2020-01-15T00:00:00.000Z", "2020-01-15T01:00:00.000Z", ]), ) await send_data( { "inp_1": FilteredSink(ref_id="sink_id_1", type="timeseries(float)", filters={}), "inp_2": FilteredSink( ref_id="sink_id_2", type=ExternalType.TIMESERIES_STR, filters={}, ), }, { "inp_1": ts_1, "inp_2": ts_2, }, adapter_key= "test_end_to_end_send_only_timeseries_data_adapter_key", ) # note: can be async! func_name_1, args_1, kwargs_1 = post_mock.mock_calls[1] func_name_2, args_2, kwargs_2 = post_mock.mock_calls[2] assert (len(kwargs_1["json"]) == 3) or (len(kwargs_2["json"]) == 3) assert (len(kwargs_1["json"]) == 2) or (len(kwargs_2["json"]) == 2) # a timeseries with attributes ts = pd.Series( [1.2, 3.4, np.nan], index=pd.to_datetime([ "2020-01-15T00:00:00.000Z", "2020-01-15T01:00:00.000Z", "2020-01-15T02:00:00.000Z", ]), ) ts_1_attrs = {"a": 1} ts_1.attrs = ts_1_attrs await send_data( { "inp_1": FilteredSink(ref_id="sink_id_1", type="timeseries(float)", filters={}), }, {"inp_1": ts_1}, adapter_key= "test_end_to_end_send_only_timeseries_data_adapter_key", ) # note: can be async! func_name_3, args_3, kwargs_3 = post_mock.mock_calls[3] assert "Data-Attributes" in kwargs_3["headers"] received_attrs = decode_attributes( kwargs_3["headers"]["Data-Attributes"]) for key, value in ts_1_attrs.items(): key in received_attrs assert received_attrs[key] == value
async def test_end_to_end_send_only_metadata_data(): response = mock.Mock() response.status_code = 200 post_mock = mock.AsyncMock(return_value=response) with mock.patch( "hetdesrun.adapters.generic_rest.send_metadata.get_generic_rest_adapter_base_url", return_value="https://hetida.de", ): with mock.patch( "hetdesrun.adapters.generic_rest.send_metadata.httpx.AsyncClient.post", new=post_mock, ): # more than one await send_data( { "outp_1": FilteredSink( ref_id="th_node_id", type="metadata(string)", ref_id_type="THINGNODE", ref_key="description", filters={}, ), "outp_2": FilteredSink( ref_id="sink_id", type="metadata(float)", ref_id_type="SINK", ref_key="upper_lim", filters={}, ), }, { "outp_1": "some description", "outp_2": 47.8 }, adapter_key= "test_end_to_end_send_only_metadata_data_adapter_key", ) func_name_1, args_1, kwargs_1 = post_mock.mock_calls[0] func_name_2, args_2, kwargs_2 = post_mock.mock_calls[1] assert ({ "key": "description", "value": "some description", "dataType": "string", }) in [kwargs_1["json"], kwargs_2["json"]] assert ({ "key": "upper_lim", "value": 47.8, "dataType": "float", }) in [kwargs_1["json"], kwargs_2["json"]] assert "https://hetida.de/thingNodes/th_node_id/metadata/description" in [ args_1[0], args_2[0], ] assert "https://hetida.de/sinks/sink_id/metadata/upper_lim" in [ args_1[0], args_2[0], ]
async def test_end_to_end_send_only_single_metadata_data(): response = mock.Mock() response.status_code = 200 post_mock = mock.AsyncMock(return_value=response) with mock.patch( "hetdesrun.adapters.generic_rest.send_metadata.get_generic_rest_adapter_base_url", return_value="https://hetida.de", ): with mock.patch( "hetdesrun.adapters.generic_rest.send_metadata.httpx.AsyncClient.post", new=post_mock, ): # one frame await send_data( { "inp_1": FilteredSink( ref_id="sink_id_1", type="metadata(int)", ref_id_type="SOURCE", ref_key="number", filters={}, ) }, {"inp_1": 55}, adapter_key= "test_end_to_end_send_only_metadata_data_adapter_key", ) assert post_mock.called # we got through to actually posting! func_name, args, kwargs = post_mock.mock_calls[0] assert kwargs["json"] == { "key": "number", "value": 55, "dataType": "int" } assert args[ 0] == "https://hetida.de/sources/sink_id_1/metadata/number" response.status_code = 400 with pytest.raises(AdapterConnectionError): await send_data( { "inp_1": FilteredSink( ref_id="sink_id_1", type="metadata(int)", ref_id_type="SOURCE", ref_key="number", filters={}, ) }, {"inp_1": 55}, adapter_key= "test_end_to_end_send_only_metadata_data_adapter_key", )
async def test_resources_offered_from_structure_hierarchy(async_test_client): """Walks through the hierarchy provided by structure endpoint and gets/posts offered resources""" async with async_test_client as client: response_obj = (await client.get("/adapters/localfile/structure")).json() assert len(response_obj["sources"]) == 0 assert len(response_obj["sinks"]) == 0 roots = response_obj["thingNodes"] assert len(roots) == 1 root = roots[0] all_tns = [] all_srcs = [] all_snks = [] tn_attached_metadata_dict = {} src_attached_metadata_dict = {} snk_attached_metadata_dict = {} await walk_thing_nodes( root["id"], tn_append_list=all_tns, src_append_list=all_srcs, snk_append_list=all_snks, src_attached_metadata_dict=src_attached_metadata_dict, snk_attached_metadata_dict=snk_attached_metadata_dict, tn_attached_metadata_dict=tn_attached_metadata_dict, open_async_test_client=client, ) assert len(all_tns) == 4 assert len(all_srcs) == 7 assert len(all_snks) == 2 assert len(src_attached_metadata_dict) == 0 assert len(snk_attached_metadata_dict) == 0 assert len(tn_attached_metadata_dict) == 0 for src in all_srcs: response_obj = ( await client.get(f'/adapters/localfile/sources/{src["id"]}') ).json() for key in src.keys(): assert response_obj[key] == src[key] for snk in all_snks: response_obj = ( await client.get(f'/adapters/localfile/sinks/{snk["id"]}') ).json() for key in snk.keys(): print(response_obj) assert response_obj[key] == snk[key] for tn in all_tns: response_obj = ( await client.get(f'/adapters/localfile/thingNodes/{tn["id"]}') ).json() for key in tn.keys(): print(response_obj) assert response_obj[key] == tn[key] # we actually get all metadata that is available as attached to something: for ((src_id, key), md) in src_attached_metadata_dict.items(): response_obj = ( await client.get(f"/adapters/localfile/sources/{src_id}/metadata/{key}") ).json() print(response_obj, "versus", md) assert response_obj["key"] == key assert response_obj["value"] == md["value"] assert response_obj["dataType"] == md["dataType"] if md.get("isSink", False): assert response_obj["isSink"] resp = await client.post( f"/adapters/localfile/sources/{src_id}/metadata/{key}", json=md ) assert resp.status_code == 200 for ((snk_id, key), md) in snk_attached_metadata_dict.items(): response_obj = ( await client.get(f"/adapters/localfile/sinks/{snk_id}/metadata/{key}") ).json() print(response_obj, "versus", md) assert response_obj["key"] == key assert response_obj["value"] == md["value"] assert response_obj["dataType"] == md["dataType"] if md.get("isSink", False): assert response_obj["isSink"] resp = await client.post( f"/adapters/localfile/sinks/{snk_id}/metadata/{key}", json=md ) assert resp.status_code == 200 for ((tn_id, key), md) in tn_attached_metadata_dict.items(): response_obj = ( await client.get( f"/adapters/localfile/thingNodes/{tn_id}/metadata/{key}" ) ).json() print(response_obj, "versus", md) assert response_obj["key"] == key assert response_obj["value"] == md["value"] assert response_obj["dataType"] == md["dataType"] if md.get("isSink", False): assert response_obj["isSink"] resp = await client.post( f"/adapters/localfile/thingNodes/{snk_id}/metadata/{key}", json=md ) assert resp.status_code == 200 # all metadata that is a source in the tree is also found for src in all_srcs: if src["type"].startswith("metadata"): response_obj = ( await client.get( f'/adapters/localfile/thingNodes/{src["thingNodeId"]}/metadata/{src["metadataKey"]}' ) ).json() print(response_obj, "versus", src) assert response_obj["key"] == src["metadataKey"] assert response_obj["dataType"] == ( ExternalType(src["type"]).value_datatype.value ) if src["type"].startswith("dataframe"): loaded_df = ( await load_data( { "wf_input": FilteredSource( ref_id=src["id"], ref_id_type="SOURCE", ref_key=None, type="dataframe", ), }, adapter_key="local-file-adapter", ) )["wf_input"] assert isinstance(loaded_df, pd.DataFrame) if src["type"].startswith("timeseries"): raise AssertionError( "No timeseries type expected in local file adapter sources" ) # metadata that is a sink in the tree is also always obtainable for snk in all_snks: if snk["type"].startswith("metadata"): response_obj = ( await client.get( f'/adapters/localfile/thingNodes/{snk["thingNodeId"]}/metadata/{snk["metadataKey"]}' ) ).json() print(response_obj, "versus", snk) assert response_obj["key"] == snk["metadataKey"] assert response_obj["dataType"] == ( ExternalType(snk["type"]).value_datatype.value ) resp = await client.post( f'/adapters/localfile/thingNodes/{snk["thingNodeId"]}/metadata/{snk["metadataKey"]}', json=response_obj, ) assert resp.status_code == 200 if snk["type"].startswith("dataframe"): with mock.patch( "hetdesrun.adapters.local_file.write_file.pd.DataFrame.to_csv" ) as to_csv_mock: with mock.patch( "hetdesrun.adapters.local_file.write_file.pd.DataFrame.to_excel" ) as to_excel_mock: await send_data( { "wf_output": FilteredSink( ref_id=snk["id"], ref_id_type="SINK", ref_key=None, type="dataframe", ), }, { "wf_output": pd.DataFrame( {"a": [1, 2, 3], "b": [12.2, 13.3, 14.4]} ) }, adapter_key="local-file-adapter", ) if snk["type"].startswith("timeseries"): raise AssertionError( "No timeseries type expected in local file adapter sinks" ) assert to_csv_mock.called_once func_name, args, kwargs = to_csv_mock.mock_calls[0] if snk["id"].endswith(".csv"): assert ( kwargs["sep"] == ";" ) # option from the settings file of the only test sink assert to_excel_mock.called_once
async def test_end_to_end_send_only_dataframe_data(): response = mock.Mock() response.status_code = 200 post_mock = mock.AsyncMock(return_value=response) with mock.patch( "hetdesrun.adapters.generic_rest.send_framelike.get_generic_rest_adapter_base_url", return_value="https://hetida.de", ): with mock.patch( "hetdesrun.adapters.generic_rest.send_dataframe.AsyncClient.post", new=post_mock, ): # one frame await send_data( { "inp_1": FilteredSink( ref_id="sink_id_1", type="dataframe", filters={}) }, { "inp_1": pd.DataFrame({ "a": [1.2, 3.4, 5.9], "b": [2.9, 8.7, 2.2] }) }, adapter_key= "test_end_to_end_send_only_dataframe_data_adapter_key", ) assert post_mock.called # we got through to actually posting! func_name, args, kwargs = post_mock.mock_calls[0] assert kwargs["params"] == [("id", "sink_id_1")] assert kwargs["json"] == [ { "a": 1.2, "b": 2.9 }, { "a": 3.4, "b": 8.7 }, { "a": 5.9, "b": 2.2 }, ] # more than one frame await send_data( { "inp_1": FilteredSink( ref_id="sink_id_1", type="dataframe", filters={}), "inp_2": FilteredSink( ref_id="sink_id_2", type=ExternalType.DATAFRAME, filters={}, ), }, { "inp_1": pd.DataFrame({ "a": [1.2, 3.4, 5.9], "b": [2.9, 8.7, 2.2] }), "inp_2": pd.DataFrame({"c": [1.9, np.nan]}), }, adapter_key= "test_end_to_end_send_only_dataframe_data_adapter_key", ) # note: can be async! func_name_1, args_1, kwargs_1 = post_mock.mock_calls[1] func_name_2, args_2, kwargs_2 = post_mock.mock_calls[2] assert (len(kwargs_1["json"]) == 3) or (len(kwargs_2["json"]) == 3) assert (len(kwargs_1["json"]) == 2) or (len(kwargs_2["json"]) == 2) # one dataframe frame with timestamps and attributes df = pd.DataFrame({ "a": [1.2, 3.4, 5.9], "b": [2.9, 8.7, 2.2], "timestamp": [ pd.Timestamp("2020-08-03 15:30:00+0000", tz="UTC"), pd.Timestamp("2020-12-01 07:15:00+0000", tz="UTC"), pd.Timestamp("2021-01-05 09:20:00+0000", tz="UTC"), ], }) df_attrs = {"c": "test"} df.attrs = df_attrs await send_data( { "inp_1": FilteredSink( ref_id="sink_id_1", type="dataframe", filters={}) }, {"inp_1": df}, adapter_key= "test_end_to_end_send_only_dataframe_data_adapter_key", ) assert post_mock.called # we got through to actually posting! func_name, args, kwargs = post_mock.mock_calls[3] assert kwargs["params"] == [("id", "sink_id_1")] assert kwargs["json"] == [ { "a": 1.2, "b": 2.9, "timestamp": "2020-08-03T15:30:00+00:00" }, { "a": 3.4, "b": 8.7, "timestamp": "2020-12-01T07:15:00+00:00" }, { "a": 5.9, "b": 2.2, "timestamp": "2021-01-05T09:20:00+00:00" }, ] assert "Data-Attributes" in kwargs["headers"]