async def test_updating_and_keeping_existing_attrs_for_dataframe( async_test_client: AsyncClient, ) -> None: async with async_test_client as client: df_attrs_1 = {"test": "Hello world!", "answer": 42} base64_str_1 = encode_attributes(df_attrs_1) response_1 = await client.post( "/dataframe?id=root.plantA.alerts", json=[{ "column1": 1, "column2": 1.3 }, { "column1": 2, "column2": 2.8 }], headers={"Data-Attributes": base64_str_1}, ) assert response_1.status_code == 200 df_from_store_1 = get_value_from_store("root.plantA.alerts") assert len(df_from_store_1.attrs) != 0 assert "test" in df_from_store_1.attrs for key, value in df_attrs_1.items(): assert df_from_store_1.attrs[key] == value df_attrs_2 = {"test": "test"} base64_str_2 = encode_attributes(df_attrs_2) response_2 = await client.post( "/dataframe?id=root.plantA.alerts", json=[{ "column1": 1, "column2": 1.3 }, { "column1": 2, "column2": 2.8 }], headers={"Data-Attributes": base64_str_2}, ) assert response_2.status_code == 200 df_from_store_2 = get_value_from_store("root.plantA.alerts") df_attrs = deepcopy(df_attrs_1) df_attrs.update(df_attrs_2) assert len(df_from_store_2.attrs) != 0 assert "test" in df_from_store_2.attrs for key, value in df_attrs.items(): assert df_from_store_2.attrs[key] == value
async def test_receiving_attrs_via_post_timeseries( async_test_client: AsyncClient, ) -> None: async with async_test_client as client: df_attrs = {"test": "Hello world!", "answer": 42} base64_str = encode_attributes(df_attrs) ts_id = "root.plantA.picklingUnit.influx.anomaly_score" response = await client.post( f"/timeseries?timeseriesId={ts_id}", json=[ { "timestamp": "2020-01-01T00:00:00.000000000Z", "value": 12.3 }, { "timestamp": "2020-01-02T00:00:00.000000000Z", "value": 11.9 }, ], headers={"Data-Attributes": base64_str}, ) assert response.status_code == 200 df_from_store = get_value_from_store(ts_id) assert len(df_from_store.attrs) != 0 assert "test" in df_from_store.attrs for key, value in df_attrs.items(): assert df_from_store.attrs[key] == value
async def test_receiving_attrs_via_post_dataframe( async_test_client: AsyncClient, ) -> None: async with async_test_client as client: df_attrs = {"test": "Hello world!", "answer": 42} base64_str = encode_attributes(df_attrs) response = await client.post( "/dataframe?id=root.plantA.alerts", json=[{ "column1": 1, "column2": 1.3 }, { "column1": 2, "column2": 2.8 }], headers={"Data-Attributes": base64_str}, ) assert response.status_code == 200 df_from_store = get_value_from_store("root.plantA.alerts") assert len(df_from_store.attrs) != 0 assert "test" in df_from_store.attrs for key, value in df_attrs.items(): assert df_from_store.attrs[key] == value
async def post_dataframe( df_body: List[ Dict] = Body( ..., example=[ { "column_A": 42.0, "column_B": "example" }, { "column_A": 11.97, "column_B": "example" }, ], ), df_id: str = Query(..., alias="id"), data_attributes: Optional[str] = Header(None), ) -> dict: if df_id.endswith("alerts"): df = pd.DataFrame.from_dict(df_body, orient="columns") if data_attributes is not None and len(data_attributes) != 0: df_from_store: pd.DataFrame = get_value_from_store(df_id) df.attrs = df_from_store.attrs df.attrs.update(decode_attributes(data_attributes)) logger.debug("storing %s", json.dumps(df_body)) logger.debug("which has attributes %s", str(df.attrs)) set_value_in_store(df_id, df) return {"message": "success"} raise HTTPException(404, f"No writable dataframe with id {df_id}")
async def timeseries( ids: List[str] = Query(..., alias="id", min_length=1), from_timestamp: datetime.datetime = Query(..., alias="from"), to_timestamp: datetime.datetime = Query(..., alias="to"), ) -> StreamingResponse: io_stream = StringIO() dt_range = pd.date_range(start=from_timestamp, end=to_timestamp, freq="1h", tz=datetime.timezone.utc) for ts_id in ids: ts_df = None if ts_id.endswith("temp"): offset = 100.0 if "plantA" in ts_id else 30.0 factor = 10.0 if "plantA" in ts_id else 5.0 elif ts_id.endswith("press"): offset = 14.5 if "plantA" in ts_id else 1.0 factor = 0.73 if "plantA" in ts_id else 0.05 elif ts_id.endswith("anomaly_score"): stored_df = get_value_from_store(ts_id).copy() if "timestamp" in stored_df.columns: stored_df.index = pd.to_datetime(stored_df["timestamp"]) stored_df = stored_df[from_timestamp.isoformat():to_timestamp. isoformat() # type: ignore ] ts_df = pd.DataFrame({ "timestamp": stored_df.index, "timeseriesId": ts_id, "value": stored_df["value"], }) else: ts_df = pd.DataFrame({ "timestamp": [], "timeseriesId": [], "value": [], }) else: offset = 0.0 factor = 1.0 if ts_df is None: ts_df = pd.DataFrame({ "timestamp": dt_range, "timeseriesId": ts_id, "value": np.random.randn(len(dt_range)) * factor + offset, }) # throws warning during pytest: ts_df.to_json(io_stream, lines=True, orient="records", date_format="iso") io_stream.seek(0) return StreamingResponse(io_stream, media_type="application/json")
async def dataframe(df_id: str = Query( ..., alias="id")) -> Union[HTTPException, StreamingResponse]: if df_id.endswith("plantA.maintenance_events"): df = pd.DataFrame( { # has timestamp column "component_id": ["AB4217", "AB4217", "CD7776"], "component_name": ["central gearbox", "central gearbox", "connector"], "maintenance_type": ["repair", "replacement", "repair"], "timestamp": [ "2020-08-03T15:30:00.000Z", "2020-12-01T07:15:00.000Z", "2021-01-05T09:20:00.000Z", ], } ) elif df_id.endswith("plantB.maintenance_events"): df = pd.DataFrame( { # has timestamp column "component_id": ["GH5300"], "component_name": ["plug"], "maintenance_type": ["replacement"], "timestamp": [ "2020-07-16T22:30:00.000Z", ], } ) elif df_id.endswith("plantA.masterdata"): df = pd.DataFrame({ "name": [ "plant_construction_date", "plant_country", "num_current_employees", ], "value": ["2012-12-07T00:00:00.000Z", "US", 17], }) elif df_id.endswith("plantB.masterdata"): df = pd.DataFrame({ "name": [ "plant_construction_date", "plant_country", "num_current_employees", ], "value": ["2017-08-22T00:00:00.000Z", "DE", 13], }) elif df_id.endswith("alerts"): df = get_value_from_store(df_id) else: return HTTPException( 404, f"no dataframe data available with provided id {df_id}") io_stream = StringIO() df.to_json(io_stream, lines=True, orient="records", date_format="iso") io_stream.seek(0) return StreamingResponse(io_stream, media_type="application/json")
async def post_timeseries( ts_body: List[TimeseriesRecord], ts_id: str = Query(..., alias="timeseriesId"), data_attributes: Optional[str] = Header(None), ) -> Dict: logger.info("Received ts_body for id %s:\n%s", ts_id, str(ts_body)) if ts_id.endswith("anomaly_score"): df = pd.DataFrame.from_dict((x.dict() for x in ts_body), orient="columns") if data_attributes is not None and len(data_attributes) != 0: df_from_store: pd.DataFrame = get_value_from_store(ts_id) df.attrs = df_from_store.attrs df.attrs.update(decode_attributes(data_attributes)) set_value_in_store(ts_id, df) logger.info( "stored timeseries %s in store: %s\n with columns %s", ts_id, str(df), str(df.columns), ) return {"message": "success"} raise HTTPException(404, f"No writable timeseries with id {ts_id}")
async def test_updating_and_keeping_existing_attrs_for_timeseries( async_test_client: AsyncClient, ) -> None: async with async_test_client as client: ts_id = "root.plantA.picklingUnit.influx.anomaly_score" df_attrs_1 = {"test": "Hello world!", "answer": 42} base64_str_1 = encode_attributes(df_attrs_1) response_1 = await client.post( f"/timeseries?timeseriesId={ts_id}", json=[ { "timestamp": "2020-01-01T00:00:00.000000000Z", "value": 12.3 }, { "timestamp": "2020-01-02T00:00:00.000000000Z", "value": 11.9 }, ], headers={"Data-Attributes": base64_str_1}, ) assert response_1.status_code == 200 df_from_store_1 = get_value_from_store(ts_id) assert len(df_from_store_1.attrs) != 0 assert "test" in df_from_store_1.attrs for key, value in df_attrs_1.items(): assert df_from_store_1.attrs[key] == value df_attrs_2 = {"test": "test"} base64_str_2 = encode_attributes(df_attrs_2) response_2 = await client.post( f"/timeseries?timeseriesId={ts_id}", json=[ { "timestamp": "2020-01-01T00:00:00.000000000Z", "value": 12.3 }, { "timestamp": "2020-01-02T00:00:00.000000000Z", "value": 11.9 }, ], headers={"Data-Attributes": base64_str_2}, ) assert response_2.status_code == 200 df_from_store_2 = get_value_from_store(ts_id) df_attrs = deepcopy(df_attrs_1) df_attrs.update(df_attrs_2) assert len(df_from_store_2.attrs) != 0 assert "test" in df_from_store_2.attrs for key, value in df_attrs.items(): assert df_from_store_2.attrs[key] == value
async def timeseries( ids: List[str] = Query(..., alias="id", min_length=1), from_timestamp: datetime.datetime = Query( ..., alias="from", example=datetime.datetime.now(datetime.timezone.utc)), to_timestamp: datetime.datetime = Query( ..., alias="to", example=datetime.datetime.now(datetime.timezone.utc)), ) -> StreamingResponse: collected_attrs = {} io_stream = StringIO() dt_range = pd.date_range(start=from_timestamp, end=to_timestamp, freq="1h", tz=datetime.timezone.utc) for ts_id in ids: logger.debug("loading timeseries dataframe with id %s", str(ts_id)) ts_df = None if ts_id.endswith("temp"): offset = 100.0 if "plantA" in ts_id else 30.0 factor = 10.0 if "plantA" in ts_id else 5.0 elif ts_id.endswith("press"): offset = 14.5 if "plantA" in ts_id else 1.0 factor = 0.73 if "plantA" in ts_id else 0.05 elif ts_id.endswith("anomaly_score"): stored_df = get_value_from_store(ts_id).copy() if "timestamp" in stored_df.columns: stored_df.index = pd.to_datetime(stored_df["timestamp"]) stored_df = stored_df[from_timestamp.isoformat():to_timestamp. isoformat() # type: ignore ] ts_df = pd.DataFrame({ "timestamp": stored_df.index, "timeseriesId": ts_id, "value": stored_df["value"], }) ts_df.attrs = stored_df.attrs else: ts_df = pd.DataFrame({ "timestamp": [], "timeseriesId": [], "value": [], }) # do not overwrite stored attributes! ts_df.attrs.update({ "from_timestamp": from_timestamp.isoformat(), "to_timestamp": to_timestamp.isoformat(), }) else: offset = 0.0 factor = 1.0 if ts_df is None: ts_df = pd.DataFrame({ "timestamp": dt_range, "timeseriesId": ts_id, "value": np.random.randn(len(dt_range)) * factor + offset, }) # throws warning during pytest: ts_df.to_json(io_stream, lines=True, orient="records", date_format="iso") if len(ts_df.attrs) != 0: logger.debug("which has attributes %s", str(ts_df.attrs)) collected_attrs[ts_id] = ts_df.attrs io_stream.seek(0) headers = {} if len(collected_attrs) != 0: headers["Data-Attributes"] = encode_attributes(collected_attrs) return StreamingResponse(io_stream, media_type="application/json", headers=headers)