async def test_updating_and_keeping_existing_attrs_for_dataframe(
async_test_client: AsyncClient, ) -> None:
async with async_test_client as client:
df_attrs_1 = {"test": "Hello world!", "answer": 42}
base64_str_1 = encode_attributes(df_attrs_1)
response_1 = await client.post(
"/dataframe?id=root.plantA.alerts",
json=[{
"column1": 1,
"column2": 1.3
}, {
"column1": 2,
"column2": 2.8
}],
headers={"Data-Attributes": base64_str_1},
)
assert response_1.status_code == 200
df_from_store_1 = get_value_from_store("root.plantA.alerts")
assert len(df_from_store_1.attrs) != 0
assert "test" in df_from_store_1.attrs
for key, value in df_attrs_1.items():
assert df_from_store_1.attrs[key] == value
df_attrs_2 = {"test": "test"}
base64_str_2 = encode_attributes(df_attrs_2)
response_2 = await client.post(
"/dataframe?id=root.plantA.alerts",
json=[{
"column1": 1,
"column2": 1.3
}, {
"column1": 2,
"column2": 2.8
}],
headers={"Data-Attributes": base64_str_2},
)
assert response_2.status_code == 200
df_from_store_2 = get_value_from_store("root.plantA.alerts")
df_attrs = deepcopy(df_attrs_1)
df_attrs.update(df_attrs_2)
assert len(df_from_store_2.attrs) != 0
assert "test" in df_from_store_2.attrs
for key, value in df_attrs.items():
assert df_from_store_2.attrs[key] == value
async def test_receiving_attrs_via_post_timeseries(
async_test_client: AsyncClient, ) -> None:
async with async_test_client as client:
df_attrs = {"test": "Hello world!", "answer": 42}
base64_str = encode_attributes(df_attrs)
ts_id = "root.plantA.picklingUnit.influx.anomaly_score"
response = await client.post(
f"/timeseries?timeseriesId={ts_id}",
json=[
{
"timestamp": "2020-01-01T00:00:00.000000000Z",
"value": 12.3
},
{
"timestamp": "2020-01-02T00:00:00.000000000Z",
"value": 11.9
},
],
headers={"Data-Attributes": base64_str},
)
assert response.status_code == 200
df_from_store = get_value_from_store(ts_id)
assert len(df_from_store.attrs) != 0
assert "test" in df_from_store.attrs
for key, value in df_attrs.items():
assert df_from_store.attrs[key] == value
async def test_receiving_attrs_via_post_dataframe(
async_test_client: AsyncClient, ) -> None:
async with async_test_client as client:
df_attrs = {"test": "Hello world!", "answer": 42}
base64_str = encode_attributes(df_attrs)
response = await client.post(
"/dataframe?id=root.plantA.alerts",
json=[{
"column1": 1,
"column2": 1.3
}, {
"column1": 2,
"column2": 2.8
}],
headers={"Data-Attributes": base64_str},
)
assert response.status_code == 200
df_from_store = get_value_from_store("root.plantA.alerts")
assert len(df_from_store.attrs) != 0
assert "test" in df_from_store.attrs
for key, value in df_attrs.items():
assert df_from_store.attrs[key] == value
async def post_dataframe(
df_body: List[
Dict] = Body(
...,
example=[
{
"column_A": 42.0,
"column_B": "example"
},
{
"column_A": 11.97,
"column_B": "example"
},
],
),
df_id: str = Query(..., alias="id"),
data_attributes: Optional[str] = Header(None),
) -> dict:
if df_id.endswith("alerts"):
df = pd.DataFrame.from_dict(df_body, orient="columns")
if data_attributes is not None and len(data_attributes) != 0:
df_from_store: pd.DataFrame = get_value_from_store(df_id)
df.attrs = df_from_store.attrs
df.attrs.update(decode_attributes(data_attributes))
logger.debug("storing %s", json.dumps(df_body))
logger.debug("which has attributes %s", str(df.attrs))
set_value_in_store(df_id, df)
return {"message": "success"}
raise HTTPException(404, f"No writable dataframe with id {df_id}")
async def timeseries(
ids: List[str] = Query(..., alias="id", min_length=1),
from_timestamp: datetime.datetime = Query(..., alias="from"),
to_timestamp: datetime.datetime = Query(..., alias="to"),
) -> StreamingResponse:
io_stream = StringIO()
dt_range = pd.date_range(start=from_timestamp,
end=to_timestamp,
freq="1h",
tz=datetime.timezone.utc)
for ts_id in ids:
ts_df = None
if ts_id.endswith("temp"):
offset = 100.0 if "plantA" in ts_id else 30.0
factor = 10.0 if "plantA" in ts_id else 5.0
elif ts_id.endswith("press"):
offset = 14.5 if "plantA" in ts_id else 1.0
factor = 0.73 if "plantA" in ts_id else 0.05
elif ts_id.endswith("anomaly_score"):
stored_df = get_value_from_store(ts_id).copy()
if "timestamp" in stored_df.columns:
stored_df.index = pd.to_datetime(stored_df["timestamp"])
stored_df = stored_df[from_timestamp.isoformat():to_timestamp.
isoformat() # type: ignore
]
ts_df = pd.DataFrame({
"timestamp": stored_df.index,
"timeseriesId": ts_id,
"value": stored_df["value"],
})
else:
ts_df = pd.DataFrame({
"timestamp": [],
"timeseriesId": [],
"value": [],
})
else:
offset = 0.0
factor = 1.0
if ts_df is None:
ts_df = pd.DataFrame({
"timestamp":
dt_range,
"timeseriesId":
ts_id,
"value":
np.random.randn(len(dt_range)) * factor + offset,
})
# throws warning during pytest:
ts_df.to_json(io_stream,
lines=True,
orient="records",
date_format="iso")
io_stream.seek(0)
return StreamingResponse(io_stream, media_type="application/json")
async def dataframe(df_id: str = Query(
..., alias="id")) -> Union[HTTPException, StreamingResponse]:
if df_id.endswith("plantA.maintenance_events"):
df = pd.DataFrame(
{ # has timestamp column
"component_id": ["AB4217", "AB4217", "CD7776"],
"component_name": ["central gearbox", "central gearbox", "connector"],
"maintenance_type": ["repair", "replacement", "repair"],
"timestamp": [
"2020-08-03T15:30:00.000Z",
"2020-12-01T07:15:00.000Z",
"2021-01-05T09:20:00.000Z",
],
}
)
elif df_id.endswith("plantB.maintenance_events"):
df = pd.DataFrame(
{ # has timestamp column
"component_id": ["GH5300"],
"component_name": ["plug"],
"maintenance_type": ["replacement"],
"timestamp": [
"2020-07-16T22:30:00.000Z",
],
}
)
elif df_id.endswith("plantA.masterdata"):
df = pd.DataFrame({
"name": [
"plant_construction_date",
"plant_country",
"num_current_employees",
],
"value": ["2012-12-07T00:00:00.000Z", "US", 17],
})
elif df_id.endswith("plantB.masterdata"):
df = pd.DataFrame({
"name": [
"plant_construction_date",
"plant_country",
"num_current_employees",
],
"value": ["2017-08-22T00:00:00.000Z", "DE", 13],
})
elif df_id.endswith("alerts"):
df = get_value_from_store(df_id)
else:
return HTTPException(
404, f"no dataframe data available with provided id {df_id}")
io_stream = StringIO()
df.to_json(io_stream, lines=True, orient="records", date_format="iso")
io_stream.seek(0)
return StreamingResponse(io_stream, media_type="application/json")
async def post_timeseries(
ts_body: List[TimeseriesRecord],
ts_id: str = Query(..., alias="timeseriesId"),
data_attributes: Optional[str] = Header(None),
) -> Dict:
logger.info("Received ts_body for id %s:\n%s", ts_id, str(ts_body))
if ts_id.endswith("anomaly_score"):
df = pd.DataFrame.from_dict((x.dict() for x in ts_body),
orient="columns")
if data_attributes is not None and len(data_attributes) != 0:
df_from_store: pd.DataFrame = get_value_from_store(ts_id)
df.attrs = df_from_store.attrs
df.attrs.update(decode_attributes(data_attributes))
set_value_in_store(ts_id, df)
logger.info(
"stored timeseries %s in store: %s\n with columns %s",
ts_id,
str(df),
str(df.columns),
)
return {"message": "success"}
raise HTTPException(404, f"No writable timeseries with id {ts_id}")
async def test_updating_and_keeping_existing_attrs_for_timeseries(
async_test_client: AsyncClient, ) -> None:
async with async_test_client as client:
ts_id = "root.plantA.picklingUnit.influx.anomaly_score"
df_attrs_1 = {"test": "Hello world!", "answer": 42}
base64_str_1 = encode_attributes(df_attrs_1)
response_1 = await client.post(
f"/timeseries?timeseriesId={ts_id}",
json=[
{
"timestamp": "2020-01-01T00:00:00.000000000Z",
"value": 12.3
},
{
"timestamp": "2020-01-02T00:00:00.000000000Z",
"value": 11.9
},
],
headers={"Data-Attributes": base64_str_1},
)
assert response_1.status_code == 200
df_from_store_1 = get_value_from_store(ts_id)
assert len(df_from_store_1.attrs) != 0
assert "test" in df_from_store_1.attrs
for key, value in df_attrs_1.items():
assert df_from_store_1.attrs[key] == value
df_attrs_2 = {"test": "test"}
base64_str_2 = encode_attributes(df_attrs_2)
response_2 = await client.post(
f"/timeseries?timeseriesId={ts_id}",
json=[
{
"timestamp": "2020-01-01T00:00:00.000000000Z",
"value": 12.3
},
{
"timestamp": "2020-01-02T00:00:00.000000000Z",
"value": 11.9
},
],
headers={"Data-Attributes": base64_str_2},
)
assert response_2.status_code == 200
df_from_store_2 = get_value_from_store(ts_id)
df_attrs = deepcopy(df_attrs_1)
df_attrs.update(df_attrs_2)
assert len(df_from_store_2.attrs) != 0
assert "test" in df_from_store_2.attrs
for key, value in df_attrs.items():
assert df_from_store_2.attrs[key] == value
async def timeseries(
ids: List[str] = Query(..., alias="id", min_length=1),
from_timestamp: datetime.datetime = Query(
...,
alias="from",
example=datetime.datetime.now(datetime.timezone.utc)),
to_timestamp: datetime.datetime = Query(
..., alias="to", example=datetime.datetime.now(datetime.timezone.utc)),
) -> StreamingResponse:
collected_attrs = {}
io_stream = StringIO()
dt_range = pd.date_range(start=from_timestamp,
end=to_timestamp,
freq="1h",
tz=datetime.timezone.utc)
for ts_id in ids:
logger.debug("loading timeseries dataframe with id %s", str(ts_id))
ts_df = None
if ts_id.endswith("temp"):
offset = 100.0 if "plantA" in ts_id else 30.0
factor = 10.0 if "plantA" in ts_id else 5.0
elif ts_id.endswith("press"):
offset = 14.5 if "plantA" in ts_id else 1.0
factor = 0.73 if "plantA" in ts_id else 0.05
elif ts_id.endswith("anomaly_score"):
stored_df = get_value_from_store(ts_id).copy()
if "timestamp" in stored_df.columns:
stored_df.index = pd.to_datetime(stored_df["timestamp"])
stored_df = stored_df[from_timestamp.isoformat():to_timestamp.
isoformat() # type: ignore
]
ts_df = pd.DataFrame({
"timestamp": stored_df.index,
"timeseriesId": ts_id,
"value": stored_df["value"],
})
ts_df.attrs = stored_df.attrs
else:
ts_df = pd.DataFrame({
"timestamp": [],
"timeseriesId": [],
"value": [],
})
# do not overwrite stored attributes!
ts_df.attrs.update({
"from_timestamp": from_timestamp.isoformat(),
"to_timestamp": to_timestamp.isoformat(),
})
else:
offset = 0.0
factor = 1.0
if ts_df is None:
ts_df = pd.DataFrame({
"timestamp":
dt_range,
"timeseriesId":
ts_id,
"value":
np.random.randn(len(dt_range)) * factor + offset,
})
# throws warning during pytest:
ts_df.to_json(io_stream,
lines=True,
orient="records",
date_format="iso")
if len(ts_df.attrs) != 0:
logger.debug("which has attributes %s", str(ts_df.attrs))
collected_attrs[ts_id] = ts_df.attrs
io_stream.seek(0)
headers = {}
if len(collected_attrs) != 0:
headers["Data-Attributes"] = encode_attributes(collected_attrs)
return StreamingResponse(io_stream,
media_type="application/json",
headers=headers)
