def validate_metadatum_filtered_source_sink(
filtered_source_sink: FilteredSourceSink, ) -> FilteredSourceSink:
if filtered_source_sink.ref_id_type is None:
raise AdapterClientWiringInvalidError(
"Unset ref_id_type in metadatum wiring.")
if filtered_source_sink.ref_key is None:
raise AdapterClientWiringInvalidError(
"Unset ref_key in metadatum wiring.")
return filtered_source_sink
def validate_type_and_ref_id(
wf_in_out_name_to_filtered_source_or_sink_mapping: Mapping[str, Union[
FilteredSource, FilteredSink]]
) -> Tuple[List[str], List[str], List[ExternalType]]:
"""Validate generic rest adapter specific requirements of wirings
* ref_ids can't be None
* types must be provided and be a known rest adapter type
Raises AdapterClientWiringInvalidError if requirements are not fullfilled.
Returns a triple of compatibly ordered lists of
workflow input / output names
ref ids
types (ExternalType)
"""
wf_in_out_names = list(
wf_in_out_name_to_filtered_source_or_sink_mapping.keys())
ref_ids: List[str] = [
filtered_source.ref_id # type: ignore
for wf_input_name in wf_in_out_names
if (filtered_source :=
wf_in_out_name_to_filtered_source_or_sink_mapping[wf_input_name]
).ref_id is not None
]
if len(ref_ids) < len(wf_in_out_name_to_filtered_source_or_sink_mapping):
raise AdapterClientWiringInvalidError(
"Unset ref id in a wiring using generic rest adapter.")
try:
corresponding_types: List[ExternalType] = [
ExternalType(fs.type) for wf_input_name in wf_in_out_names if
(fs :=
wf_in_out_name_to_filtered_source_or_sink_mapping[wf_input_name]
).type is not None
]
except ValueError as e:
raise AdapterClientWiringInvalidError(
"Unknown type in a wiring using generic rest adapter.") from e
if len(corresponding_types) < len(
wf_in_out_name_to_filtered_source_or_sink_mapping):
raise AdapterClientWiringInvalidError(
"Unset type in a wiring using generic rest adapter.")
if not all((isinstance(rest_adapter_data_type, ExternalType)
for rest_adapter_data_type in corresponding_types)):
raise AdapterClientWiringInvalidError(
"Got unknown type in wiring for generic rest adapter")
return wf_in_out_names, ref_ids, corresponding_types
def load_directly_provisioned_data(
wf_input_name_to_filtered_source_mapping_dict: Dict[str, FilteredSource]
) -> Dict[str, Any]:
try:
return {
wf_inp_name: direct_provisioning_filtered_source.filters["value"]
for (
wf_inp_name,
direct_provisioning_filtered_source,
) in wf_input_name_to_filtered_source_mapping_dict.items()
}
except KeyError as e:
raise AdapterClientWiringInvalidError(
"Direct Input Provisioning without 'value' field in filters"
) from e
async def load_grouped_timeseries_data_together(
data_to_load: Dict[str, FilteredSource],
adapter_key: str) -> Dict[str, pd.Series]:
"""Reorganize query information by timestamp pairs and load timeseries data
Generic Rest Adapter allows to query for multiple timeseries in one request but then only with
one timestamp filter pair and same (requested) value type for all those timeseries.
This function expects data refs of the timeseries type,
groups them together if they have same filter timestamp pairs and same value type,
loads each such group in one request
and returns all results gathered.
"""
loaded_data = {}
# group by occuring timestamp pairs
group_by_timestamp_pair: Dict[Tuple[str, str, ExternalType],
Dict[str,
FilteredSource], ] = defaultdict(dict)
for filtered_source in data_to_load.values():
if (not isinstance(filtered_source.filters.get(
"timestampFrom", None), str)) or (not isinstance(
filtered_source.filters.get("timestampTo", None), str)):
raise AdapterClientWiringInvalidError(
"Timeseries data with no to/from filters.")
for key, filtered_source in data_to_load.items():
group_by_timestamp_pair[(
filtered_source.filters["timestampFrom"],
filtered_source.filters["timestampTo"],
ExternalType(filtered_source.type),
)][key] = filtered_source
# load each group together:
for (group_tuple, grouped_source_dict) in group_by_timestamp_pair.items():
loaded_ts_data_from_adapter = await load_ts_data_from_adapter(
list(grouped_source_dict.values()),
group_tuple[0],
group_tuple[1],
adapter_key=adapter_key,
)
loaded_data.update({
key: extract_one_channel_series_from_loaded_data(
loaded_ts_data_from_adapter,
filtered_source.ref_id, # type: ignore
)
for key, filtered_source in grouped_source_dict.items()
})
try:
received_ids = loaded_ts_data_from_adapter["timeseriesId"].unique()
except KeyError as e:
msg = (
f"Missing keys in received timeseries records."
f" Got columns {str(loaded_ts_data_from_adapter.columns)}"
f" with dataframe of shape {str(loaded_ts_data_from_adapter.shape)}:\n"
f"{str(loaded_ts_data_from_adapter)}")
logger.info(msg)
raise AdapterHandlingException(msg) from e
queried_ids = [fs.ref_id for fs in grouped_source_dict.values()]
if not np.isin(received_ids, np.array(queried_ids)).all():
msg = (
f"Found timeseries ids in received data that were not queried."
f" Received timeseriesId unique values were:\n{str(received_ids.tolist())}"
f" \nQueried ids were:\n{str(queried_ids)}."
"\nThis unassignable data will be discarded. This indicates an error in the adapter"
f" implementation of the adapter {str(adapter_key)}!")
logger.warning(msg)
return loaded_data