def load(kcca_data: dict, **kwargs):
from airqo_etl_utils.kcca_utils import (
transform_kcca_measurements_for_api,
transform_kcca_hourly_data_for_bigquery,
transform_kcca_data_for_message_broker,
)
from airqo_etl_utils.commons import un_fill_nan
from airqo_etl_utils.airqo_api import AirQoApi
from airqo_etl_utils.message_broker import KafkaBrokerClient
from airqo_etl_utils.bigquery_api import BigQueryApi
from airqo_etl_utils.config import configuration
data = un_fill_nan(kcca_data.get("data"))
try:
dag_run = kwargs.get("dag_run")
destination = dag_run.conf["destination"]
except KeyError:
destination = "bigquery"
if destination == "bigquery":
kcca_transformed_data = transform_kcca_hourly_data_for_bigquery(
data)
big_query_api = BigQueryApi()
big_query_api.save_data(
data=kcca_transformed_data,
table=big_query_api.hourly_measurements_table,
)
elif destination == "message-broker":
kcca_transformed_data = transform_kcca_data_for_message_broker(
data=data, frequency="hourly")
info = {
"data": kcca_transformed_data,
"action": "insert",
"tenant": "kcca"
}
kafka = KafkaBrokerClient()
kafka.send_data(info=info,
topic=configuration.HOURLY_MEASUREMENTS_TOPIC)
elif destination == "api":
kcca_transformed_data = transform_kcca_measurements_for_api(data)
airqo_api = AirQoApi()
airqo_api.save_events(measurements=kcca_transformed_data,
tenant="kcca")
else:
raise Exception(
"Invalid data destination. Valid values are bigquery, message-broker and api"
)
def load(inputs: dict):
from airqo_etl_utils.commons import un_fill_nan
from airqo_etl_utils.airqo_api import AirQoApi
kcca_data = un_fill_nan(inputs.get("data"))
airqo_api = AirQoApi()
airqo_api.save_events(measurements=kcca_data, tenant="kcca")
def send_raw_measurements_to_api(airqo_data: dict):
from airqo_etl_utils.commons import un_fill_nan
from airqo_etl_utils.airqo_utils import restructure_airqo_data
from airqo_etl_utils.airqo_api import AirQoApi
data = un_fill_nan(airqo_data.get("data"))
airqo_restructured_data = restructure_airqo_data(data=data, destination="api")
airqo_api = AirQoApi()
airqo_api.save_events(measurements=airqo_restructured_data, tenant="airqo")
def send_hourly_measurements_to_api(inputs: dict):
from airqo_etl_utils.kcca_utils import transform_kcca_measurements_for_api
from airqo_etl_utils.commons import un_fill_nan
from airqo_etl_utils.airqo_api import AirQoApi
data = un_fill_nan(inputs.get("data"))
kcca_data = transform_kcca_measurements_for_api(data)
airqo_api = AirQoApi()
airqo_api.save_events(measurements=kcca_data, tenant="kcca")
def load(airqo_data: dict, **kwargs):
from airqo_etl_utils.commons import un_fill_nan
from airqo_etl_utils.bigquery_api import BigQueryApi
from airqo_etl_utils.airqo_api import AirQoApi
from airqo_etl_utils.airqo_utils import restructure_airqo_data
from airqo_etl_utils.config import configuration
from airqo_etl_utils.message_broker import KafkaBrokerClient
data = un_fill_nan(airqo_data.get("data"))
try:
dag_run = kwargs.get("dag_run")
destination = dag_run.conf["destination"]
except KeyError:
destination = "bigquery"
if destination == "bigquery":
airqo_restructured_data = restructure_airqo_data(
data=data, destination="bigquery"
)
big_query_api = BigQueryApi()
big_query_api.save_data(
data=airqo_restructured_data,
table=big_query_api.hourly_measurements_table,
)
elif destination == "message-broker":
airqo_restructured_data = restructure_airqo_data(
data=data, destination="message-broker"
)
info = {
"data": airqo_restructured_data,
"action": "insert",
"tenant": "airqo",
}
kafka = KafkaBrokerClient()
kafka.send_data(info=info, topic=configuration.HOURLY_MEASUREMENTS_TOPIC)
elif destination == "api":
airqo_restructured_data = restructure_airqo_data(
data=data, destination="api"
)
airqo_api = AirQoApi()
airqo_api.save_events(measurements=airqo_restructured_data, tenant="airqo")
else:
raise Exception(
"Invalid data destination. Valid values are bigquery, message-broker and api"
)
