def load(kcca_data: dict, **kwargs): from airqo_etl_utils.kcca_utils import ( transform_kcca_measurements_for_api, transform_kcca_hourly_data_for_bigquery, transform_kcca_data_for_message_broker, ) from airqo_etl_utils.commons import un_fill_nan from airqo_etl_utils.airqo_api import AirQoApi from airqo_etl_utils.message_broker import KafkaBrokerClient from airqo_etl_utils.bigquery_api import BigQueryApi from airqo_etl_utils.config import configuration data = un_fill_nan(kcca_data.get("data")) try: dag_run = kwargs.get("dag_run") destination = dag_run.conf["destination"] except KeyError: destination = "bigquery" if destination == "bigquery": kcca_transformed_data = transform_kcca_hourly_data_for_bigquery( data) big_query_api = BigQueryApi() big_query_api.save_data( data=kcca_transformed_data, table=big_query_api.hourly_measurements_table, ) elif destination == "message-broker": kcca_transformed_data = transform_kcca_data_for_message_broker( data=data, frequency="hourly") info = { "data": kcca_transformed_data, "action": "insert", "tenant": "kcca" } kafka = KafkaBrokerClient() kafka.send_data(info=info, topic=configuration.HOURLY_MEASUREMENTS_TOPIC) elif destination == "api": kcca_transformed_data = transform_kcca_measurements_for_api(data) airqo_api = AirQoApi() airqo_api.save_events(measurements=kcca_transformed_data, tenant="kcca") else: raise Exception( "Invalid data destination. Valid values are bigquery, message-broker and api" )
def load(inputs: dict): from airqo_etl_utils.commons import un_fill_nan from airqo_etl_utils.airqo_api import AirQoApi kcca_data = un_fill_nan(inputs.get("data")) airqo_api = AirQoApi() airqo_api.save_events(measurements=kcca_data, tenant="kcca")
def send_raw_measurements_to_api(airqo_data: dict): from airqo_etl_utils.commons import un_fill_nan from airqo_etl_utils.airqo_utils import restructure_airqo_data from airqo_etl_utils.airqo_api import AirQoApi data = un_fill_nan(airqo_data.get("data")) airqo_restructured_data = restructure_airqo_data(data=data, destination="api") airqo_api = AirQoApi() airqo_api.save_events(measurements=airqo_restructured_data, tenant="airqo")
def send_hourly_measurements_to_api(inputs: dict): from airqo_etl_utils.kcca_utils import transform_kcca_measurements_for_api from airqo_etl_utils.commons import un_fill_nan from airqo_etl_utils.airqo_api import AirQoApi data = un_fill_nan(inputs.get("data")) kcca_data = transform_kcca_measurements_for_api(data) airqo_api = AirQoApi() airqo_api.save_events(measurements=kcca_data, tenant="kcca")
def load(airqo_data: dict, **kwargs): from airqo_etl_utils.commons import un_fill_nan from airqo_etl_utils.bigquery_api import BigQueryApi from airqo_etl_utils.airqo_api import AirQoApi from airqo_etl_utils.airqo_utils import restructure_airqo_data from airqo_etl_utils.config import configuration from airqo_etl_utils.message_broker import KafkaBrokerClient data = un_fill_nan(airqo_data.get("data")) try: dag_run = kwargs.get("dag_run") destination = dag_run.conf["destination"] except KeyError: destination = "bigquery" if destination == "bigquery": airqo_restructured_data = restructure_airqo_data( data=data, destination="bigquery" ) big_query_api = BigQueryApi() big_query_api.save_data( data=airqo_restructured_data, table=big_query_api.hourly_measurements_table, ) elif destination == "message-broker": airqo_restructured_data = restructure_airqo_data( data=data, destination="message-broker" ) info = { "data": airqo_restructured_data, "action": "insert", "tenant": "airqo", } kafka = KafkaBrokerClient() kafka.send_data(info=info, topic=configuration.HOURLY_MEASUREMENTS_TOPIC) elif destination == "api": airqo_restructured_data = restructure_airqo_data( data=data, destination="api" ) airqo_api = AirQoApi() airqo_api.save_events(measurements=airqo_restructured_data, tenant="airqo") else: raise Exception( "Invalid data destination. Valid values are bigquery, message-broker and api" )