def power_flows_week( time_series: TimeSeries, network_region_code: str, ) -> Optional[OpennemDataSet]: engine = get_database_engine() query = interconnector_power_flow(time_series=time_series, network_region=network_region_code) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise Exception("No results from query: {}".format(query)) imports = [ DataQueryResult(interval=i[0], result=i[2], group_by="imports" if len(i) > 1 else None) for i in row ] exports = [ DataQueryResult(interval=i[0], result=i[3], group_by="exports" if len(i) > 1 else None) for i in row ] result = stats_factory( imports, # code=network_region_code or network.code, network=time_series.network, period=human_to_period("7d"), interval=human_to_interval("5m"), units=get_unit("power"), region=network_region_code, fueltech_group=True, ) if not result: raise Exception("No results") result_exports = stats_factory( exports, # code=network_region_code or network.code, network=time_series.network, period=human_to_period("7d"), interval=human_to_interval("5m"), units=get_unit("power"), region=network_region_code, fueltech_group=True, ) result.append_set(result_exports) return result
def test_load_unit(self): power = get_unit("power") assert isinstance(power, UnitDefinition), "Returns a definition" assert power.name == "power_mega", "Returns power name" assert power.name_alias == "power", "Returns power name" assert power.unit == "MW", "Schema returns correct unit"
def gov_stats_cpi() -> Optional[OpennemDataSet]: engine = get_database_engine() query = country_stats_query(StatTypes.CPI) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in row ] if len(stats) < 1: logger.error("No results for gov_stats_cpi returing blank set") return None result = stats_factory( stats, code="au.cpi", network=NetworkNEM, interval=human_to_interval("1Q"), period=human_to_period("all"), units=get_unit("cpi"), group_field="gov", ) return result
def power_unit( unit_code: str = Query(..., description="Unit code"), network_code: str = Query(..., description="Network code"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") stats = [] facility_codes = [normalize_duid(unit_code)] query = power_facility_query(facility_codes, network.code, interval=interval, period=period) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Unit stats not found", ) output = stats_factory( stats, code=unit_code, interval=interval, period=period, units=units, network=network, ) if not output: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No stats found", ) return output
def energy_network_api( engine=Depends(get_database_engine), network_code: str = Query(..., description="Network code"), interval_human: str = Query("1d", description="Interval"), period_human: str = Query("1Y", description="Period"), ) -> OpennemDataSet: results = [] network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("energy_giga") query = energy_network(network=network, interval=interval, period=period) with engine.connect() as c: results = list(c.execute(query)) if len(results) < 1: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No results") stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period, units=units, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def demand_week( time_series: TimeSeries, network_region_code: Optional[str], networks_query: Optional[List[NetworkSchema]] = None, ) -> Optional[OpennemDataSet]: engine = get_database_engine() query = network_demand_query( time_series=time_series, network_region=network_region_code, networks_query=networks_query, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: logger.error( "No results from network_demand_query with {}".format(time_series)) return None demand = [ DataQueryResult(interval=i[0], result=i[2], group_by="demand" if len(i) > 1 else None) for i in row ] result = stats_factory( demand, # code=network_region_code or network.code, network=time_series.network, period=human_to_period("7d"), interval=human_to_interval("5m"), units=get_unit("demand"), region=network_region_code, ) if not result: logger.error( "No results from network_demand_query with {}".format(time_series)) return None return result
def emissions_for_network_interval( time_series: TimeSeries, network_region_code: str = None, ) -> OpennemDataSet | None: engine = get_database_engine() if network_region_code and not re.match(_valid_region, network_region_code): raise OpenNEMInvalidNetworkRegion() query = emission_network_fueltech_query( time_series=time_series, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] if len(stats) < 1: logger.error( "No results from emissions_for_network_interval query with {}". format(time_series)) return None result = stats_factory( stats, # code=network_region_code or network.code, network=time_series.network, interval=time_series.interval, period=time_series.period, units=get_unit("emissions"), region=network_region_code, fueltech_group=True, ) return result
def get_power_example() -> OpennemDataSet: network = network_from_network_code("NEM") interval = human_to_interval("5m") units = get_unit("power") period = human_to_period("7d") network_region_code = "NSW1" test_rows = [] dt = datetime.fromisoformat("2021-01-15 10:00:00") for ft in ["coal_black", "coal_brown"]: for v in range(0, 3): test_rows.append([dt, ft, v]) dt = dt + timedelta(minutes=5) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in test_rows ] assert len(stats) == 6, "Should have 6 stats" result = stats_factory( stats, code=network_region_code or network.code, network=network, interval=interval, period=period, units=units, region=network_region_code, fueltech_group=True, ) if not result: raise Exception("Bad unit test data") with open("power-nsw1.json", "w") as fh: fh.write(result.json(indent=4)) return result
def power_flows_network_week( time_series: TimeSeries, ) -> Optional[OpennemDataSet]: engine = get_database_engine() query = interconnector_flow_network_regions_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: logger.error( "No results from interconnector_flow_network_regions_query with {}".format(time_series) ) return None imports = [ DataQueryResult(interval=i[0], result=i[4], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( imports, # code=network_region_code or network.code, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("regional_trade"), # fueltech_group=True, group_field="power", include_group_code=True, include_code=True, ) if not result: logger.error( "No results from interconnector_flow_network_regions_query with {}".format(time_series) ) return None return result
def energy_station( engine=Depends(get_database_engine), session: Session = Depends(get_database_session), network_code: str = Query(..., description="Network code"), station_code: str = Query(..., description="Station Code"), interval: str = Query(None, description="Interval"), period: str = Query("7d", description="Period"), ) -> OpennemDataSet: """ Get energy output for a station (list of facilities) over a period """ network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval: # @NOTE rooftop data is 15m if station_code.startswith("ROOFTOP"): interval = "15m" else: interval = "{}m".format(network.interval_size) interval_obj = human_to_interval(interval) period_obj = human_to_period(period) units = get_unit("energy") station = ( session.query(Station) .join(Station.facilities) .filter(Station.code == station_code) .filter(Facility.network_id == network.code) .one_or_none() ) if not station: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Station not found") if len(station.facilities) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station has no facilities", ) facility_codes = list(set([f.code for f in station.facilities])) query = energy_facility_query( facility_codes, network=network, interval=interval_obj, period=period_obj, ) logger.debug(query) with engine.connect() as c: row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) results_energy = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[2] if len(i) > 1 else None) for i in row ] results_market_value = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[3] if len(i) > 1 else None) for i in row ] results_emissions = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[4] if len(i) > 1 else None) for i in row ] if len(results_energy) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) stats = stats_factory( stats=results_energy, units=units, network=network, interval=interval_obj, period=period_obj, code=station_code, include_group_code=True, ) if not stats: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) stats_market_value = stats_factory( stats=results_market_value, units=get_unit("market_value"), network=network, interval=interval_obj, period=period_obj, code=station_code, include_group_code=True, ) stats.append_set(stats_market_value) stats_emissions = stats_factory( stats=results_emissions, units=get_unit("emissions"), network=network, interval=interval_obj, period=period_obj, code=network.code.lower(), include_group_code=True, ) stats.append_set(stats_emissions) return stats
def power_network_fueltech_api( network_code: str = Query(..., description="Network code"), network_region: str = Query(None, description="Network region"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") scada_range = get_scada_range(network=network) query = power_network_fueltech( network=network, interval=interval, period=period, network_region=network_region, scada_range=scada_range, ) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period, units=units, region=network_region, fueltech_group=True, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def power_station( station_code: str = Query(..., description="Station code"), network_code: str = Query(..., description="Network code"), since: datetime = Query(None, description="Since time"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), session: Session = Depends(get_database_session), engine=Depends(get_database_engine), ) -> OpennemDataSet: if not since: since = datetime.now() - human_to_timedelta("7d") network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: # @NOTE rooftop data is 15m if station_code.startswith("ROOFTOP"): interval_human = "15m" else: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") station = ( session.query(Station) .join(Facility) .filter(Station.code == station_code) .filter(Facility.network_id == network.code) .filter(Station.approved.is_(True)) .one_or_none() ) if not station: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Station not found") facility_codes = list(set([f.code for f in station.facilities])) stats = [] query = power_facility_query(facility_codes, network=network, interval=interval, period=period) logger.debug(query) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats, code=station_code, network=network, interval=interval, period=period, include_group_code=True, units=units, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def energy_interconnector_emissions_region_daily( time_series: TimeSeries, network_region_code: str, networks_query: Optional[List[NetworkSchema]] = None, ) -> Optional[OpennemDataSet]: engine = get_database_engine() period: TimePeriod = human_to_period("1Y") units = get_unit("emissions") query = energy_network_interconnector_emissions_query( time_series=time_series, network_region=network_region_code, networks_query=networks_query, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: return None stats = [ RegionFlowEmissionsResult( interval=i[0], flow_from=i[1], flow_to=i[2], energy=i[3], flow_from_emissions=i[4], flow_to_emissions=i[5], ) for i in row ] stats_grouped = net_flows_emissions(network_region_code, stats, time_series.interval) imports = stats_grouped["imports"] exports = stats_grouped["exports"] # imports = [DataQueryResult(interval=i[0], group_by="imports", result=i[5]) for i in row] # exports = [DataQueryResult(interval=i[0], group_by="exports", result=i[4]) for i in row] result = stats_factory( imports, network=time_series.network, period=period, interval=time_series.interval, units=units, region=network_region_code, fueltech_group=True, localize=False, ) # Bail early on no interconnector # don't error if not result: return result result_exports = stats_factory( exports, network=time_series.network, period=period, interval=time_series.interval, units=units, region=network_region_code, fueltech_group=True, localize=False, ) result.append_set(result_exports) return result
def energy_interconnector_region_daily( time_series: TimeSeries, network_region_code: str, networks_query: Optional[List[NetworkSchema]] = None, ) -> Optional[OpennemDataSet]: engine = get_database_engine() period: TimePeriod = human_to_period("1Y") units = get_unit("energy_giga") query = energy_network_flow_query( time_series=time_series, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: return None imports = [DataQueryResult(interval=i[0], group_by="imports", result=i[1]) for i in row] exports = [DataQueryResult(interval=i[0], group_by="exports", result=i[2]) for i in row] imports_mv = [DataQueryResult(interval=i[0], group_by="imports", result=i[3]) for i in row] exports_mv = [DataQueryResult(interval=i[0], group_by="exports", result=i[4]) for i in row] result = stats_factory( imports, network=time_series.network, period=period, interval=time_series.interval, units=units, region=network_region_code, fueltech_group=True, # localize=False, ) # Bail early on no interconnector # don't error if not result: logger.warn("No interconnector energy result") return result result_exports = stats_factory( exports, network=time_series.network, period=period, interval=time_series.interval, units=units, region=network_region_code, fueltech_group=True, ) result.append_set(result_exports) result_imports_mv = stats_factory( imports_mv, units=get_unit("market_value"), network=time_series.network, fueltech_group=True, interval=time_series.interval, region=network_region_code, period=time_series.period, code=time_series.network.code.lower(), localize=False, ) result.append_set(result_imports_mv) result_export_mv = stats_factory( exports_mv, units=get_unit("market_value"), network=time_series.network, fueltech_group=True, interval=time_series.interval, region=network_region_code, period=time_series.period, code=time_series.network.code.lower(), localize=False, ) result.append_set(result_export_mv) return result
def station_observations_api( station_code: str = Query(None, description="Station code"), interval_human: str = Query("15m", description="Interval"), period_human: str = Query("7d", description="Period"), station_codes: List[str] = [], timezone: str = None, offset: str = None, year: int = None, engine=Depends(get_database_engine), ) -> OpennemDataSet: units = get_unit("temperature") if not interval_human: interval_human = "15m" if not period_human: period_human = "7d" if station_code: station_codes = [station_code] interval = human_to_interval(interval_human) period = human_to_period(period_human) if timezone: timezone = ZoneInfo(timezone) if offset: timezone = get_fixed_timezone(offset) query = observation_query( station_codes=station_codes, interval=interval, period=period, year=year, ) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats=stats, units=units, interval=interval, period=period, code="bom", group_field="temperature", ) return result
def energy_network_fueltech_api( network_code: str = Query(None, description="Network code"), network_region: str = Query(None, description="Network region"), interval_human: str = Query("1d", description="Interval"), year: int = Query(None, description="Year to query"), period_human: str = Query("1Y", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) interval = human_to_interval(interval_human) period_obj: TimePeriod = human_to_period("1Y") if period_human: period_obj = human_to_period(period_human) units = get_unit("energy_giga") query = "" if year and isinstance(year, int): period_obj = human_to_period("1Y") if year > datetime.now().year or year < 1996: raise HTTPException( status_code=status.HTTP_406_NOT_ACCEPTABLE, detail="Not a valid year", ) scada_range = get_scada_range(network=network) query = energy_network_fueltech_year( network=network, interval=interval, year=year, network_region=network_region, scada_range=scada_range, ) elif period_obj and period_obj.period_human == "all": scada_range = get_scada_range(network=network) query = energy_network_fueltech_all( network=network, network_region=network_region, scada_range=scada_range, ) else: query = energy_network_fueltech( network=network, interval=interval, period=period_obj, network_region=network_region, ) # print(query) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Energy stats not found", ) result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period_obj, units=units, region=network_region, fueltech_group=True, ) if not result: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No stats") return result
def power_week( time_series: TimeSeries, network_region_code: str = None, networks_query: Optional[List[NetworkSchema]] = None, include_capacities: bool = False, include_code: Optional[bool] = True, ) -> Optional[OpennemDataSet]: engine = get_database_engine() query = power_network_fueltech_query( time_series=time_series, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] if len(stats) < 1: logger.error("No results from power week query with {}".format(time_series)) return None result = stats_factory( stats, # code=network_region_code or network.code, network=time_series.network, interval=time_series.interval, period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, include_code=include_code, ) if not result: logger.error("No results from power week status factory with {}".format(time_series)) return None if include_capacities and network_region_code: region_fueltech_capacities = get_facility_capacities( time_series.network, network_region_code ) for ft in result.data: if ft.fuel_tech in region_fueltech_capacities: ft.x_capacity_at_present = region_fueltech_capacities[ft.fuel_tech] # price time_series_price = time_series.copy() time_series_price.interval = human_to_interval("30m") query = price_network_query( time_series=time_series_price, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats_price = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] stats_market_value = stats_factory( stats=stats_price, code=network_region_code or time_series.network.code.lower(), units=get_unit("price_energy_mega"), network=time_series.network, interval=human_to_interval("30m"), region=network_region_code, period=time_series.period, include_code=include_code, ) result.append_set(stats_market_value) # rooftop solar time_series_rooftop = time_series.copy() time_series_rooftop.interval = human_to_interval("30m") query = power_network_rooftop_query( time_series=time_series_rooftop, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) rooftop_power = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] rooftop = stats_factory( rooftop_power, # code=network_region_code or network.code, network=time_series.network, interval=human_to_interval("30m"), period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, include_code=include_code, cast_nulls=False, ) # rooftop forecast rooftop_forecast = None if rooftop and rooftop.data and len(rooftop.data) > 0: time_series_rooftop_forecast = time_series_rooftop.copy() time_series_rooftop_forecast.start = rooftop.data[0].history.last time_series_rooftop_forecast.forecast = True query = power_network_rooftop_query( time_series=time_series_rooftop_forecast, networks_query=networks_query, network_region=network_region_code, forecast=True, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) rooftop_forecast_power = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] rooftop_forecast = stats_factory( rooftop_forecast_power, # code=network_region_code or network.code, network=time_series.network, interval=human_to_interval("30m"), period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, include_code=include_code, cast_nulls=False, ) if rooftop and rooftop_forecast: if ( hasattr(rooftop, "data") and len(rooftop.data) > 0 and rooftop_forecast.data and len(rooftop_forecast.data) > 0 ): rooftop.data[0].forecast = rooftop_forecast.data[0].history result.append_set(rooftop) return result
def energy_interconnector_flows_and_emissions( time_series: TimeSeries, network_region_code: str, networks_query: Optional[List[NetworkSchema]] = None, ) -> Optional[OpennemDataSet]: engine = get_database_engine() period: TimePeriod = human_to_period("1Y") unit_energy = get_unit("energy_giga") unit_emissions = get_unit("emissions") query = energy_network_interconnector_emissions_query( time_series=time_series, network_region=network_region_code, networks_query=networks_query, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: return None imports = [ DataQueryResult(interval=i[0], group_by="imports", result=i[1]) for i in row ] exports = [ DataQueryResult(interval=i[0], group_by="exports", result=i[2]) for i in row ] import_emissions = [ DataQueryResult(interval=i[0], group_by="imports", result=i[3]) for i in row ] export_emissions = [ DataQueryResult(interval=i[0], group_by="exports", result=i[4]) for i in row ] import_mv = [ DataQueryResult(interval=i[0], group_by="imports", result=i[5]) for i in row ] export_mv = [ DataQueryResult(interval=i[0], group_by="exports", result=i[6]) for i in row ] result = stats_factory( imports, network=time_series.network, period=period, interval=time_series.interval, units=unit_energy, region=network_region_code, fueltech_group=True, ) if not result: raise Exception("No results from flow controller") result_exports = stats_factory( exports, network=time_series.network, period=period, interval=time_series.interval, units=unit_energy, region=network_region_code, fueltech_group=True, ) result.append_set(result_exports) result_import_emissions = stats_factory( import_emissions, network=time_series.network, period=period, interval=time_series.interval, units=unit_emissions, region=network_region_code, fueltech_group=True, localize=False, ) result.append_set(result_import_emissions) result_export_emissions = stats_factory( export_emissions, network=time_series.network, period=period, interval=time_series.interval, units=unit_emissions, region=network_region_code, fueltech_group=True, localize=False, ) result.append_set(result_export_emissions) # market value for flows result_import_mv = stats_factory( import_mv, network=time_series.network, period=period, interval=time_series.interval, units=get_unit("market_value"), region=network_region_code, fueltech_group=True, localize=False, ) result.append_set(result_import_mv) result_export_mv = stats_factory( export_mv, network=time_series.network, period=period, interval=time_series.interval, units=get_unit("market_value"), region=network_region_code, fueltech_group=True, localize=False, ) result.append_set(result_export_mv) return result
def fueltech_demand_mix( engine=Depends(get_database_engine), # type: ignore network_code: str = Query(..., description="Network code"), ) -> OpennemDataSet: """Return fueltech proportion of demand for a network Args: engine ([type], optional): Database engine. Defaults to Depends(get_database_engine). Raises: HTTPException: No results Returns: OpennemData: data set """ engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval("5m") period_obj = human_to_period("1d") scada_range = get_scada_range(network=network) if not scada_range: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) query = network_fueltech_demand_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) result_set = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( result_set, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("emissions_factor"), group_field="emission_factor", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result
def emission_factor_per_network( # type: ignore engine=Depends(get_database_engine), # type: ignore network_code: str = Query(..., description="Network code"), interval: str = Query("30m", description="Interval size"), ) -> Optional[OpennemDataSet]: engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval(interval) period_obj = human_to_period("7d") if not interval_obj: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Invalid interval size") scada_range = get_scada_range(network=network) if not scada_range: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) query = emission_factor_region_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) emission_factors = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( emission_factors, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("emissions_factor"), group_field="emission_factor", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result
def power_flows_network_week( engine=Depends(get_database_engine), # type: ignore network_code: str = Query(..., description="Network code"), month: date = Query(datetime.now().date(), description="Month to query"), ) -> Optional[OpennemDataSet]: engine = get_database_engine() network = network_from_network_code(network_code) interval_obj = network.get_interval() period_obj = human_to_period("1M") scada_range = get_scada_range(network=network) if not scada_range: raise Exception("Require a scada range") if not network: raise Exception("Network not found") time_series = TimeSeries( start=scada_range.start, month=month, network=network, interval=interval_obj, period=period_obj, ) query = interconnector_flow_network_regions_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise Exception("No results from query: {}".format(query)) imports = [ DataQueryResult(interval=i[0], result=i[4], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( imports, # code=network_region_code or network.code, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("regional_trade"), # fueltech_group=True, group_field="power", include_group_code=True, include_code=True, ) if not result or not result.data: raise Exception("No results") INVERT_SETS = ["VIC1->NSW1", "VIC1->SA1"] inverted_data = [] for ds in result.data: if ds.code in INVERT_SETS: ds_inverted = invert_flow_set(ds) inverted_data.append(ds_inverted) else: inverted_data.append(ds) result.data = inverted_data return result
def demand_network_region_daily( time_series: TimeSeries, network_region_code: str | None = None, networks: list[NetworkSchema] = [], ) -> OpennemDataSet | None: """Gets demand market_value and energy for a network -> network_region""" engine = get_database_engine() query = demand_network_region_query(time_series=time_series, network_region=network_region_code, networks=networks) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) results_energy = [ DataQueryResult(interval=i[0], group_by=i[2], result=i[3] if len(i) > 1 else None) for i in row ] results_market_value = [ DataQueryResult(interval=i[0], group_by=i[2], result=i[4] if len(i) > 1 else None) for i in row ] if len(results_energy) < 1: logger.error("No results from query: {}".format(query)) return None # demand based values for VWP stats = stats_factory( stats=results_energy, units=get_unit("demand.energy_giga"), network=time_series.network, fueltech_group=False, interval=time_series.interval, period=time_series.period, localize=True, ) if not stats: raise Exception( f"Not stats for demand_network_region_daily: {network_region_code}" ) stats_market_value = stats_factory( stats=results_market_value, units=get_unit("demand.market_value"), network=time_series.network, fueltech_group=False, interval=time_series.interval, period=time_series.period, code=time_series.network.code.lower(), localize=True, ) if stats_market_value: stats.append_set(stats_market_value) return stats
def weather_daily( time_series: TimeSeries, station_code: str, unit_name: str = "temperature_mean", include_min_max: bool = True, network_region: Optional[str] = None, ) -> Optional[OpennemDataSet]: engine = get_database_engine() units = get_unit(unit_name) query = weather_observation_query( time_series=time_series, station_codes=[station_code], ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) temp_avg = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[2] if len(i) > 1 else None) for i in row ] temp_min = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[3] if len(i) > 1 else None) for i in row ] temp_max = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[4] if len(i) > 1 else None) for i in row ] if len(temp_avg) < 1: logger.error("No results from weather_observation_query with {}".format(time_series)) return None stats = stats_factory( stats=temp_avg, units=units, network=time_series.network, interval=time_series.interval, region=network_region, code="bom", group_field="temperature", localize=False, ) if not stats: logger.error( "No results from weather_observation_query stats factory with {}".format(time_series) ) return None if include_min_max: stats_min = stats_factory( stats=temp_min, units=get_unit("temperature_min"), network=time_series.network, interval=time_series.interval, region=network_region, code="bom", group_field="temperature", localize=False, ) stats_max = stats_factory( stats=temp_max, units=get_unit("temperature_max"), network=time_series.network, interval=time_series.interval, region=network_region, code="bom", group_field="temperature", localize=False, ) stats.append_set(stats_min) stats.append_set(stats_max) return stats
def power_week( time_series: TimeSeries, network_region_code: str = None, networks_query: Optional[List[NetworkSchema]] = None, ) -> Optional[OpennemDataSet]: engine = get_database_engine() query = power_network_fueltech_query( time_series=time_series, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] if len(stats) < 1: raise Exception("No results from query: {}".format(query)) result = stats_factory( stats, # code=network_region_code or network.code, network=time_series.network, interval=time_series.interval, period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, ) if not result: raise Exception("No results") # price time_series_price = time_series.copy() time_series_price.interval = human_to_interval("30m") query = price_network_query( time_series=time_series_price, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats_price = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] stats_market_value = stats_factory( stats=stats_price, code=network_region_code or time_series.network.code.lower(), units=get_unit("price_energy_mega"), network=time_series.network, interval=human_to_interval("30m"), region=network_region_code, period=time_series.period, ) result.append_set(stats_market_value) # rooftop solar time_series_rooftop = time_series.copy() time_series_rooftop.interval = human_to_interval("30m") query = power_network_rooftop_query( time_series=time_series_rooftop, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) rooftop_power = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] rooftop = stats_factory( rooftop_power, # code=network_region_code or network.code, network=time_series.network, interval=human_to_interval("30m"), period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, ) # rooftop forecast time_series_rooftop_forecast = time_series.copy() time_series_rooftop_forecast.interval = human_to_interval("30m") time_series_rooftop_forecast.forecast = True query = power_network_rooftop_query( time_series=time_series_rooftop, networks_query=networks_query, network_region=network_region_code, forecast=True, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) rooftop_forecast_power = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] rooftop_forecast = stats_factory( rooftop_forecast_power, # code=network_region_code or network.code, network=time_series.network, interval=human_to_interval("30m"), period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, ) if rooftop and rooftop_forecast: if (hasattr(rooftop, "data") and len(rooftop.data) > 0 and rooftop_forecast.data and len(rooftop_forecast.data) > 0): rooftop.data[0].forecast = rooftop_forecast.data[0].history result.append_set(rooftop) return result
def energy_fueltech_daily( time_series: TimeSeries, network_region_code: Optional[str] = None, networks_query: Optional[List[NetworkSchema]] = None, ) -> Optional[OpennemDataSet]: engine = get_database_engine() units = get_unit("energy_giga") query = energy_network_fueltech_query( time_series=time_series, network_region=network_region_code, networks_query=networks_query, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) results_energy = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[2] if len(i) > 1 else None) for i in row ] results_market_value = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[3] if len(i) > 1 else None) for i in row ] results_emissions = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[4] if len(i) > 1 else None) for i in row ] if len(results_energy) < 1: logger.error("No results from query: {}".format(query)) return None stats = stats_factory( stats=results_energy, units=units, network=time_series.network, fueltech_group=True, interval=time_series.interval, region=network_region_code, period=time_series.period, localize=True, # code=network.code.lower(), ) if not stats: return None stats_market_value = stats_factory( stats=results_market_value, units=get_unit("market_value"), network=time_series.network, fueltech_group=True, interval=time_series.interval, region=network_region_code, period=time_series.period, code=time_series.network.code.lower(), localize=True, ) stats.append_set(stats_market_value) stats_emissions = stats_factory( stats=results_emissions, units=get_unit("emissions"), network=time_series.network, fueltech_group=True, interval=time_series.interval, region=network_region_code, period=time_series.period, code=time_series.network.code.lower(), localize=True, ) stats.append_set(stats_emissions) return stats
def price_network_region_api( engine=Depends(get_database_engine), network_code: str = Query(..., description="Network code"), network_region_code: str = Query(..., description="Region code"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), year: Optional[int] = None, ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period_obj = None if period_human: period_obj = human_to_period(period_human) units = get_unit("price") scada_range = get_scada_range(network=network) if period_obj and period_obj.period_human == "all" and interval.interval_human == "1M": query = price_network_monthly( network=network, network_region_code=network_region_code, scada_range=scada_range, ) else: query = price_network_region( network=network, network_region_code=network_region_code, interval=interval, period=period_obj, scada_range=scada_range, year=year, ) with engine.connect() as c: results = list(c.execute(query)) if len(results) < 1: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No data found") stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in results ] result = stats_factory( stats, code=network.code, region=network_region_code, network=network, interval=interval, period=period_obj, units=units, group_field="price", ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def price_network_endpoint( engine: Engine = Depends(get_database_engine), network_code: str = Path(..., description="Network code"), network_region: Optional[str] = Query(None, description="Network region code"), forecasts: bool = Query(False, description="Include price forecasts"), ) -> OpennemDataSet: """Returns network and network region price info for interval which defaults to network interval size Args: engine ([type], optional): Database engine. Defaults to Depends(get_database_engine). Raises: HTTPException: No results Returns: OpennemData: data set """ engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval("5m") period_obj = human_to_period("1d") scada_range = get_balancing_range(network=network, include_forecasts=forecasts) if not scada_range: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) if network_region: time_series.network.regions = [NetworkNetworkRegion(code=network_region)] query = network_region_price_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) result_set = [ DataQueryResult(interval=i[0], result=i[3], group_by=i[2] if len(i) > 1 else None) for i in row ] result = stats_factory( result_set, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("price"), group_field="price", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result