def power_flows_week( time_series: TimeSeries, network_region_code: str, ) -> Optional[OpennemDataSet]: engine = get_database_engine() query = interconnector_power_flow(time_series=time_series, network_region=network_region_code) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise Exception("No results from query: {}".format(query)) imports = [ DataQueryResult(interval=i[0], result=i[2], group_by="imports" if len(i) > 1 else None) for i in row ] exports = [ DataQueryResult(interval=i[0], result=i[3], group_by="exports" if len(i) > 1 else None) for i in row ] result = stats_factory( imports, # code=network_region_code or network.code, network=time_series.network, period=human_to_period("7d"), interval=human_to_interval("5m"), units=get_unit("power"), region=network_region_code, fueltech_group=True, ) if not result: raise Exception("No results") result_exports = stats_factory( exports, # code=network_region_code or network.code, network=time_series.network, period=human_to_period("7d"), interval=human_to_interval("5m"), units=get_unit("power"), region=network_region_code, fueltech_group=True, ) result.append_set(result_exports) return result
def gov_stats_cpi() -> Optional[OpennemDataSet]: engine = get_database_engine() query = country_stats_query(StatTypes.CPI) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in row ] if len(stats) < 1: logger.error("No results for gov_stats_cpi returing blank set") return None result = stats_factory( stats, code="au.cpi", network=NetworkNEM, interval=human_to_interval("1Q"), period=human_to_period("all"), units=get_unit("cpi"), group_field="gov", ) return result
def values(self) -> List[Tuple[datetime, float]]: interval_obj = get_human_interval(self.interval) interval_def = human_to_interval(self.interval) inclusive = False dt = self.start if interval_def.interval < 1440: inclusive = True # return as list rather than generate timeseries_data = [] # rewind back one interval if inclusive: # dt -= interval_obj pass for v in self.data: timeseries_data.append((dt, v)) dt = dt + interval_obj # @TODO do some sanity checking here # if dt != self.last: # raise Exception( # "Mismatch between start, last and interval size. Got {} and {}".format( # dt, self.last # ) # ) return timeseries_data
def power_unit( unit_code: str = Query(..., description="Unit code"), network_code: str = Query(..., description="Network code"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") stats = [] facility_codes = [normalize_duid(unit_code)] query = power_facility_query(facility_codes, network.code, interval=interval, period=period) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Unit stats not found", ) output = stats_factory( stats, code=unit_code, interval=interval, period=period, units=units, network=network, ) if not output: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No stats found", ) return output
def energy_network_api( engine=Depends(get_database_engine), network_code: str = Query(..., description="Network code"), interval_human: str = Query("1d", description="Interval"), period_human: str = Query("1Y", description="Period"), ) -> OpennemDataSet: results = [] network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("energy_giga") query = energy_network(network=network, interval=interval, period=period) with engine.connect() as c: results = list(c.execute(query)) if len(results) < 1: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No results") stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period, units=units, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def get_date_range(network: NetworkSchema) -> DatetimeRange: date_range = get_scada_range(network=NetworkNEM) time_series = TimeSeries( start=date_range.start, end=date_range.end, interval=human_to_interval("1d"), period=human_to_period("all"), network=network, ) return time_series.get_range()
def demand_week( time_series: TimeSeries, network_region_code: Optional[str], networks_query: Optional[List[NetworkSchema]] = None, ) -> Optional[OpennemDataSet]: engine = get_database_engine() query = network_demand_query( time_series=time_series, network_region=network_region_code, networks_query=networks_query, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: logger.error( "No results from network_demand_query with {}".format(time_series)) return None demand = [ DataQueryResult(interval=i[0], result=i[2], group_by="demand" if len(i) > 1 else None) for i in row ] result = stats_factory( demand, # code=network_region_code or network.code, network=time_series.network, period=human_to_period("7d"), interval=human_to_interval("5m"), units=get_unit("demand"), region=network_region_code, ) if not result: logger.error( "No results from network_demand_query with {}".format(time_series)) return None return result
def get_power_example() -> OpennemDataSet: network = network_from_network_code("NEM") interval = human_to_interval("5m") units = get_unit("power") period = human_to_period("7d") network_region_code = "NSW1" test_rows = [] dt = datetime.fromisoformat("2021-01-15 10:00:00") for ft in ["coal_black", "coal_brown"]: for v in range(0, 3): test_rows.append([dt, ft, v]) dt = dt + timedelta(minutes=5) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in test_rows ] assert len(stats) == 6, "Should have 6 stats" result = stats_factory( stats, code=network_region_code or network.code, network=network, interval=interval, period=period, units=units, region=network_region_code, fueltech_group=True, ) if not result: raise Exception("Bad unit test data") with open("power-nsw1.json", "w") as fh: fh.write(result.json(indent=4)) return result
def test_nem_nsw1_201512() -> None: query_result = get_test_fixture("emissions_nem_nsw1_201512.csv") region_flows = [ RegionFlowEmissionsResult( interval=i["trading_interval"], flow_from=i["flow_from"], flow_to=i["flow_to"], energy=i["energy"], flow_from_emissions=i["flow_from_emissions"], flow_to_emissions=i["flow_to_emissions"], ) for i in query_result ] interval = human_to_interval("1M") flows = net_flows_emissions("NSW1", region_flows, interval) print(flows)
def energy_station( engine=Depends(get_database_engine), session: Session = Depends(get_database_session), network_code: str = Query(..., description="Network code"), station_code: str = Query(..., description="Station Code"), interval: str = Query(None, description="Interval"), period: str = Query("7d", description="Period"), ) -> OpennemDataSet: """ Get energy output for a station (list of facilities) over a period """ network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval: # @NOTE rooftop data is 15m if station_code.startswith("ROOFTOP"): interval = "15m" else: interval = "{}m".format(network.interval_size) interval_obj = human_to_interval(interval) period_obj = human_to_period(period) units = get_unit("energy") station = ( session.query(Station) .join(Station.facilities) .filter(Station.code == station_code) .filter(Facility.network_id == network.code) .one_or_none() ) if not station: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Station not found") if len(station.facilities) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station has no facilities", ) facility_codes = list(set([f.code for f in station.facilities])) query = energy_facility_query( facility_codes, network=network, interval=interval_obj, period=period_obj, ) logger.debug(query) with engine.connect() as c: row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) results_energy = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[2] if len(i) > 1 else None) for i in row ] results_market_value = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[3] if len(i) > 1 else None) for i in row ] results_emissions = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[4] if len(i) > 1 else None) for i in row ] if len(results_energy) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) stats = stats_factory( stats=results_energy, units=units, network=network, interval=interval_obj, period=period_obj, code=station_code, include_group_code=True, ) if not stats: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) stats_market_value = stats_factory( stats=results_market_value, units=get_unit("market_value"), network=network, interval=interval_obj, period=period_obj, code=station_code, include_group_code=True, ) stats.append_set(stats_market_value) stats_emissions = stats_factory( stats=results_emissions, units=get_unit("emissions"), network=network, interval=interval_obj, period=period_obj, code=network.code.lower(), include_group_code=True, ) stats.append_set(stats_emissions) return stats
def power_network_fueltech_api( network_code: str = Query(..., description="Network code"), network_region: str = Query(None, description="Network region"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") scada_range = get_scada_range(network=network) query = power_network_fueltech( network=network, interval=interval, period=period, network_region=network_region, scada_range=scada_range, ) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period, units=units, region=network_region, fueltech_group=True, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def power_station( station_code: str = Query(..., description="Station code"), network_code: str = Query(..., description="Network code"), since: datetime = Query(None, description="Since time"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), session: Session = Depends(get_database_session), engine=Depends(get_database_engine), ) -> OpennemDataSet: if not since: since = datetime.now() - human_to_timedelta("7d") network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: # @NOTE rooftop data is 15m if station_code.startswith("ROOFTOP"): interval_human = "15m" else: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") station = ( session.query(Station) .join(Facility) .filter(Station.code == station_code) .filter(Facility.network_id == network.code) .filter(Station.approved.is_(True)) .one_or_none() ) if not station: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Station not found") facility_codes = list(set([f.code for f in station.facilities])) stats = [] query = power_facility_query(facility_codes, network=network, interval=interval, period=period) logger.debug(query) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats, code=station_code, network=network, interval=interval, period=period, include_group_code=True, units=units, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def power_week( time_series: TimeSeries, network_region_code: str = None, networks_query: Optional[List[NetworkSchema]] = None, include_capacities: bool = False, include_code: Optional[bool] = True, ) -> Optional[OpennemDataSet]: engine = get_database_engine() query = power_network_fueltech_query( time_series=time_series, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] if len(stats) < 1: logger.error("No results from power week query with {}".format(time_series)) return None result = stats_factory( stats, # code=network_region_code or network.code, network=time_series.network, interval=time_series.interval, period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, include_code=include_code, ) if not result: logger.error("No results from power week status factory with {}".format(time_series)) return None if include_capacities and network_region_code: region_fueltech_capacities = get_facility_capacities( time_series.network, network_region_code ) for ft in result.data: if ft.fuel_tech in region_fueltech_capacities: ft.x_capacity_at_present = region_fueltech_capacities[ft.fuel_tech] # price time_series_price = time_series.copy() time_series_price.interval = human_to_interval("30m") query = price_network_query( time_series=time_series_price, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) stats_price = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] stats_market_value = stats_factory( stats=stats_price, code=network_region_code or time_series.network.code.lower(), units=get_unit("price_energy_mega"), network=time_series.network, interval=human_to_interval("30m"), region=network_region_code, period=time_series.period, include_code=include_code, ) result.append_set(stats_market_value) # rooftop solar time_series_rooftop = time_series.copy() time_series_rooftop.interval = human_to_interval("30m") query = power_network_rooftop_query( time_series=time_series_rooftop, networks_query=networks_query, network_region=network_region_code, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) rooftop_power = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] rooftop = stats_factory( rooftop_power, # code=network_region_code or network.code, network=time_series.network, interval=human_to_interval("30m"), period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, include_code=include_code, cast_nulls=False, ) # rooftop forecast rooftop_forecast = None if rooftop and rooftop.data and len(rooftop.data) > 0: time_series_rooftop_forecast = time_series_rooftop.copy() time_series_rooftop_forecast.start = rooftop.data[0].history.last time_series_rooftop_forecast.forecast = True query = power_network_rooftop_query( time_series=time_series_rooftop_forecast, networks_query=networks_query, network_region=network_region_code, forecast=True, ) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) rooftop_forecast_power = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] rooftop_forecast = stats_factory( rooftop_forecast_power, # code=network_region_code or network.code, network=time_series.network, interval=human_to_interval("30m"), period=time_series.period, units=get_unit("power"), region=network_region_code, fueltech_group=True, include_code=include_code, cast_nulls=False, ) if rooftop and rooftop_forecast: if ( hasattr(rooftop, "data") and len(rooftop.data) > 0 and rooftop_forecast.data and len(rooftop_forecast.data) > 0 ): rooftop.data[0].forecast = rooftop_forecast.data[0].history result.append_set(rooftop) return result
def generate_weekly_export_map() -> StatMetadata: """ Generate export map for weekly power series @TODO deconstruct this into separate methods and schema ex. network.get_scada_range(), network_region.get_bom_station() etc. """ session = get_scoped_session() networks = session.query(Network).filter( Network.export_set.is_(True)).all() if not networks: raise Exception("No networks") countries = list(set([network.country for network in networks])) _exmap = [] # Loop countries for country in countries: # @TODO derive this scada_range = get_scada_range(network=NetworkAU, networks=[NetworkNEM, NetworkWEM]) if not scada_range: raise Exception("Require a scada range for NetworkAU") for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=country, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], year=year, week=week, date_range=date_range_from_week(year, week, NetworkAU), interval=human_to_interval("30m"), period=human_to_period("7d"), ) _exmap.append(export) # Loop networks for network in networks: network_schema = network_from_network_code(network.code) scada_range = get_scada_range(network=network_schema) if not scada_range: raise Exception("Require a scada range for network: {}".format( network.code)) for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=network.country, network=network_schema, year=year, week=week, date_range=date_range_from_week(year, week, NetworkAU), interval=human_to_interval(f"{network.interval_size}m"), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) # Skip cases like wem/wem where region is supurfelous if len(network.regions) < 2: continue for region in network.regions: scada_range = get_scada_range(network=network_schema, network_region=region.code) if not scada_range: logger.error( "Require a scada range for network {} and region {}". format(network_schema.code, region.code)) continue for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=network.country, network=network_schema, year=year, week=week, date_range=date_range_from_week( year, week, network_from_network_code(network.code)), interval=human_to_interval(f"{network.interval_size}m"), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export_meta = StatMetadata(date_created=datetime.now(), version=get_version(), resources=_exmap) return export_meta
period=human_to_period("7d"), ), # Also testing timezone shift from UTC to NEM time datetime.fromisoformat("2021-01-08 22:45:00+10:00"), datetime.fromisoformat("2021-01-15 22:45:00+10:00"), "5m", 2017, # number of 5 minute intervals in a year ), # Years ( TimeSeries( start=datetime.fromisoformat("1997-05-05 12:45:00+10:00"), end=datetime.fromisoformat("2021-01-15 12:45:00+10:00"), network=NetworkNEM, year=2021, interval=human_to_interval("1d"), period=human_to_period("1Y"), ), # Expected datetime.fromisoformat("2021-01-01 00:00:00+10:00"), datetime.fromisoformat("2021-01-15 00:00:00+10:00"), "1d", 15, ), ( TimeSeries( start=datetime.fromisoformat("1997-05-05 12:45:00+00:00"), end=datetime.fromisoformat("2021-02-15 02:45:00+00:00"), network=NetworkNEM, year=2021, interval=human_to_interval("1d"),
def energy_network_fueltech_api( network_code: str = Query(None, description="Network code"), network_region: str = Query(None, description="Network region"), interval_human: str = Query("1d", description="Interval"), year: int = Query(None, description="Year to query"), period_human: str = Query("1Y", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) interval = human_to_interval(interval_human) period_obj: TimePeriod = human_to_period("1Y") if period_human: period_obj = human_to_period(period_human) units = get_unit("energy_giga") query = "" if year and isinstance(year, int): period_obj = human_to_period("1Y") if year > datetime.now().year or year < 1996: raise HTTPException( status_code=status.HTTP_406_NOT_ACCEPTABLE, detail="Not a valid year", ) scada_range = get_scada_range(network=network) query = energy_network_fueltech_year( network=network, interval=interval, year=year, network_region=network_region, scada_range=scada_range, ) elif period_obj and period_obj.period_human == "all": scada_range = get_scada_range(network=network) query = energy_network_fueltech_all( network=network, network_region=network_region, scada_range=scada_range, ) else: query = energy_network_fueltech( network=network, interval=interval, period=period_obj, network_region=network_region, ) # print(query) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Energy stats not found", ) result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period_obj, units=units, region=network_region, fueltech_group=True, ) if not result: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No stats") return result
def emission_factor_per_network( # type: ignore engine=Depends(get_database_engine), # type: ignore network_code: str = Query(..., description="Network code"), interval: str = Query("30m", description="Interval size"), ) -> Optional[OpennemDataSet]: engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval(interval) period_obj = human_to_period("7d") if not interval_obj: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Invalid interval size") scada_range = get_scada_range(network=network) if not scada_range: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) query = emission_factor_region_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) emission_factors = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( emission_factors, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("emissions_factor"), group_field="emission_factor", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result
def export_all_monthly() -> None: session = get_scoped_session() all_monthly = OpennemDataSet(code="au", data=[], version=get_version(), created_at=datetime.now()) cpi = gov_stats_cpi() all_monthly.append_set(cpi) # Iterate networks and network regions networks = [NetworkNEM, NetworkWEM] for network in networks: network_regions = session.query(NetworkRegion).filter( NetworkRegion.network_id == network.code).all() for network_region in network_regions: networks = [] logging.info( "Exporting monthly for network {} and region {}".format( network.code, network_region.code)) if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] if network == NetworkNEM: networks = [NetworkNEM, NetworkAEMORooftop] logger.debug("Running monthlies for {} and {}".format( network.code, network_region.code)) scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks, energy=True) if not scada_range or not scada_range.start: logger.error( "Could not get scada range for network {} and energy {}". format(network, True)) continue time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1M"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue demand_energy_and_value = demand_network_region_daily( time_series=time_series, network_region_code=network_region.code, networks=networks) stat_set.append_set(demand_energy_and_value) if network == NetworkNEM: interconnector_flows = energy_interconnector_flows_and_emissions( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) all_monthly.append_set(stat_set) bom_station = get_network_region_weather_station( network_region.code) if bom_station: try: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) all_monthly.append_set(weather_stats) except Exception: pass write_output("v3/stats/au/all/monthly.json", all_monthly)
def export_all_daily( networks: List[NetworkSchema] = [NetworkNEM, NetworkWEM], network_region_code: Optional[str] = None, ) -> None: session = get_scoped_session() cpi = gov_stats_cpi() for network in networks: network_regions = session.query(NetworkRegion).filter_by( export_set=True).filter_by(network_id=network.code) if network_region_code: network_regions = network_regions.filter_by( code=network_region_code) network_regions = network_regions.all() for network_region in network_regions: logging.info("Exporting for network {} and region {}".format( network.code, network_region.code)) networks = [ NetworkNEM, NetworkAEMORooftop, NetworkAEMORooftopBackfill ] if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks, energy=True) if not scada_range or not scada_range.start: logger.error( "Could not get scada range for network {} and energy {}". format(network, True)) continue time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1d"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue demand_energy_and_value = demand_network_region_daily( time_series=time_series, network_region_code=network_region.code, networks=networks) stat_set.append_set(demand_energy_and_value) # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if network == NetworkNEM: interconnector_flows = energy_interconnector_flows_and_emissions( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) bom_station = get_network_region_weather_station( network_region.code) if bom_station: try: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) stat_set.append_set(weather_stats) except Exception: pass if cpi: stat_set.append_set(cpi) write_output(f"v3/stats/au/{network_region.code}/daily.json", stat_set)
def station_observations_api( station_code: str = Query(None, description="Station code"), interval_human: str = Query("15m", description="Interval"), period_human: str = Query("7d", description="Period"), station_codes: List[str] = [], timezone: str = None, offset: str = None, year: int = None, engine=Depends(get_database_engine), ) -> OpennemDataSet: units = get_unit("temperature") if not interval_human: interval_human = "15m" if not period_human: period_human = "7d" if station_code: station_codes = [station_code] interval = human_to_interval(interval_human) period = human_to_period(period_human) if timezone: timezone = ZoneInfo(timezone) if offset: timezone = get_fixed_timezone(offset) query = observation_query( station_codes=station_codes, interval=interval, period=period, year=year, ) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats=stats, units=units, interval=interval, period=period, code="bom", group_field="temperature", ) return result
def get_range(self) -> DatetimeRange: """Return a DatetimeRange from the time series for queries""" start = self.start end = self.end # If its a forward looking forecast # jump out early if self.forecast: start = self.end + timedelta(minutes=self.interval.interval) end = self.end + get_human_interval(self.forecast_period) start = start.astimezone(self.network.get_fixed_offset()) end = end.astimezone(self.network.get_fixed_offset()) return DatetimeRange(start=start, end=end, interval=self.interval) # subtract the period (ie. 7d from the end for start if not all) if self.period == human_to_period("all"): start = date_trunc(start, self.interval.trunc) start = start.replace( hour=0, minute=0, second=0, tzinfo=self.network.get_fixed_offset() ) # If its all per month take the end of the last month if self.interval == human_to_interval("1M"): end = date_trunc(get_end_of_last_month(end), "day") end = end.replace( hour=23, minute=59, second=59, tzinfo=self.network.get_fixed_offset() ) self.year = None else: start = self.end - get_human_interval(self.period.period_human) if self.year: if self.year > end.year: raise Exception("Specified year is great than end year") start = start.replace( year=self.year, month=1, day=1, hour=0, minute=0, second=0, tzinfo=self.network.get_fixed_offset(), ) end = datetime( year=self.year, month=12, day=31, hour=23, minute=59, second=59, tzinfo=self.network.get_fixed_offset(), ) if self.year == CUR_YEAR: today = datetime.now(tz=self.network.get_fixed_offset()) end = datetime( year=CUR_YEAR, month=today.month, day=today.day, hour=23, minute=59, second=59 ) end = end - timedelta(days=1) end = end.replace(tzinfo=self.network.get_fixed_offset()) if self.end.date() < today.date(): end = self.end if self.month: start = datetime( year=self.month.year, month=self.month.month, day=1, hour=0, minute=0, second=0, tzinfo=self.network.get_fixed_offset(), ) end = start + get_human_interval("1M") - timedelta(days=1) end = end.replace( hour=23, minute=59, second=59, ) # localize times if not start.tzinfo or start.tzinfo != self.network.get_fixed_offset(): start = start.astimezone(self.network.get_fixed_offset()) if not end.tzinfo or end.tzinfo != self.network.get_fixed_offset(): end = end.astimezone(self.network.get_fixed_offset()) dtr = DatetimeRange(start=start, end=end, interval=self.interval) return dtr
def price_network_endpoint( engine: Engine = Depends(get_database_engine), network_code: str = Path(..., description="Network code"), network_region: Optional[str] = Query(None, description="Network region code"), forecasts: bool = Query(False, description="Include price forecasts"), ) -> OpennemDataSet: """Returns network and network region price info for interval which defaults to network interval size Args: engine ([type], optional): Database engine. Defaults to Depends(get_database_engine). Raises: HTTPException: No results Returns: OpennemData: data set """ engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval("5m") period_obj = human_to_period("1d") scada_range = get_balancing_range(network=network, include_forecasts=forecasts) if not scada_range: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) if network_region: time_series.network.regions = [NetworkNetworkRegion(code=network_region)] query = network_region_price_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) result_set = [ DataQueryResult(interval=i[0], result=i[3], group_by=i[2] if len(i) > 1 else None) for i in row ] result = stats_factory( result_set, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("price"), group_field="price", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result
def export_all_daily() -> None: session = SessionLocal() network_regions = session.query(NetworkRegion).all() cpi = gov_stats_cpi() for network_region in network_regions: network = network_from_network_code(network_region.network.code) networks = None if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks) time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1d"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if network == NetworkNEM: interconnector_flows = energy_interconnector_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) interconnector_emissions = energy_interconnector_emissions_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_emissions) bom_station = get_network_region_weather_station(network_region.code) if bom_station: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) stat_set.append_set(weather_stats) if cpi: stat_set.append_set(cpi) write_output(f"v3/stats/au/{network_region.code}/daily.json", stat_set)
def fueltech_demand_mix( engine=Depends(get_database_engine), # type: ignore network_code: str = Query(..., description="Network code"), ) -> OpennemDataSet: """Return fueltech proportion of demand for a network Args: engine ([type], optional): Database engine. Defaults to Depends(get_database_engine). Raises: HTTPException: No results Returns: OpennemData: data set """ engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval("5m") period_obj = human_to_period("1d") scada_range = get_scada_range(network=network) if not scada_range: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) query = network_fueltech_demand_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) result_set = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( result_set, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("emissions_factor"), group_field="emission_factor", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result
def export_all_monthly() -> None: session = SessionLocal() network_regions = session.query(NetworkRegion).all() all_monthly = OpennemDataSet(code="au", data=[], version=get_version(), created_at=datetime.now()) cpi = gov_stats_cpi() all_monthly.append_set(cpi) for network_region in network_regions: network = network_from_network_code(network_region.network.code) networks = None if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks) time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1M"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue if network == NetworkNEM: interconnector_flows = energy_interconnector_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) interconnector_emissions = energy_interconnector_emissions_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_emissions) all_monthly.append_set(stat_set) bom_station = get_network_region_weather_station(network_region.code) if bom_station: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) all_monthly.append_set(weather_stats) write_output("v3/stats/au/all/monthly.json", all_monthly)
def stats_factory( stats: List[DataQueryResult], units: UnitDefinition, interval: TimeInterval, period: Optional[TimePeriod] = None, network: Optional[NetworkSchema] = None, timezone: Optional[Union[timezone, str]] = None, code: Optional[str] = None, region: Optional[str] = None, include_group_code: bool = False, fueltech_group: Optional[bool] = False, group_field: Optional[str] = None, data_id: Optional[str] = None, localize: Optional[bool] = True, include_code: Optional[bool] = True, cast_nulls: Optional[bool] = True, ) -> Optional[OpennemDataSet]: """ Takes a list of data query results and returns OpennemDataSets @TODO optional groupby field @TODO multiple groupings / slight refactor """ if network: timezone = network.get_timezone() group_codes = list(set([i.group_by for i in stats if i.group_by])) stats_grouped = [] for group_code in group_codes: data_grouped: Dict[datetime, Any] = dict() for stat in stats: if stat.group_by != group_code: continue if stat.interval not in data_grouped: data_grouped[stat.interval] = None # if stat.result: data_grouped[stat.interval] = stat.result data_sorted = OrderedDict(sorted(data_grouped.items())) data_value = list(data_sorted.values()) # Skip null series if len([i for i in data_value if i]) == 0: continue # @TODO possible bring this back # Skip zero series # if sum([i for i in data_value if i]) == 0: # continue # Cast trailing nulls if (not units.name.startswith("temperature") or (units.cast_nulls is True)) and (cast_nulls is True): data_value = cast_trailing_nulls(data_value) data_trimmed = dict(zip(data_sorted.keys(), data_value)) data_trimmed = trim_nulls(data_trimmed) # Find start/end dates dates = list(data_trimmed.keys()) if not dates: return None start = min(dates) end = max(dates) # should probably make sure these are the same TZ if localize: if timezone and not is_aware(start): start = make_aware(start, timezone) if timezone and not is_aware(end): end = make_aware(end, timezone) if timezone and localize and network and network.offset: tz = network.get_timezone() if tz: start = start.astimezone(tz) end = end.astimezone(tz) # Everything needs a timezone even flat dates if network and timezone and not is_aware(start): start = start.replace(tzinfo=network.get_fixed_offset()) if network and timezone and not is_aware(end): end = end.replace(tzinfo=network.get_fixed_offset()) # @TODO compose this and make it generic - some intervals # get truncated. # trunc the date for days and months if interval == human_to_interval("1d"): start = date_trunc(start, truncate_to="day") end = date_trunc(end, truncate_to="day") if interval == human_to_interval("1M"): start = date_trunc(start, truncate_to="month") end = date_trunc(end, truncate_to="month") # free dates = [] history = OpennemDataHistory( start=start, last=end, interval=interval.interval_human, data=data_trimmed.values(), ) data = OpennemData( data_type=units.unit_type, units=units.unit, # interval=interval, # period=period, history=history, ) if include_code: data.code = group_code if network: data.network = network.code.lower() # *sigh* - not the most flexible model # @TODO fix this schema and make it more flexible if fueltech_group: data.fuel_tech = group_code data_comps = [ # @NOTE disable for now since FE doesn't # support it network.country if network else None, network.code.lower() if network else None, region.lower() if region and region.lower() != network.code.lower() else None, "fuel_tech", group_code, units.unit_type, ] data.id = ".".join(i for i in data_comps if i) # @TODO make this an alias data.type = units.unit_type if group_field: group_fields = [] # setattr(data, group_field, group_code) if network: group_fields.append(network.country.lower()) group_fields.append(network.code.lower()) if region: if region.lower() != network.code.lower(): group_fields.append(region.lower()) if units.name_alias: group_fields.append(units.name_alias) elif units.unit_type: group_fields.append(units.unit_type) if group_code and include_group_code: group_fields.append(group_code) group_fields.append(group_field) data.id = ".".join([f for f in group_fields if f]) data.type = units.unit_type if data_id: data.id = data_id if not data.id: _id_list = [] # @NOTE disable for now since FE doesn't # support it # network.country if network else None, if network: _id_list.append(network.code.lower()) if region and (region.lower() != network.code.lower()): _id_list.append(region.lower()) if group_code: _id_list.append(group_code.lower()) if units and units.name_alias: _id_list.append(units.name_alias) elif units and units.name: _id_list.append(units.name) data.id = ".".join([f for f in _id_list if f]) data.type = units.unit_type if region: data.region = region stats_grouped.append(data) dt_now = datetime.now() if network: dt_now = dt_now.astimezone(network.get_timezone()) # @NOTE this should probably be # country.network.region if not code: if network: code = network.code if region: code = region stat_set = OpennemDataSet( type=units.unit_type, data=stats_grouped, created_at=dt_now, version=get_version(), ) if include_code: stat_set.code = code if network: stat_set.network = network.code if region: stat_set.region = region return stat_set
def price_network_region_api( engine=Depends(get_database_engine), network_code: str = Query(..., description="Network code"), network_region_code: str = Query(..., description="Region code"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), year: Optional[int] = None, ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period_obj = None if period_human: period_obj = human_to_period(period_human) units = get_unit("price") scada_range = get_scada_range(network=network) if period_obj and period_obj.period_human == "all" and interval.interval_human == "1M": query = price_network_monthly( network=network, network_region_code=network_region_code, scada_range=scada_range, ) else: query = price_network_region( network=network, network_region_code=network_region_code, interval=interval, period=period_obj, scada_range=scada_range, year=year, ) with engine.connect() as c: results = list(c.execute(query)) if len(results) < 1: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No data found") stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in results ] result = stats_factory( stats, code=network.code, region=network_region_code, network=network, interval=interval, period=period_obj, units=units, group_field="price", ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def export_energy( stats: List[StatExport] = None, priority: Optional[PriorityType] = None, latest: Optional[bool] = False, ) -> None: """ Export energy stats from the export map """ if not stats: export_map = get_export_map().get_by_stat_type(StatType.energy) if priority: export_map = export_map.get_by_priority(priority) stats = export_map.resources CURRENT_YEAR = datetime.now().year logger.info(f"Running export_energy with {len(stats)} stats") for energy_stat in stats: if energy_stat.stat_type != StatType.energy: continue # @FIX trim to NEM since it's the one with the shortest # data time span. # @TODO find a better and more flexible way to do this in the # range method date_range_networks = energy_stat.networks or [] if NetworkNEM in date_range_networks: date_range_networks = [NetworkNEM] date_range: ScadaDateRange = get_scada_range( network=energy_stat.network, networks=date_range_networks, energy=True) if not date_range: logger.error( "Skipping - Could not get date range for energy {} {}".format( energy_stat.network, date_range_networks)) continue logger.debug("Date range is: {} {} => {}".format( energy_stat.network.code, date_range.start, date_range.end)) # Migrate to this time_series time_series = TimeSeries( start=date_range.start, end=date_range.end, network=energy_stat.network, year=energy_stat.year, interval=energy_stat.interval, period=human_to_period("1Y"), ) if energy_stat.year: if latest and energy_stat.year != CURRENT_YEAR: continue stat_set = energy_fueltech_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) if not stat_set: continue demand_energy_and_value = demand_network_region_daily( time_series=time_series, network_region_code=energy_stat.network_region, networks=energy_stat.networks) stat_set.append_set(demand_energy_and_value) # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if energy_stat.network == NetworkNEM and energy_stat.network_region: interconnector_flows = energy_interconnector_flows_and_emissions( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) stat_set.append_set(interconnector_flows) if energy_stat.bom_station: try: weather_stats = weather_daily( time_series=time_series, station_code=energy_stat.bom_station, network_region=energy_stat.network_region, ) stat_set.append_set(weather_stats) except NoResults as e: logger.info("No results for weather result: {}".format(e)) except Exception as e: logger.error("weather_stat exception: {}".format(e)) pass else: logger.info("Stat set has no bom station") write_output(energy_stat.path, stat_set) elif energy_stat.period and energy_stat.period.period_human == "all" and not latest: time_series.period = human_to_period("all") time_series.interval = human_to_interval("1M") time_series.year = None stat_set = energy_fueltech_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) if not stat_set: continue demand_energy_and_value = demand_network_region_daily( time_series=time_series, network_region_code=energy_stat.network_region, networks=energy_stat.networks) stat_set.append_set(demand_energy_and_value) # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if energy_stat.network == NetworkNEM and energy_stat.network_region: interconnector_flows = energy_interconnector_flows_and_emissions( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) stat_set.append_set(interconnector_flows) if energy_stat.bom_station: try: weather_stats = weather_daily( time_series=time_series, station_code=energy_stat.bom_station, network_region=energy_stat.network_region, ) stat_set.append_set(weather_stats) except NoResults as e: logger.info("No weather results: {}".format(e)) except Exception: pass write_output(energy_stat.path, stat_set)
def get_export_map() -> StatMetadata: """ Generates a map of all export JSONs """ session = SessionLocal() networks = session.query(Network).filter(Network.export_set.is_(True)).all() if not networks: raise Exception("No networks") countries = list(set([network.country for network in networks])) _exmap = [] for country in countries: # @TODO derive this scada_range = get_scada_range(network=NetworkAU, networks=[NetworkNEM, NetworkWEM]) if not scada_range: raise Exception("Require a scada range") export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], interval=NetworkAU.get_interval(), period=human_to_period("7d"), ) _exmap.append(export) for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], year=year, interval=human_to_interval("1d"), period=human_to_period("1Y"), ) _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], interval=human_to_interval("1M"), period=human_to_period("all"), ) _exmap.append(export) for network in networks: network_schema = network_from_network_code(network.code) scada_range = get_scada_range(network=network_schema) bom_station = get_network_region_weather_station(network.code) export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, interval=network_schema.get_interval(), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) if not scada_range: raise Exception("Require a scada range") for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, year=year, period=human_to_period("1Y"), interval=human_to_interval("1d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, interval=human_to_interval("1M"), period=human_to_period("all"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) # Skip cases like wem/wem where region is supurfelous if len(network.regions) < 2: continue for region in network.regions: scada_range = get_scada_range(network=network_schema, network_region=region) bom_station = get_network_region_weather_station(region.code) if not scada_range: raise Exception("Require a scada range") export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, period=human_to_period("7d"), interval=network_schema.get_interval(), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, year=year, period=human_to_period("1Y"), interval=human_to_interval("1d"), ) _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, period=human_to_period("all"), interval=human_to_interval("1M"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export_meta = StatMetadata( date_created=datetime.now(), version=get_version(), resources=_exmap ) return export_meta
def export_power( stats: List[StatExport] = None, priority: Optional[PriorityType] = None, latest: Optional[bool] = False, ) -> None: """ Export power stats from the export map """ # Not passed a stat map so go and get one if not stats: export_map = None if priority and priority == PriorityType.history: export_map = get_weekly_export_map() else: export_map = get_export_map().get_by_stat_type(StatType.power) if priority: export_map = export_map.get_by_priority(priority) stats = export_map.resources output_count: int = 0 logger.info("Running {}export {} with {} stats".format( "latest " if latest else "", priority, len(stats))) for power_stat in stats: if power_stat.stat_type != StatType.power: continue if output_count >= 1 and latest: return None date_range_networks = power_stat.networks or [] if NetworkNEM in date_range_networks: date_range_networks = [NetworkNEM] date_range: ScadaDateRange = get_scada_range( network=power_stat.network, networks=date_range_networks) logger.debug("Date range for {}: {} => {}".format( power_stat.network.code, date_range.start, date_range.end)) # Migrate to this time_series time_series = TimeSeries( start=date_range.start, end=date_range.end, network=power_stat.network, year=power_stat.year, interval=power_stat.interval, period=power_stat.period, ) stat_set = power_week( time_series=time_series, network_region_code=power_stat.network_region_query or power_stat.network_region, networks_query=power_stat.networks, ) if not stat_set: logger.info("No power stat set for {} {} {}".format( power_stat.period, power_stat.networks, power_stat.network_region, )) continue demand_set = demand_week( time_series=time_series, networks_query=power_stat.networks, network_region_code=power_stat.network_region_query or power_stat.network_region, ) stat_set.append_set(demand_set) if power_stat.network_region: flow_set = power_flows_region_week( time_series=time_series, network_region_code=power_stat.network_region, ) if flow_set: stat_set.append_set(flow_set) time_series_weather = time_series.copy() time_series_weather.interval = human_to_interval("30m") if power_stat.bom_station: try: weather_set = weather_daily( time_series=time_series_weather, station_code=power_stat.bom_station, network_region=power_stat.network_region, include_min_max=False, unit_name="temperature", ) stat_set.append_set(weather_set) except Exception: pass write_output(power_stat.path, stat_set) output_count += 1