def run_energy_gapfill_for_network(network: NetworkSchema = NetworkNEM, days: int = 7, run_all: bool = False) -> None: energy_gaps = query_energy_gaps(network, days=days) energy_gaps_filtered = list( filter(lambda x: x.has_power is True and x.has_energy is False, energy_gaps)) if run_all: energy_gaps_filtered = energy_gaps logger.info("Found {} energy gaps interval hours".format( len(energy_gaps_filtered))) for gap in energy_gaps_filtered: logger.info( "{} Running for interval {} with power: {} energy: {} energy_value: {}" .format(gap.network_id, gap.interval, gap.has_power, gap.has_energy, gap.total_energy)) if settings.dry_run: continue dmin = gap.interval dmax = dmin + timedelta(hours=1) try: run_energy_calc(dmin, dmax, network=network_from_network_code(gap.network_id)) except Exception: logger.error("Error running {} energy gapfill for {} => {}".format( gap.network_id, dmin, dmax))
def power_network_region_fueltech( network_code: str = Query(..., description="Network code"), network_region_code: str = Query(..., description="Network region code"), month: date = Query(datetime.now().date(), description="Month to query"), ) -> OpennemDataSet: network = network_from_network_code(network_code) interval_obj = network.get_interval() period_obj = human_to_period("1M") scada_range = get_scada_range(network=network) if not scada_range: raise Exception("Require a scada range") if not network: raise Exception("Network not found") time_series = TimeSeries( start=scada_range.start, month=month, network=network, interval=interval_obj, period=period_obj, ) stat_set = power_week(time_series, network_region_code, include_capacities=True) if not stat_set: raise Exception("No results") return stat_set
def power_unit( unit_code: str = Query(..., description="Unit code"), network_code: str = Query(..., description="Network code"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") stats = [] facility_codes = [normalize_duid(unit_code)] query = power_facility_query(facility_codes, network.code, interval=interval, period=period) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Unit stats not found", ) output = stats_factory( stats, code=unit_code, interval=interval, period=period, units=units, network=network, ) if not output: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No stats found", ) return output
def check_opennem_interval_delays(network_code: str) -> bool: """Runs periodically and alerts if there is a current delay in output of power intervals""" network = network_from_network_code(network_code) env = "" if settings.debug: env = ".dev" url = f"https://data{env}.opennem.org.au/v3/stats/au/{network.code}/power/7d.json" resp = http.get(url) if resp.status_code != 200 or not resp.ok: logger.error("Error retrieving: {}".format(url)) return False resp_json = resp.json() if "data" not in resp_json: logger.error("Error retrieving wem power: malformed response") return False data = resp_json["data"] fueltech_data = data.pop() history_end_date = fueltech_data["history"]["last"] history_date = parse_date(history_end_date, dayfirst=False) if not history_date: logger.error( "Could not read history date for opennem interval monitor") return False now_date = datetime.now().astimezone( network.get_timezone()) # type: ignore time_delta = chop_delta_microseconds(now_date - history_date) - timedelta( minutes=network.interval_size) logger.debug("Live time: {}, delay: {}".format(history_date, time_delta)) alert_threshold = network.monitor_interval_alert_threshold or settings.monitor_interval_alert_threshold or 60 if time_delta > timedelta(minutes=alert_threshold): slack_message( f"*WARNING*: OpenNEM {network.code} interval delay on {settings.env} currently: {time_delta}.\n", tag_users=settings.monitoring_alert_slack_user, ) return True
def energy_network_api( engine=Depends(get_database_engine), network_code: str = Query(..., description="Network code"), interval_human: str = Query("1d", description="Interval"), period_human: str = Query("1Y", description="Period"), ) -> OpennemDataSet: results = [] network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("energy_giga") query = energy_network(network=network, interval=interval, period=period) with engine.connect() as c: results = list(c.execute(query)) if len(results) < 1: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No results") stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period, units=units, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def power_network_region_fueltech( network_code: str = Query(..., description="Network code"), network_region_code: str = Query(..., description="Network region code"), month: date = Query(datetime.now().date(), description="Month to query"), ) -> OpennemDataSet: network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = network.get_interval() period_obj = human_to_period("1M") scada_range = get_scada_range(network=network) if not scada_range: raise Exception("Require a scada range") if not network: raise Exception("Network not found") networks = [network] if network == NetworkNEM: networks.append(NetworkAEMORooftop) networks.append(NetworkAEMORooftopBackfill) elif network == NetworkWEM: networks.append(NetworkAPVI) time_series = TimeSeries( start=scada_range.start, month=month, network=network, networks=networks, interval=interval_obj, period=period_obj, ) stat_set = power_week( time_series, network_region_code, include_capacities=True, networks_query=networks ) if not stat_set: raise Exception("No results") return stat_set
def check_opennem_interval_delays(network_code: str) -> bool: network = network_from_network_code(network_code) if settings.debug: env = ".dev" url = ( f"https://data.opennem.org.au/v3/stats/au/{network.code}/power/7d.json" f"https://data{env}.opennem.org.au/v3/stats/au/{network.code}/power/7d.json" ) resp = http.get(url) if resp.status_code != 200 or not resp.ok: logger.error("Error retrieving: {}".format(url)) return False resp_json = resp.json() if "data" not in resp_json: logger.error("Error retrieving wem power: malformed response") return False data = resp_json["data"] fueltech_data = data.pop() history_end_date = fueltech_data["history"]["last"] history_date = parse_date(history_end_date, dayfirst=False) if not history_date: logger.error( "Could not read history date for opennem interval monitor") return False now_date = datetime.now().astimezone(network.get_timezone()) time_delta = chop_microseconds(now_date - history_date) logger.debug("Live time: {}, delay: {}".format(history_date, time_delta)) if time_delta > timedelta(hours=3): slack_message( "*WARNING*: OpenNEM {} interval delay on {} currently: {}\n". format(network.code, settings.env, time_delta)) return True
def api_export_energy_year( engine: Any = Depends(get_database_engine), network_code: str = Query("WEM", description="Network code"), year: int = Query(YEAR_CURRENT, description="Year to query"), ) -> OpennemDataSet: if year > YEAR_CURRENT or year < 1996: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Invalid year" ) network: NetworkSchema = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Invalid network", ) stats = energy_network_fueltech_api( network_code=network.code, network_region=None, interval="1d", year=year, period="1Y", engine=engine, ) weather = station_observations_api( station_code="009021", interval="1d", year=year, network_code="WEM", engine=engine, period=None, ) price = price_network_region_api( engine=engine, network_code="WEM", network_region_code="WEM", interval="1d", period=None, year=year, ) stats.data += weather.data + price.data return stats
def get_power_example() -> OpennemDataSet: network = network_from_network_code("NEM") interval = human_to_interval("5m") units = get_unit("power") period = human_to_period("7d") network_region_code = "NSW1" test_rows = [] dt = datetime.fromisoformat("2021-01-15 10:00:00") for ft in ["coal_black", "coal_brown"]: for v in range(0, 3): test_rows.append([dt, ft, v]) dt = dt + timedelta(minutes=5) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in test_rows ] assert len(stats) == 6, "Should have 6 stats" result = stats_factory( stats, code=network_region_code or network.code, network=network, interval=interval, period=period, units=units, region=network_region_code, fueltech_group=True, ) if not result: raise Exception("Bad unit test data") with open("power-nsw1.json", "w") as fh: fh.write(result.json(indent=4)) return result
def api_export_energy_month( engine: Any = Depends(get_database_engine), network_code: str = Query("WEM", description="Network code"), month: str = Query("all", description="Month to query"), ) -> OpennemDataSet: network: NetworkSchema = network_from_network_code(network_code) if month != "all": raise HTTPException( status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Other months not yet supported", ) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Invalid network", ) stats = energy_network_fueltech_api( network_code=network.code, period="all", engine=engine, interval="1M", network_region=None, ) price = price_network_region_api( engine=engine, network_code="WEM", network_region_code="WEM", interval="1M", period="all", ) stats.data += price.data return stats
def energy_station( engine=Depends(get_database_engine), session: Session = Depends(get_database_session), network_code: str = Query(..., description="Network code"), station_code: str = Query(..., description="Station Code"), interval: str = Query(None, description="Interval"), period: str = Query("7d", description="Period"), ) -> OpennemDataSet: """ Get energy output for a station (list of facilities) over a period """ network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval: # @NOTE rooftop data is 15m if station_code.startswith("ROOFTOP"): interval = "15m" else: interval = "{}m".format(network.interval_size) interval_obj = human_to_interval(interval) period_obj = human_to_period(period) units = get_unit("energy") station = ( session.query(Station) .join(Station.facilities) .filter(Station.code == station_code) .filter(Facility.network_id == network.code) .one_or_none() ) if not station: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Station not found") if len(station.facilities) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station has no facilities", ) facility_codes = list(set([f.code for f in station.facilities])) query = energy_facility_query( facility_codes, network=network, interval=interval_obj, period=period_obj, ) logger.debug(query) with engine.connect() as c: row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) results_energy = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[2] if len(i) > 1 else None) for i in row ] results_market_value = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[3] if len(i) > 1 else None) for i in row ] results_emissions = [ DataQueryResult(interval=i[0], group_by=i[1], result=i[4] if len(i) > 1 else None) for i in row ] if len(results_energy) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) stats = stats_factory( stats=results_energy, units=units, network=network, interval=interval_obj, period=period_obj, code=station_code, include_group_code=True, ) if not stats: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) stats_market_value = stats_factory( stats=results_market_value, units=get_unit("market_value"), network=network, interval=interval_obj, period=period_obj, code=station_code, include_group_code=True, ) stats.append_set(stats_market_value) stats_emissions = stats_factory( stats=results_emissions, units=get_unit("emissions"), network=network, interval=interval_obj, period=period_obj, code=network.code.lower(), include_group_code=True, ) stats.append_set(stats_emissions) return stats
def export_all_monthly() -> None: session = SessionLocal() network_regions = session.query(NetworkRegion).all() all_monthly = OpennemDataSet(code="au", data=[], version=get_version(), created_at=datetime.now()) cpi = gov_stats_cpi() all_monthly.append_set(cpi) for network_region in network_regions: network = network_from_network_code(network_region.network.code) networks = None if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks) time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1M"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue if network == NetworkNEM: interconnector_flows = energy_interconnector_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) interconnector_emissions = energy_interconnector_emissions_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_emissions) all_monthly.append_set(stat_set) bom_station = get_network_region_weather_station(network_region.code) if bom_station: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) all_monthly.append_set(weather_stats) write_output("v3/stats/au/all/monthly.json", all_monthly)
def power_station( station_code: str = Query(..., description="Station code"), network_code: str = Query(..., description="Network code"), since: datetime = Query(None, description="Since time"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), session: Session = Depends(get_database_session), engine=Depends(get_database_engine), ) -> OpennemDataSet: if not since: since = datetime.now() - human_to_timedelta("7d") network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: # @NOTE rooftop data is 15m if station_code.startswith("ROOFTOP"): interval_human = "15m" else: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") station = ( session.query(Station) .join(Facility) .filter(Station.code == station_code) .filter(Facility.network_id == network.code) .filter(Station.approved.is_(True)) .one_or_none() ) if not station: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Station not found") facility_codes = list(set([f.code for f in station.facilities])) stats = [] query = power_facility_query(facility_codes, network=network, interval=interval, period=period) logger.debug(query) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats, code=station_code, network=network, interval=interval, period=period, include_group_code=True, units=units, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def power_network_fueltech_api( network_code: str = Query(..., description="Network code"), network_region: str = Query(None, description="Network region"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period = human_to_period(period_human) units = get_unit("power") scada_range = get_scada_range(network=network) query = power_network_fueltech( network=network, interval=interval, period=period, network_region=network_region, scada_range=scada_range, ) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period, units=units, region=network_region, fueltech_group=True, ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def emission_factor_per_network( # type: ignore engine=Depends(get_database_engine), # type: ignore network_code: str = Query(..., description="Network code"), interval: str = Query("30m", description="Interval size"), ) -> Optional[OpennemDataSet]: engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval(interval) period_obj = human_to_period("7d") if not interval_obj: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Invalid interval size") scada_range = get_scada_range(network=network) if not scada_range: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) query = emission_factor_region_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) emission_factors = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( emission_factors, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("emissions_factor"), group_field="emission_factor", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result
def fueltech_demand_mix( engine=Depends(get_database_engine), # type: ignore network_code: str = Query(..., description="Network code"), ) -> OpennemDataSet: """Return fueltech proportion of demand for a network Args: engine ([type], optional): Database engine. Defaults to Depends(get_database_engine). Raises: HTTPException: No results Returns: OpennemData: data set """ engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval("5m") period_obj = human_to_period("1d") scada_range = get_scada_range(network=network) if not scada_range: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) query = network_fueltech_demand_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) result_set = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( result_set, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("emissions_factor"), group_field="emission_factor", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result
def get_export_map() -> StatMetadata: """ Generates a map of all export JSONs """ session = SessionLocal() networks = session.query(Network).filter(Network.export_set.is_(True)).all() if not networks: raise Exception("No networks") countries = list(set([network.country for network in networks])) _exmap = [] for country in countries: # @TODO derive this scada_range = get_scada_range(network=NetworkAU, networks=[NetworkNEM, NetworkWEM]) if not scada_range: raise Exception("Require a scada range") export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], interval=NetworkAU.get_interval(), period=human_to_period("7d"), ) _exmap.append(export) for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], year=year, interval=human_to_interval("1d"), period=human_to_period("1Y"), ) _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], interval=human_to_interval("1M"), period=human_to_period("all"), ) _exmap.append(export) for network in networks: network_schema = network_from_network_code(network.code) scada_range = get_scada_range(network=network_schema) bom_station = get_network_region_weather_station(network.code) export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, interval=network_schema.get_interval(), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) if not scada_range: raise Exception("Require a scada range") for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, year=year, period=human_to_period("1Y"), interval=human_to_interval("1d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, interval=human_to_interval("1M"), period=human_to_period("all"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) # Skip cases like wem/wem where region is supurfelous if len(network.regions) < 2: continue for region in network.regions: scada_range = get_scada_range(network=network_schema, network_region=region) bom_station = get_network_region_weather_station(region.code) if not scada_range: raise Exception("Require a scada range") export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, period=human_to_period("7d"), interval=network_schema.get_interval(), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, year=year, period=human_to_period("1Y"), interval=human_to_interval("1d"), ) _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, period=human_to_period("all"), interval=human_to_interval("1M"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export_meta = StatMetadata( date_created=datetime.now(), version=get_version(), resources=_exmap ) return export_meta
def power_flows_network_week( engine=Depends(get_database_engine), # type: ignore network_code: str = Query(..., description="Network code"), month: date = Query(datetime.now().date(), description="Month to query"), ) -> Optional[OpennemDataSet]: engine = get_database_engine() network = network_from_network_code(network_code) interval_obj = network.get_interval() period_obj = human_to_period("1M") scada_range = get_scada_range(network=network) if not scada_range: raise Exception("Require a scada range") if not network: raise Exception("Network not found") time_series = TimeSeries( start=scada_range.start, month=month, network=network, interval=interval_obj, period=period_obj, ) query = interconnector_flow_network_regions_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise Exception("No results from query: {}".format(query)) imports = [ DataQueryResult(interval=i[0], result=i[4], group_by=i[1] if len(i) > 1 else None) for i in row ] result = stats_factory( imports, # code=network_region_code or network.code, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("regional_trade"), # fueltech_group=True, group_field="power", include_group_code=True, include_code=True, ) if not result or not result.data: raise Exception("No results") INVERT_SETS = ["VIC1->NSW1", "VIC1->SA1"] inverted_data = [] for ds in result.data: if ds.code in INVERT_SETS: ds_inverted = invert_flow_set(ds) inverted_data.append(ds_inverted) else: inverted_data.append(ds) result.data = inverted_data return result
def export_all_daily() -> None: session = SessionLocal() network_regions = session.query(NetworkRegion).all() cpi = gov_stats_cpi() for network_region in network_regions: network = network_from_network_code(network_region.network.code) networks = None if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks) time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1d"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if network == NetworkNEM: interconnector_flows = energy_interconnector_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) interconnector_emissions = energy_interconnector_emissions_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_emissions) bom_station = get_network_region_weather_station(network_region.code) if bom_station: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) stat_set.append_set(weather_stats) if cpi: stat_set.append_set(cpi) write_output(f"v3/stats/au/{network_region.code}/daily.json", stat_set)
dmax, network=network_from_network_code(gap.network_id)) except Exception: logger.error("Error running {} energy gapfill for {} => {}".format( gap.network_id, dmin, dmax)) def run_energy_gapfill( days: int = 14, networks: List[NetworkSchema] = [ NetworkNEM, NetworkWEM, NetworkAPVI, NetworkAEMORooftop ], run_all: bool = False, ) -> None: """Run gapfill of energy values - will find gaps in energy values and run energy_sum""" for network in networks: logger.info("Running energy gapfill for {}".format(network.code)) try: run_energy_gapfill_for_network(network, days=days, run_all=run_all) except Exception as e: logger.error("gap_fill run error: {}".format(e)) # debug entry point if __name__ == "__main__": # run_energy_gapfill(days=365/2) dmin = datetime.fromisoformat("2022-02-28T00:00:00+10:00") dmax = dmin + timedelta(days=3) run_energy_calc(dmin, dmax, network=network_from_network_code("NEM"))
def energy_network_fueltech_api( network_code: str = Query(None, description="Network code"), network_region: str = Query(None, description="Network region"), interval_human: str = Query("1d", description="Interval"), year: int = Query(None, description="Year to query"), period_human: str = Query("1Y", description="Period"), engine=Depends(get_database_engine), ) -> OpennemDataSet: network = network_from_network_code(network_code) interval = human_to_interval(interval_human) period_obj: TimePeriod = human_to_period("1Y") if period_human: period_obj = human_to_period(period_human) units = get_unit("energy_giga") query = "" if year and isinstance(year, int): period_obj = human_to_period("1Y") if year > datetime.now().year or year < 1996: raise HTTPException( status_code=status.HTTP_406_NOT_ACCEPTABLE, detail="Not a valid year", ) scada_range = get_scada_range(network=network) query = energy_network_fueltech_year( network=network, interval=interval, year=year, network_region=network_region, scada_range=scada_range, ) elif period_obj and period_obj.period_human == "all": scada_range = get_scada_range(network=network) query = energy_network_fueltech_all( network=network, network_region=network_region, scada_range=scada_range, ) else: query = energy_network_fueltech( network=network, interval=interval, period=period_obj, network_region=network_region, ) # print(query) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Energy stats not found", ) result = stats_factory( stats, code=network.code, network=network, interval=interval, period=period_obj, units=units, region=network_region, fueltech_group=True, ) if not result: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No stats") return result
def price_network_endpoint( engine: Engine = Depends(get_database_engine), network_code: str = Path(..., description="Network code"), network_region: Optional[str] = Query(None, description="Network region code"), forecasts: bool = Query(False, description="Include price forecasts"), ) -> OpennemDataSet: """Returns network and network region price info for interval which defaults to network interval size Args: engine ([type], optional): Database engine. Defaults to Depends(get_database_engine). Raises: HTTPException: No results Returns: OpennemData: data set """ engine = get_database_engine() network = None try: network = network_from_network_code(network_code) except Exception: raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND) interval_obj = human_to_interval("5m") period_obj = human_to_period("1d") scada_range = get_balancing_range(network=network, include_forecasts=forecasts) if not scada_range: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Could not find a date range", ) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Network not found", ) time_series = TimeSeries( start=scada_range.start, network=network, interval=interval_obj, period=period_obj, ) if network_region: time_series.network.regions = [NetworkNetworkRegion(code=network_region)] query = network_region_price_query(time_series=time_series) with engine.connect() as c: logger.debug(query) row = list(c.execute(query)) if len(row) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) result_set = [ DataQueryResult(interval=i[0], result=i[3], group_by=i[2] if len(i) > 1 else None) for i in row ] result = stats_factory( result_set, network=time_series.network, period=time_series.period, interval=time_series.interval, units=get_unit("price"), group_field="price", include_group_code=True, include_code=True, ) if not result or not result.data: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results", ) return result
def generate_weekly_export_map() -> StatMetadata: """ Generate export map for weekly power series @TODO deconstruct this into separate methods and schema ex. network.get_scada_range(), network_region.get_bom_station() etc. """ session = get_scoped_session() networks = session.query(Network).filter( Network.export_set.is_(True)).all() if not networks: raise Exception("No networks") countries = list(set([network.country for network in networks])) _exmap = [] # Loop countries for country in countries: # @TODO derive this scada_range = get_scada_range(network=NetworkAU, networks=[NetworkNEM, NetworkWEM]) if not scada_range: raise Exception("Require a scada range for NetworkAU") for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=country, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], year=year, week=week, date_range=date_range_from_week(year, week, NetworkAU), interval=human_to_interval("30m"), period=human_to_period("7d"), ) _exmap.append(export) # Loop networks for network in networks: network_schema = network_from_network_code(network.code) scada_range = get_scada_range(network=network_schema) if not scada_range: raise Exception("Require a scada range for network: {}".format( network.code)) for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=network.country, network=network_schema, year=year, week=week, date_range=date_range_from_week(year, week, NetworkAU), interval=human_to_interval(f"{network.interval_size}m"), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) # Skip cases like wem/wem where region is supurfelous if len(network.regions) < 2: continue for region in network.regions: scada_range = get_scada_range(network=network_schema, network_region=region.code) if not scada_range: logger.error( "Require a scada range for network {} and region {}". format(network_schema.code, region.code)) continue for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=network.country, network=network_schema, year=year, week=week, date_range=date_range_from_week( year, week, network_from_network_code(network.code)), interval=human_to_interval(f"{network.interval_size}m"), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export_meta = StatMetadata(date_created=datetime.now(), version=get_version(), resources=_exmap) return export_meta
def price_network_region_api( engine=Depends(get_database_engine), network_code: str = Query(..., description="Network code"), network_region_code: str = Query(..., description="Region code"), interval_human: str = Query(None, description="Interval"), period_human: str = Query("7d", description="Period"), year: Optional[int] = None, ) -> OpennemDataSet: network = network_from_network_code(network_code) if not network: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No such network", ) if not interval_human: interval_human = "{}m".format(network.interval_size) interval = human_to_interval(interval_human) period_obj = None if period_human: period_obj = human_to_period(period_human) units = get_unit("price") scada_range = get_scada_range(network=network) if period_obj and period_obj.period_human == "all" and interval.interval_human == "1M": query = price_network_monthly( network=network, network_region_code=network_region_code, scada_range=scada_range, ) else: query = price_network_region( network=network, network_region_code=network_region_code, interval=interval, period=period_obj, scada_range=scada_range, year=year, ) with engine.connect() as c: results = list(c.execute(query)) if len(results) < 1: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No data found") stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in results ] result = stats_factory( stats, code=network.code, region=network_region_code, network=network, interval=interval, period=period_obj, units=units, group_field="price", ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No results found", ) return result
def station_observations_api( station_code: str = Query(None, description="Station code"), interval_human: str = Query("15m", description="Interval"), period_human: str = Query("7d", description="Period"), station_codes: List[str] = [], network_code: str = "NEM", timezone: str = None, offset: str = None, year: int = None, engine=Depends(get_database_engine), ) -> OpennemDataSet: units = get_unit("temperature") if not interval_human: interval = "15m" if not period_human: period = "7d" network = None if network_code: network = network_from_network_code(network_code) if station_code: station_codes = [station_code] interval = human_to_interval(interval_human) period = human_to_period(period_human) if timezone: timezone = pytz.timezone(timezone) if offset: timezone = get_fixed_timezone(offset) scada_range = None if network: scada_range = get_scada_range(network=network) query = observation_query( station_codes=station_codes, interval=interval, period=period, network=network, scada_range=scada_range, year=year, ) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats=stats, units=units, interval=interval, period=period, network=network, code="bom", group_field="temperature", ) return result