def export_flows() -> None: date_range = get_scada_range(network=NetworkNEM) interchange_stat = StatExport( stat_type=StatType.power, priority=PriorityType.live, country="au", date_range=date_range, network=NetworkNEM, interval=NetworkNEM.get_interval(), period=human_to_period("7d"), ) time_series = TimeSeries( start=date_range.start, end=date_range.end, network=interchange_stat.network, interval=interchange_stat.interval, period=interchange_stat.period, ) stat_set = power_flows_network_week(time_series=time_series) if stat_set: write_output( f"v3/stats/au/{interchange_stat.network.code}/flows/7d.json", stat_set)
def export_historic_intervals(limit: int | None = None) -> None: """ """ session = get_scoped_session() networks = [NetworkNEM, NetworkWEM] for network in networks: network_regions: list[NetworkRegion] = ( session.query(NetworkRegion).filter( NetworkRegion.network_id == network.code).all()) for network_region in network_regions: scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks, energy=False) if not scada_range or not scada_range.start: logger.error( "Could not get scada range for network {}".format(network)) continue for week_start, week_end in week_series_datetimes( start=scada_range.end, end=scada_range.start, length=limit): if week_start > get_today_opennem(): continue try: export_network_intervals_for_week( week_start, week_end, network=network, network_region=network_region) except Exception as e: logger.error(f"export_historic_intervals error: {e}")
def run_aggregates_facility_all(network: NetworkSchema) -> None: scada_range: ScadaDateRange = get_scada_range(network=network) if not scada_range: logger.error("Could not find a scada range for {}".format( network.code)) return None exec_aggregates_facility_daily_query(date_min=scada_range.start, date_max=scada_range.end, network=network)
def get_date_range(network: NetworkSchema) -> DatetimeRange: date_range = get_scada_range(network=NetworkNEM) time_series = TimeSeries( start=date_range.start, end=date_range.end, interval=human_to_interval("1d"), period=human_to_period("all"), network=network, ) return time_series.get_range()
def generate_weekly_export_map() -> StatMetadata: """ Generate export map for weekly power series @TODO deconstruct this into separate methods and schema ex. network.get_scada_range(), network_region.get_bom_station() etc. """ session = get_scoped_session() networks = session.query(Network).filter( Network.export_set.is_(True)).all() if not networks: raise Exception("No networks") countries = list(set([network.country for network in networks])) _exmap = [] # Loop countries for country in countries: # @TODO derive this scada_range = get_scada_range(network=NetworkAU, networks=[NetworkNEM, NetworkWEM]) if not scada_range: raise Exception("Require a scada range for NetworkAU") for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=country, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], year=year, week=week, date_range=date_range_from_week(year, week, NetworkAU), interval=human_to_interval("30m"), period=human_to_period("7d"), ) _exmap.append(export) # Loop networks for network in networks: network_schema = network_from_network_code(network.code) scada_range = get_scada_range(network=network_schema) if not scada_range: raise Exception("Require a scada range for network: {}".format( network.code)) for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=network.country, network=network_schema, year=year, week=week, date_range=date_range_from_week(year, week, NetworkAU), interval=human_to_interval(f"{network.interval_size}m"), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) # Skip cases like wem/wem where region is supurfelous if len(network.regions) < 2: continue for region in network.regions: scada_range = get_scada_range(network=network_schema, network_region=region.code) if not scada_range: logger.error( "Require a scada range for network {} and region {}". format(network_schema.code, region.code)) continue for year, week in week_series(scada_range.end, scada_range.start): export = StatExport( stat_type=StatType.power, priority=PriorityType.history, country=network.country, network=network_schema, year=year, week=week, date_range=date_range_from_week( year, week, network_from_network_code(network.code)), interval=human_to_interval(f"{network.interval_size}m"), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export_meta = StatMetadata(date_created=datetime.now(), version=get_version(), resources=_exmap) return export_meta
def export_electricitymap() -> None: date_range = get_scada_range(network=NetworkNEM) if not date_range.start: raise Exception("Could not get a scada range in EM export") interchange_stat = StatExport( stat_type=StatType.power, priority=PriorityType.live, country="au", date_range=date_range, network=NetworkNEM, interval=NetworkNEM.get_interval(), period=human_to_period("1d"), ) time_series = TimeSeries( start=date_range.start, end=date_range.end, network=interchange_stat.network, networks=[NetworkNEM, NetworkAEMORooftop, NetworkAEMORooftopBackfill], interval=interchange_stat.interval, period=interchange_stat.period, ) stat_set = power_flows_network_week(time_series=time_series) if not stat_set: raise Exception("No flow results for electricitymap export") em_set = OpennemDataSet(type="custom", version=get_version(), created_at=datetime.now(), data=[]) INVERT_SETS = ["VIC1->NSW1", "VIC1->SA1"] for ds in stat_set.data: if ds.code in INVERT_SETS: ds_inverted = invert_flow_set(ds) em_set.data.append(ds_inverted) logging.info("Inverted {}".format(ds.code)) else: em_set.data.append(ds) for region in ["NSW1", "QLD1", "VIC1", "TAS1", "SA1"]: power_set = power_week( time_series, region, include_capacities=True, include_code=False, networks_query=[ NetworkNEM, NetworkAEMORooftop, NetworkAEMORooftopBackfill ], ) if power_set: em_set.append_set(power_set) date_range = get_scada_range(network=NetworkWEM) # WEM custom time_series = TimeSeries( start=date_range.start, end=date_range.end, network=NetworkWEM, networks=[NetworkWEM, NetworkAPVI], interval=NetworkWEM.get_interval(), period=interchange_stat.period, ) power_set = power_week( time_series, "WEM", include_capacities=True, networks_query=[NetworkWEM, NetworkAPVI], include_code=False, ) if power_set: em_set.append_set(power_set) write_output("v3/clients/em/latest.json", em_set)
def export_power( stats: List[StatExport] = None, priority: Optional[PriorityType] = None, latest: Optional[bool] = False, ) -> None: """ Export power stats from the export map """ # Not passed a stat map so go and get one if not stats: export_map = None if priority and priority == PriorityType.history: export_map = get_weekly_export_map() else: export_map = get_export_map().get_by_stat_type(StatType.power) if priority: export_map = export_map.get_by_priority(priority) stats = export_map.resources output_count: int = 0 logger.info("Running {}export {} with {} stats".format( "latest " if latest else "", priority, len(stats))) for power_stat in stats: if power_stat.stat_type != StatType.power: continue if output_count >= 1 and latest: return None date_range_networks = power_stat.networks or [] if NetworkNEM in date_range_networks: date_range_networks = [NetworkNEM] date_range: ScadaDateRange = get_scada_range( network=power_stat.network, networks=date_range_networks) logger.debug("Date range for {}: {} => {}".format( power_stat.network.code, date_range.start, date_range.end)) # Migrate to this time_series time_series = TimeSeries( start=date_range.start, end=date_range.end, network=power_stat.network, year=power_stat.year, interval=power_stat.interval, period=power_stat.period, ) stat_set = power_week( time_series=time_series, network_region_code=power_stat.network_region_query or power_stat.network_region, networks_query=power_stat.networks, ) if not stat_set: logger.info("No power stat set for {} {} {}".format( power_stat.period, power_stat.networks, power_stat.network_region, )) continue demand_set = demand_week( time_series=time_series, networks_query=power_stat.networks, network_region_code=power_stat.network_region_query or power_stat.network_region, ) stat_set.append_set(demand_set) if power_stat.network_region: flow_set = power_flows_region_week( time_series=time_series, network_region_code=power_stat.network_region, ) if flow_set: stat_set.append_set(flow_set) time_series_weather = time_series.copy() time_series_weather.interval = human_to_interval("30m") if power_stat.bom_station: try: weather_set = weather_daily( time_series=time_series_weather, station_code=power_stat.bom_station, network_region=power_stat.network_region, include_min_max=False, unit_name="temperature", ) stat_set.append_set(weather_set) except Exception: pass write_output(power_stat.path, stat_set) output_count += 1
def export_all_daily( networks: List[NetworkSchema] = [NetworkNEM, NetworkWEM], network_region_code: Optional[str] = None, ) -> None: session = get_scoped_session() cpi = gov_stats_cpi() for network in networks: network_regions = session.query(NetworkRegion).filter_by( export_set=True).filter_by(network_id=network.code) if network_region_code: network_regions = network_regions.filter_by( code=network_region_code) network_regions = network_regions.all() for network_region in network_regions: logging.info("Exporting for network {} and region {}".format( network.code, network_region.code)) networks = [ NetworkNEM, NetworkAEMORooftop, NetworkAEMORooftopBackfill ] if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks, energy=True) if not scada_range or not scada_range.start: logger.error( "Could not get scada range for network {} and energy {}". format(network, True)) continue time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1d"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue demand_energy_and_value = demand_network_region_daily( time_series=time_series, network_region_code=network_region.code, networks=networks) stat_set.append_set(demand_energy_and_value) # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if network == NetworkNEM: interconnector_flows = energy_interconnector_flows_and_emissions( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) bom_station = get_network_region_weather_station( network_region.code) if bom_station: try: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) stat_set.append_set(weather_stats) except Exception: pass if cpi: stat_set.append_set(cpi) write_output(f"v3/stats/au/{network_region.code}/daily.json", stat_set)
def export_all_monthly() -> None: session = get_scoped_session() all_monthly = OpennemDataSet(code="au", data=[], version=get_version(), created_at=datetime.now()) cpi = gov_stats_cpi() all_monthly.append_set(cpi) # Iterate networks and network regions networks = [NetworkNEM, NetworkWEM] for network in networks: network_regions = session.query(NetworkRegion).filter( NetworkRegion.network_id == network.code).all() for network_region in network_regions: networks = [] logging.info( "Exporting monthly for network {} and region {}".format( network.code, network_region.code)) if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] if network == NetworkNEM: networks = [NetworkNEM, NetworkAEMORooftop] logger.debug("Running monthlies for {} and {}".format( network.code, network_region.code)) scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks, energy=True) if not scada_range or not scada_range.start: logger.error( "Could not get scada range for network {} and energy {}". format(network, True)) continue time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1M"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue demand_energy_and_value = demand_network_region_daily( time_series=time_series, network_region_code=network_region.code, networks=networks) stat_set.append_set(demand_energy_and_value) if network == NetworkNEM: interconnector_flows = energy_interconnector_flows_and_emissions( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) all_monthly.append_set(stat_set) bom_station = get_network_region_weather_station( network_region.code) if bom_station: try: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) all_monthly.append_set(weather_stats) except Exception: pass write_output("v3/stats/au/all/monthly.json", all_monthly)
def station_observations_api( station_code: str = Query(None, description="Station code"), interval_human: str = Query("15m", description="Interval"), period_human: str = Query("7d", description="Period"), station_codes: List[str] = [], network_code: str = "NEM", timezone: str = None, offset: str = None, year: int = None, engine=Depends(get_database_engine), ) -> OpennemDataSet: units = get_unit("temperature") if not interval_human: interval = "15m" if not period_human: period = "7d" network = None if network_code: network = network_from_network_code(network_code) if station_code: station_codes = [station_code] interval = human_to_interval(interval_human) period = human_to_period(period_human) if timezone: timezone = pytz.timezone(timezone) if offset: timezone = get_fixed_timezone(offset) scada_range = None if network: scada_range = get_scada_range(network=network) query = observation_query( station_codes=station_codes, interval=interval, period=period, network=network, scada_range=scada_range, year=year, ) with engine.connect() as c: results = list(c.execute(query)) stats = [ DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None) for i in results ] if len(stats) < 1: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Station stats not found", ) result = stats_factory( stats=stats, units=units, interval=interval, period=period, network=network, code="bom", group_field="temperature", ) return result
def export_all_daily() -> None: session = SessionLocal() network_regions = session.query(NetworkRegion).all() cpi = gov_stats_cpi() for network_region in network_regions: network = network_from_network_code(network_region.network.code) networks = None if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks) time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1d"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if network == NetworkNEM: interconnector_flows = energy_interconnector_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) interconnector_emissions = energy_interconnector_emissions_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_emissions) bom_station = get_network_region_weather_station(network_region.code) if bom_station: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) stat_set.append_set(weather_stats) if cpi: stat_set.append_set(cpi) write_output(f"v3/stats/au/{network_region.code}/daily.json", stat_set)
def export_all_monthly() -> None: session = SessionLocal() network_regions = session.query(NetworkRegion).all() all_monthly = OpennemDataSet(code="au", data=[], version=get_version(), created_at=datetime.now()) cpi = gov_stats_cpi() all_monthly.append_set(cpi) for network_region in network_regions: network = network_from_network_code(network_region.network.code) networks = None if network_region.code == "WEM": networks = [NetworkWEM, NetworkAPVI] scada_range: ScadaDateRange = get_scada_range(network=network, networks=networks) time_series = TimeSeries( start=scada_range.start, end=scada_range.end, network=network, interval=human_to_interval("1M"), period=human_to_period("all"), ) stat_set = energy_fueltech_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) if not stat_set: continue if network == NetworkNEM: interconnector_flows = energy_interconnector_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_flows) interconnector_emissions = energy_interconnector_emissions_region_daily( time_series=time_series, networks_query=networks, network_region_code=network_region.code, ) stat_set.append_set(interconnector_emissions) all_monthly.append_set(stat_set) bom_station = get_network_region_weather_station(network_region.code) if bom_station: weather_stats = weather_daily( time_series=time_series, station_code=bom_station, network_region=network_region.code, ) all_monthly.append_set(weather_stats) write_output("v3/stats/au/all/monthly.json", all_monthly)
def export_energy( stats: List[StatExport] = None, priority: Optional[PriorityType] = None, latest: Optional[bool] = False, ) -> None: """ Export energy stats from the export map """ if not stats: export_map = get_export_map().get_by_stat_type(StatType.energy) if priority: export_map = export_map.get_by_priority(priority) stats = export_map.resources CURRENT_YEAR = datetime.now().year for energy_stat in stats: if energy_stat.stat_type != StatType.energy: continue # @FIX trim to NEM since it's the one with the shortest # data time span. # @TODO find a better and more flexible way to do this in the # range method date_range_networks = energy_stat.networks or [] if NetworkNEM in date_range_networks: date_range_networks = [NetworkNEM] date_range: ScadaDateRange = get_scada_range( network=energy_stat.network, networks=date_range_networks) # Migrate to this time_series time_series = TimeSeries( start=date_range.start, end=date_range.end, network=energy_stat.network, year=energy_stat.year, interval=energy_stat.interval, period=human_to_period("1Y"), ) if energy_stat.year: if latest and energy_stat.year != CURRENT_YEAR: continue stat_set = energy_fueltech_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) if not stat_set: continue # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if energy_stat.network == NetworkNEM and energy_stat.network_region: interconnector_flows = energy_interconnector_region_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) stat_set.append_set(interconnector_flows) interconnector_emissions = energy_interconnector_emissions_region_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) stat_set.append_set(interconnector_emissions) if energy_stat.bom_station: weather_stats = weather_daily( time_series=time_series, station_code=energy_stat.bom_station, network_region=energy_stat.network_region, ) stat_set.append_set(weather_stats) write_output(energy_stat.path, stat_set) elif energy_stat.period and energy_stat.period.period_human == "all" and not latest: time_series.period = human_to_period("all") time_series.interval = human_to_interval("1M") time_series.year = None stat_set = energy_fueltech_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) if not stat_set: continue # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if energy_stat.network == NetworkNEM and energy_stat.network_region: interconnector_flows = energy_interconnector_region_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) stat_set.append_set(interconnector_flows) interconnector_emissions = energy_interconnector_emissions_region_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) stat_set.append_set(interconnector_emissions) if energy_stat.bom_station: weather_stats = weather_daily( time_series=time_series, station_code=energy_stat.bom_station, network_region=energy_stat.network_region, ) stat_set.append_set(weather_stats) stat_set = pad_stat_set(stat_set) write_output(energy_stat.path, stat_set)
def power_facility_query( facility_codes: List[str], network: NetworkSchema, period: TimePeriod, interval: Optional[TimeInterval] = None, date_range: Optional[ScadaDateRange] = None, ) -> str: timezone = network.get_timezone(postgres_format=True) if not date_range: date_range = get_scada_range(network=network, facilities=facility_codes) timezone = network.timezone_database __query = """ select t.trading_interval at time zone '{timezone}', coalesce(avg(t.facility_power), 0), t.facility_code from ( select time_bucket_gapfill('{trunc}', fs.trading_interval) AS trading_interval, coalesce( avg(fs.generated), 0 ) as facility_power, fs.facility_code from facility_scada fs join facility f on fs.facility_code = f.code where fs.trading_interval <= '{date_max}' and fs.trading_interval > '{date_min}' and fs.facility_code in ({facility_codes_parsed}) group by 1, 3 ) as t group by 1, 3 order by 1 desc """ if not interval: interval = network.get_interval() if not date_range: raise Exception("Require a date range for query") if not interval: raise Exception("Require an interval") date_max = date_range.get_end() date_min = date_range.get_start() if period: date_min = date_range.get_end() - timedelta(minutes=period.period) query = __query.format( facility_codes_parsed=duid_in_case(facility_codes), trunc=interval.interval_sql, period=period.period_sql, timezone=timezone, date_max=date_max, date_min=date_min, ) return query
def energy_facility_query( facility_codes: List[str], network: NetworkSchema, period: TimePeriod, interval: Optional[TimeInterval] = None, ) -> str: """ Get Energy for a list of facility codes """ __query = """ select date_trunc('{trunc}', t.trading_interval at time zone '{timezone}') as trading_day, t.code, sum(t.energy) / 1000 as fueltech_energy, sum(t.market_value) as fueltech_market_value, sum(t.emissions) as fueltech_emissions from mv_facility_all t where t.trading_interval <= '{date_max}' and t.trading_interval >= '{date_min}' and t.code in ({facility_codes_parsed}) group by 1, 2 order by trading_day desc; """ timezone = network.timezone_database offset = network.get_timezone(postgres_format=True) date_range: ScadaDateRange = get_scada_range(network=network, facilities=facility_codes) if not interval: interval = network.get_interval() if not date_range: raise Exception("Require a date range for query") if not period: raise Exception("Require a period") if not interval: interval = network.get_interval() if not interval: raise Exception("Require an interval") trunc = interval.trunc date_max = date_range.get_end() date_min = date_range.get_start() if period.period_human == "1M": date_min = date_range.get_end() - timedelta(minutes=period.period) elif period.period_human == "1Y": # might have to do +offset times year = datetime.now().year date_min = "{}-01-01 00:00:00{}".format(year, offset) elif period.period_human in ["7d", "5Y", "10Y"]: date_min = date_range.get_end() - timedelta(minutes=period.period) # elif period.period_human == "all": # else: # date_min = date_range.get_end() - timedelta(minutes=period.period) query = dedent( __query.format( facility_codes_parsed=duid_in_case(facility_codes), trunc=trunc, date_max=date_max, date_min=date_min, timezone=timezone, )) return query
def export_power( stats: List[StatExport] = None, priority: Optional[PriorityType] = None, latest: Optional[bool] = False, ) -> None: """ Export power stats from the export map """ if not stats: export_map = get_export_map().get_by_stat_type(StatType.power) if priority: export_map = export_map.get_by_priority(priority) stats = export_map.resources output_count: int = 0 for power_stat in stats: if power_stat.stat_type != StatType.power: continue if output_count >= 1 and latest: return None date_range_networks = power_stat.networks or [] if NetworkNEM in date_range_networks: date_range_networks = [NetworkNEM] date_range: ScadaDateRange = get_scada_range( network=power_stat.network, networks=date_range_networks) # Migrate to this time_series time_series = TimeSeries( start=date_range.start, end=date_range.end, network=power_stat.network, year=power_stat.year, interval=power_stat.interval, period=power_stat.period, ) stat_set = power_week( time_series=time_series, network_region_code=power_stat.network_region_query or power_stat.network_region, networks_query=power_stat.networks, ) if not stat_set: logger.info("No power stat set for {} {} {}".format( power_stat.period, power_stat.networks, power_stat.network_region, )) continue demand_set = demand_week( time_series=time_series, networks_query=power_stat.networks, network_region_code=power_stat.network_region_query or power_stat.network_region, ) stat_set.append_set(demand_set) if power_stat.network_region: flow_set = power_flows_week( time_series=time_series, network_region_code=power_stat.network_region, ) if flow_set: stat_set.append_set(flow_set) if power_stat.bom_station: weather_set = weather_daily( time_series=time_series, station_code=power_stat.bom_station, network_region=power_stat.network_region, include_min_max=False, unit_name="temperature", ) stat_set.append_set(weather_set) write_output(power_stat.path, stat_set) output_count += 1
def get_export_map() -> StatMetadata: """ Generates a map of all export JSONs """ session = SessionLocal() networks = session.query(Network).filter(Network.export_set.is_(True)).all() if not networks: raise Exception("No networks") countries = list(set([network.country for network in networks])) _exmap = [] for country in countries: # @TODO derive this scada_range = get_scada_range(network=NetworkAU, networks=[NetworkNEM, NetworkWEM]) if not scada_range: raise Exception("Require a scada range") export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], interval=NetworkAU.get_interval(), period=human_to_period("7d"), ) _exmap.append(export) for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], year=year, interval=human_to_interval("1d"), period=human_to_period("1Y"), ) _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=country, date_range=scada_range, network=NetworkAU, networks=[NetworkNEM, NetworkWEM], interval=human_to_interval("1M"), period=human_to_period("all"), ) _exmap.append(export) for network in networks: network_schema = network_from_network_code(network.code) scada_range = get_scada_range(network=network_schema) bom_station = get_network_region_weather_station(network.code) export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, interval=network_schema.get_interval(), period=human_to_period("7d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) if not scada_range: raise Exception("Require a scada range") for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, year=year, period=human_to_period("1Y"), interval=human_to_interval("1d"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=network.country, date_range=scada_range, network=network_schema, bom_station=bom_station, interval=human_to_interval("1M"), period=human_to_period("all"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) # Skip cases like wem/wem where region is supurfelous if len(network.regions) < 2: continue for region in network.regions: scada_range = get_scada_range(network=network_schema, network_region=region) bom_station = get_network_region_weather_station(region.code) if not scada_range: raise Exception("Require a scada range") export = StatExport( stat_type=StatType.power, priority=PriorityType.live, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, period=human_to_period("7d"), interval=network_schema.get_interval(), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) for year in range( datetime.now().year, scada_range.start.year - 1, -1, ): export = StatExport( stat_type=StatType.energy, priority=PriorityType.daily, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, year=year, period=human_to_period("1Y"), interval=human_to_interval("1d"), ) _exmap.append(export) export = StatExport( stat_type=StatType.energy, priority=PriorityType.monthly, country=network.country, date_range=scada_range, network=network_schema, network_region=region.code, bom_station=bom_station, period=human_to_period("all"), interval=human_to_interval("1M"), ) if network.code == "WEM": export.networks = [NetworkWEM, NetworkAPVI] export.network_region_query = "WEM" _exmap.append(export) export_meta = StatMetadata( date_created=datetime.now(), version=get_version(), resources=_exmap ) return export_meta
def export_energy( stats: List[StatExport] = None, priority: Optional[PriorityType] = None, latest: Optional[bool] = False, ) -> None: """ Export energy stats from the export map """ if not stats: export_map = get_export_map().get_by_stat_type(StatType.energy) if priority: export_map = export_map.get_by_priority(priority) stats = export_map.resources CURRENT_YEAR = datetime.now().year logger.info(f"Running export_energy with {len(stats)} stats") for energy_stat in stats: if energy_stat.stat_type != StatType.energy: continue # @FIX trim to NEM since it's the one with the shortest # data time span. # @TODO find a better and more flexible way to do this in the # range method date_range_networks = energy_stat.networks or [] if NetworkNEM in date_range_networks: date_range_networks = [NetworkNEM] date_range: ScadaDateRange = get_scada_range( network=energy_stat.network, networks=date_range_networks, energy=True) if not date_range: logger.error( "Skipping - Could not get date range for energy {} {}".format( energy_stat.network, date_range_networks)) continue logger.debug("Date range is: {} {} => {}".format( energy_stat.network.code, date_range.start, date_range.end)) # Migrate to this time_series time_series = TimeSeries( start=date_range.start, end=date_range.end, network=energy_stat.network, year=energy_stat.year, interval=energy_stat.interval, period=human_to_period("1Y"), ) if energy_stat.year: if latest and energy_stat.year != CURRENT_YEAR: continue stat_set = energy_fueltech_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) if not stat_set: continue demand_energy_and_value = demand_network_region_daily( time_series=time_series, network_region_code=energy_stat.network_region, networks=energy_stat.networks) stat_set.append_set(demand_energy_and_value) # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if energy_stat.network == NetworkNEM and energy_stat.network_region: interconnector_flows = energy_interconnector_flows_and_emissions( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) stat_set.append_set(interconnector_flows) if energy_stat.bom_station: try: weather_stats = weather_daily( time_series=time_series, station_code=energy_stat.bom_station, network_region=energy_stat.network_region, ) stat_set.append_set(weather_stats) except NoResults as e: logger.info("No results for weather result: {}".format(e)) except Exception as e: logger.error("weather_stat exception: {}".format(e)) pass else: logger.info("Stat set has no bom station") write_output(energy_stat.path, stat_set) elif energy_stat.period and energy_stat.period.period_human == "all" and not latest: time_series.period = human_to_period("all") time_series.interval = human_to_interval("1M") time_series.year = None stat_set = energy_fueltech_daily( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) if not stat_set: continue demand_energy_and_value = demand_network_region_daily( time_series=time_series, network_region_code=energy_stat.network_region, networks=energy_stat.networks) stat_set.append_set(demand_energy_and_value) # Hard coded to NEM only atm but we'll put has_interconnectors # in the metadata to automate all this if energy_stat.network == NetworkNEM and energy_stat.network_region: interconnector_flows = energy_interconnector_flows_and_emissions( time_series=time_series, networks_query=energy_stat.networks, network_region_code=energy_stat.network_region_query or energy_stat.network_region, ) stat_set.append_set(interconnector_flows) if energy_stat.bom_station: try: weather_stats = weather_daily( time_series=time_series, station_code=energy_stat.bom_station, network_region=energy_stat.network_region, ) stat_set.append_set(weather_stats) except NoResults as e: logger.info("No weather results: {}".format(e)) except Exception: pass write_output(energy_stat.path, stat_set)