def get_export_map() -> StatMetadata:
"""
Generates a map of all export JSONs
"""
session = SessionLocal()
networks = session.query(Network).filter(Network.export_set.is_(True)).all()
if not networks:
raise Exception("No networks")
countries = list(set([network.country for network in networks]))
_exmap = []
for country in countries:
# @TODO derive this
scada_range = get_scada_range(network=NetworkAU, networks=[NetworkNEM, NetworkWEM])
if not scada_range:
raise Exception("Require a scada range")
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.live,
country=country,
date_range=scada_range,
network=NetworkAU,
networks=[NetworkNEM, NetworkWEM],
interval=NetworkAU.get_interval(),
period=human_to_period("7d"),
)
_exmap.append(export)
for year in range(
datetime.now().year,
scada_range.start.year - 1,
-1,
):
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.daily,
country=country,
date_range=scada_range,
network=NetworkAU,
networks=[NetworkNEM, NetworkWEM],
year=year,
interval=human_to_interval("1d"),
period=human_to_period("1Y"),
)
_exmap.append(export)
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.monthly,
country=country,
date_range=scada_range,
network=NetworkAU,
networks=[NetworkNEM, NetworkWEM],
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
_exmap.append(export)
for network in networks:
network_schema = network_from_network_code(network.code)
scada_range = get_scada_range(network=network_schema)
bom_station = get_network_region_weather_station(network.code)
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.live,
country=network.country,
date_range=scada_range,
network=network_schema,
bom_station=bom_station,
interval=network_schema.get_interval(),
period=human_to_period("7d"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
if not scada_range:
raise Exception("Require a scada range")
for year in range(
datetime.now().year,
scada_range.start.year - 1,
-1,
):
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.daily,
country=network.country,
date_range=scada_range,
network=network_schema,
bom_station=bom_station,
year=year,
period=human_to_period("1Y"),
interval=human_to_interval("1d"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.monthly,
country=network.country,
date_range=scada_range,
network=network_schema,
bom_station=bom_station,
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
# Skip cases like wem/wem where region is supurfelous
if len(network.regions) < 2:
continue
for region in network.regions:
scada_range = get_scada_range(network=network_schema, network_region=region)
bom_station = get_network_region_weather_station(region.code)
if not scada_range:
raise Exception("Require a scada range")
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.live,
country=network.country,
date_range=scada_range,
network=network_schema,
network_region=region.code,
bom_station=bom_station,
period=human_to_period("7d"),
interval=network_schema.get_interval(),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
for year in range(
datetime.now().year,
scada_range.start.year - 1,
-1,
):
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.daily,
country=network.country,
date_range=scada_range,
network=network_schema,
network_region=region.code,
bom_station=bom_station,
year=year,
period=human_to_period("1Y"),
interval=human_to_interval("1d"),
)
_exmap.append(export)
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.monthly,
country=network.country,
date_range=scada_range,
network=network_schema,
network_region=region.code,
bom_station=bom_station,
period=human_to_period("all"),
interval=human_to_interval("1M"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
export_meta = StatMetadata(
date_created=datetime.now(), version=get_version(), resources=_exmap
)
return export_meta
def test_get_network_region_weather_station(network_region,
weather_station_id):
date_subject_dt = get_network_region_weather_station(network_region)
assert date_subject_dt == weather_station_id
def export_all_daily(
networks: List[NetworkSchema] = [NetworkNEM, NetworkWEM],
network_region_code: Optional[str] = None,
) -> None:
session = get_scoped_session()
cpi = gov_stats_cpi()
for network in networks:
network_regions = session.query(NetworkRegion).filter_by(
export_set=True).filter_by(network_id=network.code)
if network_region_code:
network_regions = network_regions.filter_by(
code=network_region_code)
network_regions = network_regions.all()
for network_region in network_regions:
logging.info("Exporting for network {} and region {}".format(
network.code, network_region.code))
networks = [
NetworkNEM, NetworkAEMORooftop, NetworkAEMORooftopBackfill
]
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks,
energy=True)
if not scada_range or not scada_range.start:
logger.error(
"Could not get scada range for network {} and energy {}".
format(network, True))
continue
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1d"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
demand_energy_and_value = demand_network_region_daily(
time_series=time_series,
network_region_code=network_region.code,
networks=networks)
stat_set.append_set(demand_energy_and_value)
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if network == NetworkNEM:
interconnector_flows = energy_interconnector_flows_and_emissions(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
bom_station = get_network_region_weather_station(
network_region.code)
if bom_station:
try:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
stat_set.append_set(weather_stats)
except Exception:
pass
if cpi:
stat_set.append_set(cpi)
write_output(f"v3/stats/au/{network_region.code}/daily.json",
stat_set)
def export_all_monthly() -> None:
session = get_scoped_session()
all_monthly = OpennemDataSet(code="au",
data=[],
version=get_version(),
created_at=datetime.now())
cpi = gov_stats_cpi()
all_monthly.append_set(cpi)
# Iterate networks and network regions
networks = [NetworkNEM, NetworkWEM]
for network in networks:
network_regions = session.query(NetworkRegion).filter(
NetworkRegion.network_id == network.code).all()
for network_region in network_regions:
networks = []
logging.info(
"Exporting monthly for network {} and region {}".format(
network.code, network_region.code))
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
if network == NetworkNEM:
networks = [NetworkNEM, NetworkAEMORooftop]
logger.debug("Running monthlies for {} and {}".format(
network.code, network_region.code))
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks,
energy=True)
if not scada_range or not scada_range.start:
logger.error(
"Could not get scada range for network {} and energy {}".
format(network, True))
continue
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
demand_energy_and_value = demand_network_region_daily(
time_series=time_series,
network_region_code=network_region.code,
networks=networks)
stat_set.append_set(demand_energy_and_value)
if network == NetworkNEM:
interconnector_flows = energy_interconnector_flows_and_emissions(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
all_monthly.append_set(stat_set)
bom_station = get_network_region_weather_station(
network_region.code)
if bom_station:
try:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
all_monthly.append_set(weather_stats)
except Exception:
pass
write_output("v3/stats/au/all/monthly.json", all_monthly)
def export_all_daily() -> None:
session = SessionLocal()
network_regions = session.query(NetworkRegion).all()
cpi = gov_stats_cpi()
for network_region in network_regions:
network = network_from_network_code(network_region.network.code)
networks = None
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks)
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1d"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if network == NetworkNEM:
interconnector_flows = energy_interconnector_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
interconnector_emissions = energy_interconnector_emissions_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_emissions)
bom_station = get_network_region_weather_station(network_region.code)
if bom_station:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
stat_set.append_set(weather_stats)
if cpi:
stat_set.append_set(cpi)
write_output(f"v3/stats/au/{network_region.code}/daily.json", stat_set)
def export_all_monthly() -> None:
session = SessionLocal()
network_regions = session.query(NetworkRegion).all()
all_monthly = OpennemDataSet(code="au",
data=[],
version=get_version(),
created_at=datetime.now())
cpi = gov_stats_cpi()
all_monthly.append_set(cpi)
for network_region in network_regions:
network = network_from_network_code(network_region.network.code)
networks = None
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks)
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
if network == NetworkNEM:
interconnector_flows = energy_interconnector_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
interconnector_emissions = energy_interconnector_emissions_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_emissions)
all_monthly.append_set(stat_set)
bom_station = get_network_region_weather_station(network_region.code)
if bom_station:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
all_monthly.append_set(weather_stats)
write_output("v3/stats/au/all/monthly.json", all_monthly)
def export_network_intervals_for_week(
week_start: datetime,
week_end: datetime,
network: NetworkSchema,
network_region: NetworkRegion,
) -> int | None:
""" """
week_number = get_week_number_from_datetime(week_start)
logging.info(
"Exporting historic intervals for network {} and region {} and year {} and week {} ({} => {})"
.format(network.code, network_region.code, week_start.year,
week_number, week_start, week_end))
time_series = TimeSeries(
start=week_start,
end=week_end + timedelta(days=1),
network=network,
interval=network.get_interval(),
period=human_to_period("7d"),
)
stat_set = power_week(
time_series=time_series,
networks_query=network.get_networks_query(),
network_region_code=network_region.code,
)
if not stat_set:
return None
# emissions for network
emission_intervals = emissions_for_network_interval(
time_series=time_series, network_region_code=network_region.code)
stat_set.append_set(emission_intervals)
# demand and pricing
demand_energy_and_value = demand_network_region_daily(
time_series=time_series,
network_region_code=network_region.code,
networks=network.get_networks_query())
stat_set.append_set(demand_energy_and_value)
if network == NetworkNEM:
interconnector_flows = energy_interconnector_flows_and_emissions(
time_series=time_series,
networks_query=network.get_networks_query(),
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
bom_station = get_network_region_weather_station(network_region.code)
if bom_station:
try:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
stat_set.append_set(weather_stats)
except Exception:
pass
save_path = (
f"v3/stats/historic/weekly/{network.code}/{network_region.code}/year/{week_start.year}/week/{week_number}.json"
)
written_bytes = write_output(save_path, stat_set)
return written_bytes
