def export_all_daily(
networks: List[NetworkSchema] = [NetworkNEM, NetworkWEM],
network_region_code: Optional[str] = None,
) -> None:
session = get_scoped_session()
cpi = gov_stats_cpi()
for network in networks:
network_regions = session.query(NetworkRegion).filter_by(
export_set=True).filter_by(network_id=network.code)
if network_region_code:
network_regions = network_regions.filter_by(
code=network_region_code)
network_regions = network_regions.all()
for network_region in network_regions:
logging.info("Exporting for network {} and region {}".format(
network.code, network_region.code))
networks = [
NetworkNEM, NetworkAEMORooftop, NetworkAEMORooftopBackfill
]
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks,
energy=True)
if not scada_range or not scada_range.start:
logger.error(
"Could not get scada range for network {} and energy {}".
format(network, True))
continue
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1d"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
demand_energy_and_value = demand_network_region_daily(
time_series=time_series,
network_region_code=network_region.code,
networks=networks)
stat_set.append_set(demand_energy_and_value)
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if network == NetworkNEM:
interconnector_flows = energy_interconnector_flows_and_emissions(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
bom_station = get_network_region_weather_station(
network_region.code)
if bom_station:
try:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
stat_set.append_set(weather_stats)
except Exception:
pass
if cpi:
stat_set.append_set(cpi)
write_output(f"v3/stats/au/{network_region.code}/daily.json",
stat_set)
def export_all_monthly() -> None:
session = get_scoped_session()
all_monthly = OpennemDataSet(code="au",
data=[],
version=get_version(),
created_at=datetime.now())
cpi = gov_stats_cpi()
all_monthly.append_set(cpi)
# Iterate networks and network regions
networks = [NetworkNEM, NetworkWEM]
for network in networks:
network_regions = session.query(NetworkRegion).filter(
NetworkRegion.network_id == network.code).all()
for network_region in network_regions:
networks = []
logging.info(
"Exporting monthly for network {} and region {}".format(
network.code, network_region.code))
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
if network == NetworkNEM:
networks = [NetworkNEM, NetworkAEMORooftop]
logger.debug("Running monthlies for {} and {}".format(
network.code, network_region.code))
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks,
energy=True)
if not scada_range or not scada_range.start:
logger.error(
"Could not get scada range for network {} and energy {}".
format(network, True))
continue
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
demand_energy_and_value = demand_network_region_daily(
time_series=time_series,
network_region_code=network_region.code,
networks=networks)
stat_set.append_set(demand_energy_and_value)
if network == NetworkNEM:
interconnector_flows = energy_interconnector_flows_and_emissions(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
all_monthly.append_set(stat_set)
bom_station = get_network_region_weather_station(
network_region.code)
if bom_station:
try:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
all_monthly.append_set(weather_stats)
except Exception:
pass
write_output("v3/stats/au/all/monthly.json", all_monthly)
def export_energy(
stats: List[StatExport] = None,
priority: Optional[PriorityType] = None,
latest: Optional[bool] = False,
) -> None:
"""
Export energy stats from the export map
"""
if not stats:
export_map = get_export_map().get_by_stat_type(StatType.energy)
if priority:
export_map = export_map.get_by_priority(priority)
stats = export_map.resources
CURRENT_YEAR = datetime.now().year
logger.info(f"Running export_energy with {len(stats)} stats")
for energy_stat in stats:
if energy_stat.stat_type != StatType.energy:
continue
# @FIX trim to NEM since it's the one with the shortest
# data time span.
# @TODO find a better and more flexible way to do this in the
# range method
date_range_networks = energy_stat.networks or []
if NetworkNEM in date_range_networks:
date_range_networks = [NetworkNEM]
date_range: ScadaDateRange = get_scada_range(
network=energy_stat.network,
networks=date_range_networks,
energy=True)
if not date_range:
logger.error(
"Skipping - Could not get date range for energy {} {}".format(
energy_stat.network, date_range_networks))
continue
logger.debug("Date range is: {} {} => {}".format(
energy_stat.network.code, date_range.start, date_range.end))
# Migrate to this time_series
time_series = TimeSeries(
start=date_range.start,
end=date_range.end,
network=energy_stat.network,
year=energy_stat.year,
interval=energy_stat.interval,
period=human_to_period("1Y"),
)
if energy_stat.year:
if latest and energy_stat.year != CURRENT_YEAR:
continue
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
if not stat_set:
continue
demand_energy_and_value = demand_network_region_daily(
time_series=time_series,
network_region_code=energy_stat.network_region,
networks=energy_stat.networks)
stat_set.append_set(demand_energy_and_value)
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if energy_stat.network == NetworkNEM and energy_stat.network_region:
interconnector_flows = energy_interconnector_flows_and_emissions(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
stat_set.append_set(interconnector_flows)
if energy_stat.bom_station:
try:
weather_stats = weather_daily(
time_series=time_series,
station_code=energy_stat.bom_station,
network_region=energy_stat.network_region,
)
stat_set.append_set(weather_stats)
except NoResults as e:
logger.info("No results for weather result: {}".format(e))
except Exception as e:
logger.error("weather_stat exception: {}".format(e))
pass
else:
logger.info("Stat set has no bom station")
write_output(energy_stat.path, stat_set)
elif energy_stat.period and energy_stat.period.period_human == "all" and not latest:
time_series.period = human_to_period("all")
time_series.interval = human_to_interval("1M")
time_series.year = None
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
if not stat_set:
continue
demand_energy_and_value = demand_network_region_daily(
time_series=time_series,
network_region_code=energy_stat.network_region,
networks=energy_stat.networks)
stat_set.append_set(demand_energy_and_value)
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if energy_stat.network == NetworkNEM and energy_stat.network_region:
interconnector_flows = energy_interconnector_flows_and_emissions(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
stat_set.append_set(interconnector_flows)
if energy_stat.bom_station:
try:
weather_stats = weather_daily(
time_series=time_series,
station_code=energy_stat.bom_station,
network_region=energy_stat.network_region,
)
stat_set.append_set(weather_stats)
except NoResults as e:
logger.info("No weather results: {}".format(e))
except Exception:
pass
write_output(energy_stat.path, stat_set)
def export_all_daily() -> None:
session = SessionLocal()
network_regions = session.query(NetworkRegion).all()
cpi = gov_stats_cpi()
for network_region in network_regions:
network = network_from_network_code(network_region.network.code)
networks = None
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks)
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1d"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if network == NetworkNEM:
interconnector_flows = energy_interconnector_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
interconnector_emissions = energy_interconnector_emissions_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_emissions)
bom_station = get_network_region_weather_station(network_region.code)
if bom_station:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
stat_set.append_set(weather_stats)
if cpi:
stat_set.append_set(cpi)
write_output(f"v3/stats/au/{network_region.code}/daily.json", stat_set)
def export_all_monthly() -> None:
session = SessionLocal()
network_regions = session.query(NetworkRegion).all()
all_monthly = OpennemDataSet(code="au",
data=[],
version=get_version(),
created_at=datetime.now())
cpi = gov_stats_cpi()
all_monthly.append_set(cpi)
for network_region in network_regions:
network = network_from_network_code(network_region.network.code)
networks = None
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks)
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
if network == NetworkNEM:
interconnector_flows = energy_interconnector_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
interconnector_emissions = energy_interconnector_emissions_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_emissions)
all_monthly.append_set(stat_set)
bom_station = get_network_region_weather_station(network_region.code)
if bom_station:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
all_monthly.append_set(weather_stats)
write_output("v3/stats/au/all/monthly.json", all_monthly)
def export_energy(
stats: List[StatExport] = None,
priority: Optional[PriorityType] = None,
latest: Optional[bool] = False,
) -> None:
"""
Export energy stats from the export map
"""
if not stats:
export_map = get_export_map().get_by_stat_type(StatType.energy)
if priority:
export_map = export_map.get_by_priority(priority)
stats = export_map.resources
CURRENT_YEAR = datetime.now().year
for energy_stat in stats:
if energy_stat.stat_type != StatType.energy:
continue
# @FIX trim to NEM since it's the one with the shortest
# data time span.
# @TODO find a better and more flexible way to do this in the
# range method
date_range_networks = energy_stat.networks or []
if NetworkNEM in date_range_networks:
date_range_networks = [NetworkNEM]
date_range: ScadaDateRange = get_scada_range(
network=energy_stat.network, networks=date_range_networks)
# Migrate to this time_series
time_series = TimeSeries(
start=date_range.start,
end=date_range.end,
network=energy_stat.network,
year=energy_stat.year,
interval=energy_stat.interval,
period=human_to_period("1Y"),
)
if energy_stat.year:
if latest and energy_stat.year != CURRENT_YEAR:
continue
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
if not stat_set:
continue
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if energy_stat.network == NetworkNEM and energy_stat.network_region:
interconnector_flows = energy_interconnector_region_daily(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
stat_set.append_set(interconnector_flows)
interconnector_emissions = energy_interconnector_emissions_region_daily(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
stat_set.append_set(interconnector_emissions)
if energy_stat.bom_station:
weather_stats = weather_daily(
time_series=time_series,
station_code=energy_stat.bom_station,
network_region=energy_stat.network_region,
)
stat_set.append_set(weather_stats)
write_output(energy_stat.path, stat_set)
elif energy_stat.period and energy_stat.period.period_human == "all" and not latest:
time_series.period = human_to_period("all")
time_series.interval = human_to_interval("1M")
time_series.year = None
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
if not stat_set:
continue
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if energy_stat.network == NetworkNEM and energy_stat.network_region:
interconnector_flows = energy_interconnector_region_daily(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
stat_set.append_set(interconnector_flows)
interconnector_emissions = energy_interconnector_emissions_region_daily(
time_series=time_series,
networks_query=energy_stat.networks,
network_region_code=energy_stat.network_region_query
or energy_stat.network_region,
)
stat_set.append_set(interconnector_emissions)
if energy_stat.bom_station:
weather_stats = weather_daily(
time_series=time_series,
station_code=energy_stat.bom_station,
network_region=energy_stat.network_region,
)
stat_set.append_set(weather_stats)
stat_set = pad_stat_set(stat_set)
write_output(energy_stat.path, stat_set)