def run_energy_gapfill_for_network(network: NetworkSchema = NetworkNEM,
days: int = 7,
run_all: bool = False) -> None:
energy_gaps = query_energy_gaps(network, days=days)
energy_gaps_filtered = list(
filter(lambda x: x.has_power is True and x.has_energy is False,
energy_gaps))
if run_all:
energy_gaps_filtered = energy_gaps
logger.info("Found {} energy gaps interval hours".format(
len(energy_gaps_filtered)))
for gap in energy_gaps_filtered:
logger.info(
"{} Running for interval {} with power: {} energy: {} energy_value: {}"
.format(gap.network_id, gap.interval, gap.has_power,
gap.has_energy, gap.total_energy))
if settings.dry_run:
continue
dmin = gap.interval
dmax = dmin + timedelta(hours=1)
try:
run_energy_calc(dmin,
dmax,
network=network_from_network_code(gap.network_id))
except Exception:
logger.error("Error running {} energy gapfill for {} => {}".format(
gap.network_id, dmin, dmax))
def power_network_region_fueltech(
network_code: str = Query(..., description="Network code"),
network_region_code: str = Query(..., description="Network region code"),
month: date = Query(datetime.now().date(), description="Month to query"),
) -> OpennemDataSet:
network = network_from_network_code(network_code)
interval_obj = network.get_interval()
period_obj = human_to_period("1M")
scada_range = get_scada_range(network=network)
if not scada_range:
raise Exception("Require a scada range")
if not network:
raise Exception("Network not found")
time_series = TimeSeries(
start=scada_range.start,
month=month,
network=network,
interval=interval_obj,
period=period_obj,
)
stat_set = power_week(time_series,
network_region_code,
include_capacities=True)
if not stat_set:
raise Exception("No results")
return stat_set
def power_unit(
unit_code: str = Query(..., description="Unit code"),
network_code: str = Query(..., description="Network code"),
interval_human: str = Query(None, description="Interval"),
period_human: str = Query("7d", description="Period"),
engine=Depends(get_database_engine),
) -> OpennemDataSet:
network = network_from_network_code(network_code)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such network",
)
if not interval_human:
interval_human = "{}m".format(network.interval_size)
interval = human_to_interval(interval_human)
period = human_to_period(period_human)
units = get_unit("power")
stats = []
facility_codes = [normalize_duid(unit_code)]
query = power_facility_query(facility_codes, network.code, interval=interval, period=period)
with engine.connect() as c:
results = list(c.execute(query))
stats = [
DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None)
for i in results
]
if len(stats) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Unit stats not found",
)
output = stats_factory(
stats,
code=unit_code,
interval=interval,
period=period,
units=units,
network=network,
)
if not output:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No stats found",
)
return output
def check_opennem_interval_delays(network_code: str) -> bool:
"""Runs periodically and alerts if there is a current delay in output of power intervals"""
network = network_from_network_code(network_code)
env = ""
if settings.debug:
env = ".dev"
url = f"https://data{env}.opennem.org.au/v3/stats/au/{network.code}/power/7d.json"
resp = http.get(url)
if resp.status_code != 200 or not resp.ok:
logger.error("Error retrieving: {}".format(url))
return False
resp_json = resp.json()
if "data" not in resp_json:
logger.error("Error retrieving wem power: malformed response")
return False
data = resp_json["data"]
fueltech_data = data.pop()
history_end_date = fueltech_data["history"]["last"]
history_date = parse_date(history_end_date, dayfirst=False)
if not history_date:
logger.error(
"Could not read history date for opennem interval monitor")
return False
now_date = datetime.now().astimezone(
network.get_timezone()) # type: ignore
time_delta = chop_delta_microseconds(now_date - history_date) - timedelta(
minutes=network.interval_size)
logger.debug("Live time: {}, delay: {}".format(history_date, time_delta))
alert_threshold = network.monitor_interval_alert_threshold or settings.monitor_interval_alert_threshold or 60
if time_delta > timedelta(minutes=alert_threshold):
slack_message(
f"*WARNING*: OpenNEM {network.code} interval delay on {settings.env} currently: {time_delta}.\n",
tag_users=settings.monitoring_alert_slack_user,
)
return True
def energy_network_api(
engine=Depends(get_database_engine),
network_code: str = Query(..., description="Network code"),
interval_human: str = Query("1d", description="Interval"),
period_human: str = Query("1Y", description="Period"),
) -> OpennemDataSet:
results = []
network = network_from_network_code(network_code)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such network",
)
if not interval_human:
interval_human = "{}m".format(network.interval_size)
interval = human_to_interval(interval_human)
period = human_to_period(period_human)
units = get_unit("energy_giga")
query = energy_network(network=network, interval=interval, period=period)
with engine.connect() as c:
results = list(c.execute(query))
if len(results) < 1:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No results")
stats = [
DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None)
for i in results
]
result = stats_factory(
stats,
code=network.code,
network=network,
interval=interval,
period=period,
units=units,
)
if not result:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results found",
)
return result
def power_network_region_fueltech(
network_code: str = Query(..., description="Network code"),
network_region_code: str = Query(..., description="Network region code"),
month: date = Query(datetime.now().date(), description="Month to query"),
) -> OpennemDataSet:
network = None
try:
network = network_from_network_code(network_code)
except Exception:
raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND)
interval_obj = network.get_interval()
period_obj = human_to_period("1M")
scada_range = get_scada_range(network=network)
if not scada_range:
raise Exception("Require a scada range")
if not network:
raise Exception("Network not found")
networks = [network]
if network == NetworkNEM:
networks.append(NetworkAEMORooftop)
networks.append(NetworkAEMORooftopBackfill)
elif network == NetworkWEM:
networks.append(NetworkAPVI)
time_series = TimeSeries(
start=scada_range.start,
month=month,
network=network,
networks=networks,
interval=interval_obj,
period=period_obj,
)
stat_set = power_week(
time_series, network_region_code, include_capacities=True, networks_query=networks
)
if not stat_set:
raise Exception("No results")
return stat_set
def check_opennem_interval_delays(network_code: str) -> bool:
network = network_from_network_code(network_code)
if settings.debug:
env = ".dev"
url = (
f"https://data.opennem.org.au/v3/stats/au/{network.code}/power/7d.json"
f"https://data{env}.opennem.org.au/v3/stats/au/{network.code}/power/7d.json"
)
resp = http.get(url)
if resp.status_code != 200 or not resp.ok:
logger.error("Error retrieving: {}".format(url))
return False
resp_json = resp.json()
if "data" not in resp_json:
logger.error("Error retrieving wem power: malformed response")
return False
data = resp_json["data"]
fueltech_data = data.pop()
history_end_date = fueltech_data["history"]["last"]
history_date = parse_date(history_end_date, dayfirst=False)
if not history_date:
logger.error(
"Could not read history date for opennem interval monitor")
return False
now_date = datetime.now().astimezone(network.get_timezone())
time_delta = chop_microseconds(now_date - history_date)
logger.debug("Live time: {}, delay: {}".format(history_date, time_delta))
if time_delta > timedelta(hours=3):
slack_message(
"*WARNING*: OpenNEM {} interval delay on {} currently: {}\n".
format(network.code, settings.env, time_delta))
return True
def api_export_energy_year(
engine: Any = Depends(get_database_engine),
network_code: str = Query("WEM", description="Network code"),
year: int = Query(YEAR_CURRENT, description="Year to query"),
) -> OpennemDataSet:
if year > YEAR_CURRENT or year < 1996:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Invalid year"
)
network: NetworkSchema = network_from_network_code(network_code)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Invalid network",
)
stats = energy_network_fueltech_api(
network_code=network.code,
network_region=None,
interval="1d",
year=year,
period="1Y",
engine=engine,
)
weather = station_observations_api(
station_code="009021",
interval="1d",
year=year,
network_code="WEM",
engine=engine,
period=None,
)
price = price_network_region_api(
engine=engine,
network_code="WEM",
network_region_code="WEM",
interval="1d",
period=None,
year=year,
)
stats.data += weather.data + price.data
return stats
def get_power_example() -> OpennemDataSet:
network = network_from_network_code("NEM")
interval = human_to_interval("5m")
units = get_unit("power")
period = human_to_period("7d")
network_region_code = "NSW1"
test_rows = []
dt = datetime.fromisoformat("2021-01-15 10:00:00")
for ft in ["coal_black", "coal_brown"]:
for v in range(0, 3):
test_rows.append([dt, ft, v])
dt = dt + timedelta(minutes=5)
stats = [
DataQueryResult(interval=i[0],
result=i[2],
group_by=i[1] if len(i) > 1 else None)
for i in test_rows
]
assert len(stats) == 6, "Should have 6 stats"
result = stats_factory(
stats,
code=network_region_code or network.code,
network=network,
interval=interval,
period=period,
units=units,
region=network_region_code,
fueltech_group=True,
)
if not result:
raise Exception("Bad unit test data")
with open("power-nsw1.json", "w") as fh:
fh.write(result.json(indent=4))
return result
def api_export_energy_month(
engine: Any = Depends(get_database_engine),
network_code: str = Query("WEM", description="Network code"),
month: str = Query("all", description="Month to query"),
) -> OpennemDataSet:
network: NetworkSchema = network_from_network_code(network_code)
if month != "all":
raise HTTPException(
status_code=status.HTTP_501_NOT_IMPLEMENTED,
detail="Other months not yet supported",
)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Invalid network",
)
stats = energy_network_fueltech_api(
network_code=network.code,
period="all",
engine=engine,
interval="1M",
network_region=None,
)
price = price_network_region_api(
engine=engine,
network_code="WEM",
network_region_code="WEM",
interval="1M",
period="all",
)
stats.data += price.data
return stats
def energy_station(
engine=Depends(get_database_engine),
session: Session = Depends(get_database_session),
network_code: str = Query(..., description="Network code"),
station_code: str = Query(..., description="Station Code"),
interval: str = Query(None, description="Interval"),
period: str = Query("7d", description="Period"),
) -> OpennemDataSet:
"""
Get energy output for a station (list of facilities)
over a period
"""
network = network_from_network_code(network_code)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such network",
)
if not interval:
# @NOTE rooftop data is 15m
if station_code.startswith("ROOFTOP"):
interval = "15m"
else:
interval = "{}m".format(network.interval_size)
interval_obj = human_to_interval(interval)
period_obj = human_to_period(period)
units = get_unit("energy")
station = (
session.query(Station)
.join(Station.facilities)
.filter(Station.code == station_code)
.filter(Facility.network_id == network.code)
.one_or_none()
)
if not station:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Station not found")
if len(station.facilities) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Station has no facilities",
)
facility_codes = list(set([f.code for f in station.facilities]))
query = energy_facility_query(
facility_codes,
network=network,
interval=interval_obj,
period=period_obj,
)
logger.debug(query)
with engine.connect() as c:
row = list(c.execute(query))
if len(row) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Station stats not found",
)
results_energy = [
DataQueryResult(interval=i[0], group_by=i[1], result=i[2] if len(i) > 1 else None)
for i in row
]
results_market_value = [
DataQueryResult(interval=i[0], group_by=i[1], result=i[3] if len(i) > 1 else None)
for i in row
]
results_emissions = [
DataQueryResult(interval=i[0], group_by=i[1], result=i[4] if len(i) > 1 else None)
for i in row
]
if len(results_energy) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Station stats not found",
)
stats = stats_factory(
stats=results_energy,
units=units,
network=network,
interval=interval_obj,
period=period_obj,
code=station_code,
include_group_code=True,
)
if not stats:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Station stats not found",
)
stats_market_value = stats_factory(
stats=results_market_value,
units=get_unit("market_value"),
network=network,
interval=interval_obj,
period=period_obj,
code=station_code,
include_group_code=True,
)
stats.append_set(stats_market_value)
stats_emissions = stats_factory(
stats=results_emissions,
units=get_unit("emissions"),
network=network,
interval=interval_obj,
period=period_obj,
code=network.code.lower(),
include_group_code=True,
)
stats.append_set(stats_emissions)
return stats
def export_all_monthly() -> None:
session = SessionLocal()
network_regions = session.query(NetworkRegion).all()
all_monthly = OpennemDataSet(code="au",
data=[],
version=get_version(),
created_at=datetime.now())
cpi = gov_stats_cpi()
all_monthly.append_set(cpi)
for network_region in network_regions:
network = network_from_network_code(network_region.network.code)
networks = None
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks)
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
if network == NetworkNEM:
interconnector_flows = energy_interconnector_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
interconnector_emissions = energy_interconnector_emissions_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_emissions)
all_monthly.append_set(stat_set)
bom_station = get_network_region_weather_station(network_region.code)
if bom_station:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
all_monthly.append_set(weather_stats)
write_output("v3/stats/au/all/monthly.json", all_monthly)
def power_station(
station_code: str = Query(..., description="Station code"),
network_code: str = Query(..., description="Network code"),
since: datetime = Query(None, description="Since time"),
interval_human: str = Query(None, description="Interval"),
period_human: str = Query("7d", description="Period"),
session: Session = Depends(get_database_session),
engine=Depends(get_database_engine),
) -> OpennemDataSet:
if not since:
since = datetime.now() - human_to_timedelta("7d")
network = network_from_network_code(network_code)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such network",
)
if not interval_human:
# @NOTE rooftop data is 15m
if station_code.startswith("ROOFTOP"):
interval_human = "15m"
else:
interval_human = "{}m".format(network.interval_size)
interval = human_to_interval(interval_human)
period = human_to_period(period_human)
units = get_unit("power")
station = (
session.query(Station)
.join(Facility)
.filter(Station.code == station_code)
.filter(Facility.network_id == network.code)
.filter(Station.approved.is_(True))
.one_or_none()
)
if not station:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Station not found")
facility_codes = list(set([f.code for f in station.facilities]))
stats = []
query = power_facility_query(facility_codes, network=network, interval=interval, period=period)
logger.debug(query)
with engine.connect() as c:
results = list(c.execute(query))
stats = [
DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None)
for i in results
]
if len(stats) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Station stats not found",
)
result = stats_factory(
stats,
code=station_code,
network=network,
interval=interval,
period=period,
include_group_code=True,
units=units,
)
if not result:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results found",
)
return result
def power_network_fueltech_api(
network_code: str = Query(..., description="Network code"),
network_region: str = Query(None, description="Network region"),
interval_human: str = Query(None, description="Interval"),
period_human: str = Query("7d", description="Period"),
engine=Depends(get_database_engine),
) -> OpennemDataSet:
network = network_from_network_code(network_code)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such network",
)
if not interval_human:
interval_human = "{}m".format(network.interval_size)
interval = human_to_interval(interval_human)
period = human_to_period(period_human)
units = get_unit("power")
scada_range = get_scada_range(network=network)
query = power_network_fueltech(
network=network,
interval=interval,
period=period,
network_region=network_region,
scada_range=scada_range,
)
with engine.connect() as c:
results = list(c.execute(query))
stats = [
DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None)
for i in results
]
if len(stats) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Station stats not found",
)
result = stats_factory(
stats,
code=network.code,
network=network,
interval=interval,
period=period,
units=units,
region=network_region,
fueltech_group=True,
)
if not result:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results found",
)
return result
def emission_factor_per_network( # type: ignore
engine=Depends(get_database_engine), # type: ignore
network_code: str = Query(..., description="Network code"),
interval: str = Query("30m", description="Interval size"),
) -> Optional[OpennemDataSet]:
engine = get_database_engine()
network = None
try:
network = network_from_network_code(network_code)
except Exception:
raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND)
interval_obj = human_to_interval(interval)
period_obj = human_to_period("7d")
if not interval_obj:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Invalid interval size")
scada_range = get_scada_range(network=network)
if not scada_range:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Could not find a date range",
)
if not network:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Network not found",
)
time_series = TimeSeries(
start=scada_range.start,
network=network,
interval=interval_obj,
period=period_obj,
)
query = emission_factor_region_query(time_series=time_series)
with engine.connect() as c:
logger.debug(query)
row = list(c.execute(query))
if len(row) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results",
)
emission_factors = [
DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None)
for i in row
]
result = stats_factory(
emission_factors,
network=time_series.network,
period=time_series.period,
interval=time_series.interval,
units=get_unit("emissions_factor"),
group_field="emission_factor",
include_group_code=True,
include_code=True,
)
if not result or not result.data:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results",
)
return result
def fueltech_demand_mix(
engine=Depends(get_database_engine), # type: ignore
network_code: str = Query(..., description="Network code"),
) -> OpennemDataSet:
"""Return fueltech proportion of demand for a network
Args:
engine ([type], optional): Database engine. Defaults to Depends(get_database_engine).
Raises:
HTTPException: No results
Returns:
OpennemData: data set
"""
engine = get_database_engine()
network = None
try:
network = network_from_network_code(network_code)
except Exception:
raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND)
interval_obj = human_to_interval("5m")
period_obj = human_to_period("1d")
scada_range = get_scada_range(network=network)
if not scada_range:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Could not find a date range",
)
if not network:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Network not found",
)
time_series = TimeSeries(
start=scada_range.start,
network=network,
interval=interval_obj,
period=period_obj,
)
query = network_fueltech_demand_query(time_series=time_series)
with engine.connect() as c:
logger.debug(query)
row = list(c.execute(query))
if len(row) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results",
)
result_set = [
DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None)
for i in row
]
result = stats_factory(
result_set,
network=time_series.network,
period=time_series.period,
interval=time_series.interval,
units=get_unit("emissions_factor"),
group_field="emission_factor",
include_group_code=True,
include_code=True,
)
if not result or not result.data:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results",
)
return result
def get_export_map() -> StatMetadata:
"""
Generates a map of all export JSONs
"""
session = SessionLocal()
networks = session.query(Network).filter(Network.export_set.is_(True)).all()
if not networks:
raise Exception("No networks")
countries = list(set([network.country for network in networks]))
_exmap = []
for country in countries:
# @TODO derive this
scada_range = get_scada_range(network=NetworkAU, networks=[NetworkNEM, NetworkWEM])
if not scada_range:
raise Exception("Require a scada range")
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.live,
country=country,
date_range=scada_range,
network=NetworkAU,
networks=[NetworkNEM, NetworkWEM],
interval=NetworkAU.get_interval(),
period=human_to_period("7d"),
)
_exmap.append(export)
for year in range(
datetime.now().year,
scada_range.start.year - 1,
-1,
):
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.daily,
country=country,
date_range=scada_range,
network=NetworkAU,
networks=[NetworkNEM, NetworkWEM],
year=year,
interval=human_to_interval("1d"),
period=human_to_period("1Y"),
)
_exmap.append(export)
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.monthly,
country=country,
date_range=scada_range,
network=NetworkAU,
networks=[NetworkNEM, NetworkWEM],
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
_exmap.append(export)
for network in networks:
network_schema = network_from_network_code(network.code)
scada_range = get_scada_range(network=network_schema)
bom_station = get_network_region_weather_station(network.code)
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.live,
country=network.country,
date_range=scada_range,
network=network_schema,
bom_station=bom_station,
interval=network_schema.get_interval(),
period=human_to_period("7d"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
if not scada_range:
raise Exception("Require a scada range")
for year in range(
datetime.now().year,
scada_range.start.year - 1,
-1,
):
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.daily,
country=network.country,
date_range=scada_range,
network=network_schema,
bom_station=bom_station,
year=year,
period=human_to_period("1Y"),
interval=human_to_interval("1d"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.monthly,
country=network.country,
date_range=scada_range,
network=network_schema,
bom_station=bom_station,
interval=human_to_interval("1M"),
period=human_to_period("all"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
# Skip cases like wem/wem where region is supurfelous
if len(network.regions) < 2:
continue
for region in network.regions:
scada_range = get_scada_range(network=network_schema, network_region=region)
bom_station = get_network_region_weather_station(region.code)
if not scada_range:
raise Exception("Require a scada range")
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.live,
country=network.country,
date_range=scada_range,
network=network_schema,
network_region=region.code,
bom_station=bom_station,
period=human_to_period("7d"),
interval=network_schema.get_interval(),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
for year in range(
datetime.now().year,
scada_range.start.year - 1,
-1,
):
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.daily,
country=network.country,
date_range=scada_range,
network=network_schema,
network_region=region.code,
bom_station=bom_station,
year=year,
period=human_to_period("1Y"),
interval=human_to_interval("1d"),
)
_exmap.append(export)
export = StatExport(
stat_type=StatType.energy,
priority=PriorityType.monthly,
country=network.country,
date_range=scada_range,
network=network_schema,
network_region=region.code,
bom_station=bom_station,
period=human_to_period("all"),
interval=human_to_interval("1M"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
export_meta = StatMetadata(
date_created=datetime.now(), version=get_version(), resources=_exmap
)
return export_meta
def power_flows_network_week(
engine=Depends(get_database_engine), # type: ignore
network_code: str = Query(..., description="Network code"),
month: date = Query(datetime.now().date(), description="Month to query"),
) -> Optional[OpennemDataSet]:
engine = get_database_engine()
network = network_from_network_code(network_code)
interval_obj = network.get_interval()
period_obj = human_to_period("1M")
scada_range = get_scada_range(network=network)
if not scada_range:
raise Exception("Require a scada range")
if not network:
raise Exception("Network not found")
time_series = TimeSeries(
start=scada_range.start,
month=month,
network=network,
interval=interval_obj,
period=period_obj,
)
query = interconnector_flow_network_regions_query(time_series=time_series)
with engine.connect() as c:
logger.debug(query)
row = list(c.execute(query))
if len(row) < 1:
raise Exception("No results from query: {}".format(query))
imports = [
DataQueryResult(interval=i[0], result=i[4], group_by=i[1] if len(i) > 1 else None)
for i in row
]
result = stats_factory(
imports,
# code=network_region_code or network.code,
network=time_series.network,
period=time_series.period,
interval=time_series.interval,
units=get_unit("regional_trade"),
# fueltech_group=True,
group_field="power",
include_group_code=True,
include_code=True,
)
if not result or not result.data:
raise Exception("No results")
INVERT_SETS = ["VIC1->NSW1", "VIC1->SA1"]
inverted_data = []
for ds in result.data:
if ds.code in INVERT_SETS:
ds_inverted = invert_flow_set(ds)
inverted_data.append(ds_inverted)
else:
inverted_data.append(ds)
result.data = inverted_data
return result
def export_all_daily() -> None:
session = SessionLocal()
network_regions = session.query(NetworkRegion).all()
cpi = gov_stats_cpi()
for network_region in network_regions:
network = network_from_network_code(network_region.network.code)
networks = None
if network_region.code == "WEM":
networks = [NetworkWEM, NetworkAPVI]
scada_range: ScadaDateRange = get_scada_range(network=network,
networks=networks)
time_series = TimeSeries(
start=scada_range.start,
end=scada_range.end,
network=network,
interval=human_to_interval("1d"),
period=human_to_period("all"),
)
stat_set = energy_fueltech_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
if not stat_set:
continue
# Hard coded to NEM only atm but we'll put has_interconnectors
# in the metadata to automate all this
if network == NetworkNEM:
interconnector_flows = energy_interconnector_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_flows)
interconnector_emissions = energy_interconnector_emissions_region_daily(
time_series=time_series,
networks_query=networks,
network_region_code=network_region.code,
)
stat_set.append_set(interconnector_emissions)
bom_station = get_network_region_weather_station(network_region.code)
if bom_station:
weather_stats = weather_daily(
time_series=time_series,
station_code=bom_station,
network_region=network_region.code,
)
stat_set.append_set(weather_stats)
if cpi:
stat_set.append_set(cpi)
write_output(f"v3/stats/au/{network_region.code}/daily.json", stat_set)
dmax,
network=network_from_network_code(gap.network_id))
except Exception:
logger.error("Error running {} energy gapfill for {} => {}".format(
gap.network_id, dmin, dmax))
def run_energy_gapfill(
days: int = 14,
networks: List[NetworkSchema] = [
NetworkNEM, NetworkWEM, NetworkAPVI, NetworkAEMORooftop
],
run_all: bool = False,
) -> None:
"""Run gapfill of energy values - will find gaps in energy values and run energy_sum"""
for network in networks:
logger.info("Running energy gapfill for {}".format(network.code))
try:
run_energy_gapfill_for_network(network, days=days, run_all=run_all)
except Exception as e:
logger.error("gap_fill run error: {}".format(e))
# debug entry point
if __name__ == "__main__":
# run_energy_gapfill(days=365/2)
dmin = datetime.fromisoformat("2022-02-28T00:00:00+10:00")
dmax = dmin + timedelta(days=3)
run_energy_calc(dmin, dmax, network=network_from_network_code("NEM"))
def energy_network_fueltech_api(
network_code: str = Query(None, description="Network code"),
network_region: str = Query(None, description="Network region"),
interval_human: str = Query("1d", description="Interval"),
year: int = Query(None, description="Year to query"),
period_human: str = Query("1Y", description="Period"),
engine=Depends(get_database_engine),
) -> OpennemDataSet:
network = network_from_network_code(network_code)
interval = human_to_interval(interval_human)
period_obj: TimePeriod = human_to_period("1Y")
if period_human:
period_obj = human_to_period(period_human)
units = get_unit("energy_giga")
query = ""
if year and isinstance(year, int):
period_obj = human_to_period("1Y")
if year > datetime.now().year or year < 1996:
raise HTTPException(
status_code=status.HTTP_406_NOT_ACCEPTABLE,
detail="Not a valid year",
)
scada_range = get_scada_range(network=network)
query = energy_network_fueltech_year(
network=network,
interval=interval,
year=year,
network_region=network_region,
scada_range=scada_range,
)
elif period_obj and period_obj.period_human == "all":
scada_range = get_scada_range(network=network)
query = energy_network_fueltech_all(
network=network,
network_region=network_region,
scada_range=scada_range,
)
else:
query = energy_network_fueltech(
network=network,
interval=interval,
period=period_obj,
network_region=network_region,
)
# print(query)
with engine.connect() as c:
results = list(c.execute(query))
stats = [
DataQueryResult(interval=i[0], result=i[1], group_by=i[2] if len(i) > 1 else None)
for i in results
]
if len(stats) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Energy stats not found",
)
result = stats_factory(
stats,
code=network.code,
network=network,
interval=interval,
period=period_obj,
units=units,
region=network_region,
fueltech_group=True,
)
if not result:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No stats")
return result
def price_network_endpoint(
engine: Engine = Depends(get_database_engine),
network_code: str = Path(..., description="Network code"),
network_region: Optional[str] = Query(None, description="Network region code"),
forecasts: bool = Query(False, description="Include price forecasts"),
) -> OpennemDataSet:
"""Returns network and network region price info for interval which defaults to network
interval size
Args:
engine ([type], optional): Database engine. Defaults to Depends(get_database_engine).
Raises:
HTTPException: No results
Returns:
OpennemData: data set
"""
engine = get_database_engine()
network = None
try:
network = network_from_network_code(network_code)
except Exception:
raise HTTPException(detail="Network not found", status_code=status.HTTP_404_NOT_FOUND)
interval_obj = human_to_interval("5m")
period_obj = human_to_period("1d")
scada_range = get_balancing_range(network=network, include_forecasts=forecasts)
if not scada_range:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Could not find a date range",
)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Network not found",
)
time_series = TimeSeries(
start=scada_range.start,
network=network,
interval=interval_obj,
period=period_obj,
)
if network_region:
time_series.network.regions = [NetworkNetworkRegion(code=network_region)]
query = network_region_price_query(time_series=time_series)
with engine.connect() as c:
logger.debug(query)
row = list(c.execute(query))
if len(row) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results",
)
result_set = [
DataQueryResult(interval=i[0], result=i[3], group_by=i[2] if len(i) > 1 else None)
for i in row
]
result = stats_factory(
result_set,
network=time_series.network,
period=time_series.period,
interval=time_series.interval,
units=get_unit("price"),
group_field="price",
include_group_code=True,
include_code=True,
)
if not result or not result.data:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results",
)
return result
def generate_weekly_export_map() -> StatMetadata:
"""
Generate export map for weekly power series
@TODO deconstruct this into separate methods and schema
ex. network.get_scada_range(), network_region.get_bom_station() etc.
"""
session = get_scoped_session()
networks = session.query(Network).filter(
Network.export_set.is_(True)).all()
if not networks:
raise Exception("No networks")
countries = list(set([network.country for network in networks]))
_exmap = []
# Loop countries
for country in countries:
# @TODO derive this
scada_range = get_scada_range(network=NetworkAU,
networks=[NetworkNEM, NetworkWEM])
if not scada_range:
raise Exception("Require a scada range for NetworkAU")
for year, week in week_series(scada_range.end, scada_range.start):
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.history,
country=country,
network=NetworkAU,
networks=[NetworkNEM, NetworkWEM],
year=year,
week=week,
date_range=date_range_from_week(year, week, NetworkAU),
interval=human_to_interval("30m"),
period=human_to_period("7d"),
)
_exmap.append(export)
# Loop networks
for network in networks:
network_schema = network_from_network_code(network.code)
scada_range = get_scada_range(network=network_schema)
if not scada_range:
raise Exception("Require a scada range for network: {}".format(
network.code))
for year, week in week_series(scada_range.end, scada_range.start):
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.history,
country=network.country,
network=network_schema,
year=year,
week=week,
date_range=date_range_from_week(year, week, NetworkAU),
interval=human_to_interval(f"{network.interval_size}m"),
period=human_to_period("7d"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
# Skip cases like wem/wem where region is supurfelous
if len(network.regions) < 2:
continue
for region in network.regions:
scada_range = get_scada_range(network=network_schema,
network_region=region.code)
if not scada_range:
logger.error(
"Require a scada range for network {} and region {}".
format(network_schema.code, region.code))
continue
for year, week in week_series(scada_range.end, scada_range.start):
export = StatExport(
stat_type=StatType.power,
priority=PriorityType.history,
country=network.country,
network=network_schema,
year=year,
week=week,
date_range=date_range_from_week(
year, week, network_from_network_code(network.code)),
interval=human_to_interval(f"{network.interval_size}m"),
period=human_to_period("7d"),
)
if network.code == "WEM":
export.networks = [NetworkWEM, NetworkAPVI]
export.network_region_query = "WEM"
_exmap.append(export)
export_meta = StatMetadata(date_created=datetime.now(),
version=get_version(),
resources=_exmap)
return export_meta
def price_network_region_api(
engine=Depends(get_database_engine),
network_code: str = Query(..., description="Network code"),
network_region_code: str = Query(..., description="Region code"),
interval_human: str = Query(None, description="Interval"),
period_human: str = Query("7d", description="Period"),
year: Optional[int] = None,
) -> OpennemDataSet:
network = network_from_network_code(network_code)
if not network:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such network",
)
if not interval_human:
interval_human = "{}m".format(network.interval_size)
interval = human_to_interval(interval_human)
period_obj = None
if period_human:
period_obj = human_to_period(period_human)
units = get_unit("price")
scada_range = get_scada_range(network=network)
if period_obj and period_obj.period_human == "all" and interval.interval_human == "1M":
query = price_network_monthly(
network=network,
network_region_code=network_region_code,
scada_range=scada_range,
)
else:
query = price_network_region(
network=network,
network_region_code=network_region_code,
interval=interval,
period=period_obj,
scada_range=scada_range,
year=year,
)
with engine.connect() as c:
results = list(c.execute(query))
if len(results) < 1:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No data found")
stats = [
DataQueryResult(interval=i[0], result=i[2], group_by=i[1] if len(i) > 1 else None)
for i in results
]
result = stats_factory(
stats,
code=network.code,
region=network_region_code,
network=network,
interval=interval,
period=period_obj,
units=units,
group_field="price",
)
if not result:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No results found",
)
return result
def station_observations_api(
station_code: str = Query(None, description="Station code"),
interval_human: str = Query("15m", description="Interval"),
period_human: str = Query("7d", description="Period"),
station_codes: List[str] = [],
network_code: str = "NEM",
timezone: str = None,
offset: str = None,
year: int = None,
engine=Depends(get_database_engine),
) -> OpennemDataSet:
units = get_unit("temperature")
if not interval_human:
interval = "15m"
if not period_human:
period = "7d"
network = None
if network_code:
network = network_from_network_code(network_code)
if station_code:
station_codes = [station_code]
interval = human_to_interval(interval_human)
period = human_to_period(period_human)
if timezone:
timezone = pytz.timezone(timezone)
if offset:
timezone = get_fixed_timezone(offset)
scada_range = None
if network:
scada_range = get_scada_range(network=network)
query = observation_query(
station_codes=station_codes,
interval=interval,
period=period,
network=network,
scada_range=scada_range,
year=year,
)
with engine.connect() as c:
results = list(c.execute(query))
stats = [
DataQueryResult(interval=i[0],
result=i[2],
group_by=i[1] if len(i) > 1 else None) for i in results
]
if len(stats) < 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Station stats not found",
)
result = stats_factory(
stats=stats,
units=units,
interval=interval,
period=period,
network=network,
code="bom",
group_field="temperature",
)
return result
