def registry_to_station(registry_station: dict,
_id: int) -> StationImportSchema:
station = StationImportSchema(
id=_id,
code=registry_station.get("station_id", ""),
name=registry_station.get("display_name", ""),
network_name=registry_station.get("display_name", ""),
)
station.location = LocationSchema(
state=registry_station.get("location", {}).get("state", None),
geocode_by="opennem.registry",
geocode_approved=True,
lat=registry_station.get("location", {}).get("latitude", None),
lng=registry_station.get("location", {}).get("longitude", None),
)
if "duid_data" not in registry_station:
logger.info("Registry: station has no duid data: {}".format(
registry_station.get("display_name", "")))
return station
for duid, registry_facility in registry_station["duid_data"].items():
network_region = map_compat_network_region(
registry_station.get("region_id", ""))
facility = FacilitySchema(
**{
"code":
duid,
"created_by":
"opennem.registry",
"created_at":
datetime.now(),
"network":
network_from_network_region(network_region),
"network_region":
network_region,
"station_code":
registry_station.get("station_id", ""),
"dispatch_type":
"GENERATOR",
"status":
parse_facility_status(
map_compat_facility_state(
registry_station.get("status", {}).get("state", ""))),
"fueltech":
parse_facility_fueltech(
map_compat_fueltech(
registry_facility.get("fuel_tech", None))),
"capacity_registered":
registry_facility.get("registered_capacity", None),
})
station.facilities.append(facility)
return station
def dudetailsummary_grouper(tables):
if "PARTICIPANT_REGISTRATION_DUDETAILSUMMARY" not in tables:
raise Exception("No dudetailsummary table")
records = tables["PARTICIPANT_REGISTRATION_DUDETAILSUMMARY"]
mms = tables["mms"] if "mms" in tables else {}
records = [
{
"date_start": parse_mms_date(i["START_DATE"]),
"date_end": parse_mms_date(i["END_DATE"]),
# "status": "operating",
"code": i["DUID"],
"network_region": i["REGIONID"],
"station_code": i["STATIONID"],
"participant_id": i["PARTICIPANTID"],
"dispatch_type": i["DISPATCHTYPE"],
} for i in records
]
now = datetime.now()
records = [{
**i,
"status":
parse_facility_status("retired")
if i["date_end"] <= now else parse_facility_status("operating"),
} for i in records]
grouped_records = {}
# First pass sorts facilities into stations
for k, v in groupby(records, lambda x: (x["station_code"], x["code"])):
key = k[0]
duid = k[1]
if key not in grouped_records:
grouped_records[key] = {}
grouped_records[key]["station_code"] = k[0]
# grouped_records[key]["participant"] = v[0]["PARTICIPANTID"]
grouped_records[key]["details"] = {}
grouped_records[key]["facilities"] = []
if duid not in grouped_records[key]["details"]:
grouped_records[key]["details"][duid] = []
grouped_records[key]["details"][duid] += list(v)
# Second pass flatten the records and we should get start and end dates
# and a derived status
for rec in grouped_records.keys():
for _, facility_group_records in grouped_records[rec]["details"].items(
):
# date_end_min = min(
# facility_group_records, key=lambda x: x["date_end"]
# )
date_end_max = max(facility_group_records,
key=lambda x: x["date_end"])
date_start_min = min(facility_group_records,
key=lambda x: x["date_start"])
grouped_rec = {
**date_end_max,
"date_start": date_start_min["date_start"],
}
if grouped_rec["date_end"].year == 2999:
grouped_rec["date_end"] = None
grouped_records[rec]["facilities"].append(grouped_rec)
grouped_records = [{
"station_code": i,
"facilities": v["facilities"]
} for i, v in grouped_records.items()]
tables["PARTICIPANT_REGISTRATION_DUDETAILSUMMARY"] = grouped_records
for record in grouped_records:
station_code = record["station_code"]
if station_code not in mms:
print("dudetailsummary: {} not in mms".format(station_code))
continue
# mms[station_code] = {**record, **mms[station_code]}
mms[station_code]["facilities"] = record["facilities"]
tables["mms"] = mms
return tables
def rel_grouper(records, station_code_map):
records_parsed = []
for _id, i in enumerate(records, start=2000):
name = station_name_cleaner(i["station_name"])
duid = normalize_duid(i["duid"])
unit = parse_unit_duid(i["unit_no"], duid)
fueltech = lookup_fueltech(
i["fuel_source_primary"],
i["fuel_source_descriptor"],
i["tech_primary"],
i["tech_primary_descriptor"],
i["dispatch_type"],
)
station_code = lookup_station_code([duid], i["station_name"],
station_code_map)
records_parsed.append({
"name":
name,
"code":
duid,
"status":
parse_facility_status("operating"),
"station_code":
station_code,
"network_region":
i["region"].strip(),
"network_name":
i["station_name"].strip(),
"unit_size":
clean_capacity(i["unit_size"]),
"unit_code":
get_unit_code(unit, duid, name),
"dispatch_type":
parse_dispatch_type(i["dispatch_type"]),
"fueltech":
parse_facility_fueltech(fueltech),
"capacity_registered":
clean_capacity(i["reg_cap"]),
"capacity_maximum":
clean_capacity(i["max_cap"]),
})
grouped_records = {}
for key, v in groupby(records_parsed, key=lambda v: v["station_code"]):
# key = k[1
if key not in grouped_records:
grouped_records[key] = []
grouped_records[key] += list(v)
coded_records = {}
_id = 2000
for station_code, rel in grouped_records.items():
station_name = rel[0]["network_name"]
if station_code in coded_records:
raise Exception("Code conflict: {}. {} {}".format(
station_code, station_name, coded_records[station_code]))
if not station_code:
raise Exception("Unmapped station: {}".format(rel))
coded_records[station_code] = {
"name": station_name_cleaner(station_name),
"network_name": station_name,
"code": station_code,
"id": _id,
"facilities": rel,
}
_id += 1
return coded_records
def gi_grouper(records, station_code_map):
# filter out records we don't want
records = list(filter(gi_filter, records))
records = [{
"name": station_name_cleaner(i["station_name"]),
**i
} for i in records]
grouped_records = {}
for k, v in groupby(records,
key=lambda v: (v["name"].strip(), v["owner"].strip())):
v = list(v)
key = k[0]
if key not in grouped_records:
grouped_records[key] = []
grouped_records[key] += v
records_parsed = []
for station_name, facilities in grouped_records.items():
facility_index = 0
for i in facilities:
name = station_name_cleaner(i["station_name"])
duid = normalize_duid(i["duid"])
unit = parse_unit_duid(i["unit_num"], duid)
units_num = i["unit_num"] or 1
unit_id = facility_index + (units_num - 1)
unit = parse_unit_duid(unit_id, duid)
fueltech = lookup_fueltech(i["FuelType"], techtype=i["TechType"])
if not duid:
duid = get_unit_code(unit, duid, i["station_name"])
facility_duids = [duid]
station_code = lookup_station_code(facility_duids, station_name,
station_code_map)
records_parsed.append({
# not a real station id
# "id": _id,
"name":
name,
"code":
duid,
"network_code":
duid,
"station_code":
station_code,
"network_region":
normalize_aemo_region(i["Region"]),
"network_name":
i["station_name"].strip(),
"dispatch_type":
"GENERATOR",
"fueltech":
parse_facility_fueltech(fueltech) if fueltech else None,
"status":
parse_facility_status(map_aemo_facility_status(i["status"])),
"registered":
parse_comissioned_date(i["SurveyEffective"]),
**get_capacities(i),
})
facility_index += 1
coded_records = {}
_id = 3000
for station_code, facilities in groupby(records_parsed,
key=lambda x: x["station_code"]):
# station_name = facilities[0]["name"]
facilities = list(facilities)
if not station_code:
raise Exception("Unmapped station {}: {}".format(
station_code, facilities))
if station_code not in coded_records:
coded_records[station_code] = {
"id": _id,
"code": station_code,
# "name": station_name_cleaner(station_name),
# "network_name": station_name,
"facilities": [],
}
_id += 1
coded_records[station_code]["facilities"] += facilities
grouped_records = {}
for station_code, station_record in coded_records.items():
grouped_records[station_code] = {
"id": station_record.get("id"),
"code": station_code,
"name": station_name_cleaner(station_name),
"network_name": station_name,
"facilities": [],
}
for facility in station_record["facilities"]:
grouped_records[station_code]["name"] = facility["name"]
grouped_records[station_code]["network_name"] = facility[
"network_name"]
facility.pop("name")
facility.pop("network_name")
grouped_records[station_code]["facilities"].append(facility)
return grouped_records