def registry_to_station(registry_station: dict, _id: int) -> StationImportSchema: station = StationImportSchema( id=_id, code=registry_station.get("station_id", ""), name=registry_station.get("display_name", ""), network_name=registry_station.get("display_name", ""), ) station.location = LocationSchema( state=registry_station.get("location", {}).get("state", None), geocode_by="opennem.registry", geocode_approved=True, lat=registry_station.get("location", {}).get("latitude", None), lng=registry_station.get("location", {}).get("longitude", None), ) if "duid_data" not in registry_station: logger.info("Registry: station has no duid data: {}".format( registry_station.get("display_name", ""))) return station for duid, registry_facility in registry_station["duid_data"].items(): network_region = map_compat_network_region( registry_station.get("region_id", "")) facility = FacilitySchema( **{ "code": duid, "created_by": "opennem.registry", "created_at": datetime.now(), "network": network_from_network_region(network_region), "network_region": network_region, "station_code": registry_station.get("station_id", ""), "dispatch_type": "GENERATOR", "status": parse_facility_status( map_compat_facility_state( registry_station.get("status", {}).get("state", ""))), "fueltech": parse_facility_fueltech( map_compat_fueltech( registry_facility.get("fuel_tech", None))), "capacity_registered": registry_facility.get("registered_capacity", None), }) station.facilities.append(facility) return station
def dudetailsummary_grouper(tables): if "PARTICIPANT_REGISTRATION_DUDETAILSUMMARY" not in tables: raise Exception("No dudetailsummary table") records = tables["PARTICIPANT_REGISTRATION_DUDETAILSUMMARY"] mms = tables["mms"] if "mms" in tables else {} records = [ { "date_start": parse_mms_date(i["START_DATE"]), "date_end": parse_mms_date(i["END_DATE"]), # "status": "operating", "code": i["DUID"], "network_region": i["REGIONID"], "station_code": i["STATIONID"], "participant_id": i["PARTICIPANTID"], "dispatch_type": i["DISPATCHTYPE"], } for i in records ] now = datetime.now() records = [{ **i, "status": parse_facility_status("retired") if i["date_end"] <= now else parse_facility_status("operating"), } for i in records] grouped_records = {} # First pass sorts facilities into stations for k, v in groupby(records, lambda x: (x["station_code"], x["code"])): key = k[0] duid = k[1] if key not in grouped_records: grouped_records[key] = {} grouped_records[key]["station_code"] = k[0] # grouped_records[key]["participant"] = v[0]["PARTICIPANTID"] grouped_records[key]["details"] = {} grouped_records[key]["facilities"] = [] if duid not in grouped_records[key]["details"]: grouped_records[key]["details"][duid] = [] grouped_records[key]["details"][duid] += list(v) # Second pass flatten the records and we should get start and end dates # and a derived status for rec in grouped_records.keys(): for _, facility_group_records in grouped_records[rec]["details"].items( ): # date_end_min = min( # facility_group_records, key=lambda x: x["date_end"] # ) date_end_max = max(facility_group_records, key=lambda x: x["date_end"]) date_start_min = min(facility_group_records, key=lambda x: x["date_start"]) grouped_rec = { **date_end_max, "date_start": date_start_min["date_start"], } if grouped_rec["date_end"].year == 2999: grouped_rec["date_end"] = None grouped_records[rec]["facilities"].append(grouped_rec) grouped_records = [{ "station_code": i, "facilities": v["facilities"] } for i, v in grouped_records.items()] tables["PARTICIPANT_REGISTRATION_DUDETAILSUMMARY"] = grouped_records for record in grouped_records: station_code = record["station_code"] if station_code not in mms: print("dudetailsummary: {} not in mms".format(station_code)) continue # mms[station_code] = {**record, **mms[station_code]} mms[station_code]["facilities"] = record["facilities"] tables["mms"] = mms return tables
def rel_grouper(records, station_code_map): records_parsed = [] for _id, i in enumerate(records, start=2000): name = station_name_cleaner(i["station_name"]) duid = normalize_duid(i["duid"]) unit = parse_unit_duid(i["unit_no"], duid) fueltech = lookup_fueltech( i["fuel_source_primary"], i["fuel_source_descriptor"], i["tech_primary"], i["tech_primary_descriptor"], i["dispatch_type"], ) station_code = lookup_station_code([duid], i["station_name"], station_code_map) records_parsed.append({ "name": name, "code": duid, "status": parse_facility_status("operating"), "station_code": station_code, "network_region": i["region"].strip(), "network_name": i["station_name"].strip(), "unit_size": clean_capacity(i["unit_size"]), "unit_code": get_unit_code(unit, duid, name), "dispatch_type": parse_dispatch_type(i["dispatch_type"]), "fueltech": parse_facility_fueltech(fueltech), "capacity_registered": clean_capacity(i["reg_cap"]), "capacity_maximum": clean_capacity(i["max_cap"]), }) grouped_records = {} for key, v in groupby(records_parsed, key=lambda v: v["station_code"]): # key = k[1 if key not in grouped_records: grouped_records[key] = [] grouped_records[key] += list(v) coded_records = {} _id = 2000 for station_code, rel in grouped_records.items(): station_name = rel[0]["network_name"] if station_code in coded_records: raise Exception("Code conflict: {}. {} {}".format( station_code, station_name, coded_records[station_code])) if not station_code: raise Exception("Unmapped station: {}".format(rel)) coded_records[station_code] = { "name": station_name_cleaner(station_name), "network_name": station_name, "code": station_code, "id": _id, "facilities": rel, } _id += 1 return coded_records
def gi_grouper(records, station_code_map): # filter out records we don't want records = list(filter(gi_filter, records)) records = [{ "name": station_name_cleaner(i["station_name"]), **i } for i in records] grouped_records = {} for k, v in groupby(records, key=lambda v: (v["name"].strip(), v["owner"].strip())): v = list(v) key = k[0] if key not in grouped_records: grouped_records[key] = [] grouped_records[key] += v records_parsed = [] for station_name, facilities in grouped_records.items(): facility_index = 0 for i in facilities: name = station_name_cleaner(i["station_name"]) duid = normalize_duid(i["duid"]) unit = parse_unit_duid(i["unit_num"], duid) units_num = i["unit_num"] or 1 unit_id = facility_index + (units_num - 1) unit = parse_unit_duid(unit_id, duid) fueltech = lookup_fueltech(i["FuelType"], techtype=i["TechType"]) if not duid: duid = get_unit_code(unit, duid, i["station_name"]) facility_duids = [duid] station_code = lookup_station_code(facility_duids, station_name, station_code_map) records_parsed.append({ # not a real station id # "id": _id, "name": name, "code": duid, "network_code": duid, "station_code": station_code, "network_region": normalize_aemo_region(i["Region"]), "network_name": i["station_name"].strip(), "dispatch_type": "GENERATOR", "fueltech": parse_facility_fueltech(fueltech) if fueltech else None, "status": parse_facility_status(map_aemo_facility_status(i["status"])), "registered": parse_comissioned_date(i["SurveyEffective"]), **get_capacities(i), }) facility_index += 1 coded_records = {} _id = 3000 for station_code, facilities in groupby(records_parsed, key=lambda x: x["station_code"]): # station_name = facilities[0]["name"] facilities = list(facilities) if not station_code: raise Exception("Unmapped station {}: {}".format( station_code, facilities)) if station_code not in coded_records: coded_records[station_code] = { "id": _id, "code": station_code, # "name": station_name_cleaner(station_name), # "network_name": station_name, "facilities": [], } _id += 1 coded_records[station_code]["facilities"] += facilities grouped_records = {} for station_code, station_record in coded_records.items(): grouped_records[station_code] = { "id": station_record.get("id"), "code": station_code, "name": station_name_cleaner(station_name), "network_name": station_name, "facilities": [], } for facility in station_record["facilities"]: grouped_records[station_code]["name"] = facility["name"] grouped_records[station_code]["network_name"] = facility[ "network_name"] facility.pop("name") facility.pop("network_name") grouped_records[station_code]["facilities"].append(facility) return grouped_records