def registry_init(): registry = registry_import() for station in registry: station_dict = station.dict(exclude={"id"}) # pprint(station_dict) # pylint:disable no-member station_model = fromdict(Station(), station_dict) station_model.approved = True station_model.approved_at = datetime.now() station_model.approved_by = "opennem.registry" station_model.created_by = "opennem.registry" # location station_model.location = fromdict( Location(), station_dict["location"], exclude=["id"], ) if station.location.lat and station.location.lng: station_model.location.geom = "SRID=4326;POINT({} {})".format( station.location.lng, station.location.lat) for fac in station.facilities: f = Facility(**fac.dict( exclude={ "id", "fueltech", "status", "network", "revision_ids", "scada_power", })) f.network_id = fac.network.code if fac.fueltech: f.fueltech_id = fac.fueltech.code f.status_id = fac.status.code f.approved = True f.approved_by = "opennem.registry" f.created_by = "opennem.registry" f.approved_at = datetime.now() f.created_at = datetime.now() station_model.facilities.append(f) s.add(station_model) s.commit()
def setup_network_flow_stations(network: NetworkSchema = NetworkNEM) -> None: """Creats a station for each network region and a facility for each of imports exports so that energy values in facility_scada can be matched up per-region""" session = SessionLocal() network_regions = get_network_region_schema(network) for network_region in network_regions: logger.info( f"Setting up for network {network.code} and region: {network_region.code}" ) flow_station_id = generated_flow_station_id(network, network_region) flow_station = (session.query(Station).filter_by( code=flow_station_id).filter_by( network_code=flow_station_id).one_or_none()) if not flow_station: flow_station = Station(code=flow_station_id, network_code=flow_station_id) flow_station.approved = False flow_station.created_by = IMPORTER_NAME if not flow_station.location: flow_station.location = Location( state=state_from_network_region(network_region.code)) flow_station.name = "Flows for {} state {}".format( network.code.upper(), state_from_network_region(network_region.code.upper())) flow_facilities = [(i, generated_flow_station_id(network, network_region, i)) for i in FlowDirection] for (flow_direction, flow_facility_id) in flow_facilities: flow_facility_model = (session.query(Facility).filter_by( code=flow_facility_id).filter_by( dispatch_type=DispatchType.GENERATOR).filter_by( network_id=network.code).filter_by( network_region=network_region.code).one_or_none()) if not flow_facility_model: flow_facility_model = Facility( # type: ignore code=flow_facility_id, dispatch_type=DispatchType.GENERATOR, network_id=network.code, network_region=network_region.code, ) flow_facility_model.status_id = "operating" flow_facility_model.approved = False flow_facility_model.created_by = IMPORTER_NAME flow_facility_model.fueltech_id = flow_direction.value flow_facility_model.updated_at = datetime.now() flow_station.facilities.append(flow_facility_model) session.add(flow_station) logger.info( "Created network trading flow station facility: {}".format( flow_facility_id)) session.commit() return None
def opennem_init() -> None: session = SessionLocal() station_data = load_data("opennem_stations.json", from_project=True) stations = StationSet() for s in station_data: stations.add_dict(s) for station in stations: logger.debug("Adding station: {}".format(station.code)) station_model = session.query(Station).filter_by( code=station.code).one_or_none() if not station_model: station_model = Station(code=station.code) station_model.approved = True station_model.approved_at = datetime.now() station_model.approved_by = "opennem.init" station_model.created_by = "opennem.init" station_model.description = station.description station_model.name = station.name station_model.network_name = station.network_name if not station_model.location: station_model.location = Location() if station.location: station_model.location.locality = station.location.locality station_model.location.state = station.location.state station_model.location.postcode = station.location.postcode station_model.location.country = station.location.country if station.location.lat and station.location.lng: station_model.location.geom = "SRID=4326;POINT({} {})".format( station.location.lng, station.location.lat) for fac in station.facilities: facility_model = (session.query(Facility).filter_by( code=fac.code).filter_by( network_id=fac.network.code).one_or_none()) if not facility_model: facility_model = Facility(code=fac.code, network_id=fac.network.code) facility_model.network_region = fac.network_region facility_model.fueltech_id = fac.fueltech.code facility_model.status_id = fac.status.code facility_model.dispatch_type = fac.dispatch_type facility_model.capacity_registered = fac.capacity_registered facility_model.registered = fac.registered facility_model.unit_id = fac.unit_id facility_model.unit_number = fac.unit_number facility_model.unit_alias = fac.unit_alias facility_model.unit_capacity = fac.unit_capacity facility_model.emissions_factor_co2 = fac.emissions_factor_co2 facility_model.approved = fac.approved facility_model.approved_by = fac.approved_by facility_model.created_by = "opennem.init" facility_model.approved_by = "opennem.init" session.add(facility_model) station_model.facilities.append(facility_model) logger.debug(" => Added facility {}".format(fac.code)) print(station_model) session.add(station_model) session.commit()
def registry_init() -> None: registry = registry_import() session = SessionLocal() for station in registry: station_dict = station.dict(exclude={"id"}) # pprint(station_dict) station_model = session.query(Station).filter_by( code=station.code).one_or_none() if not station_model: # pylint:disable no-member station_model = fromdict(Station(), station_dict) station_model.approved = True station_model.approved_at = datetime.now() station_model.approved_by = "opennem.registry" station_model.created_by = "opennem.registry" # location station_model.location = fromdict( Location(), station_dict["location"], exclude=["id"], ) if station.location.lat and station.location.lng: station_model.location.geom = "SRID=4326;POINT({} {})".format( station.location.lng, station.location.lat) session.add(station_model) session.commit() for fac in station.facilities: f = (session.query(Facility).filter_by(code=fac.code).filter_by( network_id=fac.network.code).one_or_none()) if not f: print("new facility {} {}".format(fac.code, fac.network.code)) f = Facility(**fac.dict( exclude={ "id", "fueltech", "status", "network", "revision_ids", "scada_power", })) f.approved_by = "opennem.registry" f.created_by = "opennem.registry" f.approved_at = datetime.now() f.created_at = datetime.now() f.network_id = fac.network.code if fac.fueltech: f.fueltech_id = fac.fueltech.code if fac.network.code: f.network_code = fac.network.code f.status_id = fac.status.code f.approved = True if station_model.id: f.station_id = station_model.id else: station_model.facilities.append(f) session.add(station_model) session.add(f) try: session.commit() except IntegrityError as e: logger.error(e)
def import_station_set(stations: StationSet, only_insert_facilities: bool = False) -> None: session = SessionLocal() for station in stations: add_or_update: str = "Updating" station_model = session.query(Station).filter_by( code=station.code).one_or_none() if not station_model: add_or_update = "Adding" station_model = Station(code=station.code) station_model.created_by = "opennem.init" logger.debug("{} station: {}".format(add_or_update, station.code)) if station.description: station_model.description = station.description if station.name: station_model.name = station.name station_model.approved = station.approved if station.approved: station_model.approved = True station_model.approved_at = datetime.now() station_model.approved_by = "opennem.init" else: station_model.approved_at = None station_model.approved_by = None if station.website_url: station_model.website_url = station.website_url if station.network_name: station_model.network_name = station.network_name if not station_model.location: station_model.location = Location() if station.location: station_model.location.locality = station.location.locality station_model.location.state = station.location.state station_model.location.postcode = station.location.postcode station_model.location.country = station.location.country if station.location.lat and station.location.lng: station_model.location.geom = "SRID=4326;POINT({} {})".format( station.location.lng, station.location.lat) session.add(station_model) session.commit() for fac in station.facilities: facility_added = False facility_model = (session.query(Facility).filter_by( code=fac.code).filter_by( network_id=fac.network.code).one_or_none()) if facility_model and only_insert_facilities: logger.debug(" => skip updating {}".format( facility_model.code)) continue if not facility_model: facility_model = Facility(code=fac.code, network_id=fac.network.code) facility_added = True if facility_model.station_id != station_model.id: facility_model.station_id = station_model.id logger.debug(" => Reassigned facility {} to station {}".format( facility_model.code, station_model.code)) # fueltech if fac.fueltech: facility_model.fueltech_id = fac.fueltech.code if fac.fueltech_id: facility_model.fueltech_id = fac.fueltech_id # network if fac.network: facility_model.network_id = fac.network.code if fac.network_id: facility_model.network_id = fac.network_id # status if fac.status: facility_model.status_id = fac.status.code if fac.status_id: facility_model.status_id = fac.status_id # rest if fac.dispatch_type: facility_model.dispatch_type = fac.dispatch_type if fac.capacity_registered: facility_model.capacity_registered = fac.capacity_registered if fac.registered: facility_model.registered = fac.registered if fac.network_region: facility_model.network_region = fac.network_region facility_model.unit_id = fac.unit_id facility_model.unit_number = fac.unit_number facility_model.unit_alias = fac.unit_alias facility_model.unit_capacity = fac.unit_capacity if fac.emissions_factor_co2: facility_model.emissions_factor_co2 = fac.emissions_factor_co2 if fac.approved: facility_model.approved = fac.approved if fac.approved: facility_model.approved_by = "opennem.importer" else: facility_model.approved_by = None if not facility_model.created_by: facility_model.created_by = "opennem.init" session.add(facility_model) station_model.facilities.append(facility_model) logger.debug(" => {} facility {} to {} {}".format( "Added" if facility_added else "Updated", fac.code, facility_model.network_id, facility_model.network_region, )) session.add(station_model) session.commit()
def import_nem_interconnects() -> None: session = SessionLocal() # Load the MMS CSV file that contains interconnector info csv_data = load_data( "mms/PUBLIC_DVD_INTERCONNECTOR_202006010000.CSV", from_project=True, ) # gotta be a string otherwise decode if not isinstance(csv_data, str): csv_data = decode_bytes(csv_data) # parse the AEMO CSV into schemas aemo_table_set = None try: aemo_table_set = parse_aemo_csv(csv_data) except AEMOParserException as e: logger.error(e) return None records: List[MarketConfigInterconnector] = aemo_table_set.get_table( "MARKET_CONFIG_INTERCONNECTOR").get_records() for interconnector in records: if not isinstance(interconnector, MarketConfigInterconnector): raise Exception("Not what we're looking for ") # skip SNOWY # @TODO do these need to be remapped for historical if interconnector.regionfrom in [ "SNOWY1" ] or interconnector.regionto in ["SNOWY1"]: continue logger.debug(interconnector) interconnector_station = (session.query(Station).filter_by( code=interconnector.interconnectorid).filter_by( network_code=interconnector.interconnectorid).one_or_none()) if not interconnector_station: interconnector_station = Station( code=interconnector.interconnectorid, network_code=interconnector.interconnectorid, ) interconnector_station.approved = False interconnector_station.created_by = "opennem.importer.interconnectors" if not interconnector_station.location: interconnector_station.location = Location( state=state_from_network_region(interconnector.regionfrom)) interconnector_station.name = interconnector.description # for network_region in [interconnector.regionfrom, interconnector.regionto]: # Fac1 int_facility = (session.query(Facility).filter_by( code=interconnector.interconnectorid).filter_by( dispatch_type=DispatchType.GENERATOR).filter_by( network_id="NEM").filter_by(network_region=interconnector. regionfrom).one_or_none()) if not int_facility: int_facility = Facility( # type: ignore code=interconnector.interconnectorid, dispatch_type=DispatchType.GENERATOR, network_id="NEM", network_region=interconnector.regionfrom, ) int_facility.status_id = "operating" int_facility.approved = False int_facility.created_by = "opennem.importer.interconnectors" int_facility.fueltech_id = None int_facility.interconnector = True int_facility.interconnector_region_to = interconnector.regionto interconnector_station.facilities.append(int_facility) session.add(interconnector_station) logger.debug("Created interconnector station: {}".format( interconnector_station.code)) session.commit() return None