def process_item(self, item, spider=None): s = self.session() records_updated = 0 records_created = 0 for record in item: created = False duid = normalize_duid(record["DUID"]) station_code = facility_map_station( duid, normalize_duid(record["STATIONID"])) station = (s.query(Station).filter( Station.network_code == station_code).one_or_none()) facility = (s.query(Facility).filter( Facility.network_code == duid).one_or_none()) if not station: station = Station( code=station_code, network_code=station_code, network_id="NEM", created_by="au.nem.mms.statdualloc", ) if not facility: facility = Facility( code=duid, network_code=duid, network_id="NEM", status_id="retired", created_by="au.nem.mms.statdualloc", ) records_created += 1 created = True else: facility.updated_by = "au.nem.mms.statdualloc" records_updated += 1 facility.station = station try: s.add(facility) s.commit() except Exception as e: logger.error(e) logger.debug("{} facility record with id {}".format( "Created" if created else "Updated", duid)) logger.info("Created {} facility records and updated {}".format( records_created, records_updated))
def process_item(self, item, spider=None): s = self.session() records_updated = 0 records_created = 0 for record in item: created = False participant_code = normalize_duid( record["facilities"][0]["PARTICIPANTID"]) # Step 1. Find participant by code or create participant = (s.query(ParticipantModel).filter( ParticipantModel.code == participant_code).one_or_none()) if not participant: participant = ParticipantModel( code=participant_code, network_code=participant_code, created_by="au.nem.mms.dudetail_summary", ) logger.debug("Created participant {}".format(participant_code)) else: participant.updated_by = "au.nem.mms.dudetail_summary" # Step 3. now create the facilities and associate for facility_record in record["facilities"]: duid = normalize_duid(facility_record["DUID"]) station_code = facility_map_station( duid, normalize_duid(record["id"])) network_region = normalize_aemo_region( facility_record["REGIONID"]) date_start = facility_record["date_start"] date_end = facility_record["date_end"] facility_state = "retired" # Step 2. Find station or create station = (s.query(Station).filter( Station.network_code == station_code).one_or_none()) if not station: station = Station( code=station_code, network_code=station_code, network_id="NEM", created_by="au.nem.mms.dudetail_summary", ) logger.debug("Created station {}".format(station_code)) else: station.updated_by = "au.nem.mms.dudetail_summary" station.participant = participant if date_end == None: facility_state = "operating" if not "DISPATCHTYPE" in facility_record: logger.error( "MMS dudetailsummary: Invalid record: {}".format( facility_record)) continue dispatch_type = parse_dispatch_type( facility_record["DISPATCHTYPE"]) facility = (s.query(Facility).filter( Facility.network_code == duid).one_or_none()) if not facility: facility = Facility( code=duid, network_code=duid, dispatch_type=dispatch_type, created_by="au.nem.mms.dudetail_summary", ) records_created += 1 created = True else: facility.updated_by = "au.nem.mms.dudetail_summary" records_updated += 1 facility.network_region = network_region facility.deregistered = date_end facility.registered = date_start facility.status_id = facility_state if not facility.dispatch_type: facility.dispatch_type = dispatch_type # Associations facility_station_id = facility_map_station(duid, station.id) facility.station_id = station.id try: s.add(facility) s.commit() except Exception as e: logger.error(e) logger.debug( "MMS DudetailSummary:{} facility record with id {}".format( "Created" if created else "Updated", duid)) logger.info( "MMS DudetailSummary: Created {} facility records and updated {}". format(records_created, records_updated))
def load_opennem_facilities(): station_fixture = load_data("facility_registry.json", from_fixture=True) stations = [{"station_code": k, **v} for k, v in station_fixture.items()] engine = db_connect() session = sessionmaker(bind=engine) s = session() for station_data in stations: station = None facilities = [{ "code": k, **v } for k, v in station_data["duid_data"].items()] # update facilities for facility_data in facilities: facility_duid = facility_data["code"] station_name = station_name_cleaner(station_data["display_name"]) # station_code = normalize_duid(station_data["station_code"]) station_code = facility_map_station( facility_duid, normalize_duid(station_data["station_code"])) station_state = map_compat_facility_state( station_data["status"]["state"]) station_network = "WEM" if station_data["location"][ "state"] == "WA" else "NEM" station = s.query(Station).filter( Station.network_code == station_code).one_or_none() if not station: station = Station( network_id=station_network, code=station_code, network_code=station_code, name=station_name, network_name=station_data["display_name"], created_by="opennem.load_facilities", ) logger.info("Created station: {} {} ".format( station_name, station_code)) s.add(station) s.commit() facility_status = station_state # Network region facility_network_region = map_compat_network_region( station_data["region_id"]) # Fueltech facility_fueltech = None if "fuel_tech" in facility_data and facility_data["fuel_tech"]: facility_fueltech = map_compat_fueltech( facility_data["fuel_tech"]) # Capacity facility_capacity = None if "registered_capacity" in facility_data and facility_data[ "registered_capacity"]: facility_capacity = clean_capacity( facility_data["registered_capacity"]) facility = None try: facility = (s.query(Facility).filter( Facility.network_code == facility_duid).one_or_none()) except MultipleResultsFound: logger.error("Multiple facilities found for duid {}".format( facility_duid)) # facility = ( # s.query(Facility) # .filter(Facility.network_code == facility_duid) # .first() # ) continue if not facility: facility = Facility( code=facility_duid, network_code=facility_duid, network_region=facility_network_region, created_by="opennem.load_facilities", ) logger.info("Created facility: {} {} to station {} ".format( facility_duid, facility_fueltech, station_code)) if not facility.unit_id: facility.unit_id = 1 facility.unit_number = 1 if facility_capacity and not facility.unit_capacity: facility.unit_capacity = facility_capacity if facility_capacity and not facility.capacity_registered: facility.capacity_registered = facility_capacity if facility_fueltech and not facility.fueltech_id: facility.fueltech_id = facility_fueltech if not facility.status_id: facility.status_id = facility_status if not facility.station: facility.station = station s.add(facility) s.commit()