def process_item(self, item, spider=None): s = self.session() records_updated = 0 records_created = 0 for record in item: created = False participant_code = normalize_duid( record["facilities"][0]["PARTICIPANTID"]) # Step 1. Find participant by code or create participant = (s.query(ParticipantModel).filter( ParticipantModel.code == participant_code).one_or_none()) if not participant: participant = ParticipantModel( code=participant_code, network_code=participant_code, created_by="au.nem.mms.dudetail_summary", ) logger.debug("Created participant {}".format(participant_code)) else: participant.updated_by = "au.nem.mms.dudetail_summary" # Step 3. now create the facilities and associate for facility_record in record["facilities"]: duid = normalize_duid(facility_record["DUID"]) station_code = facility_map_station( duid, normalize_duid(record["id"])) network_region = normalize_aemo_region( facility_record["REGIONID"]) date_start = facility_record["date_start"] date_end = facility_record["date_end"] facility_state = "retired" # Step 2. Find station or create station = (s.query(Station).filter( Station.network_code == station_code).one_or_none()) if not station: station = Station( code=station_code, network_code=station_code, network_id="NEM", created_by="au.nem.mms.dudetail_summary", ) logger.debug("Created station {}".format(station_code)) else: station.updated_by = "au.nem.mms.dudetail_summary" station.participant = participant if date_end == None: facility_state = "operating" if not "DISPATCHTYPE" in facility_record: logger.error( "MMS dudetailsummary: Invalid record: {}".format( facility_record)) continue dispatch_type = parse_dispatch_type( facility_record["DISPATCHTYPE"]) facility = (s.query(Facility).filter( Facility.network_code == duid).one_or_none()) if not facility: facility = Facility( code=duid, network_code=duid, dispatch_type=dispatch_type, created_by="au.nem.mms.dudetail_summary", ) records_created += 1 created = True else: facility.updated_by = "au.nem.mms.dudetail_summary" records_updated += 1 facility.network_region = network_region facility.deregistered = date_end facility.registered = date_start facility.status_id = facility_state if not facility.dispatch_type: facility.dispatch_type = dispatch_type # Associations facility_station_id = facility_map_station(duid, station.id) facility.station_id = station.id try: s.add(facility) s.commit() except Exception as e: logger.error(e) logger.debug( "MMS DudetailSummary:{} facility record with id {}".format( "Created" if created else "Updated", duid)) logger.info( "MMS DudetailSummary: Created {} facility records and updated {}". format(records_created, records_updated))
def registry_init() -> None: registry = registry_import() session = SessionLocal() for station in registry: station_dict = station.dict(exclude={"id"}) # pprint(station_dict) station_model = session.query(Station).filter_by( code=station.code).one_or_none() if not station_model: # pylint:disable no-member station_model = fromdict(Station(), station_dict) station_model.approved = True station_model.approved_at = datetime.now() station_model.approved_by = "opennem.registry" station_model.created_by = "opennem.registry" # location station_model.location = fromdict( Location(), station_dict["location"], exclude=["id"], ) if station.location.lat and station.location.lng: station_model.location.geom = "SRID=4326;POINT({} {})".format( station.location.lng, station.location.lat) session.add(station_model) session.commit() for fac in station.facilities: f = (session.query(Facility).filter_by(code=fac.code).filter_by( network_id=fac.network.code).one_or_none()) if not f: print("new facility {} {}".format(fac.code, fac.network.code)) f = Facility(**fac.dict( exclude={ "id", "fueltech", "status", "network", "revision_ids", "scada_power", })) f.approved_by = "opennem.registry" f.created_by = "opennem.registry" f.approved_at = datetime.now() f.created_at = datetime.now() f.network_id = fac.network.code if fac.fueltech: f.fueltech_id = fac.fueltech.code if fac.network.code: f.network_code = fac.network.code f.status_id = fac.status.code f.approved = True if station_model.id: f.station_id = station_model.id else: station_model.facilities.append(f) session.add(station_model) session.add(f) try: session.commit() except IntegrityError as e: logger.error(e)
def import_station_set(stations: StationSet, only_insert_facilities: bool = False) -> None: session = SessionLocal() for station in stations: add_or_update: str = "Updating" station_model = session.query(Station).filter_by( code=station.code).one_or_none() if not station_model: add_or_update = "Adding" station_model = Station(code=station.code) station_model.created_by = "opennem.init" logger.debug("{} station: {}".format(add_or_update, station.code)) if station.description: station_model.description = station.description if station.name: station_model.name = station.name station_model.approved = station.approved if station.approved: station_model.approved = True station_model.approved_at = datetime.now() station_model.approved_by = "opennem.init" else: station_model.approved_at = None station_model.approved_by = None if station.website_url: station_model.website_url = station.website_url if station.network_name: station_model.network_name = station.network_name if not station_model.location: station_model.location = Location() if station.location: station_model.location.locality = station.location.locality station_model.location.state = station.location.state station_model.location.postcode = station.location.postcode station_model.location.country = station.location.country if station.location.lat and station.location.lng: station_model.location.geom = "SRID=4326;POINT({} {})".format( station.location.lng, station.location.lat) session.add(station_model) session.commit() for fac in station.facilities: facility_added = False facility_model = (session.query(Facility).filter_by( code=fac.code).filter_by( network_id=fac.network.code).one_or_none()) if facility_model and only_insert_facilities: logger.debug(" => skip updating {}".format( facility_model.code)) continue if not facility_model: facility_model = Facility(code=fac.code, network_id=fac.network.code) facility_added = True if facility_model.station_id != station_model.id: facility_model.station_id = station_model.id logger.debug(" => Reassigned facility {} to station {}".format( facility_model.code, station_model.code)) # fueltech if fac.fueltech: facility_model.fueltech_id = fac.fueltech.code if fac.fueltech_id: facility_model.fueltech_id = fac.fueltech_id # network if fac.network: facility_model.network_id = fac.network.code if fac.network_id: facility_model.network_id = fac.network_id # status if fac.status: facility_model.status_id = fac.status.code if fac.status_id: facility_model.status_id = fac.status_id # rest if fac.dispatch_type: facility_model.dispatch_type = fac.dispatch_type if fac.capacity_registered: facility_model.capacity_registered = fac.capacity_registered if fac.registered: facility_model.registered = fac.registered if fac.network_region: facility_model.network_region = fac.network_region facility_model.unit_id = fac.unit_id facility_model.unit_number = fac.unit_number facility_model.unit_alias = fac.unit_alias facility_model.unit_capacity = fac.unit_capacity if fac.emissions_factor_co2: facility_model.emissions_factor_co2 = fac.emissions_factor_co2 if fac.approved: facility_model.approved = fac.approved if fac.approved: facility_model.approved_by = "opennem.importer" else: facility_model.approved_by = None if not facility_model.created_by: facility_model.created_by = "opennem.init" session.add(facility_model) station_model.facilities.append(facility_model) logger.debug(" => {} facility {} to {} {}".format( "Added" if facility_added else "Updated", fac.code, facility_model.network_id, facility_model.network_region, )) session.add(station_model) session.commit()