def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: """Run scraper for SDGE MyAccount if enabled. Retrying a bad login will lock the account. If a login fails, mark all data sources for this account as disabled. """ configuration = SdgeMyAccountConfiguration( meter.utility_account_id, meter.service_id, meter.direction, meter.interval, meter.commodity, ) return run_datafeed( SdgeMyAccountScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, disable_login_on_error=True, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: """Check if datasource is enabled; disable on bad login attempts. Retrying a bad login will lock the account. If a login fails, mark all data sources for this account as disabled. """ meta = datasource.meta or {} configuration = epo_schneider.EnergyProfilerConfiguration( base_url="https://csapps.pacificpower.net/idm/business-insights", account_id=meter.utility_account_id, epo_meter_id=meta.get("pacificPowerORMeterNumber"), channel_id=meta.get("channelId", None), ) return run_datafeed( PacificPowerIntervalScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, disable_login_on_error=True, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = SceWebsiteConfiguration( account=account, meter=meter, datasource=datasource, params=params, task_id=task_id, ) return run_datafeed( SceWebsiteScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, disable_login_on_error=True, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: """Check if datasource is enabled; disable on bad login attempts. Retrying a bad login will lock the account. If a login fails, mark all data sources for this account as disabled. """ configuration = epo_schneider.EnergyProfilerConfiguration( base_url= "https://smudpm.epo.schneider-electric.com/smudpm/cgi/eponline.exe", account_id=meter.utility_account_id, epo_meter_id=meter.utility_service.service_id, ) return run_datafeed( epo_schneider.EnergyProfilerScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, disable_login_on_error=True, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, metascraper=False, ) -> Status: # If there's a generation service id for the meter, get generation partials (with gen_service_id) # and T&D partials (with service_id). Otherwise, get bundled bills. is_partial = meter.utility_service.gen_service_id is not None configuration = SceReactBasicBillingConfiguration( service_id=meter.service_id, gen_service_id=meter.utility_service.gen_service_id, utility_account_id=meter.utility_service.utility_account_id, scrape_bills=not is_partial, scrape_partial_bills=is_partial, metascraper=metascraper, account_datasource_id=datasource._account_data_source, ) return run_datafeed( SceReactBasicBillingScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: meter_numbers = meter.service_id.split(",") # if totalized is set in meta, get list of meter numbers totalized = (datasource.meta or {}).get("totalized") if totalized: meter_numbers = totalized.split(",") configuration = SCLMeterWatchConfiguration(meter_numbers=meter_numbers, meter=meter) return run_datafeed( SCLMeterWatchScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: try: site_id = datasource.meta.get("site_id") except AttributeError: log.info("Site ID not set") site_id = None configuration = PowerTrackConfiguration(meter_id=meter.service_id, site_id=site_id) return run_datafeed( PowerTrackScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = LADWPBillPdfConfiguration( meter_number=meter.service_id, utility_account_id=meter.utility_service.utility_account_id, commodity=meter.commodity, account_name=(datasource.meta or {}).get("accountName"), ) # If meter has a recent bill, don't go to website since ladwp.com is fragile. # last_closing is last element of tuple latest_closing = meter.bills_range[-1] if latest_closing and latest_closing >= date.today() - timedelta(days=21): log.info("latest bill is fresh (%s); stopping now", latest_closing) return Status.COMPLETED return run_datafeed( LADWPBillPdfScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, metascraper=False, ) -> Status: # if there's a generation service id, get partial bills instead of full bills is_partial = meter.utility_service.gen_service_id is not None configuration = SceReactEnergyManagerBillingConfiguration( utility=meter.utility_service.utility, utility_account_id=meter.utility_account_id, service_id=meter.service_id, scrape_bills=not is_partial, scrape_partial_bills=is_partial, metascraper=metascraper, ) return run_datafeed( SceReactEnergyManagerBillingScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, disable_login_on_error=True, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, metascraper=False, ) -> Status: configuration = SceReactEnergyManagerGreenButtonConfiguration( service_id=meter.service_id, meta=datasource.meta, meter=meter, metascraper=metascraper, ) return run_datafeed( SceReactEnergyManagerGreenButtonScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, disable_login_on_error=True, )
def test_run_datafeed_interval(self): """Run a test scraper that creates interval data.""" us = self.meter.utility_service configuration = TestConfiguration( us.service_id, us.gen_service_id, scrape_readings=True ) meter_ds = ( db.session.query(SnapmeterMeterDataSource) .filter_by(meter=self.meter) .first() ) start_dt = datetime.now() - timedelta(days=7) params = { "data_start": start_dt.strftime("%Y-%m-%d"), "data_end": datetime.now().strftime("%Y-%m-%d"), } rval = run_datafeed( TestIntervalScraper, self.account, self.meter, meter_ds, params, configuration=configuration, task_id=uuid.uuid4().hex, ) self.assertEqual(Status.SUCCEEDED, rval) # created readings readings = [ row for row in db.session.query(MeterReading).filter_by(meter=self.meter.oid) ] self.assertEqual(1, len(readings)) self.assertEqual(start_dt.date(), readings[0].occurred) self.assertEqual([1.0] * 96, readings[0].readings)
def test_run_datafeed_partial(self): """Run a test scraper that creates partial bills.""" us = self.meter.utility_service configuration = TestConfiguration( us.service_id, us.gen_service_id, scrape_partial_bills=True ) meter_ds = ( db.session.query(SnapmeterMeterDataSource) .filter_by(meter=self.meter) .first() ) params = { "data_start": (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d"), "data_end": datetime.now().strftime("%Y-%m-%d"), } rval = run_datafeed( TestPartialBillScraper, self.account, self.meter, meter_ds, params, configuration=configuration, task_id=uuid.uuid4().hex, ) self.assertEqual(Status.SUCCEEDED, rval) # created a partial bill bills = db.session.query(PartialBill).filter_by(service=us.oid) self.assertEqual(1, bills.count()) bill = bills.first() self.assertEqual(date.today(), bill.closing) self.assertEqual(100, bill.cost) self.assertEqual(25, bill.used) self.assertEqual( PartialBillProviderType.GENERATION_ONLY.value, bill.provider_type )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: utility_service = meter.utility_service configuration = PgeBillPdfConfiguration( utility_service.utility, utility_service.utility_account_id, utility_service.gen_utility, utility_service.gen_utility_account_id, datasource, ) return run_datafeed( PgeBillPdfScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, disable_login_on_error=True, notify_on_login_error=False, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: """Run the SMT API integration to gather interval data.""" esiid = (datasource.meta or {}).get("esiid") if esiid is None: msg = "Missing ESIID for datasource {}, meter {}.".format( datasource.oid, meter.oid) raise DataSourceConfigurationError(msg) configuration = SmartMeterTexasConfiguration(esiid) return run_datafeed( SmartMeterTexasScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: """ pge-urjanet-v3 has three jobs: 1) Scrape any generation partial bills 2) Set generation utility, if we have it. 3) Look for any PDF's on the service that we can attach to existing SMD Bills or SMD Partial Bills. """ utility_service = meter.utility_service urja_datasource = PacificGasElectricXMLDatasource( utility_service.utility, utility_service.utility_account_id, utility_service.service_id, utility_service.gen_utility, utility_service.gen_utility_account_id, utility_service, ) transformer = PacificGasElectricUrjaXMLTransformer() conn = db.urjanet_connection() try: urja_datasource.conn = conn scraper_config = BaseUrjanetConfiguration( urja_datasource=urja_datasource, urja_transformer=transformer, utility_name=meter.utility_service.utility, fetch_attachments=True, partial_type=PartialBillProviderType.GENERATION_ONLY, ) scraper_config.scrape_pdfs = True return run_datafeed( PacGAndEPDFAndGenPartialScraper, account, meter, datasource, params, configuration=scraper_config, task_id=task_id, meter_only= True, # Upload PDF's found to just this meter, not others in account. ) finally: conn.close()
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: SnapmeterMeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: meta = datasource.meta or {} configuration = Configuration(mvweb_id=meta.get("mvWebId"), interval=meter.interval) # reduce load on MVWeb servers: skip if meter has data from within the last 3 days and there are no gaps max_reading = meter.readings_range.max_date or date.today() - timedelta( days=365) interval_age = (date.today() - max_reading).days date_range = DateRange( *iso_to_dates(params.get("data_start"), params.get("data_end"))) # freshest we can expect is 3 days old date_range = DateRange( date_range.start_date, min(date_range.end_date, date.today() - timedelta(days=3)), ) expected = (date_range.end_date - date_range.start_date).days + 1 days_with_data = (db.session.query(MeterReading).filter( MeterReading.meter == meter.oid, MeterReading.occurred >= date_range.start_date, MeterReading.occurred <= date_range.end_date, ).count()) log.info( "days with data from %s - %s = %s", date_range.start_date, date_range.end_date, days_with_data, ) if interval_age <= 3 and days_with_data == expected: log.info( "skipping MVWeb run: meter %s has recent interval data (%s) and no gaps", meter.oid, max_reading, ) return Status.SKIPPED return run_datafeed( LADWPMVWebScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def test_run_datafeed_bill(self): """Run a test scraper that scrapes a bill.""" us = self.meter.utility_service configuration = TestConfiguration( us.service_id, us.gen_service_id, scrape_bills=True ) meter_ds = ( db.session.query(SnapmeterMeterDataSource) .filter_by(meter=self.meter) .first() ) params = { "data_start": (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d"), "data_end": datetime.now().strftime("%Y-%m-%d"), } rval = run_datafeed( TestBillScraper, self.account, self.meter, meter_ds, params, configuration=configuration, task_id=uuid.uuid4().hex, ) self.assertEqual(Status.SUCCEEDED, rval) # writes bill data to csv path = os.path.join(config.WORKING_DIRECTORY, "bills.csv") with open(path) as f: reader = csv.reader(f) rows = [row for row in reader] self.assertEqual(2, len(rows)) # header row self.assertEqual( ["Service ID", "Start", "End", "Cost", "Used", "Peak"], rows[0] ) # data row self.assertEqual( [ str(us.service_id), (date.today() - timedelta(days=30)).strftime("%Y-%m-%d"), date.today().strftime("%Y-%m-%d"), "100", "25", "10", ], rows[1], )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: conf = SaltRiverBillingConfiguration(account_id=meter.utility_account_id) return run_datafeed( SaltRiverBillingScraper, account, meter, datasource, params, configuration=conf, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = _UtilityName_Configuration(meter_id=meter.service_id) return run_datafeed( _UtilityName_Scraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = SmdPartialBillingScraperConfiguration(meter) return run_datafeed( SmdPartialBillingScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, disable_login_on_error=True, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: SnapmeterMeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = Configuration(metering_point=meter.service_id) return run_datafeed( Scraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = PGEEnergyExpertConfiguration(datasource.meta.get("itemId")) return run_datafeed( PGEEnergyExpertScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = FPLMyAccountConfiguration( meter.utility_service.utility_account_id) return run_datafeed( FPLMyAccountScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = PSEIntervalConfiguration(meter.service_id, datasource.meta.get("siteName")) return run_datafeed( PSEIntervalScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = HECOGridConfiguration( meter_id=meter.service_id, interval=meter.interval ) return run_datafeed( HECOScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: acct_ds = datasource.account_data_source configuration = SolarEdgeConfiguration(meter_id=meter.service_id, site_id=acct_ds.username) return run_datafeed( SolarEdgeScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = PowayWaterConfiguration( account_id=meter.utility_account_id) return run_datafeed( PowayWaterScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = SolrenGridConfiguration( inverter_id=meter.service_id, site_id=datasource.meta.get("site_id")) return run_datafeed( SolrenScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = DukeIntervalConfiguration( meter.service_id, channel_id=datasource.meta.get("channelId", None) ) return run_datafeed( DukeIntervalScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )
def datafeed( account: SnapmeterAccount, meter: Meter, datasource: MeterDataSource, params: dict, task_id: Optional[str] = None, ) -> Status: configuration = SMUDMyAccountBillingConfiguration( utility=meter.utility_service.utility, account_id=meter.utility_service.utility_account_id, ) return run_datafeed( SMUDMyAccountBillingScraper, account, meter, datasource, params, configuration=configuration, task_id=task_id, )