def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
"""Run scraper for SDGE MyAccount if enabled.
Retrying a bad login will lock the account. If a login fails, mark all data sources
for this account as disabled.
"""
configuration = SdgeMyAccountConfiguration(
meter.utility_account_id,
meter.service_id,
meter.direction,
meter.interval,
meter.commodity,
)
return run_datafeed(
SdgeMyAccountScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
disable_login_on_error=True,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
"""Check if datasource is enabled; disable on bad login attempts.
Retrying a bad login will lock the account. If a login fails, mark all data sources
for this account as disabled.
"""
meta = datasource.meta or {}
configuration = epo_schneider.EnergyProfilerConfiguration(
base_url="https://csapps.pacificpower.net/idm/business-insights",
account_id=meter.utility_account_id,
epo_meter_id=meta.get("pacificPowerORMeterNumber"),
channel_id=meta.get("channelId", None),
)
return run_datafeed(
PacificPowerIntervalScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
disable_login_on_error=True,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = SceWebsiteConfiguration(
account=account,
meter=meter,
datasource=datasource,
params=params,
task_id=task_id,
)
return run_datafeed(
SceWebsiteScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
disable_login_on_error=True,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
"""Check if datasource is enabled; disable on bad login attempts.
Retrying a bad login will lock the account. If a login fails, mark all data sources
for this account as disabled.
"""
configuration = epo_schneider.EnergyProfilerConfiguration(
base_url=
"https://smudpm.epo.schneider-electric.com/smudpm/cgi/eponline.exe",
account_id=meter.utility_account_id,
epo_meter_id=meter.utility_service.service_id,
)
return run_datafeed(
epo_schneider.EnergyProfilerScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
disable_login_on_error=True,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
metascraper=False,
) -> Status:
# If there's a generation service id for the meter, get generation partials (with gen_service_id)
# and T&D partials (with service_id). Otherwise, get bundled bills.
is_partial = meter.utility_service.gen_service_id is not None
configuration = SceReactBasicBillingConfiguration(
service_id=meter.service_id,
gen_service_id=meter.utility_service.gen_service_id,
utility_account_id=meter.utility_service.utility_account_id,
scrape_bills=not is_partial,
scrape_partial_bills=is_partial,
metascraper=metascraper,
account_datasource_id=datasource._account_data_source,
)
return run_datafeed(
SceReactBasicBillingScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
meter_numbers = meter.service_id.split(",")
# if totalized is set in meta, get list of meter numbers
totalized = (datasource.meta or {}).get("totalized")
if totalized:
meter_numbers = totalized.split(",")
configuration = SCLMeterWatchConfiguration(meter_numbers=meter_numbers,
meter=meter)
return run_datafeed(
SCLMeterWatchScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
try:
site_id = datasource.meta.get("site_id")
except AttributeError:
log.info("Site ID not set")
site_id = None
configuration = PowerTrackConfiguration(meter_id=meter.service_id,
site_id=site_id)
return run_datafeed(
PowerTrackScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = LADWPBillPdfConfiguration(
meter_number=meter.service_id,
utility_account_id=meter.utility_service.utility_account_id,
commodity=meter.commodity,
account_name=(datasource.meta or {}).get("accountName"),
)
# If meter has a recent bill, don't go to website since ladwp.com is fragile.
# last_closing is last element of tuple
latest_closing = meter.bills_range[-1]
if latest_closing and latest_closing >= date.today() - timedelta(days=21):
log.info("latest bill is fresh (%s); stopping now", latest_closing)
return Status.COMPLETED
return run_datafeed(
LADWPBillPdfScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
metascraper=False,
) -> Status:
# if there's a generation service id, get partial bills instead of full bills
is_partial = meter.utility_service.gen_service_id is not None
configuration = SceReactEnergyManagerBillingConfiguration(
utility=meter.utility_service.utility,
utility_account_id=meter.utility_account_id,
service_id=meter.service_id,
scrape_bills=not is_partial,
scrape_partial_bills=is_partial,
metascraper=metascraper,
)
return run_datafeed(
SceReactEnergyManagerBillingScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
disable_login_on_error=True,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
metascraper=False,
) -> Status:
configuration = SceReactEnergyManagerGreenButtonConfiguration(
service_id=meter.service_id,
meta=datasource.meta,
meter=meter,
metascraper=metascraper,
)
return run_datafeed(
SceReactEnergyManagerGreenButtonScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
disable_login_on_error=True,
)
def test_run_datafeed_interval(self):
"""Run a test scraper that creates interval data."""
us = self.meter.utility_service
configuration = TestConfiguration(
us.service_id, us.gen_service_id, scrape_readings=True
)
meter_ds = (
db.session.query(SnapmeterMeterDataSource)
.filter_by(meter=self.meter)
.first()
)
start_dt = datetime.now() - timedelta(days=7)
params = {
"data_start": start_dt.strftime("%Y-%m-%d"),
"data_end": datetime.now().strftime("%Y-%m-%d"),
}
rval = run_datafeed(
TestIntervalScraper,
self.account,
self.meter,
meter_ds,
params,
configuration=configuration,
task_id=uuid.uuid4().hex,
)
self.assertEqual(Status.SUCCEEDED, rval)
# created readings
readings = [
row
for row in db.session.query(MeterReading).filter_by(meter=self.meter.oid)
]
self.assertEqual(1, len(readings))
self.assertEqual(start_dt.date(), readings[0].occurred)
self.assertEqual([1.0] * 96, readings[0].readings)
def test_run_datafeed_partial(self):
"""Run a test scraper that creates partial bills."""
us = self.meter.utility_service
configuration = TestConfiguration(
us.service_id, us.gen_service_id, scrape_partial_bills=True
)
meter_ds = (
db.session.query(SnapmeterMeterDataSource)
.filter_by(meter=self.meter)
.first()
)
params = {
"data_start": (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d"),
"data_end": datetime.now().strftime("%Y-%m-%d"),
}
rval = run_datafeed(
TestPartialBillScraper,
self.account,
self.meter,
meter_ds,
params,
configuration=configuration,
task_id=uuid.uuid4().hex,
)
self.assertEqual(Status.SUCCEEDED, rval)
# created a partial bill
bills = db.session.query(PartialBill).filter_by(service=us.oid)
self.assertEqual(1, bills.count())
bill = bills.first()
self.assertEqual(date.today(), bill.closing)
self.assertEqual(100, bill.cost)
self.assertEqual(25, bill.used)
self.assertEqual(
PartialBillProviderType.GENERATION_ONLY.value, bill.provider_type
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
utility_service = meter.utility_service
configuration = PgeBillPdfConfiguration(
utility_service.utility,
utility_service.utility_account_id,
utility_service.gen_utility,
utility_service.gen_utility_account_id,
datasource,
)
return run_datafeed(
PgeBillPdfScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
disable_login_on_error=True,
notify_on_login_error=False,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
"""Run the SMT API integration to gather interval data."""
esiid = (datasource.meta or {}).get("esiid")
if esiid is None:
msg = "Missing ESIID for datasource {}, meter {}.".format(
datasource.oid, meter.oid)
raise DataSourceConfigurationError(msg)
configuration = SmartMeterTexasConfiguration(esiid)
return run_datafeed(
SmartMeterTexasScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
"""
pge-urjanet-v3 has three jobs: 1) Scrape any generation partial bills 2) Set generation utility, if we have it.
3) Look for any PDF's on the service that we can attach to existing SMD Bills or SMD Partial Bills.
"""
utility_service = meter.utility_service
urja_datasource = PacificGasElectricXMLDatasource(
utility_service.utility,
utility_service.utility_account_id,
utility_service.service_id,
utility_service.gen_utility,
utility_service.gen_utility_account_id,
utility_service,
)
transformer = PacificGasElectricUrjaXMLTransformer()
conn = db.urjanet_connection()
try:
urja_datasource.conn = conn
scraper_config = BaseUrjanetConfiguration(
urja_datasource=urja_datasource,
urja_transformer=transformer,
utility_name=meter.utility_service.utility,
fetch_attachments=True,
partial_type=PartialBillProviderType.GENERATION_ONLY,
)
scraper_config.scrape_pdfs = True
return run_datafeed(
PacGAndEPDFAndGenPartialScraper,
account,
meter,
datasource,
params,
configuration=scraper_config,
task_id=task_id,
meter_only=
True, # Upload PDF's found to just this meter, not others in account.
)
finally:
conn.close()
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: SnapmeterMeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
meta = datasource.meta or {}
configuration = Configuration(mvweb_id=meta.get("mvWebId"),
interval=meter.interval)
# reduce load on MVWeb servers: skip if meter has data from within the last 3 days and there are no gaps
max_reading = meter.readings_range.max_date or date.today() - timedelta(
days=365)
interval_age = (date.today() - max_reading).days
date_range = DateRange(
*iso_to_dates(params.get("data_start"), params.get("data_end")))
# freshest we can expect is 3 days old
date_range = DateRange(
date_range.start_date,
min(date_range.end_date,
date.today() - timedelta(days=3)),
)
expected = (date_range.end_date - date_range.start_date).days + 1
days_with_data = (db.session.query(MeterReading).filter(
MeterReading.meter == meter.oid,
MeterReading.occurred >= date_range.start_date,
MeterReading.occurred <= date_range.end_date,
).count())
log.info(
"days with data from %s - %s = %s",
date_range.start_date,
date_range.end_date,
days_with_data,
)
if interval_age <= 3 and days_with_data == expected:
log.info(
"skipping MVWeb run: meter %s has recent interval data (%s) and no gaps",
meter.oid,
max_reading,
)
return Status.SKIPPED
return run_datafeed(
LADWPMVWebScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def test_run_datafeed_bill(self):
"""Run a test scraper that scrapes a bill."""
us = self.meter.utility_service
configuration = TestConfiguration(
us.service_id, us.gen_service_id, scrape_bills=True
)
meter_ds = (
db.session.query(SnapmeterMeterDataSource)
.filter_by(meter=self.meter)
.first()
)
params = {
"data_start": (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d"),
"data_end": datetime.now().strftime("%Y-%m-%d"),
}
rval = run_datafeed(
TestBillScraper,
self.account,
self.meter,
meter_ds,
params,
configuration=configuration,
task_id=uuid.uuid4().hex,
)
self.assertEqual(Status.SUCCEEDED, rval)
# writes bill data to csv
path = os.path.join(config.WORKING_DIRECTORY, "bills.csv")
with open(path) as f:
reader = csv.reader(f)
rows = [row for row in reader]
self.assertEqual(2, len(rows))
# header row
self.assertEqual(
["Service ID", "Start", "End", "Cost", "Used", "Peak"], rows[0]
)
# data row
self.assertEqual(
[
str(us.service_id),
(date.today() - timedelta(days=30)).strftime("%Y-%m-%d"),
date.today().strftime("%Y-%m-%d"),
"100",
"25",
"10",
],
rows[1],
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
conf = SaltRiverBillingConfiguration(account_id=meter.utility_account_id)
return run_datafeed(
SaltRiverBillingScraper,
account,
meter,
datasource,
params,
configuration=conf,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = _UtilityName_Configuration(meter_id=meter.service_id)
return run_datafeed(
_UtilityName_Scraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = SmdPartialBillingScraperConfiguration(meter)
return run_datafeed(
SmdPartialBillingScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
disable_login_on_error=True,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: SnapmeterMeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = Configuration(metering_point=meter.service_id)
return run_datafeed(
Scraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = PGEEnergyExpertConfiguration(datasource.meta.get("itemId"))
return run_datafeed(
PGEEnergyExpertScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = FPLMyAccountConfiguration(
meter.utility_service.utility_account_id)
return run_datafeed(
FPLMyAccountScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = PSEIntervalConfiguration(meter.service_id,
datasource.meta.get("siteName"))
return run_datafeed(
PSEIntervalScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = HECOGridConfiguration(
meter_id=meter.service_id, interval=meter.interval
)
return run_datafeed(
HECOScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
acct_ds = datasource.account_data_source
configuration = SolarEdgeConfiguration(meter_id=meter.service_id,
site_id=acct_ds.username)
return run_datafeed(
SolarEdgeScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = PowayWaterConfiguration(
account_id=meter.utility_account_id)
return run_datafeed(
PowayWaterScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = SolrenGridConfiguration(
inverter_id=meter.service_id, site_id=datasource.meta.get("site_id"))
return run_datafeed(
SolrenScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = DukeIntervalConfiguration(
meter.service_id, channel_id=datasource.meta.get("channelId", None)
)
return run_datafeed(
DukeIntervalScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
def datafeed(
account: SnapmeterAccount,
meter: Meter,
datasource: MeterDataSource,
params: dict,
task_id: Optional[str] = None,
) -> Status:
configuration = SMUDMyAccountBillingConfiguration(
utility=meter.utility_service.utility,
account_id=meter.utility_service.utility_account_id,
)
return run_datafeed(
SMUDMyAccountBillingScraper,
account,
meter,
datasource,
params,
configuration=configuration,
task_id=task_id,
)
