def set_history_type(self):
WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located(self.UsageRadioButtonLocator))
usage_radio_button = self.driver.find_element(
*self.UsageRadioButtonLocator)
scroll_to(self.driver, usage_radio_button)
usage_radio_button.click()
def set_displayed_bills(self):
WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located(self.NumBillsDisplayedLocator))
num_bills_displayed_select = self.driver.find_element(
*self.NumBillsDisplayedLocator)
scroll_to(self.driver, num_bills_displayed_select)
num_bills_displayed_select.click()
WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located(
self.NumBillsDisplayedOptionLocator))
num_bills_displayed_option = self.driver.find_element(
*self.NumBillsDisplayedOptionLocator)
num_bills_displayed_option.click()
def click(
driver,
css_selector: str = None,
xpath: str = None,
elem: WebElement = None,
scroll: bool = True,
):
"""helper method to click an element, if it is blocked by blockOverlay, waits for the overlay to disappear"""
if elem:
pass
elif css_selector:
elem = driver.find_element_by_css_selector(css_selector)
elif xpath:
elem = driver.find_element_by_xpath(xpath)
else:
raise ValueError("one of css_selector, xpath or elem must be provided")
retries_left = 5
while retries_left > 0:
try:
scroll_to(driver, elem) if scroll is True else None
elem.click()
break
except ElementClickInterceptedException as e:
if "blockUI blockOverlay" in e.msg:
log.info(
"blocked by overlay, waiting for it to go before clicking again"
)
wait_for_block_overlay(driver)
continue
else:
if not close_modal(driver):
raise
finally:
retries_left -= 1
def get_bills(self, account_id: str, start: date,
end: date) -> List[BillingDatum]:
"""Get bills from the table.
for each row:
get end from Read date column (date)
get start date from end date - (Days column (date) - 1)
get statement date from Bill date column (date)
if not start - end overlaps passed in start / end, continue
get peak from On-peak Billed kW (float)
get used from (Off-peak kWh + Shoulder kWh + On-peak kWh) (float)
get cost from New charges (float)
click eye icon to download PDF; wait for download to complete to self.driver.download_dir
"""
WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located(self.UsageTableBodyLocator))
usage_table_rows = self.driver.find_elements(
*self.UsageTableRowsLocator)
bill_data: List[BillingDatum] = []
self.driver.screenshot(BaseWebScraper.screenshot_path("bill table"))
for row in usage_table_rows:
cols = row.find_elements_by_tag_name("td")
cols = [
c for c in cols
if "display: none" not in c.get_attribute("style")
]
col = lambda x: cols[x].text
to_num = lambda x: "".join(d for d in col(x)
if d.isdigit() or d == ".")
to_float = lambda x: float(to_num(x)) if len(to_num(x)) > 0 else 0
log.debug(f"statement={col(1)} end={col(2)} days={col(7)}")
# statement date
statement_date = date_parser.parse(col(1)).date()
# bill end
period_year = statement_date.year
if statement_date.month == 1 and col(2).startswith("12"):
period_year = statement_date.year - 1
end_str = f"{col(2)}/{period_year}"
bill_end = date_parser.parse(end_str).date()
# bill start
bill_start = bill_end - timedelta(days=int(to_float(7)) - 1)
log.debug(f"start={bill_start} end={bill_end}")
if not self._overlap(start, end, bill_start, bill_end):
log.info(
f"skipping bill {bill_start} - {bill_end}: does not overlap requested range {start} - {end}"
)
continue
# cost
new_charges = to_float(8)
# used
used = to_float(4) + to_float(5) + to_float(6)
# peak
peak = to_float(3)
bill_datum = BillingDatum(
start=bill_start,
end=bill_end,
statement=statement_date,
cost=new_charges,
used=used,
peak=peak,
items=None,
attachments=None,
utility_code=None,
)
try:
bill_pdf_name = "SRPbill{}{}.pdf".format(
statement_date.strftime("%B"), statement_date.year)
pdf_download_link = cols[0].find_element_by_tag_name("a")
scroll_to(self.driver, pdf_download_link)
pdf_download_link.click()
log.info("looking for %s in %s", bill_pdf_name,
self.driver.download_dir)
self.driver.wait(60).until(
file_exists_in_dir(self.driver.download_dir,
bill_pdf_name))
except Exception as e:
raise Exception(
f"Failed to download bill {bill_pdf_name} for statement date {statement_date}:\n {e}"
)
log.info(
f"Bill {bill_pdf_name} for statement date {statement_date} downloaded successfully"
)
attachment_entry = None
# open downloaded PDF and upload
if config.enabled("S3_BILL_UPLOAD"):
key = hash_bill_datum(account_id, bill_datum)
with open(f"{self.driver.download_dir}/{bill_pdf_name}",
"rb") as pdf_data:
attachment_entry = upload_bill_to_s3(
BytesIO(pdf_data.read()),
key,
source="myaccount.srpnet.com",
statement=bill_datum.statement,
utility="utility:salt-river-project",
utility_account_id=account_id,
)
if attachment_entry:
bill_data.append(
bill_datum._replace(attachments=[attachment_entry]))
else:
bill_data.append(bill_datum)
return bill_data
def export_csv(self, service_id, start: date, end: date) -> str:
"""Export CSV file and return path to downloaded file.
Select meter service_id from Meter drop down
Click triple bar button, then Export All Data (CSV)
Adjust end date if needed: get latest to date from form, parse into a date, and set end to max(end, form_max_dt)
Set from and to dates (mm/dd/yyyy) and click Download.
Wait for file to download (.csv)
Return path to csv file
"""
self.driver.wait().until(
EC.invisibility_of_element_located(
(By.CSS_SELECTOR, "div.spinner-container")))
self.driver.sleep(2)
self.driver.find(self.MeterDropdownSelector).click()
# wait for loading
self.driver.sleep(5)
meter_dropdown_selector = f'//table[@id="sdp_selector_table"]//a[contains(@class,"sdp-dropdown") and contains(.,"{service_id}")]'
meter_id_dropdown_option = self.driver.find(meter_dropdown_selector,
xpath=True)
scroll_to(self.driver, meter_id_dropdown_option)
if not meter_id_dropdown_option:
raise DataSourceConfigurationError(
f"No meter found with service_id: {service_id}")
meter_id_dropdown_option.click()
self.driver.wait().until(
EC.invisibility_of_element_located(
(By.CSS_SELECTOR, "div.spinner-container")))
self.driver.sleep(2)
self.driver.find_element(*self.ExportCSVDropDownButtonLocator).click()
self.driver.sleep(2)
self.driver.find(self.ExportCSVLinkSelector).click()
self.driver.wait().until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR, self.FromDateInputSelector)))
self.driver.sleep(2)
from_date_input_field = self.driver.find(self.FromDateInputSelector)
from_date_input_field.clear()
from_date_input_field.send_keys(start.strftime("%m/%d/%Y"))
to_date_input_field = self.driver.find(self.ToDateInputSelector)
max_available_to_date = parse_date(
to_date_input_field.get_attribute("placeholder")).date()
end = max(max_available_to_date, end)
to_date_input_field.clear()
to_date_input_field.send_keys(end.strftime("%m/%d/%Y"))
self.driver.find(self.DownloadButtonSelector).click()
# Wait for csv to download
download_dir = self.driver.download_dir
filename = self.driver.wait().until(
file_exists_in_dir(download_dir, r".*\.{}$".format("csv")))
return os.path.join(download_dir, filename)