def get_water_and_sewer_charges(soup):
def find_water_consumption(tag):
return (tag.name == u'div') and \
(tag.decode().find('Consumption') >= 0) and \
(tag.decode().find('Total Consumption') == -1)
water_div = soup.find_all(find_water_consumption)[0]
water_type = format_fields(water_div.next_sibling.contents[0])
result = {u'Time period': water_type[0]}
water_type = water_type[1:]
consumption = format_fields(
water_div.next_sibling.next_sibling.contents[0])
rates = format_fields(water_div.next_sibling.next_sibling.next_sibling.
next_sibling.contents[0])
charges = format_fields(water_div.next_sibling.next_sibling.next_sibling.
next_sibling.next_sibling.contents[0])
for x in range(len(water_type)):
result[water_type[x]] = {
u'Consumption': consumption[x],
u'Rate': rates[x],
u'Charges': charges[x]
}
return result
def get_water_consumption(soup):
def find_total_consumption(tag):
return tag.name == u'div' and tag.decode().find(
'Total Consumption') >= 0
div_list = soup.find_all(find_total_consumption)
# Find the div containing 3 fields (gas has an extra
# 'Billing Conversion Multiplier'). Note that it is possible to have
# more than one consumption section.
tags = [x for x in div_list if len(format_fields(x.contents[0])) == 3]
consumption = []
for tag in tags:
# Extract the top pixel coordinate.
match = re.search('top:(?P<top>\d+)px', tag.decode())
top = match.groups()[0]
# Match all divs with the same top pixel coordinate.
def find_matching_top(tag):
return tag.name == u'div' and tag.decode().find(
'top:%spx' % top) >= 0
divs = [
format_fields(x.contents[0])
for x in soup.find_all(find_matching_top)
]
consumption.append(dict(zip(divs[0], divs[2])))
return consumption
def get_summary(soup):
def find_seq_id(tag):
return tag.name == u'div' and tag.decode().find('SEQ-ID') >= 0
def find_account_summary(tag):
return tag.name == u'span' and tag.decode(). \
find('Your Account Summary') >= 0
summary_fields = format_fields(
soup.find_all(find_account_summary)[0].contents)
summary_data = format_fields(
soup.find_all(find_seq_id)[0].next_sibling.contents[0].contents)
summary_dict = dict(zip(summary_fields[1:], summary_data))
def find_charges(name):
def find_matching_div(tag):
return tag.name == u'div' and tag.decode().find(name) >= 0
tag = soup.find(find_matching_div)
# Extract the top pixel coordinate.
match = re.search('top:(?P<top>\d+)px', tag.decode())
top = match.groups()[0]
# Find the second div with the same top pixel coordinate.
return format_fields(
soup.find_all(style=re.compile('top:%spx' %
top))[1].span.contents)[0]
summary_dict[u'Water Charges'] = find_charges('Water charges')
summary_dict[u'Gas Charges'] = find_charges('Gas charges')
return summary_dict
def get_amount_due(soup):
def find_new_charges(tag):
return tag.name == u'div' and tag.decode().find('New Charges') >= 0
tag = soup.find(find_new_charges)
match = re.search('top:(?P<top>\d+)px', tag.decode())
top = float(match.groups()[0])
def find_left_pos(tag):
if tag.name == u'div':
match = re.search('left:(?P<left>\d+)px', tag.decode())
if match:
left = float(match.groups()[0])
return left >= 120 and left <= 132
return False
tags = soup.find_all(find_left_pos)
distance = []
for tag in tags:
match = re.search('top:(?P<top>\d+)px', tag.decode())
distance.append(abs(float(match.groups()[0]) - top))
amount_due = format_fields(
tags[np.nonzero(distance == np.min(distance))[0][0]].span)[0]
index = str(amount_due).find('CR')
if index >= 0:
amount_due = (-1) * float(amount_due[:index])
return amount_due
def get_summary(soup):
def find_gas_used_this_period(tag):
return (tag.name == u'div'
and tag.decode().find('Gas used this period') >= 0)
div = soup.find(find_gas_used_this_period)
field_data = format_fields(div.next_sibling.span.contents)
"""
field_names = [format_fields(x.contents) for x in div.contents]
# Flatten the list of lists.
field_names = [item for sublist in field_names for item in sublist]
"""
# Dynamic discovery of field names failing. Hard-code for now.
field_names = [
u'Meter Number', u'Estimated Reading', u'Previous Reading',
u'Gas used this period', u'PEF Value', u'Adjusted volume'
]
summary_dict = dict(zip(field_names, field_data))
summary_dict[u'Bill Date'] = get_bill_date(soup)
summary_dict[u'Amount Due'] = get_amount_due(soup)
return summary_dict
def get_amount_due(soup):
pos_re = ('left:(?P<left>\d+)px.*top:(?P<top>\d+)px.*'
'width:(?P<width>\d+)px.*height:(?P<height>\d+)')
def find_amount_due_now(tag):
return tag.name == u'div' and tag.decode().find('Amount due now') >= 0
tag = soup.find_all(find_amount_due_now)[-1]
pos = re.search(pos_re, tag.decode()).groupdict()
pos = {k: int(v) for (k, v) in pos.items()}
pos['bottom'] = pos['top'] + pos['height']
pos['right'] = pos['left'] + pos['width']
def find_divs_on_same_line(tag):
if tag.name == u'div':
match = re.search(pos_re, tag.decode())
if match:
top = int(match.groupdict()['top'])
bottom = top + int(match.groupdict()['height'])
left = int(match.groupdict()['left'])
right = left + int(match.groupdict()['width'])
return ((left > pos['right']) and
((top >= pos['top'] and top <= pos['bottom']) or
(bottom >= pos['top'] and bottom <= pos['bottom'])))
return False
return format_fields(
soup.find(find_divs_on_same_line).span.contents)[0][1:]
def get_gas_rates(soup):
def find_gas_rates(tag):
return tag.name == u'div' and tag.decode().find(
'Gas Fixed Delivery Charge') >= 0
gas_div = soup.find_all(find_gas_rates)[0]
gas_fields = format_fields(gas_div.contents[0])
gas_fields = gas_fields[1:]
gas_rates = format_fields(
gas_div.next_sibling.next_sibling.next_sibling.contents[0])
gas_charges = format_fields(gas_div.next_sibling.next_sibling.next_sibling.
next_sibling.contents[0])
return dict(
zip([
x + ' Rate' for x in gas_fields
if (x.find('HST') == -1) and x.find('Fixed') == -1
], gas_rates))
def get_billing_date(soup):
def find_billing(tag):
return tag.name == u'div' and tag.decode().find('BILLING DATE') >= 0
match = re.search(
'([A-Z]+)\s+(\d+)\s+(\d+)',
format_fields(
soup.find_all(find_billing)
[0].next_sibling.next_sibling.span.contents)[0])
month, day, year = match.groups()
return arrow.get("%s %s %s" % (month, day, year),
'MMM DD YYYY').date().isoformat()
def find_charges(name):
def find_matching_div(tag):
return tag.name == u'div' and tag.decode().find(name) >= 0
tag = soup.find(find_matching_div)
# Extract the top pixel coordinate.
match = re.search('top:(?P<top>\d+)px', tag.decode())
top = match.groups()[0]
# Find the second div with the same top pixel coordinate.
return format_fields(
soup.find_all(style=re.compile('top:%spx' %
top))[1].span.contents)[0]
def get_electricity_rates(soup):
def find_kWhOffPeak(tag):
return tag.name == u'div' and tag.decode().find('kWh Off Peak') >= 0
tag = soup.find(find_kWhOffPeak)
match = re.search('top:(?P<top>\d+)px', tag.decode())
top = match.groups()[0]
# match all divs with the same top pixel coordinate
def find_matching_top(tag):
return tag.name == u'div' and tag.decode().find('top:%spx' % top) >= 0
for x in soup.find_all(find_matching_top):
fields = format_fields(x.span)
if len(fields) > 0 and str(fields[0]).find('at $') == 0:
rates = [float(x[4:]) for x in fields]
break
return dict(zip(['off peak', 'on peak', 'mid peak'], rates))
def get_electricity_consumption(soup):
def find_kWh(tag):
return tag.name == u'span' and (
tag.decode().find('kWh Off Peak') >= 0
or tag.decode().find('kWh Mid Peak') >= 0
or tag.decode().find('kWh On Peak') >= 0)
fields = []
for x in soup.find_all(find_kWh):
fields += format_fields(x)
data = {'off peak': 0, 'mid peak': 0, 'on peak': 0}
for x in fields:
if x.find('kWh Off Peak') > 0:
data['off peak'] += float(x[:x.find('kWh Off Peak')])
elif x.find('kWh Mid Peak') > 0:
data['mid peak'] += float(x[:x.find('kWh Mid Peak')])
elif x.find('kWh On Peak') > 0:
data['on peak'] += float(x[:x.find('kWh On Peak')])
return data
def get_bill_date(soup):
def find_bill_date(tag):
return tag.name == u'div' and tag.decode().find('Bill Date') >= 0
return format_fields(soup.find(find_bill_date).contents[1].contents)[0]