def split_previous_bill(self):
"""All parsing and wireless bill splitting happen here.
First Parse for U-verse TV, then U-verse Internet, and finally
Wireless. Wireless account monthly charges (after discount if there
is any) are split among all users.
"""
logger.info('Start processing new bill...')
# billing cycle
new_charge_title = self.browser.find_element_by_xpath(
"//h3[contains(text(), 'New charges for')]").text
billing_cycle_name = new_charge_title[16:]
if BillingCycle.select().where(
BillingCycle.name == billing_cycle_name):
logger.warning('Billing Cycle %s already processed.',
billing_cycle_name)
return
billing_cycle = BillingCycle.create(name=billing_cycle_name)
# parse user name and number
users = self.parse_user_info()
# set account holder
account_holder = users[0]
# ---------------------------------------------------------------------
# U-verse tv
# ---------------------------------------------------------------------
# beginning div
beginning_div_xpath = (
"div[starts-with(@class, 'MarLeft12 MarRight90') and "
"descendant::div[contains(text(), 'U-verse TV')]]")
# # end div
end_div_xpath = (
"div[@class='Padbot5 topDotBorder MarLeft12 MarRight90' and "
"descendant::div[contains(text(), 'Total U-verse TV Charges')]]")
charge_elems = self.browser.find_elements_by_xpath(
"//div[(starts-with(@class, 'accSummary') or "
"@id='UTV-monthly') and preceding-sibling::{} and "
"following-sibling::{}]".format(beginning_div_xpath,
end_div_xpath))
if charge_elems:
utv_charge_category, _ = ChargeCategory.get_or_create(
category='utv', text='U-verse TV')
for elem in charge_elems:
charge_type_text = elem.find_element_by_xpath("div[1]").text
if charge_type_text.startswith('Monthly Charges'):
charge_type_text = 'Monthly Charges'
m = re.search(
'Total {}.*?\$([0-9.]+)'.format(charge_type_text),
elem.text,
flags=re.DOTALL)
charge_total = float(m.group(1))
# save data to db
charge_type_name = slugify(charge_type_text)
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type=charge_type_name,
text=charge_type_text,
charge_category=utv_charge_category)
# Charge
try:
new_charge = Charge(user=account_holder,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=charge_total)
new_charge.save()
except IntegrityError:
logger.warning(
'Trying to write duplicate charge record!\n%s',
new_charge.__dict__)
else:
logger.info('No U-verse TV Charge Elements Found, skipped.')
# ---------------------------------------------------------------------
# U-verse Internet
# ---------------------------------------------------------------------
# beginning div
beginning_div_xpath = (
"div[starts-with(@class, 'MarLeft12 MarRight90') and "
"descendant::div[contains(text(), 'U-verse Internet')]]")
# end div
end_div_xpath = (
"div[@class='Padbot5 topDotBorder MarLeft12 MarRight90' and "
"descendant::div[contains(text(), "
"'Total U-verse Internet Charges')]]")
charge_elems = self.browser.find_elements_by_xpath(
"//div[(starts-with(@class, 'accSummary') or "
"@id='UVI-monthly') and preceding-sibling::{} and "
"following-sibling::{}]".format(beginning_div_xpath,
end_div_xpath))
if charge_elems:
uvi_charge_category, _ = ChargeCategory.get_or_create(
category='uvi', text='U-verse Internet')
for elem in charge_elems:
charge_type_text = elem.find_element_by_xpath("div[1]").text
if charge_type_text.startswith('Monthly Charges'):
charge_type_text = 'Monthly Charges'
m = re.search(
'Total {}.*?\$([0-9.]+)'.format(charge_type_text),
elem.text,
flags=re.DOTALL)
charge_total = float(m.group(1))
# save data to db
charge_type_name = slugify(charge_type_text)
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type=charge_type_name,
text=charge_type_text,
charge_category=uvi_charge_category)
# Charge
try:
new_charge = Charge(user=account_holder,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=charge_total)
new_charge.save()
except IntegrityError:
logger.warning(
'Trying to write duplicate charge record!\n%s',
new_charge.__dict__)
else:
logger.info('No U-verse Internet Charge Elements Found, skipped.')
# ---------------------------------------------------------------------
# Wireless
# ---------------------------------------------------------------------
# beginning div
charged_users = users[:1] # users who have a positive balance
beginning_div_xpath = (
"div[@class='MarLeft12 MarRight90 ' and "
"descendant::div[contains(text(), '{name} {num}')]]")
# end div
end_div_xpath = (
"div[@class='topDotBorder accSummary MarLeft12 "
"Padbot5 botMar10 botMar23ie' and "
"descendant::div[contains(text(), 'Total for {num}')]]")
# first parse charges under account holder
charge_elems = self.browser.find_elements_by_xpath(
"//div[starts-with(@class, 'accSummary') and "
"preceding-sibling::{} and following-sibling::{}]".format(
beginning_div_xpath.format(name=account_holder.name,
num=account_holder.number),
end_div_xpath.format(num=account_holder.number)))
if charge_elems:
wireless_charge_category, _ = ChargeCategory.get_or_create(
category='wireless', text='Wireless')
for elem in charge_elems:
charge_type_text = elem.find_element_by_xpath("div[1]").text
offset = 0
if charge_type_text.startswith('Monthly Charges'):
charge_type_text = 'Monthly Charges'
# get account monthly charge and discount
act_m_elem = elem.find_element_by_xpath("div[4]/div[1]")
m = re.search('.*?\$([0-9.]+)', act_m_elem.text, re.DOTALL)
w_act_m = float(m.group(1))
m = re.search('National Account Discount.*?\$([0-9.]+)',
elem.text,
flags=re.DOTALL)
w_act_m_disc = float(m.group(1)) if m else 0
offset = w_act_m - w_act_m_disc
m = re.search(
'Total {}.*?\$([0-9.]+)'.format(charge_type_text),
elem.text,
flags=re.DOTALL)
charge_total = float(m.group(1)) - offset
# save data to db
charge_type_name = slugify(charge_type_text)
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type=charge_type_name,
text=charge_type_text,
charge_category=wireless_charge_category)
# Charge
try:
new_charge = Charge(user=account_holder,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=charge_total)
new_charge.save()
except IntegrityError:
logger.warning(
'Trying to write duplicate charge record!\n%s',
new_charge.__dict__)
else:
raise ParsingError('No charges found for account holder.')
# iterate regular users
for user in users[1:]:
charge_total = 0
charge_elems = self.browser.find_elements_by_xpath(
"//div[starts-with(@class, 'accSummary') and "
"preceding-sibling::{} and following-sibling::{}]".format(
beginning_div_xpath.format(name=user.name,
num=user.number),
end_div_xpath.format(num=user.number)))
for elem in charge_elems:
charge_type_text = elem.find_element_by_xpath("div[1]").text
if charge_type_text.startswith('Monthly Charges'):
charge_type_text = 'Monthly Charges'
m = re.search(
'Total {}.*?\$([0-9.]+)'.format(charge_type_text),
elem.text,
flags=re.DOTALL)
charge_total = float(m.group(1)) - offset
# save data to db
charge_type_name = slugify(charge_type_text)
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type=charge_type_name,
text=charge_type_text,
charge_category=wireless_charge_category)
# Charge
try:
new_charge = Charge(user=user,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=charge_total)
new_charge.save()
except IntegrityError:
logger.warning(
'Trying to write duplicate charge record!\n%s',
new_charge.__dict__)
if charge_total > 0:
charged_users.append(user)
# update share of account monthly charges for each line
# also calculate total wireless charges (for verification later)
act_m_share = (w_act_m - w_act_m_disc) / len(charged_users)
wireless_total = 0
for user in charged_users:
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type='wireless-acount-monthly-charges-share',
text='Account Monthly Charges Share',
charge_category=wireless_charge_category)
# Charge
try:
new_charge = Charge(user=user,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=act_m_share)
new_charge.save()
except IntegrityError:
logger.warning('Trying to write duplicate charge record!\n%s',
new_charge.__dict__)
else:
user_total = Charge.select(
pw.fn.Sum(Charge.amount).alias('total')).join(
ChargeType,
on=Charge.charge_type_id == ChargeType.id).where(
(Charge.user == user),
Charge.billing_cycle == billing_cycle,
ChargeType.charge_category ==
wireless_charge_category)
wireless_total += user_total[0].total
# now that we have wireless_total calculated from user charges
# let's compare if it matches with what the bill says
wireless_total_elem = self.browser.find_element_by_xpath(
"//div[preceding-sibling::"
"div[contains(text(), 'Total Wireless Charges')]]")
bill_wireless_total = float(wireless_total_elem.text.strip('$'))
if abs(bill_wireless_total - wireless_total) > 0.01:
raise CalculationError(
'Wireless total calculated does not match with the bill')
logger.info('Wireless charges calculation results verified.')
logger.info('Finished procesessing bill %s.', billing_cycle_name)
def split_bill(self, bc_name, bill_statement_id):
"""Parse bill and split wireless charges among users.
Currently not parsing U-Verse charges.
:param bc_name: billing cycle name
:type bc_name: str
:param bill_statement_id: bill statement id, used in link
:type bill_statement_id: str
:returns: None
"""
bill_link_template = (
'https://www.att.com/olam/billPrintPreview.myworld?'
'fromPage=history&billStatementID={}')
bill_req = self.session.get(
bill_link_template.format(bill_statement_id))
bill_html = bill_req.text
if 'Account Details' not in bill_html:
raise ParsingError('Failed to retrieve billing page')
soup = BeautifulSoup(bill_html, 'html.parser')
start_date, end_date = get_start_end_date(bc_name)
billing_cycle = BillingCycle.create(name=bc_name,
start_date=start_date,
end_date=end_date)
# parse user name and number
users = self.parse_user_info(bill_html)
if not users:
return
account_holder_name = soup.find(
'span', class_='hidden-spoken ng-binding'
).parent.next_sibling.next_sibling.text.strip()
account_holder = [
user for user in users if user.name == account_holder_name
][0]
# --------------------------------------------------------------------
# Wireless
# --------------------------------------------------------------------
wireless_charge_category, _ = ChargeCategory.get_or_create(
category='wireless', text='Wireless')
charged_users = [account_holder]
name, number = account_holder.name, account_holder.number
offset = 0.0
# charge section starts with user name followed by his/her number
target = soup.find('div',
string=re.compile('{} {}'.format(name, number)))
# fetch data usage in case there is an overage
usage_req = self.session.post(
'https://www.att.com/olam/billUsageTiles.myworld',
data={'billStatementID': bill_statement_id})
usage_soup = BeautifulSoup(usage_req.text, 'html.parser')
usages = {}
overage_charge_type_name = 'data-text-usage-charges'
overage_charge_type_text = 'Data & Text Usage Charges'
data_overused = False
for tag in target.parent.next_siblings:
# all charge data are in divs
if not isinstance(tag, Tag) or tag.name != 'div':
continue
# charge section ends with Total for number
if 'Total for {}'.format(number) in tag.text:
break
# each charge type has 'accSummary' as one of its css classes
if 'accSummary' in tag.get('class', []):
charge_type_text = tag.find('div').text.strip('\n\t')
if charge_type_text.startswith('Monthly Charges'):
charge_type_text = 'Monthly Charges'
# account monthly fee will be shared by all users
w_act_m = float(
re.search(r'\$([0-9.]+)', tag.text).group(1))
# national discount is applied to account monthly fee
m = re.search(r'National Account Discount.*?\$([0-9.]+)',
tag.text, re.DOTALL)
w_act_m_disc = float(m.group(1)) if m else 0.0
# this non-zero offset will be used to adjust account
# holder's total monthly charge
offset = w_act_m - w_act_m_disc
m = re.search(
r'Total {}.*?\$([0-9.]+)'.format(charge_type_text),
tag.text,
flags=re.DOTALL)
charge_type_name = slugify(charge_type_text)
# check if it's a data overage which needs to be shared proportionaly
if charge_type_name == overage_charge_type_name:
data_overused = True
total_overage_charge = float(m.group(1))
user_tag = usage_soup.find('p',
string=re.compile(
account_holder.name))
usage_tag = list(
user_tag.parent.parent.parent.next_siblings)[1]
usage = float(usage_tag.findChild('strong').text)
usages[account_holder] = usage
total_data_allowance = float(
list(usage_tag.findChild('strong').next_siblings)
[-1].split()[0])
else:
charge_total = float(m.group(1)) - offset
# save data to db
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type=charge_type_name,
text=charge_type_text,
charge_category=wireless_charge_category)
# Charge
new_charge = Charge(user=account_holder,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=charge_total)
new_charge.save()
offset = 0.0
# iterate regular users
remaining_users = [u for u in users if u.number != number]
for user in remaining_users:
charge_total = 0.0
name, number = user.name, user.number
# charge section starts with user name followed by his/her number
target = soup.find('div',
string=re.compile('{} {}'.format(name, number)))
for tag in target.parent.next_siblings:
# all charge data are in divs
if not isinstance(tag, Tag) or tag.name != 'div':
continue
# charge section ends with Total for number
if 'Total for {}'.format(number) in tag.text:
break
# each charge type has 'accSummary' as one of its css classes
if 'accSummary' in tag.get('class', []):
charge_type_text = tag.find('div').text.strip('\n\t')
if charge_type_text.startswith('Monthly Charges'):
charge_type_text = 'Monthly Charges'
m = re.search(
r'Total {}.*?\$([0-9.]+)'.format(charge_type_text),
tag.text,
flags=re.DOTALL)
charge_total = float(m.group(1))
# save data to db
charge_type_name = slugify(charge_type_text)
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type=charge_type_name,
text=charge_type_text,
charge_category=wireless_charge_category)
# Charge
new_charge = Charge(user=user,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=charge_total)
new_charge.save()
if charge_total > 0:
charged_users.append(user)
charge_type, _ = ChargeType.get_or_create(
type='data-text-usage-charges',
text='Data & Text Usage Charges',
charge_category=wireless_charge_category)
# data usages
user_tag = usage_soup.find('p', string=re.compile(user.name))
if user_tag:
usage_tag = list(
user_tag.parent.parent.parent.next_siblings)[1]
usage = float(usage_tag.findChild('strong').text)
usages[user] = usage
# update share of account monthly charges for each user
# also calculate total wireless charges (for verification later)
act_m_share = (w_act_m - w_act_m_disc) / len(charged_users)
wireless_total = 0
for user in charged_users:
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type='wireless-acount-monthly-charges-share',
text='Account Monthly Charges Share',
charge_category=wireless_charge_category)
# Charge
new_charge = Charge(user=user,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=act_m_share)
new_charge.save()
user_total = Charge.select(
pw.fn.Sum(Charge.amount).alias('total')).join(
ChargeType,
on=Charge.charge_type_id == ChargeType.id).where(
(Charge.user == user),
Charge.billing_cycle == billing_cycle,
ChargeType.charge_category == wireless_charge_category)
wireless_total += user_total[0].total
if data_overused:
user_share = total_data_allowance / len(charged_users)
overages = {
k: v - user_share
for k, v in usages.iteritems() if v > user_share
}
total_overage = sum(overages.values())
for user, overage in overages.iteritems():
overage_charge = overage / total_overage * total_overage_charge
print(
'User {} over used {} GB data, will be charged extra ${:.2f}'
.format(user.name, overage, overage_charge))
charge_type, _ = ChargeType.get_or_create(
type=overage_charge_type_name,
text=overage_charge_type_text,
charge_category=wireless_charge_category)
# Charge
new_charge = Charge(user=user,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=overage_charge)
new_charge.save()
# aggregate
aggregate_wireless_monthly(billing_cycle)
def split_bill(self, bc_name, bill_link):
"""Parse bill and split wireless charges among users.
Currently not parsing U-Verse charges.
:param bc_name: billing cycle name
:type bc_name: str
:param bill_link: url to bill
:type bc_name: str
:returns: None
"""
bill_req = self.session.get(bill_link)
bill_html = bill_req.text
if 'Account Details' not in bill_html:
raise ParsingError('Failed to retrieve billing page')
soup = BeautifulSoup(bill_html, 'html.parser')
start_date, end_date = get_start_end_date(bc_name)
billing_cycle = BillingCycle.create(name=bc_name,
start_date=start_date,
end_date=end_date)
# parse user name and number
users = self.parse_user_info(bill_html)
if not users:
return
account_holder = users[0]
# --------------------------------------------------------------------
# Wireless
# --------------------------------------------------------------------
wireless_charge_category, _ = ChargeCategory.get_or_create(
category='wireless', text='Wireless')
charged_users = users[:1]
name, number = account_holder.name, account_holder.number
offset = 0.0
# charge section starts with user name followed by his/her number
target = soup.find('div',
string=re.compile('{} {}'.format(name, number)))
for tag in target.parent.next_siblings:
# all charge data are in divs
if not isinstance(tag, Tag) or tag.name != 'div':
continue
# charge section ends with Total for number
if 'Total for {}'.format(number) in tag.text:
break
# each charge type has 'accSummary' as one of its css classes
if 'accSummary' in tag.get('class', []):
charge_type_text = tag.find('div').text.strip('\n\t')
if charge_type_text.startswith('Monthly Charges'):
charge_type_text = 'Monthly Charges'
# account monthly fee will be shared by all users
w_act_m = float(
re.search(r'\$([0-9.]+)', tag.text).group(1))
# national discount is applied to account monthly fee
m = re.search(r'National Account Discount.*?\$([0-9.]+)',
tag.text, re.DOTALL)
w_act_m_disc = float(m.group(1)) if m else 0.0
# this non-zero offset will be used to adjust account
# holder's total monthly charge
offset = w_act_m - w_act_m_disc
m = re.search(
r'Total {}.*?\$([0-9.]+)'.format(charge_type_text),
tag.text,
flags=re.DOTALL)
charge_total = float(m.group(1)) - offset
# save data to db
charge_type_name = slugify(charge_type_text)
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type=charge_type_name,
text=charge_type_text,
charge_category=wireless_charge_category)
# Charge
new_charge = Charge(user=account_holder,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=charge_total)
new_charge.save()
offset = 0.0
# iterate regular users
for user in users[1:]:
charge_total = 0.0
name, number = user.name, user.number
# charge section starts with user name followed by his/her number
target = soup.find('div',
string=re.compile('{} {}'.format(name, number)))
for tag in target.parent.next_siblings:
# all charge data are in divs
if not isinstance(tag, Tag) or tag.name != 'div':
continue
# charge section ends with Total for number
if 'Total for {}'.format(number) in tag.text:
break
# each charge type has 'accSummary' as one of its css classes
if 'accSummary' in tag.get('class', []):
charge_type_text = tag.find('div').text.strip('\n\t')
if charge_type_text.startswith('Monthly Charges'):
charge_type_text = 'Monthly Charges'
m = re.search(
r'Total {}.*?\$([0-9.]+)'.format(charge_type_text),
tag.text,
flags=re.DOTALL)
charge_total = float(m.group(1))
# save data to db
charge_type_name = slugify(charge_type_text)
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type=charge_type_name,
text=charge_type_text,
charge_category=wireless_charge_category)
# Charge
new_charge = Charge(user=user,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=charge_total)
new_charge.save()
if charge_total > 0:
charged_users.append(user)
# update share of account monthly charges for each user
# also calculate total wireless charges (for verification later)
act_m_share = (w_act_m - w_act_m_disc) / len(charged_users)
wireless_total = 0
for user in charged_users:
# ChargeType
charge_type, _ = ChargeType.get_or_create(
type='wireless-acount-monthly-charges-share',
text='Account Monthly Charges Share',
charge_category=wireless_charge_category)
# Charge
new_charge = Charge(user=user,
charge_type=charge_type,
billing_cycle=billing_cycle,
amount=act_m_share)
new_charge.save()
user_total = Charge.select(
pw.fn.Sum(Charge.amount).alias('total')).join(
ChargeType,
on=Charge.charge_type_id == ChargeType.id).where(
(Charge.user == user),
Charge.billing_cycle == billing_cycle,
ChargeType.charge_category == wireless_charge_category)
wireless_total += user_total[0].total
# aggregate
aggregate_wireless_monthly(billing_cycle)