def split_previous_bill(self): """All parsing and wireless bill splitting happen here. First Parse for U-verse TV, then U-verse Internet, and finally Wireless. Wireless account monthly charges (after discount if there is any) are split among all users. """ logger.info('Start processing new bill...') # billing cycle new_charge_title = self.browser.find_element_by_xpath( "//h3[contains(text(), 'New charges for')]").text billing_cycle_name = new_charge_title[16:] if BillingCycle.select().where( BillingCycle.name == billing_cycle_name): logger.warning('Billing Cycle %s already processed.', billing_cycle_name) return billing_cycle = BillingCycle.create(name=billing_cycle_name) # parse user name and number users = self.parse_user_info() # set account holder account_holder = users[0] # --------------------------------------------------------------------- # U-verse tv # --------------------------------------------------------------------- # beginning div beginning_div_xpath = ( "div[starts-with(@class, 'MarLeft12 MarRight90') and " "descendant::div[contains(text(), 'U-verse TV')]]") # # end div end_div_xpath = ( "div[@class='Padbot5 topDotBorder MarLeft12 MarRight90' and " "descendant::div[contains(text(), 'Total U-verse TV Charges')]]") charge_elems = self.browser.find_elements_by_xpath( "//div[(starts-with(@class, 'accSummary') or " "@id='UTV-monthly') and preceding-sibling::{} and " "following-sibling::{}]".format(beginning_div_xpath, end_div_xpath)) if charge_elems: utv_charge_category, _ = ChargeCategory.get_or_create( category='utv', text='U-verse TV') for elem in charge_elems: charge_type_text = elem.find_element_by_xpath("div[1]").text if charge_type_text.startswith('Monthly Charges'): charge_type_text = 'Monthly Charges' m = re.search( 'Total {}.*?\$([0-9.]+)'.format(charge_type_text), elem.text, flags=re.DOTALL) charge_total = float(m.group(1)) # save data to db charge_type_name = slugify(charge_type_text) # ChargeType charge_type, _ = ChargeType.get_or_create( type=charge_type_name, text=charge_type_text, charge_category=utv_charge_category) # Charge try: new_charge = Charge(user=account_holder, charge_type=charge_type, billing_cycle=billing_cycle, amount=charge_total) new_charge.save() except IntegrityError: logger.warning( 'Trying to write duplicate charge record!\n%s', new_charge.__dict__) else: logger.info('No U-verse TV Charge Elements Found, skipped.') # --------------------------------------------------------------------- # U-verse Internet # --------------------------------------------------------------------- # beginning div beginning_div_xpath = ( "div[starts-with(@class, 'MarLeft12 MarRight90') and " "descendant::div[contains(text(), 'U-verse Internet')]]") # end div end_div_xpath = ( "div[@class='Padbot5 topDotBorder MarLeft12 MarRight90' and " "descendant::div[contains(text(), " "'Total U-verse Internet Charges')]]") charge_elems = self.browser.find_elements_by_xpath( "//div[(starts-with(@class, 'accSummary') or " "@id='UVI-monthly') and preceding-sibling::{} and " "following-sibling::{}]".format(beginning_div_xpath, end_div_xpath)) if charge_elems: uvi_charge_category, _ = ChargeCategory.get_or_create( category='uvi', text='U-verse Internet') for elem in charge_elems: charge_type_text = elem.find_element_by_xpath("div[1]").text if charge_type_text.startswith('Monthly Charges'): charge_type_text = 'Monthly Charges' m = re.search( 'Total {}.*?\$([0-9.]+)'.format(charge_type_text), elem.text, flags=re.DOTALL) charge_total = float(m.group(1)) # save data to db charge_type_name = slugify(charge_type_text) # ChargeType charge_type, _ = ChargeType.get_or_create( type=charge_type_name, text=charge_type_text, charge_category=uvi_charge_category) # Charge try: new_charge = Charge(user=account_holder, charge_type=charge_type, billing_cycle=billing_cycle, amount=charge_total) new_charge.save() except IntegrityError: logger.warning( 'Trying to write duplicate charge record!\n%s', new_charge.__dict__) else: logger.info('No U-verse Internet Charge Elements Found, skipped.') # --------------------------------------------------------------------- # Wireless # --------------------------------------------------------------------- # beginning div charged_users = users[:1] # users who have a positive balance beginning_div_xpath = ( "div[@class='MarLeft12 MarRight90 ' and " "descendant::div[contains(text(), '{name} {num}')]]") # end div end_div_xpath = ( "div[@class='topDotBorder accSummary MarLeft12 " "Padbot5 botMar10 botMar23ie' and " "descendant::div[contains(text(), 'Total for {num}')]]") # first parse charges under account holder charge_elems = self.browser.find_elements_by_xpath( "//div[starts-with(@class, 'accSummary') and " "preceding-sibling::{} and following-sibling::{}]".format( beginning_div_xpath.format(name=account_holder.name, num=account_holder.number), end_div_xpath.format(num=account_holder.number))) if charge_elems: wireless_charge_category, _ = ChargeCategory.get_or_create( category='wireless', text='Wireless') for elem in charge_elems: charge_type_text = elem.find_element_by_xpath("div[1]").text offset = 0 if charge_type_text.startswith('Monthly Charges'): charge_type_text = 'Monthly Charges' # get account monthly charge and discount act_m_elem = elem.find_element_by_xpath("div[4]/div[1]") m = re.search('.*?\$([0-9.]+)', act_m_elem.text, re.DOTALL) w_act_m = float(m.group(1)) m = re.search('National Account Discount.*?\$([0-9.]+)', elem.text, flags=re.DOTALL) w_act_m_disc = float(m.group(1)) if m else 0 offset = w_act_m - w_act_m_disc m = re.search( 'Total {}.*?\$([0-9.]+)'.format(charge_type_text), elem.text, flags=re.DOTALL) charge_total = float(m.group(1)) - offset # save data to db charge_type_name = slugify(charge_type_text) # ChargeType charge_type, _ = ChargeType.get_or_create( type=charge_type_name, text=charge_type_text, charge_category=wireless_charge_category) # Charge try: new_charge = Charge(user=account_holder, charge_type=charge_type, billing_cycle=billing_cycle, amount=charge_total) new_charge.save() except IntegrityError: logger.warning( 'Trying to write duplicate charge record!\n%s', new_charge.__dict__) else: raise ParsingError('No charges found for account holder.') # iterate regular users for user in users[1:]: charge_total = 0 charge_elems = self.browser.find_elements_by_xpath( "//div[starts-with(@class, 'accSummary') and " "preceding-sibling::{} and following-sibling::{}]".format( beginning_div_xpath.format(name=user.name, num=user.number), end_div_xpath.format(num=user.number))) for elem in charge_elems: charge_type_text = elem.find_element_by_xpath("div[1]").text if charge_type_text.startswith('Monthly Charges'): charge_type_text = 'Monthly Charges' m = re.search( 'Total {}.*?\$([0-9.]+)'.format(charge_type_text), elem.text, flags=re.DOTALL) charge_total = float(m.group(1)) - offset # save data to db charge_type_name = slugify(charge_type_text) # ChargeType charge_type, _ = ChargeType.get_or_create( type=charge_type_name, text=charge_type_text, charge_category=wireless_charge_category) # Charge try: new_charge = Charge(user=user, charge_type=charge_type, billing_cycle=billing_cycle, amount=charge_total) new_charge.save() except IntegrityError: logger.warning( 'Trying to write duplicate charge record!\n%s', new_charge.__dict__) if charge_total > 0: charged_users.append(user) # update share of account monthly charges for each line # also calculate total wireless charges (for verification later) act_m_share = (w_act_m - w_act_m_disc) / len(charged_users) wireless_total = 0 for user in charged_users: # ChargeType charge_type, _ = ChargeType.get_or_create( type='wireless-acount-monthly-charges-share', text='Account Monthly Charges Share', charge_category=wireless_charge_category) # Charge try: new_charge = Charge(user=user, charge_type=charge_type, billing_cycle=billing_cycle, amount=act_m_share) new_charge.save() except IntegrityError: logger.warning('Trying to write duplicate charge record!\n%s', new_charge.__dict__) else: user_total = Charge.select( pw.fn.Sum(Charge.amount).alias('total')).join( ChargeType, on=Charge.charge_type_id == ChargeType.id).where( (Charge.user == user), Charge.billing_cycle == billing_cycle, ChargeType.charge_category == wireless_charge_category) wireless_total += user_total[0].total # now that we have wireless_total calculated from user charges # let's compare if it matches with what the bill says wireless_total_elem = self.browser.find_element_by_xpath( "//div[preceding-sibling::" "div[contains(text(), 'Total Wireless Charges')]]") bill_wireless_total = float(wireless_total_elem.text.strip('$')) if abs(bill_wireless_total - wireless_total) > 0.01: raise CalculationError( 'Wireless total calculated does not match with the bill') logger.info('Wireless charges calculation results verified.') logger.info('Finished procesessing bill %s.', billing_cycle_name)
def split_bill(self, bc_name, bill_statement_id): """Parse bill and split wireless charges among users. Currently not parsing U-Verse charges. :param bc_name: billing cycle name :type bc_name: str :param bill_statement_id: bill statement id, used in link :type bill_statement_id: str :returns: None """ bill_link_template = ( 'https://www.att.com/olam/billPrintPreview.myworld?' 'fromPage=history&billStatementID={}') bill_req = self.session.get( bill_link_template.format(bill_statement_id)) bill_html = bill_req.text if 'Account Details' not in bill_html: raise ParsingError('Failed to retrieve billing page') soup = BeautifulSoup(bill_html, 'html.parser') start_date, end_date = get_start_end_date(bc_name) billing_cycle = BillingCycle.create(name=bc_name, start_date=start_date, end_date=end_date) # parse user name and number users = self.parse_user_info(bill_html) if not users: return account_holder_name = soup.find( 'span', class_='hidden-spoken ng-binding' ).parent.next_sibling.next_sibling.text.strip() account_holder = [ user for user in users if user.name == account_holder_name ][0] # -------------------------------------------------------------------- # Wireless # -------------------------------------------------------------------- wireless_charge_category, _ = ChargeCategory.get_or_create( category='wireless', text='Wireless') charged_users = [account_holder] name, number = account_holder.name, account_holder.number offset = 0.0 # charge section starts with user name followed by his/her number target = soup.find('div', string=re.compile('{} {}'.format(name, number))) # fetch data usage in case there is an overage usage_req = self.session.post( 'https://www.att.com/olam/billUsageTiles.myworld', data={'billStatementID': bill_statement_id}) usage_soup = BeautifulSoup(usage_req.text, 'html.parser') usages = {} overage_charge_type_name = 'data-text-usage-charges' overage_charge_type_text = 'Data & Text Usage Charges' data_overused = False for tag in target.parent.next_siblings: # all charge data are in divs if not isinstance(tag, Tag) or tag.name != 'div': continue # charge section ends with Total for number if 'Total for {}'.format(number) in tag.text: break # each charge type has 'accSummary' as one of its css classes if 'accSummary' in tag.get('class', []): charge_type_text = tag.find('div').text.strip('\n\t') if charge_type_text.startswith('Monthly Charges'): charge_type_text = 'Monthly Charges' # account monthly fee will be shared by all users w_act_m = float( re.search(r'\$([0-9.]+)', tag.text).group(1)) # national discount is applied to account monthly fee m = re.search(r'National Account Discount.*?\$([0-9.]+)', tag.text, re.DOTALL) w_act_m_disc = float(m.group(1)) if m else 0.0 # this non-zero offset will be used to adjust account # holder's total monthly charge offset = w_act_m - w_act_m_disc m = re.search( r'Total {}.*?\$([0-9.]+)'.format(charge_type_text), tag.text, flags=re.DOTALL) charge_type_name = slugify(charge_type_text) # check if it's a data overage which needs to be shared proportionaly if charge_type_name == overage_charge_type_name: data_overused = True total_overage_charge = float(m.group(1)) user_tag = usage_soup.find('p', string=re.compile( account_holder.name)) usage_tag = list( user_tag.parent.parent.parent.next_siblings)[1] usage = float(usage_tag.findChild('strong').text) usages[account_holder] = usage total_data_allowance = float( list(usage_tag.findChild('strong').next_siblings) [-1].split()[0]) else: charge_total = float(m.group(1)) - offset # save data to db # ChargeType charge_type, _ = ChargeType.get_or_create( type=charge_type_name, text=charge_type_text, charge_category=wireless_charge_category) # Charge new_charge = Charge(user=account_holder, charge_type=charge_type, billing_cycle=billing_cycle, amount=charge_total) new_charge.save() offset = 0.0 # iterate regular users remaining_users = [u for u in users if u.number != number] for user in remaining_users: charge_total = 0.0 name, number = user.name, user.number # charge section starts with user name followed by his/her number target = soup.find('div', string=re.compile('{} {}'.format(name, number))) for tag in target.parent.next_siblings: # all charge data are in divs if not isinstance(tag, Tag) or tag.name != 'div': continue # charge section ends with Total for number if 'Total for {}'.format(number) in tag.text: break # each charge type has 'accSummary' as one of its css classes if 'accSummary' in tag.get('class', []): charge_type_text = tag.find('div').text.strip('\n\t') if charge_type_text.startswith('Monthly Charges'): charge_type_text = 'Monthly Charges' m = re.search( r'Total {}.*?\$([0-9.]+)'.format(charge_type_text), tag.text, flags=re.DOTALL) charge_total = float(m.group(1)) # save data to db charge_type_name = slugify(charge_type_text) # ChargeType charge_type, _ = ChargeType.get_or_create( type=charge_type_name, text=charge_type_text, charge_category=wireless_charge_category) # Charge new_charge = Charge(user=user, charge_type=charge_type, billing_cycle=billing_cycle, amount=charge_total) new_charge.save() if charge_total > 0: charged_users.append(user) charge_type, _ = ChargeType.get_or_create( type='data-text-usage-charges', text='Data & Text Usage Charges', charge_category=wireless_charge_category) # data usages user_tag = usage_soup.find('p', string=re.compile(user.name)) if user_tag: usage_tag = list( user_tag.parent.parent.parent.next_siblings)[1] usage = float(usage_tag.findChild('strong').text) usages[user] = usage # update share of account monthly charges for each user # also calculate total wireless charges (for verification later) act_m_share = (w_act_m - w_act_m_disc) / len(charged_users) wireless_total = 0 for user in charged_users: # ChargeType charge_type, _ = ChargeType.get_or_create( type='wireless-acount-monthly-charges-share', text='Account Monthly Charges Share', charge_category=wireless_charge_category) # Charge new_charge = Charge(user=user, charge_type=charge_type, billing_cycle=billing_cycle, amount=act_m_share) new_charge.save() user_total = Charge.select( pw.fn.Sum(Charge.amount).alias('total')).join( ChargeType, on=Charge.charge_type_id == ChargeType.id).where( (Charge.user == user), Charge.billing_cycle == billing_cycle, ChargeType.charge_category == wireless_charge_category) wireless_total += user_total[0].total if data_overused: user_share = total_data_allowance / len(charged_users) overages = { k: v - user_share for k, v in usages.iteritems() if v > user_share } total_overage = sum(overages.values()) for user, overage in overages.iteritems(): overage_charge = overage / total_overage * total_overage_charge print( 'User {} over used {} GB data, will be charged extra ${:.2f}' .format(user.name, overage, overage_charge)) charge_type, _ = ChargeType.get_or_create( type=overage_charge_type_name, text=overage_charge_type_text, charge_category=wireless_charge_category) # Charge new_charge = Charge(user=user, charge_type=charge_type, billing_cycle=billing_cycle, amount=overage_charge) new_charge.save() # aggregate aggregate_wireless_monthly(billing_cycle)
def split_bill(self, bc_name, bill_link): """Parse bill and split wireless charges among users. Currently not parsing U-Verse charges. :param bc_name: billing cycle name :type bc_name: str :param bill_link: url to bill :type bc_name: str :returns: None """ bill_req = self.session.get(bill_link) bill_html = bill_req.text if 'Account Details' not in bill_html: raise ParsingError('Failed to retrieve billing page') soup = BeautifulSoup(bill_html, 'html.parser') start_date, end_date = get_start_end_date(bc_name) billing_cycle = BillingCycle.create(name=bc_name, start_date=start_date, end_date=end_date) # parse user name and number users = self.parse_user_info(bill_html) if not users: return account_holder = users[0] # -------------------------------------------------------------------- # Wireless # -------------------------------------------------------------------- wireless_charge_category, _ = ChargeCategory.get_or_create( category='wireless', text='Wireless') charged_users = users[:1] name, number = account_holder.name, account_holder.number offset = 0.0 # charge section starts with user name followed by his/her number target = soup.find('div', string=re.compile('{} {}'.format(name, number))) for tag in target.parent.next_siblings: # all charge data are in divs if not isinstance(tag, Tag) or tag.name != 'div': continue # charge section ends with Total for number if 'Total for {}'.format(number) in tag.text: break # each charge type has 'accSummary' as one of its css classes if 'accSummary' in tag.get('class', []): charge_type_text = tag.find('div').text.strip('\n\t') if charge_type_text.startswith('Monthly Charges'): charge_type_text = 'Monthly Charges' # account monthly fee will be shared by all users w_act_m = float( re.search(r'\$([0-9.]+)', tag.text).group(1)) # national discount is applied to account monthly fee m = re.search(r'National Account Discount.*?\$([0-9.]+)', tag.text, re.DOTALL) w_act_m_disc = float(m.group(1)) if m else 0.0 # this non-zero offset will be used to adjust account # holder's total monthly charge offset = w_act_m - w_act_m_disc m = re.search( r'Total {}.*?\$([0-9.]+)'.format(charge_type_text), tag.text, flags=re.DOTALL) charge_total = float(m.group(1)) - offset # save data to db charge_type_name = slugify(charge_type_text) # ChargeType charge_type, _ = ChargeType.get_or_create( type=charge_type_name, text=charge_type_text, charge_category=wireless_charge_category) # Charge new_charge = Charge(user=account_holder, charge_type=charge_type, billing_cycle=billing_cycle, amount=charge_total) new_charge.save() offset = 0.0 # iterate regular users for user in users[1:]: charge_total = 0.0 name, number = user.name, user.number # charge section starts with user name followed by his/her number target = soup.find('div', string=re.compile('{} {}'.format(name, number))) for tag in target.parent.next_siblings: # all charge data are in divs if not isinstance(tag, Tag) or tag.name != 'div': continue # charge section ends with Total for number if 'Total for {}'.format(number) in tag.text: break # each charge type has 'accSummary' as one of its css classes if 'accSummary' in tag.get('class', []): charge_type_text = tag.find('div').text.strip('\n\t') if charge_type_text.startswith('Monthly Charges'): charge_type_text = 'Monthly Charges' m = re.search( r'Total {}.*?\$([0-9.]+)'.format(charge_type_text), tag.text, flags=re.DOTALL) charge_total = float(m.group(1)) # save data to db charge_type_name = slugify(charge_type_text) # ChargeType charge_type, _ = ChargeType.get_or_create( type=charge_type_name, text=charge_type_text, charge_category=wireless_charge_category) # Charge new_charge = Charge(user=user, charge_type=charge_type, billing_cycle=billing_cycle, amount=charge_total) new_charge.save() if charge_total > 0: charged_users.append(user) # update share of account monthly charges for each user # also calculate total wireless charges (for verification later) act_m_share = (w_act_m - w_act_m_disc) / len(charged_users) wireless_total = 0 for user in charged_users: # ChargeType charge_type, _ = ChargeType.get_or_create( type='wireless-acount-monthly-charges-share', text='Account Monthly Charges Share', charge_category=wireless_charge_category) # Charge new_charge = Charge(user=user, charge_type=charge_type, billing_cycle=billing_cycle, amount=act_m_share) new_charge.save() user_total = Charge.select( pw.fn.Sum(Charge.amount).alias('total')).join( ChargeType, on=Charge.charge_type_id == ChargeType.id).where( (Charge.user == user), Charge.billing_cycle == billing_cycle, ChargeType.charge_category == wireless_charge_category) wireless_total += user_total[0].total # aggregate aggregate_wireless_monthly(billing_cycle)