Example #1
0
def get_product_pages(static,base_url,ext):
	logger = logging.getLogger('retrieve')
	bsobj = themortgagemeter_utils.get_page(static,'static_html/hsbc/savings-accounts.html',base_url + ext,logger)
	# foreach item in the doormatCol, in the ul, get each li's a element href attribute.
	doormatCols = bsobj.find_all(attrs={'class' : 'doormatCol'})
	for d in doormatCols:
		anchors = d.find_all('a')
		for a in anchors:
			href = a.get('href')
			if href and re.match('.*savings-accounts/.*',href):
				url = base_url + href
				savings_data = savings_util.get_savings_data_object()
				# Set online to and branch to default to Y for HSBC
				savings_data['online'] = 'Y'
				savings_data['branch'] = 'Y'
				#print savings_data
				get_product_page_interest_rates(url + '/interest-rates',savings_data)
				get_product_page_details(url + '/details',savings_data)
def get_product_pages(static, base_url, ext):
    logger = logging.getLogger('retrieve')
    bsobj = themortgagemeter_utils.get_page(
        static, 'static_html/hsbc/savings-accounts.html', base_url + ext,
        logger)
    # foreach item in the doormatCol, in the ul, get each li's a element href attribute.
    doormatCols = bsobj.find_all(attrs={'class': 'doormatCol'})
    for d in doormatCols:
        anchors = d.find_all('a')
        for a in anchors:
            href = a.get('href')
            if href and re.match('.*savings-accounts/.*', href):
                url = base_url + href
                savings_data = savings_util.get_savings_data_object()
                # Set online to and branch to default to Y for HSBC
                savings_data['online'] = 'Y'
                savings_data['branch'] = 'Y'
                #print savings_data
                get_product_page_interest_rates(url + '/interest-rates',
                                                savings_data)
                get_product_page_details(url + '/details', savings_data)
Example #3
0
def get_product_pages(static,base_url,ext_url,logger):
	url = base_url + ext_url
	urls_seen = []
	bsobj = themortgagemeter_utils.get_page(static,'static_html/halifax/savings-accounts.html',url,logger)
	# let's see how much info we can extract from the page
	# Get all the sortable tables and divine as much info as possible from that.
	sortable_tables = doormatCols = bsobj.find_all(attrs={'class' : 'sortableTable'})
	for table in sortable_tables:
		#print table
		for tr in table.find_all('tr'):
			td_idx = 0
			savings_data = savings_util.get_savings_data_object()
			for td in tr.find_all('td'):
				td_text = td.text.encode('utf-8').strip().lower()
				if td_idx == 0:
					# title of account - Junior == child
					#re.match('/product/A[0-9]+.*',href)
					##0 ##<td style="text-align: left;"><a href="/savings/accounts/cash-isas/isa-saver-online/">ISA Saver Online</a></td>
					##1 ##<td style="text-align: left;"><strong class="apr">1.35%</strong> tax free/AER variable including 12 month fixed bonus of<strong> </strong>1.10%</td>
					##2 ##<td>£1</td>
					##3 ##<td>Variable</td>
					##4 ##<td>Unlimited</td>
					##5 ##<td style="text-align: center;"><img alt="Online" src="/common/images/icons/mousegrey.gif" title="Online"/></td>
					##6 ##<td><a href="/savings/accounts/cash-isas/isa-saver-online/"><img alt="Find out more" src="/common/images/Buttons/primary_find_out_more.gif"/></a></td>
					#print "0: " + td_text
					if re.match('^.*isa.*$',td_text):
						savings_data['isa'] = 'Y'
					if re.match('^.*junior.*$',td_text):
						savings_data['child'] = 'Y'
				elif td_idx == 1:
					#print "1: " + td_text
					# We don't bother with this at the moment - TODO - sort this out
					#if re.match('.*bonus.*',td_text):
					#	savings_data['bonus'] = 'Y'
					pass
				elif td_idx == 2:
					#print "2: " + td_text
					# minimum investment, max is always infinity
					min_amt = themortgagemeter_utils.get_money(td_text,logger)
					savings_data['min_amt'] = min_amt
				elif td_idx == 3:
					#print "3: " + td_text
					# Variable/Fixed
					if re.match('.*variable.*',td_text):
						savings_data['variability'] = 'V'
					elif re.match('.*fixed.*',td_text):
						savings_data['variability'] = 'F'
					else:
						themortgagemeter_utils.record_alert('ERROR: unknown variability: ' + td_text,logger,themortgagemeter_db.db_connection,themortgagemeter_db.cursor)
						exit()
				elif td_idx == 4:
					#print "4: " + td_text
					# Let's assume we'll get this info from the sub-page.
					# Withdrawals allowed: "None, by closure only", "Unlimited", "None, until child is 18"
					pass
				elif td_idx == 5:
					#print "5: " + td_text
					for img in td.find_all('img'):
						title = img.get('title').lower().strip()
						if title == 'online':
							savings_data['online'] = 'Y'
						elif title == 'branch':
							savings_data['branch'] = 'Y'
							# I'm going to ignore "phone"
						elif title == 'phone':
							pass
				elif td_idx == 6:
					#print "6: " + td_text
					# more details link
					new_url = base_url + td.find_all('a')[0].get('href')
					if new_url in urls_seen:
						continue
					savings_array = process_more_info_page(savings_data,new_url,logger)
					print new_url
					for this_savings_data in savings_array:
						# insert savings here TODO.
						print this_savings_data
						isa = this_savings_data['isa']
						regular_saver = this_savings_data['regular_saver']
						regular_saver_frequency_period = this_savings_data['regular_saver_frequency_period']
						regular_saver_frequency_type = this_savings_data['regular_saver_frequency_type']
						regular_saver_min_amt = this_savings_data['regular_saver_min_amt']
						regular_saver_max_amt = this_savings_data['regular_saver_max_amt']
						bonus = this_savings_data['bonus']
						bonus_frequency_period = this_savings_data['bonus_frequency_period']
						bonus_frequency_type = this_savings_data['bonus_frequency_type']
						online = this_savings_data['online']
						branch = this_savings_data['branch']
						variability = this_savings_data['variability']
						min_amt = this_savings_data['min_amt']
						max_amt = this_savings_data['max_amt']
						gross_percent = this_savings_data['gross_percent']
						aer_percent = this_savings_data['aer_percent']
						interest_paid = this_savings_data['interest_paid']
						child = this_savings_data['child']
						savings_period = this_savings_data['savings_period']
						savings_util.handle_savings_insert(institution_code, isa, regular_saver, regular_saver_frequency_period, regular_saver_frequency_type, regular_saver_min_amt, regular_saver_max_amt, bonus, bonus_frequency_period, bonus_frequency_type, online, branch, variability, savings_period, min_amt, max_amt, gross_percent, aer_percent, child, interest_paid, url, logger)
					urls_seen.insert(0,new_url)
				else:
					themortgagemeter_utils.record_alert('ERROR: too many tds in tr: ' + tr,logger,themortgagemeter_db.db_connection,themortgagemeter_db.cursor)
					exit()
				td_idx = td_idx + 1
def get_product_pages(static, base_url, ext_url, logger):
    url = base_url + ext_url
    urls_seen = []
    bsobj = themortgagemeter_utils.get_page(
        static, 'static_html/halifax/savings-accounts.html', url, logger)
    # let's see how much info we can extract from the page
    # Get all the sortable tables and divine as much info as possible from that.
    sortable_tables = doormatCols = bsobj.find_all(
        attrs={'class': 'sortableTable'})
    for table in sortable_tables:
        #print table
        for tr in table.find_all('tr'):
            td_idx = 0
            savings_data = savings_util.get_savings_data_object()
            for td in tr.find_all('td'):
                td_text = td.text.encode('utf-8').strip().lower()
                if td_idx == 0:
                    # title of account - Junior == child
                    #re.match('/product/A[0-9]+.*',href)
                    ##0 ##<td style="text-align: left;"><a href="/savings/accounts/cash-isas/isa-saver-online/">ISA Saver Online</a></td>
                    ##1 ##<td style="text-align: left;"><strong class="apr">1.35%</strong> tax free/AER variable including 12 month fixed bonus of<strong> </strong>1.10%</td>
                    ##2 ##<td>£1</td>
                    ##3 ##<td>Variable</td>
                    ##4 ##<td>Unlimited</td>
                    ##5 ##<td style="text-align: center;"><img alt="Online" src="/common/images/icons/mousegrey.gif" title="Online"/></td>
                    ##6 ##<td><a href="/savings/accounts/cash-isas/isa-saver-online/"><img alt="Find out more" src="/common/images/Buttons/primary_find_out_more.gif"/></a></td>
                    #print "0: " + td_text
                    if re.match('^.*isa.*$', td_text):
                        savings_data['isa'] = 'Y'
                    if re.match('^.*junior.*$', td_text):
                        savings_data['child'] = 'Y'
                elif td_idx == 1:
                    #print "1: " + td_text
                    # We don't bother with this at the moment - TODO - sort this out
                    #if re.match('.*bonus.*',td_text):
                    #	savings_data['bonus'] = 'Y'
                    pass
                elif td_idx == 2:
                    #print "2: " + td_text
                    # minimum investment, max is always infinity
                    min_amt = themortgagemeter_utils.get_money(td_text, logger)
                    savings_data['min_amt'] = min_amt
                elif td_idx == 3:
                    #print "3: " + td_text
                    # Variable/Fixed
                    if re.match('.*variable.*', td_text):
                        savings_data['variability'] = 'V'
                    elif re.match('.*fixed.*', td_text):
                        savings_data['variability'] = 'F'
                    else:
                        themortgagemeter_utils.record_alert(
                            'ERROR: unknown variability: ' + td_text, logger,
                            themortgagemeter_db.db_connection,
                            themortgagemeter_db.cursor)
                        exit()
                elif td_idx == 4:
                    #print "4: " + td_text
                    # Let's assume we'll get this info from the sub-page.
                    # Withdrawals allowed: "None, by closure only", "Unlimited", "None, until child is 18"
                    pass
                elif td_idx == 5:
                    #print "5: " + td_text
                    for img in td.find_all('img'):
                        title = img.get('title').lower().strip()
                        if title == 'online':
                            savings_data['online'] = 'Y'
                        elif title == 'branch':
                            savings_data['branch'] = 'Y'
                            # I'm going to ignore "phone"
                        elif title == 'phone':
                            pass
                elif td_idx == 6:
                    #print "6: " + td_text
                    # more details link
                    new_url = base_url + td.find_all('a')[0].get('href')
                    if new_url in urls_seen:
                        continue
                    savings_array = process_more_info_page(
                        savings_data, new_url, logger)
                    print new_url
                    for this_savings_data in savings_array:
                        # insert savings here TODO.
                        print this_savings_data
                        isa = this_savings_data['isa']
                        regular_saver = this_savings_data['regular_saver']
                        regular_saver_frequency_period = this_savings_data[
                            'regular_saver_frequency_period']
                        regular_saver_frequency_type = this_savings_data[
                            'regular_saver_frequency_type']
                        regular_saver_min_amt = this_savings_data[
                            'regular_saver_min_amt']
                        regular_saver_max_amt = this_savings_data[
                            'regular_saver_max_amt']
                        bonus = this_savings_data['bonus']
                        bonus_frequency_period = this_savings_data[
                            'bonus_frequency_period']
                        bonus_frequency_type = this_savings_data[
                            'bonus_frequency_type']
                        online = this_savings_data['online']
                        branch = this_savings_data['branch']
                        variability = this_savings_data['variability']
                        min_amt = this_savings_data['min_amt']
                        max_amt = this_savings_data['max_amt']
                        gross_percent = this_savings_data['gross_percent']
                        aer_percent = this_savings_data['aer_percent']
                        interest_paid = this_savings_data['interest_paid']
                        child = this_savings_data['child']
                        savings_period = this_savings_data['savings_period']
                        savings_util.handle_savings_insert(
                            institution_code, isa, regular_saver,
                            regular_saver_frequency_period,
                            regular_saver_frequency_type,
                            regular_saver_min_amt, regular_saver_max_amt,
                            bonus, bonus_frequency_period,
                            bonus_frequency_type, online, branch, variability,
                            savings_period, min_amt, max_amt, gross_percent,
                            aer_percent, child, interest_paid, url, logger)
                    urls_seen.insert(0, new_url)
                else:
                    themortgagemeter_utils.record_alert(
                        'ERROR: too many tds in tr: ' + tr, logger,
                        themortgagemeter_db.db_connection,
                        themortgagemeter_db.cursor)
                    exit()
                td_idx = td_idx + 1