def get_product_pages(url, mortgage_type, ltv_percent, eligibilities, logger):
resp = themortgagemeter_utils.get_page(False,
'',
url,
logger,
tostring=True)
json_obj = json.loads(resp)
#print json_obj
mortgage_list = json_obj['mortgageList']
if mortgage_list == 'none':
logger.info('URL returned nothing: ' + url)
return
for item in mortgage_list:
#print item['mortgages']
#mortgage_type #product_fee #ltv #offer #initial_rate #homeowner_variable_rate #repayment #erc #buyer_type #loan_size #overall_cost_for_comparison #payment #initial_term
mortgage = item['mortgages']
#print mortgage
if mortgage['initial_term'] == None:
#print "continuing"
continue
rate_percent = mortgage['initial_rate']['rate']
svr_percent = mortgage['homeowner_variable_rate']['rate']
apr_percent = mortgage['overall_cost_for_comparison']['rate']
initial_period = str(int(float(mortgage['initial_term']) * 12))
booking_fee = mortgage['product_fee']['rate']
buyer_types = mortgage['buyer_type']
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code, mortgage_type,
rate_percent, svr_percent,
apr_percent, ltv_percent,
initial_period, booking_fee, term,
'http://www.lloydsbank.com',
eligibility, logger)
def halifax_ftb_page(static,url,mortgage_type,eligibility,logger):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page(static,'static_html/halifax/fixed.html',url,logger)
trs = bsobj.find_all('tr')
for tr in trs:
mortgage_details = []
for d in tr.strings:
mortgage_details.append(string.strip(d.encode('utf-8')))
if len(mortgage_details) > 19 and len(mortgage_details) < 25:
if mortgage_details[3].find('%') != -1:
initial_period = mortgage_details[1]
if initial_period[0] == 'x':
# handle special case of "dummy row"
continue
rate_percent = mortgage_details[3][:-1]
svr_percent = mortgage_details[6].split()[0][:-1].strip('\xc2').strip('\xa0')
apr_percent = mortgage_details[10].split()[0][:-1]
booking_fee = mortgage_details[12][2:].replace(',','')
# handle special nonsense case
if re.search(r'years',initial_period) and not re.search(r'[0-9]+ years',initial_period):
years = initial_period[0]
initial_period = str(int(years) * 12)
elif re.search(r'months',initial_period) and not re.search(r'[0-9]+ month',initial_period):
initial_period = initial_period[0:2]
else:
initial_period = str(themortgagemeter_utils.get_months(initial_period,logger))
#print mortgage_details
if len(mortgage_details[14].split('-')) > 1:
ltv_percent = str(100 - int(mortgage_details[14].split('-')[0]))
else:
ltv_percent = str(100 - int(mortgage_details[14][0:2]))
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
def process_page(static, url, mortgage_type, eligibility):
logger = logging.getLogger('retrieve')
resp = themortgagemeter_utils.get_page(static,
'static_html/natwest/fix_ftb.html',
url,
logger,
tostring=True)
json_obj = json.loads(resp)
if json_obj['Mortgages'] == None:
logger.info('URL returned nothing: ' + url)
return
#themortgagemeter_utils.pretty_print_json(json_obj)
mortgages = json_obj['Mortgages']['mortgage']
for mortgage in mortgages:
rate_percent = mortgage['initialRate']['value']
svr_percent = mortgage['followOnRate']
apr_percent = mortgage['overallCostAPR']
ltv_percent = mortgage['LTV']
initial_period = str(int(mortgage['initialRate']['duration']) * 12)
booking_fee = mortgage['arrangementFee']
mc_util.handle_mortgage_insert(institution_code, mortgage_type,
rate_percent, svr_percent, apr_percent,
ltv_percent, initial_period,
booking_fee, term, url, eligibility,
logger)
def get_product_pages(static,url,logger):
logger.debug("In get_product_pages: " + url)
# Get the svr first (it's global)
lines = themortgagemeter_utils.get_page(False,'',url,logger,True).split('\n')
# Now get the mortgage data
if static:
tree = ET.parse('static_html/tesco/Products.xml')
root = tree.getroot()
else:
root = ET.fromstring(themortgagemeter_utils.get_page(False,'',url,logger,True))
term = str(25 * 12)
for purchase in ('HousePurchase','Remortgage'):
if purchase == 'HousePurchase':
eligibilities = ['NFTB','NMH']
elif purchase == 'Remortgage':
eligibilities = ['NRM']
for rate_type in ('FixedRate','TrackerRate'):
if rate_type == 'FixedRate':
mortgage_type = 'F'
elif rate_type == 'TrackerRate':
mortgage_type = 'T'
rate_set = root.find(purchase).find(rate_type)
for rate in rate_set.findall('LTV'):
ltv_percent = rate.get('max')
for mortgage in rate.findall('Mortgage'):
#ET.dump(mortgage)
#print "--------------------"
rate_percent = mortgage.find('initialRate').text
apr_percent = mortgage.find('APR').text
svr_percent = mortgage.find('variableRate').text
name = mortgage.find('name').text.split('\n')[0]
initial_period = themortgagemeter_utils.get_months(name,logger)
booking_fee = str(int(mortgage.find('bookingFee').text) + int(mortgage.find('productFee').text))
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,'http://www.tescobank.com/personal/finance/mortgages',eligibility,logger)
def process_page(static, url, mortgage_type, eligibility):
logger = logging.getLogger("retrieve")
resp = themortgagemeter_utils.get_page(static, "static_html/natwest/fix_ftb.html", url, logger, tostring=True)
json_obj = json.loads(resp)
if json_obj["Mortgages"] == None:
logger.info("URL returned nothing: " + url)
return
# themortgagemeter_utils.pretty_print_json(json_obj)
mortgages = json_obj["Mortgages"]["mortgage"]
for mortgage in mortgages:
rate_percent = mortgage["initialRate"]["value"]
svr_percent = mortgage["followOnRate"]
apr_percent = mortgage["overallCostAPR"]
ltv_percent = mortgage["LTV"]
initial_period = str(int(mortgage["initialRate"]["duration"]) * 12)
booking_fee = mortgage["arrangementFee"]
mc_util.handle_mortgage_insert(
institution_code,
mortgage_type,
rate_percent,
svr_percent,
apr_percent,
ltv_percent,
initial_period,
booking_fee,
term,
url,
eligibility,
logger,
)
def process_page(static, base_url, url_suffix, eligibility):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page(
static, 'static_html/northernrock/First-Time-Buyer',
base_url + url_suffix, logger)
anchors = bsobj.find_all(attrs={'class': 'continue moreinfo'})
for anchor in anchors:
url = base_url + anchor['href']
logger.info(url)
anchor_bsobj = themortgagemeter_utils.get_page(
static, 'static_html/northernrock/5yr_everyday_fixed_5ct5', url,
logger)
title = anchor_bsobj.find_all('h1')[0].string
for class_str in ('fixedpanel', 'trackerpanel'):
trs = anchor_bsobj.find_all('tr', 'list ' + class_str)
if trs:
(initial_period, mortgage_type) = process_title(title, logger)
#print title
#print initial_period
#print mortgage_type
# TODO get time period and type from title
# TODO: I think this is wrong! fixedpanel is different from trackerpanel!
#print trs
for tr in trs:
spans = tr.find_all('span')
count = 0
for span in spans:
# Skip the first one.
#print span
#print count
count += 1
if count > 5:
continue
else:
s = span.string
if count == 1:
if s == None:
s = span.em.string
rate_percent = s.split('%')[0]
elif count == 2:
svr_percent = s.split('%')[0]
elif count == 3:
apr_percent = s.split('%')[0]
elif count == 4:
booking_fee = s[1:].replace(',', '')
elif count == 5:
ltv_percent = s.split('%')[0].replace(',', '')
#print spans
if spans:
mc_util.handle_mortgage_insert(
institution_code, mortgage_type, rate_percent,
svr_percent, apr_percent, ltv_percent,
initial_period, booking_fee, term, url,
eligibility, logger)
#print 'rate_percent:' + rate_percent + ' apr_percent:' + apr_percent + ' booking_fee:' + booking_fee + ' ltv_percent:' + ltv_percent + ' mortgage_type:' + mortgage_type + ' initial_period:' + initial_period + ' svr_percent:' + svr_percent
else:
logger.critical('No data from url: ' + url)
def get_mortgage_page_details(static, url, eligibility):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page_headless(url, logger)
# assume default of 25 years
term = str(25 * 12)
#logger.info(bsobj)
# rate
rate_percent = bsobj.find_all(id='InterestRate', limit=1)[0].find_all(
attrs={'class': 'last'}, limit=1)[0].string.string.split('%')[0]
# fee(n)
booking_fee = bsobj.find_all(id='BookingFee', limit=1)[0].find_all(
attrs={'class': 'last'},
limit=1)[0].string.encode('utf_8')[2:].replace(',', '')
if booking_fee == '':
booking_fee = str(0)
# LTV
ltv_percent = bsobj.find_all(id='maxLTV', limit=1)[0].find_all(
attrs={'class': 'last'}, limit=1)[0].string.split('%')[0]
# APR for comparison %
apr_percent = bsobj.find_all(id='OverallCost', limit=1)[0].find_all(
attrs={'class': 'last'}, limit=1)[0].strong.string.strip().split()[0]
# SVR - To test!
svr_percent = bsobj.find_all(id='RevertingVariableRate',
limit=1)[0].find_all(
attrs={'class': 'last'},
limit=1)[0].string.strip().split('%')[0]
# Sometimes not displayed, presumably because not applicable so let's assume it's the headline rate.
if svr_percent == '-':
svr_percent = rate_percent
# fixed/tracker/discount/variable
desc_lower = bsobj.find_all(attrs={'class': 'productResults'},
limit=1)[0].div.h3.string.lower()
if desc_lower.find('fixed') != -1:
mortgage_type = 'F'
elif desc_lower.find('discount') != -1:
mortgage_type = 'D'
elif desc_lower.find('tracker') != -1:
mortgage_type = 'T'
else:
# default to variable
mortgage_type = 'V'
# fix period (months)
for initial_period in (bsobj.find_all(
id='RatePeriod', limit=1)[0].find_all(attrs={'class': 'last'},
limit=1)[0].children):
if initial_period == 'Term of Loan':
initial_period = term
else:
initial_period = themortgagemeter_utils.get_months(
initial_period, logger)
break
mc_util.handle_mortgage_insert(institution_code, mortgage_type,
rate_percent, svr_percent, apr_percent,
ltv_percent, initial_period, booking_fee,
term, url, eligibility, logger)
def process_page(url,ltv,eligibility,mortgage_type,logger):
# "Rates": [
# {
# "Apr": 3.7,
# "AssetUrl": "58305",
# "Availability": "All",
# "BaseRateDifferential": 0,
# "BookingFee": 99,
# "CashBackAmount": 0,
# "Changed": "Changed",
# "Eligibility": "First Time Buyer",
# "EligibilityFeatures": null,
# "ErcPercentage": 3,
# "FeesPayable": 499,
# "HasFreeLegals": false,
# "HasFreeValuations": false,
# "InitialRate": 1.94,
# "MaxLoanAmount": 1000000,
# "MaxLoanToValue": 60,
# "MinLoanAmount": 25000,
# "MinLoanToValue": 0,
# "MonthlyRepayment": 420.93,
# "MortgageType": 0,
# "OverpaymentAmountAllowed": 0,
# "ProductDescription": "2 Year Fixed Rate (First Time Buyer)",
# "ProductFee": 400,
# "RequiresExistingBorrower": false,
# "RequiresFirstTimeBuyer": true,
# "RequiresFlexAccount": false,
# "RequiresFurtherAdvance": false,
# "RequiresHomeMover": true,
# "RequiresOwnSolicitor": false,
# "RequiresRemortgage": false,
# "ReservationFeeScale": "",
# "RevertRate1": 3.99,
# "SpecialProductFee": 900,
# "Term": 24,
# "Withdrawn": false
resp = themortgagemeter_utils.get_page(False,'',url,logger,tostring=True)
json_obj = json.loads(resp)
mortgage_list = json_obj['Rates']
if mortgage_list == 'none':
logger.info('URL returned nothing: ' + url)
return
for mortgage in mortgage_list:
rate_percent = str(mortgage['InitialRate'])
svr_percent = str(mortgage['RevertRate1'])
apr_percent = str(mortgage['Apr'])
initial_period = str(mortgage['Term'])
booking_fee = str(int(mortgage['BookingFee'] + mortgage['SpecialProductFee'] + mortgage['FeesPayable']))
ltv_percent = str(mortgage['MaxLoanToValue'])
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,'http://www.nationwide.co.uk',eligibility,logger)
def get_product_page(static,url):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page(static,'static_html/first_direct/mortgage-rates',url,logger)
print bsobj
sections = bsobj.find_all(attrs={'class':'section'})
for section in sections:
#print section
#print "============================="
tbodys = section.find_all("tbody")
for tbody in tbodys:
print tbody
trs = tbody.find_all("tr")
for tr in trs:
tds = tr.find_all("td")
booking_fee_int = 0
count = 0
for td in tds:
# assume default of 25 years
term = str(25 * 12)
td_text = td.text.strip().encode('utf-8')
#print count
#print td
if count == 0:
#initial_period
initial_period = themortgagemeter_utils.get_months(td_text,logger)
#mortgage_type F/D/T/O/V
mortgage_type = mc_util.get_mortgage_type(td_text,logger)
#eligibility
print td_text
pass
elif count == 1:
#ltv_percent
ltv_percent = themortgagemeter_utils.get_percentage(td_text,logger)
elif count == 2:
#rate_percent
rate_percent = themortgagemeter_utils.get_percentage(td_text,logger)
elif count == 3:
#svr_percent
svr_percent = themortgagemeter_utils.get_percentage(td_text,logger)
elif count == 4:
#apr_percent
apr_percent = themortgagemeter_utils.get_percentage(td_text,logger)
elif count == 5:
booking_fee_int = booking_fee_int + int(themortgagemeter_utils.get_money(td_text,logger))
elif count == 6:
booking_fee_int = booking_fee_int + int(themortgagemeter_utils.get_money(td_text,logger))
count = count + 1
booking_fee = str(booking_fee_int)
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
def process_page(static,base_url,url_suffix,eligibility):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page(static,'static_html/northernrock/First-Time-Buyer',base_url + url_suffix,logger)
anchors = bsobj.find_all(attrs={'class' : 'continue moreinfo'})
for anchor in anchors:
url = base_url + anchor['href']
logger.info(url)
anchor_bsobj = themortgagemeter_utils.get_page(static,'static_html/northernrock/5yr_everyday_fixed_5ct5',url,logger)
title = anchor_bsobj.find_all('h1')[0].string
for class_str in ('fixedpanel','trackerpanel'):
trs = anchor_bsobj.find_all('tr','list ' + class_str)
if trs:
(initial_period,mortgage_type) = process_title(title,logger)
#print title
#print initial_period
#print mortgage_type
# TODO get time period and type from title
# TODO: I think this is wrong! fixedpanel is different from trackerpanel!
#print trs
for tr in trs:
spans = tr.find_all('span')
count = 0
for span in spans:
# Skip the first one.
#print span
#print count
count += 1
if count > 5:
continue
else:
s = span.string
if count == 1:
if s == None:
s = span.em.string
rate_percent = s.split('%')[0]
elif count == 2:
svr_percent = s.split('%')[0]
elif count == 3:
apr_percent = s.split('%')[0]
elif count == 4:
booking_fee = s[1:].replace(',','')
elif count == 5:
ltv_percent = s.split('%')[0].replace(',','')
#print spans
if spans:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
#print 'rate_percent:' + rate_percent + ' apr_percent:' + apr_percent + ' booking_fee:' + booking_fee + ' ltv_percent:' + ltv_percent + ' mortgage_type:' + mortgage_type + ' initial_period:' + initial_period + ' svr_percent:' + svr_percent
else:
logger.critical('No data from url: ' + url)
def halifax_ftb_page(static, url, mortgage_type, eligibility, logger):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page(static,
'static_html/halifax/fixed.html',
url, logger)
trs = bsobj.find_all('tr')
for tr in trs:
mortgage_details = []
for d in tr.strings:
mortgage_details.append(string.strip(d.encode('utf-8')))
if len(mortgage_details) > 19 and len(mortgage_details) < 25:
if mortgage_details[3].find('%') != -1:
initial_period = mortgage_details[1]
if initial_period[0] == 'x':
# handle special case of "dummy row"
continue
rate_percent = mortgage_details[3][:-1]
svr_percent = mortgage_details[6].split()[0][:-1].strip(
'\xc2').strip('\xa0')
apr_percent = mortgage_details[10].split()[0][:-1]
booking_fee = mortgage_details[12][2:].replace(',', '')
# handle special nonsense case
if re.search(r'years', initial_period) and not re.search(
r'[0-9]+ years', initial_period):
years = initial_period[0]
initial_period = str(int(years) * 12)
elif re.search(r'months',
initial_period) and not re.search(
r'[0-9]+ month', initial_period):
initial_period = initial_period[0:2]
else:
initial_period = str(
themortgagemeter_utils.get_months(
initial_period, logger))
#print mortgage_details
if len(mortgage_details[14].split('-')) > 1:
ltv_percent = str(
100 - int(mortgage_details[14].split('-')[0]))
else:
ltv_percent = str(100 - int(mortgage_details[14][0:2]))
mc_util.handle_mortgage_insert(institution_code,
mortgage_type, rate_percent,
svr_percent, apr_percent,
ltv_percent, initial_period,
booking_fee, term, url,
eligibility, logger)
def halifax_remortgage_page(static,url,mortgage_type,eligibility,logger):
bsobj = themortgagemeter_utils.get_page(static,'static_html/halifax/remortgage-fixed-75ltv.asp',url,logger)
trs = bsobj.find_all('tr')
for tr in trs:
mortgage_details = []
for d in tr.strings:
mortgage_details.append(string.strip(d.encode('utf-8')))
#['\n', 'Term', 'Initial rate', '\xc2\xa0', 'Halifax Homeowner Variable rate thereafter', '\xc2\xa0', 'For the remainder of the term from', '\xc2\xa0', 'The overall cost for comparison is', '\xc2\xa0', 'Product fee', '\xc2\xa0', 'LTV\xc2\xa0\xc2\xa0\xc2\xa0\xc2\xa0\xc2\xa0\xc2\xa0 ', 'Early Repayment Charges until', '\xc2\xa0', 'Loan amount', '\n', 'Extra benefits', '\xc2\xa0', '\n', '\xc2\xa0', '\n']
#['\n', '2 years', '\n', '4.44%', '\n', 'Currently', ' \xc2\xa03.99%', '\n', '30/11/2014', '\n', '4.3% APR', '\n', '\xc2\xa3995', '\n', '75-80%', '\n', '30/11/2014', '\n', '\xc2\xa30-\xc2\xa31m', '\n', 'Halifax Remortgage Service*', '\n', '\n']
logger.debug(mortgage_details)
if len(mortgage_details) > 19 and len(mortgage_details) < 25:
if mortgage_details[3].find('%') != -1:
rate_percent = mortgage_details[3][:-1]
svr_percent = mortgage_details[6].split()[0][:-1].strip('\xc2').strip('\xa0')
apr_percent = mortgage_details[10].split()[0][:-1]
booking_fee = mortgage_details[12][2:].replace(',','')
initial_period = mortgage_details[1]
# handle special nonsense case
if re.search(r'years',initial_period) and not re.search(r'[0-9]+ years',initial_period):
years = initial_period[0]
initial_period = str(int(years) * 12)
elif re.search(r'months',initial_period) and not re.search(r'[0-9]+ month',initial_period):
initial_period = initial_period[0:2]
else:
initial_period = str(themortgagemeter_utils.get_months(initial_period,logger))
ltv_percent = mortgage_details[14].split('-')[1].strip('%')
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
elif len(mortgage_details) == 25:
if mortgage_details[3].find('%') != -1:
rate_percent = mortgage_details[3][:-1]
svr_percent = mortgage_details[8].split()[0][:-1].strip('\xc2').strip('\xa0')
apr_percent = mortgage_details[12].split()[0][:-1]
booking_fee = mortgage_details[14][2:].replace(',','')
initial_period = mortgage_details[1]
if re.search(r'years',initial_period) and not re.search(r'[0-9]+ years',initial_period):
years = initial_period[0]
initial_period = str(int(years) * 12)
elif re.search(r'months',initial_period) and not re.search(r'[0-9]+ month',initial_period):
initial_period = initial_period[0:2]
else:
initial_period = str(themortgagemeter_utils.get_months(initial_period,logger))
ltv_percent = mortgage_details[16].split('-')[1].strip('%')
# handle special nonsense case
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
elif len(mortgage_details) > 3:
logger.debug('Should this be handled?: %s',(mortgage_details))
def get_product_pages(static, url, logger):
logger.debug("In get_product_pages: " + url)
# Get the svr first (it's global)
lines = themortgagemeter_utils.get_page(False, '', url, logger,
True).split('\n')
# Now get the mortgage data
if static:
tree = ET.parse('static_html/tesco/Products.xml')
root = tree.getroot()
else:
root = ET.fromstring(
themortgagemeter_utils.get_page(False, '', url, logger, True))
term = str(25 * 12)
for purchase in ('HousePurchase', 'Remortgage'):
if purchase == 'HousePurchase':
eligibilities = ['NFTB', 'NMH']
elif purchase == 'Remortgage':
eligibilities = ['NRM']
for rate_type in ('FixedRate', 'TrackerRate'):
if rate_type == 'FixedRate':
mortgage_type = 'F'
elif rate_type == 'TrackerRate':
mortgage_type = 'T'
rate_set = root.find(purchase).find(rate_type)
for rate in rate_set.findall('LTV'):
ltv_percent = rate.get('max')
for mortgage in rate.findall('Mortgage'):
#ET.dump(mortgage)
#print "--------------------"
rate_percent = mortgage.find('initialRate').text
apr_percent = mortgage.find('APR').text
svr_percent = mortgage.find('variableRate').text
name = mortgage.find('name').text.split('\n')[0]
initial_period = themortgagemeter_utils.get_months(
name, logger)
booking_fee = str(
int(mortgage.find('bookingFee').text) +
int(mortgage.find('productFee').text))
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(
institution_code, mortgage_type, rate_percent,
svr_percent, apr_percent, ltv_percent,
initial_period, booking_fee, term,
'http://www.tescobank.com/personal/finance/mortgages',
eligibility, logger)
def get_mortgage_page_details(static,url,eligibility):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page_headless(url,logger)
# assume default of 25 years
term = str(25 * 12)
#logger.info(bsobj)
# rate
rate_percent = bsobj.find_all(id='InterestRate',limit=1)[0].find_all(attrs={'class' : 'last'},limit=1)[0].string.string.split('%')[0]
# fee(n)
booking_fee = bsobj.find_all(id='BookingFee',limit=1)[0].find_all(attrs={'class' : 'last'},limit=1)[0].string.encode('utf_8')[2:].replace(',','')
if booking_fee == '':
booking_fee = str(0)
# LTV
ltv_percent = bsobj.find_all(id='maxLTV',limit=1)[0].find_all(attrs={'class' : 'last'},limit=1)[0].string.split('%')[0]
# APR for comparison %
apr_percent = bsobj.find_all(id='OverallCost',limit=1)[0].find_all(attrs={'class' : 'last'},limit=1)[0].strong.string.strip().split()[0]
# SVR - To test!
svr_percent = bsobj.find_all(id='RevertingVariableRate',limit=1)[0].find_all(attrs={'class' : 'last'},limit=1)[0].string.strip().split('%')[0]
# Sometimes not displayed, presumably because not applicable so let's assume it's the headline rate.
if svr_percent == '-':
svr_percent = rate_percent
# fixed/tracker/discount/variable
desc_lower = bsobj.find_all(attrs={'class' : 'productResults'},limit=1)[0].div.h3.string.lower()
if desc_lower.find('fixed') != -1:
mortgage_type = 'F'
elif desc_lower.find('discount') != -1:
mortgage_type = 'D'
elif desc_lower.find('tracker') != -1:
mortgage_type = 'T'
else:
# default to variable
mortgage_type = 'V'
# fix period (months)
for initial_period in (bsobj.find_all(id='RatePeriod',limit=1)[0].find_all(attrs={'class' : 'last'},limit=1)[0].children):
if initial_period == 'Term of Loan':
initial_period = term
else:
initial_period = themortgagemeter_utils.get_months(initial_period,logger)
break
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
def get_product_pages(url,mortgage_type,ltv_percent,eligibilities,logger):
resp = themortgagemeter_utils.get_page(False,'',url,logger,tostring=True)
json_obj = json.loads(resp)
#print json_obj
mortgage_list = json_obj['mortgageList']
if mortgage_list == 'none':
logger.info('URL returned nothing: ' + url)
return
for item in mortgage_list:
#print item['mortgages']
#mortgage_type #product_fee #ltv #offer #initial_rate #homeowner_variable_rate #repayment #erc #buyer_type #loan_size #overall_cost_for_comparison #payment #initial_term
mortgage = item['mortgages']
#print mortgage
if mortgage['initial_term'] == None:
#print "continuing"
continue
rate_percent = mortgage['initial_rate']['rate']
svr_percent = mortgage['homeowner_variable_rate']['rate']
apr_percent = mortgage['overall_cost_for_comparison']['rate']
initial_period = str(int(float(mortgage['initial_term']) * 12))
booking_fee = mortgage['product_fee']['rate']
buyer_types = mortgage['buyer_type']
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,'http://www.lloydsbank.com',eligibility,logger)
def halifax_remortgage_page(static, url, mortgage_type, eligibility, logger):
bsobj = themortgagemeter_utils.get_page(
static, 'static_html/halifax/remortgage-fixed-75ltv.asp', url, logger)
trs = bsobj.find_all('tr')
for tr in trs:
mortgage_details = []
for d in tr.strings:
mortgage_details.append(string.strip(d.encode('utf-8')))
#['\n', 'Term', 'Initial rate', '\xc2\xa0', 'Halifax Homeowner Variable rate thereafter', '\xc2\xa0', 'For the remainder of the term from', '\xc2\xa0', 'The overall cost for comparison is', '\xc2\xa0', 'Product fee', '\xc2\xa0', 'LTV\xc2\xa0\xc2\xa0\xc2\xa0\xc2\xa0\xc2\xa0\xc2\xa0 ', 'Early Repayment Charges until', '\xc2\xa0', 'Loan amount', '\n', 'Extra benefits', '\xc2\xa0', '\n', '\xc2\xa0', '\n']
#['\n', '2 years', '\n', '4.44%', '\n', 'Currently', ' \xc2\xa03.99%', '\n', '30/11/2014', '\n', '4.3% APR', '\n', '\xc2\xa3995', '\n', '75-80%', '\n', '30/11/2014', '\n', '\xc2\xa30-\xc2\xa31m', '\n', 'Halifax Remortgage Service*', '\n', '\n']
logger.debug(mortgage_details)
if len(mortgage_details) > 19 and len(mortgage_details) < 25:
if mortgage_details[3].find('%') != -1:
rate_percent = mortgage_details[3][:-1]
svr_percent = mortgage_details[6].split()[0][:-1].strip(
'\xc2').strip('\xa0')
apr_percent = mortgage_details[10].split()[0][:-1]
booking_fee = mortgage_details[12][2:].replace(',', '')
initial_period = mortgage_details[1]
# handle special nonsense case
if re.search(r'years', initial_period) and not re.search(
r'[0-9]+ years', initial_period):
years = initial_period[0]
initial_period = str(int(years) * 12)
elif re.search(r'months', initial_period) and not re.search(
r'[0-9]+ month', initial_period):
initial_period = initial_period[0:2]
else:
initial_period = str(
themortgagemeter_utils.get_months(
initial_period, logger))
ltv_percent = mortgage_details[14].split('-')[1].strip('%')
mc_util.handle_mortgage_insert(institution_code, mortgage_type,
rate_percent, svr_percent,
apr_percent, ltv_percent,
initial_period, booking_fee,
term, url, eligibility, logger)
elif len(mortgage_details) == 25:
if mortgage_details[3].find('%') != -1:
rate_percent = mortgage_details[3][:-1]
svr_percent = mortgage_details[8].split()[0][:-1].strip(
'\xc2').strip('\xa0')
apr_percent = mortgage_details[12].split()[0][:-1]
booking_fee = mortgage_details[14][2:].replace(',', '')
initial_period = mortgage_details[1]
if re.search(r'years', initial_period) and not re.search(
r'[0-9]+ years', initial_period):
years = initial_period[0]
initial_period = str(int(years) * 12)
elif re.search(r'months', initial_period) and not re.search(
r'[0-9]+ month', initial_period):
initial_period = initial_period[0:2]
else:
initial_period = str(
themortgagemeter_utils.get_months(
initial_period, logger))
ltv_percent = mortgage_details[16].split('-')[1].strip('%')
# handle special nonsense case
mc_util.handle_mortgage_insert(institution_code, mortgage_type,
rate_percent, svr_percent,
apr_percent, ltv_percent,
initial_period, booking_fee,
term, url, eligibility, logger)
elif len(mortgage_details) > 3:
logger.debug('Should this be handled?: %s', (mortgage_details))
def get_product_pages(static,url,logger):
logger.debug("In get_product_pages: " + url)
# Get the svr first (it's global)
lines = themortgagemeter_utils.get_page(False,'','http://www.thechelsea.co.uk/js/mortgage-finder.js',logger,True).split('\n')
for line in lines:
if re.match(r'^var chelseaSVR = "[^%]*%".*',line) != None:
svr_percent = re.match(r'^var chelseaSVR = "([^%]*)%".*$',line).group(1)
break
# Now get the mortgage data
if static:
tree = ET.parse('static_html/chelsea/mortage-product-data-0031.xml')
root = tree.getroot()
else:
root = ET.fromstring(themortgagemeter_utils.get_page(False,'',url,logger,True))
term = str(25 * 12)
for product in root.findall('product'):
apr_percent = product.get('apr').split('%')[0]
rate_percent = product.get('interestRate').split('%')[0]
# No svr supplied, take apr
ltv_percent = product.get('maxLTV').split('%')[0]
mortgage_type_raw = product.get('mortgageType')
name = product.get('name')
booking_fee = product.get('completionFee')
if booking_fee == '':
booking_fee = '0'
existing_borrower = product.get('existingBorrower')
new_borrower = product.get('newBorrower')
first_time_buyer = product.get('firstTimeBuyer')
moving_home = product.get('movingHome')
remortgaging = product.get('remortgaging')
# Gathered data, now let's marshall before submitting.
if mortgage_type_raw == 'fixed':
mortgage_type = 'F'
elif mortgage_type_raw == 'fixedoffset':
mortgage_type = 'F'
elif mortgage_type_raw == 'ftbfixed':
mortgage_type = 'F'
elif mortgage_type_raw == 'ftbfixedoffset':
mortgage_type = 'F'
elif mortgage_type_raw == 'fixedtracker':
# Presumably fixed, then a tracker??
mortgage_type = 'F'
elif mortgage_type_raw == 'tracker':
mortgage_type = 'T'
elif mortgage_type_raw == 'trackeroffset':
mortgage_type = 'T'
elif mortgage_type_raw == 'offset':
mortgage_type = 'T'
elif mortgage_type_raw == 'mixedoffset':
mortgage_type = 'T'
elif mortgage_type_raw == 'rollover':
# rollover? no example, but exists in the docs
#print 'rollover'
#ET.dump(product)
mortgage_type = 'T'
elif mortgage_type_raw == 'mixed':
# WTF is mixed?
mortgage_type = 'T'
else:
# default to variable
#print mortgage_type_raw
mortgage_type = 'V'
# Get a mortgage eligibility dictionary to submit.
mortgage_eligibility_dict = mc_util.get_mortgage_eligibility_dict()
if existing_borrower == 'Y':
mortgage_eligibility_dict['existing_customer'] = 'B'
if new_borrower == 'Y':
mortgage_eligibility_dict['moving_home'] = 'B'
if first_time_buyer == 'Y':
mortgage_eligibility_dict['ftb'] = 'B'
if moving_home == 'Y':
mortgage_eligibility_dict['moving_home'] = 'B'
if remortgaging == 'Y':
mortgage_eligibility_dict['remortgage']= 'B'
eligibilities = mc_util.validate_eligibility_dict(mortgage_eligibility_dict,[])
# use get_months to determine period
initial_period = themortgagemeter_utils.get_months(name,logger)
#ET.dump(product)
#print eligibilities
#print initial_period
#print mortgage_eligibility_dict
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
def process_page(url, logger):
#var mortgages = [
#{
#data: "65543",
#name: "2 year fixed",
#offer: " ",
#customer: "Existing Customer",
#type: "Fixed Rate",
#lowltv: 0,
#highltv: 60,
#initialrate: 2.19,
#until: 2,
#rateafter: 4.95,
#apr: 4.6,
#fee: 499,
#minloan: 1,
#maxloan: 1000000,
#links: "/personal/mortgages/all-our-mortgages/fixed-rate-mortgages/mortgages-fixed-rate-2year-60ltv"},{
#[...]
#data: "655434",
#name: "3 year fixed - Fee Offer ",
#offer: " ",
#customer: "First Time Buyer",
#type: "Fixed Rate",
#lowltv: 90,
#highltv: 95,
#initialrate: 4.99,
#until: 3,
#rateafter: 4.95,
#apr: 5.2,
#fee: 0,
#minloan: 1,
#maxloan: 1000000,
#links: "/personal/mortgages/all-our-mortgages/fixed-rate-mortgages/mortgages-three-year-fixed-rate-95ltv"}
#]
resp = themortgagemeter_utils.get_page(False,
'',
url,
logger,
tostring=True)
# Tidy up json
# http://stackoverflow.com/questions/4033633/handling-lazy-json-in-python-expecting-property-name
resp = re.sub(r"{\s*'?(\w)", r'{"\1', resp)
resp = re.sub(r",\s*'?(\w)", r',"\1', resp)
resp = re.sub(r"(\w)'?\s*:", r'\1":', resp)
resp = re.sub(r":\s*'(\w+)'\s*([,}])", r':"\1"\2', resp)
json_obj = json.loads(resp[16:])
print json_obj
#mortgage_list = json_obj['mortgages']
#if mortgage_list == 'none':
# logger.info('URL returned nothing: ' + url)
# return
for mortgage in json_obj:
customer = mortgage['customer']
if customer == "Existing Customer":
eligibilities = ("EMH", "EBM", "EDE", "EED")
elif customer == "First Time Buyer":
eligibilities = ("NFTB", )
elif customer == "New Customer":
eligibilities = ("NRM", "NMH")
else:
raise Exception('Unrecognised eligibility: ' + eligibility,
eligibility, l)
mortgage_type = mc_util.get_mortgage_type(mortgage['name'], logger)
rate_percent = str(mortgage['initialrate'])
svr_percent = str(mortgage['rateafter'])
apr_percent = str(mortgage['apr'])
initial_period = str(int(mortgage['until'] * 12.0))
booking_fee = str(mortgage['fee'])
ltv_percent = str(mortgage['highltv'])
for eligibility in eligibilities:
print eligibility
mc_util.handle_mortgage_insert(institution_code, mortgage_type,
rate_percent, svr_percent,
apr_percent, ltv_percent,
initial_period, booking_fee, term,
'http://www.nationwide.co.uk',
eligibility, logger)
def get_product_page(static,url,eligibilities):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page(static,'static_html/post_office/our-full-range.html',url,logger)
#print bsobj
term = str(25 * 12)
ltv_elems = bsobj.find_all('h2')
# foreach h2 element, determine the ltv.
# then get the next element (which is the div, class displaytable). Then, for each tr:
# td0 = years of fixed or tracker
# td1 = initial rate
# td2 = svr
# td3 = apr
# td4 = fees
for ltv_elem in ltv_elems:
# For post office, first reported % is 100 - LTV
ltv_elem_str = ltv_elem.string
if (ltv_elem_str):
ltv_percent = themortgagemeter_utils.get_percentage(ltv_elem_str,logger)
if ltv_percent != '':
ltv_percent = str(100 - int(ltv_percent))
else:
continue
else:
continue
div = ltv_elem.fetchNextSiblings(attrs={'class' : 'displaytable'},limit=1)
if (div):
logger.debug('here')
logger.debug(div)
trs = div[0].find_all('tr')
for tr in trs:
logger.debug(tr)
# initialise:
rate_percent = ''
svr_percent = ''
apr_percent = ''
booking_fee = ''
tds = tr.find_all('td')
i = 0
# If there are tds and there are more than 1 of them then we can extract a mortgage...
logger.debug(tr)
if tds and len(tds) > 1:
logger.debug(tds[0].text.encode('utf-8').split('\n'))
s = tds[0].text.encode('utf-8').split('\n')
# Sometimes we get empty fields - we remove them here.
while '' in s:
s.remove('')
initial_period = str(themortgagemeter_utils.get_months(s[i],logger))
#logger.debug('type_str before split: ' + tds[i].text.encode('utf-8'))
#logger.debug('tds i: ' + str(i) + ' tds: ' + str(tds))
#logger.debug('tds i: ' + str(i) + ' tds[i]: ' + str(tds[i].text.encode('utf-8')))
#logger.debug(re.sub('\xa0','',tds[i].text.encode('utf-8')).split())
# TODO: generic text cleansing function
type_str = re.sub('\xa0','',re.sub('\xc2',' ',tds[i].text.encode('utf-8'))).split()[2]
logger.debug('type_str: ' + type_str)
if type_str == 'fixed':
mortgage_type = 'F'
elif type_str == 'tracker':
mortgage_type = 'T'
else:
themortgagemeter_utils.record_alert('ERROR: PSTFFC neither fixed nor tracker: ' + type_str,logger,themortgagemeter_db.db_connection,themortgagemeter_db.cursor)
i+=1
j = 0
for td in tds[i].text.encode('utf-8').split('\n'):
t = tds[i].text.encode('utf-8').split('\n')[j]
rate_percent = themortgagemeter_utils.get_percentage(t,logger)
if rate_percent != '':
break
j += 1
while svr_percent == '':
i+=1
for t in tds[i].text.encode('utf-8').split('\n'):
svr_percent = themortgagemeter_utils.get_percentage(t,logger)
if svr_percent != '':
break
while apr_percent == '':
i+=1
for t in tds[i].text.encode('utf-8').split('\n'):
apr_percent = themortgagemeter_utils.get_percentage(t,logger)
if apr_percent != '':
break
i+=1
booking_fee = tds[i].text.strip().encode('utf-8')[2:].replace(',','')
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
else:
pass
def get_product_page(static, url, eligibilities):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page(
static, 'static_html/post_office/our-full-range.html', url, logger)
#print bsobj
term = str(25 * 12)
ltv_elems = bsobj.find_all('h2')
# foreach h2 element, determine the ltv.
# then get the next element (which is the div, class displaytable). Then, for each tr:
# td0 = years of fixed or tracker
# td1 = initial rate
# td2 = svr
# td3 = apr
# td4 = fees
for ltv_elem in ltv_elems:
# For post office, first reported % is 100 - LTV
ltv_elem_str = ltv_elem.string
if (ltv_elem_str):
ltv_percent = themortgagemeter_utils.get_percentage(
ltv_elem_str, logger)
if ltv_percent != '':
ltv_percent = str(100 - int(ltv_percent))
else:
continue
else:
continue
div = ltv_elem.fetchNextSiblings(attrs={'class': 'displaytable'},
limit=1)
if (div):
logger.debug('here')
logger.debug(div)
trs = div[0].find_all('tr')
for tr in trs:
logger.debug(tr)
# initialise:
rate_percent = ''
svr_percent = ''
apr_percent = ''
booking_fee = ''
tds = tr.find_all('td')
i = 0
# If there are tds and there are more than 1 of them then we can extract a mortgage...
logger.debug(tr)
if tds and len(tds) > 1:
logger.debug(tds[0].text.encode('utf-8').split('\n'))
s = tds[0].text.encode('utf-8').split('\n')
# Sometimes we get empty fields - we remove them here.
while '' in s:
s.remove('')
initial_period = str(
themortgagemeter_utils.get_months(s[i], logger))
#logger.debug('type_str before split: ' + tds[i].text.encode('utf-8'))
#logger.debug('tds i: ' + str(i) + ' tds: ' + str(tds))
#logger.debug('tds i: ' + str(i) + ' tds[i]: ' + str(tds[i].text.encode('utf-8')))
#logger.debug(re.sub('\xa0','',tds[i].text.encode('utf-8')).split())
# TODO: generic text cleansing function
type_str = re.sub(
'\xa0', '',
re.sub('\xc2', ' ',
tds[i].text.encode('utf-8'))).split()[2]
logger.debug('type_str: ' + type_str)
if type_str == 'fixed':
mortgage_type = 'F'
elif type_str == 'tracker':
mortgage_type = 'T'
else:
themortgagemeter_utils.record_alert(
'ERROR: PSTFFC neither fixed nor tracker: ' +
type_str, logger,
themortgagemeter_db.db_connection,
themortgagemeter_db.cursor)
i += 1
j = 0
for td in tds[i].text.encode('utf-8').split('\n'):
t = tds[i].text.encode('utf-8').split('\n')[j]
rate_percent = themortgagemeter_utils.get_percentage(
t, logger)
if rate_percent != '':
break
j += 1
while svr_percent == '':
i += 1
for t in tds[i].text.encode('utf-8').split('\n'):
svr_percent = themortgagemeter_utils.get_percentage(
t, logger)
if svr_percent != '':
break
while apr_percent == '':
i += 1
for t in tds[i].text.encode('utf-8').split('\n'):
apr_percent = themortgagemeter_utils.get_percentage(
t, logger)
if apr_percent != '':
break
i += 1
booking_fee = tds[i].text.strip().encode(
'utf-8')[2:].replace(',', '')
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(
institution_code, mortgage_type, rate_percent,
svr_percent, apr_percent, ltv_percent,
initial_period, booking_fee, term, url,
eligibility, logger)
else:
pass
def get_product_pages(static,url):
logger = logging.getLogger('retrieve')
logger.debug('In get_product pages, base_url: %s',(url))
# URL
if static:
f = open('static_html/santander/productInfo.js')
the_page = f.read()
f.close()
else:
req = urllib2.Request(url)
# Make request.
response = urllib2.urlopen(req)
# Get 'page' content.
the_page = response.read()
logger.debug('In get_product pages, response: \n%s',(the_page))
r = re.compile('\[.*')
for l in the_page.split('\n'):
#print l
if re.match(r,l):
# FORMAT OF ARRAY
# // array format ["OUR INPUT","Product","Maximum loan size formatted","Rate Type","Product Type","Eligibility","Customer Type","Benefit solution","Maximum LTV","Initial rate","Differential to BBR","Standard Variable Rate","APR","Maximum loan size","Booking fee","Charge end date"]]
# ["N907H","DIRECT 2 yr Fixed Homebuyer","£1m","Fixed","2 year Fixed rate","All","Mover|FTB","Free valuation and £250 cashback","0.60","0.0239","n/a","0.0474","0.0460","1000000","1995","03/12/2014","6000"]
def strip_quotes(x): return x.strip('"')
if len(map(strip_quotes,l.strip('[],').split(','))) == 17:
(ref,product,max_loan_size,rate_type,product_type,eligibility,customer_type,benefit_solution,ltv,initial_rate,differential_to_bbr,svr,apr,max_loan_size,booking_fee,end_date,mystery_num) = map(strip_quotes,l.strip('[],').split(','))
else:
logger.critical("Len of array wrong! Continuing. If there are too many of these, fix the bug")
logger.critical(l)
continue
logger.info('ref: ' + ref + ' product: ' + product + ' max_loan_size: ' + max_loan_size + ' rate_type: ' + rate_type+ ' product_type: ' + product_type+ ' eligibility: ' + eligibility+ ' customer_type: ' + customer_type + ' benefit_solution: ' + benefit_solution+ ' ltv: ' + ltv+ ' initial_rate: ' + initial_rate+ ' differential_to_bbr: ' + differential_to_bbr+ ' svr: ' + svr + ' apr: ' + apr+ ' max_loan_size: ' + max_loan_size+ ' booking_fee: ' + booking_fee+ ' end_date: ' + end_date)
#- rate (%)
#- fixed/tracker/discount (fixed/tracker/discount)
#- fix period (n months)
#- fee (n)
#- LTV (%)
#- APR for comparison (%)
#- term (eg 25 years) (n months)
# term (months)
# assume default of 25 years
mortgage_eligibility_dict = mc_util.get_mortgage_eligibility_dict()
eligibilities=[]
if eligibility == 'Loyalty':
mortgage_eligibility_dict['existing_customer'] = 'T'
elif eligibility== 'All':
mortgage_eligibility_dict['existing_customer'] = 'B'
elif eligibility== 'Non-loyalty':
mortgage_eligibility_dict['existing_customer'] = 'F'
elif eligibility== 'Current Account Exclusive':
# Count this as "existing customer"
mortgage_eligibility_dict['existing_customer'] = 'T'
elif eligibility== 'Existing Mover Exclusive':
# Count this as "existing customer"
mortgage_eligibility_dict['existing_customer'] = 'T'
else:
raise Exception('Unrecognised eligibility: ' + eligibility, eligibility, l)
for ct in customer_type.split('|'):
c = mortgage_eligibility_dict.copy()
if ct == 'Mover':
c['moving_home'] = 'T'
eligibilities += mc_util.validate_eligibility_dict(c,[])
elif ct.split()[0].strip() == 'FTB':
c['ftb'] = 'T'
c['existing_customer'] = 'F'
eligibilities += mc_util.validate_eligibility_dict(c,[])
elif ct == 'Remortgage':
c['remortgage'] = 'T'
eligibilities += mc_util.validate_eligibility_dict(c,[])
elif ct == 'FTBRemortgage':
c['remortgage'] = 'T'
c['ftb'] = 'T'
eligibilities += mc_util.validate_eligibility_dict(c,[])
else:
raise Exception('Unrecognised customer type: ', ct, l)
term = str(25 * 12)
rate_percent = str(float(initial_rate) * 100)
if product_type.find('Fixed') != -1:
mortgage_type = 'F'
elif product_type.find('Offset') != -1:
mortgage_type = 'O'
elif product_type.find('Tracker') != -1:
mortgage_type = 'T'
else:
raise Exception('Product type not recognised: ', product_type, l)
if end_date == 'Lifetime Tracker':
initial_period = term
elif product_type.find('year') != -1:
years = re.match(r'^([0-9]+).*$',product_type).group(1)
initial_period = str(int(years) * 12)
elif end_date.find('anniversary') != -1:
years = re.match(r'^([0-9]+).*$',end_date).group(1)
initial_period = str(int(years) * 12)
elif end_date.find('N/A') != -1 or end_date == 'd-M':
initial_period = term
else:
raise Exception('Unrecognised fix period: ', end_date, l)
ltv_percent = str(float(ltv) * 100)
apr_percent = str(float(apr) * 100)
if svr == "n/a" or svr == "N/A":
svr_percent = global_svr
else:
svr_percent = str(float(svr) * 100)
global_svr = svr_percent
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
def get_product_pages(static,base_url,suffix,mortgage_type,href_re):
logger = logging.getLogger('retrieve')
bsobj = themortgagemeter_utils.get_page(static,'static_html/skipton/fixed_rate_mortgages.html',base_url + suffix,logger)
term = str(25 * 12)
#print bsobj
anchors = bsobj.find_all(href=href_re)
for anchor in anchors:
#print anchor
# Get from the anchor the ltv and the term
link = anchor.get('href')
url = base_url + link
# Still to get:
rate_percent = 0
svr_percent = 0
apr_percent = 0
booking_fee = 0
application_fee = 0
# eligibilities - first time buyers have own page, so all others?
eligibilities = ['NMH','NRM','ERM','EMH','EBM','EED']
#print link
if re.search(fr_re,link):
initial_period = str(int(re.search(fr_re,link).group(1)) * 12)
ltv_percent = str(int(re.search(fr_re,link).group(2)))
# Now go to link
subpage_bsobj = themortgagemeter_utils.get_page(static,'N/A',url,logger)
table = subpage_bsobj.find_all(attrs={'id' : 'centralContent'},limit=1)[0].find_all('table',limit=1)[0]
#print '==================================================='
#print table
tr_count = 0
for tr in table.find_all('tr'):
tr_count += 1
if tr_count == 3:
rate_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[1].string,logger)
elif tr_count == 4:
svr_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[1].string,logger)
elif tr_count == 5:
apr_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[0].string,logger)
elif tr_count == 7:
application_fee = tr.find_all('td')[0].string.encode('utf_8')[2:].replace(',','')
elif tr_count == 8:
booking_fee = tr.find_all('td')[0].string.encode('utf_8')[2:].replace(',','')
elif re.search(tracker_re,link):
initial_period = str(int(re.search(tracker_re,link).group(1)) * 10)
ltv_percent = str(int(re.search(tracker_re,link).group(2)))
# Now go to link
subpage_bsobj = themortgagemeter_utils.get_page(static,'N/A',url,logger)
#print subpage_bsobj
table = subpage_bsobj.find_all(attrs={'id' : 'centralContent'},limit=1)[0].find_all('table',limit=1)[0]
#print '==================================================='
#print table
tr_count = 0
for tr in table.find_all('tr'):
tr_count += 1
if tr_count == 3:
rate_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[0].string,logger)
elif tr_count == 4:
svr_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[1].string,logger)
elif tr_count == 5:
apr_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[0].string,logger)
elif tr_count == 7:
application_fee = tr.find_all('td')[0].string.encode('utf_8')[2:].replace(',','')
elif tr_count == 8:
booking_fee = tr.find_all('td')[0].string.encode('utf_8')[2:].replace(',','')
elif re.search(discount_re,link):
initial_period = str(int(re.search(discount_re,link).group(1)) * 10)
ltv_percent = str(int(re.search(discount_re,link).group(2)))
# Now go to link
subpage_bsobj = themortgagemeter_utils.get_page(static,'N/A',url,logger)
#print subpage_bsobj
table = subpage_bsobj.find_all(attrs={'id' : 'centralContent'},limit=1)[0].find_all('table',limit=1)[0]
#print '==================================================='
#print table
tr_count = 0
for tr in table.find_all('tr'):
tr_count += 1
if tr_count == 3:
rate_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[1].string,logger)
elif tr_count == 4:
svr_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[1].string,logger)
elif tr_count == 5:
apr_percent = themortgagemeter_utils.get_percentage(tr.find_all('td')[0].string,logger)
elif tr_count == 7:
application_fee = tr.find_all('td')[0].string.encode('utf_8')[2:].replace(',','')
elif tr_count == 8:
booking_fee = tr.find_all('td')[0].string.encode('utf_8')[2:].replace(',','')
elif re.search(ftb_re,link):
themortgagemeter_utils.record_alert('ERROR: SKIPTON first time buyer seen for the first time',logger,themortgagemeter_db.db_connection,themortgagemeter_db.cursor)
continue
else:
raise Exception("Unhandled link " + url,'')
# set up the booking fee
# Sometimes it's "No Fee" on the page
if booking_fee.strip() == "Fee":
booking_fee = "0"
if application_fee.strip() == "Fee":
application_fee = "0"
booking_fee = str(int(booking_fee) + int(application_fee))
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
