def get_product_pages(static,url,logger):
logger.debug("In get_product_pages: " + url)
# Get the svr first (it's global)
lines = themortgagemeter_utils.get_page(False,'','http://www.thechelsea.co.uk/js/mortgage-finder.js',logger,True).split('\n')
for line in lines:
if re.match(r'^var chelseaSVR = "[^%]*%".*',line) != None:
svr_percent = re.match(r'^var chelseaSVR = "([^%]*)%".*$',line).group(1)
break
# Now get the mortgage data
if static:
tree = ET.parse('static_html/chelsea/mortage-product-data-0031.xml')
root = tree.getroot()
else:
root = ET.fromstring(themortgagemeter_utils.get_page(False,'',url,logger,True))
term = str(25 * 12)
for product in root.findall('product'):
apr_percent = product.get('apr').split('%')[0]
rate_percent = product.get('interestRate').split('%')[0]
# No svr supplied, take apr
ltv_percent = product.get('maxLTV').split('%')[0]
mortgage_type_raw = product.get('mortgageType')
name = product.get('name')
booking_fee = product.get('completionFee')
if booking_fee == '':
booking_fee = '0'
existing_borrower = product.get('existingBorrower')
new_borrower = product.get('newBorrower')
first_time_buyer = product.get('firstTimeBuyer')
moving_home = product.get('movingHome')
remortgaging = product.get('remortgaging')
# Gathered data, now let's marshall before submitting.
if mortgage_type_raw == 'fixed':
mortgage_type = 'F'
elif mortgage_type_raw == 'fixedoffset':
mortgage_type = 'F'
elif mortgage_type_raw == 'ftbfixed':
mortgage_type = 'F'
elif mortgage_type_raw == 'ftbfixedoffset':
mortgage_type = 'F'
elif mortgage_type_raw == 'fixedtracker':
# Presumably fixed, then a tracker??
mortgage_type = 'F'
elif mortgage_type_raw == 'tracker':
mortgage_type = 'T'
elif mortgage_type_raw == 'trackeroffset':
mortgage_type = 'T'
elif mortgage_type_raw == 'offset':
mortgage_type = 'T'
elif mortgage_type_raw == 'mixedoffset':
mortgage_type = 'T'
elif mortgage_type_raw == 'rollover':
# rollover? no example, but exists in the docs
#print 'rollover'
#ET.dump(product)
mortgage_type = 'T'
elif mortgage_type_raw == 'mixed':
# WTF is mixed?
mortgage_type = 'T'
else:
# default to variable
#print mortgage_type_raw
mortgage_type = 'V'
# Get a mortgage eligibility dictionary to submit.
mortgage_eligibility_dict = mc_util.get_mortgage_eligibility_dict()
if existing_borrower == 'Y':
mortgage_eligibility_dict['existing_customer'] = 'B'
if new_borrower == 'Y':
mortgage_eligibility_dict['moving_home'] = 'B'
if first_time_buyer == 'Y':
mortgage_eligibility_dict['ftb'] = 'B'
if moving_home == 'Y':
mortgage_eligibility_dict['moving_home'] = 'B'
if remortgaging == 'Y':
mortgage_eligibility_dict['remortgage']= 'B'
eligibilities = mc_util.validate_eligibility_dict(mortgage_eligibility_dict,[])
# use get_months to determine period
initial_period = themortgagemeter_utils.get_months(name,logger)
#ET.dump(product)
#print eligibilities
#print initial_period
#print mortgage_eligibility_dict
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
def get_product_pages(static,url):
logger = logging.getLogger('retrieve')
logger.debug('In get_product pages, base_url: %s',(url))
# URL
if static:
f = open('static_html/santander/productInfo.js')
the_page = f.read()
f.close()
else:
req = urllib2.Request(url)
# Make request.
response = urllib2.urlopen(req)
# Get 'page' content.
the_page = response.read()
logger.debug('In get_product pages, response: \n%s',(the_page))
r = re.compile('\[.*')
for l in the_page.split('\n'):
#print l
if re.match(r,l):
# FORMAT OF ARRAY
# // array format ["OUR INPUT","Product","Maximum loan size formatted","Rate Type","Product Type","Eligibility","Customer Type","Benefit solution","Maximum LTV","Initial rate","Differential to BBR","Standard Variable Rate","APR","Maximum loan size","Booking fee","Charge end date"]]
# ["N907H","DIRECT 2 yr Fixed Homebuyer","£1m","Fixed","2 year Fixed rate","All","Mover|FTB","Free valuation and £250 cashback","0.60","0.0239","n/a","0.0474","0.0460","1000000","1995","03/12/2014","6000"]
def strip_quotes(x): return x.strip('"')
if len(map(strip_quotes,l.strip('[],').split(','))) == 17:
(ref,product,max_loan_size,rate_type,product_type,eligibility,customer_type,benefit_solution,ltv,initial_rate,differential_to_bbr,svr,apr,max_loan_size,booking_fee,end_date,mystery_num) = map(strip_quotes,l.strip('[],').split(','))
else:
logger.critical("Len of array wrong! Continuing. If there are too many of these, fix the bug")
logger.critical(l)
continue
logger.info('ref: ' + ref + ' product: ' + product + ' max_loan_size: ' + max_loan_size + ' rate_type: ' + rate_type+ ' product_type: ' + product_type+ ' eligibility: ' + eligibility+ ' customer_type: ' + customer_type + ' benefit_solution: ' + benefit_solution+ ' ltv: ' + ltv+ ' initial_rate: ' + initial_rate+ ' differential_to_bbr: ' + differential_to_bbr+ ' svr: ' + svr + ' apr: ' + apr+ ' max_loan_size: ' + max_loan_size+ ' booking_fee: ' + booking_fee+ ' end_date: ' + end_date)
#- rate (%)
#- fixed/tracker/discount (fixed/tracker/discount)
#- fix period (n months)
#- fee (n)
#- LTV (%)
#- APR for comparison (%)
#- term (eg 25 years) (n months)
# term (months)
# assume default of 25 years
mortgage_eligibility_dict = mc_util.get_mortgage_eligibility_dict()
eligibilities=[]
if eligibility == 'Loyalty':
mortgage_eligibility_dict['existing_customer'] = 'T'
elif eligibility== 'All':
mortgage_eligibility_dict['existing_customer'] = 'B'
elif eligibility== 'Non-loyalty':
mortgage_eligibility_dict['existing_customer'] = 'F'
elif eligibility== 'Current Account Exclusive':
# Count this as "existing customer"
mortgage_eligibility_dict['existing_customer'] = 'T'
elif eligibility== 'Existing Mover Exclusive':
# Count this as "existing customer"
mortgage_eligibility_dict['existing_customer'] = 'T'
else:
raise Exception('Unrecognised eligibility: ' + eligibility, eligibility, l)
for ct in customer_type.split('|'):
c = mortgage_eligibility_dict.copy()
if ct == 'Mover':
c['moving_home'] = 'T'
eligibilities += mc_util.validate_eligibility_dict(c,[])
elif ct.split()[0].strip() == 'FTB':
c['ftb'] = 'T'
c['existing_customer'] = 'F'
eligibilities += mc_util.validate_eligibility_dict(c,[])
elif ct == 'Remortgage':
c['remortgage'] = 'T'
eligibilities += mc_util.validate_eligibility_dict(c,[])
elif ct == 'FTBRemortgage':
c['remortgage'] = 'T'
c['ftb'] = 'T'
eligibilities += mc_util.validate_eligibility_dict(c,[])
else:
raise Exception('Unrecognised customer type: ', ct, l)
term = str(25 * 12)
rate_percent = str(float(initial_rate) * 100)
if product_type.find('Fixed') != -1:
mortgage_type = 'F'
elif product_type.find('Offset') != -1:
mortgage_type = 'O'
elif product_type.find('Tracker') != -1:
mortgage_type = 'T'
else:
raise Exception('Product type not recognised: ', product_type, l)
if end_date == 'Lifetime Tracker':
initial_period = term
elif product_type.find('year') != -1:
years = re.match(r'^([0-9]+).*$',product_type).group(1)
initial_period = str(int(years) * 12)
elif end_date.find('anniversary') != -1:
years = re.match(r'^([0-9]+).*$',end_date).group(1)
initial_period = str(int(years) * 12)
elif end_date.find('N/A') != -1 or end_date == 'd-M':
initial_period = term
else:
raise Exception('Unrecognised fix period: ', end_date, l)
ltv_percent = str(float(ltv) * 100)
apr_percent = str(float(apr) * 100)
if svr == "n/a" or svr == "N/A":
svr_percent = global_svr
else:
svr_percent = str(float(svr) * 100)
global_svr = svr_percent
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
