def get_product_pages(static,url,logger): logger.debug("In get_product_pages: " + url) # Get the svr first (it's global) lines = themortgagemeter_utils.get_page(False,'','http://www.thechelsea.co.uk/js/mortgage-finder.js',logger,True).split('\n') for line in lines: if re.match(r'^var chelseaSVR = "[^%]*%".*',line) != None: svr_percent = re.match(r'^var chelseaSVR = "([^%]*)%".*$',line).group(1) break # Now get the mortgage data if static: tree = ET.parse('static_html/chelsea/mortage-product-data-0031.xml') root = tree.getroot() else: root = ET.fromstring(themortgagemeter_utils.get_page(False,'',url,logger,True)) term = str(25 * 12) for product in root.findall('product'): apr_percent = product.get('apr').split('%')[0] rate_percent = product.get('interestRate').split('%')[0] # No svr supplied, take apr ltv_percent = product.get('maxLTV').split('%')[0] mortgage_type_raw = product.get('mortgageType') name = product.get('name') booking_fee = product.get('completionFee') if booking_fee == '': booking_fee = '0' existing_borrower = product.get('existingBorrower') new_borrower = product.get('newBorrower') first_time_buyer = product.get('firstTimeBuyer') moving_home = product.get('movingHome') remortgaging = product.get('remortgaging') # Gathered data, now let's marshall before submitting. if mortgage_type_raw == 'fixed': mortgage_type = 'F' elif mortgage_type_raw == 'fixedoffset': mortgage_type = 'F' elif mortgage_type_raw == 'ftbfixed': mortgage_type = 'F' elif mortgage_type_raw == 'ftbfixedoffset': mortgage_type = 'F' elif mortgage_type_raw == 'fixedtracker': # Presumably fixed, then a tracker?? mortgage_type = 'F' elif mortgage_type_raw == 'tracker': mortgage_type = 'T' elif mortgage_type_raw == 'trackeroffset': mortgage_type = 'T' elif mortgage_type_raw == 'offset': mortgage_type = 'T' elif mortgage_type_raw == 'mixedoffset': mortgage_type = 'T' elif mortgage_type_raw == 'rollover': # rollover? no example, but exists in the docs #print 'rollover' #ET.dump(product) mortgage_type = 'T' elif mortgage_type_raw == 'mixed': # WTF is mixed? mortgage_type = 'T' else: # default to variable #print mortgage_type_raw mortgage_type = 'V' # Get a mortgage eligibility dictionary to submit. mortgage_eligibility_dict = mc_util.get_mortgage_eligibility_dict() if existing_borrower == 'Y': mortgage_eligibility_dict['existing_customer'] = 'B' if new_borrower == 'Y': mortgage_eligibility_dict['moving_home'] = 'B' if first_time_buyer == 'Y': mortgage_eligibility_dict['ftb'] = 'B' if moving_home == 'Y': mortgage_eligibility_dict['moving_home'] = 'B' if remortgaging == 'Y': mortgage_eligibility_dict['remortgage']= 'B' eligibilities = mc_util.validate_eligibility_dict(mortgage_eligibility_dict,[]) # use get_months to determine period initial_period = themortgagemeter_utils.get_months(name,logger) #ET.dump(product) #print eligibilities #print initial_period #print mortgage_eligibility_dict for eligibility in eligibilities: mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)
def get_product_pages(static,url): logger = logging.getLogger('retrieve') logger.debug('In get_product pages, base_url: %s',(url)) # URL if static: f = open('static_html/santander/productInfo.js') the_page = f.read() f.close() else: req = urllib2.Request(url) # Make request. response = urllib2.urlopen(req) # Get 'page' content. the_page = response.read() logger.debug('In get_product pages, response: \n%s',(the_page)) r = re.compile('\[.*') for l in the_page.split('\n'): #print l if re.match(r,l): # FORMAT OF ARRAY # // array format ["OUR INPUT","Product","Maximum loan size formatted","Rate Type","Product Type","Eligibility","Customer Type","Benefit solution","Maximum LTV","Initial rate","Differential to BBR","Standard Variable Rate","APR","Maximum loan size","Booking fee","Charge end date"]] # ["N907H","DIRECT 2 yr Fixed Homebuyer","£1m","Fixed","2 year Fixed rate","All","Mover|FTB","Free valuation and £250 cashback","0.60","0.0239","n/a","0.0474","0.0460","1000000","1995","03/12/2014","6000"] def strip_quotes(x): return x.strip('"') if len(map(strip_quotes,l.strip('[],').split(','))) == 17: (ref,product,max_loan_size,rate_type,product_type,eligibility,customer_type,benefit_solution,ltv,initial_rate,differential_to_bbr,svr,apr,max_loan_size,booking_fee,end_date,mystery_num) = map(strip_quotes,l.strip('[],').split(',')) else: logger.critical("Len of array wrong! Continuing. If there are too many of these, fix the bug") logger.critical(l) continue logger.info('ref: ' + ref + ' product: ' + product + ' max_loan_size: ' + max_loan_size + ' rate_type: ' + rate_type+ ' product_type: ' + product_type+ ' eligibility: ' + eligibility+ ' customer_type: ' + customer_type + ' benefit_solution: ' + benefit_solution+ ' ltv: ' + ltv+ ' initial_rate: ' + initial_rate+ ' differential_to_bbr: ' + differential_to_bbr+ ' svr: ' + svr + ' apr: ' + apr+ ' max_loan_size: ' + max_loan_size+ ' booking_fee: ' + booking_fee+ ' end_date: ' + end_date) #- rate (%) #- fixed/tracker/discount (fixed/tracker/discount) #- fix period (n months) #- fee (n) #- LTV (%) #- APR for comparison (%) #- term (eg 25 years) (n months) # term (months) # assume default of 25 years mortgage_eligibility_dict = mc_util.get_mortgage_eligibility_dict() eligibilities=[] if eligibility == 'Loyalty': mortgage_eligibility_dict['existing_customer'] = 'T' elif eligibility== 'All': mortgage_eligibility_dict['existing_customer'] = 'B' elif eligibility== 'Non-loyalty': mortgage_eligibility_dict['existing_customer'] = 'F' elif eligibility== 'Current Account Exclusive': # Count this as "existing customer" mortgage_eligibility_dict['existing_customer'] = 'T' elif eligibility== 'Existing Mover Exclusive': # Count this as "existing customer" mortgage_eligibility_dict['existing_customer'] = 'T' else: raise Exception('Unrecognised eligibility: ' + eligibility, eligibility, l) for ct in customer_type.split('|'): c = mortgage_eligibility_dict.copy() if ct == 'Mover': c['moving_home'] = 'T' eligibilities += mc_util.validate_eligibility_dict(c,[]) elif ct.split()[0].strip() == 'FTB': c['ftb'] = 'T' c['existing_customer'] = 'F' eligibilities += mc_util.validate_eligibility_dict(c,[]) elif ct == 'Remortgage': c['remortgage'] = 'T' eligibilities += mc_util.validate_eligibility_dict(c,[]) elif ct == 'FTBRemortgage': c['remortgage'] = 'T' c['ftb'] = 'T' eligibilities += mc_util.validate_eligibility_dict(c,[]) else: raise Exception('Unrecognised customer type: ', ct, l) term = str(25 * 12) rate_percent = str(float(initial_rate) * 100) if product_type.find('Fixed') != -1: mortgage_type = 'F' elif product_type.find('Offset') != -1: mortgage_type = 'O' elif product_type.find('Tracker') != -1: mortgage_type = 'T' else: raise Exception('Product type not recognised: ', product_type, l) if end_date == 'Lifetime Tracker': initial_period = term elif product_type.find('year') != -1: years = re.match(r'^([0-9]+).*$',product_type).group(1) initial_period = str(int(years) * 12) elif end_date.find('anniversary') != -1: years = re.match(r'^([0-9]+).*$',end_date).group(1) initial_period = str(int(years) * 12) elif end_date.find('N/A') != -1 or end_date == 'd-M': initial_period = term else: raise Exception('Unrecognised fix period: ', end_date, l) ltv_percent = str(float(ltv) * 100) apr_percent = str(float(apr) * 100) if svr == "n/a" or svr == "N/A": svr_percent = global_svr else: svr_percent = str(float(svr) * 100) global_svr = svr_percent for eligibility in eligibilities: mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,url,eligibility,logger)