def get_cnt(self): list_tiers = [] url = "https://www.tirerack.com/tires/TireSearchResults.jsp" querystring = { "zip-code": "", "width": self.s1 + "/", "ratio": self.s2, "diameter": self.s3, "rearWidth": self.s1 + "/", "rearRatio": self.s2, "rearDiameter": self.s3 } response = self.request_data(url, querystring) soup = bs(response.text.encode('utf-8'), 'html.parser') cnt = soup.findAll('script', {'type': 'text/javascript'}) for jst in cnt: if 'tireList[i] =' in jst.text: up_cnt = jst.text.split('tireList[i] =')[1:] for x, element in enumerate(up_cnt): try: item = element.split("i++;")[0] js_item = parse(item) except: continue try: tier_url_t = 'https://www.tirerack.com' + js_item[ 'url'].replace("'", "").encode('utf-8') except: tier_url_t = '' try: partNumber = js_item['partNumber'].replace( "'", "").encode('utf-8') except: partNumber = '' try: tireMake = js_item['tireMake'].replace( "'", "").encode('utf-8') except: tireMake = '' try: tireModel = js_item['tireModel'].replace( "'", "").encode('utf-8') except: tireModel = '' try: displaySize = js_item['displaySize'].replace( "'", "").encode('utf-8') except: displaySize = '' try: tier_url = tier_url_t + '&partnum=' + partNumber + '&vehicleSearch=false&fromCompare1=yes' except: tier_url = '' if self.brand == tireMake: compare_result = fuzz.token_set_ratio( self.model, tireModel) else: compare_result = 0 tier_list_item = [ tireMake, tireModel, partNumber, displaySize, tier_url, compare_result ] list_tiers.append(tier_list_item) return list_tiers
def _read_invalid_json(path): return barely_json.parse(_open_file(path))
for agency in agencies: planFile = agencies[agency] + fileLocation print(agency, planFile) try: # Give at least ten seconds for slower agencies to respond r = requests.get(planFile, timeout=10, headers=headers) except: print('! Cannot download file. {}'.format(sys.exc_info()[0])) missingAgencies.append(agency) continue if r.status_code == 200 and len(r.text): text = filter_nonprintable(r.text) data = parse(text) # print(data) # DEBUG # TODO: Validate # Insert into database. # loop over our categories. for type in types: if isinstance(type, str): if type in data: row = data[type] elif isinstance(type, tuple): (key, type) = type
def sanitize_response(self, text): try: return json.loads(text) except Exception as e: return parse(text)