def get_cnt(self):
list_tiers = []
url = "https://www.tirerack.com/tires/TireSearchResults.jsp"
querystring = {
"zip-code": "",
"width": self.s1 + "/",
"ratio": self.s2,
"diameter": self.s3,
"rearWidth": self.s1 + "/",
"rearRatio": self.s2,
"rearDiameter": self.s3
}
response = self.request_data(url, querystring)
soup = bs(response.text.encode('utf-8'), 'html.parser')
cnt = soup.findAll('script', {'type': 'text/javascript'})
for jst in cnt:
if 'tireList[i] =' in jst.text:
up_cnt = jst.text.split('tireList[i] =')[1:]
for x, element in enumerate(up_cnt):
try:
item = element.split("i++;")[0]
js_item = parse(item)
except:
continue
try:
tier_url_t = 'https://www.tirerack.com' + js_item[
'url'].replace("'", "").encode('utf-8')
except:
tier_url_t = ''
try:
partNumber = js_item['partNumber'].replace(
"'", "").encode('utf-8')
except:
partNumber = ''
try:
tireMake = js_item['tireMake'].replace(
"'", "").encode('utf-8')
except:
tireMake = ''
try:
tireModel = js_item['tireModel'].replace(
"'", "").encode('utf-8')
except:
tireModel = ''
try:
displaySize = js_item['displaySize'].replace(
"'", "").encode('utf-8')
except:
displaySize = ''
try:
tier_url = tier_url_t + '&partnum=' + partNumber + '&vehicleSearch=false&fromCompare1=yes'
except:
tier_url = ''
if self.brand == tireMake:
compare_result = fuzz.token_set_ratio(
self.model, tireModel)
else:
compare_result = 0
tier_list_item = [
tireMake, tireModel, partNumber, displaySize, tier_url,
compare_result
]
list_tiers.append(tier_list_item)
return list_tiers
def _read_invalid_json(path):
return barely_json.parse(_open_file(path))
for agency in agencies:
planFile = agencies[agency] + fileLocation
print(agency, planFile)
try:
# Give at least ten seconds for slower agencies to respond
r = requests.get(planFile, timeout=10, headers=headers)
except:
print('! Cannot download file. {}'.format(sys.exc_info()[0]))
missingAgencies.append(agency)
continue
if r.status_code == 200 and len(r.text):
text = filter_nonprintable(r.text)
data = parse(text)
# print(data) # DEBUG
# TODO: Validate
# Insert into database.
# loop over our categories.
for type in types:
if isinstance(type, str):
if type in data:
row = data[type]
elif isinstance(type, tuple):
(key, type) = type
def sanitize_response(self, text):
try:
return json.loads(text)
except Exception as e:
return parse(text)