def results_leasingfinder():
if request.method == 'POST':
month_list_entity = ds.get_saved_months('month_list_leasing')
if month_list_entity != None:
tmp_months_to_show = month_list_entity.month_list_leasing
leasingfinder_data_final = []
one_entry_dict = {}
f = request.files['file']
header = f.headers['Content-Type']
parsed_header = parse_options_header(header)
blob_key = parsed_header[1]['blob-key']
blob_info = blobstore.get(blob_key)
blob_data = blob_info.open().read()
if blob_info.content_type == 'application/json':
leasingfinder_data = leasingfinder.parse_csv_or_json(json.load(StringIO.StringIO(blob_data)))
elif blob_info.content_type == 'text/csv' or blob_info.content_type == 'application/octet-stream':
leasingfinder_data = leasingfinder.parse_csv_or_json(csv.DictReader(StringIO.StringIO(blob_data)))
elif blob_info.content_type == 'text/html':
leasingfinder_data = leasingfinder.parse_html_hertz(blob_data)
else:
blobstore.delete(blob_key)
return "Error, not able to parse the file uploaded, file format: %s not supported."%blob_info.content_type
for i in leasingfinder_data:
entry_key = str(i['year'])+str(i['brand'])+str(i['name'])+str(i['price_date'])
if int(i['leasing_months']) != 0:
i['average leasing'] = (int(i['leasing_months'])*int(i['lease'])+int(i['due_at_signing']))/int(i['leasing_months'])
if int(i['hwy_mpg']) != 0:
i['average mpg'] = (int(i['hwy_mpg'])+int(i['city_mpg']))/2
i['price_date'] = i['price_date'][:6]
ds.save_general_entity(entry_key, i)
# leasingfinder_data_final.append(json.dumps(i))
# one_entry_dict = {}
#use orderdict!
# for j,key in enumerate(i.keys()):
# one_entry_dict['index'+str(j)] = i[key]
# index_nums = one_entry_dict.keys()
# leasingfinder_data_final_keys = i.keys()
# str_to_index_mapping = dict(zip(leasingfinder_data_final_keys, index_nums))
tmp_month = i['price_date']
if tmp_month not in tmp_months_to_show:
for i,m in enumerate(tmp_months_to_show):
if int(tmp_month) > int(m):
tmp_months_to_show.insert(i, tmp_month)
ds.save_updated_months('month_list_leasing', tmp_months_to_show)
blobstore.delete(blob_key)
#new URL for next upload
# upload_url = blobstore.create_upload_url('/leasingfinder/results')
# print leasingfinder_data_final
# my_title = "Yeah! Totally %d cars found. Check out the lower left corner ones!"%len(leasingfinder_data_final)
# return render_template('leasing_finder_show.html', uploadurl=upload_url,
# leasingfinder_data = leasingfinder_data_final, my_title = my_title)
return redirect('leasingfinder')
def getdata():
get_save_tweet(debug=False)
print "Get data done!"
month_list_entity = ds.get_saved_months('month_list')
if month_list_entity != None:
tmp_months_to_show = month_list_entity.month_list
else:
tmp_months_to_show = ["201512"]
ds.save_updated_months('month_list', tmp_months_to_show)
current_month = time.strftime("%Y%m")
if current_month != tmp_months_to_show[0]:
print "####current month is not same as latest month from saved month list, insert&save at the list head"
tmp_months_to_show.insert(0, current_month)
ds.save_updated_months('month_list', tmp_months_to_show)
return "Cron task done!"
def get_save_selling_num(url):
printset = set(string.printable)
all_rows = []
ret_dict = {}
prius_num = 0
prius_ytd = 0
tmp_months_to_show = []
month_list_entity = ds.get_saved_months('month_list')
if month_list_entity != None:
tmp_months_to_show = month_list_entity.month_list
soup = BeautifulSoup(urllib2.urlopen(url).read(), "html.parser")
table = soup.find('table', attrs={'border':'1'})
if table == None:
table = soup.find('table', attrs={'border':'3'})
table_body = table.find('tbody')
rows = table_body.find_all('tr')
for row in rows:
one_row = []
cols = row.find_all('td')
for entry in cols:
one_row.append(entry.text.strip().lower().replace('\n',' '))
# print one_row
all_rows.append(one_row)
month_str = all_rows[0][2].split()
month = month_str[1] + month_dict[month_str[0].replace('.','')]
if len(tmp_months_to_show) == 0:
tmp_months_to_show.append(month)
ds.save_updated_months('month_list', tmp_months_to_show)
elif month not in tmp_months_to_show:
#tmp_months_to_show is sorted list of months
for i,m in enumerate(tmp_months_to_show):
if int(month) > int(m):
tmp_months_to_show.insert(i, month)
break
else:
continue
if i == len(tmp_months_to_show)-1:
tmp_months_to_show.append(month)
ds.save_updated_months('month_list', tmp_months_to_show)
for v in all_rows[1:]:
name_tmp = v[1].replace('*','').strip().split()
brand = name_tmp[0].strip()
if 'prius' in name_tmp:
brand = 'toyota'
if name_tmp[0] == 'prius':
model = 'prius x'
else:
model = ' '.join(name_tmp[1:])
elif 'ram' in name_tmp:
brand = 'ram'
model = 'p/u'
else:
if not set(name_tmp[-1]).issubset(printset):
del name_tmp[-1]
model = ' '.join(name_tmp[1:])
if brand == 'bmw':
model = "3&4 series"
# print brand,model
ret_dict['brand'] = brand
ret_dict['model'] = model
ret_dict['month'] = month
selling_no = int(v[2].replace(',',''))
if len(v) < 5:
if month_dict[month_str[0]] == '01':
selling_no_ytd = selling_no
else:
selling_no_ytd = 0
else:
if int(month) < 201310:
if v[4] == 'n/a':
selling_no_ytd = 0
else:
selling_no_ytd = int(v[4].replace(',',''))
else:
selling_no_ytd = int(v[5].replace(',',''))
if model == 'prius':
prius_num = selling_no
prius_ytd = selling_no_ytd
selling_no = prius_num
selling_no_ytd = prius_ytd
elif 'prius' in model:
prius_num += selling_no
prius_ytd += selling_no_ytd
selling_no = prius_num
selling_no_ytd = prius_ytd
model = 'prius'
ret_dict['model'] = 'prius'
elif model == 'corolla/matrix':
model = 'corolla'
ret_dict['model'] = 'corolla'
elif model in ['ram p/u','ram','p/u']:
model = 'p/u'
ret_dict['model'] = 'p/u'
if model in TOP_30_TRUCKS:
ret_dict['car_type'] = 'truck'
elif model in TOP_30_CARS:
ret_dict['car_type'] = 'car'
elif model in TOP_30_SUVS:
ret_dict['car_type'] = 'suv'
elif model in TOP_VANS:
ret_dict['car_type'] = 'van'
else:
ret_dict['car_type'] = 'car'
print "##########model type not found for %s"%model
ret_dict['selling_no'] = selling_no
ret_dict['selling_no_ytd'] = selling_no_ytd
entry_key = str(brand)+str(model)+str(month)
ds.save_entity(entry_key, ret_dict)
monthly_total_perbrand = {}
monthly_total_perbrand['month'] = month
data_entities = ds.get_all_entities_bymonth(month)
for b in TREND_TOP_V_BRAND:
monthly_total_perbrand['brand'] = b
monthly_total_perbrand['monthly_total_no'] = 0
for i in data_entities:
if b == i.brand:
monthly_total_perbrand['monthly_total_no'] += i.selling_no
# print monthly_total_perbrand
ds.save_monthly_total_perbrand(b+month, monthly_total_perbrand)
monthly_total_pertype = {}
monthly_total_pertype['month'] = month
for t in TREND_TOP_V_TYPE:
monthly_total_pertype['car_type'] = t
monthly_total_pertype['monthly_total_no'] = 0
for i in data_entities:
if t == "all":
monthly_total_pertype['monthly_total_no'] += i.selling_no
elif t == i.car_type:
monthly_total_pertype['monthly_total_no'] += i.selling_no
# print monthly_total_pertype
ds.save_monthly_total_pertype(t+month, monthly_total_pertype)
