def results_leasingfinder():
if request.method == 'POST':
month_list_entity = ds.get_saved_months('month_list_leasing')
if month_list_entity != None:
tmp_months_to_show = month_list_entity.month_list_leasing
leasingfinder_data_final = []
one_entry_dict = {}
f = request.files['file']
header = f.headers['Content-Type']
parsed_header = parse_options_header(header)
blob_key = parsed_header[1]['blob-key']
blob_info = blobstore.get(blob_key)
blob_data = blob_info.open().read()
if blob_info.content_type == 'application/json':
leasingfinder_data = leasingfinder.parse_csv_or_json(json.load(StringIO.StringIO(blob_data)))
elif blob_info.content_type == 'text/csv' or blob_info.content_type == 'application/octet-stream':
leasingfinder_data = leasingfinder.parse_csv_or_json(csv.DictReader(StringIO.StringIO(blob_data)))
elif blob_info.content_type == 'text/html':
leasingfinder_data = leasingfinder.parse_html_hertz(blob_data)
else:
blobstore.delete(blob_key)
return "Error, not able to parse the file uploaded, file format: %s not supported."%blob_info.content_type
for i in leasingfinder_data:
entry_key = str(i['year'])+str(i['brand'])+str(i['name'])+str(i['price_date'])
if int(i['leasing_months']) != 0:
i['average leasing'] = (int(i['leasing_months'])*int(i['lease'])+int(i['due_at_signing']))/int(i['leasing_months'])
if int(i['hwy_mpg']) != 0:
i['average mpg'] = (int(i['hwy_mpg'])+int(i['city_mpg']))/2
i['price_date'] = i['price_date'][:6]
ds.save_general_entity(entry_key, i)
# leasingfinder_data_final.append(json.dumps(i))
# one_entry_dict = {}
#use orderdict!
# for j,key in enumerate(i.keys()):
# one_entry_dict['index'+str(j)] = i[key]
# index_nums = one_entry_dict.keys()
# leasingfinder_data_final_keys = i.keys()
# str_to_index_mapping = dict(zip(leasingfinder_data_final_keys, index_nums))
tmp_month = i['price_date']
if tmp_month not in tmp_months_to_show:
for i,m in enumerate(tmp_months_to_show):
if int(tmp_month) > int(m):
tmp_months_to_show.insert(i, tmp_month)
ds.save_updated_months('month_list_leasing', tmp_months_to_show)
blobstore.delete(blob_key)
#new URL for next upload
# upload_url = blobstore.create_upload_url('/leasingfinder/results')
# print leasingfinder_data_final
# my_title = "Yeah! Totally %d cars found. Check out the lower left corner ones!"%len(leasingfinder_data_final)
# return render_template('leasing_finder_show.html', uploadurl=upload_url,
# leasingfinder_data = leasingfinder_data_final, my_title = my_title)
return redirect('leasingfinder')
def getdata():
get_save_tweet(debug=False)
print "Get data done!"
month_list_entity = ds.get_saved_months('month_list')
if month_list_entity != None:
tmp_months_to_show = month_list_entity.month_list
else:
tmp_months_to_show = ["201512"]
ds.save_updated_months('month_list', tmp_months_to_show)
current_month = time.strftime("%Y%m")
if current_month != tmp_months_to_show[0]:
print "####current month is not same as latest month from saved month list, insert&save at the list head"
tmp_months_to_show.insert(0, current_month)
ds.save_updated_months('month_list', tmp_months_to_show)
return "Cron task done!"
def get_share():
data_list_share = []
months_list = []
month_list_entity = ds.get_saved_months('month_list')
if month_list_entity != None:
#months_list is list, months_to_show is string
months_list = month_list_entity.month_list
months_to_show = months_list
# months_to_show = json.dumps(months_list)
else:
months_to_show = '201501'
if request.method == 'POST':
month = request.form.get("month","")
else:
#use latest month when GET by default
month = months_list[0]
data_entities = ds.get_all_entities()
for i in data_entities:
data_list_share.append(json.dumps(i.to_dict()))
monthly_total_pertype_dict_list = []
per_car_type_list = []
for car_type in TREND_TOP_V_TYPE:
monthly_total_pertype_entities = ds.get_monthly_total_pertype_bytypemonth(car_type,month)
for i in monthly_total_pertype_entities:
per_car_type_list.append({car_type:i.monthly_total_no})
monthly_total_pertype_dict_list.append({'month':str(month), 'per_type_num':per_car_type_list})
# print monthly_total_pertype_dict_list
monthly_total_perbrand_dict_list = []
per_car_brand_list = {}
for car_brand in TREND_TOP_V_BRAND:
monthly_total_perbrand_entities = ds.get_monthly_total_perbrand_bybrandmonth(car_brand,month)
for i in monthly_total_perbrand_entities:
per_car_brand_list[car_brand] = i.monthly_total_no
monthly_total_perbrand_dict_list.append({'month':str(month), 'per_brand_num':per_car_brand_list})
# print monthly_total_perbrand_dict_list
return render_template('market_share.html', data_list_share=data_list_share, months_show=months_to_show,\
cur_month = month, brands_all = TREND_TOP_V_BRAND, \
monthly_total_pertype_dict_list = monthly_total_pertype_dict_list, \
monthly_total_perbrand_dict_list = monthly_total_perbrand_dict_list)
def main():
ret_list = []
tweets = []
display_tweets = []
models_to_show = ALL_TOP_V_SHOW
month_list_entity = ds.get_saved_months('month_list')
if month_list_entity != None:
months_to_show = month_list_entity.month_list
else:
months_to_show = ["201512"]
print "####months_to_show not found, use default:%s"%months_to_show
if request.method == 'POST':
month = request.form.get("month","")
model = request.form.get("model","")
else:
#use latest month when GET by default
month = months_to_show[0]
model = models_to_show[0]
if model in ALL_TOP_V_ALIAS_DICT.keys():
models_alias = ALL_TOP_V_ALIAS_DICT[model]
else:
models_alias = [model]
for model_query in models_alias:
#get the tweets per model, month from db
entities = ds.get_all_entities_bymodelmonth(model_query, month)
for i in entities:
#gmap uses [lat,lng], while twitter uses [lng,lat], here's using [lng,lat]
tmp_coordinate = [(i.coordinate[0]),(i.coordinate[1])]
#WAR for heat in middle of US! remove the 'if' after Dec.
if cmp([-95.712891, 37.09024], tmp_coordinate) != 0:
ret_list.append(tmp_coordinate)
tweets.append([i.month, i.place, i.text])
if len(tweets) > 10:
display_tweets = random.sample(tweets, 10)
return render_template('index.html', coor_data=ret_list, tweet_list=display_tweets, \
model_list=models_to_show, month_list=months_to_show, model=model, month=month, \
total_num=len(tweets))
def process_leasingfinder():
leasingfinder_data_final = []
default_month = datetime.today().strftime("%Y%m")
months_to_show = [default_month]
month_list_entity = ds.get_saved_months('month_list_leasing')
if month_list_entity != None:
months_to_show = month_list_entity.month_list_leasing
default_month = months_to_show[0]
if request.method == 'POST':
default_month = request.form.get("default_month","")
print "Get data for month %s"%default_month
leasingfinder_data = ds.get_general_entities_bymonth(default_month)
if leasingfinder_data != None:
for i in leasingfinder_data:
leasingfinder_data_final.append(json.dumps(json.loads(i.json_str)))
upload_url = blobstore.create_upload_url('/leasingfinder/results')
return render_template('leasing_finder_show.html', uploadurl=upload_url,
leasingfinder_data = leasingfinder_data_final, months_to_show = months_to_show,
default_month = default_month )
def main():
month_list_entity = ds.get_saved_months('month_list')
if month_list_entity != None:
months_to_show = month_list_entity.month_list
else:
months_to_show = ["201501"]
print "####months_to_show not found, use default:%s"%months_to_show
if request.method == 'POST':
month = request.form.get("month","")
else:
#use latest month when GET by default
month = months_to_show[0]
selling_data_list = []
selling_data_entities = ds.get_all_entities_bymonth(month)
for i in selling_data_entities:
selling_data_list.append(json.dumps(i.to_dict()))
# print selling_data_list
monthly_total_pertype_list = []
for car_type in TREND_TOP_V_TYPE:
monthly_total_pertype_entities = ds.get_monthly_total_pertype_bytypemonth(car_type,month)
for i in monthly_total_pertype_entities:
monthly_total_pertype_list.append(i.monthly_total_no)
# print monthly_total_pertype_list
monthly_total_perbrand_list = []
for b in TREND_TOP_V_BRAND:
monthly_total_perbrand_entities = ds.get_monthly_total_perbrand_bybrandmonth(b,month)
for i in monthly_total_perbrand_entities:
monthly_total_perbrand_list.append({str(i.brand):i.monthly_total_no})
# print monthly_total_perbrand_list
return render_template('selling_num.html', data_list=selling_data_list,\
monthly_total_pertype_list = monthly_total_pertype_list,\
monthly_total_perbrand_list = monthly_total_perbrand_list, \
month=month, months_show=months_to_show)
def get_save_selling_num(url):
printset = set(string.printable)
all_rows = []
ret_dict = {}
prius_num = 0
prius_ytd = 0
tmp_months_to_show = []
month_list_entity = ds.get_saved_months('month_list')
if month_list_entity != None:
tmp_months_to_show = month_list_entity.month_list
soup = BeautifulSoup(urllib2.urlopen(url).read(), "html.parser")
table = soup.find('table', attrs={'border':'1'})
if table == None:
table = soup.find('table', attrs={'border':'3'})
table_body = table.find('tbody')
rows = table_body.find_all('tr')
for row in rows:
one_row = []
cols = row.find_all('td')
for entry in cols:
one_row.append(entry.text.strip().lower().replace('\n',' '))
# print one_row
all_rows.append(one_row)
month_str = all_rows[0][2].split()
month = month_str[1] + month_dict[month_str[0].replace('.','')]
if len(tmp_months_to_show) == 0:
tmp_months_to_show.append(month)
ds.save_updated_months('month_list', tmp_months_to_show)
elif month not in tmp_months_to_show:
#tmp_months_to_show is sorted list of months
for i,m in enumerate(tmp_months_to_show):
if int(month) > int(m):
tmp_months_to_show.insert(i, month)
break
else:
continue
if i == len(tmp_months_to_show)-1:
tmp_months_to_show.append(month)
ds.save_updated_months('month_list', tmp_months_to_show)
for v in all_rows[1:]:
name_tmp = v[1].replace('*','').strip().split()
brand = name_tmp[0].strip()
if 'prius' in name_tmp:
brand = 'toyota'
if name_tmp[0] == 'prius':
model = 'prius x'
else:
model = ' '.join(name_tmp[1:])
elif 'ram' in name_tmp:
brand = 'ram'
model = 'p/u'
else:
if not set(name_tmp[-1]).issubset(printset):
del name_tmp[-1]
model = ' '.join(name_tmp[1:])
if brand == 'bmw':
model = "3&4 series"
# print brand,model
ret_dict['brand'] = brand
ret_dict['model'] = model
ret_dict['month'] = month
selling_no = int(v[2].replace(',',''))
if len(v) < 5:
if month_dict[month_str[0]] == '01':
selling_no_ytd = selling_no
else:
selling_no_ytd = 0
else:
if int(month) < 201310:
if v[4] == 'n/a':
selling_no_ytd = 0
else:
selling_no_ytd = int(v[4].replace(',',''))
else:
selling_no_ytd = int(v[5].replace(',',''))
if model == 'prius':
prius_num = selling_no
prius_ytd = selling_no_ytd
selling_no = prius_num
selling_no_ytd = prius_ytd
elif 'prius' in model:
prius_num += selling_no
prius_ytd += selling_no_ytd
selling_no = prius_num
selling_no_ytd = prius_ytd
model = 'prius'
ret_dict['model'] = 'prius'
elif model == 'corolla/matrix':
model = 'corolla'
ret_dict['model'] = 'corolla'
elif model in ['ram p/u','ram','p/u']:
model = 'p/u'
ret_dict['model'] = 'p/u'
if model in TOP_30_TRUCKS:
ret_dict['car_type'] = 'truck'
elif model in TOP_30_CARS:
ret_dict['car_type'] = 'car'
elif model in TOP_30_SUVS:
ret_dict['car_type'] = 'suv'
elif model in TOP_VANS:
ret_dict['car_type'] = 'van'
else:
ret_dict['car_type'] = 'car'
print "##########model type not found for %s"%model
ret_dict['selling_no'] = selling_no
ret_dict['selling_no_ytd'] = selling_no_ytd
entry_key = str(brand)+str(model)+str(month)
ds.save_entity(entry_key, ret_dict)
monthly_total_perbrand = {}
monthly_total_perbrand['month'] = month
data_entities = ds.get_all_entities_bymonth(month)
for b in TREND_TOP_V_BRAND:
monthly_total_perbrand['brand'] = b
monthly_total_perbrand['monthly_total_no'] = 0
for i in data_entities:
if b == i.brand:
monthly_total_perbrand['monthly_total_no'] += i.selling_no
# print monthly_total_perbrand
ds.save_monthly_total_perbrand(b+month, monthly_total_perbrand)
monthly_total_pertype = {}
monthly_total_pertype['month'] = month
for t in TREND_TOP_V_TYPE:
monthly_total_pertype['car_type'] = t
monthly_total_pertype['monthly_total_no'] = 0
for i in data_entities:
if t == "all":
monthly_total_pertype['monthly_total_no'] += i.selling_no
elif t == i.car_type:
monthly_total_pertype['monthly_total_no'] += i.selling_no
# print monthly_total_pertype
ds.save_monthly_total_pertype(t+month, monthly_total_pertype)
def trends():
set_tab = 0
data_list_trends_brand = []
selling_data_list_trends = []
monthly_total_alltype_list = []
monthly_total_list = []
per_model_total_ytd = 0
per_brand_total_ytd = 0
models_to_show = TREND_TOP_V
brands_to_show = TREND_TOP_V_BRAND
tmp_months_to_show = []
tmp_month_list = []
month_list_entity = ds.get_saved_months('month_list')
if month_list_entity != None:
tmp_month_list = month_list_entity.month_list
#default to use latest year
current_year = tmp_month_list[0][:4]
if current_year == None:
current_year = '2015'
if request.method == 'POST':
model_full = request.form.get("model","")
brand = request.form.get("brand","")
current_year = request.form.get("year","")
if model_full == "":
model_full = models_to_show[0]
if brand != "":
set_tab = 1
else:
brand = brands_to_show[0]
else:
model_full = models_to_show[0]
brand = brands_to_show[0]
tmp_months_to_show = []
if tmp_month_list != []:
tmp_year_list = []
years_show_list = []
for m in tmp_month_list:
tmp_year_list.append(m[:4])
if current_year in m:
tmp_months_to_show.append(m)
tmp_year_list = list(set(tmp_year_list))
tmp_year_list.sort(reverse=True)
# print tmp_year_list
years_show_list = tmp_year_list
else:
tmp_months_to_show = ['201501']
if model_full == "top 30 in total":
per_model_total_ytd = 0
for tmp_month in reversed(tmp_months_to_show):
data_dict_brand = {}
brand_total = 0
data_dict_brand['month'] = str(tmp_month)
data_entities_brand_a_month = ds.get_monthly_total_perbrand_bybrandmonth(brand, tmp_month)
for i in data_entities_brand_a_month:
brand_total += i.monthly_total_no
per_brand_total_ytd += brand_total
data_dict_brand['brand_total'] = brand_total
data_list_trends_brand.append(data_dict_brand)
if model_full == "top 30 in total":
monthly_total_alltype_entities = ds.get_monthly_total_pertype_bytypemonth("all",tmp_month)
for i in monthly_total_alltype_entities:
monthly_total_alltype_list.append({'month':str(tmp_month),'monthly_total_no':i.monthly_total_no})
per_model_total_ytd += i.monthly_total_no
# print data_list_trends_brand
# print monthly_total_alltype_list
model =' '.join(model_full.split()[1:])
for tmp_month in reversed(tmp_months_to_show):
selling_data_entities_model = ds.get_all_entities_bymodelmonth(model, tmp_month)
if selling_data_entities_model != None:
for i in selling_data_entities_model:
per_model_total_ytd = i.selling_no_ytd
selling_data_list_trends.append(json.dumps(i.to_dict()))
# print selling_data_list_trends
return render_template('selling_trends.html', data_list_trends = selling_data_list_trends, \
monthly_total_list = monthly_total_alltype_list, model = model_full, models_show = models_to_show, \
per_model_total_ytd = per_model_total_ytd, per_brand_total_ytd = per_brand_total_ytd, \
data_list_trends_brand = data_list_trends_brand, brand = brand, brands_show = brands_to_show, \
years_show = years_show_list, year = current_year,set_tab = set_tab)
