def results_leasingfinder(): if request.method == 'POST': month_list_entity = ds.get_saved_months('month_list_leasing') if month_list_entity != None: tmp_months_to_show = month_list_entity.month_list_leasing leasingfinder_data_final = [] one_entry_dict = {} f = request.files['file'] header = f.headers['Content-Type'] parsed_header = parse_options_header(header) blob_key = parsed_header[1]['blob-key'] blob_info = blobstore.get(blob_key) blob_data = blob_info.open().read() if blob_info.content_type == 'application/json': leasingfinder_data = leasingfinder.parse_csv_or_json(json.load(StringIO.StringIO(blob_data))) elif blob_info.content_type == 'text/csv' or blob_info.content_type == 'application/octet-stream': leasingfinder_data = leasingfinder.parse_csv_or_json(csv.DictReader(StringIO.StringIO(blob_data))) elif blob_info.content_type == 'text/html': leasingfinder_data = leasingfinder.parse_html_hertz(blob_data) else: blobstore.delete(blob_key) return "Error, not able to parse the file uploaded, file format: %s not supported."%blob_info.content_type for i in leasingfinder_data: entry_key = str(i['year'])+str(i['brand'])+str(i['name'])+str(i['price_date']) if int(i['leasing_months']) != 0: i['average leasing'] = (int(i['leasing_months'])*int(i['lease'])+int(i['due_at_signing']))/int(i['leasing_months']) if int(i['hwy_mpg']) != 0: i['average mpg'] = (int(i['hwy_mpg'])+int(i['city_mpg']))/2 i['price_date'] = i['price_date'][:6] ds.save_general_entity(entry_key, i) # leasingfinder_data_final.append(json.dumps(i)) # one_entry_dict = {} #use orderdict! # for j,key in enumerate(i.keys()): # one_entry_dict['index'+str(j)] = i[key] # index_nums = one_entry_dict.keys() # leasingfinder_data_final_keys = i.keys() # str_to_index_mapping = dict(zip(leasingfinder_data_final_keys, index_nums)) tmp_month = i['price_date'] if tmp_month not in tmp_months_to_show: for i,m in enumerate(tmp_months_to_show): if int(tmp_month) > int(m): tmp_months_to_show.insert(i, tmp_month) ds.save_updated_months('month_list_leasing', tmp_months_to_show) blobstore.delete(blob_key) #new URL for next upload # upload_url = blobstore.create_upload_url('/leasingfinder/results') # print leasingfinder_data_final # my_title = "Yeah! Totally %d cars found. Check out the lower left corner ones!"%len(leasingfinder_data_final) # return render_template('leasing_finder_show.html', uploadurl=upload_url, # leasingfinder_data = leasingfinder_data_final, my_title = my_title) return redirect('leasingfinder')
def getdata(): get_save_tweet(debug=False) print "Get data done!" month_list_entity = ds.get_saved_months('month_list') if month_list_entity != None: tmp_months_to_show = month_list_entity.month_list else: tmp_months_to_show = ["201512"] ds.save_updated_months('month_list', tmp_months_to_show) current_month = time.strftime("%Y%m") if current_month != tmp_months_to_show[0]: print "####current month is not same as latest month from saved month list, insert&save at the list head" tmp_months_to_show.insert(0, current_month) ds.save_updated_months('month_list', tmp_months_to_show) return "Cron task done!"
def get_save_selling_num(url): printset = set(string.printable) all_rows = [] ret_dict = {} prius_num = 0 prius_ytd = 0 tmp_months_to_show = [] month_list_entity = ds.get_saved_months('month_list') if month_list_entity != None: tmp_months_to_show = month_list_entity.month_list soup = BeautifulSoup(urllib2.urlopen(url).read(), "html.parser") table = soup.find('table', attrs={'border':'1'}) if table == None: table = soup.find('table', attrs={'border':'3'}) table_body = table.find('tbody') rows = table_body.find_all('tr') for row in rows: one_row = [] cols = row.find_all('td') for entry in cols: one_row.append(entry.text.strip().lower().replace('\n',' ')) # print one_row all_rows.append(one_row) month_str = all_rows[0][2].split() month = month_str[1] + month_dict[month_str[0].replace('.','')] if len(tmp_months_to_show) == 0: tmp_months_to_show.append(month) ds.save_updated_months('month_list', tmp_months_to_show) elif month not in tmp_months_to_show: #tmp_months_to_show is sorted list of months for i,m in enumerate(tmp_months_to_show): if int(month) > int(m): tmp_months_to_show.insert(i, month) break else: continue if i == len(tmp_months_to_show)-1: tmp_months_to_show.append(month) ds.save_updated_months('month_list', tmp_months_to_show) for v in all_rows[1:]: name_tmp = v[1].replace('*','').strip().split() brand = name_tmp[0].strip() if 'prius' in name_tmp: brand = 'toyota' if name_tmp[0] == 'prius': model = 'prius x' else: model = ' '.join(name_tmp[1:]) elif 'ram' in name_tmp: brand = 'ram' model = 'p/u' else: if not set(name_tmp[-1]).issubset(printset): del name_tmp[-1] model = ' '.join(name_tmp[1:]) if brand == 'bmw': model = "3&4 series" # print brand,model ret_dict['brand'] = brand ret_dict['model'] = model ret_dict['month'] = month selling_no = int(v[2].replace(',','')) if len(v) < 5: if month_dict[month_str[0]] == '01': selling_no_ytd = selling_no else: selling_no_ytd = 0 else: if int(month) < 201310: if v[4] == 'n/a': selling_no_ytd = 0 else: selling_no_ytd = int(v[4].replace(',','')) else: selling_no_ytd = int(v[5].replace(',','')) if model == 'prius': prius_num = selling_no prius_ytd = selling_no_ytd selling_no = prius_num selling_no_ytd = prius_ytd elif 'prius' in model: prius_num += selling_no prius_ytd += selling_no_ytd selling_no = prius_num selling_no_ytd = prius_ytd model = 'prius' ret_dict['model'] = 'prius' elif model == 'corolla/matrix': model = 'corolla' ret_dict['model'] = 'corolla' elif model in ['ram p/u','ram','p/u']: model = 'p/u' ret_dict['model'] = 'p/u' if model in TOP_30_TRUCKS: ret_dict['car_type'] = 'truck' elif model in TOP_30_CARS: ret_dict['car_type'] = 'car' elif model in TOP_30_SUVS: ret_dict['car_type'] = 'suv' elif model in TOP_VANS: ret_dict['car_type'] = 'van' else: ret_dict['car_type'] = 'car' print "##########model type not found for %s"%model ret_dict['selling_no'] = selling_no ret_dict['selling_no_ytd'] = selling_no_ytd entry_key = str(brand)+str(model)+str(month) ds.save_entity(entry_key, ret_dict) monthly_total_perbrand = {} monthly_total_perbrand['month'] = month data_entities = ds.get_all_entities_bymonth(month) for b in TREND_TOP_V_BRAND: monthly_total_perbrand['brand'] = b monthly_total_perbrand['monthly_total_no'] = 0 for i in data_entities: if b == i.brand: monthly_total_perbrand['monthly_total_no'] += i.selling_no # print monthly_total_perbrand ds.save_monthly_total_perbrand(b+month, monthly_total_perbrand) monthly_total_pertype = {} monthly_total_pertype['month'] = month for t in TREND_TOP_V_TYPE: monthly_total_pertype['car_type'] = t monthly_total_pertype['monthly_total_no'] = 0 for i in data_entities: if t == "all": monthly_total_pertype['monthly_total_no'] += i.selling_no elif t == i.car_type: monthly_total_pertype['monthly_total_no'] += i.selling_no # print monthly_total_pertype ds.save_monthly_total_pertype(t+month, monthly_total_pertype)