def extract_by_phone(phone_list=None): print "====Begin extraction by phone." columns_list = mysqldao.column_names(db_name, tb_yelp_phone) biz_list = [] if phone_list == None: phone_list = [] temp_phone_list = mysqldao.select(db_name, tb_insp_norm, ['PHONE']) for p in temp_phone_list: phone_list.append(p[0]) exist_phone_list = [] exist_phone_tuple = mysqldao.select_unique_column(db_name, tb_log_phone, 'phone') count = 0 for phone in phone_list: biz_data = {} biz_list = [] phone = str(phone).replace(' ', '').replace('_', '') if len(phone) == 11: phone = phone[1:] if phone not in exist_phone_tuple: url_params = {"phone": phone, 'ywsid': 'bxtstnNlHgO8c6W4X2yuYA'} biz_data = yelp_api.request(API_HOST, '/phone_search', url_params=url_params)[BUSINESS] if len(biz_data) != 0: data_phone = dic_look_up(biz_data[0], 'phone') if data_phone == phone: biz_list.append(biz_data[0]) print "Phone:", phone, "count", len(biz_list), 'data' tran_list = json_transform_phone(biz_list, columns_list) load_data_db(db_name, tb_yelp_phone, tran_list) extracttime = str(datetime.datetime.now())[0:19] log_para_list = [{'extracttime': extracttime, 'phone': phone}] mysqldao.insert(db_name, tb_log_phone, log_para_list)
def extract_by_phone(phone_list=None): print "====Begin extraction by phone." columns_list = mysqldao.column_names(db_name, tb_yelp_phone) biz_list=[] if phone_list==None: phone_list=[] temp_phone_list=mysqldao.select(db_name, tb_insp_norm, ['PHONE']) for p in temp_phone_list: phone_list.append(p[0]) exist_phone_list=[] exist_phone_tuple = mysqldao.select_unique_column(db_name,tb_log_phone,'phone') count = 0 for phone in phone_list: biz_data={} biz_list=[] phone=str(phone).replace(' ','').replace('_','') if len(phone) ==11: phone = phone[1:] if phone not in exist_phone_tuple: url_params={ "phone":phone, 'ywsid':'bxtstnNlHgO8c6W4X2yuYA' } biz_data = yelp_api.request(API_HOST, '/phone_search', url_params=url_params)[BUSINESS] if len(biz_data) != 0: data_phone=dic_look_up(biz_data[0],'phone') if data_phone == phone: biz_list.append(biz_data[0]) print "Phone:",phone,"count",len(biz_list),'data' tran_list=json_transform_phone(biz_list,columns_list) load_data_db(db_name, tb_yelp_phone, tran_list) extracttime = str(datetime.datetime.now())[0:19] log_para_list = [{'extracttime':extracttime, 'phone':phone}] mysqldao.insert(db_name,tb_log_phone,log_para_list)
def extract_by_zipcode(url_params=url_params): print "====Begin extraction by zipcode." zip_tuple_all=mysqldao.select(db_name,tb_zipcode,['zipcode']) exist_zipcode_tuple = mysqldao.select_unique_column(db_name,tb_log_zipcode,'zipcode') for code in zip_tuple_all: zipcode = str(code[0]) url_params_copy=url_params if zipcode not in exist_zipcode_tuple: url_params_copy['location']="New York, "+str(zipcode)+", NY" url_params_copy['zipcode']=str(zipcode) extract_full(url_params_copy) else: print zipcode, "existed."
def extract_by_zipcode(url_params=url_params): print "====Begin extraction by zipcode." zip_tuple_all = mysqldao.select(db_name, tb_zipcode, ['zipcode']) exist_zipcode_tuple = mysqldao.select_unique_column( db_name, tb_log_zipcode, 'zipcode') for code in zip_tuple_all: zipcode = str(code[0]) url_params_copy = url_params if zipcode not in exist_zipcode_tuple: url_params_copy['location'] = "New York, " + str(zipcode) + ", NY" url_params_copy['zipcode'] = str(zipcode) extract_full(url_params_copy) else: print zipcode, "existed."
def load_data_db(dbname, tbname, data_entry_list): id_key = 'id' insert_list = [] update_list = [] exist_id_tuple = mysqldao.select_unique_column(dbname, tbname, id_key) for data in data_entry_list: if data[id_key] in exist_id_tuple: update_list.append(data) else: insert_list.append(data) #print 'Update exist '+str(len(update_list))+" data." #mysqldao.update(dbname, tbname, '', ['id'], update_list) #print 'Insert new '+str(len(insert_list))+" data." mysqldao.insert(dbname, tbname, insert_list) return len(insert_list)
def load_data_db(dbname, tbname, data_entry_list): id_key='id' insert_list=[] update_list=[] exist_id_tuple=mysqldao.select_unique_column(dbname, tbname, id_key) for data in data_entry_list: if data[id_key] in exist_id_tuple: update_list.append(data) else: insert_list.append(data) #print 'Update exist '+str(len(update_list))+" data." #mysqldao.update(dbname, tbname, '', ['id'], update_list) #print 'Insert new '+str(len(insert_list))+" data." mysqldao.insert(dbname,tbname,insert_list) return len(insert_list)