def extract_by_phone(phone_list=None):
print "====Begin extraction by phone."
columns_list = mysqldao.column_names(db_name, tb_yelp_phone)
biz_list = []
if phone_list == None:
phone_list = []
temp_phone_list = mysqldao.select(db_name, tb_insp_norm, ['PHONE'])
for p in temp_phone_list:
phone_list.append(p[0])
exist_phone_list = []
exist_phone_tuple = mysqldao.select_unique_column(db_name, tb_log_phone,
'phone')
count = 0
for phone in phone_list:
biz_data = {}
biz_list = []
phone = str(phone).replace(' ', '').replace('_', '')
if len(phone) == 11:
phone = phone[1:]
if phone not in exist_phone_tuple:
url_params = {"phone": phone, 'ywsid': 'bxtstnNlHgO8c6W4X2yuYA'}
biz_data = yelp_api.request(API_HOST,
'/phone_search',
url_params=url_params)[BUSINESS]
if len(biz_data) != 0:
data_phone = dic_look_up(biz_data[0], 'phone')
if data_phone == phone:
biz_list.append(biz_data[0])
print "Phone:", phone, "count", len(biz_list), 'data'
tran_list = json_transform_phone(biz_list, columns_list)
load_data_db(db_name, tb_yelp_phone, tran_list)
extracttime = str(datetime.datetime.now())[0:19]
log_para_list = [{'extracttime': extracttime, 'phone': phone}]
mysqldao.insert(db_name, tb_log_phone, log_para_list)
def extract_by_phone(phone_list=None):
print "====Begin extraction by phone."
columns_list = mysqldao.column_names(db_name, tb_yelp_phone)
biz_list=[]
if phone_list==None:
phone_list=[]
temp_phone_list=mysqldao.select(db_name, tb_insp_norm, ['PHONE'])
for p in temp_phone_list:
phone_list.append(p[0])
exist_phone_list=[]
exist_phone_tuple = mysqldao.select_unique_column(db_name,tb_log_phone,'phone')
count = 0
for phone in phone_list:
biz_data={}
biz_list=[]
phone=str(phone).replace(' ','').replace('_','')
if len(phone) ==11:
phone = phone[1:]
if phone not in exist_phone_tuple:
url_params={
"phone":phone,
'ywsid':'bxtstnNlHgO8c6W4X2yuYA'
}
biz_data = yelp_api.request(API_HOST, '/phone_search', url_params=url_params)[BUSINESS]
if len(biz_data) != 0:
data_phone=dic_look_up(biz_data[0],'phone')
if data_phone == phone:
biz_list.append(biz_data[0])
print "Phone:",phone,"count",len(biz_list),'data'
tran_list=json_transform_phone(biz_list,columns_list)
load_data_db(db_name, tb_yelp_phone, tran_list)
extracttime = str(datetime.datetime.now())[0:19]
log_para_list = [{'extracttime':extracttime, 'phone':phone}]
mysqldao.insert(db_name,tb_log_phone,log_para_list)
def extract_by_zipcode(url_params=url_params):
print "====Begin extraction by zipcode."
zip_tuple_all=mysqldao.select(db_name,tb_zipcode,['zipcode'])
exist_zipcode_tuple = mysqldao.select_unique_column(db_name,tb_log_zipcode,'zipcode')
for code in zip_tuple_all:
zipcode = str(code[0])
url_params_copy=url_params
if zipcode not in exist_zipcode_tuple:
url_params_copy['location']="New York, "+str(zipcode)+", NY"
url_params_copy['zipcode']=str(zipcode)
extract_full(url_params_copy)
else:
print zipcode, "existed."
def extract_by_zipcode(url_params=url_params):
print "====Begin extraction by zipcode."
zip_tuple_all = mysqldao.select(db_name, tb_zipcode, ['zipcode'])
exist_zipcode_tuple = mysqldao.select_unique_column(
db_name, tb_log_zipcode, 'zipcode')
for code in zip_tuple_all:
zipcode = str(code[0])
url_params_copy = url_params
if zipcode not in exist_zipcode_tuple:
url_params_copy['location'] = "New York, " + str(zipcode) + ", NY"
url_params_copy['zipcode'] = str(zipcode)
extract_full(url_params_copy)
else:
print zipcode, "existed."
def load_data_db(dbname, tbname, data_entry_list):
id_key = 'id'
insert_list = []
update_list = []
exist_id_tuple = mysqldao.select_unique_column(dbname, tbname, id_key)
for data in data_entry_list:
if data[id_key] in exist_id_tuple:
update_list.append(data)
else:
insert_list.append(data)
#print 'Update exist '+str(len(update_list))+" data."
#mysqldao.update(dbname, tbname, '', ['id'], update_list)
#print 'Insert new '+str(len(insert_list))+" data."
mysqldao.insert(dbname, tbname, insert_list)
return len(insert_list)
def load_data_db(dbname, tbname, data_entry_list):
id_key='id'
insert_list=[]
update_list=[]
exist_id_tuple=mysqldao.select_unique_column(dbname, tbname, id_key)
for data in data_entry_list:
if data[id_key] in exist_id_tuple:
update_list.append(data)
else:
insert_list.append(data)
#print 'Update exist '+str(len(update_list))+" data."
#mysqldao.update(dbname, tbname, '', ['id'], update_list)
#print 'Insert new '+str(len(insert_list))+" data."
mysqldao.insert(dbname,tbname,insert_list)
return len(insert_list)
