def save_gaode_phone_data(data_df):
'''
存储高德手机号数据
@return:
'''
if data_df.empty:
print('plz check data')
return False
table_name = conf.gaodemap_baidu_data_table
# 删除旧数据
data_df = data_df.where(data_df.notnull(), None)
data_df['cal_time'] = datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')
all_data = np.array(data_df).tolist()
sql = dbhandler.con_insert_sql(data_df, table_name)
in_bo = dbhandler.inser_many_date(sql, table_name, all_data)
return in_bo
def save_dzdp_shoplist(data_df, select_method):
'''
存储大众点评店铺列表数据
@return:
'''
in_bo = False
if data_df.empty:
print('plz check data')
return False
table_name = conf.dzdp_shop_table
# 删除旧数据
# if 1 == select_method:
# region = data_df['region'].tolist()[0]
# if 1 < len(data_df):
# del_sql = '''delete from {}
# where url in {} and region='{}' '''.format(table_name,
# tuple(data_df['url'].tolist()),
# region)
# else:
# del_sql = '''delete from {}
# where url = '{}' and region='{}' '''.format(table_name,
# data_df['url'].tolist()[0],
# region)
# elif 0 == select_method:
# small_type = data_df['small_type'].tolist()[0]
# if 1 < len(data_df):
# del_sql = '''delete from {}
# where url in {} and small_type='{}' '''.format(table_name,
# tuple(data_df['url'].tolist()),
# small_type)
# else:
# del_sql = '''delete from {}
# where url = '{}' and small_type='{}' '''.format(table_name,
# tuple(data_df['url'].tolist()),
# small_type)
# else:
# pass
shop_id_list = data_df['shop_id'].tolist()
del_sql = '''delete from {} where shop_id in {} '''.format(table_name, tuple(shop_id_list))
del_bo = dbhandler.exec_sql(del_sql, table_name)
if del_bo:
data_df = data_df.where(data_df.notnull(), None)
data_df['cal_time'] = datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')
all_data = np.array(data_df).tolist()
sql = dbhandler.con_insert_sql(data_df, table_name)
in_bo = dbhandler.inser_many_date(sql, table_name, all_data)
return in_bo
def save_douban_book(data_df):
'''
数据存储
@param save_df:
@return:
'''
if data_df.empty:
print('plz check data')
return False
table_name = conf.douban_book_table
data_df = data_df.where(data_df.notnull(), None)
data_df['cal_time'] = datetime.strftime(datetime.now(),
'%Y-%m-%d %H:%M:%S')
all_data = np.array(data_df).tolist()
sql = dbhandler.con_insert_sql(data_df, table_name)
in_bo = dbhandler.inser_many_date(sql, table_name, all_data)
return in_bo
def save_maoyan_movie(data_df):
'''
存储猫眼电影数据 名称 url 评分
@param data_df: 要存储的数据
@return:
'''
if data_df.empty:
print('plz check data')
return False
table_name = conf.maoyan_movie_table
del_sql = '''delete from {} where url in {}'''.format(
table_name, tuple(data_df['url'].tolist()))
del_bo = dbhandler.exec_sql(del_sql, table_name)
if del_bo:
data_df = data_df.where(data_df.notnull(), None)
data_df['cal_time'] = datetime.strftime(datetime.now(),
'%Y-%m-%d %H:%M:%S')
all_data = np.array(data_df).tolist()
sql = dbhandler.con_insert_sql(data_df, table_name)
in_bo = dbhandler.inser_many_date(sql, table_name, all_data)
return in_bo