def get_one_kaola_data(**kwargs):
'''
抓取一个考拉 url的data
:param kwargs:
:return:
'''
username = kwargs.get('username', DEFAULT_USERNAME)
wait_to_deal_with_url = kwargs.get('wait_to_deal_with_url', '')
my_lg = kwargs.get('my_lg')
kaola = KaoLaParse(logger=my_lg)
goods_id = kaola.get_goods_id_from_url(
wait_to_deal_with_url) # 获取goods_id, 这里返回的是一个list
if goods_id == '': # 如果得不到goods_id, 则return error
my_lg.info('获取到的goods_id为空!')
try:
del kaola # 每次都回收一下
except Exception:
pass
gc.collect()
return {'goods_id': ''} # 错误1: goods_id为空值
tmp_result = kaola._get_goods_data(goods_id=goods_id)
data = kaola._deal_with_data() # 如果成功获取的话, 返回的是一个data的dict对象
if data == {} or tmp_result == {}:
my_lg.error('获取到的data为空!出错地址: {0}'.format(wait_to_deal_with_url))
try:
del kaola
except:
pass
gc.collect()
return {'goods_id': goods_id, 'msg': 'data为空!'} # 错误2: 抓取失败
wait_to_save_data = add_base_info_2_processed_data(
data=data,
spider_url=wait_to_deal_with_url,
username=username,
goods_id=goods_id)
try:
del kaola
except:
pass
return wait_to_save_data
def run_forever():
while True:
# ** 不能写成全局变量并放在循环中, 否则会一直记录到同一文件中
my_lg = set_logger(
log_file_name=MY_SPIDER_LOGS_PATH + '/网易考拉/实时更新/' + str(get_shanghai_time())[0:10] + '.txt',
console_log_level=INFO,
file_log_level=ERROR
)
#### 实时更新数据
tmp_sql_server = SqlServerMyPageInfoSaveItemPipeline()
# and GETDATE()-ModfiyTime>0.2
# and MainGoodsID is not null
sql_str = '''
select SiteID, GoodsID, IsDelete, Price, TaoBaoPrice, shelf_time, delete_time
from dbo.GoodsInfoAutoGet
where SiteID=29 and GETDATE()-ModfiyTime>0.3 and MainGoodsID is not null
order by ID asc'''
try:
result = list(tmp_sql_server._select_table(sql_str=sql_str))
except TypeError:
my_lg.error('TypeError错误, 原因数据库连接失败...(可能维护中)')
result = None
if result is None:
pass
else:
my_lg.info('------>>> 下面是数据库返回的所有符合条件的goods_id <<<------')
my_lg.info(str(result))
my_lg.info('--------------------------------------------------------')
my_lg.info('总计待更新个数: {0}'.format(len(result)))
my_lg.info('即将开始实时更新数据, 请耐心等待...'.center(100, '#'))
index = 1
# 释放内存,在外面声明就会占用很大的，所以此处优化内存的方法是声明后再删除释放
kaola = KaoLaParse(logger=my_lg)
for item in result: # 实时更新数据
if index % 5 == 0:
try:
del kaola
except:
pass
kaola = KaoLaParse(logger=my_lg)
gc.collect()
if index % 10 == 0: # 每10次重连一次，避免单次长连无响应报错
my_lg.info('正在重置，并与数据库建立新连接中...')
tmp_sql_server = SqlServerMyPageInfoSaveItemPipeline()
my_lg.info('与数据库的新连接成功建立...')
if tmp_sql_server.is_connect_success:
my_lg.info('------>>>| 正在更新的goods_id为(%s) | --------->>>@ 索引值为(%s)' % (str(item[1]), str(index)))
data = kaola._get_goods_data(goods_id=item[1])
if data.get('is_delete') == 1: # 单独处理下架商品
my_lg.info('@@@ 该商品已下架...')
data['goods_id'] = item[1]
data['shelf_time'], data['delete_time'] = get_shelf_time_and_delete_time(
tmp_data=data,
is_delete=item[2],
shelf_time=item[5],
delete_time=item[6])
# my_lg.info('------>>>| 爬取到的数据为: %s' % str(data))
kaola.to_right_and_update_data(data, pipeline=tmp_sql_server)
sleep(TMALL_REAL_TIMES_SLEEP_TIME)
index += 1
gc.collect()
continue
data = kaola._deal_with_data()
if data != {}:
data['goods_id'] = item[1]
data['shelf_time'], data['delete_time'] = get_shelf_time_and_delete_time(
tmp_data=data,
is_delete=item[2],
shelf_time=item[5],
delete_time=item[6])
data['_is_price_change'], data['_price_change_info'] = _get_price_change_info(
old_price=item[3],
old_taobao_price=item[4],
new_price=data['price'],
new_taobao_price=data['taobao_price']
)
# my_lg.info(str(data['_is_price_change']) + ' ' +str(data['_price_change_info']))
# my_lg.info('------>>>| 爬取到的数据为: %s' % str(data))
kaola.to_right_and_update_data(data, pipeline=tmp_sql_server)
else: # 表示返回的data值为空值
my_lg.info('------>>>| 休眠8s中...')
sleep(8)
else: # 表示返回的data值为空值
my_lg.error('数据库连接失败，数据库可能关闭或者维护中')
sleep(5)
pass
index += 1
gc.collect()
sleep(TMALL_REAL_TIMES_SLEEP_TIME)
my_lg.info('全部数据更新完毕'.center(100, '#')) # sleep(60*60)
if get_shanghai_time().hour == 0: # 0点以后不更新
sleep(60 * 60 * 5.5)
else:
sleep(60)
gc.collect()
def run_forever():
while True:
# ** 不能写成全局变量并放在循环中, 否则会一直记录到同一文件中
my_lg = set_logger(log_file_name=MY_SPIDER_LOGS_PATH + '/网易考拉/实时更新/' +
str(get_shanghai_time())[0:10] + '.txt',
console_log_level=INFO,
file_log_level=ERROR)
#### 实时更新数据
sql_cli = SqlServerMyPageInfoSaveItemPipeline()
try:
result = list(sql_cli._select_table(sql_str=kl_select_str_1))
except TypeError:
my_lg.error('TypeError错误, 原因数据库连接失败...(可能维护中)')
result = None
if result is None:
pass
else:
_block_print_db_old_data(result=result, logger=my_lg)
index = 1
# 释放内存,在外面声明就会占用很大的，所以此处优化内存的方法是声明后再删除释放
kaola = KaoLaParse(logger=my_lg, is_real_times_update_call=True)
for item in result: # 实时更新数据
goods_id = item[1]
if index % 5 == 0:
try:
del kaola
except:
pass
kaola = KaoLaParse(logger=my_lg,
is_real_times_update_call=True)
collect()
sql_cli = _block_get_new_db_conn(
db_obj=sql_cli,
index=index,
logger=my_lg,
remainder=10,
)
if sql_cli.is_connect_success:
my_lg.info(
'------>>>| 正在更新的goods_id为(%s) | --------->>>@ 索引值为(%s)'
% (str(goods_id), str(index)))
db_goods_info_obj = KLDbGoodsInfoObj(item=item,
logger=my_lg)
data = kaola._get_goods_data(goods_id=goods_id)
if data.get('is_delete', 0) == 1:
# 单独处理下架商品
data['goods_id'] = goods_id
data['shelf_time'], data[
'delete_time'] = get_shelf_time_and_delete_time(
tmp_data=data,
is_delete=db_goods_info_obj.is_delete,
shelf_time=db_goods_info_obj.shelf_time,
delete_time=db_goods_info_obj.delete_time,
)
try:
kaola.to_right_and_update_data(data,
pipeline=sql_cli)
except Exception:
my_lg.error(exc_info=True)
sleep(TMALL_REAL_TIMES_SLEEP_TIME)
index += 1
collect()
continue
data = kaola._deal_with_data()
if data != {}:
if data.get('is_delete', 0) == 1:
_handle_goods_shelves_in_auto_goods_table(
goods_id=goods_id,
logger=my_lg,
sql_cli=sql_cli,
)
sleep(TMALL_REAL_TIMES_SLEEP_TIME)
continue
else:
data = get_goods_info_change_data(
target_short_name='kl',
logger=my_lg,
data=data,
db_goods_info_obj=db_goods_info_obj,
)
kaola.to_right_and_update_data(data, pipeline=sql_cli)
else: # 表示返回的data值为空值
my_lg.info('------>>>| 休眠3s中...')
sleep(3.)
else: # 表示返回的data值为空值
my_lg.error('数据库连接失败，数据库可能关闭或者维护中')
sleep(5)
pass
index += 1
collect()
sleep(TMALL_REAL_TIMES_SLEEP_TIME)
my_lg.info('全部数据更新完毕'.center(100, '#')) # sleep(60*60)
if get_shanghai_time().hour == 0:
# 0点以后不更新
sleep(60 * 60 * 5.5)
else:
sleep(60)
collect()
def run_forever():
while True:
# ** 不能写成全局变量并放在循环中, 否则会一直记录到同一文件中
my_lg = set_logger(
log_file_name=MY_SPIDER_LOGS_PATH + '/网易考拉/实时更新/' + str(get_shanghai_time())[0:10] + '.txt',
console_log_level=INFO,
file_log_level=ERROR
)
#### 实时更新数据
tmp_sql_server = SqlServerMyPageInfoSaveItemPipeline()
try:
result = list(tmp_sql_server._select_table(sql_str=kl_select_str_1))
except TypeError:
my_lg.error('TypeError错误, 原因数据库连接失败...(可能维护中)')
result = None
if result is None:
pass
else:
my_lg.info('------>>> 下面是数据库返回的所有符合条件的goods_id <<<------')
my_lg.info(str(result))
my_lg.info('--------------------------------------------------------')
my_lg.info('总计待更新个数: {0}'.format(len(result)))
my_lg.info('即将开始实时更新数据, 请耐心等待...'.center(100, '#'))
index = 1
# 释放内存,在外面声明就会占用很大的，所以此处优化内存的方法是声明后再删除释放
kaola = KaoLaParse(logger=my_lg)
for item in result: # 实时更新数据
if index % 5 == 0:
try:
del kaola
except:
pass
kaola = KaoLaParse(logger=my_lg)
gc.collect()
if index % 10 == 0: # 每10次重连一次，避免单次长连无响应报错
my_lg.info('正在重置，并与数据库建立新连接中...')
tmp_sql_server = SqlServerMyPageInfoSaveItemPipeline()
my_lg.info('与数据库的新连接成功建立...')
if tmp_sql_server.is_connect_success:
my_lg.info('------>>>| 正在更新的goods_id为(%s) | --------->>>@ 索引值为(%s)' % (str(item[1]), str(index)))
data = kaola._get_goods_data(goods_id=item[1])
if data.get('is_delete') == 1: # 单独处理下架商品
data['goods_id'] = item[1]
data['shelf_time'], data['delete_time'] = get_shelf_time_and_delete_time(
tmp_data=data,
is_delete=item[2],
shelf_time=item[5],
delete_time=item[6])
# my_lg.info('------>>>| 爬取到的数据为: %s' % str(data))
kaola.to_right_and_update_data(data, pipeline=tmp_sql_server)
sleep(TMALL_REAL_TIMES_SLEEP_TIME)
index += 1
gc.collect()
continue
data = kaola._deal_with_data()
if data != {}:
data['goods_id'] = item[1]
data['shelf_time'], data['delete_time'] = get_shelf_time_and_delete_time(
tmp_data=data,
is_delete=item[2],
shelf_time=item[5],
delete_time=item[6])
if data.get('is_delete') == 1:
my_lg.info('@@@ 该商品已下架...')
tmp_sql_server._update_table_2(sql_str=kl_update_str_2, params=(item[1],), logger=my_lg)
sleep(TMALL_REAL_TIMES_SLEEP_TIME)
continue
data['_is_price_change'], data['_price_change_info'] = _get_price_change_info(
old_price=item[3],
old_taobao_price=item[4],
new_price=data['price'],
new_taobao_price=data['taobao_price']
)
try:
old_sku_info = format_price_info_list(price_info_list=json_2_dict(item[7]), site_id=29)
except AttributeError: # 处理已被格式化过的
old_sku_info = item[7]
data['_is_price_change'], data['sku_info_trans_time'] = get_sku_info_trans_record(
old_sku_info=old_sku_info,
new_sku_info=format_price_info_list(data['price_info_list'], site_id=29),
is_price_change=item[8] if item[8] is not None else 0
)
kaola.to_right_and_update_data(data, pipeline=tmp_sql_server)
else: # 表示返回的data值为空值
my_lg.info('------>>>| 休眠8s中...')
sleep(8)
else: # 表示返回的data值为空值
my_lg.error('数据库连接失败，数据库可能关闭或者维护中')
sleep(5)
pass
index += 1
gc.collect()
sleep(TMALL_REAL_TIMES_SLEEP_TIME)
my_lg.info('全部数据更新完毕'.center(100, '#')) # sleep(60*60)
if get_shanghai_time().hour == 0: # 0点以后不更新
sleep(60 * 60 * 5.5)
else:
sleep(60)
gc.collect()