def old_tmall_goods_insert_into_new_table(self, data, pipeline):
"""
老库数据规范，然后存入
:param data:
:param pipeline:
:return:
"""
site_id = from_tmall_type_get_site_id(type=data.get('type'))
if site_id is False:
self.lg.error('获取到的site_id为False!出错!请检查!出错goods_id: {0}'.format(
data.get('goods_id')))
return None
tmp = _get_right_model_data(data=data, site_id=site_id, logger=self.lg)
params = self._get_db_insert_params(item=tmp)
if tmp.get('main_goods_id') is not None:
sql_str = tm_insert_str_1
else:
sql_str = tm_insert_str_2
result = pipeline._insert_into_table_2(sql_str=sql_str,
params=params,
logger=self.lg)
return result
def old_taobao_goods_insert_into_new_table(self, data, pipeline):
'''
得到规范格式的data并且存入数据库
:param data:
:param pipeline:
:return:
'''
goods_id = data.get('goods_id', '')
try:
tmp = _get_right_model_data(data=data, site_id=1, logger=self.lg)
except:
self.lg.error('遇到错误, 先跳过处理!出错goods_id={0}'.format(goods_id),
exc_info=True)
return
params = self._get_db_insert_params(item=tmp)
if tmp.get('main_goods_id') is not None:
# main_goods_id不为空
sql_str = tb_insert_str_1
else:
# main_goods_id为空
sql_str = tb_insert_str_2
result = pipeline._insert_into_table_2(sql_str=sql_str,
params=params,
logger=self.lg)
return result
def old_jd_goods_insert_into_new_table(self, data, pipeline):
"""
老数据转到新表
:param data:
:param pipeline:
:return:
"""
site_id = from_jd_type_get_site_id(jd_type=data.get('jd_type'))
if site_id is False:
self.lg.error('site_id获取异常, 请检查!')
return False
tmp = _get_right_model_data(data=data, site_id=site_id)
self.lg.info('------>>>| 待存储的数据信息为: {0}'.format(tmp.get('goods_id')))
params = self._get_db_insert_params(item=tmp)
if tmp.get('main_goods_id') is not None:
sql_str = jd_insert_str_1
else:
sql_str = jd_insert_str_2
result = pipeline._insert_into_table_2(sql_str=sql_str,
params=params,
logger=self.lg)
return result
async def update_jumeiyoupin_pintuan_table_2(self, data, pipeline,
logger) -> bool:
'''
异步更新数据
:param data:
:param pipeline:
:param logger:
:return:
'''
try:
tmp = _get_right_model_data(data=data, site_id=27, logger=self.lg)
except:
self.lg.error('此处抓到的可能是聚美优品拼团券所以跳过')
return False
# print('------>>> | 待存储的数据信息为: |', tmp)
logger.info('------>>>| 待存储的数据信息为: ' + str(tmp.get('goods_id')))
params = await self._get_db_update_pintuan_params_2(item=tmp)
# pprint(params)
try:
result = pipeline._update_table_2(sql_str=jm_update_str_3,
params=params,
logger=logger)
return result
except Exception as e:
logger.exception(e)
return False
async def update_taobao_tiantiantejia_table(self, data, pipeline):
'''
更新天天秒杀特价的商品信息
:param data:
:param pipeline:
:param logger
:return:
'''
try:
data['miaosha_begin_time'] = data.get('tejia_begin_time')
data['miaosha_end_time'] = data.get('tejia_end_time')
tmp = _get_right_model_data(data=data, site_id=19)
except Exception:
self.lg.error('遇到错误, 先跳过处理!出错goods_id={0}'.format(
data['goods_id']),
exc_info=True)
return False
self.lg.info('------>>>| 待存储的数据信息为: |' + tmp.get('goods_id'))
params = self._get_db_update_tejia_params(item=tmp)
await pipeline._update_table_3(sql_str=tb_update_str_2,
params=params,
logger=self.lg,
error_msg_dict={
'other_error': [{
'field_name':
'goods_id',
'field_value':
tmp.get('goods_id', ''),
}]
})
def to_update_pinduoduo_xianshimiaosha_table(self, data, pipeline):
tmp = _get_right_model_data(data=data, site_id=16)
# print('------>>> | 待存储的数据信息为: |', tmp)
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_update_miaosha_params(item=tmp)
pipeline._update_table(sql_str=pd_update_str_2, params=params)
async def insert_into_jumeiyoupin_pintuan_table(self, data, pipeline,
logger) -> bool:
'''
存储数据
:param data:
:param pipeline:
:param logger
:return:
'''
try:
tmp = _get_right_model_data(data=data, site_id=27,
logger=self.lg) # 采集来源地(聚美优品拼团商品)
except:
self.lg.error('此处抓到的可能是聚美优品拼团券所以跳过')
return False
self.msg = '------>>>| 待存储的数据信息为: |' + str(tmp.get('goods_id'))
logger.info(self.msg)
params = await self._get_db_insert_pintuan_params(item=tmp)
try:
pipeline._insert_into_table_2(sql_str=jm_insert_str_2,
params=params,
logger=logger)
return True
except Exception as e:
logger.exception(e)
return False
def insert_into_juanpi_xianshimiaosha_table(self, data, pipeline) -> bool:
tmp = _get_right_model_data(data=data, site_id=15)
print('------>>> | 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_insert_miaosha_params(item=tmp)
res = pipeline._insert_into_table(sql_str=jp_insert_str_1, params=params)
return res
def to_update_juanpi_xianshimiaosha_table(self, data, pipeline):
tmp = _get_right_model_data(data=data, site_id=15)
# print('------>>> | 待存储的数据信息为: |', tmp)
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_update_miaosha_params(item=tmp)
res = pipeline._update_table(sql_str=jp_update_str_3, params=params)
return res
def to_right_and_update_pintuan_data(self, data, pipeline):
try:
tmp = _get_right_model_data(data=data, site_id=18)
except:
print('此处抓到的可能是卷皮拼团券所以跳过')
return None
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_update_pintuan_params(item=tmp)
pipeline._update_table(sql_str=jp_update_str_4, params=params)
def to_right_and_update_data(self, data, pipeline):
try:
tmp = _get_right_model_data(data=data, site_id=17)
except:
print('此处抓到的可能是折800拼团券所以跳过')
return None
print('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id')))
params = self._get_db_update_pintuan_params(item=tmp)
pipeline._update_table(sql_str=z8_update_str_3, params=params)
def update_mogujie_xianshimiaosha_table(self, data, pipeline):
try:
tmp = _get_right_model_data(data=data, site_id=22)
except:
print('此处抓到的可能是蘑菇街券所以跳过')
return None
# print('------>>> | 待存储的数据信息为: |', tmp)
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_update_miaosha_params(item=tmp)
pipeline._update_table(sql_str=mg_update_str_2, params=params)
def update_mia_pintuan_table(self, data, pipeline):
try:
tmp = _get_right_model_data(data=data, site_id=21)
except:
print('此处抓到的可能是蜜芽拼团券所以跳过')
return None
# print('------>>> | 待存储的数据信息为: |', tmp)
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_update_pintuan_params(item=tmp)
pipeline._update_table(sql_str=mia_update_str_3, params=params)
def to_right_and_update_data(self, data, pipeline):
tmp = _get_right_model_data(data=data, site_id=12)
params = get_db_commom_goods_update_params(item=tmp)
base_sql_str = jp_update_str_2
if tmp['delete_time'] == '':
sql_str = base_sql_str.format('shelf_time=%s', '')
elif tmp['shelf_time'] == '':
sql_str = base_sql_str.format('delete_time=%s', '')
else:
sql_str = base_sql_str.format('shelf_time=%s,', 'delete_time=%s')
pipeline._update_table(sql_str=sql_str, params=params)
def insert_into_zhe_800_pintuan_table(self, data, pipeline) -> bool:
try:
tmp = _get_right_model_data(data=data, site_id=17) # 采集来源地(折800拼团商品)
except:
print('此处抓到的可能是折800拼团券所以跳过')
return False
print('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id')))
params = self._get_db_insert_pintuan_params(item=tmp)
res = pipeline._insert_into_table(sql_str=z8_insert_str_2, params=params)
return res
def insert_into_mogujie_xianshimiaosha_table(self, data, pipeline) -> bool:
try:
tmp = _get_right_model_data(data=data, site_id=22) # 采集来源地(蘑菇街秒杀商品)
except:
print('此处抓到的可能是蘑菇街券所以跳过')
return False
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_insert_miaosha_params(item=tmp)
res = pipeline._insert_into_table(sql_str=mg_insert_str_1, params=params)
return res
def insert_into_jd_table(self, data, pipeline):
site_id = from_jd_type_get_site_id(jd_type=data.get('jd_type'))
if site_id is False:
self.lg.error('site_id获取异常, 请检查!')
return False
tmp = _get_right_model_data(data=data, site_id=site_id)
self.lg.info('------>>>| 待存储的数据信息为:{0}'.format(tmp.get('goods_id')))
pipeline.insert_into_jd_table(item=tmp)
return True
def insert_into_mia_pintuan_table(self, data, pipeline) -> bool:
try:
tmp = _get_right_model_data(data=data, site_id=21) # 采集来源地(蜜芽拼团商品)
except:
print('此处抓到的可能是蜜芽拼团券所以跳过')
return False
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_insert_pintuan_params(item=tmp)
_r = pipeline._insert_into_table(sql_str=mia_insert_str_2, params=params)
return _r
def to_update_zhe_800_xianshimiaosha_table(self, data, pipeline) -> bool:
try:
tmp = _get_right_model_data(data=data, site_id=14)
except:
print('此处抓到的可能是折800秒杀券所以跳过')
return False
tmp = self._set_detail_price_to_miaosha_price(tmp=tmp)
print('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id')))
params = self._get_db_update_miaosha_params(item=tmp)
res = pipeline._update_table(sql_str=z8_update_str_2, params=params)
return res
def insert_into_zhe_800_xianshimiaosha_table(self, data, pipeline):
try:
tmp = _get_right_model_data(data=data, site_id=14) # 采集来源地(折800秒杀商品)
except:
print('此处抓到的可能是折800秒杀券所以跳过')
return False
tmp = self._set_detail_price_to_miaosha_price(tmp=tmp)
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_insert_miaosha_params(item=tmp)
res = pipeline._insert_into_table(sql_str=z8_insert_str_1, params=params)
return res
def update_jumeiyoupin_xianshimiaosha_table(self, data, pipeline) -> bool:
try:
tmp = _get_right_model_data(data=data, site_id=26)
except:
print('此处抓到的可能是聚美优品券所以跳过')
return False
tmp = self._set_detail_price_to_miaosha_price(tmp=tmp)
print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id'))
params = self._get_db_update_miaosha_params(item=tmp)
res = pipeline._update_table(sql_str=jm_update_str_1, params=params)
return res
def old_ali_1688_goods_insert_into_new_table(self, data, pipeline):
tmp = _get_right_model_data(data=data, site_id=2)
params = self._get_db_insert_params(item=tmp)
if tmp.get('main_goods_id') is not None:
sql_str = al_insert_str_1
else:
sql_str = al_insert_str_2
result = pipeline._insert_into_table_2(sql_str=sql_str,
params=params,
logger=self.lg)
return result
def update_chuchujie_xianshimiaosha_table(self, data, pipeline) -> bool:
try:
tmp = _get_right_model_data(data=data, site_id=24)
except:
print('此处抓到的可能是楚楚街券所以跳过')
return False
# print('------>>> | 待存储的数据信息为: |', tmp)
print('------>>>| 待存储的数据信息为: ', tmp.get('goods_id'))
params = self._get_db_update_miaosha_params(item=tmp)
sql_str = cc_update_str_1
res = pipeline._update_table(sql_str=sql_str, params=params)
return res
def to_right_and_update_data(self, data, pipeline):
tmp = _get_right_model_data(data=data, site_id=2)
params = self._get_db_update_params(item=tmp)
base_sql_str = al_update_str_2
if tmp['delete_time'] == '':
sql_str = base_sql_str.format('shelf_time=%s', '')
elif tmp['shelf_time'] == '':
sql_str = base_sql_str.format('delete_time=%s', '')
else:
sql_str = base_sql_str.format('shelf_time=%s,', 'delete_time=%s')
res = pipeline._update_table_2(sql_str=sql_str,
params=params,
logger=self.lg)
return res
def to_right_and_update_data(self, data, pipeline):
'''
更新商品数据
:param data:
:param pipeline:
:return:
'''
tmp = _get_right_model_data(data=data, site_id=25)
params = get_db_commom_goods_update_params(item=tmp)
base_sql_str = vip_update_str_1
if tmp['delete_time'] == '':
sql_str = base_sql_str.format('shelf_time=%s', '')
elif tmp['shelf_time'] == '':
sql_str = base_sql_str.format('delete_time=%s', '')
else:
sql_str = base_sql_str.format('shelf_time=%s,', 'delete_time=%s')
pipeline._update_table(sql_str=sql_str, params=params)
def _to_right_and_update_data(self, data, pipeline):
'''
实时更新数据
:param data:
:param pipeline:
:return:
'''
tmp = _get_right_model_data(data, site_id=31, logger=self.lg)
params = get_db_commom_goods_update_params(item=tmp)
base_sql_str = yp_update_str_1
if tmp['delete_time'] == '':
sql_str = base_sql_str.format('shelf_time=%s', '')
elif tmp['shelf_time'] == '':
sql_str = base_sql_str.format('delete_time=%s', '')
else:
sql_str = base_sql_str.format('shelf_time=%s,', 'delete_time=%s')
result = pipeline._update_table_2(sql_str=sql_str, params=params, logger=self.lg)
return result
async def update_expired_goods_id_taobao_tiantiantejia_table(
self, data, pipeline):
'''
更新过期商品的信息，使其转为普通常规商品
:param data:
:param pipeline:
:return:
'''
try:
data['miaosha_begin_time'] = data.get('tejia_begin_time')
data['miaosha_end_time'] = data.get('tejia_end_time')
tmp = _get_right_model_data(data=data, site_id=19)
except Exception:
self.lg.error('遇到错误, 先跳过处理!出错goods_id={0}'.format(
data['goods_id']),
exc_info=True)
return False
self.lg.info('------>>>| 待存储的数据信息为: |' + tmp.get('goods_id'))
await pipeline.update_expired_goods_id_taobao_tiantiantejia_table(
item=tmp, logger=self.lg)
async def _update_taoqianggou_xianshimiaosha_table(self, data, pipeline):
"""
update对应表的数据
:param data:
:param pipeline:
:return:
"""
try:
tmp = _get_right_model_data(data=data, site_id=28, logger=self.lg)
except:
self.lg.error('获取规范化数据失败!出错goods_id:{0}'.format(
data.get('goods_id')))
return None
self.lg.info('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id')))
params = await self._get_db_update_miaosha_params(item=tmp)
pipeline._update_table_2(sql_str=tm_update_str_2,
params=params,
logger=self.lg)
return
def insert_into_taoqianggou_xianshimiaosha_table(self, data,
pipeline) -> bool:
"""
将数据规范化插入淘抢购表
:param data:
:param pipeline:
:return:
"""
try:
tmp = _get_right_model_data(data=data, site_id=28,
logger=self.lg) # 采集来源地(淘抢购)
except:
print('此处抓到的可能是淘宝秒杀券所以跳过')
return False
self.lg.info('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id')))
params = self._get_db_insert_taoqianggou_miaosha_params(item=tmp)
res = pipeline._insert_into_table_2(sql_str=tm_insert_str_3,
params=params,
logger=self.lg)
return res
def _deal_with_tm_goods(goods_link, my_lg):
'''
处理天猫商品
:param goods_link:
:return: json_str
'''
my_lg.info('进入天猫商品处理接口...')
goods_id = _get_tm_goods_id(goods_link)
if goods_id == '':
msg = 'goods_id匹配失败!请检查url是否正确!'
return _error_data(msg=msg)
tm_url = 'https://detail.tmall.com/item.htm?id={0}'.format(goods_id)
data = get_one_tm_data(wait_to_deal_with_url=tm_url, my_lg=my_lg)
if data.get('msg', '') == 'data为空!':
msg = '该goods_id:{0}, 抓取数据失败!'.format(goods_id)
return _error_data(msg=msg)
else:
pass
site_id = _from_tmall_type_get_site_id(type=data.get('type'))
data = _get_right_model_data(data=data, site_id=site_id, logger=my_lg)
my_pipeline = SqlServerMyPageInfoSaveItemPipeline()
my_lg.info('------>>>| 正在存储的数据为: ' + data.get('goods_id', ''))
params = _get_db_tmall_insert_params(item=data)
sql_str = 'insert into dbo.GoodsInfoAutoGet(GoodsID, GoodsUrl, UserName, CreateTime, ModfiyTime, ShopName, Account, GoodsName, SubTitle, LinkName, Price, TaoBaoPrice, PriceInfo, SKUName, SKUInfo, ImageUrl, PropertyInfo, DetailInfo, SellCount, SiteID, IsDelete) values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'
is_insert_into = my_pipeline._insert_into_table_2(sql_str=sql_str,
params=params,
logger=my_lg)
if is_insert_into: # 如果返回值为True
pass
else: # 不处理存储结果
# msg = '存储该goods_id:{0}失败!'.format(goods_id)
# return _error_data(msg=msg)
pass
return compatible_api_goods_data(data=data, my_lg=my_lg)
