def old_tmall_goods_insert_into_new_table(self, data, pipeline): """ 老库数据规范,然后存入 :param data: :param pipeline: :return: """ site_id = from_tmall_type_get_site_id(type=data.get('type')) if site_id is False: self.lg.error('获取到的site_id为False!出错!请检查!出错goods_id: {0}'.format( data.get('goods_id'))) return None tmp = _get_right_model_data(data=data, site_id=site_id, logger=self.lg) params = self._get_db_insert_params(item=tmp) if tmp.get('main_goods_id') is not None: sql_str = tm_insert_str_1 else: sql_str = tm_insert_str_2 result = pipeline._insert_into_table_2(sql_str=sql_str, params=params, logger=self.lg) return result
def old_taobao_goods_insert_into_new_table(self, data, pipeline): ''' 得到规范格式的data并且存入数据库 :param data: :param pipeline: :return: ''' goods_id = data.get('goods_id', '') try: tmp = _get_right_model_data(data=data, site_id=1, logger=self.lg) except: self.lg.error('遇到错误, 先跳过处理!出错goods_id={0}'.format(goods_id), exc_info=True) return params = self._get_db_insert_params(item=tmp) if tmp.get('main_goods_id') is not None: # main_goods_id不为空 sql_str = tb_insert_str_1 else: # main_goods_id为空 sql_str = tb_insert_str_2 result = pipeline._insert_into_table_2(sql_str=sql_str, params=params, logger=self.lg) return result
def old_jd_goods_insert_into_new_table(self, data, pipeline): """ 老数据转到新表 :param data: :param pipeline: :return: """ site_id = from_jd_type_get_site_id(jd_type=data.get('jd_type')) if site_id is False: self.lg.error('site_id获取异常, 请检查!') return False tmp = _get_right_model_data(data=data, site_id=site_id) self.lg.info('------>>>| 待存储的数据信息为: {0}'.format(tmp.get('goods_id'))) params = self._get_db_insert_params(item=tmp) if tmp.get('main_goods_id') is not None: sql_str = jd_insert_str_1 else: sql_str = jd_insert_str_2 result = pipeline._insert_into_table_2(sql_str=sql_str, params=params, logger=self.lg) return result
async def update_jumeiyoupin_pintuan_table_2(self, data, pipeline, logger) -> bool: ''' 异步更新数据 :param data: :param pipeline: :param logger: :return: ''' try: tmp = _get_right_model_data(data=data, site_id=27, logger=self.lg) except: self.lg.error('此处抓到的可能是聚美优品拼团券所以跳过') return False # print('------>>> | 待存储的数据信息为: |', tmp) logger.info('------>>>| 待存储的数据信息为: ' + str(tmp.get('goods_id'))) params = await self._get_db_update_pintuan_params_2(item=tmp) # pprint(params) try: result = pipeline._update_table_2(sql_str=jm_update_str_3, params=params, logger=logger) return result except Exception as e: logger.exception(e) return False
async def update_taobao_tiantiantejia_table(self, data, pipeline): ''' 更新天天秒杀特价的商品信息 :param data: :param pipeline: :param logger :return: ''' try: data['miaosha_begin_time'] = data.get('tejia_begin_time') data['miaosha_end_time'] = data.get('tejia_end_time') tmp = _get_right_model_data(data=data, site_id=19) except Exception: self.lg.error('遇到错误, 先跳过处理!出错goods_id={0}'.format( data['goods_id']), exc_info=True) return False self.lg.info('------>>>| 待存储的数据信息为: |' + tmp.get('goods_id')) params = self._get_db_update_tejia_params(item=tmp) await pipeline._update_table_3(sql_str=tb_update_str_2, params=params, logger=self.lg, error_msg_dict={ 'other_error': [{ 'field_name': 'goods_id', 'field_value': tmp.get('goods_id', ''), }] })
def to_update_pinduoduo_xianshimiaosha_table(self, data, pipeline): tmp = _get_right_model_data(data=data, site_id=16) # print('------>>> | 待存储的数据信息为: |', tmp) print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_update_miaosha_params(item=tmp) pipeline._update_table(sql_str=pd_update_str_2, params=params)
async def insert_into_jumeiyoupin_pintuan_table(self, data, pipeline, logger) -> bool: ''' 存储数据 :param data: :param pipeline: :param logger :return: ''' try: tmp = _get_right_model_data(data=data, site_id=27, logger=self.lg) # 采集来源地(聚美优品拼团商品) except: self.lg.error('此处抓到的可能是聚美优品拼团券所以跳过') return False self.msg = '------>>>| 待存储的数据信息为: |' + str(tmp.get('goods_id')) logger.info(self.msg) params = await self._get_db_insert_pintuan_params(item=tmp) try: pipeline._insert_into_table_2(sql_str=jm_insert_str_2, params=params, logger=logger) return True except Exception as e: logger.exception(e) return False
def insert_into_juanpi_xianshimiaosha_table(self, data, pipeline) -> bool: tmp = _get_right_model_data(data=data, site_id=15) print('------>>> | 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_insert_miaosha_params(item=tmp) res = pipeline._insert_into_table(sql_str=jp_insert_str_1, params=params) return res
def to_update_juanpi_xianshimiaosha_table(self, data, pipeline): tmp = _get_right_model_data(data=data, site_id=15) # print('------>>> | 待存储的数据信息为: |', tmp) print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_update_miaosha_params(item=tmp) res = pipeline._update_table(sql_str=jp_update_str_3, params=params) return res
def to_right_and_update_pintuan_data(self, data, pipeline): try: tmp = _get_right_model_data(data=data, site_id=18) except: print('此处抓到的可能是卷皮拼团券所以跳过') return None print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_update_pintuan_params(item=tmp) pipeline._update_table(sql_str=jp_update_str_4, params=params)
def to_right_and_update_data(self, data, pipeline): try: tmp = _get_right_model_data(data=data, site_id=17) except: print('此处抓到的可能是折800拼团券所以跳过') return None print('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id'))) params = self._get_db_update_pintuan_params(item=tmp) pipeline._update_table(sql_str=z8_update_str_3, params=params)
def update_mogujie_xianshimiaosha_table(self, data, pipeline): try: tmp = _get_right_model_data(data=data, site_id=22) except: print('此处抓到的可能是蘑菇街券所以跳过') return None # print('------>>> | 待存储的数据信息为: |', tmp) print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_update_miaosha_params(item=tmp) pipeline._update_table(sql_str=mg_update_str_2, params=params)
def update_mia_pintuan_table(self, data, pipeline): try: tmp = _get_right_model_data(data=data, site_id=21) except: print('此处抓到的可能是蜜芽拼团券所以跳过') return None # print('------>>> | 待存储的数据信息为: |', tmp) print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_update_pintuan_params(item=tmp) pipeline._update_table(sql_str=mia_update_str_3, params=params)
def to_right_and_update_data(self, data, pipeline): tmp = _get_right_model_data(data=data, site_id=12) params = get_db_commom_goods_update_params(item=tmp) base_sql_str = jp_update_str_2 if tmp['delete_time'] == '': sql_str = base_sql_str.format('shelf_time=%s', '') elif tmp['shelf_time'] == '': sql_str = base_sql_str.format('delete_time=%s', '') else: sql_str = base_sql_str.format('shelf_time=%s,', 'delete_time=%s') pipeline._update_table(sql_str=sql_str, params=params)
def insert_into_zhe_800_pintuan_table(self, data, pipeline) -> bool: try: tmp = _get_right_model_data(data=data, site_id=17) # 采集来源地(折800拼团商品) except: print('此处抓到的可能是折800拼团券所以跳过') return False print('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id'))) params = self._get_db_insert_pintuan_params(item=tmp) res = pipeline._insert_into_table(sql_str=z8_insert_str_2, params=params) return res
def insert_into_mogujie_xianshimiaosha_table(self, data, pipeline) -> bool: try: tmp = _get_right_model_data(data=data, site_id=22) # 采集来源地(蘑菇街秒杀商品) except: print('此处抓到的可能是蘑菇街券所以跳过') return False print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_insert_miaosha_params(item=tmp) res = pipeline._insert_into_table(sql_str=mg_insert_str_1, params=params) return res
def insert_into_jd_table(self, data, pipeline): site_id = from_jd_type_get_site_id(jd_type=data.get('jd_type')) if site_id is False: self.lg.error('site_id获取异常, 请检查!') return False tmp = _get_right_model_data(data=data, site_id=site_id) self.lg.info('------>>>| 待存储的数据信息为:{0}'.format(tmp.get('goods_id'))) pipeline.insert_into_jd_table(item=tmp) return True
def insert_into_mia_pintuan_table(self, data, pipeline) -> bool: try: tmp = _get_right_model_data(data=data, site_id=21) # 采集来源地(蜜芽拼团商品) except: print('此处抓到的可能是蜜芽拼团券所以跳过') return False print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_insert_pintuan_params(item=tmp) _r = pipeline._insert_into_table(sql_str=mia_insert_str_2, params=params) return _r
def to_update_zhe_800_xianshimiaosha_table(self, data, pipeline) -> bool: try: tmp = _get_right_model_data(data=data, site_id=14) except: print('此处抓到的可能是折800秒杀券所以跳过') return False tmp = self._set_detail_price_to_miaosha_price(tmp=tmp) print('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id'))) params = self._get_db_update_miaosha_params(item=tmp) res = pipeline._update_table(sql_str=z8_update_str_2, params=params) return res
def insert_into_zhe_800_xianshimiaosha_table(self, data, pipeline): try: tmp = _get_right_model_data(data=data, site_id=14) # 采集来源地(折800秒杀商品) except: print('此处抓到的可能是折800秒杀券所以跳过') return False tmp = self._set_detail_price_to_miaosha_price(tmp=tmp) print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_insert_miaosha_params(item=tmp) res = pipeline._insert_into_table(sql_str=z8_insert_str_1, params=params) return res
def update_jumeiyoupin_xianshimiaosha_table(self, data, pipeline) -> bool: try: tmp = _get_right_model_data(data=data, site_id=26) except: print('此处抓到的可能是聚美优品券所以跳过') return False tmp = self._set_detail_price_to_miaosha_price(tmp=tmp) print('------>>>| 待存储的数据信息为: |', tmp.get('goods_id')) params = self._get_db_update_miaosha_params(item=tmp) res = pipeline._update_table(sql_str=jm_update_str_1, params=params) return res
def old_ali_1688_goods_insert_into_new_table(self, data, pipeline): tmp = _get_right_model_data(data=data, site_id=2) params = self._get_db_insert_params(item=tmp) if tmp.get('main_goods_id') is not None: sql_str = al_insert_str_1 else: sql_str = al_insert_str_2 result = pipeline._insert_into_table_2(sql_str=sql_str, params=params, logger=self.lg) return result
def update_chuchujie_xianshimiaosha_table(self, data, pipeline) -> bool: try: tmp = _get_right_model_data(data=data, site_id=24) except: print('此处抓到的可能是楚楚街券所以跳过') return False # print('------>>> | 待存储的数据信息为: |', tmp) print('------>>>| 待存储的数据信息为: ', tmp.get('goods_id')) params = self._get_db_update_miaosha_params(item=tmp) sql_str = cc_update_str_1 res = pipeline._update_table(sql_str=sql_str, params=params) return res
def to_right_and_update_data(self, data, pipeline): tmp = _get_right_model_data(data=data, site_id=2) params = self._get_db_update_params(item=tmp) base_sql_str = al_update_str_2 if tmp['delete_time'] == '': sql_str = base_sql_str.format('shelf_time=%s', '') elif tmp['shelf_time'] == '': sql_str = base_sql_str.format('delete_time=%s', '') else: sql_str = base_sql_str.format('shelf_time=%s,', 'delete_time=%s') res = pipeline._update_table_2(sql_str=sql_str, params=params, logger=self.lg) return res
def to_right_and_update_data(self, data, pipeline): ''' 更新商品数据 :param data: :param pipeline: :return: ''' tmp = _get_right_model_data(data=data, site_id=25) params = get_db_commom_goods_update_params(item=tmp) base_sql_str = vip_update_str_1 if tmp['delete_time'] == '': sql_str = base_sql_str.format('shelf_time=%s', '') elif tmp['shelf_time'] == '': sql_str = base_sql_str.format('delete_time=%s', '') else: sql_str = base_sql_str.format('shelf_time=%s,', 'delete_time=%s') pipeline._update_table(sql_str=sql_str, params=params)
def _to_right_and_update_data(self, data, pipeline): ''' 实时更新数据 :param data: :param pipeline: :return: ''' tmp = _get_right_model_data(data, site_id=31, logger=self.lg) params = get_db_commom_goods_update_params(item=tmp) base_sql_str = yp_update_str_1 if tmp['delete_time'] == '': sql_str = base_sql_str.format('shelf_time=%s', '') elif tmp['shelf_time'] == '': sql_str = base_sql_str.format('delete_time=%s', '') else: sql_str = base_sql_str.format('shelf_time=%s,', 'delete_time=%s') result = pipeline._update_table_2(sql_str=sql_str, params=params, logger=self.lg) return result
async def update_expired_goods_id_taobao_tiantiantejia_table( self, data, pipeline): ''' 更新过期商品的信息,使其转为普通常规商品 :param data: :param pipeline: :return: ''' try: data['miaosha_begin_time'] = data.get('tejia_begin_time') data['miaosha_end_time'] = data.get('tejia_end_time') tmp = _get_right_model_data(data=data, site_id=19) except Exception: self.lg.error('遇到错误, 先跳过处理!出错goods_id={0}'.format( data['goods_id']), exc_info=True) return False self.lg.info('------>>>| 待存储的数据信息为: |' + tmp.get('goods_id')) await pipeline.update_expired_goods_id_taobao_tiantiantejia_table( item=tmp, logger=self.lg)
async def _update_taoqianggou_xianshimiaosha_table(self, data, pipeline): """ update对应表的数据 :param data: :param pipeline: :return: """ try: tmp = _get_right_model_data(data=data, site_id=28, logger=self.lg) except: self.lg.error('获取规范化数据失败!出错goods_id:{0}'.format( data.get('goods_id'))) return None self.lg.info('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id'))) params = await self._get_db_update_miaosha_params(item=tmp) pipeline._update_table_2(sql_str=tm_update_str_2, params=params, logger=self.lg) return
def insert_into_taoqianggou_xianshimiaosha_table(self, data, pipeline) -> bool: """ 将数据规范化插入淘抢购表 :param data: :param pipeline: :return: """ try: tmp = _get_right_model_data(data=data, site_id=28, logger=self.lg) # 采集来源地(淘抢购) except: print('此处抓到的可能是淘宝秒杀券所以跳过') return False self.lg.info('------>>>| 待存储的数据信息为: {0}'.format(data.get('goods_id'))) params = self._get_db_insert_taoqianggou_miaosha_params(item=tmp) res = pipeline._insert_into_table_2(sql_str=tm_insert_str_3, params=params, logger=self.lg) return res
def _deal_with_tm_goods(goods_link, my_lg): ''' 处理天猫商品 :param goods_link: :return: json_str ''' my_lg.info('进入天猫商品处理接口...') goods_id = _get_tm_goods_id(goods_link) if goods_id == '': msg = 'goods_id匹配失败!请检查url是否正确!' return _error_data(msg=msg) tm_url = 'https://detail.tmall.com/item.htm?id={0}'.format(goods_id) data = get_one_tm_data(wait_to_deal_with_url=tm_url, my_lg=my_lg) if data.get('msg', '') == 'data为空!': msg = '该goods_id:{0}, 抓取数据失败!'.format(goods_id) return _error_data(msg=msg) else: pass site_id = _from_tmall_type_get_site_id(type=data.get('type')) data = _get_right_model_data(data=data, site_id=site_id, logger=my_lg) my_pipeline = SqlServerMyPageInfoSaveItemPipeline() my_lg.info('------>>>| 正在存储的数据为: ' + data.get('goods_id', '')) params = _get_db_tmall_insert_params(item=data) sql_str = 'insert into dbo.GoodsInfoAutoGet(GoodsID, GoodsUrl, UserName, CreateTime, ModfiyTime, ShopName, Account, GoodsName, SubTitle, LinkName, Price, TaoBaoPrice, PriceInfo, SKUName, SKUInfo, ImageUrl, PropertyInfo, DetailInfo, SellCount, SiteID, IsDelete) values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)' is_insert_into = my_pipeline._insert_into_table_2(sql_str=sql_str, params=params, logger=my_lg) if is_insert_into: # 如果返回值为True pass else: # 不处理存储结果 # msg = '存储该goods_id:{0}失败!'.format(goods_id) # return _error_data(msg=msg) pass return compatible_api_goods_data(data=data, my_lg=my_lg)