def deal_with_data(self): ''' 处理得到规范的data数据 :return: result 类型 dict ''' data = self.result_data if data != {}: shop_name = data['shop_name'] account = '' title = data['title'] sub_title = data['sub_title'] price = data['price'] # 商品价格 taobao_price = data['taobao_price'] # 淘宝价 detail_name_list = data['detail_name_list'] price_info_list = data['price_info_list'] all_img_url = data['all_img_url'] p_info = data['p_info'] div_desc = data['div_desc'] is_delete = 0 if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): print('违禁物品下架...') is_delete = 1 else: pass result = { # 'goods_url': data['goods_url'], # goods_url 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list,# 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'is_delete': is_delete # 用于判断商品是否已经下架 } # pprint(result) # print(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) return result else: print('待处理的data为空的dict, 该商品可能已经转移或者下架') return {}
def deal_with_data(self): ''' 解析data数据,得到需要的东西 :return: dict ''' data = self.result_data if data != {}: shop_name = self._get_shop_name(data=data) # 掌柜 account = '' title = self._get_title(data=data) sub_title = '' detail_name_list = self._get_detail_name_list(data=data) # print(detail_name_list) '''单独处理下架的情况''' if isinstance(detail_name_list, str): if detail_name_list == 'is_delete=1': _handle_goods_shelves_in_auto_goods_table( goods_id=self.result_data.get('goods_id', ''), ) else: pass if detail_name_list == {}: return self._data_error_init() price_info_list, price, taobao_price = self._get_price_info_list_and_price_and_taobao_price( data=data) all_img_url = self._get_all_img_url(data=data) p_info = self._get_p_info(data=data) div_desc = self._get_div_desc(data=data) # 商品销售时间段 schedule = self._get_goods_schedule(data=data) # pprint(schedule) is_delete = self._get_is_delete(data=data, schedule=schedule) if price == 0 or taobao_price == 0: # 没有获取到价格说明商品已经下架了 is_delete = 1 parent_dir = data.get('parent_dir', '') all_sell_count = '' if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): print('违禁物品下架...') is_delete = 1 else: pass result = { 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list, # 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'is_delete': is_delete, # 是否下架判断 'schedule': schedule, # 商品销售时间段 'parent_dir': parent_dir, 'all_sell_count': all_sell_count, } # pprint(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) collect() return result else: print('待处理的data为空的dict') return {}
def deal_with_data(self) -> dict: ''' 处理得到规范的data数据 :return: result 类型 dict ''' data = self.result_data # pprint(data) if data != {}: shop_name = '' account = '' title = data['title'] sub_title = data['sub_title'] # 商品价格和淘宝价 try: tmp_price_list = sorted([round(float(item.get('pintuan_price', '')), 2) for item in data['price_info_list']]) price = tmp_price_list[-1] # 商品价格 taobao_price = tmp_price_list[0] # 淘宝价 except IndexError: return self._data_error_init() detail_name_list = data['detail_name_list'] price_info_list = data['price_info_list'] all_img_url = data['all_img_url'] p_info = data['p_info'] div_desc = data['div_desc'] parent_dir = data['parent_dir'] is_delete = 0 if price_info_list == [] \ or data['pintuan_time'] == {}: is_delete = 1 else: pass if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): print('违禁物品下架...') is_delete = 1 else: pass result = { 'goods_url': data['goods_url'], # goods_url 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list,# 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'pintuan_time': data['pintuan_time'], # 拼团开始和结束时间 'all_sell_count': data['all_sell_count'], # 总销量 'is_delete': is_delete, # 用于判断商品是否已经下架 'parent_dir': parent_dir, } # pprint(result) # print(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) return result else: print('待处理的data为空的dict, 该商品可能已经转移或者下架') return {}
def _deal_with_data(self): ''' 处理得到需求data :return: ''' data = self.result_data if data != {}: shop_name = data['shop_name'] account = '' title = data['title'] sub_title = data['sub_title'] detail_name_list = data['detail_name_list'] price_info_list = data['price_info_list'] all_img_url = data['all_img_url'] p_info = data['p_info'] # pprint(p_info) div_desc = data['div_desc'] is_delete = data['is_delete'] parent_dir = data['parent_dir'] # 上下架时间 if data.get('sell_time', {}) != {}: schedule = [{ 'begin_time': data.get('sell_time', {}).get('begin_time', ''), 'end_time': data.get('sell_time', {}).get('end_time', ''), }] else: schedule = [] all_sell_count = '' price, taobao_price = data['price'], data['taobao_price'] if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): self.lg.info('违禁物品下架...') is_delete = 1 else: pass result = { 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list, # 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'schedule': schedule, # 商品特价销售时间段 'all_sell_count': all_sell_count, # 销售总量 'is_delete': is_delete, # 是否下架 'parent_dir': parent_dir, } # pprint(result) # print(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) self.result_data = {} return result else: self.lg.error('待处理的data为空的dict, 该商品可能已经转移或者下架') return self._get_data_error_init()
def deal_with_data(self, goods_id) -> dict: """ 处理数据 :param goods_id: :return: {'is_delete': 1} 表示下架商品 """ data = self.result_data # pprint(data) if data == {}: self.lg.info('待处理的data为空的dict' + self.error_record) return {} if data.get('is_delete', 1) == 1: self.lg.info('**** 该商品{}已下架...'.format(goods_id[1])) return {'is_delete': 1} try: title = self._get_title(data=data) sub_title = self._get_sub_title(data=data) shop_name = self._get_shop_name(data=data) account = '' all_img_url = self._get_all_img_url(data=data) detail_name_list = self._get_detail_name_list(data=data) price_info_list = self._get_price_info_list(data=data) if price_info_list == []: # 避免手机有规格但是电脑没有规格的bug detail_name_list = [] p_info = self._get_p_info(data=data) div_desc = self._get_div_desc(data=data) price, taobao_price = self._get_price_and_taobao_price( price_info_list=price_info_list, base_price_info=data.get('base_price_info', {})) is_delete = self._get_is_delete(data=data) if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): self.lg.info('违禁物品下架...') is_delete = 1 else: pass except (AssertionError, Exception): self.lg.error('遇到错误:', exc_info=True) return self._data_error_init() jd_type = 7 # 不进行区分全部都为7, 即京东常规商品 all_sell_count = data.get('all_sell_count', '600') res = { 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list, # 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存(京东隐藏库存无法爬取,只能能买或不能买) 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 # 'pc_div_url': pc_div_url, # pc端描述地址 'div_desc': div_desc, # div_desc 'is_delete': is_delete, # 是否下架判断 'jd_type': jd_type, # 京东类型,(京东常规商品为7,京东超市为8) 'all_sell_count': all_sell_count, # 商品总销售量 } # pprint(result) # self.lg.info(str(result)) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # self.lg.info(str(json_data)) collect() return res
def deal_with_data(self): """ 得到需求数据 :return: """ data = self.result_data # pprint(data) if data == {}: return self._data_error_init() taobao = TaoBaoLoginAndParse( logger=self.lg, is_real_times_update_call=self.is_real_times_update_call) goods_id = data['goods_id'] # 天猫类型 tmall_type = data.get('type', 33) # 33用于表示无法正确获取 # 可能不存在shopName这个字段 shop_name = data['seller'].get('shopName', '') account = data['seller'].get('sellerNick', '') title = data['item']['title'] sub_title = data['item'].get('subtitle', '') if sub_title is None: # 处理tm 2版为None的情况 sub_title = '' else: pass sub_title = re.compile(r'\n').sub('', sub_title) price, taobao_price = taobao._get_price_and_taobao_price(data=data) # 商品库存 goods_stock = data['apiStack'][0]['value']\ .get('skuCore', {})\ .get('sku2info', {})\ .get('0', {})\ .get('quantity', '') # 商品标签属性名称,及其对应id值 detail_name_list, detail_value_list = taobao._get_detail_name_and_value_list( data=data) # 上下架判断放在这里避免已下架的不进行taobao_price为空断言判断 is_delete = self._get_is_delete(data=data, title=title) # self.lg.info('is_delete: {}'.format(is_delete)) try: # 每个标签对应值的价格及其库存 price_info_list = taobao._get_price_info_list( data=data, detail_value_list=detail_value_list, is_delete=is_delete, ) # 多规格进行重新赋值 price, taobao_price = taobao._get_new_price_and_taobao_price_when_price_info_list_not_null_list( price_info_list=price_info_list, price=price, taobao_price=taobao_price, is_delete=is_delete) except Exception: self.lg.error('遇到错误[goods_id: {}]:'.format(goods_id), exc_info=True) return self._data_error_init() # 所有示例图片地址 all_img_url = taobao._get_all_img_url( tmp_all_img_url=data['item']['images']) # self.lg.info(str(all_img_url)) # tmp_p_info 一个list [{'内存容量': '32GB'}, ...] p_info = taobao._get_p_info( tmp_p_info=data.get('props').get('groupProps')) if p_info != []: p_info = [{ 'id': 0, 'name': _i.get('p_name', ''), 'value': _i.get('p_value', ''), } for _i in p_info] """div_desc""" div_desc = taobao.get_div_from_pc_div_url(goods_id=goods_id) # self.lg.info(div_desc) if div_desc == '': self.lg.error('该商品的div_desc为空! 出错goods_id: {}'.format(goods_id)) return self._data_error_init() else: pass """后期处理""" # 后期处理detail_name_list, detail_value_list detail_name_list = [{ 'spec_name': i[0], 'img_here': i[2], } for i in detail_name_list] # 商品标签属性对应的值, 及其对应id值 if data.get('skuBase').get('props') is None: pass else: tmp_detail_value_list = [ item['values'] for item in data.get('skuBase', '').get('props', '') ] # self.lg.info(str(tmp_detail_value_list)) detail_value_list = [] for item in tmp_detail_value_list: tmp = [i['name'] for i in item] # self.lg.info(str(tmp)) detail_value_list.append(tmp) # 商品标签属性对应的值 # pprint(detail_value_list) # 月销量 sell_count = '0' try: sell_count = str( data.get('apiStack', [])[0].get('value', {}).get('item', {}).get('sellCount', '')) # self.lg.info(sell_count) except: pass if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): self.lg.info('违禁物品下架...') is_delete = 1 else: pass try: del taobao except: pass result = { 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 'detail_value_list': detail_value_list, # 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'sell_count': sell_count, # 月销量 'is_delete': is_delete, # 是否下架判断 'type': tmall_type, # 天猫类型 } # pprint(result) # self.lg.info(str(result)) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) return result
def deal_with_data(self): ''' 处理result_data, 返回需要的信息 :return: 字典类型 ''' data = self.result_data if data != {}: try: shop_name = self._get_shop_name(data=data) title = self._get_title(data=data) sub_title = self._get_sub_title(data=data) detail_name_list = self._get_detail_name_list(data=data) price_info_list = self._get_price_info_list( data=data, detail_name_list=detail_name_list) price_info_list, price, taobao_price = self._get_price_and_tb_price(price_info_list=price_info_list, data=data) all_img_url = self._get_all_img_url(data=data) p_info = self._get_p_info(data=data) # 总销量(shop_sales字段) all_sell_count = self._get_all_sell_count(data=data) div_desc = self._get_div_desc(data=data) # 商品销售时间区间(sale_begin_time和sale_end_time字段) schedule = self._get_schedule(data=data) # pprint(schedule) is_delete = self._get_is_delete(data=data, schedule=schedule) # print(is_delete) parent_dir = data.get('parent_dir', '') if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): print('违禁物品下架...') is_delete = 1 else: pass except Exception as e: print('遇到错误:', e) return self._data_error_init() result = { 'shop_name': shop_name, # 店铺名称 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list,# 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'schedule': schedule, # 商品开卖时间和结束开卖时间 'all_sell_count': all_sell_count, # 商品总销售量 'is_delete': is_delete, # 用于判断商品是否已经下架 'parent_dir': parent_dir, } # pprint(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) return result else: print('待处理的data为空的dict, 该商品可能已经转移或者下架') return {}
def deal_with_data(self): ''' 处理得到规范的data数据 :return: result 类型 dict ''' data = self.result_data if data != {}: shop_name = data['shop_name'] account = '' title = data['title'] sub_title = data['sub_title'] detail_name_list = data['detail_name_list'] price_info_list = data['price_info_list'] all_img_url = data['all_img_url'] p_info = data['p_info'] div_desc = data['div_desc'] is_delete = data['is_delete'] # 上下架时间 schedule = data['sell_time'] # 销售总量 all_sell_count = data['all_sell_count'] # 商品价格和淘宝价 # pprint(data['price_info_list']) try: tmp_price_list = sorted([ round(float(item.get('detail_price', '')), 2) for item in data['price_info_list'] ]) price = tmp_price_list[-1] # 商品价格 taobao_price = tmp_price_list[0] # 淘宝价 except IndexError: print('获取price和taobao_price时出错, 请检查!' ) # 商品下架时, detail_price为空str, 所以会IndexError报错 self.result_data = {} price = 0. taobao_price = 0. is_delete = 1 # return {} if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): print('违禁物品下架...') is_delete = 1 else: pass result = { 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list,# 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'schedule': schedule, # 商品特价销售时间段 'all_sell_count': all_sell_count, # 销售总量 'is_delete': is_delete # 用于判断商品是否已经下架 } # pprint(result) # print(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) self.result_data = {} return result else: print('待处理的data为空的dict, 该商品可能已经转移或者下架') self.result_data = {} return {}
def publish_one_article(self, driver: BaseDriver, article_url: str): """ 发布一篇图文 :param driver: :param article_url: :return: """ try: # 切换到目标iframe(用index有时候不准, pass) # driver.switch_to_frame(frame_reference=1) iframe_ele_list = driver.find_elements(by=By.TAG_NAME, value='iframe') # pprint(iframe_ele_list) assert iframe_ele_list != [] target_iframe_ele = iframe_ele_list[1] if len( iframe_ele_list) > 1 else iframe_ele_list[0] driver.switch_to_frame(frame_reference=target_iframe_ele) except (NoSuchFrameException, ) as e: # 没匹配到frame(可能是原先就在目标iframe, eg: title过长的, 再切回iframe, 但是iframe_ele_list为0) raise e # 清空输入框 input_box_ele = driver.find_element(value='input#SnatchUrl') input_box_ele.clear() # 输入待采集地址 input_box_ele.send_keys(article_url) # 点击采集按钮 driver.find_elements(value='span.input-group-btn button')[0].click() try: self.wait_for_delete_img_appear(driver=driver) except (FZTimeoutError, NoSuchElementException): # 发布某文章超时失败or无元素存在, 则抛出发布异常 raise PublishOneArticleFailException # 获取输入框的值 title = driver.find_element( value='input#RecommendName').get_attribute('value') self.lg.info('title: {}'.format(title)) if target_str_contain_some_char_check( target_str=title, check_char_obj=ARTICLE_TITLE_SENSITIVE_STR_TUPLE): raise ArticleTitleContainSensitiveWordsException else: pass if isinstance(title, str) and len(title) > 30: # 标题过长则return, 不发布 self.lg.info('@@@ title 标题过长, 无法发布!! 跳过!') # 由于标题过长后, 无法处理后续文章, 故不return, 直接抛出异常 # return raise ArticleTitleOverLongException else: pass try: # 点击发布按钮 driver.find_elements( value='span.input-group-btn button')[1].click() except WebDriverException: # 处理发布单篇异常! # 处理报错: Message: unknown error: Element <iframe class="J_iframe" name="iframe0" raise PublishOneArticleFailException # 切换至主页面 driver.switch_to_default_content() # 填写被发布人 random_phone = self.get_random_phone() driver.find_element( value='input.layui-layer-input').send_keys(random_phone) # 点击确定 driver.find_element(value='a.layui-layer-btn0').click() self.lg.info('url: {} 发布成功!'.format(article_url)) # 发布成功, 等待5.秒, 等待页面元素置空 sleep(5.) return
def deal_with_data(self): ''' 处理返回的result_data, 并返回需要的信息 :return: 字典类型 ''' data = self.result_data # pprint(data) try: assert data != {}, '待处理的data为空dict!' company_name = data.get('companyName', '') title = self._wash_sensitive_words(data.get('subject', '')) link_name = '' price_info = self._get_price_info(data=data) # self.lg.info(str(price_info)) # 标签属性名称及其对应的值 # (可能有图片(url), 无图(imageUrl=None)) # eg: [{'value': [{'imageUrl': 'https://cbu01.alicdn.com/img/ibank/2017/520/684/4707486025_608602289.jpg', 'name': '白色'}, {'imageUrl': 'https://cbu01.alicdn.com/img/ibank/2017/554/084/4707480455_608602289.jpg', 'name': '卡其色'}, {'imageUrl': 'https://cbu01.alicdn.com/img/ibank/2017/539/381/4705183935_608602289.jpg', 'name': '黑色'}], 'prop': '颜色'}, {'value': [{'imageUrl': None, 'name': 'L'}, {'imageUrl': None, 'name': 'XL'}, {'imageUrl': None, 'name': '2XL'}], 'prop': '尺码'}] sku_props = self._get_detail_name_list(data=data) # self.lg.info(str(sku_props)) # 每个规格对应价格, 及其库存量 sku_map = self._get_sku_info(data=data, price_info=price_info, detail_name_list=sku_props) # pprint(sku_map) except Exception: self.lg.error('遇到错误:' + self.error_base_record, exc_info=True) self.is_activity_goods = False return self._data_error_init() price, taobao_price = self._get_price_and_taobao_price( price_info=price_info) all_img_url = self._get_all_img_url(data=data) # 即: p_info property_info = self._get_p_info(data=data) # 即: div_desc detail_info_url = data.get('detailUrl', '') detail_info = self._get_div_desc( detail_info_url) if detail_info_url != '' else '' # self.lg.info(str(detail_info)) is_delete = self._get_is_delete(title=title, price_info=price_info) if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): self.lg.info('违禁物品下架...') is_delete = 1 else: pass result = { 'company_name': company_name, # 公司名称 'title': title, # 商品名称 'link_name': link_name, # 卖家姓名 'price_info': price_info, # 商品价格信息, 及其对应起批量 'price': price, # 起批的最高价 'taobao_price': taobao_price, # 起批的最低价 'sku_props': sku_props, # 标签属性名称及其对应的值 (可能有图片(url), 无图(imageUrl=None)) 'sku_map': sku_map, # 每个规格对应价格, 及其库存量 'all_img_url': all_img_url, # 所有示例图片地址 'property_info': property_info, # 详细信息的标签名, 及其对应的值 'detail_info': detail_info, # 下方详细div块 'is_delete': is_delete, # 判断是否下架 } # pprint(result) # self.lg.info(str(result)) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # self.lg.info(str(json_data)) # 重置self.is_activity_goods = False self.is_activity_goods = False return result
def deal_with_data(self): ''' 处理result_data, 返回需要的信息 :return: 字典类型 ''' data = self.result_data if data != {}: shop_name = data.get('mall', {}).get('mallName', '') \ if data.get('mall') is not None else '' account = '' title = data.get('goods', {}).get('goodsName', '') sub_title = '' detail_name_list = self._get_detail_name_list(data=data) # print(detail_name_list) price_info_list = self._get_price_info_list(data=data) if price_info_list == []: print('price_info_list为空值') return {} # 商品价格和淘宝价 tmp_price_list = sorted([ round(float(item.get('detail_price', '')), 2) for item in price_info_list ]) price = tmp_price_list[-1] # 商品价格 taobao_price = tmp_price_list[0] # 淘宝价 if detail_name_list == []: print('## detail_name_list为空值 ##') price_info_list = [] # print('最高价为: ', price) # print('最低价为: ', taobao_price) # print(len(price_info_list)) # pprint(price_info_list) all_img_url = self._get_all_img_url(data=data) # print(all_img_url) p_info = self._get_p_info(data=data) # print(p_info) # 总销量 all_sell_count = data.get('goods', {}).get('sales', 0) div_desc = data.get('div_desc', '') # 商品销售时间区间 schedule = [{ 'begin_time': timestamp_to_regulartime( data.get('goods', {}).get('groupTypes', [])[0].get('startTime')), 'end_time': timestamp_to_regulartime( data.get('goods', {}).get('groupTypes', [])[0].get('endTime')), }] # pprint(schedule) # 用于判断商品是否已经下架 is_delete = 0 if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): print('违禁物品下架...') is_delete = 1 else: pass result = { 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 # 'shop_name_url': shop_name_url, # 店铺主页地址 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list,# 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'schedule': schedule, # 商品开卖时间和结束开卖时间 'all_sell_count': all_sell_count, # 商品总销售量 'is_delete': is_delete # 用于判断商品是否已经下架 } # pprint(result) # print(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) return result else: print('待处理的data为空的dict, 该商品可能已经转移或者下架') return {}
async def deal_with_data(self, jumei_pintuan_url) -> dict: ''' 得到规范数据并处理 :return: ''' data = await self.get_goods_data(jumei_pintuan_url=jumei_pintuan_url) if data != {}: shop_name = data['shop_name'] account = '' title = data['title'] sub_title = data['sub_title'] # 商品价格和淘宝价 try: tmp_price_list = sorted([round(float(item.get('pintuan_price', '')), 2) for item in data['price_info_list']]) price = tmp_price_list[-1] # 商品价格 taobao_price = tmp_price_list[0] # 淘宝价 except IndexError: self.msg = '获取price or taobao_price时出错请检查!' + ' 出错地址: ' + data['goods_url'] self.lg.error(self.msg) return await self._data_error() detail_name_list = data['detail_name_list'] price_info_list = data['price_info_list'] all_img_url = data['all_img_url'] p_info = data['p_info'] div_desc = data['div_desc'] is_delete = data['is_delete'] if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): self.lg.info('违禁物品下架...') is_delete = 1 else: pass result = { 'goods_url': data['goods_url'], # goods_url 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 'detail_name_list': detail_name_list, # 商品标签属性名称 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'all_sell_count': data['all_sell_count'], # 总销量 'is_delete': is_delete # 用于判断商品是否已经下架 } # pprint(result) # print(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) return result else: try: self.msg = '待处理的data为空的dict, 该商品可能已经转移或者下架' + ' 出错地址: ' + data['goods_url'] self.lg.error(self.msg) except KeyError: pass return {}
def deal_with_data(self): ''' 处理result_data, 返回需要的信息 :return: 字典类型 ''' data = self.result_data goods_id = data.get('goods_id', '') if data != {}: try: shop_name = data.get('shop_name', '') account = '' title = data.get('/app/detail/product/base', {}).get('title', '') sub_title = '' # 要存储的每个标签对应规格的价格及其库存 tmp_price_info_list = data.get('/app/detail/product/sku', {}).get('items') # pprint(tmp_price_info_list) cache = self._get_detail_name_list_and_price_info_list_and_price_and_taobao_price( data=data, tmp_price_info_list=tmp_price_info_list) all_img_url = self._get_all_img_url(tmp_all_img_url=data.get( '/app/detail/product/base', {}).get('images', [])) detail_name_list = cache[0] price_info_list = cache[1] price = cache[2] taobao_price = cache[3] # print('最高价为: ', price) # print('最低价为: ', taobao_price) # print(detail_name_list) # pprint(price_info_list) p_info = self._get_p_info(data=data) # pprint(p_info) # div_desc div_desc = data.get('/app/detail/graph/detail', '') is_delete = self._get_is_delete( price_info_list=price_info_list) schedule, is_delete = self._get_schedule(data=data, is_delete=is_delete) # pprint(schedule) parent_dir = str(data.get('parent_dir', '')) all_sell_count = '' if target_str_contain_some_char_check( target_str=title, check_char_obj=CONTRABAND_GOODS_KEY_TUPLE): print('违禁物品下架...') is_delete = 1 else: pass except GoodsShelvesException: _handle_goods_shelves_in_auto_goods_table(goods_id=goods_id, ) return self._data_error() except (AttributeError, Exception) as e: print('遇到错误:', e) return self._data_error() result = { 'shop_name': shop_name, # 店铺名称 'account': account, # 掌柜 'title': title, # 商品名称 'sub_title': sub_title, # 子标题 # 'shop_name_url': shop_name_url, # 店铺主页地址 'price': price, # 商品价格 'taobao_price': taobao_price, # 淘宝价 # 'goods_stock': goods_stock, # 商品库存 'detail_name_list': detail_name_list, # 商品标签属性名称 # 'detail_value_list': detail_value_list, # 商品标签属性对应的值 'price_info_list': price_info_list, # 要存储的每个标签对应规格的价格及其库存 'all_img_url': all_img_url, # 所有示例图片地址 'p_info': p_info, # 详细信息标签名对应属性 'div_desc': div_desc, # div_desc 'schedule': schedule, # 商品开卖时间和结束开卖时间 'is_delete': is_delete, # 用于判断商品是否已经下架 'parent_dir': parent_dir, 'all_sell_count': all_sell_count, } # pprint(result) # print(result) # wait_to_send_data = { # 'reason': 'success', # 'data': result, # 'code': 1 # } # json_data = json.dumps(wait_to_send_data, ensure_ascii=False) # print(json_data) return result else: print('待处理的data为空的dict, 该商品可能已经转移或者下架') self.result_data = {} return {}