def sub_product_update(): sql = "update shopee_sub_product set sales_attributes=%s ,price=%s ," \ "before_price=%s ,sold=%s ,quantity=%s ,status = %s ,update_time=%s ,update_by= %s" \ " where spu= %s and sku= %s" start_time6 = get_time() count6 = 0 data_list6 = [] while True: sub_product = cache_sub_product_update.get_nowait() if sub_product is not None: sub_product = json.loads(sub_product.decode()) params = (sub_product['sales_attributes'], sub_product['price'], sub_product['before_price'], sub_product['sold'], sub_product['quantity'], sub_product['status'], datetime.datetime.now(), '007', str(sub_product['spu']), str(sub_product['sku'])) data_list6.append(params) count6 += 1 else: time.sleep(5) # 首先判断数组内数据个数是否达到100,达到,就批量存入msyql if count6 > 200: try: dbpool.Pool().update_many(sql, 'shopee_sub_product update', data_list6) except Exception as e: print('-shopee_sub_product update---', e) for product in data_list6: cache_sub_product_update.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count start_time count6 = 0 start_time6 = get_time() data_list6 = [] else: # 虽然数组个数没有100,但接受数据时间超过了10分钟,就批量存入msyql time_c = get_time() - start_time6 if time_c > 300: if len(data_list6) > 0: try: dbpool.Pool().update_many(sql, 'shopee_sub_product update', data_list6) except Exception: traceback.print_exc() for product in data_list6: cache_sub_product_update.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count = 0 start_time count6 = 0 start_time6 = get_time() data_list6 = []
def sub_product_change_insert(): start_time5 = get_time() count5 = 0 data_list5 = [] while True: sub_product = cache_sub_product_change_insert.get_nowait() if sub_product is not None: sub_product = json.loads(sub_product.decode()) data_list5.append(sub_product) count5 += 1 else: time.sleep(5) # 首先判断数组内数据个数是否达到100,达到,就批量存入msyql if count5 > 200: try: dbpool.Pool().insert_many_temp('shopee_sub_product_change', data_list5, 'shopee_sub_product_change') except Exception as e: print('-shopee_sub_product_change---', e) for product in data_list5: print(product) cache_sub_product_change_insert.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count start_time count5 = 0 start_time5 = get_time() data_list5 = [] else: # 虽然数组个数没有100,但接受数据时间超过了10分钟,就批量存入msyql time_c = get_time() - start_time5 if time_c > 300: if len(data_list5) > 0: try: dbpool.Pool().insert_many_temp( 'shopee_sub_product_change', data_list5, 'shopee_sub_product_change') except Exception: traceback.print_exc() for product in data_list5: cache_sub_product_change_insert.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count = 0 start_time count5 = 0 start_time5 = get_time() data_list5 = []
def product_update_time(): sql = "update shopee_product set update_time=%s ,status = 1 , update_by= %s where spu= %s" start_time2 = get_time() count2 = 0 data_list2 = [] while True: spu = cache_product_update_time.get_nowait() if spu is not None: spu = spu.decode() now_time = datetime.datetime.now() data_list2.append((now_time, '007', spu)) count2 += 1 else: time.sleep(5) # 首先判断数组内数据个数是否达到100,达到,就批量存入msyql if count2 > 200: try: dbpool.Pool().update_many(sql, 'shopee_product update time', data_list2) except Exception: traceback.print_exc() for product in data_list2: cache_product_update_time.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count start_time count2 = 0 start_time2 = get_time() data_list2 = [] else: # 虽然数组个数没有100,但接受数据时间超过了10分钟,就批量存入msyql time_c = get_time() - start_time2 if time_c > 100: if len(data_list2) > 0: try: dbpool.Pool().update_many( sql, 'shopee_product update time', data_list2) except Exception: traceback.print_exc() for product in data_list2: cache_product_update_time.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count = 0 start_time count2 = 0 start_time2 = get_time() data_list2 = []
def product_insert(): start_time0 = get_time() count0 = 0 data_list0 = [] while True: product = cache_product_insert.get_nowait() if product is not None: product = json.loads(product.decode()) product = trans_product_entity(product) data_list0.append(product) count0 += 1 else: time.sleep(10) # 首先判断数组内数据个数是否达到100,达到,就批量存入msyql if count0 > 1: try: dbpool.Pool().insert_many_temp('shopee_product', data_list0, 'shopee_product') except Exception as e: print('-shopee_product---', e) for product in data_list0: cache_product_insert.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count start_time count0 = 0 start_time0 = get_time() data_list0 = [] else: # 虽然数组个数没有100,但接受数据时间超过了10分钟,就批量存入msyql time_c = get_time() - start_time0 if time_c > 300: if len(data_list0) > 0: try: dbpool.Pool().insert_many_temp('shopee_product', data_list0, 'shopee_product') except Exception: traceback.print_exc() for product in data_list0: cache_product_insert.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count = 0 start_time count0 = 0 start_time0 = get_time() data_list0 = []
def push_search(): pool = dbpool.Pool() result = dbpool.Pool().get_all( "select * from shopee_product_copy1 where create_by is null ") for item in result: shopid = item['shopid'] spu = item['spu'] task = { "shopid": shopid, "webid": 1, "country": "PH", "parse_type": "goods", "level": 1, "spu": spu } print(task) queue_shopee_search.put(json.dumps(dict(task), cls=DateEnconding)) # product(task) sql = "update shopee_product_copy1 set create_by=1 where shopid='{}' and spu='{}'".format( shopid, spu) print(sql) pool.update(sql)
def shop_insert(): # 比较少,不做批量处理,有就直接存储 while True: shop = cache_shop_insert.get_nowait() if shop is not None: shop = json.loads(shop.decode()) try: dbpool.Pool().insert_temp('shopee_shope', shop, 'shopee_shope') except Exception: traceback.print_exc() cache_shop_insert.put(json.dumps(dict(shop), cls=DateEnconding)) else: time.sleep(10)
def search(task): sort = task['sort'] keyword = task['keyword'] page = task['page'] try: url = "https://ph.xiapibuy.com/api/v2/search_items/?by={}&keyword={}&limit=50&newest={}&order=desc&page_type=search&version=2".format( sort, keyword, (page - 1) * 50) # shipeeed from过滤条件 Overseas # url = "https://ph.xiapibuy.com/api/v2/search_items/?by={}&keyword={}&limit=50&locations=-2&newest=0&order=desc&page_type=search&skip_autocorrect=1&version=2".format(sort,keyword) # print(url) html = get_html(url) # print(html) json_str = json.loads(html) search_list = [] num = 1 for item in json_str['items']: search = trans_search(task) search['position'] = '{}-{}'.format(page, num) shopid = item['shopid'] search['shopid'] = shopid search['create_time'] = datetime.datetime.now() # search['update_time']= datetime.datetime.now() search['spu'] = item['itemid'] search['historical_sold'] = item['historical_sold'] search['sold'] = item['sold'] search['shipping_from'] = item['shop_location'] search['ad'] = item['adsid'] try: adsid = item['adsid'] if adsid is not None: search['ad'] = 1 else: search['ad'] = 0 except Exception: search['ad'] = 0 num += 1 search_list.append(search) redis_db.hset('shops', shopid, 0) dbpool.Pool().insert_many_temp('shopee_search', search_list, 'shopee_search') except Exception as e: traceback.print_exc() print('search异常--', e) queue_shopee_search.put(json.dumps(dict(task), cls=DateEnconding)) print('存入redis队列')
def shop_update_time(): # 比较少,不做批量处理,有就直接存储 while True: shopid = cache_shop_update_time.get_nowait() if shopid is not None: shopid = shopid.decode() try: sql = "update shopee_shope set update_time=%s , update_by= %s where shopid= %s" now_time = datetime.datetime.now() dbpool.Pool().update(sql, (now_time, '007', shopid)) except Exception: traceback.print_exc() cache_shop_update_time.put(shopid) else: time.sleep(10)
def shop_update(): # 比较少,不做批量处理,有就直接存储 while True: shop = cache_shop_update.get_nowait() if shop is not None: shop = json.loads(shop.decode()) try: sql = "update shopee_shope set " \ "shopname=%s ," \ "icon=%s ," \ "products=%s ," \ "followers=%s ," \ "following=%s ," \ "rating_num=%s ," \ "rating=%s ," \ "chatperformance=%s ," \ " abstract=%s ," \ " update_time=%s ," \ " update_by= %s" \ " where shopid= %s" abstract = None try: abstract = shop['abstract'] except Exception: pass params = (shop['shopname'], shop['icon'], shop['products'], shop['followers'], shop['following'], shop['rating_num'], shop['rating'], shop['chatperformance'], abstract, datetime.datetime.now(), '007', shop['shopid']) dbpool.Pool().update(sql, params) except Exception: traceback.print_exc() cache_shop_update.put(json.dumps(dict(shop), cls=DateEnconding)) else: time.sleep(10)
def product_update(): sql = "update shopee_product set " \ "historical_sold=%s ," \ "price=%s ," \ "height_price=%s ," \ "product_title=%s ," \ " quantity=%s ," \ " status = 1 ," \ " update_time=%s ," \ " update_by= %s" \ " where spu= %s" start_time3 = get_time() count3 = 0 data_list3 = [] while True: product = cache_product_update.get_nowait() if product is not None: product = json.loads(product.decode()) params = (product['historical_sold'], product['price'], product['height_price'], product['product_title'], product['quantity'], datetime.datetime.now(), '007', str(product['spu'])) data_list3.append(params) count3 += 1 else: time.sleep(5) # 首先判断数组内数据个数是否达到100,达到,就批量存入msyql if count3 > 100: try: dbpool.Pool().update_many(sql, 'shopee_product update ', data_list3) except Exception as e: print('-shopee_product update---', e) for product in data_list3: cache_product_update.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count start_time count3 = 0 start_time3 = get_time() data_list3 = [] else: # 虽然数组个数没有100,但接受数据时间超过了10分钟,就批量存入msyql time_c = get_time() - start_time3 if time_c > 300: if len(data_list3) > 0: try: dbpool.Pool().update_many(sql, 'shopee_product update ', data_list3) except Exception: traceback.print_exc() for product in data_list3: cache_product_update.put( json.dumps(dict(product), cls=DateEnconding)) # 重置 count = 0 start_time count3 = 0 start_time3 = get_time() data_list3 = []
import json import sys import os import time curPath = os.path.abspath(os.path.dirname(__file__)) rootPath = os.path.split(curPath)[0] rootPath = os.path.split(rootPath)[0] rootPath = os.path.split(rootPath)[0] sys.path.append(rootPath) from by.pipline import dbpool from by.pipline.redisclient import RedisClient from by.pipline.redisqueue import RedisQueue from by.utils.tools import DateEnconding db = dbpool.Pool() queue_shopee_search = RedisQueue('shopee_search', 'mz') queue_shopee = RedisQueue('shopee', 'mz') redis_db = RedisClient() # 重新同步mysql 和redis shop def update_shop_to_redis(): result = db.get_all("select distinct(shopid) from shopee_search ") for item in result: shopid = item['shopid'] redis_db.hset('shops',shopid,0) # 将搜索词推送到search搜索任务队列中 def push_search():