def sub_product_update():
sql = "update shopee_sub_product set sales_attributes=%s ,price=%s ," \
"before_price=%s ,sold=%s ,quantity=%s ,status = %s ,update_time=%s ,update_by= %s" \
" where spu= %s and sku= %s"
start_time6 = get_time()
count6 = 0
data_list6 = []
while True:
sub_product = cache_sub_product_update.get_nowait()
if sub_product is not None:
sub_product = json.loads(sub_product.decode())
params = (sub_product['sales_attributes'], sub_product['price'],
sub_product['before_price'], sub_product['sold'],
sub_product['quantity'], sub_product['status'],
datetime.datetime.now(), '007', str(sub_product['spu']),
str(sub_product['sku']))
data_list6.append(params)
count6 += 1
else:
time.sleep(5)
# 首先判断数组内数据个数是否达到100，达到，就批量存入msyql
if count6 > 200:
try:
dbpool.Pool().update_many(sql, 'shopee_sub_product update',
data_list6)
except Exception as e:
print('-shopee_sub_product update---', e)
for product in data_list6:
cache_sub_product_update.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count start_time
count6 = 0
start_time6 = get_time()
data_list6 = []
else:
# 虽然数组个数没有100，但接受数据时间超过了10分钟，就批量存入msyql
time_c = get_time() - start_time6
if time_c > 300:
if len(data_list6) > 0:
try:
dbpool.Pool().update_many(sql,
'shopee_sub_product update',
data_list6)
except Exception:
traceback.print_exc()
for product in data_list6:
cache_sub_product_update.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count = 0 start_time
count6 = 0
start_time6 = get_time()
data_list6 = []
def sub_product_change_insert():
start_time5 = get_time()
count5 = 0
data_list5 = []
while True:
sub_product = cache_sub_product_change_insert.get_nowait()
if sub_product is not None:
sub_product = json.loads(sub_product.decode())
data_list5.append(sub_product)
count5 += 1
else:
time.sleep(5)
# 首先判断数组内数据个数是否达到100，达到，就批量存入msyql
if count5 > 200:
try:
dbpool.Pool().insert_many_temp('shopee_sub_product_change',
data_list5,
'shopee_sub_product_change')
except Exception as e:
print('-shopee_sub_product_change---', e)
for product in data_list5:
print(product)
cache_sub_product_change_insert.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count start_time
count5 = 0
start_time5 = get_time()
data_list5 = []
else:
# 虽然数组个数没有100，但接受数据时间超过了10分钟，就批量存入msyql
time_c = get_time() - start_time5
if time_c > 300:
if len(data_list5) > 0:
try:
dbpool.Pool().insert_many_temp(
'shopee_sub_product_change', data_list5,
'shopee_sub_product_change')
except Exception:
traceback.print_exc()
for product in data_list5:
cache_sub_product_change_insert.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count = 0 start_time
count5 = 0
start_time5 = get_time()
data_list5 = []
def product_update_time():
sql = "update shopee_product set update_time=%s ,status = 1 , update_by= %s where spu= %s"
start_time2 = get_time()
count2 = 0
data_list2 = []
while True:
spu = cache_product_update_time.get_nowait()
if spu is not None:
spu = spu.decode()
now_time = datetime.datetime.now()
data_list2.append((now_time, '007', spu))
count2 += 1
else:
time.sleep(5)
# 首先判断数组内数据个数是否达到100，达到，就批量存入msyql
if count2 > 200:
try:
dbpool.Pool().update_many(sql, 'shopee_product update time',
data_list2)
except Exception:
traceback.print_exc()
for product in data_list2:
cache_product_update_time.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count start_time
count2 = 0
start_time2 = get_time()
data_list2 = []
else:
# 虽然数组个数没有100，但接受数据时间超过了10分钟，就批量存入msyql
time_c = get_time() - start_time2
if time_c > 100:
if len(data_list2) > 0:
try:
dbpool.Pool().update_many(
sql, 'shopee_product update time', data_list2)
except Exception:
traceback.print_exc()
for product in data_list2:
cache_product_update_time.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count = 0 start_time
count2 = 0
start_time2 = get_time()
data_list2 = []
def product_insert():
start_time0 = get_time()
count0 = 0
data_list0 = []
while True:
product = cache_product_insert.get_nowait()
if product is not None:
product = json.loads(product.decode())
product = trans_product_entity(product)
data_list0.append(product)
count0 += 1
else:
time.sleep(10)
# 首先判断数组内数据个数是否达到100，达到，就批量存入msyql
if count0 > 1:
try:
dbpool.Pool().insert_many_temp('shopee_product', data_list0,
'shopee_product')
except Exception as e:
print('-shopee_product---', e)
for product in data_list0:
cache_product_insert.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count start_time
count0 = 0
start_time0 = get_time()
data_list0 = []
else:
# 虽然数组个数没有100，但接受数据时间超过了10分钟，就批量存入msyql
time_c = get_time() - start_time0
if time_c > 300:
if len(data_list0) > 0:
try:
dbpool.Pool().insert_many_temp('shopee_product',
data_list0,
'shopee_product')
except Exception:
traceback.print_exc()
for product in data_list0:
cache_product_insert.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count = 0 start_time
count0 = 0
start_time0 = get_time()
data_list0 = []
def push_search():
pool = dbpool.Pool()
result = dbpool.Pool().get_all(
"select * from shopee_product_copy1 where create_by is null ")
for item in result:
shopid = item['shopid']
spu = item['spu']
task = {
"shopid": shopid,
"webid": 1,
"country": "PH",
"parse_type": "goods",
"level": 1,
"spu": spu
}
print(task)
queue_shopee_search.put(json.dumps(dict(task), cls=DateEnconding))
# product(task)
sql = "update shopee_product_copy1 set create_by=1 where shopid='{}' and spu='{}'".format(
shopid, spu)
print(sql)
pool.update(sql)
def shop_insert():
# 比较少，不做批量处理，有就直接存储
while True:
shop = cache_shop_insert.get_nowait()
if shop is not None:
shop = json.loads(shop.decode())
try:
dbpool.Pool().insert_temp('shopee_shope', shop, 'shopee_shope')
except Exception:
traceback.print_exc()
cache_shop_insert.put(json.dumps(dict(shop),
cls=DateEnconding))
else:
time.sleep(10)
def search(task):
sort = task['sort']
keyword = task['keyword']
page = task['page']
try:
url = "https://ph.xiapibuy.com/api/v2/search_items/?by={}&keyword={}&limit=50&newest={}&order=desc&page_type=search&version=2".format(
sort, keyword, (page - 1) * 50)
# shipeeed from过滤条件 Overseas
# url = "https://ph.xiapibuy.com/api/v2/search_items/?by={}&keyword={}&limit=50&locations=-2&newest=0&order=desc&page_type=search&skip_autocorrect=1&version=2".format(sort,keyword)
# print(url)
html = get_html(url)
# print(html)
json_str = json.loads(html)
search_list = []
num = 1
for item in json_str['items']:
search = trans_search(task)
search['position'] = '{}-{}'.format(page, num)
shopid = item['shopid']
search['shopid'] = shopid
search['create_time'] = datetime.datetime.now()
# search['update_time']= datetime.datetime.now()
search['spu'] = item['itemid']
search['historical_sold'] = item['historical_sold']
search['sold'] = item['sold']
search['shipping_from'] = item['shop_location']
search['ad'] = item['adsid']
try:
adsid = item['adsid']
if adsid is not None:
search['ad'] = 1
else:
search['ad'] = 0
except Exception:
search['ad'] = 0
num += 1
search_list.append(search)
redis_db.hset('shops', shopid, 0)
dbpool.Pool().insert_many_temp('shopee_search', search_list,
'shopee_search')
except Exception as e:
traceback.print_exc()
print('search异常--', e)
queue_shopee_search.put(json.dumps(dict(task), cls=DateEnconding))
print('存入redis队列')
def shop_update_time():
# 比较少，不做批量处理，有就直接存储
while True:
shopid = cache_shop_update_time.get_nowait()
if shopid is not None:
shopid = shopid.decode()
try:
sql = "update shopee_shope set update_time=%s , update_by= %s where shopid= %s"
now_time = datetime.datetime.now()
dbpool.Pool().update(sql, (now_time, '007', shopid))
except Exception:
traceback.print_exc()
cache_shop_update_time.put(shopid)
else:
time.sleep(10)
def shop_update():
# 比较少，不做批量处理，有就直接存储
while True:
shop = cache_shop_update.get_nowait()
if shop is not None:
shop = json.loads(shop.decode())
try:
sql = "update shopee_shope set " \
"shopname=%s ," \
"icon=%s ," \
"products=%s ," \
"followers=%s ," \
"following=%s ," \
"rating_num=%s ," \
"rating=%s ," \
"chatperformance=%s ," \
" abstract=%s ," \
" update_time=%s ," \
" update_by= %s" \
" where shopid= %s"
abstract = None
try:
abstract = shop['abstract']
except Exception:
pass
params = (shop['shopname'], shop['icon'], shop['products'],
shop['followers'], shop['following'],
shop['rating_num'], shop['rating'],
shop['chatperformance'], abstract,
datetime.datetime.now(), '007', shop['shopid'])
dbpool.Pool().update(sql, params)
except Exception:
traceback.print_exc()
cache_shop_update.put(json.dumps(dict(shop),
cls=DateEnconding))
else:
time.sleep(10)
def product_update():
sql = "update shopee_product set " \
"historical_sold=%s ," \
"price=%s ," \
"height_price=%s ," \
"product_title=%s ," \
" quantity=%s ," \
" status = 1 ," \
" update_time=%s ," \
" update_by= %s" \
" where spu= %s"
start_time3 = get_time()
count3 = 0
data_list3 = []
while True:
product = cache_product_update.get_nowait()
if product is not None:
product = json.loads(product.decode())
params = (product['historical_sold'], product['price'],
product['height_price'], product['product_title'],
product['quantity'], datetime.datetime.now(), '007',
str(product['spu']))
data_list3.append(params)
count3 += 1
else:
time.sleep(5)
# 首先判断数组内数据个数是否达到100，达到，就批量存入msyql
if count3 > 100:
try:
dbpool.Pool().update_many(sql, 'shopee_product update ',
data_list3)
except Exception as e:
print('-shopee_product update---', e)
for product in data_list3:
cache_product_update.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count start_time
count3 = 0
start_time3 = get_time()
data_list3 = []
else:
# 虽然数组个数没有100，但接受数据时间超过了10分钟，就批量存入msyql
time_c = get_time() - start_time3
if time_c > 300:
if len(data_list3) > 0:
try:
dbpool.Pool().update_many(sql,
'shopee_product update ',
data_list3)
except Exception:
traceback.print_exc()
for product in data_list3:
cache_product_update.put(
json.dumps(dict(product), cls=DateEnconding))
# 重置 count = 0 start_time
count3 = 0
start_time3 = get_time()
data_list3 = []
import json
import sys
import os
import time
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
rootPath = os.path.split(rootPath)[0]
rootPath = os.path.split(rootPath)[0]
sys.path.append(rootPath)
from by.pipline import dbpool
from by.pipline.redisclient import RedisClient
from by.pipline.redisqueue import RedisQueue
from by.utils.tools import DateEnconding
db = dbpool.Pool()
queue_shopee_search = RedisQueue('shopee_search', 'mz')
queue_shopee = RedisQueue('shopee', 'mz')
redis_db = RedisClient()
# 重新同步mysql 和redis shop
def update_shop_to_redis():
result = db.get_all("select distinct(shopid) from shopee_search ")
for item in result:
shopid = item['shopid']
redis_db.hset('shops',shopid,0)
# 将搜索词推送到search搜索任务队列中
def push_search():