def main():
mysql = Mysql()
R = RedisClient(SKU_NAME)
list_wdetail = []
list_prop = []
list_changes = []
list_skuinfo = []
list_shopinfo = []
list_shopscore = []
items = R.r.llen('skuinfo:items')
print(items)
for i in range(0, items):
item = R.r.blpop('skuinfo:items')
# print(item[1])
contents = json.loads(item[1].decode('utf-8'))
change_set = get_change(contents)
list_changes.append(tuple(change_set))
sku_list = get_sku(contents)
if sku_list != None:
for sku in sku_list:
list_skuinfo.append(tuple(sku))
sql_change = "insert into source_taobao_goods_change(itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s)"
mysql.insert_sql(sql_change, list_changes)
sql_sku = "insert into source_taobao_goods_skuinfo(itemId,quantity,price,propPath,updatetime)values(%s,%s,%s,%s,%s)"
mysql.insert_sql(sql_sku, list_skuinfo)
mysql.close_db()
def process_item(self, item, Spider):
if isinstance(item, shopitem):
list = []
mysql = Mysql()
print(item['shopinfo'])
list.append(item['shopinfo'])
sql_shop = "replace into source_taobao_goods_shopinfo(userId,shopId,shopName,shopIcon,fans,sellerType,shopType,shopage,goodRatePercentage,sellerNick,creditLevel,describe1,service,logistics,deposittime,maintype)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
mysql.insert_sql(sql_shop, list)
mysql.close_db()
return item
def main():
mysql = Mysql()
list_changes = []
R = RedisClient(SKU_NAME)
items = R.r.llen('skuinfo:items')
print(items)
for i in range(0, items):
item = R.r.blpop('skuinfo:items')
# print(item[1])
contents = json.loads(item[1].decode('utf-8'))
change_set = get_change(contents)
# print(change_set)
list_changes.append(tuple(change_set))
sql_change = "insert into source_taobao_goods_quantity(accountId,itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s,%s)"
mysql.insert_sql(sql_change, list_changes)
mysql.close_db()
def insertsql(self):
mysql = Mysql()
R = RedisClient(DETAIL_NAME)
list_detail = []
list_prop = []
list_changes = []
list_skuinfo = []
list_shopinfo = []
list_shopscore = []
items = R.r.llen('detail:items')
print(items)
for i in range(0, items):
item = R.r.blpop('detail:items', timeout=5)
contents = json.loads(item[1])
list_detail.append(self.get_detail(contents))
changeinfo = self.get_change(contents)
if changeinfo != None:
list_changes.append(tuple(changeinfo))
propinfo = self.get_prop(contents)
if propinfo != None:
for prop in propinfo:
list_prop.append(tuple(prop))
sku_list = self.get_sku(contents)
if sku_list != None:
for sku in sku_list:
list_skuinfo.append(tuple(sku))
# list_shopinfo.append(get_shop(contents))
sql_detail = "replace into source_taobao_goods_detail(itemId,shopId,sellerId,title,headimg,itemprice,tb_state,rootCategoryId,categoryId,deposittime,shopType)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
mysql.insert_sql(sql_detail, list_detail)
# sql_prop = "replace into source_taobao_goods_prop_0621(itemId,pid,pidname,vid,vidname,deposittime)values(%s,%s,%s,%s,%s,%s)"
# mysql.insert_sql(sql_prop, list_prop)
# sql_change = "insert into source_taobao_goods_change_0614(itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s)"
# mysql.insert_sql(sql_change, list_changes)
# sql_sku = "insert into source_taobao_goods_skuinfo(itemId,skuId,quantity,price,propPath,updatetime)values(%s,%s,%s,%s,%s,%s)"
# mysql.insert_sql(sql_sku, list_skuinfo)
mysql.close_db()