def main(): mysql = Mysql() R = RedisClient(SKU_NAME) list_wdetail = [] list_prop = [] list_changes = [] list_skuinfo = [] list_shopinfo = [] list_shopscore = [] items = R.r.llen('skuinfo:items') print(items) for i in range(0, items): item = R.r.blpop('skuinfo:items') # print(item[1]) contents = json.loads(item[1].decode('utf-8')) change_set = get_change(contents) list_changes.append(tuple(change_set)) sku_list = get_sku(contents) if sku_list != None: for sku in sku_list: list_skuinfo.append(tuple(sku)) sql_change = "insert into source_taobao_goods_change(itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s)" mysql.insert_sql(sql_change, list_changes) sql_sku = "insert into source_taobao_goods_skuinfo(itemId,quantity,price,propPath,updatetime)values(%s,%s,%s,%s,%s)" mysql.insert_sql(sql_sku, list_skuinfo) mysql.close_db()
def process_item(self, item, Spider): if isinstance(item, shopitem): list = [] mysql = Mysql() print(item['shopinfo']) list.append(item['shopinfo']) sql_shop = "replace into source_taobao_goods_shopinfo(userId,shopId,shopName,shopIcon,fans,sellerType,shopType,shopage,goodRatePercentage,sellerNick,creditLevel,describe1,service,logistics,deposittime,maintype)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" mysql.insert_sql(sql_shop, list) mysql.close_db() return item
def main(): mysql = Mysql() list_changes = [] R = RedisClient(SKU_NAME) items = R.r.llen('skuinfo:items') print(items) for i in range(0, items): item = R.r.blpop('skuinfo:items') # print(item[1]) contents = json.loads(item[1].decode('utf-8')) change_set = get_change(contents) # print(change_set) list_changes.append(tuple(change_set)) sql_change = "insert into source_taobao_goods_quantity(accountId,itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s,%s)" mysql.insert_sql(sql_change, list_changes) mysql.close_db()
def insertsql(self): mysql = Mysql() R = RedisClient(DETAIL_NAME) list_detail = [] list_prop = [] list_changes = [] list_skuinfo = [] list_shopinfo = [] list_shopscore = [] items = R.r.llen('detail:items') print(items) for i in range(0, items): item = R.r.blpop('detail:items', timeout=5) contents = json.loads(item[1]) list_detail.append(self.get_detail(contents)) changeinfo = self.get_change(contents) if changeinfo != None: list_changes.append(tuple(changeinfo)) propinfo = self.get_prop(contents) if propinfo != None: for prop in propinfo: list_prop.append(tuple(prop)) sku_list = self.get_sku(contents) if sku_list != None: for sku in sku_list: list_skuinfo.append(tuple(sku)) # list_shopinfo.append(get_shop(contents)) sql_detail = "replace into source_taobao_goods_detail(itemId,shopId,sellerId,title,headimg,itemprice,tb_state,rootCategoryId,categoryId,deposittime,shopType)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" mysql.insert_sql(sql_detail, list_detail) # sql_prop = "replace into source_taobao_goods_prop_0621(itemId,pid,pidname,vid,vidname,deposittime)values(%s,%s,%s,%s,%s,%s)" # mysql.insert_sql(sql_prop, list_prop) # sql_change = "insert into source_taobao_goods_change_0614(itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s)" # mysql.insert_sql(sql_change, list_changes) # sql_sku = "insert into source_taobao_goods_skuinfo(itemId,skuId,quantity,price,propPath,updatetime)values(%s,%s,%s,%s,%s,%s)" # mysql.insert_sql(sql_sku, list_skuinfo) mysql.close_db()