def setlist(self): url = 'https://acs.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=%7B%22itemNumId%22%3A%22{id}%22%7D' mysql = Mysql() R = RedisClient(SHOP_NAME) items = mysql.get_shoplist() for item in items: R.set_url(item, url)
def set_list(self): url = 'https://detailskip.taobao.com/service/getData/1/p1/item/detail/sib.htm?itemId={id}&modules=dynStock,price,xmpPromotion,originalPrice&accountid={id2}' sql = 'SELECT itemId,accountId FROM `source_taobao_live_product_now` WHERE ishot = \'false\'' mysql = Mysql() R = RedisClient(SKU_NAME) items = mysql.get_two(sql) for item in items: R.set_two_url(item[0], item[1], url)
def main(): mysql = Mysql() R = RedisClient(SKU_NAME) list_wdetail = [] list_prop = [] list_changes = [] list_skuinfo = [] list_shopinfo = [] list_shopscore = [] items = R.r.llen('skuinfo:items') print(items) for i in range(0, items): item = R.r.blpop('skuinfo:items') # print(item[1]) contents = json.loads(item[1].decode('utf-8')) change_set = get_change(contents) list_changes.append(tuple(change_set)) sku_list = get_sku(contents) if sku_list != None: for sku in sku_list: list_skuinfo.append(tuple(sku)) sql_change = "insert into source_taobao_goods_change(itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s)" mysql.insert_sql(sql_change, list_changes) sql_sku = "insert into source_taobao_goods_skuinfo(itemId,quantity,price,propPath,updatetime)values(%s,%s,%s,%s,%s)" mysql.insert_sql(sql_sku, list_skuinfo) mysql.close_db()
class SkuSpiderPipeline(object): def open_spider(self, Spider): self.mysql = Mysql() def close_spider(self, Spider): self.mysql.close_db() def process_item(self, item, Spider): if isinstance(item, SkuinfoItem): sql_detail = "insert into source_taobao_goods_change_0622_{id}(itemId,itemprice,quantity,deposittime)values('{itemId}','{itemprice}','{quantity}','{deposittime}')".format( id=item['account'], itemId=item['itemId'], itemprice=item['itemprice'], quantity=item['quantity'], deposittime=item['time'], ) self.mysql.insert_one(sql_detail) return item
def process_item(self, item, Spider): if isinstance(item, shopitem): list = [] mysql = Mysql() print(item['shopinfo']) list.append(item['shopinfo']) sql_shop = "replace into source_taobao_goods_shopinfo(userId,shopId,shopName,shopIcon,fans,sellerType,shopType,shopage,goodRatePercentage,sellerNick,creditLevel,describe1,service,logistics,deposittime,maintype)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" mysql.insert_sql(sql_shop, list) mysql.close_db() return item
def main(): mysql = Mysql() list_changes = [] R = RedisClient(SKU_NAME) items = R.r.llen('skuinfo:items') print(items) for i in range(0, items): item = R.r.blpop('skuinfo:items') # print(item[1]) contents = json.loads(item[1].decode('utf-8')) change_set = get_change(contents) # print(change_set) list_changes.append(tuple(change_set)) sql_change = "insert into source_taobao_goods_quantity(accountId,itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s,%s)" mysql.insert_sql(sql_change, list_changes) mysql.close_db()
def parse(self, response): try: item = TbGoodsItem() item['time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') type = re.search('TShop.Setup\((.*?)\);', response.text, re.S) doc = pq(response.text) if type != None: content = json.loads(type.group(1).strip()) item['tb_state'] = content['itemDO']['auctionStatus'] if item['tb_state'] == '0': self.tmall_detail(content, item, doc) self.get_skuinfo(content, item, type='B') self.get_prop(doc, item) # self.tmall_shop(doc, item) yield item if item['tb_state'] == '-2': self.tmall_detail(content, item, doc) self.get_skuinfo(content, item, type='B') # self.tmall_shop(doc, item) yield item elif doc('.error-notice-hd'): print('商品下架不存在') id = re.search('&itemid=(\d+)&', response.url).group(1) sql = 'replace into `source_taobao_live_itemId_drop` (itemId) values ({id})'.format( id=id) mysql = Mysql() mysql.insert_one(sql) mysql.close_db() else: self.taobao_detail(response, item) # self.get_shop(response, item) if item['tb_state'] == '0': self.get_skuinfo(response, item) self.get_prop(doc, item) yield item else: item['price'] = re.search( 'name="current_price" value= "(.*?)"/>', response.text).group(1) yield item except Exception as e: with open('error_0612.txt', 'a+') as f: f.write(response.url + '\n') f.write('error: %s \n' % e)
def insertsql(self): mysql = Mysql() R = RedisClient(DETAIL_NAME) list_detail = [] list_prop = [] list_changes = [] list_skuinfo = [] list_shopinfo = [] list_shopscore = [] items = R.r.llen('detail:items') print(items) for i in range(0, items): item = R.r.blpop('detail:items', timeout=5) contents = json.loads(item[1]) list_detail.append(self.get_detail(contents)) changeinfo = self.get_change(contents) if changeinfo != None: list_changes.append(tuple(changeinfo)) propinfo = self.get_prop(contents) if propinfo != None: for prop in propinfo: list_prop.append(tuple(prop)) sku_list = self.get_sku(contents) if sku_list != None: for sku in sku_list: list_skuinfo.append(tuple(sku)) # list_shopinfo.append(get_shop(contents)) sql_detail = "replace into source_taobao_goods_detail(itemId,shopId,sellerId,title,headimg,itemprice,tb_state,rootCategoryId,categoryId,deposittime,shopType)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" mysql.insert_sql(sql_detail, list_detail) # sql_prop = "replace into source_taobao_goods_prop_0621(itemId,pid,pidname,vid,vidname,deposittime)values(%s,%s,%s,%s,%s,%s)" # mysql.insert_sql(sql_prop, list_prop) # sql_change = "insert into source_taobao_goods_change_0614(itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s)" # mysql.insert_sql(sql_change, list_changes) # sql_sku = "insert into source_taobao_goods_skuinfo(itemId,skuId,quantity,price,propPath,updatetime)values(%s,%s,%s,%s,%s,%s)" # mysql.insert_sql(sql_sku, list_skuinfo) mysql.close_db()
def create_table(self, sql): mysql = Mysql() mysql.create_table(sql)
def open_spider(self, Spider): self.mysql = Mysql()