def setlist(self):
url = 'https://acs.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=%7B%22itemNumId%22%3A%22{id}%22%7D'
mysql = Mysql()
R = RedisClient(SHOP_NAME)
items = mysql.get_shoplist()
for item in items:
R.set_url(item, url)
def set_list(self):
url = 'https://detailskip.taobao.com/service/getData/1/p1/item/detail/sib.htm?itemId={id}&modules=dynStock,price,xmpPromotion,originalPrice&accountid={id2}'
sql = 'SELECT itemId,accountId FROM `source_taobao_live_product_now` WHERE ishot = \'false\''
mysql = Mysql()
R = RedisClient(SKU_NAME)
items = mysql.get_two(sql)
for item in items:
R.set_two_url(item[0], item[1], url)
def main():
mysql = Mysql()
R = RedisClient(SKU_NAME)
list_wdetail = []
list_prop = []
list_changes = []
list_skuinfo = []
list_shopinfo = []
list_shopscore = []
items = R.r.llen('skuinfo:items')
print(items)
for i in range(0, items):
item = R.r.blpop('skuinfo:items')
# print(item[1])
contents = json.loads(item[1].decode('utf-8'))
change_set = get_change(contents)
list_changes.append(tuple(change_set))
sku_list = get_sku(contents)
if sku_list != None:
for sku in sku_list:
list_skuinfo.append(tuple(sku))
sql_change = "insert into source_taobao_goods_change(itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s)"
mysql.insert_sql(sql_change, list_changes)
sql_sku = "insert into source_taobao_goods_skuinfo(itemId,quantity,price,propPath,updatetime)values(%s,%s,%s,%s,%s)"
mysql.insert_sql(sql_sku, list_skuinfo)
mysql.close_db()
class SkuSpiderPipeline(object):
def open_spider(self, Spider):
self.mysql = Mysql()
def close_spider(self, Spider):
self.mysql.close_db()
def process_item(self, item, Spider):
if isinstance(item, SkuinfoItem):
sql_detail = "insert into source_taobao_goods_change_0622_{id}(itemId,itemprice,quantity,deposittime)values('{itemId}','{itemprice}','{quantity}','{deposittime}')".format(
id=item['account'],
itemId=item['itemId'],
itemprice=item['itemprice'],
quantity=item['quantity'],
deposittime=item['time'],
)
self.mysql.insert_one(sql_detail)
return item
def process_item(self, item, Spider):
if isinstance(item, shopitem):
list = []
mysql = Mysql()
print(item['shopinfo'])
list.append(item['shopinfo'])
sql_shop = "replace into source_taobao_goods_shopinfo(userId,shopId,shopName,shopIcon,fans,sellerType,shopType,shopage,goodRatePercentage,sellerNick,creditLevel,describe1,service,logistics,deposittime,maintype)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
mysql.insert_sql(sql_shop, list)
mysql.close_db()
return item
def main():
mysql = Mysql()
list_changes = []
R = RedisClient(SKU_NAME)
items = R.r.llen('skuinfo:items')
print(items)
for i in range(0, items):
item = R.r.blpop('skuinfo:items')
# print(item[1])
contents = json.loads(item[1].decode('utf-8'))
change_set = get_change(contents)
# print(change_set)
list_changes.append(tuple(change_set))
sql_change = "insert into source_taobao_goods_quantity(accountId,itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s,%s)"
mysql.insert_sql(sql_change, list_changes)
mysql.close_db()
def parse(self, response):
try:
item = TbGoodsItem()
item['time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
type = re.search('TShop.Setup\((.*?)\);', response.text, re.S)
doc = pq(response.text)
if type != None:
content = json.loads(type.group(1).strip())
item['tb_state'] = content['itemDO']['auctionStatus']
if item['tb_state'] == '0':
self.tmall_detail(content, item, doc)
self.get_skuinfo(content, item, type='B')
self.get_prop(doc, item)
# self.tmall_shop(doc, item)
yield item
if item['tb_state'] == '-2':
self.tmall_detail(content, item, doc)
self.get_skuinfo(content, item, type='B')
# self.tmall_shop(doc, item)
yield item
elif doc('.error-notice-hd'):
print('商品下架不存在')
id = re.search('&itemid=(\d+)&', response.url).group(1)
sql = 'replace into `source_taobao_live_itemId_drop` (itemId) values ({id})'.format(
id=id)
mysql = Mysql()
mysql.insert_one(sql)
mysql.close_db()
else:
self.taobao_detail(response, item)
# self.get_shop(response, item)
if item['tb_state'] == '0':
self.get_skuinfo(response, item)
self.get_prop(doc, item)
yield item
else:
item['price'] = re.search(
'name="current_price" value= "(.*?)"/>',
response.text).group(1)
yield item
except Exception as e:
with open('error_0612.txt', 'a+') as f:
f.write(response.url + '\n')
f.write('error: %s \n' % e)
def insertsql(self):
mysql = Mysql()
R = RedisClient(DETAIL_NAME)
list_detail = []
list_prop = []
list_changes = []
list_skuinfo = []
list_shopinfo = []
list_shopscore = []
items = R.r.llen('detail:items')
print(items)
for i in range(0, items):
item = R.r.blpop('detail:items', timeout=5)
contents = json.loads(item[1])
list_detail.append(self.get_detail(contents))
changeinfo = self.get_change(contents)
if changeinfo != None:
list_changes.append(tuple(changeinfo))
propinfo = self.get_prop(contents)
if propinfo != None:
for prop in propinfo:
list_prop.append(tuple(prop))
sku_list = self.get_sku(contents)
if sku_list != None:
for sku in sku_list:
list_skuinfo.append(tuple(sku))
# list_shopinfo.append(get_shop(contents))
sql_detail = "replace into source_taobao_goods_detail(itemId,shopId,sellerId,title,headimg,itemprice,tb_state,rootCategoryId,categoryId,deposittime,shopType)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
mysql.insert_sql(sql_detail, list_detail)
# sql_prop = "replace into source_taobao_goods_prop_0621(itemId,pid,pidname,vid,vidname,deposittime)values(%s,%s,%s,%s,%s,%s)"
# mysql.insert_sql(sql_prop, list_prop)
# sql_change = "insert into source_taobao_goods_change_0614(itemId,itemprice,quantity,deposittime)values(%s,%s,%s,%s)"
# mysql.insert_sql(sql_change, list_changes)
# sql_sku = "insert into source_taobao_goods_skuinfo(itemId,skuId,quantity,price,propPath,updatetime)values(%s,%s,%s,%s,%s,%s)"
# mysql.insert_sql(sql_sku, list_skuinfo)
mysql.close_db()
def create_table(self, sql):
mysql = Mysql()
mysql.create_table(sql)
def open_spider(self, Spider):
self.mysql = Mysql()