def add_contact(name, email, theme, question): Mongo.insert('journalist', { 'name': name, 'email': email, 'theme': theme, 'question': question })
def add_file(title, description, file, datetime): Mongo.insert( 'img', { 'title': title, 'file': file, 'description': description, 'datetime': datetime })
class GoodsList: def __init__(self): self.count = 0 db_name = 'bayan' self.collection_name = 'JDGoodsUrl' self.db = Mongo(db_name) def get_list(self, list): urllist = [] headers = { 'referer': 'https://search.jd.com/Search?keyword=minecraft&enc=utf-8&pvid=b55d6cb7986748d6a32da02876cc9874', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36' } del list['_id'] pages = list['pages'] pages = pages*2+1 for page in range(pages)[1:]: url = 'https://search.jd.com/s_new.php?keyword={}&psort=3&enc=utf-8&page={}&s={}&scrolling=y&log_id={}'.format(list['keyword'], page, page*31, int(time.time())) response = requests.get(url, headers=headers).text selector = Selector(text=response) goods = selector.xpath('//li[@class="gl-item"]') for good in goods: goodsurl = good.xpath('.//a[@target="_blank"]/@href').extract()[0] list['pageUrl'] = 'https:' + goodsurl print(list) urllist.append(deepcopy(list)) self.count = len(urllist) back = self.check(list['count'], urllist) if back == 'success': print(list['keyword'] + ":成功") return urllist else: print(list['keyword'] + ":失败,重新抓取") self.get_list(list) def check(self, count, urllist): if self.count == count: self.db.insert(self.collection_name, urllist) self.count = 0 return 'success' def run(self): self.db.drop(self.collection_name) lists = AllNumber().get_number() for list in lists: print("正在抓取关键词:" + list['keyword']) GoodsList().get_list(list) time.sleep(1)
class GoodsList: def __init__(self): db_name = 'bayan' self.save_collection_name = 'TMGoodsUrl' self.collection_name = 'TMkey' self.db = Mongo(db_name) def get_list(self, list): urllist = [] headers = { 'referer': 'http://list.tmall.com/search_product.htm?q={}&type=p&spm=a220m.6910245.a2227oh.d100&from=mallfp..m_1_searchbutton&sort=d' .format(quote(list['keyword'], encoding="gbk")), 'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', 'cookie': 'cna=l9oDFcy6VQ4CAd6AqijwI13O; _med=dw:1366&dh:768&pw:1366&ph:768&ist:0; lid=%E8%91%AC%E4%BB%AA%E4%B8%BF%E5%A4%9C%E7%A5%9E%E6%9C%88; otherx=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0; x=__ll%3D-1%26_ato%3D0; _uab_collina=156877667696726520994751; hng=CN%7Czh-CN%7CCNY%7C156; enc=twxIgD2w8bZQSql4cagTND22VE%2FhUTEOaq2XkcEtDvxxkz37BO5Mh25gdOoNdNoJF5i9aTpzn%2BrzEdT6wQL1qA%3D%3D; sm4=110100; _m_h5_tk=b33074b6753c714e2c5a32fd78d6c426_1571630922757; _m_h5_tk_enc=db58b55df9684e14b5c1aaef72e5c979; t=68aeed5a9ead7edb2d26b8d916cdf5be; _tb_token_=33b56e71b803e; cookie2=1d0efb3fed419cc8c79c62ce27633524; dnk=%5Cu846C%5Cu4EEA%5Cu4E3F%5Cu591C%5Cu795E%5Cu6708; cq=ccp%3D0; tracknick=%5Cu846C%5Cu4EEA%5Cu4E3F%5Cu591C%5Cu795E%5Cu6708; _l_g_=Ug%3D%3D; unb=2646574036; lgc=%5Cu846C%5Cu4EEA%5Cu4E3F%5Cu591C%5Cu795E%5Cu6708; cookie1=BxpRR3m3mq6u2SKR8tMIAV5PbfT0Mkqa7hIMcGbyJO8%3D; login=true; cookie17=UU6lS5IHpNO1Zw%3D%3D; _nk_=%5Cu846C%5Cu4EEA%5Cu4E3F%5Cu591C%5Cu795E%5Cu6708; sg=%E6%9C%886c; uc1=cookie14=UoTbnKU5vO8R1A%3D%3D&cookie16=V32FPkk%2FxXMk5UvIbNtImtMfJQ%3D%3D&pas=0&cookie15=Vq8l%2BKCLz3%2F65A%3D%3D&lng=zh_CN&existShop=false&tag=8&cookie21=U%2BGCWk%2F7p4mBoUyS4E9C; uc3=id2=UU6lS5IHpNO1Zw%3D%3D&nk2=tzejKGxa%2FgjcE9Gg&vt3=F8dByuckA3AzTRgxjWY%3D&lg2=VT5L2FSpMGV7TQ%3D%3D; uc4=nk4=0%40tUQ6%2FECahntTXqHnI5ioo65kDQrdQ7Y%3D&id4=0%40U2xo%2B4EAVHijItFSb4zrqlzll9lp; csg=66cb9310; x5sec=7b22746d616c6c7365617263683b32223a226433303262376437326166623365333865613263653965323938316134613934434a5846744f3046455048736b71766867504c3234674561444449324e4459314e7a51774d7a59374d673d3d227d; pnm_cku822=098%23E1hvevvUvbpvUvCkvvvvvjiPRszOAj1En2sWljD2PmPpsjrUnLSU1jE2PFMWlj3vRphvChCvvvvPvpvhvv2MMQhCvvXvovvvvvvEvpCWpSuUv8ROjovDN%2BClHdUf8B69D70Ode%2BRVA3l%2Bb8rwAtYmq0DW3CQcmx%2Fsj7J%2B3%2BijLjEIEkffvyf8j7yHdBYLjnv6nQ7RAYVEvLvq8yCvv3vpvolaufqRIyCvvXmp99he1KtvpvIphvvvvvvphCvpCBXvvCCN6CvHHyvvhn2phvZ7pvvpiivpCBXvvCmeuwCvvBvpvpZ; res=scroll%3A1349*5314-client%3A1349*318-offset%3A1349*5314-screen%3A1366*768; isg=BBkZMDWmoCVFC3xjiPXyRrtzKAUzDg1k1P50YDvO3sC_QjjUg_W7KLoQREaRf6WQ; l=dBLiViNPqB8DGUOzBOCZZuI8amQTKIRbSuPRwN4pi_5CG68_WbQOkM1H9FJ6cjWAGn8B4JuaUMvTCFJgJsl0NE8xDfpFlkM2B' } for page in range(list['page']): url = 'http://list.tmall.com/m/search_items.htm?page_size=20&page_no={}&q={}&type=p&sort=d'.format( page + 1, quote(list['keyword'], encoding="gbk")) print(url) response = requests.get(url, headers=headers) print(response.text) datas = json.loads(response.text)['item'] for data in datas: list['productName'] = data['title'] list['pageUrl'] = 'https:' + data['url'] print(list) urllist.append(deepcopy(list)) time.sleep(10) if urllist != []: self.db.insert(self.save_collection_name, urllist) print('抓取数量:' + str(len(urllist)), '显示数量:' + str(list['count'])) else: print(list['keyword'] + '无商品') def run(self): # self.db.drop(self.save_collection_name) keys = self.db.get(self.collection_name) i = 0 for key in keys: print(i) i += 1 list = AllNumber().get_number(key) print("正在抓取关键词:" + list['keyword']) GoodsList().get_list(list) time.sleep(10)
def add_reservation(name, email, date): Mongo.insert('reservation', {'name': name, 'email': email, 'date': date})
def add_contact(email): Mongo.insert('dognews', {'email': email})
class GoodsContent: def __init__(self): db_name = 'bayan' self.collection_name = 'TMGoodsUrl' self.save_collection_name = 'TMGoodsData' self.db = Mongo(db_name) self.conn = connRedis.OPRedis() def get(self, item): goods_id = re.findall(r'id=(\d+)', item['pageUrl']) item['pageUrl'] = 'https://detail.tmall.com/item.htm?id={}'.format( goods_id[0]) item['_id'] = urlmd5(item['pageUrl'] + item['keyword']) item['productId'] = int(goods_id[0]) item['custom'] = 2 # 平台 item['platformType'] = 2 url = 'https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=%7B%22itemNumId%22%3A%22{}%22%7D'.format( goods_id[0]) response = requests.get(url) print(response.url) data = json.loads(response.text)['data'] dataitem = data['item'] dataprops = data['props'] apiStack = json.loads(data['apiStack'][0]['value']) try: item['productParam'] = dataprops['groupProps'][0]['基本信息'][12][ '食品口味'] # 商品规格 except: item['productParam'] = '' item['currentPrice'] = apiStack['price']['price']['priceText'] try: item['originalPrice'] = apiStack['price']['extraPrices'][0][ 'priceText'] # 原价 except IndexError as e: try: item['originalPrice'] = apiStack['price']['transmitPrice'][ 'priceText'] except: item['originalPrice'] = item['current_price'] except KeyError as e: try: item['originalPrice'] = apiStack['price']['transmitPrice'][ 'priceText'] except: item['originalPrice'] = item['current_price'] try: item['salesNumMonth'] = int(apiStack['item']['sellCount']) # 月销量 except: item['salesNumMonth'] = 0 item['servicePromise'] = '' # 服务承诺 servicePromise = apiStack['consumerProtection']['items'] for prom in servicePromise: item['servicePromise'] += prom['title'] + ' ' item['paymentInformation'] = '信用卡快捷支付蚂蚁花呗余额宝' # 支付方式 item['stockNum'] = int( apiStack['skuCore']['sku2info']['0']['quantity']) # 库存 if item['stockNum']: item['stock'] = '有货' else: item['stock'] = '无货' item['shopName'] = data['seller']['shopName'] # 店铺名称 item['shopUrl'] = "https://shop{}.taobao.com".format( data['seller']['shopId']) # 店铺链接 item['shopId'] = data['seller']['shopId'] item['shopScore'] = [] # 店铺评分 for shop in data['seller']['evaluates']: item['shopScore'].append(shop['title']) item['shopScore'].append(shop['score']) item['commentsCount'] = dataitem['commentCount'] # 评论数量 item['collectionNum'] = dataitem['favcount'] # 收藏数量 item['departureAddress'] = apiStack['delivery']['from'] # 发货地 item['promotion'] = '' # 商品促销信息 try: shopProm = apiStack['price']['shopProm'] except: shopProm = [] for prom in shopProm: for con in prom['content']: if '登录' in con: pass else: item['promotion'] += con + ',' try: pram = apiStack['consumerProtection']['channel'][ 'title'] # 是否聚划算 商品 except: pram = '' item['promotion'] += pram item['crawlTime'] = int(time.time() * 1000) craw_date = time.localtime(item['crawlTime'] / 1000) craw_date = time.strftime("%Y-%m-%d", craw_date) url = item['shopId'] + str( item['productId']) + craw_date + item['platform'] item['connectGoodsId'] = urlmd5(url) item['productSkuDetail'] = [] # 商品sku 详情 try: skuprops = data['skuBase']['props'] except KeyError as e: skuprops = [] try: sku2info = data['skuBase']['skus'] except KeyError as e: sku2info = [] for sku2 in sku2info: sku_price = apiStack['skuCore']['sku2info'][ sku2['skuId']]['price']['priceText'] sku_stock = apiStack['skuCore']['sku2info'][ sku2['skuId']]['quantity'] propPath = sku2['propPath'].split(";")[-1].split(":") for prop in skuprops: if prop['pid'] == propPath[0]: for col in prop['values']: if col['vid'] == propPath[1]: sku_name = col['name'] item['productParam'] += sku_name + ' ' item['productSkuDetail'].append({ 'sku_id': sku2['skuId'], 'sku_name': sku_name, 'sku_price': sku_price, 'sku_stock': sku_stock }) comments, crawlCommentsTime = GoodsComment().get(item, 1) item['commentsData'] = comments item['crawlCommentsCount'] = len(comments) print('评论数:' + str(len(comments))) item['crawlCommentsTime'] = crawlCommentsTime print(item) def run(self): i = 0 items = self.db.get(self.collection_name) for item in items: print(i) goodsitem = self.get(item) self.db.insert(self.save_collection_name, goodsitem) i += 1