def getShopListFromTaobaoSoup(driverContent): soup = BeautifulSoup(driverContent, 'lxml') # soup = soup.find('shopsearch-shoplist') soup = soup.find('ul', id='list-container') if soup == None: return 'Login Error' pattern = re.compile(r'&fromid=.+') for item in soup.select(".list-info"): try: shopH4 = item.find('h4') shopName = str(shopH4.a.string).replace(' ','').replace('\n','') shopUrl = 'https:' + shopH4.a['href'] shopInfoList = item.select('.shop-info-list')[0] sellerName = str(shopInfoList.a.string).replace(' ','').replace('\n','') sellerUrl = 'https:' + shopInfoList.a['href'] sellerUid = str(shopInfoList.a['trace-uid']).replace(' ','').replace('\n','') wangwangInfo = item.select('.ww-inline')[0] sellerWangWangIsOnline = str(wangwangInfo.span.string).replace(' ','').replace('\n','') sellerWangWangUrl = str(wangwangInfo['href']).replace(pattern.findall(wangwangInfo['href'])[0], '') valuation = item.select('.valuation')[0] goodCommentRatio = str(valuation.select('.good-comt')[0].string).replace(' ','').replace('\n','') isConsumerInsure = str(valuation).find('消费者保障') != -1 isGoldenSeller = str(valuation).find('金牌卖家') != -1 except IndexError: raise("Taobao Data Grasp Error") except AttributeError: raise("CSS Selector Error") shop = Shops(shopName=shopName, shopURL=shopUrl, sellerName=sellerName, \ sellerURL=sellerUrl, sellerUid=sellerUid,sellerWangWangOnlineStatus = sellerWangWangIsOnline,\ sellerWangWangURL=sellerWangWangUrl, goodCommentRatio=goodCommentRatio,\ isConsumerInsure=isConsumerInsure, isGoldSeller =isGoldenSeller) session = SessionLocal() isSellerExist = session.query(Shops).filter(Shops.sellerUid == sellerUid).first() if not isSellerExist: session.add(shop) session.commit() session.close() return 0
def getShopInfoFromKDBSoup(wangwangName, driverContent): soup = BeautifulSoup(driverContent, 'lxml') mt10 = soup.select('.mt10')[0].find('tbody') mt10Dic = { '店铺粉丝数':0, '所在地区': None, '店铺宝贝数': 0, '创店时间': None, 'DSR': 0, '主营类目': None } for row in mt10.find_all('tr'): for col in row.children: curGrid = str(col.string) if curGrid in mt10Dic: for elem in col.next_siblings: if str(elem).find('<td>') != -1: allString = '' for string in elem.stripped_strings: allString += string mt10Dic[curGrid] = allString break pattern = re.compile(r'[\d]+') mt10Dic['店铺宝贝数'] = pattern.findall(mt10Dic['店铺宝贝数'])[0] if mt10Dic['所在地区'] == '': mt10Dic['所在地区'] = '-' session = SessionLocal() shop = session.query(Shops).filter(Shops.sellerName == wangwangName).first() shop.numberOfFans = mt10Dic['店铺粉丝数'] shop.shopZone = mt10Dic['所在地区'] shop.numberOfItems = mt10Dic['店铺宝贝数'] shop.openingDate = (datetime.strptime(mt10Dic['创店时间'], '%Y-%m-%d %H:%M:%S')) shop.dsr_value = mt10Dic['DSR'] shop.shopCategory = mt10Dic['主营类目'] session.commit() return 0