def check_stock(self, asin, reserve): if self.checkshipping: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new&f_new=true") else: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new&f_new=true&f_freeShipping=on") try: self.driver.get(f.url) elements = self.driver.find_elements_by_xpath( '//*[@name="submit.addToCart"]') prices = self.driver.find_elements_by_xpath( '//*[@class="a-size-large a-color-price olpOfferPrice a-text-bold"]' ) shipping = self.driver.find_elements_by_xpath( '//*[@class="a-color-secondary"]') except Exception as e: log.debug(e) return False for i in range(len(elements)): price = parse_price(prices[i].text) ship_price = parse_price(shipping[i].text) ship_float = ship_price.amount price_float = price.amount if price_float is None or ship_float is None: log.error("Error reading price information on row.") continue elif (price_float + ship_float) <= reserve: log.info("Item in stock and under reserve!") elements[i].click() log.info("clicking add to cart") return True return False
def check_stock(self, asin, reserve): if self.checkshipping: if self.used: f = furl(AMAZON_URLS["OFFER_URL"] + asin) else: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new&f_new=true") else: if self.used: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/f_freeShipping=on") else: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new&f_new=true&f_freeShipping=on") try: self.driver.get(f.url) elements = self.driver.find_elements_by_xpath( '//*[@name="submit.addToCart"]') prices = self.driver.find_elements_by_xpath( '//*[@class="a-size-large a-color-price olpOfferPrice a-text-bold"]' ) shipping = self.driver.find_elements_by_xpath( '//*[@class="a-color-secondary"]') except Exception as e: log.error(e) log.info("Attempting to re-open window...") self.restart_browser() return False for i in range(len(elements)): price = parse_price(prices[i].text) if SHIPPING_ONLY_IF in shipping[i].text: ship_price = parse_price("0") else: ship_price = parse_price(shipping[i].text) ship_float = ship_price.amount price_float = price.amount if price_float is None: return False if ship_float is None or not self.checkshipping: ship_float = 0 if (ship_float + price_float) <= reserve or math.isclose( (price_float + ship_float), reserve, abs_tol=0.01): log.info("Item in stock and under reserve!") elements[i].click() log.info("clicking add to cart") return True return False
def check_stock(self, asin, reserve): f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new?f_new=true") try: self.driver.get(f.url) elements = self.driver.find_elements_by_xpath( '//*[@name="submit.addToCart"]') prices = self.driver.find_elements_by_xpath( '//*[@class="a-size-large a-color-price olpOfferPrice a-text-bold"]' ) except Exception as e: log.debug(e) return False x = 0 for str_price in prices: price = parse_price(str_price.text) priceFloat = price.amount if priceFloat is None: log.error("Error reading price information on row.") x = x + 1 continue elif priceFloat <= reserve: log.info("Item in stock and under reserve!") elements[x].click() log.info("clicking add to cart") return True else: x = x + 1 return False
def check_stock(self, url, reserve): log.info("Checking stock for items.") f = furl(WALMART_URLS["OFFER_URL"] + url) self.driver.get(f.url) self.check_captcha(url) try: element = self.check_exists_by_xpath( "/html/body/div[1]/div[1]/div/div[2]/div/div[1]/div[1]/div[1]/div/div/div/div/div[3]/div[5]/div/div[3]/div/div[2]/div[2]/div[1]/section/div[1]/div[3]/button/span" ) str_price = self.driver.find_element_by_xpath('//*[@id="price"]') log.info("Found price " + str_price.text) except NoSuchElementException: return False if not element: log.info("Product not available") self.check_stock(url, reserve) price = parse_price(str_price.text) priceFloat = price.amount if priceFloat is None: log.error("Error reading price information on row.") elif priceFloat <= reserve: if element: log.info("Item in stock and under reserve!") log.info("Clicking add to cart") element.click() return True return False
def kross_kulture(string): print("functions") query = set_up_query(string) try: link = "https://krosskulture.com/search/" + str( query) + "/?type=product" req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') #print(soup.prettify()) titles = soup.find_all('div', {'class': 'post-details'}) #print(titles) prices = soup.find_all('ins') #print(prices) img_url = soup.find_all('div', {'class': 'post-thumbnail'}) #print(img_url) #item_link = soup.find_all('li',{'class':'snize-product'}) #print(item_link) for i in range(0, len(titles)): print('in loop') TITLE = titles[i].find('h3').text TITLE = TITLE.replace('\n', '') TTILE = TITLE.replace('\t', '') p = prices[i].find('span').text #print(PRICE) PRICE = parse_price(p).amount_float IMAGE = img_url[i].find('img').get('src') #print(IMAGE) #IMAGE = IMAGE.replace("?v=1599216010","") LINK = titles[i].find('a').get('href') # if Item.objects.filter(item_url=LINK): # continue #print(LINK) #LINK = "https://www.cougar.com.pk/" + x #print(LINK) BRAND = "Kross Kulture" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } #print(item) print(BRAND, query) CAT = Preferences.objects.get(title=string) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() except (ConnectionError, Exception): return False
def something_in_stock(self): #params = {"anticache": str(secrets.token_urlsafe(32))} params = {} for x in range(len(self.asin_list)): params[f"ASIN.{x + 1}"] = self.asin_list[x] params[f"Quantity.{x + 1}"] = 1 f = furl(AMAZON_URLS["CART_URL"]) f.set(params) self.driver.get(f.url) title = self.driver.title #if len(self.asin_list) > 1 and title in DOGGO_TITLES: if title in DOGGO_TITLES: good_asin_list = [] for asin in self.asin_list: checkparams = {} checkparams[f"ASIN.1"] = asin checkparams[f"Quantity.1"] = 1 check = furl(AMAZON_URLS["CART_URL"]) check.set(checkparams) self.driver.get(check.url) sanity_check = self.driver.title if sanity_check in DOGGO_TITLES: log.error(f"{asin} blocked from bulk adding by Amazon") time.sleep(1) else: log.info(f"{asin} appears to allow adding") good_asin_list.append(asin) if len(good_asin_list) > 0: log.info( "Revising ASIN list to include only good ASINs listed above" ) self.asin_list = good_asin_list else: log.error("No ASINs work in list. Try using smile.amazon.com") exit(1) self.check_if_captcha(self.wait_for_pages, ADD_TO_CART_TITLES) price_element = self.driver.find_elements_by_xpath( '//td[@class="price item-row"]') if price_element: str_price = price_element[0].text log.info(f'Item Cost: {str_price}') price = parse_price(str_price) priceFloat = price.amount if priceFloat is None: log.error("Error reading price information on page.") return False elif priceFloat <= self.reserve: log.info("One or more items in stock and under reserve!") return True else: log.info("No stock available under reserve price") #log.info("{}".format(self.asin_list)) return False return False else: return False
def beechtree(string): print("functions") query = set_up_query(string) try: link = "https://www.beechtree.pk/pk/catalogsearch/result/?q=" + str( query) + "#/page/1" req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') #print(soup) titles = soup.find_all('h2', {'class': 'product-name'}) #print(titles) prices = soup.find_all('span', {'class': 'price'}) #print(prices) img_url = soup.find_all('a', {'class': 'product-image'}) #print(img_url) #item_link = soup.find_all('div',{'class':'product-info-inner'}) #print(item_link) for i in range(0, len(titles)): print('in loop') TITLE = titles[i].text TITLE = TITLE.replace('\n', '') TTILE = TITLE.replace('\t', '') #PRICE = prices[i].text #print(PRICE) PRICE = parse_price(prices[i].text).amount_float IMAGE = img_url[i].find('img').get('src') #print(IMAGE) #IMAGE = IMAGE.replace("?v=1599216010","") LINK = img_url[i].get('href') # if Item.objects.filter(item_url=LINK): # continue #print(LINK) #LINK = "https://www.cougar.com.pk/" + x #print(LINK) BRAND = "Beechtree" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } #print(item) print(BRAND, query) CAT = Preferences.objects.get(title=string) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() except (ConnectionError, Exception): return False
def get_prices(tree): # Price collection xpath: # //div[@id='aod-offer']//div[contains(@id, "aod-price-")]//span[contains(@class,'a-offscreen')] price_nodes = tree.xpath( "//div[@id='aod-offer']//div[contains(@id, 'aod-price-')]//span[contains(@class,'a-offscreen')]" ) # log.debug(f"Found {len(price_nodes)} price nodes.") prices = [] for idx, price_node in enumerate(price_nodes): log.debug(f"Found price {idx + 1}: {price_node.text}") prices.append(parse_price(price_node.text)) return prices
def something_in_stock(self): #params = {"anticache": str(secrets.token_urlsafe(32))} params = {} for x in range(len(self.asin_list)): params[f"ASIN.{x + 1}"] = self.asin_list[x] params[f"Quantity.{x + 1}"] = 1 f = furl(AMAZON_URLS["CART_URL"]) f.set(params) self.driver.get(f.url) title = self.driver.title #if len(self.asin_list) > 1 and title in DOGGO_TITLES: if title in DOGGO_TITLES: for asin in self.asin_list: checkparams = {} checkparams[f"ASIN.1"] = asin checkparams[f"Quantity.1"] = 1 check = furl(AMAZON_URLS["CART_URL"]) check.set(checkparams) self.driver.get(check.url) sanity_check = self.driver.title if sanity_check in DOGGO_TITLES: #log.error("{} blocked from bulk adding by Amazon".format(asin)) #if len(self.asin_list) == 1: # log.info("Only one ASIN remains, refreshing") # return False #self.asin_list.remove(asin) #if len(self.asin_list) == 0: # log.error("No ASIN's left in list") # exit(1) continue else: log.info("{} appears to allow adding".format(asin)) return True self.check_if_captcha(self.wait_for_pages, ADD_TO_CART_TITLES) price_element = self.driver.find_elements_by_xpath( '//td[@class="price item-row"]') if price_element: str_price = price_element[0].text log.info(f'Item Cost: {str_price}') price = parse_price(str_price) priceFloat = price.amount if priceFloat <= self.reserve: log.info("One or more items in stock and under reserve!") return True else: log.info("No stock available under reserve price") #log.info("{}".format(self.asin_list)) return False return False else: return False
def couger(string): #print("functions") query = set_up_query(string) try: link = "https://www.cougar.com.pk/search?q=" + str( query) + "&type=product" req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') #print(soup) titles = soup.find_all('span', {'class': 'prod-title'}) prices = soup.find_all('div', {'class': 'prod-price'}) #print(prices) img_url = soup.find_all('img', {'class': 'lazyload-fade'}) #print(img_url) item_link = soup.find_all('div', {'class': 'product-info-inner'}) #print(item_link) for i in range(0, len(titles)): #print('in loop') TITLE = titles[i].text #PRICE = prices[i].text PRICE = parse_price(prices[i].text).amount_float IMAGE = img_url[i].get('data-src') #IMAGE = IMAGE.replace("?v=1599216010","") x = item_link[i].find('a').get('href') LINK = "https://www.cougar.com.pk/" + x # if Item.objects.filter(item_url=LINK): # continue #print(LINK) BRAND = "Couger" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } #print(item) print(BRAND, query) CAT = Preferences.objects.get(title=string) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() except (ConnectionError, Exception): return False
def home(): if (request.method == 'POST'): sku = request.form['sku'] result = parse_price(sku) if isinstance(result, str): title = result return render_template("index.html", title=title) else: price = result[0] title = result[1] return render_template("index.html", price=str(price), title=title) else: return render_template('index.html')
def convert(self, text): """Convert currency according to the target locale Args: text (str): currency text Returns: (str): converted currency """ cur = parse_price(re.sub('\s', ' ', text)) if cur.amount and cur.currency: amountText, currency = cur.amount_text, cur.currency if self.srcDecimalSymbol in amountText: amount = float(cur.amount) res = format_currency(amount, self.curCode, locale=self.tgtLocale) else: amount = int(cur.amount) if self.tgtLang == 'eng': res = format_currency(amount, self.curCode, format=u'¤#,##0', locale=self.tgtLocale, currency_digits=False) elif self.tgtLang == 'fra': res = format_currency(amount, self.curCode, format=u'#,##0\xa0¤', locale=self.tgtLocale, currency_digits=False) else: raise NotImplementedError( 'Target language %s is not supported' % (self.tgtLang, )) if self._curFormat == 'symbol': tgtCurrencySymbol = numbers.get_currency_symbol( self.curCode, self.srcLocale) res = re.sub( re.escape(tgtCurrencySymbol) + r'\S+', tgtCurrencySymbol, res) else: res = None return res
def eden_robe(string): query = set_up_query(string) try: link = "https://edenrobe.com/catalogsearch/result/?q=" + query req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') #print(soup.text) titles = soup.find_all('a', {'class': 'product-item-link'}) prices = soup.find_all('span', {'class': 'price'}) #img = [i['href'] for i in soup.find_all('a',{'class':'MagicZoom'}, href=True) if i['href'] != "#"] img = soup.find_all( 'img', {'class': 'img-responsive product-image-photo img-thumbnail'}) #print(img) for i in range(0, len(titles)): print("IN LOPOOP") TITLE = titles[i].text PRICE = parse_price(prices[i].text).amount_float if PRICE is None: continue IMAGE = img[i].get('data-src') print(IMAGE) BRAND = "Eden Robe" LINK = titles[i].get('href') # if Item.objects.filter(item_url=LINK): # continue item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } #print(BRAND,string) CAT = Preferences.objects.get(title=query) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() except (ConnectionError, Exception): return False
def diners(string): query = set_up_query(string) try: link = "https://diners.com.pk/search?page=&q=" + query req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') titles = soup.find_all('a', {'class': 'product-title'}) prices = soup.find_all('div', {'class': 'price-box'}) img_url = soup.find_all('span', {'class': 'images-two'}) L = (len(titles)) for i in range(0, L): TITLE = titles[i].span.text IMAGE = img_url[i].img.get('src') LINK = "https://diners.com.pk/" + titles[i].get('href') # if Item.objects.filter(item_url=LINK): # continue PRICE = parse_price(prices[i].text).amount_float if PRICE is None: pass BRAND = "Diners" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } print(item) print(BRAND, query) CAT = Preferences.objects.get(title=string) print(CAT) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() except (ConnectionError, Exception): return False
def outfitters(string): query = set_up_query(string) try: link = "https://outfitters.com.pk/search?type=product&q=" + query req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') titles = soup.find_all('a', {'class': 'product-title'}) prices = soup.find_all('span', {'class': 'money'}) img_url = soup.find_all('a', {'class': 'product-grid-image'}) item_link = [ i['href'] for i in soup.find_all('a', {'class': 'product-title'}, href=True) if i['href'] != "#" ] for i in range(0, len(titles)): TITLE = titles[i].text TITLE = TITLE.replace("\t", "") TITLE = TITLE.replace("\n", "") PRICE = parse_price(prices[i].text).amount_float IMAGE = img_url[i].find('img').get('src') IMAGE = IMAGE.replace("?v=1599216010", "") LINK = "https://outfitters.com.pk" + item_link[i] # if Item.objects.filter(item_url=LINK): # continue BRAND = "Outfitters" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } print(BRAND, query) CAT = Preferences.objects.get(title=string) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() except (ConnectionError, Exception): return False
def jdot(string): query = set_up_query(string) try: page = 1 link = "https://www.junaidjamshed.com/catalogsearch/result/index/?p=" + str( page) + "&product_list_limit=30&q=" + query query = query.replace("+", " ") req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') titles = soup.find_all('a', {'class': 'product-item-link'}) prices = soup.find_all('span', {'class': 'price'}) img_url = soup.find_all('img', {'class': 'product-image-photo lazy'}) X = (len(titles)) for i in range(0, X): TITLE = titles[i].text TITLE = TITLE.replace("\t", "") TITLE = TITLE.replace("\n", "") PRICE = parse_price(prices[i].text).amount_float IMAGE = img_url[i].get('data-original') LINK = titles[i].get('href') # if Item.objects.filter(item_url=LINK): # continue BRAND = "Junaid Jamshed" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } print(item) CAT = Preferences.objects.get(title=string) print(query) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() except (ConnectionError, Exception): return False
def something_in_stock_it(self): for x in range(len(self.asin_list)): bad_asin_list = [] for asin in self.asin_list[x]: # params = {"anticache": str(secrets.token_urlsafe(32))} params = {} params[f"ASIN.1"] = asin params[f"Quantity.1"] = 1 f = furl(AMAZON_URLS["CART_URL"]) f.set(params) self.driver.get(f.url) title = self.driver.title if title in DOGGO_TITLES: log.error( f"{asin} blocked from bulk adding by Amazon, it will be removed from ASIN list" ) bad_asin_list.append(asin) else: self.check_if_captcha(self.wait_for_pages, ADD_TO_CART_TITLES) price_element = self.driver.find_elements_by_xpath( '//td[@class="price item-row"]') if price_element: str_price = price_element[0].text log.info(f"Item Cost: {str_price}") price = parse_price(str_price) priceFloat = price.amount if priceFloat is None: log.error( "Error reading price information on page.") elif priceFloat <= self.reserve[x]: log.info("Item in stock and under reserve!") if bad_asin_list: for bad_asin in bad_asin_list: self.asin_list[x].remove(bad_asin) return asin else: log.info("Item greater than reserve price") # log.info("{}".format(self.asin_list)) if bad_asin_list: for bad_asin in bad_asin_list: self.asin_list[x].remove(bad_asin) return 0
def gulahmed(query): try: link = "https://www.gulahmedshop.com/catalogsearch/result/?q=" + query req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') titles = soup.find_all('a', {'class': 'product-item-link'}) prices = soup.find_all('span', {'class': 'price'}) img_url = soup.find_all('img', {'class': 'product-image-photo'}) #//*[@id="category-products-grid"]/ol/li[1]/div/div[1]/div[1]/a/span[1]/span/span/img item_link = [ i['href'] for i in soup.find_all('a', {'class': 'product-item-link'}, href=True) if i['href'] != "#" ] for i in range(0, len(titles)): TITLE = titles[i].text TITLE = TITLE.replace("\t", "") TITLE = TITLE.replace("\n", "") PRICE = parse_price(prices[i].text).amount_float IMAGE = img_url[i].get('src') #IMAGE = IMAGE.replace(".pagespeed.ic.py1u_bk3_f.webp","") LINK = item_link[i] # if Item.objects.filter(item_url=LINK): # continue BRAND = "Gul Ahmed" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } # query= query.replace("+"," ") # Item(title=item['title'],image_url=item['img'],item_url=item['link'],brand=item['brand'],price=item['price'],category=query).save() except (ConnectionError, Exception): return False
def focus(string): query = set_up_query(string) try: link = "https://focusclothing.pk/search?q=" + query req = requests.get(link) soup = BeautifulSoup(req.text, 'html.parser') titles = soup.find_all('div', {'class': 'prod-title'}) prices = soup.find_all('div', {'class': 'onsale'}) img_url = soup.find_all('noscript') item_link = soup.find_all('div', {'class': 'product-info'}) L = (len(titles)) for i in range(0, L): TITLE = titles[i].text IMAGE = img_url[i].img.get('src') LINK = "https://focusclothing.pk" + item_link[i].a.get('href') # if Item.objects.filter(item_url=LINK): # continue PRICE = parse_price(prices[i].text).amount_float BRAND = "Focus" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } print(BRAND, query) CAT = Preferences.objects.get(title=string) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() except (ConnectionError, Exception): return False
def uniworth(query): try: link = "https://www.uniworthshop.com/search?keywords=" + query req = requests.get(link) soup = BeautifulSoup(req.text, 'lxml') titles = soup.find_all('div', {'class': 'product-name'}) prices = soup.find_all('span', {'class': 'price'}) img_url = soup.find_all('img', {'class': 'primary'}) item_link = [ i['href'] for i in soup.find_all('a', {'class': 'product-img'}, href=True) if i['href'] != "#" ] for i in range(0, len(titles)): TITLE = titles[i].text TITLE = TITLE.replace("\t", "") TITLE = TITLE.replace("\n", "") PRICE = parse_price(prices[i].text).amount_float IMAGE = img_url[i].get('data-src') LINK = item_link[i] if Item.objects.filter(item_url=LINK): continue BRAND = "Uniworth" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } #items_list.append(item) # item = Item(title=TITLE,image_url=IMAGE,item_url=LINK,brand=BRAND,price=item['price']) # item.save() except (ConnectionError, Exception): return False
def check_stock(self, asin, reserve, retry=0): if retry > DEFAULT_MAX_ATC_TRIES: log.info("max add to cart retries hit, returning to asin check") return False if self.checkshipping: if self.used: f = furl(AMAZON_URLS["OFFER_URL"] + asin) else: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new&f_new=true") else: if self.used: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/f_freeShipping=on") else: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new&f_new=true&f_freeShipping=on") try: while True: try: try: searching_update() except Exception: pass self.driver.get(f.url) break except Exception: log.error( "Failed to get the URL, were in the exception now.") time.sleep(3) pass elements = self.driver.find_elements_by_xpath( '//*[@name="submit.addToCart"]') prices = self.driver.find_elements_by_xpath( '//*[@class="a-size-large a-color-price olpOfferPrice a-text-bold"]' ) shipping = self.driver.find_elements_by_xpath( '//*[@class="a-color-secondary"]') except Exception as e: log.error(e) return None in_stock = False for i in range(len(elements)): price = parse_price(prices[i].text) if SHIPPING_ONLY_IF in shipping[i].text: ship_price = parse_price("0") else: ship_price = parse_price(shipping[i].text) ship_float = ship_price.amount price_float = price.amount if price_float is None: return False if ship_float is None or not self.checkshipping: ship_float = 0 if (ship_float + price_float) <= reserve or math.isclose( (price_float + ship_float), reserve, abs_tol=0.01): log.info("Item in stock and under reserve!") log.info("clicking add to cart") try: buy_update() except: pass elements[i].click() time.sleep(self.page_wait_delay()) if self.driver.title in SHOPING_CART_TITLES: return True else: log.info("did not add to cart, trying again") log.debug(f"failed title was {self.driver.title}") if self.no_screenshots: self.notification_handler.send_notification( "failed-atc") else: self.save_screenshot("failed-atc") self.save_page_source("failed-atc") in_stock = self.check_stock(asin=asin, reserve=reserve, retry=retry + 1) return in_stock
def extract_price(description): if description.startswith("€"): return parse_price(description, decimal_separator=",").amount else: return parse_price(description).amount
def extract_token_specific_features(df): """ Extract specific token related features :param df: :return: """ is_date = [] is_number = [] has_currency = [] extracted_amount = [] total_len_text = [] len_digit = [] len_alpha = [] len_spaces = [] for index, row in df.iterrows(): text = row['text'] try: pd.to_datetime(text) is_date.append(1) except: is_date.append(0) pass try: float(text.replace(',', '')) is_number.append(1) except: is_number.append(0) pass try: price = parse_price(text) if (not price.amount is None) and (not price.currency is None): extracted_amount.append(price.amount) has_currency.append(1) else: extracted_amount.append(None) has_currency.append(0) except: extracted_amount.append(None) has_currency.append(0) pass try: total_len_text.append(len(text)) len_digit.append(sum(c.isdigit() for c in text)) len_alpha.append(sum(c.isalpha() for c in text)) len_spaces.append(sum(c.isspace() for c in text)) except: total_len_text.append(0) len_digit.append(0) len_alpha.append(0) len_spaces.append(0) pass df['is_date'] = is_date df['is_number'] = is_number df['has_currency'] = has_currency df['extracted_amount'] = extracted_amount df['total_len_text'] = total_len_text df['len_digit'] = len_digit df['len_alpha'] = len_alpha df['len_spaces'] = len_spaces df['len_others'] = df['total_len_text'] - df['len_digit'] - df[ 'len_alpha'] - df['len_spaces']
# -*- coding: utf-8 -*- from price_parser import Price from price_parser import parse_price price = Price.fromstring("22,90 €") print(price) # numeric price amount print(price.amount) # currency symbol, as appears in the string print(price.currency) # price amount, as appears in the string print(price.amount_text) # price amount as float, not Decimal print(price.amount_float) print(parse_price("22,90 €")) print(Price.fromstring("Price: $119.00")) print(Price.fromstring("15 130 Р")) print(Price.fromstring("151,200 تومان")) print(Price.fromstring("Rp 1.550.000")) print(Price.fromstring("Běžná cena 75 990,00 Kč")) print(Price.fromstring("1,235€ 99")) print(Price.fromstring("24€99")) print(Price.fromstring("99 € 95 €")) print(Price.fromstring("35€ 999")) print(Price.fromstring("€35,999")) print(Price.fromstring("€35,999",currency_hint="€",decimal_separator=",")) print(Price.fromstring(""))
def something_in_stock_mass(self): for i in range(len(self.asin_list)): params = {} for x in range(len(self.asin_list[i])): params[f"ASIN.{x + 1}"] = self.asin_list[i][x] params[f"Quantity.{x + 1}"] = 1 f = furl(AMAZON_URLS["CART_URL"]) f.set(params) self.driver.get(f.url) title = self.driver.title bad_list_flag = False if title in DOGGO_TITLES: good_asin_list = [] for asin in self.asin_list[i]: checkparams = {} checkparams[f"ASIN.1"] = asin checkparams[f"Quantity.1"] = 1 check = furl(AMAZON_URLS["CART_URL"]) check.set(checkparams) self.driver.get(check.url) sanity_check = self.driver.title if sanity_check in DOGGO_TITLES: log.error(f"{asin} blocked from bulk adding by Amazon") else: log.info(f"{asin} appears to allow adding") good_asin_list.append(asin) time.sleep(1) if len(good_asin_list) > 0: log.info( "Revising ASIN list to include only good ASINs listed above" ) self.asin_list[i] = good_asin_list else: log.error(f"No ASINs work in list {i+1}.") self.asin_list[i] = self.asin_list[i][ 0] # just assign one asin to list, can't remove during execution bad_list_flag = True if bad_list_flag: continue self.check_if_captcha(self.wait_for_pages, ADD_TO_CART_TITLES) price_element = self.driver.find_elements_by_xpath( '//td[@class="price item-row"]') if price_element: price_flag = False price_warning_flag = False for price_e in price_element: str_price = price_e.text log.info(f"Item Cost: {str_price}") price = parse_price(str_price) priceFloat = price.amount if priceFloat is None: log.error("Error reading price information on page.") elif priceFloat <= self.reserve[i]: log.info("Item in stock and under reserve!") price_flag = True else: log.info("Item greater than reserve price") price_warning_flag = True if price_flag: log.info("Attempting to purchase") if price_warning_flag: log.info( "Cart included items below and above reserve price, cancel unwanted items ASAP!" ) self.take_screenshot("attempting-to-purchase") return i + 1 return 0
def check_stock(self, asin, reserve_min, reserve_max, retry=0): if retry > DEFAULT_MAX_ATC_TRIES: log.info("max add to cart retries hit, returning to asin check") return False if self.checkshipping: if self.used: f = furl(AMAZON_URLS["OFFER_URL"] + asin) else: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new&f_new=true") else: if self.used: f = furl(AMAZON_URLS["OFFER_URL"] + asin + "/f_freeShipping=on") else: f = furl( AMAZON_URLS["OFFER_URL"] + asin + "/ref=olp_f_new&f_new=true&f_freeShipping=on" ) fail_counter = 0 presence.searching_update() while True: try: self.get_page(f.url) break except Exception: fail_counter += 1 log.error(f"Failed to load the offer URL {fail_counter} times.") if fail_counter < DEFAULT_MAX_URL_FAIL: log.error( f"WebDriver will restart if it fails {DEFAULT_MAX_URL_FAIL} times. Retrying now..." ) time.sleep(3) else: log.info( "Attempting to delete and recreate current chrome instance" ) if not self.delete_driver(): log.error("Failed to delete chrome processes") log.error("Please restart bot") self.send_notification( message="Bot Failed, please restart bot", page_name="Bot Failed", take_screenshot=False, ) raise RuntimeError("Failed to restart bot") elif not self.create_driver(): log.error("Failed to recreate webdriver processes") log.error("Please restart bot") self.send_notification( message="Bot Failed, please restart bot", page_name="Bot Failed", take_screenshot=False, ) raise RuntimeError("Failed to restart bot") else: # deleted driver and recreated it succesfully log.info( "WebDriver recreated successfully. Returning back to stock check" ) return False timeout = self.get_timeout() while True: atc_buttons = self.driver.find_elements_by_xpath( '//*[@name="submit.addToCart"]' ) if atc_buttons: # Early out if we found buttons break test = None try: test = self.driver.find_element_by_xpath( '//*[@id="olpOfferList"]/div/p' ) except exceptions.NoSuchElementException: pass if test and (test.text in NO_SELLERS): return False if time.time() > timeout: log.info(f"failed to load page for {asin}, going to next ASIN") return False timeout = self.get_timeout() while True: prices = self.driver.find_elements_by_xpath( '//*[@class="a-size-large a-color-price olpOfferPrice a-text-bold"]' ) if prices: break if time.time() > timeout: log.info(f"failed to load prices for {asin}, going to next ASIN") return False shipping = [] if self.checkshipping: timeout = self.get_timeout() while True: shipping = self.driver.find_elements_by_xpath( '//*[@class="a-color-secondary"]' ) if shipping: break if time.time() > timeout: log.info(f"failed to load shipping for {asin}, going to next ASIN") return False in_stock = False for idx, atc_button in enumerate(atc_buttons): try: price = parse_price(prices[idx].text) except IndexError: log.debug("Price index error") return False try: if self.checkshipping: if SHIPPING_ONLY_IF in shipping[idx].text: ship_price = parse_price("0") else: ship_price = parse_price(shipping[idx].text) else: ship_price = parse_price("0") except IndexError: log.debug("shipping index error") return False ship_float = ship_price.amount price_float = price.amount if price_float is None: return False if ship_float is None or not self.checkshipping: ship_float = 0 if ( (ship_float + price_float) <= reserve_max or math.isclose((price_float + ship_float), reserve_max, abs_tol=0.01) ) and ( (ship_float + price_float) >= reserve_min or math.isclose((price_float + ship_float), reserve_min, abs_tol=0.01) ): log.info("Item in stock and in reserve range!") log.info("clicking add to cart") self.notification_handler.play_notify_sound() if self.detailed: self.send_notification( message=f"Found Stock ASIN:{asin}", page_name="Stock Alert", take_screenshot=self.take_screenshots, ) presence.buy_update() current_title = self.driver.title # log.info(f"current page title is {current_title}") try: atc_button.click() except IndexError: log.debug("Index Error") return False self.wait_for_page_change(current_title) # log.info(f"page title is {self.driver.title}") if self.driver.title in SHOPING_CART_TITLES: return True else: log.info("did not add to cart, trying again") log.debug(f"failed title was {self.driver.title}") self.send_notification( "Failed Add to Cart", "failed-atc", self.take_screenshots ) self.save_page_source("failed-atc") in_stock = self.check_stock( asin=asin, reserve_max=reserve_max, reserve_min=reserve_min, retry=retry + 1, ) return in_stock
def breakout(string): query = set_up_query(string) try: page = 1 ##print(page) link = "https://breakout.com.pk/search?page=" + str( page) + "&q=" + query ##print("query"+query) cat = query.replace("+", " ") ##print(link) req = requests.get(link) soup = BeautifulSoup(req.text, 'html.parser') nextBtn = soup.find('a', {'class': 'btn-next'}) while nextBtn.text == "Next": #print(page) #print(nextBtn) link = "https://breakout.com.pk/search?page=" + str( page) + "&q=" + query ##print("query"+query) cat = query.replace("+", " ") #print(link) req = requests.get(link) ##print('request sussecfull') soup = BeautifulSoup(req.text, 'html.parser') ##print(soup) titles = soup.find_all( 'h2', {'class': 'pt-title prod-thumb-title-color'}) prices = soup.find_all('div', {'class': 'pt-price'}) #rint(prices) img_url = soup.find_all('img', {'class': 'lazyload'}) ##print(img_url) #item_link = soup.find_all('a',{'class':'product-title'}, href=True) L = (len(titles)) nextBtn = soup.find('a', {'class': 'btn-next'}) for i in range(0, L): #print("In Loop") TITLE = titles[i].text TITLE = TITLE.upper() IMAGE = img_url[i].get('data-src') LINK = "https://breakout.com.pk" + titles[i].find('a').get( 'href') # if Item.objects.filter(item_url=LINK): # continue PRICE = parse_price(prices[i].text).amount_float if PRICE is None: continue BRAND = "Breakout" item = { 'title': TITLE, 'price': PRICE, 'img': IMAGE, 'brand': BRAND, 'link': LINK } CAT = Preferences.objects.get(title=string) Item(title=item['title'], image_url=item['img'], item_url=item['link'], brand=item['brand'], price=item['price'], category=CAT).save() page = page + 1 except (ConnectionError, Exception): return False
def get_shipping_costs(tree, free_shipping_string): # Assume Free Shipping and change otherwise # Shipping collection xpath: # .//div[starts-with(@id, 'aod-bottlingDepositFee-')]/following-sibling::span shipping_nodes = tree.xpath( ".//div[starts-with(@id, 'aod-bottlingDepositFee-')]/following-sibling::*[1]" ) count = len(shipping_nodes) # log.debug(f"Found {count} shipping nodes.") if count == 0: log.warning("No shipping nodes found. Assuming zero.") return FREE_SHIPPING_PRICE elif count > 1: log.warning("Found multiple shipping nodes. Using the first.") shipping_node = shipping_nodes[0] # Shipping information is found within either a DIV or a SPAN following the bottleDepositFee DIV # What follows is logic to parse out the various pricing formats within the HTML. Not ideal, but # it's what we have to work with. shipping_span_text = shipping_node.text.strip() if shipping_node.tag == "div": if shipping_span_text == "": # Assume zero shipping for an empty div log.debug( "Empty div found after bottleDepositFee. Assuming zero shipping." ) return FREE_SHIPPING_PRICE else: # Assume zero shipping for unknown values in log.warning( f"Non-Empty div found after bottleDepositFee. Assuming zero. Stripped Value: '{shipping_span_text}'" ) return FREE_SHIPPING_PRICE elif shipping_node.tag == "span": # Shipping values in the span are contained in: # - another SPAN # - hanging out alone in a B tag # - Hanging out alone in an I tag # - Nested in two I tags <i><i></i></i> # - "Prime FREE Delivery" in this node shipping_spans = shipping_node.findall("span") shipping_bs = shipping_node.findall("b") # shipping_is = shipping_node.findall("i") shipping_is = shipping_node.xpath("//i[@aria-label]") if len(shipping_spans) > 0: # If the span starts with a "& " it's free shipping (right?) if shipping_spans[0].text.strip() == "&": # & Free Shipping message # log.debug("Found '& Free', assuming zero.") return FREE_SHIPPING_PRICE elif shipping_spans[0].text.startswith("+"): return parse_price(shipping_spans[0].text.strip()) elif len(shipping_bs) > 0: for message_node in shipping_bs: if message_node.text.upper() in free_shipping_string: # log.debug("Found free shipping string.") return FREE_SHIPPING_PRICE else: log.error( f"Couldn't parse price from <B>. Assuming 0. Do we need to add: '{message_node.text.upper()}'" ) return FREE_SHIPPING_PRICE elif len(shipping_is) > 0: # If it has prime icon class, assume free Prime shipping if "FREE" in shipping_is[0].attrib["aria-label"].upper(): # log.debug("Found Free shipping with Prime") return FREE_SHIPPING_PRICE elif any(shipping_span_text.upper() in free_message for free_message in free_shipping_string): # We found some version of "free" inside the span.. but this relies on a match log.warning( f"Assuming free shipping based on this message: '{shipping_span_text}'" ) else: log.error( f"Unable to locate price. Assuming 0. Found this: '{shipping_span_text}' " f"Consider reporting to #tech-support Discord.") return FREE_SHIPPING_PRICE
from decimal import Decimal from enum import Enum import attr from amazoncaptcha import AmazonCaptcha from furl import furl from lxml import html from price_parser import Price, parse_price from utils.logger import log FREE_SHIPPING_PRICE = parse_price("0.00") class AmazonItemCondition(Enum): # See https://sellercentral.amazon.com/gp/help/external/200386310?language=en_US&ref=efph_200386310_cont_G1831 New = 10 Renewed = 20 Refurbished = 20 Rental = 30 Open_box = 40 OpenBoxLikeNew = 40 Used = 40 UsedLikeNew = 40 UsedVeryGood = 50 UsedGood = 60 UsedAcceptable = 70 CollectibleLikeNew = 40 CollectibleVeryGood = 50 CollectibleGood = 60 CollectibleAcceptable = 70