def test_get_total_pages_from_body(self):
w = Wishlist()
body = self.get_body("wishlist-pagination-last.html")
page = w.get_total_pages_from_body(body)
self.assertEqual(28, page)
w = Wishlist()
body = self.get_body("wishlist-pagination-pg7.html")
page = w.get_total_pages_from_body(body)
self.assertEqual(28, page)
w = Wishlist()
body = self.get_body("wishlist-1.html")
page = w.get_total_pages_from_body(body)
self.assertEqual(28, page)
def _wishlist_items(self, list_url):
"""
Get the items on the specified wishlist.
:param list_url: wishlist URL
:type list_url: str
:return: dict of item URL to item details dict
:rtype: dict
"""
res = {}
list_name = list_url.split('/')[-1]
logger.debug('Getting wishlist items for wishlist: %s', list_name)
items = [i for i in Wishlist(list_name)]
logger.debug("Found %d items in list" % len(items))
for item in items:
item = item.jsonable()
d = {'name': item['title'], 'url': item['url']}
try:
d['quantity'] = item['wanted_count']
except Exception:
d['quantity'] = 1
if item['price'] > 0:
d['cost'] = item['price']
else:
d['cost'] = item['marketplace_price']
res[item['url']] = d
return res
def test_get_items_from_body(self):
count = 0
body = self.get_body("wishlist-1.html")
w = Wishlist()
for item in w.get_items_from_body(body):
self.assertEqual(13, len(item.jsonable()))
count += 1
self.assertEqual(25, count)
def main_auth():
"""Signin to amazon so you can access private wishlists"""
w = Wishlist()
with w.open_full() as b:
host = w.host
echo.out("Requesting {}", host)
b.location(host, ignore_cookies=True)
# If you access from another country, amazon might prompt to redirect to
# country specific store, we don't want that
if b.element_exists("#redir-opt-out"):
echo.out("Circumventing redirect")
remember = b.element("#redir-opt-out")
stay = b.element("#redir-stay-at-www")
remember.click()
stay.click()
button = b.element("#a-autoid-0-announce")
echo.out("Clicking sign in button")
button.click()
# now put in your creds
email = b.element("#ap_email")
password = b.element("#ap_password")
submit = b.element("#signInSubmit")
if email and password and submit:
echo.out("Found sign in form")
email_in = echo.prompt("Amazon email address")
password_in = echo.prompt("Amazon password")
email.send_keys(email_in)
password.send_keys(password_in)
echo.out("Signing in")
submit.click()
# for 2-factor, wait for this element
code = b.wait_for_element("#auth-mfa-otpcode", 5)
if code:
echo.out(
"2-Factor authentication is on, you should be receiving a text"
)
submit = b.element("#auth-signin-button")
remember = b.element("#auth-mfa-remember-device")
remember.click()
authcode = echo.prompt("2-Factor authcode")
code.send_keys(authcode)
submit.click()
#https://www.amazon.com/ref=gw_sgn_ib/853-0204854-22247543
if "/ref=gw_sgn_ib/" in b.current_url:
echo.out("Success, you are now signed in")
b.save()
def test_redesign_2018_06(self):
"""In mid June Amazon updated the html and it had a bug or something in it
and so my wishlist didn't parse for a few days and then it was magically
fixed, but I managed to grab the first page so I could figure out what was
wrong and it turned out I didn't need to change anything except allow
Brow to use different parsers because the more liberal non built-in parsers
were both able to parse the html"""
#body = self.get_body("redesign-2018-06")
soup = self.get_soup("html-2018-06", "html.parser")
w = Wishlist("WISHLISTNAME")
items = list(w.get_items(soup, w.get_wishlist_url()))
self.assertEqual(10, len(items))
soup = self.get_soup("html-2018-06", "lxml")
w = Wishlist("WISHLISTNAME")
items = list(w.get_items(soup, w.get_wishlist_url()))
self.assertEqual(10, len(items))
soup = self.get_soup("html-2018-06", "html5lib")
w = Wishlist("WISHLISTNAME")
items = list(w.get_items(soup, w.get_wishlist_url()))
self.assertEqual(10, len(items))
def checkWishlistPrices(self):
w = Wishlist(self.wishlistid)
for item in w:
id = item.jsonable()['url'].split('/')[4]
title = item.jsonable()['title']
url = 'https://smile.amazon.com/dp/' + id + '/'
currentPrice = ceil(float(item.jsonable()['price']))
#print(id, " |>>>", currentPrice, "<<<")
self.c.execute("SELECT price FROM items WHERE id='%s'" % id)
try:
dbPrice = self.c.fetchone()[0]
#print(currentPrice, "|||", dbPrice)
if currentPrice < dbPrice:
print()
print(title)
print("Current: ", currentPrice)
print("Stored : ", dbPrice)
self.c.execute(
"SELECT price,time FROM history WHERE id='%s' ORDER BY time"
% id)
prices = self.c.fetchall()
for p in prices:
print(p)
print("URL: %s" % url)
print(
"Wishlist: https://www.amazon.com/gp/registry/wishlist/%s"
% self.wishlistid)
self.c.execute(
"UPDATE items SET price='%s' WHERE id='%s'" %
(currentPrice, id))
except TypeError:
print("Add item id: %s" % id)
self.c.execute(
"INSERT INTO items(id,title,url,price) VALUES(?,?,?,?)",
(id, title, url, currentPrice))
self.c.execute("INSERT INTO history(id,price) VALUES(?,?)",
(id, currentPrice))
self.c.execute(
"SELECT price FROM history WHERE id='%s' AND time=(SELECT MAX(time) FROM history WHERE id='%s')"
% (id, id))
try:
dbHPrice = self.c.fetchone()[0]
if currentPrice != dbHPrice:
#print(currentPrice, "|", dbHPrice)
self.c.execute("INSERT INTO history(id,price) VALUES(?,?)",
(id, currentPrice))
except TypeError:
self.c.execute("INSERT INTO history(id,price) VALUES(?,?)",
(id, currentPrice))
def main(argv):
inputwishlist = ''
try:
opts, args = getopt.getopt(argv, "hi:o:", ["i="])
except getopt.GetoptError:
print('wishlist2csv.py -i <inputwishlist>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('wishlist2csv.py -i <inputwishlist>')
sys.exit()
elif opt in ("-i"):
inputwishlist = arg
name = inputwishlist
w = Wishlist(name)
print('"' + 'title' + '"' + ',' + '"' + 'url' + '"')
for item in w:
we_json = item.jsonable()
print('"' + we_json["title"] + '"' + "," + '"' + we_json["url"] + '"')
def main_dump(name, start_page, stop_page, **kwargs):
"""This is really here just to test that I can parse a wishlist completely and
to demonstrate (by looking at the code) how to iterate through a list"""
name = name[0]
#pout.v(name, start_page, stop_page, kwargs)
#pout.x()
pages = set()
current_url = ""
w = Wishlist()
for i, item in enumerate(w.get(name, start_page, stop_page), 1):
new_current_url = w.current_url
if new_current_url != current_url:
current_url = new_current_url
echo.h3(current_url)
try:
item_json = item.jsonable()
echo.out("{}. {} is ${:.2f}", i, item_json["title"],
item_json["price"])
echo.indent(item_json["url"])
except RobotError:
raise
except ParseError as e:
echo.err("{}. Failed!", i)
echo.err(e.body)
echo.exception(e)
except KeyboardInterrupt:
break
except Exception as e:
echo.err("{}. Failed!", i)
echo.exception(e)
finally:
pages.add(w.current_page)
echo.out("Done with wishlist, {} total pages parsed (from {} to {})",
len(pages), start_page, stop_page)
def main_dump(name, **kwargs):
"""This is really here just to test that I can parse a wishlist completely and
to demonstrate (by looking at the code) how to iterate through a list"""
name = name[0]
#pout.v(name, start_page, stop_page, kwargs)
#pout.x()
w = Wishlist(name)
i = 1
for i, item in enumerate(w, 1):
try:
item_json = item.jsonable()
echo.out("{}. {} is ${:.2f}", i, item_json["title"],
item_json["price"])
except RobotError:
raise
except ParseError as e:
echo.err("{}. Failed!", i)
echo.err(e.body)
echo.exception(e)
echo.out("Done with wishlist, {} total items", i)
def get_webpage(self, filename):
body = self.get_body(filename)
w = Wishlist()
w.set_current("http://example.com/{}".format(filename), body)
return w
def get_items(self, filename, name="WISHLIST_NAME"):
soup = self.get_soup(filename)
w = Wishlist(name)
items = list(w.get_items(soup, w.get_wishlist_url()))
return items