def test_parameter_processing(self, session, Account):
account = Account(balance=(Currency('USD'), 15))
session.add(account)
session.commit()
account = session.query(Account).first()
assert account.balance.currency == Currency('USD')
assert account.balance.amount == 15
def test_parameter_processing(self, session, type_, Account):
account = Account(balances=[
type_.type_cls(Currency('USD'), 15),
type_.type_cls(Currency('AUD'), 20)
])
session.add(account)
session.commit()
account = session.query(Account).first()
assert account.balances[0].currency == Currency('USD')
assert account.balances[0].amount == 15
assert account.balances[1].currency == Currency('AUD')
assert account.balances[1].amount == 20
def test_parameter_processing(self, session, User, set_get_locale):
user = User(currency=Currency('USD'))
session.add(user)
session.commit()
user = session.query(User).first()
assert user.currency.name == u'US Dollar'
def valid_currency(form, field):
from sqlalchemy_utils import Currency
try:
Currency(field.data)
except (TypeError, ValueError):
raise ValidationError(
field.gettext(
u'Not a valid ISO currency code (e.g. USD, EUR, CNY).'))
def test_dict_input(self, session, type_, Account):
account = Account(balances=[{
'currency': Currency('USD'),
'amount': 15
}, {
'currency': Currency('AUD'),
'amount': 20
}])
session.add(account)
session.commit()
account = session.query(Account).first()
assert account.balances[0].currency == Currency('USD')
assert account.balances[0].amount == 15
assert account.balances[1].currency == Currency('AUD')
assert account.balances[1].amount == 20
def test_dict_input(self, session, Account):
account = Account(balance={'currency': Currency('USD'), 'amount': 15})
session.add(account)
session.commit()
account = session.query(Account).first()
assert account.balance.currency == 'USD'
assert account.balance.amount == 15
def test_result_set_processing(self):
account = self.Account(balance=('USD', 15))
self.session.add(account)
self.session.commit()
account = self.session.query(self.Account).first()
assert account.balance.currency == Currency('USD')
assert account.balance.amount == 15
def test_parameter_processing(self):
user = self.User(
currency=Currency('USD')
)
self.session.add(user)
self.session.commit()
user = self.session.query(self.User).first()
assert user.currency.name == u'US Dollar'
def test_symbol_property(self, code, symbol):
assert Currency(code).symbol == symbol
def test_name_property(self, code, name):
assert Currency(code).name == name
def test_invalid_currency_code_type(self):
with raises(TypeError):
Currency(None)
def test_invalid_currency_code(self):
with raises(ValueError):
Currency('Unknown code')
def __parse_2021_auction_soup(self, soup, auction_id):
def get_embedded_image_urls():
imgs = soup.find_all('img', attrs= \
{'class' : re.compile('Thumbnail__StyledThumbnailImage')})
urls = []
for i in imgs:
urllist = i['src'].split('.')
urllist[-1] = 'jpg'
urls.append('.'.join(urllist))
return urls
json = self.__extract_data_json(soup)
item = json['item']['byId'][str(auction_id)]
item_detail = json['itemDetail']['byId'][str(auction_id)]
bidding_info = json['biddingInfo']['byId'][str(auction_id)]
catalog = json['catalog']['byId'][str(item['catalogId'])]
# Extract additional auctioneer info
seller_id = item['sellerId']
seller = json['seller']['byId'][str(seller_id)]
auctioneer = seller['name']
location = self.__address_from_seller(seller)
image_urls = ' '.join(get_embedded_image_urls())
# Construct the auction object
auction = LiveAuctioneersAuction(id=str(auction_id))
try:
auction.title = item['title']
except KeyError:
pass
except ValueError:
print('auction {} received title {} of invalid type {}' \
.format(auction_id, item['title'], \
type(item['title'])))
try:
auction.description = self._normalise_text(
item_detail['description'])
except KeyError:
pass
except ValueError:
print('auction {} received description {} of invalid type {}' \
.format(auction_id, item_detail['description'], \
type(item_detail['description'])))
try:
isostring = item['publishDate']
# datetime's ISO 8601 parser doesn't understand 'Z' at the end for UTC,
# so rewrite it into an easier variant
if isostring[-1] == 'Z':
auction.start_time = datetime.fromisoformat(isostring[:-1] +
'+00:00')
else:
raise KeyError
except KeyError:
pass
except TypeError:
print('auction {} received start_time {} of invalid type {}' \
.format(auction_id, isostring, \
type(isostring)))
try:
auction.end_time = datetime.utcfromtimestamp(
catalog['saleStartTs'])
except KeyError:
pass
except TypeError:
print('auction {} received end_time {} of invalid type {}' \
.format(auction_id, catalog['saleStartTs'], \
type(catalog['saleStartTs'])))
try:
auction.n_bids = int(bidding_info['bidCount'])
except KeyError:
pass
except ValueError:
print('auction {} received n_bids {} of invalid type {}' \
.format(auction_id, bidding_info['bidCount'], \
type(bidding_info['bidCount'])))
auction.currency = Currency('USD')
try:
auction.latest_price = float(bidding_info['salePrice'])
except KeyError:
pass
except ValueError:
print('auction {} received latest_price {} of invalid type {}' \
.format(auction_id, bidding_info['salePrice'], \
type(bidding_info['salePrice'])))
try:
auction.starting_price = float(item['startPrice'])
except KeyError:
pass
except ValueError:
print('auction {} received starting_price {} of invalid type {}' \
.format(auction_id, item['startPrice'], \
type(item['startPrice'])))
auction.location = location
try:
auction.lot_number = int(re.sub('[^0-9]', '', item['lotNumber']))
except KeyError:
pass
except ValueError:
print('auction {} received lotNumber {} of invalid type {}' \
.format(auction_id, item['lotNumber'], \
type(item['lotNumber'])))
auction.image_urls = image_urls
try:
auction.condition = item_detail['conditionReport']
except KeyError:
pass
except ValueError:
print('auction {} received condition {} of invalid type {}' \
.format(auction_id, item_detail['conditionReport'], \
type(item_detail['conditionReport'])))
try:
auction.high_bid_estimate = float(item['highBidEstimate'])
except KeyError:
pass
except ValueError:
print('auction {} received high_bid_estimate {} of invalid type {}' \
.format(auction_id, item['highBidEstimate'], \
type(item['highBidEstimate'])))
try:
auction.low_bid_estimate = float(item['lowBidEstimate'])
except KeyError:
pass
except ValueError:
print('auction {} received low_bid_estimate {} of invalid type {}' \
.format(auction_id, item['lowBidEstimate'], \
type(item['lowBidEstimate'])))
try:
auction.seller_id = str(seller_id)
except KeyError:
pass
return auction
def test_representation(self):
currency = Currency('USD')
assert repr(currency) == "Currency('USD')"
def test_unicode(self):
currency = Currency('USD')
assert six.text_type(currency) == u'USD'
def test_equality_operator(self):
assert Currency('USD') == 'USD'
assert 'USD' == Currency('USD')
assert Currency('USD') == Currency('USD')
def test_non_equality_operator(self):
assert Currency('USD') != 'EUR'
assert not (Currency('USD') != 'USD')
def test_init(self):
assert Currency('USD') == Currency(Currency('USD'))
def test_str(self):
currency = Currency('USD')
assert str(currency) == 'USD'
def test_hashability(self):
assert len(set([Currency('USD'), Currency('USD')])) == 1
def __parse_2020_auction_soup(self, soup, duplicates):
def get_embedded_json():
# Strip c from s, without exception
def strip(s, c):
if isinstance(s, str):
return s.strip(c)
return s
div = soup.find('div', id='JSDF')
scripts = div.find_all('script', src=None)
# Look for $rwidgets
script_texts = []
for script in scripts:
for s in script.contents:
if '$rwidgets' in s:
script_texts.append(s)
# Bodge until we get rid of slimit
with silence_output():
parser = Parser()
raw_values = {}
for script_text in script_texts:
tree = parser.parse(script_text)
# Parsing js
for node in nodevisitor.visit(tree):
if isinstance(node, ast.FunctionCall):
if isinstance(node.identifier, ast.Identifier):
if node.identifier.value == '$rwidgets':
# Deal with here
fields = {}
for n in nodevisitor.visit(node):
if isinstance(n, ast.Assign):
k = getattr(n.left, 'value',
'').strip('"')
v = strip(
getattr(n.right, 'value', ''), '"')
if k in duplicates:
try:
fields[k].append(v)
except KeyError:
fields[k] = [v]
else:
fields[k] = v
# Merge fields and raw_values, resolving duplicates
for (k, v) in fields.items():
if k in duplicates:
try:
raw_values[k] += v
except KeyError:
raw_values[k] = v
elif v != 'null':
raw_values[k] = v
return raw_values
def get_image_urls(raw_values):
# TODO: sometimes only displayImgUrl is given, when the s-l600 image exists
# Example: https://www.ebay.com/itm/Chubby-Blob-Seal-Plush-Toy-Animal-Cute-Ocean-Pillow-Pet-Stuffed-Doll-Kids-Gift/362995774962
# Example: https://i.ebayimg.com/images/g/6NkAAOSwkEFd50Kb/s-l600.jpg
raw_image_urls = []
if 'maxImageUrl' in raw_values.keys():
if 'displayImgUrl' in raw_values.keys():
for max_image, disp_image in zip( \
self.__get_dict_value(raw_values, 'maxImageUrl'), \
self.__get_dict_value(raw_values, 'displayImgUrl')):
if max_image == 'null':
if disp_image != 'null':
raw_image_urls.append(disp_image)
else:
raw_image_urls.append(max_image)
else:
raw_image_urls = self.__get_dict_value(
raw_values, 'maxImageUrl')
else:
if 'displayImgUrl' in raw_values.keys():
raw_image_urls = self.__get_dict_value(
raw_values, 'displayImgUrl')
def f(url):
return json.loads('"{}"'.format(url))
return list(map(f, raw_image_urls))
# Get raw values, validating API assumptions
raw_values = get_embedded_json()
#pdb.set_trace()
try:
raw_values['it']
except KeyError:
try:
raw_values['kw']
except KeyError:
# No title field available in API. Obtain it from the page instead
for c in soup.find('h1', id='itemTitle').contents:
if isinstance(c, str):
title_key = c
break
else:
title_key = 'kw'
else:
title_key = 'it'
try:
if raw_values['entityId'] != raw_values['entityName']:
print(
f'notify author: entityid==entityname assumption incorrect for domain {url}'
)
except KeyError:
print(
'notify author: entityid or entityname does not exist for auction {}, for domain {}.'
.format(raw_values['itemId'], url))
# Extract additional info
image_urls = get_image_urls(raw_values)
desc_div = soup.find('div', attrs={'id': 'desc_div'})
if desc_div is not None:
iframe = desc_div.find('iframe')
else:
iframe = soup.find('div', attrs={'class': 'vi_descsnpt_holder'})
desc = iframe.text
# Construct the auction object
auction = EbayAuction(id=int(raw_values['itemId']))
try:
auction.title = unicodedata.normalize("NFKD",
raw_values[title_key])
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received title {} of invalid type {}' \
.format(auction_id, raw_values[title_key], \
type(raw_values[title_key])))
try:
auction.description = self._normalise_text(desc)
except ValueError:
print('auction {} received description {} of invalid type {}' \
.format(auction_id, desc, \
type(desc)))
try:
auction.seller_id = str(raw_values['entityName'])
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received seller_id {} of invalid type {}' \
.format(auction_id, raw_values['entityName'], \
type(raw_values['entityName'])))
try:
auction.start_time = datetime.utcfromtimestamp( \
int(raw_values['startTime'])/1000)
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received start_time {} of invalid type {}' \
.format(auction_id, raw_values['startTime'], \
type(raw_values['startTime'])))
try:
auction.end_time = datetime.utcfromtimestamp( \
int(raw_values['endTime'])/1000)
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received end_time {} of invalid type {}' \
.format(auction_id, raw_values['endTime'], \
type(raw_values['endTime'])))
try:
auction.n_bids = int(raw_values['bids'])
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received n_bids {} of invalid type {}' \
.format(auction_id, raw_values['bids'], \
type(raw_values['bids'])))
try:
auction.currency = Currency(raw_values['ccode'])
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received currency {} of invalid type {}' \
.format(auction_id, raw_values['ccode'], \
type(raw_values['ccode'])))
try:
auction.latest_price = str(float(raw_values['bidPriceDouble']))
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received latest_price {} of invalid type {}' \
.format(auction_id, raw_values['bidPriceDouble'], \
type(raw_values['bidPriceDouble'])))
try:
auction.buy_now_price = str(float(raw_values['binPriceDouble']))
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received buy_now_price {} of invalid type {}' \
.format(auction_id, raw_values['binPriceDouble'], \
type(raw_values['binPriceDouble'])))
# TODO: add starting price, winner, location
auction.image_urls = ' '.join(image_urls)
try:
auction.locale = str(raw_values['locale'])
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received locale {} of invalid type {}' \
.format(auction_id, raw_values['locale'], \
type(raw_values['locale'])))
try:
auction.quantity = int(raw_values['totalQty'])
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received quantity {} of invalid type {}' \
.format(auction_id, raw_values['quantity'], \
type(raw_values['quantity'])))
try:
auction.video_url = str(raw_values['videoUrl'])
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received video_url {} of invalid type {}' \
.format(auction_id, raw_values['videoUrl'], \
type(raw_values['videoUrl'])))
try:
if str(raw_values['vatIncluded']) == 'true':
auction.vat_included = True
elif str(raw_values['vatIncluded']) == 'false':
auction.vat_included = False
else:
print('auction {} received vat_included {} of invalid type {}' \
.format(auction_id, raw_values['vatIncluded'], \
type(raw_values['vatIncluded'])))
except (KeyError, TypeError):
pass
try:
auction.domain = raw_values['currentDomain']
except (KeyError, TypeError):
pass
except ValueError:
print('auction {} received domain {} of invalid type {}' \
.format(auction_id, raw_values['currentDomain'], \
type(raw_values['currentDomain'])))
return auction
def __parse_2020_auction_soup(self, soup, auction_id):
def get_embedded_json():
jsonsect = self.__searcher(soup, \
'<script data-reactroot="">', '</script>')
# Bodge until we get rid of slimit
with silence_output():
parser = Parser()
tree = parser.parse(jsonsect)
j = 0
element = {}
for node in nodevisitor.visit(tree):
try:
if isinstance(node, ast.Assign):
if node.left.value == '"' + str(auction_id) + '"':
for i in node.right.children():
leftp = i.left.value[1:-1]
rightp = i.right.value
element[leftp] = self.__quote_cleaner(rightp)
except:
pass
return element
def get_embedded_image_urls():
divs = soup.find_all('div', attrs= \
{'class' : re.compile('thumbnail-container')})
urls = []
for i in divs:
urllist = i.find('img')['src'].split('.')
urllist[-1] = 'jpg'
urls.append('.'.join(urllist))
return urls
element = get_embedded_json()
# Extract additional auctioneer info
div = soup.find('div', attrs={'class': re.compile('auctioneerInfo')})
auctioneer = self.__quote_cleaner( \
div.find('h3', attrs= {'class' : re.compile('sellerName')}).text)
location = self.__quote_cleaner( \
div.find('div', attrs={'class' : re.compile('address')}).text)
image_urls = ' '.join(get_embedded_image_urls())
# Construct the auction object
auction = LiveAuctioneersAuction(id=str(auction_id))
try:
auction.title = element['title']
except KeyError:
pass
except ValueError:
print('auction {} received title {} of invalid type {}' \
.format(auction_id, element['title'], \
type(element['title'])))
try:
auction.description = self._normalise_text(element['description'])
except KeyError:
pass
except ValueError:
print('auction {} received description {} of invalid type {}' \
.format(auction_id, element['description'], \
type(element['description'])))
try:
auction.start_time = datetime.utcfromtimestamp( \
element['availableTs'])
except KeyError:
pass
except TypeError:
print('auction {} received start_time {} of invalid type {}' \
.format(auction_id, element['availableTs'], \
type(element['availableTs'])))
try:
auction.end_time = datetime.utcfromtimestamp( \
element['saleStartTs'])
except KeyError:
pass
except TypeError:
print('auction {} received end_time {} of invalid type {}' \
.format(auction_id, element['saleStartTs'], \
type(element['saleStartTs'])))
try:
auction.n_bids = int(element['bidCount'])
except KeyError:
pass
except ValueError:
print('auction {} received n_bids {} of invalid type {}' \
.format(auction_id, element['bidCount'], \
type(element['bidCount'])))
auction.currency = Currency('USD')
try:
auction.latest_price = float(element['salePrice'])
except KeyError:
pass
except ValueError:
print('auction {} received latest_price {} of invalid type {}' \
.format(auction_id, element['salePrice'], \
type(element['salePrice'])))
try:
auction.starting_price = float(element['startPrice'])
except KeyError:
pass
except ValueError:
print('auction {} received starting_price {} of invalid type {}' \
.format(auction_id, element['startPrice'], \
type(element['startPrice'])))
auction.location = location
try:
auction.lot_number = int(re.sub('[^0-9]', '',
element['lotNumber']))
except KeyError:
pass
except ValueError:
print('auction {} received lotNumber {} of invalid type {}' \
.format(auction_id, element['lotNumber'], \
type(element['lotNumber'])))
auction.image_urls = image_urls
try:
auction.condition = str(element['conditionReport'])
except KeyError:
pass
except ValueError:
print('auction {} received condition {} of invalid type {}' \
.format(auction_id, element['conditionReport'], \
type(element['conditionReport'])))
try:
auction.high_bid_estimate = float(element['highBidEstimate'])
except KeyError:
pass
except ValueError:
print('auction {} received high_bid_estimate {} of invalid type {}' \
.format(auction_id, element['highBidEstimate'], \
type(element['highBidEstimate'])))
try:
auction.low_bid_estimate = float(element['lowBidEstimate'])
except KeyError:
pass
except ValueError:
print('auction {} received low_bid_estimate {} of invalid type {}' \
.format(auction_id, element['lowBidEstimate'], \
type(element['lowBidEstimate'])))
try:
auction.seller_id = str(element['sellerId'])
except KeyError:
pass
return auction