def test_parse_to_ts() -> None:
s = '1970-01-01 00:00:00'
assert dtutil.parse_to_ts(s, '%Y-%m-%d %H:%M:%S', timezone('UTC')) == 0
s = '1970-01-01 00:01:59'
assert dtutil.parse_to_ts(s, '%Y-%m-%d %H:%M:%S', timezone('UTC')) == 119
assert str(dtutil.parse_to_ts(s, '%Y-%m-%d %H:%M:%S',
timezone('UTC'))) == '119'
def update_bugged_cards() -> None:
bugs = fetcher.bugged_cards()
if bugs is None:
return
db().begin('update_bugged_cards')
db().execute('DELETE FROM card_bug')
for bug in bugs:
last_confirmed_ts = dtutil.parse_to_ts(bug['last_updated'],
'%Y-%m-%d %H:%M:%S',
dtutil.UTC_TZ)
name = bug['card'].split(
' // '
)[0] # We need a face name from split cards - we don't have combined card names yet.
card_id = db().value('SELECT card_id FROM face WHERE name = %s',
[name])
if card_id is None:
print('UNKNOWN BUGGED CARD: {card}'.format(card=bug['card']))
continue
db().execute(
'INSERT INTO card_bug (card_id, description, classification, last_confirmed, url, from_bug_blog, bannable) VALUES (%s, %s, %s, %s, %s, %s, %s)',
[
card_id, bug['description'], bug['category'],
last_confirmed_ts, bug['url'], bug['bug_blog'], bug['bannable']
])
db().commit('update_bugged_cards')
def fetch() -> None:
all_prices, timestamps = {}, []
ch_urls = configuration.get_list('cardhoarder_urls')
if ch_urls:
for _, url in enumerate(ch_urls):
s = fetch_tools.fetch(url)
s = ftfy.fix_encoding(s)
timestamps.append(
dtutil.parse_to_ts(
s.split('\n', 1)[0].replace('UPDATED ', ''),
'%Y-%m-%dT%H:%M:%S+00:00', dtutil.CARDHOARDER_TZ))
all_prices[url] = parser.parse_cardhoarder_prices(s)
url = configuration.get_str('mtgotraders_url')
if url:
s = fetch_tools.fetch(url)
timestamps.append(dtutil.dt2ts(dtutil.now()))
all_prices['mtgotraders'] = parser.parse_mtgotraders_prices(s)
if not timestamps:
raise TooFewItemsException(
'Did not get any prices when fetching {urls} ({all_prices})'.
format(urls=itertools.chain(
configuration.get_list('cardhoarder_urls'),
[configuration.get_str('mtgotraders_url')]),
all_prices=all_prices))
count = store(min(timestamps), all_prices)
cleanup(count)
def parse_created_date(soup: BeautifulSoup) -> int:
description = str(soup.select_one('div.deck-view-description'))
try:
date_s = re.findall(r'([A-Z][a-z][a-z] \d+, \d\d\d\d)', description)[0]
except IndexError as e:
raise InvalidDataException(
f'Unable to find a date in {description} because of {e}') from e
return dtutil.parse_to_ts(date_s, '%b %d, %Y', dtutil.MTGGOLDFISH_TZ)
def scrape_created_date(d):
soup = BeautifulSoup(
fetcher.internal.fetch(d.url, character_encoding='utf-8'),
'html.parser')
description = soup.select_one(
'div.deck-view-description').renderContents().decode('utf-8')
date_s = re.findall(r'([A-Z][a-z][a-z] \d+, \d\d\d\d)', description)[0]
return dtutil.parse_to_ts(date_s, '%b %d, %Y', dtutil.MTGGOLDFISH_TZ)
def fetch():
all_prices, timestamps = {}, []
for i, url in enumerate(configuration.get('cardhoarder_urls')):
s = fetcher_internal.fetch(url)
s = ftfy.fix_encoding(s)
timestamps.append(dtutil.parse_to_ts(s.split('\n', 1)[0].replace('UPDATED ', ''), '%Y-%m-%dT%H:%M:%S+00:00', dtutil.CARDHOARDER_TZ))
all_prices[i] = parse_cardhoarder_prices(s)
url = configuration.get('mtgotraders_url')
if url:
s = fetcher_internal.fetch(configuration.get('mtgotraders_url'))
timestamps.append(dtutil.dt2ts(dtutil.now()))
all_prices['mtgotraders'] = parse_mtgotraders_prices(s)
if not timestamps:
raise TooFewItemsException('Did not get any prices when fetching {urls} ({all_prices})'.format(urls=configuration.get('cardhoarder_urls') + [configuration.get('mtgotraders_url')], all_prices=all_prices))
store(min(timestamps), all_prices)
def update_bugged_cards(use_transaction=True):
bugs = fetcher.bugged_cards()
if bugs is None:
return
if use_transaction:
db().begin()
db().execute("DELETE FROM card_bugs")
for name, bug, classification, last_confirmed in bugs:
last_confirmed_ts = dtutil.parse_to_ts(last_confirmed,
'%Y-%m-%d %H:%M:%S',
dtutil.UTC_TZ)
card_id = db().value("SELECT card_id FROM face WHERE name = ?", [name])
if card_id is None:
print("UNKNOWN BUGGED CARD: {card}".format(card=name))
continue
db().execute(
"INSERT INTO card_bugs (card_id, description, classification, last_confirmed) VALUES (?, ?, ?, ?)",
[card_id, bug, classification, last_confirmed_ts])
if use_transaction:
db().commit()
def update_bugged_cards(use_transaction: bool = True) -> None:
bugs = fetcher.bugged_cards()
if bugs is None:
return
if use_transaction:
db().begin()
db().execute('DELETE FROM card_bug')
for bug in bugs:
last_confirmed_ts = dtutil.parse_to_ts(bug['last_updated'],
'%Y-%m-%d %H:%M:%S',
dtutil.UTC_TZ)
card_id = db().value('SELECT card_id FROM face WHERE name = %s',
[bug['card']])
if card_id is None:
print('UNKNOWN BUGGED CARD: {card}'.format(card=bug['card']))
continue
db().execute(
'INSERT INTO card_bug (card_id, description, classification, last_confirmed, url, from_bug_blog, bannable) VALUES (%s, %s, %s, %s, %s, %s, %s)',
[
card_id, bug['description'], bug['category'],
last_confirmed_ts, bug['url'], bug['bug_blog'], bug['bannable']
])
if use_transaction:
db().commit()
def date2int(s: str, name: str) -> Union[str, float]:
if name == 'released_at':
return dtutil.parse_to_ts(s, '%Y-%m-%d', dtutil.WOTC_TZ)
return s
def date2int(s, name):
if name == 'release_date':
return dtutil.parse_to_ts(s, '%Y-%m-%d', dtutil.WOTC_TZ)
return s
