def test_parse_to_ts() -> None: s = '1970-01-01 00:00:00' assert dtutil.parse_to_ts(s, '%Y-%m-%d %H:%M:%S', timezone('UTC')) == 0 s = '1970-01-01 00:01:59' assert dtutil.parse_to_ts(s, '%Y-%m-%d %H:%M:%S', timezone('UTC')) == 119 assert str(dtutil.parse_to_ts(s, '%Y-%m-%d %H:%M:%S', timezone('UTC'))) == '119'
def update_bugged_cards() -> None: bugs = fetcher.bugged_cards() if bugs is None: return db().begin('update_bugged_cards') db().execute('DELETE FROM card_bug') for bug in bugs: last_confirmed_ts = dtutil.parse_to_ts(bug['last_updated'], '%Y-%m-%d %H:%M:%S', dtutil.UTC_TZ) name = bug['card'].split( ' // ' )[0] # We need a face name from split cards - we don't have combined card names yet. card_id = db().value('SELECT card_id FROM face WHERE name = %s', [name]) if card_id is None: print('UNKNOWN BUGGED CARD: {card}'.format(card=bug['card'])) continue db().execute( 'INSERT INTO card_bug (card_id, description, classification, last_confirmed, url, from_bug_blog, bannable) VALUES (%s, %s, %s, %s, %s, %s, %s)', [ card_id, bug['description'], bug['category'], last_confirmed_ts, bug['url'], bug['bug_blog'], bug['bannable'] ]) db().commit('update_bugged_cards')
def fetch() -> None: all_prices, timestamps = {}, [] ch_urls = configuration.get_list('cardhoarder_urls') if ch_urls: for _, url in enumerate(ch_urls): s = fetch_tools.fetch(url) s = ftfy.fix_encoding(s) timestamps.append( dtutil.parse_to_ts( s.split('\n', 1)[0].replace('UPDATED ', ''), '%Y-%m-%dT%H:%M:%S+00:00', dtutil.CARDHOARDER_TZ)) all_prices[url] = parser.parse_cardhoarder_prices(s) url = configuration.get_str('mtgotraders_url') if url: s = fetch_tools.fetch(url) timestamps.append(dtutil.dt2ts(dtutil.now())) all_prices['mtgotraders'] = parser.parse_mtgotraders_prices(s) if not timestamps: raise TooFewItemsException( 'Did not get any prices when fetching {urls} ({all_prices})'. format(urls=itertools.chain( configuration.get_list('cardhoarder_urls'), [configuration.get_str('mtgotraders_url')]), all_prices=all_prices)) count = store(min(timestamps), all_prices) cleanup(count)
def parse_created_date(soup: BeautifulSoup) -> int: description = str(soup.select_one('div.deck-view-description')) try: date_s = re.findall(r'([A-Z][a-z][a-z] \d+, \d\d\d\d)', description)[0] except IndexError as e: raise InvalidDataException( f'Unable to find a date in {description} because of {e}') from e return dtutil.parse_to_ts(date_s, '%b %d, %Y', dtutil.MTGGOLDFISH_TZ)
def scrape_created_date(d): soup = BeautifulSoup( fetcher.internal.fetch(d.url, character_encoding='utf-8'), 'html.parser') description = soup.select_one( 'div.deck-view-description').renderContents().decode('utf-8') date_s = re.findall(r'([A-Z][a-z][a-z] \d+, \d\d\d\d)', description)[0] return dtutil.parse_to_ts(date_s, '%b %d, %Y', dtutil.MTGGOLDFISH_TZ)
def fetch(): all_prices, timestamps = {}, [] for i, url in enumerate(configuration.get('cardhoarder_urls')): s = fetcher_internal.fetch(url) s = ftfy.fix_encoding(s) timestamps.append(dtutil.parse_to_ts(s.split('\n', 1)[0].replace('UPDATED ', ''), '%Y-%m-%dT%H:%M:%S+00:00', dtutil.CARDHOARDER_TZ)) all_prices[i] = parse_cardhoarder_prices(s) url = configuration.get('mtgotraders_url') if url: s = fetcher_internal.fetch(configuration.get('mtgotraders_url')) timestamps.append(dtutil.dt2ts(dtutil.now())) all_prices['mtgotraders'] = parse_mtgotraders_prices(s) if not timestamps: raise TooFewItemsException('Did not get any prices when fetching {urls} ({all_prices})'.format(urls=configuration.get('cardhoarder_urls') + [configuration.get('mtgotraders_url')], all_prices=all_prices)) store(min(timestamps), all_prices)
def update_bugged_cards(use_transaction=True): bugs = fetcher.bugged_cards() if bugs is None: return if use_transaction: db().begin() db().execute("DELETE FROM card_bugs") for name, bug, classification, last_confirmed in bugs: last_confirmed_ts = dtutil.parse_to_ts(last_confirmed, '%Y-%m-%d %H:%M:%S', dtutil.UTC_TZ) card_id = db().value("SELECT card_id FROM face WHERE name = ?", [name]) if card_id is None: print("UNKNOWN BUGGED CARD: {card}".format(card=name)) continue db().execute( "INSERT INTO card_bugs (card_id, description, classification, last_confirmed) VALUES (?, ?, ?, ?)", [card_id, bug, classification, last_confirmed_ts]) if use_transaction: db().commit()
def update_bugged_cards(use_transaction: bool = True) -> None: bugs = fetcher.bugged_cards() if bugs is None: return if use_transaction: db().begin() db().execute('DELETE FROM card_bug') for bug in bugs: last_confirmed_ts = dtutil.parse_to_ts(bug['last_updated'], '%Y-%m-%d %H:%M:%S', dtutil.UTC_TZ) card_id = db().value('SELECT card_id FROM face WHERE name = %s', [bug['card']]) if card_id is None: print('UNKNOWN BUGGED CARD: {card}'.format(card=bug['card'])) continue db().execute( 'INSERT INTO card_bug (card_id, description, classification, last_confirmed, url, from_bug_blog, bannable) VALUES (%s, %s, %s, %s, %s, %s, %s)', [ card_id, bug['description'], bug['category'], last_confirmed_ts, bug['url'], bug['bug_blog'], bug['bannable'] ]) if use_transaction: db().commit()
def date2int(s: str, name: str) -> Union[str, float]: if name == 'released_at': return dtutil.parse_to_ts(s, '%Y-%m-%d', dtutil.WOTC_TZ) return s
def date2int(s, name): if name == 'release_date': return dtutil.parse_to_ts(s, '%Y-%m-%d', dtutil.WOTC_TZ) return s