def fetch() -> None:
all_prices, timestamps = {}, []
ch_urls = configuration.get_list('cardhoarder_urls')
if ch_urls:
for _, url in enumerate(ch_urls):
s = fetch_tools.fetch(url)
s = ftfy.fix_encoding(s)
timestamps.append(
dtutil.parse_to_ts(
s.split('\n', 1)[0].replace('UPDATED ', ''),
'%Y-%m-%dT%H:%M:%S+00:00', dtutil.CARDHOARDER_TZ))
all_prices[url] = parser.parse_cardhoarder_prices(s)
url = configuration.get_str('mtgotraders_url')
if url:
s = fetch_tools.fetch(url)
timestamps.append(dtutil.dt2ts(dtutil.now()))
all_prices['mtgotraders'] = parser.parse_mtgotraders_prices(s)
if not timestamps:
raise TooFewItemsException(
'Did not get any prices when fetching {urls} ({all_prices})'.
format(urls=itertools.chain(
configuration.get_list('cardhoarder_urls'),
[configuration.get_str('mtgotraders_url')]),
all_prices=all_prices))
count = store(min(timestamps), all_prices)
cleanup(count)
def make_final_list() -> None:
planes = fetch_tools.fetch_json('https://api.scryfall.com/cards/search?q=t:plane%20or%20t:phenomenon')['data']
bad_names = [p['name'] for p in planes]
bad_names.extend(BANNED_CARDS)
files = rotation.files()
lines: List[str] = []
for line in fileinput.input(files):
line = text.sanitize(line)
if line.strip() in bad_names:
continue
lines.append(line)
scores = Counter(lines).most_common()
passed: List[str] = []
for name, count in scores:
if count >= rotation.TOTAL_RUNS / 2:
passed.append(name)
final = list(passed)
final.sort()
h = open(os.path.join(configuration.get_str('legality_dir'), 'legal_cards.txt'), mode='w', encoding='utf-8')
h.write(''.join(final))
h.close()
print('Generated legal_cards.txt. {0}/{1} cards.'.format(len(passed), len(scores)))
setcode = rotation.next_rotation_ex().mtgo_code
h = open(os.path.join(configuration.get_str('legality_dir'), f'{setcode}_legal_cards.txt'), mode='w', encoding='utf-8')
h.write(''.join(final))
h.close()
do_push()
def login(user: Optional[str] = None, password: Optional[str] = None) -> None:
if user is None:
user = configuration.get_str('to_username')
if password is None:
password = configuration.get_str('to_password')
if user == '' or password == '':
logger.warning('No TappedOut credentials provided')
return
url = 'https://tappedout.net/accounts/login/'
session = fetcher_internal.SESSION
response = session.get(url)
match = re.search(
r"<input type='hidden' name='csrfmiddlewaretoken' value='(\w+)' />",
response.text)
if match is None:
# Already logged in?
return
csrf = match.group(1)
data = {
'csrfmiddlewaretoken': csrf,
'next': '/',
'username': user,
'password': password,
}
headers = {
'referer': url,
}
logger.warning('Logging in to TappedOut as {0}'.format(user))
response = session.post(url, data=data, headers=headers)
if response.status_code == 403:
logger.warning('Failed to log in')
def decksite_url(path: str = '/') -> str:
hostname = configuration.get_str('decksite_hostname')
port = configuration.get_int('decksite_port')
if port != 80:
hostname = '{hostname}:{port}'.format(hostname=hostname, port=port)
url = parse.urlunparse((configuration.get_str('decksite_protocol'), hostname, path, '', '', ''))
assert url is not None
return url
def __init__(self, db: str) -> None:
warnings.filterwarnings('error', category=MySQLdb.Warning)
self.name = db
self.host = configuration.get_str('mysql_host')
self.port = configuration.get_int('mysql_port')
self.user = configuration.get_str('mysql_user')
self.passwd = configuration.get_str('mysql_passwd')
self.connect()
def do_push() -> None:
gh_repo = os.path.join(configuration.get_str('legality_dir'), 'gh_pages')
if not os.path.exists(gh_repo):
subprocess.run(['git', 'clone', 'https://github.com/PennyDreadfulMTG/pennydreadfulmtg.github.io.git', gh_repo], check=True)
setcode = rotation.next_rotation_ex().mtgo_code
files = ['legal_cards.txt', f'{setcode}_legal_cards.txt']
for fn in files:
source = os.path.join(configuration.get_str('legality_dir'), fn)
dest = os.path.join(gh_repo, fn)
shutil.copy(source, dest)
os.chdir(gh_repo)
subprocess.run(['git', 'add'] + files, check=True)
subprocess.run(['git', 'commit', '-m', f'{setcode} rotation'], check=True)
subprocess.run(['git', 'push'], check=True)
checklist = f"""{setcode} rotation checklist
https://pennydreadfulmagic.com/admin/rotation/
- [ ] upload legal_cards.txt to S3
- [ ] upload {setcode}_legal_cards.txt to S3
- [ ] ping scryfall
- [ ] email mtggoldfish
- [ ] ping tappedout
"""
if redis.get_str('discordbot:commit_id'):
redis.store('discordbot:do_reboot', True)
else:
checklist += '- [ ] restart discordbot'
ds = os.path.expanduser('~/decksite/')
failed = False
try:
if os.path.exists(ds):
os.chdir(ds)
subprocess.run(['python3', 'run.py', 'maintenance', 'post_rotation'], check=True)
else:
failed = True
except Exception: # pylint: disable=broad-except
failed = True
if failed:
checklist += '- [ ] run post_rotation\n'
try:
fetch_tools.post('https://gatherling.com/util/updateDefaultFormats.php')
except fetch_tools.FetchException:
checklist += '- [ ] Update Gatherling legal cards list'
for path in ['/etc/uwsgi/vassals/decksite.ini', '/home/discord/vassals/decksite.ini']:
srv = pathlib.Path(path)
if srv.exists():
srv.touch()
break
else:
checklist += '- [ ] touch /etc/uwsgi/vassals/decksite.ini\n'
repo.create_issue(checklist, 'rotation script', 'rotation')
def report(form: ReportForm) -> bool:
try:
db().get_lock('deck_id:{id}'.format(id=form.entry))
db().get_lock('deck_id:{id}'.format(id=form.opponent))
for m in match.get_matches(form):
if int(form.opponent) == m.opponent_deck_id:
form.errors[
'result'] = 'This match was reported as You {game_wins}–{game_losses} {opponent} {date}'.format(
game_wins=m.game_wins,
game_losses=m.game_losses,
opponent=m.opponent,
date=dtutil.display_date(m.date))
return False
counts = deck.count_matches(form.entry, form.opponent)
if counts[int(form.entry)] >= 5:
form.errors['entry'] = 'You already have 5 matches reported'
return False
if counts[int(form.opponent)] >= 5:
form.errors[
'opponent'] = 'Your opponent already has 5 matches reported'
return False
pdbot = form.get('api_token',
None) == configuration.get('pdbot_api_token')
if pdbot:
mtgo_match_id = form.get('matchID', None)
else:
mtgo_match_id = None
entry_name = deck.load_deck(int(form.entry)).person
opp_name = deck.load_deck(int(form.opponent)).person
fetcher.post_discord_webhook(
configuration.get_str('league_webhook_id'),
configuration.get_str('league_webhook_token'),
'{entry} reported {f.entry_games}-{f.opponent_games} vs {opponent}'
.format(f=form, entry=entry_name, opponent=opp_name))
db().begin()
match.insert_match(dtutil.now(), form.entry, form.entry_games,
form.opponent, form.opponent_games, None, None,
mtgo_match_id)
db().commit()
return True
except LockNotAcquiredException:
form.errors[
'entry'] = 'Cannot report right now, somebody else is reporting a match for you or your opponent. Try again a bit later'
return False
finally:
db().release_lock('deck_id:{id}'.format(id=form.opponent))
db().release_lock('deck_id:{id}'.format(id=form.entry))
def run() -> None:
files = rotation.files()
n = len(files)
time_until = min(
TIME_UNTIL_FULL_ROTATION,
TIME_UNTIL_SUPPLEMENTAL_ROTATION) - datetime.timedelta(weeks=1)
if n >= TOTAL_RUNS:
print(
'It is the moment of discovery, the triumph of the mind, and the end of this rotation.'
)
return
if n == 0 and TIME_UNTIL_FULL_ROTATION > datetime.timedelta(
7) and TIME_UNTIL_SUPPLEMENTAL_ROTATION > datetime.timedelta(7):
print(
'The monks of the North Tree rarely saw their kodama until the rotation, when it woke like a slumbering, angry bear.'
)
print('ETA: {t}'.format(
t=dtutil.display_time(time_until.total_seconds())))
return
all_prices = {}
for url in configuration.get_list('cardhoarder_urls'):
s = fetcher_internal.fetch(url)
s = ftfy.fix_encoding(s)
all_prices[url] = parse_cardhoarder_prices(s)
url = configuration.get_str('mtgotraders_url')
if url:
s = fetcher_internal.fetch(url)
all_prices['mtgotraders'] = parse_mtgotraders_prices(s)
run_number = process(all_prices)
if run_number == TOTAL_RUNS:
make_final_list()
def make_final_list() -> None:
planes = fetcher_internal.fetch_json(
'https://api.scryfall.com/cards/search?q=t:plane%20or%20t:phenomenon'
)['data']
plane_names = [p['name'] for p in planes]
files = rotation.files()
lines: List[str] = []
for line in fileinput.input(files):
line = text.sanitize(line)
if line in plane_names:
print(f'DISCARDED: [{line}] is a plane.')
continue
lines.append(line)
scores = Counter(lines).most_common()
passed: List[str] = []
for name, count in scores:
if count >= TOTAL_RUNS / 2:
passed.append(name)
final = list(passed)
if is_supplemental():
temp = set(passed)
final = list(temp.union([c + '\n' for c in fetcher.legal_cards()]))
final.sort()
h = open(os.path.join(configuration.get_str('legality_dir'),
'legal_cards.txt'),
mode='w',
encoding='utf-8')
h.write(''.join(final))
h.close()
print('Generated legal_cards.txt. {0}/{1} cards.'.format(
len(passed), len(scores)))
def make_final_list() -> None:
files = rotation.files()
lines: List[str] = []
for line in fileinput.input(files):
line = text.sanitize(line)
lines.append(line)
scores = Counter(lines).most_common()
passed: List[str] = []
for name, count in scores:
if count >= TOTAL_RUNS / 2:
passed.append(name)
final = list(passed)
if is_supplemental():
temp = set(passed)
final = list(temp.union([c + '\n' for c in fetcher.legal_cards()]))
final.sort()
h = open(os.path.join(configuration.get_str('legality_dir'),
'legal_cards.txt'),
mode='w',
encoding='utf-8')
h.write(''.join(final))
h.close()
print('Generated legal_cards.txt. {0}/{1} cards.'.format(
len(passed), len(scores)))
async def post_cards(
client: Client,
cards: List[Card],
channel: Channel,
replying_to: Optional[Member] = None,
additional_text: str = ''
) -> None:
await client.send_typing(channel)
if len(cards) == 0:
await post_no_cards(client, channel, replying_to)
return
disable_emoji = channel.id in configuration.get_str('not_pd').split(',')
cards = uniqify_cards(cards)
if len(cards) > MAX_CARDS_SHOWN:
cards = cards[:DEFAULT_CARDS_SHOWN]
if len(cards) == 1:
text = single_card_text_internal(client, cards[0], disable_emoji)
else:
text = ', '.join('{name} {legal} {price}'.format(name=card.name, legal=((emoji.legal_emoji(card)) if not disable_emoji else ''), price=((fetcher.card_price_string(card, True)) if card.get('mode', None) == '$' else '')) for card in cards)
if len(cards) > MAX_CARDS_SHOWN:
image_file = None
else:
image_file = image_fetcher.download_image(cards)
if image_file is None:
text += '\n\n'
if len(cards) == 1:
text += emoji.replace_emoji(cards[0].text, client)
else:
text += 'No image available.'
text += additional_text
if image_file is None:
await client.send_message(channel, text)
else:
await send_image_with_retry(client, channel, image_file, text)
def read_rotation_files() -> Tuple[int, int, List[Card]]:
runs_str = redis.get_str('decksite:rotation:summary:runs')
runs_percent_str = redis.get_str('decksite:rotation:summary:runs_percent')
cards = redis.get_list('decksite:rotation:summary:cards')
if runs_str is not None and runs_percent_str is not None and cards is not None:
return int(runs_str), int(runs_percent_str), [
Card(c, predetermined_values=True) for c in cards
]
lines = []
fs = files()
if len(fs) == 0:
if not os.path.isdir(configuration.get_str('legality_dir')):
raise DoesNotExistException(
'Invalid configuration. Could not find legality_dir.')
return (0, 0, [])
latest_list = open(fs[-1], 'r').read().splitlines()
for filename in fs:
for line in get_file_contents(filename):
line = text.sanitize(line)
lines.append(line.strip())
scores = Counter(lines).most_common()
runs = scores[0][1]
runs_percent = round(round(runs / TOTAL_RUNS, 2) * 100)
cs = oracle.cards_by_name()
cards = []
for name, hits in scores:
c = process_score(name, hits, cs, runs, latest_list)
if c is not None:
cards.append(c)
redis.store('decksite:rotation:summary:runs', runs, ex=604800)
redis.store('decksite:rotation:summary:runs_percent',
runs_percent,
ex=604800)
redis.store('decksite:rotation:summary:cards', cards, ex=604800)
return (runs, runs_percent, cards)
async def handle_command(message: Message, client: Client) -> None:
parts = message.content.split(' ', 1)
method = find_method(parts[0])
if parts[0].lower() in configuration.get_str('otherbot_commands').split(
','):
return
args = ''
if len(parts) > 1:
args = parts[1]
if method is not None:
try:
await method(Commands,
client=client,
channel=message.channel,
args=args,
author=message.author)
except Exception as e: # pylint: disable=broad-except
print('Caught exception processing command `{cmd}`'.format(
cmd=message.content))
tb = traceback.format_exc()
print(tb)
await client.send_message(
message.channel,
'{author}: I know the command `{cmd}` but I could not do that.'
.format(cmd=parts[0], author=message.author.mention))
await getattr(Commands, 'bug')(
Commands, client, message.channel,
'Command failed with {c}: {cmd}\n\n```\n{tb}\n```'.format(
c=e.__class__.__name__, cmd=message.content,
tb=tb), message.author)
def run() -> None:
wd = configuration.get_str('modo_bugs_dir')
if not os.path.exists(wd):
subprocess.run(['git', 'clone', 'https://github.com/PennyDreadfulMTG/modo-bugs.git', wd])
os.chdir(wd)
subprocess.run(['git', 'pull'])
args = sys.argv[2:]
if not args:
args.extend(['scrape', 'update', 'verify', 'commit'])
print('modo_bugs invoked with modes: ' + repr(args))
changes: List[str] = []
if 'scrape' in args:
args.extend(['scrape_bb', 'scrape_an'])
if 'scrape_bb' in args:
scrape_bugblog.main(changes)
if 'scrape_an' in args:
scrape_announcements.main(changes)
if 'update' in args:
update.main()
if 'verify' in args:
verification.main()
if 'commit' in args:
subprocess.run(['git', 'add', '.'])
subprocess.run(['git', 'commit', '-m', 'Updated'])
user = configuration.get('github_user')
pword = configuration.get('github_password')
subprocess.run(['git', 'push', f'https://{user}:{pword}@github.com/PennyDreadfulMTG/modo-bugs.git'])
def run() -> None:
"""Make a 'safe' (no personal info) copy of the current prod db for download by devs."""
host = configuration.get_str('mysql_host')
port = configuration.get_int('mysql_port')
usr = configuration.get_str('mysql_user')
pwd = configuration.get_str('mysql_passwd')
db = configuration.get_str('decksite_database')
if not (host or port or usr or pwd or db):
safe_pwd = 'PRESENT' if pwd else 'MISSING'
raise InvalidArgumentException(f'Unable to dump dev db with {host} {port} {usr} pwd:{safe_pwd} {db}')
base_command = ['mysqldump', '-h', host, '-P', str(port), '-u', usr, f'-p{pwd}']
structure = subprocess.check_output(base_command + ['--no-data', db])
data = subprocess.check_output(base_command + [f'--ignore-table={db}.person_note', db])
with gzip.open('shared_web/static/dev-db.sql.gz', 'wb') as f:
f.write(structure)
f.write(data)
def cache() -> None:
db = database.get_database(configuration.get_str('prices_database'))
now = round(time.time())
week = now - 60 * 60 * 24 * 7
month = now - 60 * 60 * 24 * 7 * 30
last_rotation = int(rotation.last_rotation().timestamp())
sql = 'SELECT MAX(`time`) FROM low_price'
latest = db.value(sql)
db.begin()
db.execute('DELETE FROM cache')
sql = """
INSERT INTO cache (`time`, name, price, low, high, week, month, season)
SELECT
MAX(`time`) AS `time`,
name,
MIN(CASE WHEN `time` = %s THEN price END) AS price,
MIN(CASE WHEN `time` > %s THEN price END) AS low,
MAX(CASE WHEN `time` > %s THEN price END) AS high,
AVG(CASE WHEN `time` > %s AND price = 1 THEN 1 WHEN `time` > %s THEN 0 END) AS week,
AVG(CASE WHEN `time` > %s AND price = 1 THEN 1 WHEN `time` > %s THEN 0 END) AS month,
AVG(CASE WHEN `time` > %s AND price = 1 THEN 1 WHEN `time` > %s THEN 0 END) AS season
FROM low_price
GROUP BY name;
"""
db.execute(sql, [latest, last_rotation, last_rotation, week, week, month, month, last_rotation, last_rotation])
db.commit()
def on_issues(data: dict) -> str:
if data['sender']['login'] == configuration.get_str('github_user'):
return 'Ignoring self'
number = get_number(data['issue']['url'])
issue = repo.get_repo().get_issue(number)
update.process_issue(issue)
return 'done'
def rotation_redis_store() -> Tuple[int, int, List[Card]]:
lines = []
fs = files()
if len(fs) == 0:
if not os.path.isdir(os.path.expanduser(configuration.get_str('legality_dir'))):
print('WARNING: Could not find legality_dir.')
return (0, 0, [])
with open(fs[-1], 'r') as f:
latest_list = f.read().splitlines()
for filename in fs:
for line in get_file_contents(filename):
line = text.sanitize(line)
lines.append(line.strip())
scores = Counter(lines).most_common()
runs = scores[0][1]
runs_percent = round(round(runs / TOTAL_RUNS, 2) * 100)
cs = oracle.cards_by_name()
cards = []
card_names_by_status: Dict[str, List[str]] = {}
for name, hits in scores:
c = process_score(name, hits, cs, runs, latest_list)
if c is not None:
cards.append(c)
classify_by_status(c, card_names_by_status)
redis.store('decksite:rotation:summary:runs', runs, ex=604800)
redis.store('decksite:rotation:summary:runs_percent', runs_percent, ex=604800)
redis.store('decksite:rotation:summary:cards', cards, ex=604800)
if 'Undecided' in card_names_by_status:
redis.sadd('decksite:rotation:summary:undecided', *card_names_by_status['Undecided'], ex=604800)
if 'Legal' in card_names_by_status:
redis.sadd('decksite:rotation:summary:legal', *card_names_by_status['Legal'], ex=604800)
if 'Not Legal' in card_names_by_status:
redis.sadd('decksite:rotation:summary:notlegal', *card_names_by_status['Not Legal'], ex=604800)
return (runs, runs_percent, cards)
def init() -> None:
client = Bot()
logging.info('Initializing Cards DB')
multiverse.init()
asyncio.ensure_future(multiverse.update_bugged_cards_async())
oracle.init()
logging.info('Connecting to Discord')
client.run(configuration.get_str('token'))
def determine_path(name: str) -> str:
charts_dir = configuration.get_str('charts_dir')
pathlib.Path(charts_dir).mkdir(parents=True, exist_ok=True)
if not os.path.exists(charts_dir):
raise DoesNotExistException(
'Cannot store graph images because {charts_dir} does not exist.'.
format(charts_dir=charts_dir))
return os.path.join(charts_dir, name)
def info_cached(card: Card = None, name: str = None) -> Optional[PriceDataType]:
if name is None and card is not None:
name = card.name
sql = 'SELECT `time`, low / 100.0 AS low, high / 100.0 AS high, price / 100.0 AS price, week, month, season FROM cache WHERE name = %s'
db = database.get_database(configuration.get_str('prices_database'))
try:
return db.select(sql, [name])[0] # type: ignore
except IndexError:
return None
def attempt(interval: int = 1) -> bool:
from shared import database, pd_exception
try:
database.get_database(configuration.get_str('magic_database'))
return True
except pd_exception.DatabaseConnectionRefusedException:
print(f'DB not accepting connections. Sleeping for {interval}.')
time.sleep(interval)
return False
def db() -> Database:
if flask.current_app:
context = flask.g
else:
context = DATABASE
if hasattr(context, 'magic_database'):
return context.get('magic_database')
context.magic_database = get_database(configuration.get_str('magic_database'))
init()
return context.get('magic_database')
async def notpenny(self, client: Client, channel: Channel, args: str, **_: Dict[str, Any]) -> None:
"""Don't show PD Legality in this channel"""
existing = configuration.get_str('not_pd')
if args == 'server':
cid = channel.server.id
else:
cid = channel.id
if str(cid) not in existing.split(','):
configuration.write('not_pd', '{0},{1}'.format(existing, cid))
await client.send_message(channel, 'Disable PD marks')
def db() -> Database:
if has_request_context():
ctx = request
elif g:
ctx = g
else:
ctx = Container() # Fallback context for testing.
if not hasattr(ctx, 'database'):
ctx.database = get_database(configuration.get_str('decksite_database'))
return ctx.database
def db() -> Database:
if has_request_context(): # type: ignore
ctx = request
elif g:
ctx = g
else:
ctx = TEST_CONTEXT # Fallback context for testing.
if not hasattr(ctx, 'database'):
ctx.database = get_database(
configuration.get_str('decksite_database')) # type: ignore
return ctx.database # type: ignore
def process_sets(seen_sets: Set[str], used_sets: Set[str], hits: Set[str], ignored: Set[str]) -> int:
files = rotation.files()
n = len(files) + 1
path = os.path.join(configuration.get_str('legality_dir'), 'Run_{n}.txt').format(n=n)
h = open(path, mode='w', encoding='utf-8')
for card in hits:
line = card + '\n'
h.write(line)
h.close()
print('Run {n} completed, with {ccards} cards from {csets}/{tsets} sets'.format(n=n, ccards=len(hits), csets=len(used_sets), tsets=len(seen_sets)))
print('Used: {sets}'.format(sets=repr(used_sets)))
print('Missed: {sets}'.format(sets=repr(ignored)))
return n
def parse_build_notes(h: Tag) -> None:
entries = []
for n in h.next_elements:
if isinstance(n, Tag) and n.name == 'p':
if 'posted-in' in n.attrs.get('class', []):
break
if n.text:
entries.append(n.text)
embed = {
'title': 'MTGO Build Notes',
'type': 'rich',
'description': '\n'.join(entries),
'url': fetcher.find_announcements()[0],
}
if configuration.get_optional_str('bugs_webhook_id') is not None:
fetch_tools.post_discord_webhook(
configuration.get_str('bugs_webhook_id'),
configuration.get_str('bugs_webhook_token'),
embeds=[embed],
username='******',
avatar_url='https://magic.wizards.com/sites/mtg/files/styles/auth_small/public/images/person/wizards_authorpic_larger.jpg',
)
def init_search_cache() -> None:
if len(SEARCH_CACHE) > 0:
return
submenu_entries = [
] # Accumulate the submenu entries and add them after the top-level entries as they are less important.
for entry in APP.config.get('menu', lambda: [])():
if entry.get('admin_only'):
continue
SEARCH_CACHE.append(menu_item_to_search_item(entry))
for subentry in entry.get('submenu', []):
submenu_entries.append(
menu_item_to_search_item(subentry, entry.get('name')))
for entry in submenu_entries:
if entry.get('admin_only'):
continue
SEARCH_CACHE.append(menu_item_to_search_item(entry))
with open(configuration.get_str('typeahead_data_path')) as f:
for item in json.load(f):
SEARCH_CACHE.append(item)
def run() -> None:
files = rotation.files()
n = len(files)
time_until = TIME_UNTIL_ROTATION - datetime.timedelta(weeks=1)
if n >= rotation.TOTAL_RUNS:
print(
'It is the moment of discovery, the triumph of the mind, and the end of this rotation.'
)
return
if n == 0 and TIME_UNTIL_ROTATION > datetime.timedelta(7):
print(
'The monks of the North Tree rarely saw their kodama until the rotation, when it woke like a slumbering, angry bear.'
)
print('ETA: {t}'.format(
t=dtutil.display_time(int(time_until.total_seconds()))))
return
if n == 0:
rotation.clear_redis(clear_files=True)
#else:
# rotation.clear_redis()
all_prices = {}
for url in configuration.get_list('cardhoarder_urls'):
s = fetch_tools.fetch(url)
s = ftfy.fix_encoding(s)
all_prices[url] = parse_cardhoarder_prices(s)
url = configuration.get_str('mtgotraders_url')
if url:
s = fetch_tools.fetch(url)
all_prices['mtgotraders'] = parse_mtgotraders_prices(s)
run_number = process(all_prices)
if run_number == rotation.TOTAL_RUNS:
make_final_list()
try:
url = f'{fetcher.decksite_url()}/api/rotation/clear_cache'
fetch_tools.fetch(url)
except Exception as c: # pylint: disable=broad-except
print(c)