def task(args: List[str]) -> None:
module = args[1]
if module == 'scraper':
module = 'scrapers'
if module == 'scrapers':
module = 'decksite.scrapers'
name = args.pop()
from magic import oracle, multiverse
multiverse.init()
if name != 'reprime_cache':
oracle.init()
if name == 'all':
run_all_tasks(module)
else:
s = importlib.import_module('{module}.{name}'.format(name=name,
module=module))
if getattr(s, 'REQUIRES_APP_CONTEXT', True):
from decksite.main import APP
APP.config['SERVER_NAME'] = configuration.server_name()
APP.app_context().__enter__(
) # Technically we should __exit__() at the end, but since we're terminating...
if getattr(s, 'scrape', None) is not None:
s.scrape() # type: ignore
elif getattr(s, 'run', None) is not None:
s.run() # type: ignore
# Only when called directly, not in 'all'
elif getattr(s, 'ad_hoc', None) is not None:
s.ad_hoc() # type: ignore
def run_all_tasks(module: Any, with_flag: Optional[str] = None) -> None:
setup_app_context = False
m = importlib.import_module('{module}'.format(module=module))
# pylint: disable=unused-variable
for importer, modname, ispkg in pkgutil.iter_modules(
m.__path__): # type: ignore
s = importlib.import_module('{module}.{name}'.format(name=modname,
module=module))
use_app_conext = getattr(s, 'REQUIRES_APP_CONTEXT', True)
if use_app_conext and not setup_app_context:
from decksite.main import APP
APP.config['SERVER_NAME'] = configuration.server_name()
app_context = APP.app_context() # type: ignore
app_context.__enter__()
if with_flag and not getattr(s, with_flag, False):
continue
if getattr(s, 'scrape', None) is not None:
timer = time.perf_counter()
s.scrape() # type: ignore
t = time.perf_counter() - timer
print(f'{s.__name__} completed in {t}')
elif getattr(s, 'run', None) is not None:
timer = time.perf_counter()
s.run() # type: ignore
t = time.perf_counter() - timer
print(f'{s.__name__} completed in {t}')
if setup_app_context:
app_context.__exit__(None, None, None)
def test_tappedout() -> None:
prev = APP.config['SERVER_NAME']
APP.config['SERVER_NAME'] = configuration.server_name()
with APP.app_context(): # type: ignore
# pylint: disable=no-member
tappedout.scrape() # type: ignore
APP.config['SERVER_NAME'] = prev
def task(args: List[str]) -> None:
module = args[1]
if module == 'scraper':
module = 'scrapers'
if module == 'scrapers':
module = 'decksite.scrapers'
name = args.pop()
from decksite.main import APP
APP.config['SERVER_NAME'] = '127:0.0.1:5000'
with APP.app_context():
from magic import oracle, multiverse
multiverse.init()
if name != 'reprime_cache':
oracle.init()
if name == 'all':
m = importlib.import_module('{module}'.format(module=module))
# pylint: disable=unused-variable
for importer, modname, ispkg in pkgutil.iter_modules(
m.__path__): # type: ignore
s = importlib.import_module('{module}.{name}'.format(
name=modname, module=module))
if getattr(s, 'scrape', None) is not None:
s.scrape() # type: ignore
elif getattr(s, 'run', None) is not None:
s.run() # type: ignore
else:
s = importlib.import_module('{module}.{name}'.format(
name=name, module=module))
if getattr(s, 'scrape', None) is not None:
s.scrape() # type: ignore
elif getattr(s, 'run', None) is not None:
s.run() # type: ignore
# Only when called directly, not in 'all'
elif getattr(s, 'ad_hoc', None) is not None:
s.ad_hoc() # type: ignore
def run():
if len(sys.argv) == 0:
print("No entry point specified.")
sys.exit(1)
if "discordbot" in sys.argv:
from discordbot import bot
bot.init()
elif "decksite" in sys.argv:
from decksite import main
main.init()
elif "price_grabber" in sys.argv:
from price_grabber import price_grabber
price_grabber.fetch()
price_grabber.price.cache()
elif "srv_price" in sys.argv:
from price_grabber import srv_prices
srv_prices.init()
elif sys.argv[1] in ["scraper", "scrapers", "maintenance"]:
module = sys.argv[1]
if module == "scraper":
module = "scrapers"
name = sys.argv.pop()
from decksite.main import APP
APP.config["SERVER_NAME"] = "127:0.0.1:5000"
with APP.app_context():
if name == "all":
m = importlib.import_module(
'decksite.{module}'.format(module=module))
#pylint: disable=unused-variable
for importer, modname, ispkg in pkgutil.iter_modules(
m.__path__):
s = importlib.import_module(
'decksite.{module}.{name}'.format(name=modname,
module=module))
if getattr(s, "scrape", None) is not None:
s.scrape()
elif getattr(s, "run", None) is not None:
s.run()
else:
s = importlib.import_module('decksite.{module}.{name}'.format(
name=name, module=module))
if getattr(s, "scrape", None) is not None:
s.scrape()
elif getattr(s, "run", None) is not None:
s.run()
elif "tests" in sys.argv:
import pytest
sys.argv.remove("tests")
code = pytest.main()
sys.exit(code)
else:
print("You didn't tell me what to run or I don't recognize that name")
sys.exit(1)
sys.exit(0)
def run_all_tasks(module: Any) -> None:
from decksite.main import APP
APP.config['SERVER_NAME'] = configuration.server_name()
with APP.app_context():
m = importlib.import_module('{module}'.format(module=module))
# pylint: disable=unused-variable
for importer, modname, ispkg in pkgutil.iter_modules(
m.__path__): # type: ignore
s = importlib.import_module('{module}.{name}'.format(
name=modname, module=module))
if getattr(s, 'scrape', None) is not None:
s.scrape() # type: ignore
elif getattr(s, 'run', None) is not None:
s.run() # type: ignore
def task(args: List[str]) -> None:
try:
module = args[0]
if module == 'scraper':
module = 'scrapers'
if module == 'scrapers':
module = 'decksite.scrapers'
name = args[1]
from magic import multiverse, oracle
multiverse.init()
if name != 'reprime_cache':
oracle.init()
if name == 'all':
run_all_tasks(module)
elif name == 'hourly':
run_all_tasks(module, 'HOURLY')
else:
s = importlib.import_module('{module}.{name}'.format(
name=name, module=module))
use_app_context = getattr(s, 'REQUIRES_APP_CONTEXT', True)
if use_app_context:
from decksite.main import APP
APP.config['SERVER_NAME'] = configuration.server_name()
app_context = APP.app_context() # type: ignore
app_context.__enter__() # type: ignore
if getattr(s, 'scrape', None) is not None:
exitcode = s.scrape(*args[2:]) # type: ignore
elif getattr(s, 'run', None) is not None:
exitcode = s.run() # type: ignore
# Only when called directly, not in 'all'
elif getattr(s, 'ad_hoc', None) is not None:
exitcode = s.ad_hoc() # type: ignore
if use_app_context:
app_context.__exit__(None, None, None)
if exitcode is not None:
sys.exit(exitcode)
except Exception as c:
from shared import repo
repo.create_issue(f'Error running task {args}',
'CLI',
'CLI',
'PennyDreadfulMTG/perf-reports',
exception=c)
raise
def run_all_tasks(module: Any, with_flag: Optional[str] = None) -> None:
error = None
app_context = None
m = importlib.import_module('{module}'.format(module=module))
# pylint: disable=unused-variable
for _importer, modname, _ispkg in pkgutil.iter_modules(
m.__path__): # type: ignore
try:
s = importlib.import_module('{module}.{name}'.format(
name=modname, module=module))
use_app_context = getattr(s, 'REQUIRES_APP_CONTEXT', True)
if use_app_context and app_context is None:
from decksite import APP
APP.config['SERVER_NAME'] = configuration.server_name()
app_context = APP.app_context() # type: ignore
app_context.__enter__() # type: ignore
if with_flag and not getattr(s, with_flag, False):
continue
if getattr(s, 'scrape', None) is not None:
timer = time.perf_counter()
s.scrape() # type: ignore
t = time.perf_counter() - timer
print(f'{s.__name__} completed in {t}')
elif getattr(s, 'run', None) is not None:
timer = time.perf_counter()
s.run() # type: ignore
t = time.perf_counter() - timer
print(f'{s.__name__} completed in {t}')
except Exception as c: # pylint: disable=broad-except
from shared import repo
repo.create_issue(f'Error running task {s.__name__}',
'CLI',
'CLI',
'PennyDreadfulMTG/perf-reports',
exception=c)
error = c
if app_context is not None:
app_context.__exit__(None, None, None)
if error:
raise error
def test_goldfish() -> None:
with APP.app_context(): # type: ignore
mtggoldfish.scrape(1)
def test_tappedout():
prev = APP.config["SERVER_NAME"]
APP.config["SERVER_NAME"] = "127:0.0.1:5000"
with APP.app_context():
tappedout.scrape()
APP.config["SERVER_NAME"] = prev
def test_manual_tappedout():
with APP.app_context():
tappedout.scrape_url(
'https://tappedout.net/mtg-decks/60-island/') # Best deck
def test_gatherling():
with APP.app_context():
gatherling.scrape(5)
def test_tappedout() -> None:
prev = APP.config['SERVER_NAME']
APP.config['SERVER_NAME'] = configuration.server_name()
with APP.app_context():
tappedout.scrape()
APP.config['SERVER_NAME'] = prev
def test_goldfish() -> None:
with APP.app_context():
mtggoldfish.scrape(1)
def test_tappedout() -> None:
prev = APP.config['SERVER_NAME']
APP.config['SERVER_NAME'] = '127:0.0.1:5000'
with APP.app_context():
tappedout.scrape()
APP.config['SERVER_NAME'] = prev
def test_manual_tappedout() -> None:
with APP.app_context(): # type: ignore
tappedout.scrape_url('https://tappedout.net/mtg-decks/60-island/')
def test_gatherling() -> None:
with APP.app_context(): # type: ignore
gatherling.scrape(5)
def test_gatherling():
APP.config["SERVER_NAME"] = "127:0.0.1:5000"
with APP.app_context():
gatherling.scrape(5)
def test_manual_tappedout():
APP.config["SERVER_NAME"] = "127:0.0.1:5000"
with APP.app_context():
tappedout.scrape_url('http://tappedout.net/mtg-decks/60-island/') # Best Deck
