def process(page: int, page_leap: int, parsing: mp.Value):
logger = init_logging(f'mtgtop8_scrapper_{page}.log')
try:
with psycopg2.connect(user=user, dbname=database) as con:
con.autocommit = True
with con.cursor() as cursor:
while parsing.value > 0:
base_url = re.match('.*.com/', search_url).group()
logger.info(f'Fetching for page {page} in format {url_format} from url {search_url}')
child_page_value = {'cp': page} # set page value
url_soup = get_and_wait(search_url, child_page_value)
events = prs.get_events_from_page(url_soup, base_url, logger)
if events:
for event_name, event_date, event_url in events:
dbc.insert_into_tournament_info(event_name, event_date, url_format, event_url, cursor,
logger, prod_mode)
tourny_id = dbc.get_tournament_info_id(event_name, event_date, url_format, event_url,
cursor)
parse_event(tourny_id, event_url, base_url, cursor, logger, prod_mode)
page += page_leap
else:
with parsing.get_lock():
parsing.value = 0
except Exception as e:
if prod_mode:
logger.warning(str(e))
else:
raise e
def process(process_id: int, entries: List[Tuple[str, str, str]]) -> None:
# find prices for each one
logger = su.init_logging(f'mtgtop8_scrapper_{process_id}.log')
try:
for name, printing_code, printing in entries:
paper_prices, online_prices = get_printing_prices(
name, printing_code, printing, logger)
# insert each price into database
for mapping, is_paper in ((paper_prices, True), (online_prices,
False)):
for date, price in mapping.items():
insert_price_data(name, printing_code, price, date,
is_paper, cursor, logger, prod_mode)
except Exception as e:
if prod_mode:
logger.warning(str(e))
else:
raise e
def main(prod_mode):
logger = su.init_logging('scryfall_card_scapper.log')
get_stored_card_data(dbr.DATABASE_NAME, dbr.USER, logger, prod_mode)
def main(prod_mode):
logger = su.init_logging('mtggoldfish_log.log')
get_and_store_prices(dbr.DATABASE_NAME, dbr.USER, logger, prod_mode)