async def test_w17(event_loop: asyncio.AbstractEventLoop) -> None:
"""
Very basic test. Can the scraper scrape Gatherer?
"""
w17 = mtg_builder.determine_gatherer_sets({'sets': ['W17'], 'all_sets': False})
builder = mtg_builder.MTGJSON(w17, loop=event_loop)
json = await builder.build_set(w17[0], 'en')
assert json["block"] == 'Amonkhet'
assert json["border"] == 'black'
assert len(json['cards']) == 30
assert json['code'] == 'W17'
async def main(loop: asyncio.AbstractEventLoop, session: aiohttp.ClientSession,
language_to_build: str, args: dict) -> None:
"""
Main method that starts the entire build process
:param args:
:param loop:
:param session:
:param language_to_build:
:return:
"""
def get_next_batch_of_sets(queue: Iterator[List[str]]) -> List[List[str]]:
"""
To ensure better performance, we limit the number of sets built at a time
to limit our memory impact. This will return the next group of sets to
build.
"""
max_pops = int(args['max_sets_build'][0])
# User disabled this memory protection feature
if max_pops == 0:
return list(queue)
return list(itertools.islice(queue, max_pops))
# Main Applied
mtg_storage.ensure_set_dir_exists()
sets_queue = iter(SETS_TO_BUILD)
async with session:
# Start asyncio tasks for building each set
json_builder = mtg_builder.MTGJSON(SETS_TO_BUILD, session, loop)
# We will only be building a few sets at a time, to allow for partial outputs
sets_to_build_now = get_next_batch_of_sets(sets_queue)
while sets_to_build_now:
# Create our builders for the few sets
futures = [
loop.create_task(
json_builder.build_set(set_name, language_to_build))
for set_name in sets_to_build_now
]
# Then wait until all of them are completed
await asyncio.wait(futures)
# Then queue up our next sets to build
sets_to_build_now = get_next_batch_of_sets(sets_queue)
# And we're done! :)
return
async def test_chk(event_loop: asyncio.AbstractEventLoop) -> None:
"""
Scrape Champions of Kamigawa. Make sure those Flip Cards work.
"""
chk = mtg_builder.determine_gatherer_sets({'sets': ['CHK'], 'all_sets': False})
builder = mtg_builder.MTGJSON(chk, loop=event_loop)
json = await builder.build_set(chk[0], 'en')
lavarunner = [c for c in json['cards'] if c['multiverseid'] == 78694][0]
assert lavarunner['name'] == 'Akki Lavarunner'
assert lavarunner['names'] == ["Akki Lavarunner", "Tok-Tok, Volcano Born"]
assert lavarunner["number"] == "153a"
toktok = [c for c in json['cards'] if c['multiverseid'] == 78694][1]
assert toktok['name'] == 'Tok-Tok, Volcano Born'
assert toktok['names'] == ["Akki Lavarunner", "Tok-Tok, Volcano Born"]
assert toktok["number"] == "153b"
async def test_emn(event_loop: asyncio.AbstractEventLoop) -> None:
"""
Eldrich Moon is a messy set.
"""
emn = mtg_builder.determine_gatherer_sets({
'sets': ['EMN'],
'all_sets': False
})
builder = mtg_builder.MTGJSON(emn, loop=event_loop)
json = await builder.build_set(emn[0], 'en')
chittering = [c for c in json['cards'] if c['name'] == "Chittering Host"]
# assert len(chittering) == 1
assert chittering[0]['names'] == [
"Graf Rats", "Midnight Scavengers", "Chittering Host"
]
async def test_isd(event_loop: asyncio.AbstractEventLoop) -> None:
"""
Scrape Innistrad. Make sure those DFCs work.
"""
isd = mtg_builder.determine_gatherer_sets({'sets': ['ISD'], 'all_sets': False})
builder = mtg_builder.MTGJSON(isd, loop=event_loop)
json = await builder.build_set(isd[0], 'en')
assert len(json['cards']) == 284
delver = [c for c in json['cards'] if c['multiverseid'] == 226749][0]
assert delver['name'] == 'Delver of Secrets'
assert delver['names'] == ["Delver of Secrets", "Insectile Aberration"]
assert delver["number"] == "51a"
aberration = [c for c in json['cards'] if c['multiverseid'] == 226755][0]
assert aberration['name'] == 'Insectile Aberration'
assert aberration['names'] == ["Delver of Secrets", "Insectile Aberration"]
assert aberration['number'] == '51b'
async def test_inv(event_loop: asyncio.AbstractEventLoop) -> None:
"""
Scrape Invasion. Make sure those Split Cards work.
"""
inv = mtg_builder.determine_gatherer_sets({
'sets': ['INV'],
'all_sets': False
})
builder = mtg_builder.MTGJSON(inv, loop=event_loop)
json = await builder.build_set(inv[0], 'en')
stand = [c for c in json['cards'] if c['multiverseid'] == 20573][0]
assert stand['name'] == 'Stand'
assert stand['names'] == ["Stand", "Deliver"]
assert stand["number"] == "292a"
assert stand["layout"] == 'split'
deliver = [c for c in json['cards'] if c['multiverseid'] == 20573][1]
assert deliver['name'] == 'Deliver'
assert deliver['names'] == ["Stand", "Deliver"]
assert deliver["number"] == "292b"
assert deliver["layout"] == 'split'