def generate_cards(self, *words: str):
Log.i(TAG, 'trying to generate {} cards'.format(len(words)))
visited = set()
skipped = []
cf = valid_path(self.cards_file)
with open(cf, 'a', encoding='utf8') as fp:
for word in words:
if word in visited:
Log.i(TAG, 'skipping duplicate: "{}"'.format(word))
continue
try:
actual, fields = self.get_card(word)
except Exception as e:
Log.e(TAG, e)
skipped.append(word)
Log.w(TAG, 'skipped: "{}"'.format(word))
else:
if fp.tell():
fp.write('\n')
fp.write('\t'.join(fields))
visited.add(word)
visited.add(actual)
if skipped:
Log.w(
TAG, 'skipped {} words:\n'.format(len(skipped)) +
'\n'.join(skipped))
Log.i(
TAG,
'generated {} cards to: {}'.format(len(words) - len(skipped), cf))
async def do_get(word: str) -> List[str]:
async with sem:
try:
actual, fields = await asyncio.get_running_loop().run_in_executor(None, self.get_card, word)
except Exception as e:
Log.e(TAG, 'can\'t get card: "{}", {}'.format(word, e))
async with lock:
skipped.append(word)
Log.e(TAG, 'skipped: "{}"'.format(word))
else:
async with lock:
bar.extra = actual
bar.increment()
if actual not in visited:
visited.add(word)
visited.add(actual)
return fields
def generate_cards(self, *words: str):
Log.i(TAG, 'generating {} cards'.format(len(words)))
file = valid_path(self.cards_file)
# region Access with lock in coroutines
visited = set()
skipped = []
bar = ProgressBar(len(words))
lock = asyncio.Lock()
# endregion
async def do_generate():
sem = asyncio.Semaphore(DEFAULT_CONCURRENCY)
async def do_get(word: str) -> List[str]:
async with sem:
try:
actual, fields = await asyncio.get_running_loop().run_in_executor(None, self.get_card, word)
except Exception as e:
Log.e(TAG, 'can\'t get card: "{}", {}'.format(word, e))
async with lock:
skipped.append(word)
Log.e(TAG, 'skipped: "{}"'.format(word))
else:
async with lock:
bar.extra = actual
bar.increment()
if actual not in visited:
visited.add(word)
visited.add(actual)
return fields
# gather all tasks to keep results stable
return await asyncio.gather(*[do_get(w) for w in words])
bar.update()
cards = asyncio.run(do_generate())
cards = [card for card in cards if card]
bar.done()
with open(file, 'a', encoding='utf8') as fp:
writer = csv.writer(fp)
writer.writerows(cards)
Log.i(TAG, 'generated {} cards to: {}'.format(len(cards), file))
if skipped:
Log.e(TAG, 'skipped {} words:\n{}'.format(len(skipped), '\n'.join(skipped)))