def generate_cards(self, *words: str): Log.i(TAG, 'trying to generate {} cards'.format(len(words))) visited = set() skipped = [] cf = valid_path(self.cards_file) with open(cf, 'a', encoding='utf8') as fp: for word in words: if word in visited: Log.i(TAG, 'skipping duplicate: "{}"'.format(word)) continue try: actual, fields = self.get_card(word) except Exception as e: Log.e(TAG, e) skipped.append(word) Log.w(TAG, 'skipped: "{}"'.format(word)) else: if fp.tell(): fp.write('\n') fp.write('\t'.join(fields)) visited.add(word) visited.add(actual) if skipped: Log.w( TAG, 'skipped {} words:\n'.format(len(skipped)) + '\n'.join(skipped)) Log.i( TAG, 'generated {} cards to: {}'.format(len(words) - len(skipped), cf))
async def do_get(word: str) -> List[str]: async with sem: try: actual, fields = await asyncio.get_running_loop().run_in_executor(None, self.get_card, word) except Exception as e: Log.e(TAG, 'can\'t get card: "{}", {}'.format(word, e)) async with lock: skipped.append(word) Log.e(TAG, 'skipped: "{}"'.format(word)) else: async with lock: bar.extra = actual bar.increment() if actual not in visited: visited.add(word) visited.add(actual) return fields
def generate_cards(self, *words: str): Log.i(TAG, 'generating {} cards'.format(len(words))) file = valid_path(self.cards_file) # region Access with lock in coroutines visited = set() skipped = [] bar = ProgressBar(len(words)) lock = asyncio.Lock() # endregion async def do_generate(): sem = asyncio.Semaphore(DEFAULT_CONCURRENCY) async def do_get(word: str) -> List[str]: async with sem: try: actual, fields = await asyncio.get_running_loop().run_in_executor(None, self.get_card, word) except Exception as e: Log.e(TAG, 'can\'t get card: "{}", {}'.format(word, e)) async with lock: skipped.append(word) Log.e(TAG, 'skipped: "{}"'.format(word)) else: async with lock: bar.extra = actual bar.increment() if actual not in visited: visited.add(word) visited.add(actual) return fields # gather all tasks to keep results stable return await asyncio.gather(*[do_get(w) for w in words]) bar.update() cards = asyncio.run(do_generate()) cards = [card for card in cards if card] bar.done() with open(file, 'a', encoding='utf8') as fp: writer = csv.writer(fp) writer.writerows(cards) Log.i(TAG, 'generated {} cards to: {}'.format(len(cards), file)) if skipped: Log.e(TAG, 'skipped {} words:\n{}'.format(len(skipped), '\n'.join(skipped)))