def ratings_from_ids_2(self, id_iter): with tpe() as executor: future_to_rate = { executor.submit(self.json_from_url, self.r_from_id_url(group)): (i + 1, group) for (i, group) in enumerate(self.grouper(id_iter, 100, '')) } for future in as_comp(future_to_rate): i, group = future_to_rate[future] self.ow.print(f'Getting Player ratings - {i*100}') try: ratings = future.result() if not ratings.get('status') == 'ok': print(ratings) except Exception as exc: print( f'rating-ing group {i} generated an exception: {exc}') else: r_data = ratings.get('data') if r_data is not None: for player_id, player in r_data.items(): if not player: continue player['id'] = player_id yield player
def startCrawl(self, toggle_print = True): def parse_new_urls(): #parse catalog page that returns(gets) urls of hero page while self.downloader.updatePageSource(): raw_html = self.downloader.downloadPageSource() new_urls, next_page_tag = self.parser.parse(raw_html, self.downloader.start_url) self.url_manager.addNewUrl(new_urls) def parse_detail(in_url): #parse hero page if toggle_print: print("%d: gathering data from: %s" % (self.count, in_url)) page_source = self.downloader.getPageSourceViaRequest(in_url) data = self.parser.parseHero(page_source) self.outputer.collectData(data) self.count += 1 raw_html = self.downloader.downloadPageSource() new_urls, next_page_tag = self.parser.parse(raw_html, self.downloader.start_url) self.downloader.next_page_tag = next_page_tag self.url_manager.addNewUrl(new_urls) gainNewUrlsThread = Thread(target=parse_new_urls) gainNewUrlsThread.run() executor = tpe(self.thread_pool_size) while not self.url_manager.isEmpty(): all_urls = self.url_manager.getUrls() executor.map(parse_detail, all_urls) if self.outputer.data_count > self.outputer.buffer_size: self.buffer_trigger.set() self.outputer.end_writing = True self.buffer_trigger.set() executor.shutdown(wait=True)
def startCrawl(self, root_url, toggle_print=True): def parse_once(new_url): try: if toggle_print: print("crawling %d : %s" % (self.count, new_url)) self.downloader.url = new_url html_out = self.downloader.startDownload() next_url, data = self.parser.startParse(new_url, html_out) self.urls.addNewUrl(next_url) self.outputer.collectData(data) self.count += 1 time.sleep(random.random() * 10) except: print("failed\n") else: print("successful\n") assert isinstance(toggle_print, bool) self.urls.addNewUrl(root_url) path = str(os.path.dirname( os.path.abspath(__file__))) + "/SushiReview.txt" write_thread = Thread(target=self.write_method, args=(path, )) write_thread.start() while not self.urls.isEmpty(): executor = tpe(self.thread_pool_size) all_urls = self.urls.getAllUrls() executor.map(parse_once, all_urls) executor.shutdown(wait=True) if len(self.outputer.data) > self.outputer.buffer_size: self.buffer_trigger.set() self.outputer.end_writing = True self.buffer_trigger.set() write_thread.join() print("\n\nwrote %d lines of file in total. file located at: %s" % (self.outputer.total_data_count, path))
def do_it_lambda(tag_name): net_tasks = [] with tpe(max_workers=None) as executor, requests.Session() as session: session.headers.update(request_header) tasks_with_tag = get_tasks_by_tag_name(session, tag_name) tasks = [t['gid'] for t in tasks_with_tag] tasks_details = { executor.submit(get_json_response, session, 'tasks/{}'.format(t), opt_fields=','.join(task_fields)): t for t in tasks } for f in concurrent.futures.as_completed(tasks_details): fut = tasks_details[f] try: t = f.result() net_tasks.append(t) except Exception as exc: print('Error: {}'.format(exc)) records = [task_as_record(t) for t in net_tasks] df = pd.DataFrame(records) df.to_csv(csv_file, index=False) return df
def runner(): urls = [ "https://www.gutenberg.org/cache/epub/376/pg376.txt", "https://www.gutenberg.org/files/84/84-0.txt", "https://www.gutenberg.org/cache/epub/844/pg844.txt", ] with tpe(max_workers=8) as exe: results = exe.map(download_nd_save, urls) combine_files()
def ids_from_names_generator_2(self, names): with tpe() as executor: future_to_id = { executor.submit(self.id_from_name, i + 1, len(names), name): (i + 1, name) for (i, name) in enumerate(names) } for future in as_comp(future_to_id): i, name = future_to_id[future] try: id = future.result() except Exception as exc: print( f'ID-ing {name} generated an exception: {exc}, retrying' ) # future_to_id[executor.submit(self.id_from_name, i, len(names), name)] = (i, name) else: yield id
def main(): global filename, questions if len(sys.argv) > 1 and path.isfile(sys.argv[1]): filename = sys.argv[1] elif path.isfile("sample.mp3"): filename = "sample.mp3" elif path.isfile("sample.wav"): filename = "sample.wav" else: return False if len(sys.argv) > 2 and path.isfile(sys.argv[2] + '.csv'): rows = csv.reader(open(sys.argv[2] + '.csv', 'r')) questions = [] for row in rows: tmp_ans = row[1:] answers = [] for a in tmp_ans: answers.append(a.strip()) questions.append({"question": row[0], "answer": answers}) with tpe(max_workers=2) as e: e.submit(playMusic) e.submit(showQuestions)
def allLinks(root): ans = [] for e in root.getchildren(): if e.tag == 'a': ans += [e.attrib['href']] else: ans += allLinks(e) return set(ans) def download(url): path = os.path.join(dest, url.split('/')[-1]) data = requests.get(url) with open(path, 'wb') as f: f.write(data.content) links = allLinks(root) pool = tpe(max_workers=4) t = time.clock() fs = [] for li in links: print("downloading ...", li) f = pool.submit(download, site + li) fs.append(f) print("Done") wait(fs) print(time.clock() - t)
def mt_func(*args, **kwargs): return tpe().submit(func, *args, **kwargs)
def by_thread(func, items, callback, workers): with tpe(max_workers=workers) as executor: futures = (executor.submit(func, item) for item in items) for future in comp(futures): callback(future.result())
def rewrite_files(n): for k in range(n): write_files('rewritten_files\\' + str(k) + '.txt', get_file_content('given_files\\' + str(k) + '.txt')) # запис до одного файлу def write_one_file(k): write_files('rewritten_files\\' + str(k) + '.txt', get_file_content('given_files\\' + str(k) + '.txt')) if __name__ == '__main__': start = time.time() with tpe(8) as executor: executor.map(write_one_file, range(1000)) print('Час виконання ф-ції паралельно за допомогою потоків ThreadPoolExecutor: ', time.time() - start, 'c.') start = time.time() with ppe(8) as executor: executor.map(write_one_file, range(1000)) print('Час виконання ф-ції паралельно за допомогою процесів ProcessPoolExecutor: ', time.time() - start, 'c.') start = time.time() rewrite_files(1000) print('Час виконання ф-ції послідовно: ', time.time() - start, 'c.')
pr = Predators('Вовк', parent=None) producer = Producers(f) consumer = Consumers(f) consumer.phytophages = [pp] consumer.predators = [pr] detr1 = Insects('Insect', f) detr2 = Worms('Worms', f) detrio = Detriophages(f, [detr1, detr2]) abio = AbioticFactors(f) # відтворення життєвих циклів # на основі Sequence diagram # запуск в паралельному потоці життєвого циклу консументу th = Thread(target=pr.main_life_cycle, args=(pr, )) th.start() # відтворення життєвих циклів # на основі activity diagram with tpe(max_workers=3) as executor: executor.submit(pr.breath) executor.submit(pr.main_life_cycle) executor.submit(pr.sleep)
async def run(self, queue, ctx, bot, search): opts = { 'format': 'bestaudio/best', 'outtmpl': f'{ctx.guild.id}/{self.outtmpl_seed()}%(extractor)s_%(id)s.%(ext)s', 'restrictfilenames': True, 'noplaylist': False, 'nocheckcertificate': True, 'ignoreerrors': False, 'logtostderr': False, 'quiet': True, 'no_warnings': True, 'default_search': 'auto', 'source_address': '0.0.0.0', 'playlistend': 50, } ytdl = youtube_dl.YoutubeDL(opts) ytdl.params['extract_flat'] = True ef_info = ytdl.extract_info(download=False, url=search) ytdl.params['extract_flat'] = False if 'entries' in ef_info: length = len(ef_info['entries']) else: length = 1 for v in range(1, length + 1): try: ytdl.params.update({'playlistend': v, 'playliststart': v}) tdl = functools.partial(ytdl.extract_info, download=True, url=search) info = await bot.loop.run_in_executor(tpe(max_workers=4), tdl) except Exception as e: self._ytdl_error = e if length <= 1: return await ctx.send( f'**There was an error processing your song.** ```css\n[{e}]\n```' ) else: continue if 'entries' in info: info = info['entries'][0] duration = info.get('duration') or self.get_duration( info.get('url')) song_info = { 'title': info.get('title'), 'weburl': info.get('webpage_url'), 'duration': duration, 'views': info.get('view_count'), 'thumb': info.get('thumbnail'), 'requester': ctx.author, 'upload_date': info.get('upload_date', '\uFEFF') } if length == 1: await ctx.send( f'```ini\n[Added {song_info["title"]} to the queue.]\n```', delete_after=15) try: await queue.put({ 'source': ytdl.prepare_filename(info), 'info': song_info, 'channel': ctx.channel }) except Exception as e: self._ytdl_error = e
import tkinter from easydict import EasyDict as edict from tkinter.filedialog import askdirectory, askopenfilename, asksaveasfilename import os from PIL import Image, ImageTk import numpy as np from concurrent.futures import ThreadPoolExecutor as tpe from functools import lru_cache pool = tpe(4) tk = tkinter shape = (1024, 768) args = edict({ 'input': None, 'output': '', 'cls_list': ['0', '1', '2', '3', '4'], 'filelist': [], 'filelabel': {}, 'curlabel': -1, 'curselect': 0, 'im': None, 'imname': None, 'imidx': -1, 'clsidx': -1 }) class labeled(object): num = 0