def test_lock_args(): """Test overhead of nonblocking threads""" try: from concurrent.futures import ThreadPoolExecutor from threading import RLock except ImportError: raise SkipTest total = 8 subtotal = 1000 tqdm.set_lock(RLock()) with ThreadPoolExecutor(total) as pool: sys.stderr.write('block ... ') sys.stderr.flush() with relative_timer() as time_tqdm: res = list(pool.map(worker(subtotal, True), range(total))) assert sum(res) == sum(range(total)) + total sys.stderr.write('noblock ... ') sys.stderr.flush() with relative_timer() as time_noblock: res = list(pool.map(worker(subtotal, False), range(total))) assert sum(res) == sum(range(total)) + total assert_performance(0.2, 'noblock', time_noblock(), 'tqdm', time_tqdm())
def get_available_proxies(proxies, r, thread_num, write_lock): """ 检测可用的代理并存储 :param proxies: :param r: :param thread_num: :param write_lock: :return: """ tqdm.set_lock(write_lock) for proxy_ip in tqdm(proxies, desc="bar{}".format(thread_num), position=thread_num): proxie = {"http": proxy_ip.decode(), "https": proxy_ip.decode()} try: status_code = requests.get(url="http://www.baidu.com", timeout=1, proxies=proxie).status_code if status_code == 200: logger.info("Available agents:{}".format(proxy_ip)) r.lpush(config.PROXY_AVAILABLE_66IP_KEY, proxy_ip) else: logger.info("Not available proxy") except: logger.info("Not available proxy")
def count_words(lock, pos: int, read_q: Queue, write_q: Queue): """Count words in a book.""" # pylint: disable=too-many-locals tqdm.set_lock(lock) for msg in queuer(read_q): with open(msg.data) as stream: result = {"books": [], "words": {}} soup = BeautifulSoup(stream, "xml") book = soup.Tanach.tanach.book book_name = str(soup.names.find_all("name")[0].string) book_num = int(soup.names.number.string) result["books"].append( dict(id=book_num, name=book_name, corpus="tanach.us")) desc = f"{os.getpid()} COUNT {book_name:<15}" for word in tqdm(book.find_all(["w", "q"]), desc=desc, position=pos): # NOTE: We ignore nested <x> and keep nested <s>! raw = get_word(word) clean = T.strip(raw) if clean in result: result["words"][clean]["freq"] += 1 else: chapter = word.parent.parent["n"] verse = word.parent["n"] ref = f"{book_name} {chapter}:{verse}" result["words"][clean] = dict(book_id=book_num, freq=1, ref=ref, raw=raw) write_q.put(Msg("SAVE", result))
def _parse_preamble(self): ''' @ParseDirectiveMixin._parse_preamble ''' tqdm.set_lock(parallel.RLock()) for fmt in self.args.formats: mkdir(path.join(self.args.target_parent, fmt))
def main(): parser = argparse.ArgumentParser() # Required parameters parser.add_argument("--datapath", type=str, default="data") parser.add_argument("--data_type", type=str, default="train") parser.add_argument("--pilot_version", type=int, choices=[1, 2], default=1) parser.add_argument("--processes", type=int, default=4) parser.add_argument("--data_nums", type=int, default=64) parser.add_argument("--seed", type=int, default=43) parser.add_argument("--mode", type=int, choices=[0, 1, 2], default=None) parser.add_argument("--SNRdb", type=float, default=None) parser.add_argument("--with_pure_y", action='store_true') parser.add_argument("--debug", action='store_true') args = parser.parse_args() H, Htest = read_data(args.datapath) using_H = H if args.data_type == "train" else Htest generate_data_fix = partial(generate_data, args=args, H=using_H) tqdm.set_lock(RLock()) with Pool(processes=args.processes, initializer=tqdm.set_lock, initargs=(tqdm.get_lock(), )) as pool: [ pool.map(generate_data_fix, range(args.processes * i, args.processes * (i + 1))) for i in range(args.data_nums // args.processes) ]
def outer_function(**config): """Outer function running inner function for each task in input dict""" freeze_support() # for Windows support tqdm.set_lock(RLock()) with concurrent.futures.ThreadPoolExecutor(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(), ), max_workers=3) as executor: results_list = [] outer_loop_kwarg = { 'total': len(config['package']['tasks']), 'desc': 'Outer', 'ascii': True, 'position': len(config['package']['tasks']), 'leave': True } with tqdm(**outer_loop_kwarg) as out_progress: futuresListComp = [ executor.submit(inner_function, **node) for node in config['package']['tasks'] ] # Update after each completed task for future in concurrent.futures.as_completed(futuresListComp): out_progress.update() results_list.append(future.result()) return results_list
def count_words(lock, pos: int, read_q: Queue, write_q: Queue): """Count words in a book.""" tqdm.set_lock(lock) for msg in queuer(read_q): result = {"books": [], "words": {}} book = BeautifulSoup(Path(msg.data).read_text(), "lxml").find("text") book_id = int(book["num"]) result["books"].append( dict(id=book_id, name=book["name"], corpus="chabad.org")) desc = f"{os.getpid()} COUNT {book['name']:<15}" for line in tqdm(book.find_all("line"), desc=desc, position=pos): for raw in get_words(line.string): clean = T.strip(raw) if not clean: continue if clean in result["words"]: result["words"][clean]["freq"] += 1 else: ref = line["ref"] result["words"][clean] = dict(book_id=book_id, freq=1, ref=ref, raw=raw) write_q.put(Msg("SAVE", result))
def _parse_preamble(self): ''' @ParseDirectiveMixin._parse_preamble ''' tqdm.set_lock(parallel.RLock()) for fmt in self.args.formats: mkdir(path.join(self.args.target_parent, fmt)) self.pools.progress.worker_kwargs['pcount'] = len(self.frontier) * len( self.args.formats)
def _init(lock, id_queue): """Set up tqdm lock and worker process index. See https://stackoverflow.com/a/42817946 Fixes tqdm line position when |files| > terminal-height so only |workers| progressbars are shown at a time """ global current_process_position current_process_position = id_queue.get() tqdm.set_lock(lock)
def emit(self, record): try: msg = self.format(record) tqdm.set_lock(Lock()) tqdm.write(msg) self.flush() except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)
def runBacktesting(self, position=1, task_text='1/1'): # import pdb # pdb.set_trace() """运行回测""" # 载入历史数据 self.loadHistoryData() # 首先根据回测模式,确认要使用的数据类 if self.mode == self.BAR_MODE: dataClass = VtBarData func = self.newBar else: dataClass = VtTickData func = self.newTick # self.output(u'开始回测') self.strategy.inited = True self.strategy.onInit() # self.output(u'策略初始化完成') self.strategy.trading = True self.strategy.onStart() # self.output(u'策略启动完成') # self.output(u'开始回放数据') # 进度条 count = self.dbCursor.count() # self.dbCursor.batch_size(16 * 1024) from tqdm import tqdm total = count - 1 text = "progresser #{}".format(task_text) tqdm.set_lock(progress_lock) progress = tqdm( total=total, position=position, desc=text ) for d in self.dbCursor: data = dataClass() data.__dict__ = d func(data) if self.firstVolume == 0: self.firstVolume = 10000.0 / d['close'] self.lastPrice = d['close'] progress.update() progress.close()
def prepare_dataset(video_path, frame_folder, flow_folder, name, speeds=None): tqdm.set_lock(Lock()) # manually set internal lock #Step 1, Extract frames and speed dataframe_dict = {} if not os.path.exists(frame_folder): os.makedirs(frame_folder) print("Reading the video file") video_sk = skvideo.io.vread(video_path) print("Extracting the frames") for index, frame in enumerate(tqdm(video_sk)): saving_path = os.path.join(frame_folder, str(index)+'.jpg') if speeds is None: speed = 0 else: speed = speeds[index] dataframe_dict[index] = [saving_path, index, speed] skvideo.io.vwrite(saving_path, frame) processed_dataframe = pd.DataFrame.from_dict(dataframe_dict, orient='index') processed_dataframe.columns = ['frame_path', 'frame_index', 'speed'] print("Saving the dataframe") processed_dataframe.to_csv(os.path.join(PREPARED_DATA_PATH, name +'_meta.csv'), index=False) #Step 2, compute optical flow between frames and average the speed flow_dict = {} if not os.path.exists(flow_folder): os.makedirs(flow_folder) print("Computing the optical flow") for index in tqdm(range(len(processed_dataframe ) - 1)): idx1 = index idx2 = index + 1 frame1 = processed_dataframe.iloc[[idx1]] frame2 = processed_dataframe.iloc[[idx2]] assert(frame2['frame_index'].values[0] - frame1['frame_index'].values[0] == 1) assert(frame2['frame_index'].values[0] > frame1['frame_index'].values[0]) frame1_path = frame1['frame_path'].values[0] frame1_speed = frame1['speed'].values[0] frame2_path = frame2['frame_path'].values[0] frame2_speed = frame2['speed'].values[0] flow = process(frame1_path, frame2_path, SIZE) flow_path = os.path.join(flow_folder, str(index) + '.png') cv2.imwrite(flow_path, flow) speed = np.mean([frame1_speed, frame2_speed]) flow_dict[index] = [flow_path, speed] flow_dataframe = pd.DataFrame.from_dict(flow_dict, orient='index') flow_dataframe.columns = ['flow_path', 'speed'] print("Saving the flow dataframe") flow_dataframe.to_csv(os.path.join(PREPARED_DATA_PATH, name +'_flow_meta.csv'), index=False)
def test_threadpool(): """Test concurrent.futures.ThreadPoolExecutor""" try: from concurrent.futures import ThreadPoolExecutor from threading import RLock except ImportError: raise SkipTest tqdm.set_lock(RLock()) with ThreadPoolExecutor(8) as pool: res = list(tqdm(pool.map(incr_bar, range(100)), disable=True)) assert sum(res) == sum(range(1, 101))
def save_database(lock, pos, write_q): """Save books, words, and occurrences to the database.""" # pylint: disable=too-many-locals db_create(app, db) tqdm.set_lock(lock) book_offset = 0 word_offset = 0 known_words = {} try: book_offset = Book.query.add_columns(func.max(Book.id)).first()[-1] or 0 word_offset = Word.query.add_columns(func.max(Word.id)).first()[-1] or 0 known_words = {w.hebrew: w.id for w in Word.query.all()} except Exception: pass word_offset += 1 parser = grammar.Parser() for msg in queuer(write_q): books = msg.data["books"] words = [] occur = [] for book in books: book["id"] += book_offset desc = f"{os.getpid()} SAVE {books[0]['name']:<15}" for clean, word in tqdm(msg.data["words"].items(), desc=desc, position=pos): word_id = known_words.get(clean, word_offset) if clean not in known_words: # new word parsed = parse_word(parser, word["raw"], clean, word["ref"]) if not parsed: continue parsed["id"] = word_id known_words[clean] = word_id word_offset += 1 words.append(parsed) occur.append( dict( book_id=word["book_id"] + book_offset, word_id=word_id, ref=word["ref"], freq=word["freq"], ) ) for obj, values in {Book: books, Word: words, Freq: occur}.items(): if not values: continue db.engine.execute(obj.__table__.insert().values(values))
def save_words(lock, pos, write_q): """Save words to the database.""" # pylint: disable=too-many-locals db_create(app, db) tqdm.set_lock(lock) seen = {} parser = grammar.Parser() for msg in queuer(write_q): book_id = msg.data["num"] book_name = msg.data["name"] # objects to insert into database book = {"id": book_id, "name": book_name} words = [] occur = [] desc = f"{os.getpid()} SAVE {book_name:<15}" for clean, stats in tqdm(msg.data["words"].items(), desc=desc, position=pos): word_id = seen.get(clean, len(seen) + 1) raw = stats["raw"] if clean not in seen: seen[clean] = word_id parsed = parser.parse(raw) if not "".join(parsed.vowel): # unvowelized word continue syllables = parser.syllabify(parsed) words.append( dict( id=word_id, hebrew=clean, shemot=grammar.isshemot(clean), gematria=grammar.gematria(clean), parsed=str(parsed.flat()), syllables=str(syllables), syllen=len(syllables), rules=str(parsed.rules.flat()), ) ) occur.append( dict( book_id=book_id, word_id=word_id, ref=stats["ref"], freq=stats["freq"], ) ) for obj, values in {Book: book, Word: words, Freq: occur}.items(): db.engine.execute(obj.__table__.insert().values(values))
def scp_operation(args): credentials = Credentials() targets = args.hostname.split(",") tqdm.set_lock(RLock()) with Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(), )) as pool: pool.starmap( scp_process, zip( targets, repeat(credentials), repeat(args.filename), repeat(args.dst_file_path), list(range(len(targets))), ), )
def parallel_program(flash_func, devices, dfu_file): '''Parallel programming''' queue = queue_.Queue() threads = [] results = [] tqdm.monitor_interval = 0 tqdm.set_lock(RLock()) for pos, dev in enumerate(devices): threads.append( Thread(target=flash_func, args=[dev, dfu_file, queue, pos])) for thread in threads: thread.start() for thread in threads: thread.join() results.append(queue.get()) print('\n' * len(devices)) return results
def get_proxy_info(start_idx, end_idx, thread_n, write_lock): tqdm.set_lock(write_lock) text = "bar {}".format(thread_n) for page in tqdm(range(start_idx, int(end_idx) + 1), desc=text, position=thread_n, leave=False): time.sleep(0.5) response = requests.get(url=URL_KUAIDAILI.format(str(page))) soup = BeautifulSoup(response.content, "lxml") tags_aims = soup.select(selector="#list > table > tbody > tr") for i in range(len(tags_aims)): if tags_aims[i]: info_list = list(tags_aims[i].strings) proxy_ip = info_list[1] + info_list[3] r.lpush("proxy_ips", proxy_ip) else: pass response.close()
def spawn_processes(init_fn, read_fn, write_fn, num_readers=1, num_writers=1): """Start readers and writers.""" tqdm.set_lock(RLock()) write_q = Queue() write_fn = partial(write_fn, lock=tqdm.get_lock(), write_q=write_q) writers = [ Process(daemon=True, target=partial(write_fn, pos=i)) for i in range(num_writers, 0, -1) ] read_q = Queue() read_fn = partial(read_fn, lock=tqdm.get_lock(), read_q=read_q, write_q=write_q) readers = [ Process(daemon=True, target=partial(read_fn, pos=i)) for i in range(num_readers + 1, num_writers, -1) ] for p in readers + writers: p.start() init_fn(read_q) notify_and_join(read_q, readers) notify_and_join(write_q, writers)
def assign(self, X, no_scale=False, progress=True): '''Assign the clustering of new samples using :func:`~PopPUNK.dbscan.assign_samples_dbscan` Args: X (numpy.array) Core and accessory distances no_scale (bool) Do not scale X [default = False] progress (bool) Show progress bar [default = True] Returns: y (numpy.array) Cluster assignments by samples ''' if not self.fitted: raise RuntimeError("Trying to assign using an unfitted model") else: if no_scale: scale = np.array([1, 1], dtype=X.dtype) else: scale = self.scale if progress: sys.stderr.write("Assigning distances with DBSCAN model\n") y = np.zeros(X.shape[0], dtype=int) block_size = 5000 n_blocks = (X.shape[0] - 1) // block_size + 1 with SharedMemoryManager() as smm: shm_X = smm.SharedMemory(size=X.nbytes) X_shared_array = np.ndarray(X.shape, dtype=X.dtype, buffer=shm_X.buf) X_shared_array[:] = X[:] X_shared = NumpyShared(name=shm_X.name, shape=X.shape, dtype=X.dtype) shm_y = smm.SharedMemory(size=y.nbytes) y_shared_array = np.ndarray(y.shape, dtype=y.dtype, buffer=shm_y.buf) y_shared_array[:] = y[:] y_shared = NumpyShared(name=shm_y.name, shape=y.shape, dtype=y.dtype) tqdm.set_lock(RLock()) process_map(partial(assign_samples, X=X_shared, y=y_shared, model=self, scale=scale, chunk_size=block_size, values=False), range(n_blocks), max_workers=self.threads, chunksize=min(10, max(1, n_blocks // self.threads)), disable=(progress == False)) y[:] = y_shared_array[:] return y
from multiprocessing import Pool, RLock, freeze_support from core.aesthetics import * print_cyan_bold("my text is very good") class SomeClass: def __init__(self, param1, param2): self.param1 = param1 self.param2 = param2 def some_function(self): for i in range(100): print("vad ca se misca foarte bine") def progresser(n): interval = 0.001 / (n + 2) total = 5000 text = "#{}, est. {:<04.2}s".format(n, interval * total) for _ in trange(total, desc=text, position=n): # do stuff sleep(0.1) if __name__ == '__main__': freeze_support() # for Windows support tqdm.set_lock(RLock()) # for managing output contention p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(), )) p.map(progresser, range(2))
def run_listener(write_lock): # This function is used to solve the error when tqdm is used inside thread # https://github.com/tqdm/tqdm/issues/323 tqdm.set_lock(write_lock) worker_fn()
_str = unicode _range = xrange from StringIO import StringIO string_types = (basestring,) except NameError: # python3 _str = str _range = range from io import StringIO string_types = (str,) try: from threading import RLock except ImportError: tqdm = tqdm_std else: tqdm_std.set_lock(RLock()) tqdm = partial(tqdm_std, lock_args=(False,)) __author__ = "Casper da Costa-Luis <*****@*****.**>" __date__ = "2016-2020" __licence__ = "[MPLv2.0](https://mozilla.org/MPL/2.0/)" __all__ = ["TERM_WIDTH", "int_cast_or_len", "Max", "fext", "_str", "tqdm", "tighten", "check_output", "print_unicode", "StringIO", "Str"] __copyright__ = ' '.join(("Copyright (c)", __date__, __author__, __licence__)) __license__ = __licence__ # weird foreign language log = logging.getLogger(__name__) TERM_WIDTH = _screen_shape_wrapper()(sys.stdout)[0] if not TERM_WIDTH: # non interactive pipe TERM_WIDTH = 256
def _parse_preamble(self): ''' @ParseDirectiveMixin._parse_preamble ''' tqdm.set_lock(parallel.RLock())
def init_aleph(lock=None): global api api = AlephAPI() if lock is not None: tqdm.set_lock(lock)
def initialize_tqdm(): tqdm.set_lock(threading.RLock())
tb_logger.attach( trainer, log_handler=GradsHistHandler(IGTrainer.model), event_name=Events.EPOCH_COMPLETED, ) tb_logger.close() handler = EarlyStopping( patience=5, score_function=lambda engine: engine.state.metrics["accuracy"], trainer=trainer, ) val_evaluator.add_event_handler(Events.COMPLETED, handler) pbar = ProgressBar(bar_format="") pbar.attach(trainer, output_transform=lambda x: {"loss": x}) logging_board(model_name) from tqdm import tqdm from multiprocessing import Lock tqdm.set_lock(Lock()) trainer.run(loader, max_epochs=5) checkpoint_epochs_file = ".".join( checkpoint_file.split(".")[:-1]) + ".05.ckpt" IGTrainer.save_model(checkpoint_epochs_file) trainer.run(loader, max_epochs=10) checkpoint_epochs_file = ".".join( checkpoint_file.split(".")[:-1]) + ".15.ckpt" IGTrainer.save_model(checkpoint_epochs_file)
def scrape(): global filename start_time = datetime.now() # Building the filename filename = str(filename).replace("$DATE", start_time.strftime("%Y%m%d%H%M%S")) search = str(args.search).replace(" ", "") if len(search) > 10: search = search[0:9] filename = str(filename).replace("$SEARCH", search) func_args = [] stats_dict = {} if args.engines and len(args.engines) > 0: eng = args.engines[0] for e in eng: try: if not (args.exclude and len(args.exclude) > 0 and e in args.exclude[0]): func_args.append("{}:{}".format(e, args.search)) stats_dict[e] = 0 except KeyError: print( "Error: search engine {} not in the list of supported engines" .format(e)) else: for e in supported_engines.keys(): if not (args.exclude and len(args.exclude) > 0 and e in args.exclude[0]): func_args.append("{}:{}".format(e, args.search)) stats_dict[e] = 0 # Doing multiprocessing units = min((cpu_count() - 1), len(func_args)) if args.mp_units and args.mp_units > 0: units = min(args.mp_units, len(func_args)) print("search.py started with {} processing units...".format(units)) freeze_support() results = {} with Pool(units, initializer=tqdm.set_lock(RLock()), initargs=(tqdm.get_lock(), )) as p: results_map = p.map(run_method, func_args) results = reduce(lambda a, b: a + b if b is not None else a, results_map) stop_time = datetime.now() if not args.continuous_write: with open(filename, 'w', newline='') as csv_file: csv_writer = csv.writer(csv_file, delimiter=field_delim, quoting=csv.QUOTE_ALL) for r in results: write_to_csv(csv_writer, r) total = 0 print("\nReport:") print(" Execution time: %s seconds" % (stop_time - start_time)) print(" Results per engine:") for r in results: stats_dict[r['engine']] += 1 for s in stats_dict: n = stats_dict[s] print(" {}: {}".format(s, str(n))) total += n print(" Total: {} links written to {}".format(str(total), filename))
""" Progress bars in Nuitka. This is responsible for wrapping the rendering of progress bar and emitting tracing to the user while it's being displayed. """ from nuitka import Tracing from nuitka.utils.ThreadedExecutor import RLock try: from tqdm import tqdm except ImportError: tqdm = None else: tqdm.set_lock(RLock()) class NuitkaProgessBar(object): def __init__(self, stage, total, unit): self.stage = stage self.total = total self.unit = unit # No item under work yet. self.item = None # No progress yet. self.progress = 0 # Render immediately with 0 progress.
def process_collection(collection): fid = collection["foreign_id"] fid = fid.replace("/", "") fname = f"./dataset_components/{fid}.json" if os.path.exists(fname): return try: components = calculate_components(collection) except AlephException as e: print(f"Aleph Error: {fid}: {e}") return with open(fname, "w+") as fd: data = { "components_histogram": dict(components), "collection": collection, } fd.write(json.dumps(data)) if __name__ == "__main__": init_aleph() collections = api.filter_collections("*") N = collections.result["total"] tqdm.set_lock(mp.RLock()) with mp.Pool(processes=4, initializer=init_aleph, initargs=(tqdm.get_lock(),)) as p: results = p.imap_unordered(process_collection, collections, chunksize=32) for _ in tqdm(results, total=N, position=0): pass