def screencap_h264(device: AdbDevice, executor: Executor): start = time() device.push("../scrcpy-win64/scrcpy-server.jar", "/data/local/tmp/scrcpy-server.jar") print("Pushed server component") sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) port = device.adb_output("reverse", "tcp:5556", "localabstract:scrcpy") print("[ADB]", port) # sock.bind(("127.0.0.1", 5556)) # sock.listen() executor.submit(lambda: print( "[SCRCPY]", device.shell("CLASSPATH=/data/local/tmp/scrcpy-server.jar " "app_process / com.genymobile.scrcpy.Server 0 8000000 " "false - false true"))) print("Executed scrcpy") print("Waiting for connection") # conn, info = sock.accept() # print(info) # print(conn.recv(1)) # print(conn.recv(64)) # print(conn.recv(2)) # print(conn.recv(2)) # print("Closing transmission") # device.shell("\x03") # conn.close() # sock.close() print("Recv h264 took %f seconds" % (time() - start))
def submit_tasks(executor: Executor, window_size: int, fn, *iterables): """ Assuming all task take approximately the same time. Effectively executor.map, but done in batches/windows to reduce the number of queued Future objects (seems to consume a lot of memory). """ args_iterator = zip(*iterables) fs_queue = deque( (executor.submit(fn, *args) for args in itertools.islice(args_iterator, window_size)), maxlen=window_size, ) try: while fs_queue: # following comment copied from Executor.map # Careful not to keep a reference to the popped future yield fs_queue.popleft() fs_queue.append(executor.submit(fn, *next(args_iterator))) except StopIteration: # no more tasks to submit pass # collect remaining tasks while fs_queue: # Careful not to keep a reference to the popped future yield fs_queue.popleft()
def _search_async_locked(self, pool: futures.Executor, parser: MP3Parser, depth: int) -> List[futures.Future]: """ Walk self._fullpath scheduling tasks to find all songs and sub-directories. Checking of files and subdirectories is submitted as tasks to the specified pool. Must be called with the lock held """ cache = self._load_cache() tasks: List[futures.Future] = [] assert self._fullpath is not None if not self._fullpath.is_dir(): raise IOError(f'Directory "{self._fullpath}" does not exist') self.log.debug('Search %s', self._fullpath.name) for index, filename in enumerate(self._fullpath.iterdir()): abs_fname = str(filename) fstats = os.stat(abs_fname) if stat.S_ISDIR(fstats.st_mode): subdir = Directory(self, 1000 * (self.ref_id + index), filename) self.subdirectories.append(subdir) tasks.append( pool.submit(subdir.search_async, pool, parser, depth + 1)) elif (stat.S_ISREG(fstats.st_mode) and abs_fname.lower().endswith(".mp3") and fstats.st_size <= self.maxFileSize): self._todo += 1 task = pool.submit(self._parse_song, parser, cache, filename, index) task.add_done_callback(self._after_parse_song) tasks.append(task) return tasks
def inference_all_questions(self, question_folder, cpu_num=1): base_dir = os.getcwd() print('base_dir: ', base_dir) # os.makedirs(question_folder, exist_ok=True) os.chdir(question_folder) print('numcores: ', cpu_num) audios = os.listdir() audios = [audio for audio in audios if audio.endswith('.wav')] if cpu_num > 1: audios_for_one_process = len(audios) // cpu_num if audios_for_one_process < 1: audios_for_one_process = 1 print('audios count: ', len(audios)) print('audio_for_one_process: ', audios_for_one_process) tmp = [] splitted_audio = [] for audio in audios: if audios_for_one_process == len(tmp): splitted_audio.append(tmp) tmp = [] tmp.append(audio) elif audio == audios[-1]: # 마지막까지 갔을 때.. splitted_audio.append(tmp) else: tmp.append(audio) else: splitted_audio = audios print('splitted audios: {}'.format(len(splitted_audio))) if cpu_num > 1: exe = Executor(max_workers=cpu_num) futures = [] for audio in splitted_audio: futures.append( exe.submit(self.inference_splitted_audios, audio)) results = [future.result() for future in tqdm(futures)] print(results) # print('multiprocessing Result: ', len(result)) # exe.shutdown(wait=True) # exe.shutdown(wait=True) # print(future.result) # result = parmap.map(self.inference_splitted_audios, splitted_audio, pm_pbar=True, pm_processes=cpu_num) else: with Executor(max_workers=cpu_num) as exe: exe.map(self.inference_audio, splitted_audio) exe.shutdown(wait=True) # result = parmap.map(self.inference_audio, splitted_audio, pm_pbar=True, pm_processes=cpu_num) os.chdir(base_dir) return question_folder, results
def svm_kernels(kernels, model, Cs=10.**np.arange(-3, 4), prediction_file=None, repeats=1, **params): """ Evaluates a SVM model with the specified kernels. - First, optimizes C value using a grid search for each of the data sets. - Evaluates the performance of the best C . - Trains on the full data set and generates test predictions (if prediction_file is not None). :param kernels: (train_Ks, test_Ks) containing train and test kernels for all data sets :param model: model constructor :param Cs: values of C to use in the grid search :param prediction_file: file to save the predictions to :param repeats: number of repetitions of the k-fold cross validations. :param params: parameters of the model (excluding C) :return: detailed validation score over each of the 3 data sets. """ train_Ks, test_Ks = kernels params = [dict(C=C, **params) for C in Cs] total_perf = np.zeros(4) full_results = [] with Executor(max_workers=3) as executor: futures = [ executor.submit(grid_search, model, params, K, Y, repeats=repeats) for K, Y in zip(train_Ks, train_Ys) ] res = [future.result() for future in futures] for p, performance in res: total_perf += performance percentages = tuple(100 * performance) print( 'dataset: Validation {:.2f} ± {:.2f}\t Train {:.2f} ± {:.2f}\t C={:.0e}' .format(*percentages, p['C'])) full_results.append(performance) if prediction_file is not None: with Executor(max_workers=3) as executor: futures = [ executor.submit(final_train, model, p, K, Y, K_test) for K, Y, K_test, (p, _) in zip(train_Ks, train_Ys, test_Ks, res) ] predictions = [future.result() for future in futures] save_predictions(predictions, prediction_file) total_percentages = 100 * total_perf / 3 print( 'total: Validation {:.2f} ± {:.2f}\t Train {:.2f} ± {:.2f}\t'.format( *total_percentages)) return full_results
def status(pool: Executor, cwd: PurePath) -> VCStatus: if which("git"): try: r = pool.submit(root, cwd=cwd) s_main = pool.submit(_stat_main, cwd=cwd) s_sub = pool.submit(_stat_sub_modules, cwd=cwd) wait(cast(Sequence[Future], (r, s_main, s_sub))) return _parse(r.result(), stats=chain(s_main.result(), s_sub.result())) except CalledProcessError: return VCStatus() else: return VCStatus()
def batchedPoolRunner(testgenfunc: Function, dispatchfunc: Function, pool: Executor, size: int, validator: Function) -> int: """ Given a concurrent.futures.pool run the tuples produced by testgenfunc in size chunks. Submit results back to pool using dispatchfunc and the returned result of the func. testgenfunc is a function generator that produces a tuple with function to run and parameters to the func. The testgenfunc must return a two tuple with a function in the first position and the function parameters in the second position as a dictionary. dispatchfunc must return an object, list or tuple and these should be compatible with its own inputs. POOL can be a ThreadPoolExecutor or a ProcessPoolExecutor size is the processing batch size for submitting to the pool. testgenfunc should produce tuple with the first element as the function and the second element the parameters to the function NOTE: never create a generator that produces closures. Python internally updates the closure in place instead of creating a new one so you'll effectively have the same closure produced throughout the generators life. It's a nasty bug. *** Currently Doesn't work on Process Pools. Working on a solution. """ td = testgenfunc() futures = set([pool.submit(f, **p) for f, p in chunk(size, td)]) count = 0 debug("batchedPoolRunner: Starting main loop.") while len(futures) > 0: done = set() for job in as_completed(futures): if count % 1000 == 0: gc.collect() rslt = job.result() if job is not None else None if rslt is not None: if not validator(rslt): validationFailure("Test case for %s failed validation." % rslt.Function) else: serviceError("FutureResult from thread pool is None.") done.add(job) count += 1 #futures.remove(job) if dispatchfunc is not None: #debug("Running dispatchfunc %s."%dispatchfunc.__name__) if rslt: futures.add(pool.submit(dispatchfunc, rslt)) sys.stdout.write(".") sys.stdout.flush() futures = futures - done if len(futures) < 1000: debug("Adding new jobs") futures.update( set([pool.submit(f, **p) for f, p in chunk(size, td)])) debug(count) return count
def preprocess_cycles(client: InfluxDBClient, executor: Executor, manager: SyncManager, dry_run=False): logger.info("Preprocessing charge cycles") queue = manager.Queue() series = client.list_series("samples") futures = [] # TODO merge results of different detectors for attr, where, detector in [ ('charger_acvoltage', 'charger_acvoltage>0 OR veh_speed > 0', ChargeCycleACVoltageDetection(time_epoch=client.time_epoch)), ('ischarging', 'ischarging>0 OR veh_speed > 0', ChargeCycleIsChargingDetection(time_epoch=client.time_epoch)), ('ac_hvpower', 'ac_hvpower>0 OR veh_speed > 0', ChargeCycleACHVPowerDetection(time_epoch=client.time_epoch)), ('hvbatt_soc', 'hvbatt_soc<200', ChargeCycleDerivDetection(time_epoch=client.time_epoch)) ]: fields = ["time", "participant", "hvbatt_soc", "veh_speed"] if attr not in fields: fields.append(attr) futures += [executor.submit(preprocess_cycle, nr, client, queue, sname, join_selectors([sselector, where]), fields, detector, dry_run) for nr, (sname, sselector) in enumerate(series)] logger.debug("Tasks started, waiting for results...") async_progress(futures, queue) data = [f.result() for f in futures] logger.debug("Tasks done") data.sort(key=lambda a: a[0:1]) logger.info(__("Detected charge cycles:\n{}", tabulate(data, headers=["attr", "#", "cycles", "cycles_disc"])))
def run_prediction(project_id, model_id): # 学習データ読み込み try: thread_id = "{}_{}".format(project_id, model_id) fields = 'hyper_parameters,algorithm,algorithm_params,best_epoch_weight,dataset_def_id' data = storage.fetch_model(project_id, model_id, fields=fields) (id, name, ratio, train_imgs, valid_imgs, class_map, created, updated) = storage.fetch_dataset_def(data['dataset_def_id']) # weightのh5ファイルのパスを取得して予測する with Executor(max_workers=MAX_THREAD_NUM) as prediction_executor: th = PredictionThread(thread_id, model_id, data["hyper_parameters"], data["algorithm"], data["algorithm_params"], data["best_epoch_weight"], class_map) ft = prediction_executor.submit(th) prediction_thread_pool[thread_id] = [ft, th] ft.result() if th.error_msg is not None: body = json.dumps({"error_msg": th.error_msg}) else: data = { "predict_results": th.predict_results, "csv": th.csv_filename, } body = json.dumps(data) except Exception as e: traceback.print_exc() body = json.dumps({"error_msg": e.args[0]}) ret = create_response(body) return ret
def test_exe(): cgroup_name = 'test_cgroup_executor' setup_cgroup(cgroup_name) print('after setup_cgroup') def dummy(cgroup_name): pid = os.getpid() print('My pid: ', pid) cg = Cgroup(cgroup_name) print(cg.pids) print('dummy groups', cg.cgroups) print('dummy user', cg.user) print('dummy cpu limit', cg.cpu_limit) print('dummy memory limit', cg.memory_limit) cg.add(pid) print('leaving dummy') executor = Executor(max_workers=1, initializer=dummy, initargs=(cgroup_name,)) with executor as exe: f = exe.submit(mem_grower) # exe.submit(time.sleep, 25) # exe.shutdown(wait=True) print(f.result())
def _parallel_pivot_finder(self, X): N = len(X) for k in range(self._dim): print('Working on ' + str(k) + 'D projection') left_pivot_index = random.randint(0, N) right_pivot_index = left_pivot_index for m in range(self._iters): left_pivot_index = right_pivot_index left_pivot = X[left_pivot_index] with Executor(max_workers=self._cores) as executor: map_results = executor.map(self._mapper, [(X[i], i, left_pivot, k) for i in range(N)]) distributor = defaultdict(list) for key, value in map_results: distributor[key].append(value) reduced = executor.map(self._reducer, distributor.items()) right_pivot_index = -1 max_dist = -1 for (idx, dist) in reduced: if dist > max_dist: right_pivot_index = idx max_dist = dist print('Left Pivot: ' + str(left_pivot_index) + '\nRight Pivot: ' + str(right_pivot_index)) left = X[left_pivot_index] left_proj = self._i_proj(left, k) right = X[right_pivot_index] right_proj = self._i_proj(right, k) final_pivots = Pivots(left, left_proj, right, right_proj, max_dist) self._pivots.insert(k, final_pivots)
def submit(self, num_workers=None): """ Start a scan and distribute it on `num_workers` threads. If `num_workers` is omitted, the value of `os.cpu_count()` is used. Results are stored in `self.results`. """ num_workers = os.cpu_count() if not num_workers else num_workers self.parallel = num_workers logging.info('Running on host {}.'.format(os.getenv('HOSTNAME'))) logging.info('Will work on %d threads in parallel.' % num_workers) if num_workers == 1: self.results = self.scan(self.numparas) return paras_per_thread = int(self.numparas / num_workers) remainder = self.numparas % num_workers numparas = [paras_per_thread for p in range(num_workers)] numparas[-1] += remainder with Executor(num_workers) as executor: futures = [ executor.submit(self.scan, j, i) for i, j in enumerate(numparas) ] self.results = [r.result() for r in as_completed(futures)] self.results = concat(self.results, ignore_index=True)
def main(set, config): engine = create_engine(settings.database.get("url")) DBSession.configure(bind=engine) Base.metadata.create_all(engine) """ Lists of Endpoint and Transfer objectwith locks are shared between threads. The classes and objects store all locks and necessary information to sychronized transfers. """ global endpoints global transfers endpoints = [Endpoint(e) for e in config.get("endpoints")] datasets = list(config.get("datasets").keys()) params = config.get("params") if params: GlobusTransfer.deadline = params.get("deadline", 3600) transfers = [ GlobusTransfer(set, s, d, t) for s in endpoints for d in endpoints for t in datasets ] executor = Executor(max_workers=4) while GlobusTransfer.transfers2do > 0: for t in transfers: if t.done: continue if t.acquire(): executor.submit(t.run) sleep(10)
def run_test(): data = [{ "sepal_length": 5.0, "sepal_width": 3.2, "petal_length": 1.2, "petal_width": 0.2 }, { "sepal_length": 5.5, "sepal_width": 3.5, "petal_length": 1.3, "petal_width": 0.2 }, { "sepal_length": 4.9, "sepal_width": 3.1, "petal_length": 1.5, "petal_width": 0.1 }, { "sepal_length": 4.4, "sepal_width": 3.0, "petal_length": 1.3, "petal_width": 0.2 }] with Executor(max_workers=4) as exe: jobs = [exe.submit(request_task, d) for d in data] results = [job.result() for job in jobs] print("The tasks returned these predictions: {}".format(results))
def read_all_sequences( cls, input_dir: str, executor: futures.Executor = None ) -> typing.List[typing.List[knowledge_graph.KnowledgeGraph]]: """Loads all knowledge-graph sequences that are discovered in the specified directory. Args: input_dir (str): The path of the directory that is being searched. executor (futures.Executor, optional): An optional executor for loading multiple knowledge-graph sequences concurrently. Returns: list[list[:class:`knowledge_graph.KnowledgeGraph`]]: All knowledge-graph sequences that were found in ``input_dir``. Raises: ValueError: If the specified directory does not exist. """ # sanitize args input_dir = str(input_dir) if not os.path.isdir(input_dir): raise ValueError("The specified <input_dir> does not exist: '{}'!".format(input_dir)) insanity.sanitize_type("executor", executor, futures.Executor, none_allowed=True) # find all knowledge-graph sequences in the input directory all_seq = io.find_knowledge_graph_sequences(input_dir) # load all knowledge graphs that were found if executor is None: return [cls.read_sequence(input_dir, seq) for seq in all_seq] else: all_seq = [os.path.join(input_dir, seq) for seq in all_seq] return list(executor.map(cls._read_seq_from_one, all_seq))
def start(self): self.beating = True # 监视新的beater ps = self._get_redis().pubsub() ps.subscribe(self.prefix_key + ":new-interval") def _exit(signum, frame): ps.unsubscribe() self.stop() signal.signal(signal.SIGINT, _exit) signal.signal(signal.SIGTERM, _exit) self.workers = Executor(max_workers=self.max_beaters) try: for interval in self.beaters: f = self._create_beater(interval) f.add_done_callback(lambda x: self.beaters.discard(interval)) self.logger.warning('%d beater started' % len(self.beaters)) self._watch_new_interval(ps) except KeyboardInterrupt: _exit(None, None) self.logger.warning('heartbeat maker exit')
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser() parser.add_argument('movies', type=str, nargs='+') group = parser.add_mutually_exclusive_group() group.add_argument('--jobs', '-j', type=int, default=1) group.add_argument('--csv-out', '-c', type=str) parser.add_argument('--ending', '-e', type=int) parser.add_argument('--viewer', '-v', type=str) parser.add_argument('--no-video', '-V', action='store_true') parser.add_argument('--info-dict', '-i', action='store_true') parser.add_argument('--npy-actions', '-a', action='store_true') parser.add_argument('--lossless', '-L', type=str, choices=['mp4', 'mp4rgb', 'png', 'ffv1']) args = parser.parse_args(argv) monitor_csv = None monitor_file = None if args.csv_out: m0 = retro.Movie(args.movies[0]) game = m0.get_game() reward_fields = ['r'] if m0.players == 1 else ['r%d' % i for i in range(m0.players)] monitor_file = open(args.csv_out, 'w') monitor_file.write('#{"t_start": 0.0, "gym_version": "gym_retro", "env_id": "%s"}\n' % game) monitor_csv = csv.DictWriter(monitor_file, fieldnames=reward_fields + ['l', 't']) monitor_csv.writeheader() with Executor(args.jobs or None) as pool: list(pool.map(_play, *zip(*[(movie, args, monitor_csv) for movie in args.movies]))) if monitor_file: monitor_file.close()
def test_pickle(): print_testname(test_pickle.__name__) print_runtime_flag() print_msg("* Should be in color.") print_msg("* Should contain title and status.") print_msg("* Should contain stdout and stderr messages.") print_msg("* Should contain stdout and stderr of subprocess as well.") p = Pipe(dup=True, save=True, tty=True, text=True, mute=False) with Executor() as excecutor: future = excecutor.submit(cmd_print_stdout_stderr, 'foo', with_sub=True, _stdout=p, _stderr=p) cr = future.result() print_result_flag('out') print_msg("* Should contain all stdout lines in color.") print_msg("* Should contain stdout and stderr of subprocess as well.") print(cr.stdout) print_result_flag('err') print_msg("* Should contain all stderr lines in color.") print_msg("* Should contain stdout and stderr of subprocess as well.") print(cr.stderr)
def outdated_pings(outdated_clients: dict): """ Appends comments to clients if pings or not. Used with synchronized decorator for parallel checks (required def with concurrent decorator) :param outdated_clients: dict formatted with clients as keys :return: dict with appended comments """ outdated = outdated_clients with Executor(max_workers=10) as exe: # jobs are executed in parallel with exe.submit # params passed to exe.submit are: (function, args, args) # in this case we pass client, True. True is to get a return as dict # with key as client and new comments # we use one are from a list appending a for in the list jobs = [exe.submit(check_isup, k, True) for k in outdated.keys()] # results are generated from jobs results = [job.result() for job in jobs] for i, values1 in enumerate(results): data_client = list(values1.items()) client = data_client[0][0] comment = data_client[0][1].get('comments') # Append ping information to outdated_clients # as we have same format in results as the original dict, we just update the values. # dict returned by results is {k: {'comments': comments}} outdated[client]['comments'] = comment return outdated
def main() -> None: """The main function.""" args = parse_args() # Check the target target = args.target if not os.path.isfile(target): raise Exception('Target `%s` does not exist' % target) # Maxmimum number of tasks. Two 2 tasks are required as a minimum: one for # running afl-showmap, and another for updating the coverage bitmap max_task = args.jobs if max_task == 0: max_task = multiprocessing.cpu_count() # Check afl-showmap if not which('afl-showmap'): raise Exception('Could not find afl-showmap. Check PATH') # Wait for fuzzer_stats to exist out_dir = args.out_dir fuzzer_stats_path = out_dir / 'fuzzer_stats' if not fuzzer_stats_path.exists(): raise Exception('No fuzzer_stats in `%s`' % out_dir) # Open CSV plot_data csv_path = args.csv if csv_path: csv_path = Path(csv_path) with open(csv_path, 'w') as outf: CsvDictWriter(outf, fieldnames=CSV_FIELDNAMES).writeheader() with Executor(max_workers=max_task) as executor, \ open(out_dir / 'blackbox.tar.gz', 'wb') as tar_file: # The coverage bitmap cov_bitmap = [255] * MAP_SIZE cov_queue = Queue(max_task) # Thread responsible for deduplicating entries in the output directory # and logging coverage to a CSV cov_thread = Thread(target=remove_duplicate_testcases, args=(cov_bitmap, cov_queue, tar_file, csv_path)) cov_thread.daemon = True cov_thread.start() # Start the monitor handler = TestCaseHandler(executor, cov_queue, target, afl_stats, args.timeout) observer = Observer() observer.schedule(handler, out_dir / 'queue' / '.blackbox') observer.start() # Continue until interrupted try: while observer.is_alive(): observer.join(1) except KeyboardInterrupt: print('\nCtrl-C detected, goodbye') observer.stop() observer.join()
def main(): '''stdin is assumed to be the output of ./find_commits.sh [some project directory] [some project name]''' worklist = flatten_lines(sys.stdin) from concurrent.futures import ThreadPoolExecutor as Executor with Executor(max_workers=150) as executor: utils.exhaust(executor.map(fetch_and_compose, worklist))
def submit(self, num_workers=None): """ Start a scan and distribute it on `num_workers` threads. If `num_workers` is omitted, the value of `os.cpu_count()` is used. Results are stored in `self.results`. """ num_workers = os.cpu_count() if not num_workers else num_workers if not self.scanset: self.build(num_workers) if num_workers == 1: runner = self.runner(self.config['runner']) self.results = concat([runner.run(d) for d in tqdm(self.scanset)], ignore_index=True) return chunksize = min(int(self.numparas / num_workers), 1000) chunks = range(0, self.numparas, chunksize) logging.info('Running on host {}.'.format(os.getenv('HOSTNAME'))) logging.info('Splitting dataset into %d chunks.' % len(chunks)) logging.info('Will work on %d chunks in parallel.' % num_workers) with Executor(num_workers) as executor: futures = [ executor.submit(self.scan, self.scanset[i:i + chunksize]) for i in chunks ] progresser = tqdm(as_completed(futures), total=len(chunks), unit='chunk') self.results = [r.result() for r in progresser] self.results = concat(self.results, ignore_index=True)
def _run_in_pool(executor: Executor, func: Func[T, R], items: List[T], *args: Any, chunk_size: Optional[int] = None) -> Iterable[R]: if chunk_size is None: chunk_size = 1 chunks: Iterable[List[T]] = _split_to_chunks(items, chunk_size) exceptions: List[BaseException] = [] with executor: futures: Dict[Future, int] = {} for chunk in chunks: future = executor.submit(func, chunk, *args) futures[future] = len(chunk) with tqdm(total=len(items)) as pbar: for future in as_completed(set(futures.keys())): pbar.update(futures[future]) del futures[future] e = future.exception() if e is not None: exceptions.append(e) else: yield future.result() if len(exceptions) > 0: raise exceptions[0]
def _parallel_transform(self, X): with Executor(max_workers=self._cores) as executor: map_results = executor.map( lambda x: (x[0], self._i_proj(x[1], self._dim)), enumerate(X)) map_results = list(map_results) map_results.sort(key=lambda x: x[0]) return [result[1] for result in map_results]
def find(project: str, look_back: datetime, evg_api: EvergreenApi, n_threads: int = DEFAULT_THREADS) -> Dict: """ Find test flips in the evergreen project. :param project: Evergreen project to analyze. :param look_back: Look at commits until the given project. :param evg_api: Evergreen API. :param n_threads: Number of threads to use. :return: Dictionary of commits that introduced task flips. """ LOGGER.debug("Starting find_flips iteration") version_iterator = evg_api.versions_by_project(project) with Executor(max_workers=n_threads) as exe: jobs = [] for next_version, version, prev_version in windowed_iter( version_iterator, 3): log = LOGGER.bind(version=version.version_id) log.debug("Starting to look") if version.create_time < look_back: log.debug("done", create_time=version.create_time) break work_item = WorkItem(version, next_version, prev_version) jobs.append(exe.submit(_flips_for_version, work_item)) results = [job.result() for job in jobs] return {r.revision: r.flipped_tasks for r in results if r.flipped_tasks}
def parallel_dists(dist_fn, weights, X, Y=None, tqdm=False): X = ag.tensor(X.astype(np.int)) if Y is None: Y = X Ys = [Y[i:].data for i in range(len(X))] else: Y = ag.tensor(Y.astype(np.int)) Ys = [Y.data for i in range(len(X))] weights = ag.tensor(weights) d = np.zeros((len(X), len(Y)), dtype=np.float32) g = np.empty((len(weights), len(X), len(Y)), dtype=np.float32) with Executor(max_workers=multiprocessing.cpu_count()) as executor: iterator = enumerate( executor.map(dist_fn, X.data, Ys, [weights.data] * len(X))) if tqdm: iterator = tqdm_notebook(iterator, total=len(X)) for i, (row, grad_row) in iterator: if len(row) < len(X): d[i, i:] = row d[i, :i] = d[:i, i] g[:, i, i:] = grad_row g[:, i, :i] = g[:, :i, i] else: d[i] = row g[:, i] = grad_row g_tensor = ag.Tensor(g, None, children=[]) def grad_d(leaf_id): return ag.tensordot(weights.compute_grad(leaf_id), g_tensor, axes=([-1], [0])) return ag.Tensor(d, grad_d, children=[weights])
def multi_evaluate(repetitions): """ To be called when a chromosome needs to be evaluated. Ace0 will provide the fitness score. :param repetitions: number of times for the simulation to run :return: a fitness score for the chromosome as a float """ # hold the total fitness total_fitness = 0 # process pool operator creates/ submits as many process's as we need to evaluate the chromosome with Executor(max_workers=5) as executor: processes = [ executor.submit(ace_zero.return_fitness_ace_zero) for x in range(repetitions) ] # as the processes are completed, they are summed up for f in as_completed(processes): try: total_fitness += f.result() except ValueError: print 'Multiprocessing error' return total_fitness / repetitions,
def install_packages(project_path: str): """Installs packages in virtual environment Args: project_path (str): Path to where the project lives """ venv_init() env, pip = get_env_vars(project_path) needed_packages = get_user_packages() print(to_be_installed_str(needed_packages, on=True)) start = time.perf_counter() with Executor() as executor: threads = list( tqdm(executor.map(async_install, repeat([env, pip]), needed_packages), total=len(needed_packages))) finish = time.perf_counter() timing = round(finish - start, 3) installed_packages, total_packages = get_installed_packages(project_path) total_main_packages = len(needed_packages) total_dependencies = total_packages - total_main_packages print(installed_packages_str(total_main_packages, total_dependencies, timing)) print(list_packages_str(installed_packages))
def _bulk_cleanup(self, jobs: Sequence["Job"], ex: Executor) -> Iterable["Job"]: jobs = self.bulk_sync_status(jobs) # safety check for job in jobs: assert job.driver == self.__class__ if job.status in (Job.Status.SUBMITTED, Job.Status.RUNNING): raise InvalidJobStatus( f"Job {job} might be running, please kill first") logger.debug("Cleaning up %d jobs", len(jobs)) def run(job: Job) -> Job: for d in ["log_dir", "output_dir"]: try: path = job.data[d] if os.path.exists(path): logger.debug("Path %s exists, attempting to delete", path) rmtree(path) except Exception: logger.error("Unable to remove directory %s", d) return job futures = [ex.submit(run, j) for j in jobs] for f in as_completed(futures): yield f.result()
def __init__(self, job_id=None): self.data_dir = os.path.join(file_utils.get_project_base_directory(), 'data') self.job_id = str( uuid.uuid1()) if job_id is None else "{}".format(job_id) self.meta_table = _DTable('__META__', '__META__', 'fragments', 10) self.pool = Executor() Standalone.__instance = self