def collect(self): start = default_timer() executor = Executor(max_workers=len(plugins)) futures = { executor.submit(plugin.run): name for name, plugin in plugins.items() } try: for future in concurrent.futures.as_completed( futures, timeout=cfg.scrape_timeout_seconds): fn_name = futures[future] try: future.result() except Exception: exceptions.labels(fn_name).inc() log.error("plugin %r generated an exception", fn_name) else: duration = max(default_timer() - start, 0) scrape_plugin_duration_seconds.labels(fn_name).set( duration) log.debug("%s finished in %.02fs", fn_name, duration) except concurrent.futures.TimeoutError: for future, fn_name in futures.items(): if not future.done(): timeouts.labels(fn_name).inc() log.error("plugin %r timeout exceeded", fn_name) log.info("Scrape complete") return []
async def select_now(cases: dict): futures = { key: asyncio.ensure_future(t) for key, t in cases.items() } completed = next(((k, f) for k, f in futures.items() if f.done()), (None, None)) return completed
def main(input_dir, output_dir, agency_file): PA_lex = get_PA_lexicon(agency_file) analyser = SentimentIntensityAnalyzer() if not os.path.exists(output_dir): os.makedirs(output_dir) if os.path.isfile(input_dir): result = analyize_script(PA_lex, analyser, input_dir) if result is not None: result.to_csv( os.path.join( output_dir, '%s.csv' % os.path.splitext(os.path.basename(input_dir))[0])) return # must be directory with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: futures = {} for jsonfile in [ x for x in os.listdir(input_dir) if x.endswith('.json') ]: print('Analyzing %s' % jsonfile) future = executor.submit(analyize_script, PA_lex, analyser, os.path.join(input_dir, jsonfile)) futures[jsonfile] = future print(len(futures)) for jsonfile, future in futures.items(): try: result = future.result() if result is not None: print("Writing result %s" % jsonfile) result.to_csv( os.path.join(output_dir, '%s.csv' % os.path.splitext(jsonfile)[0])) except Exception as exc: print('Failed to process file; exception: %s' % exc)
def _download_objects(self) -> None: """Download objects from the specified S3 bucket and path prefix.""" if not self._quiet: print("Starting downloads...") with concurrent.futures.ThreadPoolExecutor( max_workers=self._threads ) as executor: futures = {} while True: item = self._object_queue.get(block=True) if item is None: # Check for sentinel value break futures[item] = executor.submit(self._download_object, item) for key, future in futures.items(): try: future.result() self._successful_downloads += 1 except KeyboardInterrupt: if not self._quiet: print("\nThreads are exiting...") executor.shutdown(wait=False) self._keyboard_interrupt_exit.set() raise except Exception as e: self._failed_downloads.append((key, e))
def main(args): nltk.download('names') input_dir = args.input output_dir = args.output cast_data = args.cast_data if not os.path.exists(os.path.join(output_dir, 'adjacency')): os.makedirs(os.path.join(output_dir, 'adjacency')) if not os.path.exists(os.path.join(output_dir, 'bechdel')): os.makedirs(os.path.join(output_dir, 'bechdel')) if os.path.isfile(input_dir): analyze_scene(input_dir, cast_data, output_dir) return with concurrent.futures.ProcessPoolExecutor(max_workers=2) as executor: futures = {} for jsonfile in [ x for x in os.listdir(input_dir) if x.endswith('.json') ]: print('Analyzing %s' % jsonfile) future = executor.submit(analyze_scene, os.path.join(input_dir, jsonfile), cast_data, output_dir) futures[jsonfile] = future print(len(futures)) for jsonfile, future in futures.items(): try: result = future.result() # result.to_csv(os.path.join(output_dir, '%s.csv' % os.path.splitext(jsonfile)[0])) except Exception as exc: print('Failed to process file %s; exception: %s' % (jsonfile, exc))
def _remove_completed_rpcs(futures: Mapping[int, grpc.Future], print_response: bool) -> None: logger.debug("Removing completed RPCs") done = [] for future_id, future in futures.items(): if future.done(): _on_rpc_done(future_id, future, args.print_response) done.append(future_id) for rpc_id in done: del futures[rpc_id]
async def run_rectangles_circum(exe, output_dir: str): futures = dict() loop = asyncio.get_running_loop() for n in range(1, CIRCUM_MAX + 1): rec = rectangles_circum(n) f = loop.run_in_executor(exe, rpack.pack, rec) futures[f] = rec output_file = os.path.join(output_dir, f'circum{CIRCUM_MAX}.pickle') with open(output_file, 'wb') as out_f: for f, rec in futures.items(): pos = await f pickle.dump((rec, pos), out_f) print('Done:', output_file)
def extract_everything(): """Extract everything in components.yml, respecting order requirements.""" def q_key(comp): """Decide extract priority by pointer-chase depth, filesize in ties.""" after = {c.install_after: c.name for c in component.ALL.values()} name, seen = comp.name, [] while name in after: seen.append(name) name = after.get(name) if name in seen: raise ValueError('Cyclic "install_after" config detected: ' + " -> ".join(seen + [name])) return len(seen), os.path.getsize(comp.path) queue = list(component.ALL.values()) + [ component.ALL["Dwarf Fortress"]._replace(name=path, extract_to=path) for path in ("curr_baseline", "graphics/ASCII") ] queue.sort(key=q_key, reverse=True) with concurrent.futures.ProcessPoolExecutor(8) as pool: futures = {} while queue: while sum(f.running() for f in futures.values()) < 8: for idx, comp in enumerate(queue): if comp.extract_to is False: assert comp.filename.endswith(".ini") queue.pop(idx) continue # for Therapist, handled in build.py aft = futures.get(comp.install_after) # Even if it's highest-priority, wait for parent job(s) if aft is None or aft.done(): futures[comp.name] = extract_comp(pool, queue.pop(idx)) break # reset index or we might pop the wrong item else: break # if there was nothing eligible to extract, sleep time.sleep(0.01) failed = [k for k, v in futures.items() if v.exception() is not None] for key in failed: comp = component.ALL.pop(key, None) for lst in (component.FILES, component.GRAPHICS, component.UTILITIES): if comp in lst: lst.remove(comp) if failed: print("ERROR: Could not extract: " + ", ".join(failed))
async def run_rectangles_random(exe, output_dir: str, rec_func): """Build arguments of multiprocess task""" futures = dict() loop = asyncio.get_running_loop() progress = dict() for n in range(N_STEP, N_MAX + 1, N_STEP): for m in range(M_STEP, M_MAX + 1, M_STEP): prefix = rec_func.__name__.replace('rectangles_', '') name = f'{prefix}{n:03}n{m:04}m.pickle' output_file = os.path.join(output_dir, name) if m == M_MAX: s = no_samples(n) elif n == N_MAX: s = 100 else: s = 10 progress[output_file] = s for _ in range(s): rec = rec_func(n, m) f = loop.run_in_executor(exe, task, rec) futures[f] = output_file, rec files = dict() try: for f, (output_file, rec) in futures.items(): pos, dt = await f if output_file not in files: files[output_file] = open(output_file, 'wb') out_f = files[output_file] pickle.dump((rec, pos, dt), out_f) progress[output_file] -= 1 if progress[output_file] == 0: del progress[output_file] print('Done:', output_file, end='. ') print('Files remaining:', len(progress)) finally: for f in files.values(): f.close()
def load_places(): # create a csv writer. The headings will be written in the order shown writer = csv.DictWriter(open('data/places.csv', 'w'), ['name', 'tel', 'street', 'locality', 'region', 'postcode']) with open('data/hospital-overall-ratings.csv', 'r') as f: reader = csv.DictReader(f) # go through each row and extract the name names = [row['Org Name'] for row in reader] # create a thread pool so that we can run the requests concurrently extension = '/Service-Search/Disambiguation/Organisation?serviceName=' with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: # for every place create a future which will hold the result of # calling search for that place futures = {name: executor.submit(search, BASE + extension + name) for name in names} for name, future in futures.items(): res = future.result() if res != None: # search will return all the values needed apart from the # name of the place. Add this in and write the row res['name'] = name writer.writerow(res)
def main(): # Parse command line arguments. parser = argparse.ArgumentParser( description="Check source code for formatting and linter problems.") parser.add_argument( "-a", "--no-cache", action="store_true", help="ignore the cache and check all files", ) parser.add_argument( "-m", "--allow-modifications", action="store_true", help="allow the tools to modify files", ) parser.add_argument( "-j", "--num_workers", type=int, default=min(16, os.cpu_count() + 2), help="number of parallel workers", ) parser.add_argument( "--progressbar-wait", metavar="SECONDS", type=int, default=1, help="number seconds in between progressbar updates", ) parser.add_argument("files", metavar="FILE", nargs="*", help="files to process") args = parser.parse_args() # Say hello to the server. print( f"Running precommit checks using {args.num_workers} workers and server: {SERVER}" ) server_hello = urlopen(Request(SERVER + "/"), timeout=10).read().decode("utf8") assert server_hello.startswith("cp2k precommit server") # Store candidate before changing base directory and creating scratch dir. file_list = [os.path.abspath(fn) for fn in args.files] base_dir = Path(__file__).resolve().parent.parent.parent os.chdir(base_dir) SCRATCH_DIR.mkdir(parents=True, exist_ok=True) # Collect candidate files. if not file_list: sys.stdout.write("Searching for files...\r") sys.stdout.flush() for root, dirs, files in os.walk("."): if root.startswith("./tools/toolchain/build"): continue if root.startswith("./tools/toolchain/install"): continue if root.startswith("./tools/prettify/fprettify"): continue if root.startswith("./tools/build_utils/fypp"): continue if root.startswith("./tools/autotools"): continue if root.startswith("./tools/minimax_tools/1_xData"): continue if root.startswith("./data/DFTB/scc"): continue if root.startswith("./arch"): continue if root.startswith("./doxygen"): continue if root.startswith("./exts"): continue if root.startswith("./obj"): continue if root.startswith("./lib"): continue if root.startswith("./exe"): continue if root.startswith("./regtesting"): continue if root.startswith("./.git"): continue if "/.mypy_cache/" in root: continue file_list += [os.path.join(root, fn) for fn in files] # Filter symlinks, backup copies, and hidden files. file_list = [fn for fn in file_list if not os.path.islink(fn)] file_list = [fn for fn in file_list if not fn[-1] in ("~", "#")] file_list = [ fn for fn in file_list if not os.path.basename(fn).startswith(".") ] # Sort files by size as larger ones will take longer to process. file_list.sort(reverse=True, key=lambda fn: os.path.getsize(fn)) # Load cache. should_load_cache = CACHE_FILE.exists() and not args.no_cache cache = json.loads(CACHE_FILE.read_text()) if should_load_cache else {} # Launch async processing of files. futures = {} executor = concurrent.futures.ThreadPoolExecutor( max_workers=args.num_workers) for fn in file_list: if os.path.getmtime(fn) != cache.get(fn, -1): futures[fn] = executor.submit(process_file, fn, args.allow_modifications) num_skipped = len(file_list) - len(futures) # Continuously update progressbar, save cache file, and print errors. failed_files = set() while True: num_done = num_skipped for fn, f in futures.items(): if f.done(): num_done += 1 if not f.exception(): cache[fn] = os.path.getmtime(fn) elif fn not in failed_files: failed_files.add(fn) print_box(fn, str(f.exception())) CACHE_FILE.write_text(json.dumps(cache)) progressbar = "=" * int(60 * num_done / len(file_list)) sys.stdout.write( f"[{progressbar:60s}] {num_done} / {len(file_list)} files processed\r" ) sys.stdout.flush() if num_done == len(file_list) or len(failed_files) >= 10: executor.shutdown(wait=False) break sleep(args.progressbar_wait) # Print final message. print(f"Summary: Found {len(file_list)}, " f"skipped {num_skipped}, " f"checked {num_done - num_skipped}, " f"and failed {len(failed_files)} files." + (" " * 50)) print("Status: " + ("FAILED" if failed_files else "OK")) sys.exit(len(failed_files))
def construct_kernel_matrix(self, x1_vec, x2_vec=None, quantum_instance=None): """ Construct kernel matrix, if x2_vec is None, self-innerproduct is conducted. Args: x1_vec (numpy.ndarray): data points, 2-D array, N1xD, where N1 is the number of data, D is the feature dimension x2_vec (numpy.ndarray): data points, 2-D array, N2xD, where N2 is the number of data, D is the feature dimension quantum_instance (QuantumInstance): quantum backend with all setting Returns: numpy.ndarray: 2-D matrix, N1xN2 """ self._quantum_instance = self._quantum_instance \ if quantum_instance is None else quantum_instance from .qsvm_kernel import QSVMKernel if x2_vec is None: is_symmetric = True x2_vec = x1_vec else: is_symmetric = False is_statevector_sim = self.quantum_instance.is_statevector measurement = not is_statevector_sim measurement_basis = '0' * self.num_qubits mat = np.ones((x1_vec.shape[0], x2_vec.shape[0])) num_processes = psutil.cpu_count( logical=False) if platform.system() != "Windows" else 1 # get all to-be-computed indices if is_symmetric: mus, nus = np.triu_indices(x1_vec.shape[0], k=1) # remove diagonal term else: mus, nus = np.indices((x1_vec.shape[0], x2_vec.shape[0])) mus = np.asarray(mus.flat) nus = np.asarray(nus.flat) for idx in range(0, len(mus), QSVMKernel.BATCH_SIZE): circuits = {} to_be_simulated_circuits = [] with concurrent.futures.ProcessPoolExecutor( max_workers=num_processes) as executor: futures = {} for sub_idx in range( idx, min(idx + QSVMKernel.BATCH_SIZE, len(mus))): i = mus[sub_idx] j = nus[sub_idx] x1 = x1_vec[i] x2 = x2_vec[j] if not np.all(x1 == x2): futures["{}:{}".format(i, j)] = \ executor.submit(QSVMKernel._construct_circuit, x1, x2, self.num_qubits, self.feature_map, measurement, "circuit{}:{}".format(i, j)) for k, v in futures.items(): circuit = v.result() circuits[k] = circuit to_be_simulated_circuits.append(circuit) results = self.quantum_instance.execute(to_be_simulated_circuits) kernel_values = {} with concurrent.futures.ProcessPoolExecutor( max_workers=num_processes) as executor: for idx, circuit in circuits.items(): kernel_values[idx] = executor.submit( QSVMKernel._compute_overlap, results, circuit, is_statevector_sim, measurement_basis) for k, v in kernel_values.items(): i, j = [int(x) for x in k.split(":")] mat[i, j] = v.result() if is_symmetric: mat[j, i] = mat[i, j] return mat
def execute_together( cls, remotes, # type: typing.Iterable[SSHClientBase] command, # type: str timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] expected=None, # type: typing.Optional[typing.Iterable[int]] raise_on_err=True, # type: bool **kwargs ): # type: (...) -> typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] """Execute command on multiple remotes in async mode. :param remotes: Connections to execute on :type remotes: typing.Iterable[SSHClientBase] :param command: Command for execution :type command: str :param timeout: Timeout for command execution. :type timeout: typing.Union[int, None] :param expected: expected return codes (0 by default) :type expected: typing.Optional[typing.Iterable[]] :param raise_on_err: Raise exception on unexpected return code :type raise_on_err: bool :return: dictionary {(hostname, port): result} :rtype: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] :raises ParallelCallProcessError: Unexpected any code at lest on one target :raises ParallelCallExceptions: At lest one exception raised during execution (including timeout) .. versionchanged:: 1.2.0 default timeout 1 hour .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ @threaded.threadpooled def get_result(): # type: () -> exec_result.ExecResult """Get result from remote call.""" ( chan, _, stderr, stdout, ) = remote.execute_async(command, **kwargs) # type: _type_execute_async chan.status_event.wait(timeout) exit_code = chan.recv_exit_status() # pylint: disable=protected-access cmd_for_log = remote._mask_command(cmd=command, log_mask_re=kwargs.get( 'log_mask_re', None)) # pylint: enable=protected-access result = exec_result.ExecResult(cmd=cmd_for_log) result.read_stdout(src=stdout) result.read_stderr(src=stderr) result.exit_code = exit_code chan.close() return result expected = expected or [proc_enums.ExitCodes.EX_OK] expected = proc_enums.exit_codes_to_enums(expected) futures = {} results = {} errors = {} raised_exceptions = {} for remote in set(remotes): # Use distinct remotes futures[remote] = get_result() ( _, not_done, ) = concurrent.futures.wait( list(futures.values()), timeout=timeout ) # type: typing.Set[concurrent.futures.Future], typing.Set[concurrent.futures.Future] for future in not_done: # pragma: no cover future.cancel() for ( remote, future, ) in futures.items(): # type: SSHClientBase, concurrent.futures.Future try: result = future.result() results[(remote.hostname, remote.port)] = result if result.exit_code not in expected: errors[(remote.hostname, remote.port)] = result except Exception as e: raised_exceptions[(remote.hostname, remote.port)] = e if raised_exceptions: # always raise raise exceptions.ParallelCallExceptions(command, raised_exceptions, errors, results, expected=expected) if errors and raise_on_err: raise exceptions.ParallelCallProcessError(command, errors, results, expected=expected) return results
or opt_dst_location == 'dr') GTMs = Pri_GTM if if_has_Pri else Pub_GTM if if_has_Pri: break logger.info('{}'.format(GTMs)) device_topo_info_dict = Vividict() topo_record = {} with concurrent.futures.ThreadPoolExecutor() as pool: t = threading.currentThread() logger.info('{} {}'.format(t.getName(), t.ident)) futures = {} uri_prefix = '/mgmt/tm/gtm/topology/' for gtm_ip in GTMs: future = pool.submit(do_get_gtm_topo, gtm_ip) futures[future] = gtm_ip for future, gtm_ip in futures.items(): ldns_info, dst_info, score_info = do_collate_gtm_topo( future.result()) device_topo_info_dict[gtm_ip]['ldns'] = ldns_info device_topo_info_dict[gtm_ip]['destination'] = dst_info device_topo_info_dict[gtm_ip]['weight'] = score_info logger.info(device_topo_info_dict) region_ip_list = do_transfrom_ip_to_region(src_ip_list) random_str = ''.join( random.sample(string.ascii_letters + string.digits, 5)) data_json, src_region_name = do_generate_gtm_region_json( region_ip_list, name_prefix="SrcRegion_{}".format(random_str)) topo_record['SrcName'] = src_region_name uri_prefix = '/mgmt/tm/gtm/region' topo_futures = [] for gtm_ip in GTMs:
urlwatch_output = subprocess.check_output( [ 'urlwatch', '--urls', args.urls_yaml, '--test-filter', '{}'.format(filter_num) ], stderr=subprocess.STDOUT).decode() except subprocess.CalledProcessError as e: urlwatch_output = e.output.decode() return urlwatch_output with open(args.urls_yaml, 'r') as yaml_file: urls_list = yaml.load_all(yaml_file, Loader=yaml.FullLoader) futures = {} with concurrent.futures.ThreadPoolExecutor( max_workers=args.parallelism) as executor: for i, url in enumerate(urls_list): # filters are 1-indexed futures[url['name']] = executor.submit(evaluate_filter, i + 1) for name, future in futures.items(): urlwatch_output = str(future.result()) if args.output_dir: with open(os.path.join(args.output_dir, name), 'w') as output_file: print(name) output_file.write(urlwatch_output) else: print(f"{name}:\n{urlwatch_output}")