def farm_process(tasks): if not tasks: return try: servs_pool = pool.ThreadPool(processes=CONFIG['serv_thrds']) rrd_pool = pool.ThreadPool(processes=CONFIG['rrd_thrds']) results = servs_pool.map(server_thread, [(t, rrd_pool) for t in tasks]) servs_pool.close() if not results: return ra, fa, rs, fs = post_processing(results) for k, v in ra.iteritems(): rrd_pool.map_async(RRDWorker().work, [{'ra': {k: v}}]) for k, v in fa.iteritems(): rrd_pool.map_async(RRDWorker().work, [{'fa': {k: v}}]) for k, v in rs.iteritems(): rrd_pool.map_async(RRDWorker().work, [{'rs': {k: v}}]) for k, v in fs.iteritems(): rrd_pool.map_async(RRDWorker().work, [{'fs': {k: v}}]) except: LOG.error(helper.exc_info()) finally: servs_pool.close() servs_pool.join() rrd_pool.close() rrd_pool.join()
def multithreaded_call( fn: Callable[..., Any], args: Iterable[Tuple[Any, ...]], worker_count: int = 0, thread_pool: Optional[pool.ThreadPool] = None) -> List[Any]: """Runs the given function in a thread pool and returns the results. Args: fn: Function to call in parallel. args: An iterable of argument list to call fn with. worker_count: Number of thread workers to launch. thread_pool: Optional thread pool to use instead of creating a new one. Returns: Function results in a list. """ if not thread_pool: if worker_count: threads = pool.ThreadPool(worker_count) else: threads = pool.ThreadPool() else: threads = thread_pool results = threads.starmap(fn, args) if not thread_pool: threads.close() threads.join() return list(results)
def __init__(self, n, probe_key, ignore_clock_skew=False, metadata_encoding=None, disable_action_probes=False): # Each QR code takes about 1ms (and updates at 5fps). We do # our best to ensure the QR is processed in time for the next # step call (n/16 would put us right at the threshold). self.pool = pool.ThreadPool(max(int(n/4), 1)) self.qr_pool = pool.ThreadPool(max(int(n/8), 1)) self.lock = threading.RLock() self.instance_n = [None] * n self.ignore_clock_skew = ignore_clock_skew self.disable_action_probes = disable_action_probes self.metadata_encoding = metadata_encoding self.update(probe_key=probe_key, metadata_encoding=metadata_encoding)
def scaling_metrics(self): """ :return: list of scaling metrics :rtype: list Example:: [{ 'id': 101011, 'name': 'jmx.scaling', 'value': 1, 'error': None }, { 'id': 202020, 'name': 'app.poller', 'value': None, 'error': 'Couldnt connect to host' }] """ # Obtain scaling metrics from Scalr. scaling_metrics = bus.queryenv_service.get_scaling_metrics() if not scaling_metrics: return [] if not hasattr(threading.current_thread(), '_children'): threading.current_thread()._children = weakref.WeakKeyDictionary() wrk_pool = pool.ThreadPool(processes=10) try: return wrk_pool.map_async(_ScalingMetricStrategy.get, scaling_metrics).get() finally: wrk_pool.close() wrk_pool.join()
def parallelCCompiler(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): """ Monkey-patch for parallel compilation with distutils. """ # those lines are copied from distutils.ccompiler.CCompiler directly macros, objects, extra_postargs, pp_opts, build = self._setup_compile( output_dir, macros, include_dirs, sources, depends, extra_postargs) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) def _single_compile(obj): try: src, ext = build[obj] except KeyError: return self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) # convert to list, imap is evaluated on-demand list(pool.ThreadPool(cpu_count()).imap(_single_compile, objects)) return objects
def scaling_metrics(self): ''' @return list of scaling metrics @rtype: list Sample: [{ 'id': 101011, 'name': 'jmx.scaling', 'value': 1, 'error': None }, { 'id': 202020, 'name': 'app.poller', 'value': None, 'error': 'Couldnt connect to host' }] ''' # Obtain scaling metrics from Scalr. scaling_metrics = bus.queryenv_service.get_scaling_metrics() max_threads = 10 wrk_pool = pool.ThreadPool(processes=max_threads) try: return wrk_pool.map_async(_ScalingMetricStrategy.get, scaling_metrics).get() finally: wrk_pool.close() wrk_pool.join()
def Execute(self, thread_count, urns=None): """Runs the migration with a given thread count.""" if urns is None: blob_urns = list(aff4.FACTORY.ListChildren("aff4:/blobs")) else: blob_urns = [rdfvalue.RDFURN(urn) for urn in urns] sys.stdout.write("Blobs to migrate: {}\n".format(len(blob_urns))) sys.stdout.write("Threads to use: {}\n".format(thread_count)) self._total_count = len(blob_urns) self._migrated_count = 0 self._start_time = rdfvalue.RDFDatetime.Now() batches = collection.Batch(blob_urns, _BLOB_BATCH_SIZE) self._Progress() tp = pool.ThreadPool(processes=thread_count) tp.map(self._MigrateBatch, list(batches)) self._Progress() if self._migrated_count == self._total_count: message = "\nMigration has been finished (migrated {} blobs).\n".format( self._migrated_count) sys.stdout.write(message) else: message = "Not all blobs have been migrated ({}/{})".format( self._migrated_count, self._total_count) raise AssertionError(message)
def test_multi_instance(self): """Validates that two instances of the env can run in the same thread.""" tpool = pool.ThreadPool(processes=2) run1 = tpool.apply_async(self.check_determinism) run2 = tpool.apply_async(self.check_determinism) run1.get() run2.get()
def HandleEventsOnce(self, mv): """One turn through the loop. Separated out for unit testing. @param mv: an instance of manifest_versions.ManifestVersions. @raise EnumeratorException if we can't enumerate any supported boards. """ boards = self._enumerator.Enumerate() logging.info('%d boards currently in the lab: %r', len(boards), boards) thread_pool = pool.ThreadPool(POOL_SIZE) with contextlib.closing(thread_pool): for e in self._events.itervalues(): if not e.ShouldHandle(): continue logging.info('Handling %s event for %d boards', e.keyword, len(boards)) args = [] for board in boards: args.append({ 'scheduler': self._scheduler, 'event': e, 'board': board }) thread_pool.map(self.HandleBoard, args) logging.info('Finished handling %s event for %d boards', e.keyword, len(boards)) e.UpdateCriteria()
def _run_scenario(self, cls, method_name, context, args): times = self.config["times"] period = self.config["period"] timeout = self.config.get("timeout", 600) async_results = [] for i in range(times): pool = multiprocessing_pool.ThreadPool(processes=1) scenario_args = ((i, cls, method_name, base._get_scenario_context(context), args), ) async_result = pool.apply_async(base._run_scenario_once, scenario_args) async_results.append(async_result) if i < times - 1: time.sleep(period) results = [] for async_result in async_results: try: result = async_result.get(timeout=timeout) except multiprocessing.TimeoutError as e: result = { "duration": timeout, "idle_duration": 0, "error": utils.format_exc(e) } results.append(result) return base.ScenarioRunnerResult(results)
def main4(): pool = mp.ThreadPool(5) for i in range(5): gen = simple_generator2(i) pool.apply_async(generator_caller, (gen,)) pool.close() pool.join()
def parallel_compile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): """A parallel version of the Distutils compile method Note that this routine is modified from StackOverflow post #11013851 """ # Copy args from distutils.ccompiler.CCompiler directly macros, objects, extra_postargs, pp_opts, build = \ self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) num_cpus = multiprocessing.cpu_count() # Define routine for each thread to use to compile on its own def _single_compile(obj): try: src, ext = build[obj] except KeyError: return self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) # Convert thread mapping to C/C++/CUDA objects to a list and return list(pool.ThreadPool(num_cpus).map(_single_compile, objects)) return objects
def __init__(self, directory, image_data_generator=None, target_size=(256, 256), color_mode='rgb', classes=None, class_mode='categorical', batch_size=32, shuffle=True, data_format='channels_last', follow_links=False, subset=None, interpolation='nearest', tfrecord='dataset.tfr', num_copies=3, dtype='float32'): self.set_processing_attrs(image_data_generator, target_size, color_mode, data_format, interpolation, tfrecord, num_copies) self.directory = directory self.white_list_formats = 'png' if not classes: classes = [] for subdir in sorted(os.listdir(directory)): if os.path.isdir(os.path.join(directory, subdir)): classes.append(subdir) self.num_classes = len(classes) self.class_indices = dict(zip(classes, range(len(classes)))) p = pool.ThreadPool() # Second, build an index of the images # in the different class subfolders. results = [] self.filenames = [] i = 0 for dirpath in (os.path.join(directory, subdir) for subdir in classes): results.append( p.apply_async(_list_valid_filenames_in_directory, (dirpath, self.white_list_formats, self.class_indices, False))) classes_list = [] for res in results: classes, filenames = res.get() classes_list.append(classes) self.filenames += filenames self.samples = len(self.filenames) self.classes = np.zeros((self.samples, ), dtype='int32') for classes in classes_list: self.classes[i:i + len(classes)] = classes i += len(classes) print('Found %d images belonging to %d classes.' % (self.samples, self.num_classes)) p.close() p.join() self._filepaths = [ os.path.join(self.directory, fname) for fname in self.filenames ]
def execute(self, user, cmd): tpool = pool.ThreadPool(processes=len(self.__hosts)) async_results = [] for host in self.__hosts: def wrapExecute(host, user, cmd): try: host.execute(user, cmd) except ex.SubprocessException as e: return e async_result = tpool.apply_async(wrapExecute, (host, user, cmd)) async_results.append(async_result) for async_result in async_results: return_val = async_result.get() if return_val == None: continue elif isinstance(return_val, ex.SubprocessException): raise return_val elif isinstance(return_val, Exception): raise return_val else: raise Exception( 'Unrecognized return value from ssh: {0}'.format( str(return_val)))
def _file_per_document(exportfile): if not os.path.exists(exportfile): print "%s doesn't exist!" % exportfile return dirs, _ = os.path.split(exportfile) docspath = os.path.join(dirs, 'documents') ensure_dirpath(docspath) expfile = open(exportfile, 'r') def wat(ammapobject): x = True while x: ablob = ammapobject.readline() if ablob: yield ablob else: x = False tpool = pool.ThreadPool(pool.cpu_count() * 64) gettingweird = wat(mmap.mmap(expfile.fileno(), 0, prot=mmap.PROT_READ)) job = tpool.imap_unordered( _fpd, itertools.izip_longest(gettingweird, (), fillvalue=docspath)) while True: try: job.next() except Exception: return
def _run_command(protocol_client, shell_id, command, command_type=util.POWERSHELL, upper_timeout=CONFIG.argus.upper_timeout): command_id = None bare_command = command thread_pool = pool.ThreadPool(processes=THREADS) command = util.get_command(command, command_type) try: command_id = protocol_client.run_command(shell_id, command) result = thread_pool.apply_async( protocol_client.get_command_output, args=(shell_id, command_id)) stdout, stderr, exit_code = result.get( timeout=upper_timeout) if exit_code: output = "\n\n".join([out for out in (stdout, stderr) if out]) raise exceptions.ArgusError( "Executing command {command!r} with encoded Command" "{encoded_command!r} failed with exit code {exit_code!r}" " and output {output!r}." .format(command=bare_command, encoded_command=command, exit_code=exit_code, output=output)) return util.sanitize_command_output(stdout), stderr, exit_code except multiprocessing.TimeoutError: raise exceptions.ArgusTimeoutError( "The command '{cmd}' has timed out.".format(cmd=bare_command)) finally: thread_pool.terminate() protocol_client.cleanup_command(shell_id, command_id)
def test_mt(): pool = mppool.ThreadPool(4) start_time = time.time() lengths = pool.map(worker, range(4)) finish_time = time.time() print('Multithreading: total_length={}, time={:.2f}s.'.format( sum(lengths), finish_time - start_time))
def test_out_of_order_execution2(self): with self.test_session() as session: batcher = dynamic_batching._Batcher(minimum_batch_size=1, maximum_batch_size=1, timeout_ms=None) tp = pool.ThreadPool(10) r0 = tp.apply_async(session.run, batcher.compute([[1]], [tf.int32])) (input0, ), computation_id0 = session.run( batcher.get_inputs([tf.int32])) r1 = tp.apply_async(session.run, batcher.compute([[2]], [tf.int32])) (input1, ), computation_id1 = session.run( batcher.get_inputs([tf.int32])) self.assertAllEqual([1], input0) self.assertAllEqual([2], input1) # These two runs are switched from testOutOfOrderExecution1. session.run(batcher.set_outputs([input1 + 42], computation_id1)) session.run(batcher.set_outputs([input0 + 42], computation_id0)) self.assertAllEqual([43], r0.get()) self.assertAllEqual([44], r1.get())
def main(): # Init database conn = sqlite3.connect('data.db') c = conn.cursor() c.execute('CREATE TABLE IF NOT EXISTS game_urls (url text, data text)') c.execute('CREATE UNIQUE INDEX IF NOT EXISTS unique_url ON game_urls (url)') conn.commit() p = pool.ThreadPool(8) urls = sorted(url for (url,) in conn.execute('SELECT url FROM game_urls WHERE data = ""')) print('{} games to fetch'.format(len(urls))) writer = GameWriter(conn) try: for url, game_data in p.imap_unordered(RetrieveGameData, urls): if not game_data: print('Failed to update {}'.format(url)) continue writer.Insert(url, game_data) finally: writer.Flush()
def farm_process(tasks): servs_pool = pool.ThreadPool(processes=config['serv_thrds']) try: results = servs_pool.map_async(server_thread, tasks).get() except Exception: logger.exception('Exception') finally: servs_pool.close() servs_pool.join() try: ra, fa, rs, fs = post_processing(results) global rrd_queue for k, v in ra.iteritems(): rrd_queue.put({'ra': {k: v}}) for k, v in fa.iteritems(): rrd_queue.put({'fa': {k: v}}) for k, v in rs.iteritems(): rrd_queue.put({'rs': {k: v}}) for k, v in fs.iteritems(): rrd_queue.put({'fs': {k: v}}) except Exception: logger.exception('Exception')
def test_input_shapes_should_be_equal(self): with self.test_session() as session: @dynamic_batching.batch_fn def f(a, b): return a + b output0 = f(tf.constant([1]), tf.constant([2])) output1 = f(tf.constant([[2]]), tf.constant([3])) tp = pool.ThreadPool(2) f0 = tp.apply_async(session.run, [output0]) f1 = tp.apply_async(session.run, [output1]) time.sleep(_SLEEP_TIME) coord = tf.train.Coordinator() tf.train.start_queue_runners(coord=coord) with self.assertRaises(tf.errors.CancelledError): f0.get() f1.get() with self.assertRaisesRegexp(tf.errors.InvalidArgumentError, 'Shapes of inputs much be equal'): coord.join()
def _resolve_once_providers(self): if self._providers_resolved: return votes = VoteDict() for pvd in self.providers: votes[pvd] = VoteCapabilityDict.fromkeys(self.capabilities, 0) def vote(pvd): try: pvd.vote(votes) except: LOG.debug('{0}.vote raised: {1}'.format( pvd.__class__.__name__, sys.exc_info()[1])) pool = process_pool.ThreadPool(processes=len(self.providers)) try: pool.map(vote, self.providers) finally: pool.close() for cap in self.capabilities: cap_votes = ((pvd, votes[pvd][cap]) for pvd in votes) cap_votes = sorted(cap_votes, key=operator.itemgetter(1)) pvd, vote = cap_votes[-1] if not vote: pvd = self._nodata_pvd LOG.debug("provider for '{0}': {1}".format(cap, pvd)) self.provider_for_capability[cap] = pvd self._providers_resolved = True
def test_two(self): with self.test_session() as session: @dynamic_batching.batch_fn def f(a, b): batch_size = tf.shape(a)[0] return a + b, tf.tile([batch_size], [batch_size]) output0 = f(tf.constant([1]), tf.constant([2])) output1 = f(tf.constant([2]), tf.constant([3])) tp = pool.ThreadPool(2) f0 = tp.apply_async(session.run, [output0]) f1 = tp.apply_async(session.run, [output1]) # Make sure both inputs are in the batcher before starting it. time.sleep(_SLEEP_TIME) tf.train.start_queue_runners() result0, batch_size0 = f0.get() result1, batch_size1 = f1.get() self.assertAllEqual([3], result0) self.assertAllEqual([2], batch_size0) self.assertAllEqual([5], result1) self.assertAllEqual([2], batch_size1)
def main(): """ Main script """ try: username, password, url, collection_name, cluster_id, ranker_id, relevance_path,\ output_path, num_threads, fl, use_debug = parse_args() if use_debug: logger.info('Setting logger to level=DEBUG') logger.setLevel(logging.DEBUG) relevance_dict, (succ_rows, fail_rows) = read_relevance_file(relevance_path) logger.info('Total Number of Queries in Relevance File = %d' % (succ_rows + fail_rows)) logger.info('Total Number of Queries being sent to re-rank API = %d' % succ_rows) if ranker_id: thread_obj = RetrieveAndRankQueryThread(username, password, url, \ collection_name, cluster_id, ranker_id, fl=fl) else: thread_obj = SolrThread(username, password, url, collection_name, \ cluster_id, fl=fl) thread_pool = multi_pool.ThreadPool(processes=num_threads) question_results = thread_pool.map_async(func=thread_obj, iterable=[{'query': q} for (q) in relevance_dict.iteritems()]).get() print ('Responses retrieved from Retrieve and Rank') experiment_entries = create_experiment_object(question_results, relevance_dict) experiment_metadata = {'ranker_id': ranker_id, 'solr_collection': collection_name, 'solr_cluster_id': cluster_id, 'username': username, 'password': password, 'url': url, 'time':str(datetime.datetime.now())} experiment_obj = dict(experiment_entries=experiment_entries, experiment_metadata=experiment_metadata) print ('Writing results to output_path=%r' % output_path) with open(output_path, 'wt') as outfile: json.dump(experiment_obj, outfile) print ('Exiting with status code 0') sys.exit(0) except Exception as e: logging.warning('Exception %r in main thread' % e) print ('Exiting with status code 1') sys.exit(1)
def retry_build_loop(build_func: Callable, inputs: List[Tuple], num_concurrent_builds: int) -> List: """Calls |build_func| in parallel on |inputs|. Repeat on failures up to |NUM_BUILD_RETRIES| times. Returns the list of inputs that |build_func| was called successfully on.""" successes = [] logs.info('Concurrent builds: %d.', num_concurrent_builds) with mp_pool.ThreadPool(num_concurrent_builds) as pool: for _ in range(NUM_BUILD_RETRIES): logs.info('Building using (%s): %s', build_func, inputs) results = pool.starmap(build_func, inputs) curr_successes, curr_failures = split_successes_and_failures( inputs, results) logs.info('Build successes: %s', curr_successes) successes.extend(curr_successes) if not curr_failures: break logs.error('Build failures: %s', curr_failures) inputs = curr_failures sleep_interval = random.uniform(1, BUILD_FAIL_WAIT) logs.info('Sleeping for %d secs before retrying.', sleep_interval) time.sleep(sleep_interval) return successes
def solve(din): prog = list(din) tp, *mqs = pool.ThreadPool(processes=2), Queue(), Queue() tp.apply_async(run, (0, prog, mqs)) p1 = tp.apply_async(run, (1, prog, mqs)) return run(0, prog, None), p1.get()
def MigrateClients(self, client_urns): """Migrates entire VFS of given client list to the relational data store.""" self._start_time = rdfvalue.RDFDatetime.Now() self._client_urns_to_migrate = client_urns self._client_urns_migrated = [] self._client_urns_failed = [] to_migrate_count = len(self._client_urns_to_migrate) sys.stdout.write("Clients to migrate: {}\n".format(to_migrate_count)) batches = collection.Batch(client_urns, self.client_batch_size) tp = pool.ThreadPool(processes=self.thread_count) tp.map(self.MigrateClientBatch, list(batches)) migrated_count = len(self._client_urns_migrated) sys.stdout.write("Migrated clients: {}\n".format(migrated_count)) if to_migrate_count == migrated_count: sys.stdout.write("All clients migrated successfully!\n") else: message = "Not all clients have been migrated ({}/{})".format( migrated_count, to_migrate_count) raise RuntimeError(message)
def _invoke_prefetch(self): assert self._next is None if not self._repeat and self.epoch > 0: return if self._pool is None: self._pool = pool.ThreadPool(self.n_threads) n = len(self.dataset) i = self.current_position order = self._order args = [] dataset = self.dataset epoch = self.epoch is_new_epoch = False for _ in six.moves.range(self.batch_size): index = i if order is None else order[i] args.append((dataset, index)) i += 1 if i >= n: epoch += 1 is_new_epoch = True i = 0 if not self._repeat: break if order is not None: # We cannot shuffle the order directly here, since the # iterator may be serialized before the prefetched data are # consumed by the user, in which case an inconsistency # appears. order = order.copy() numpy.random.shuffle(order) self._next = self._pool.map_async(MultithreadIterator._read, args) self._next_state = (i, epoch, is_new_epoch, order)
def main(): blog_urls = load_blog_urls() pool = mpool.ThreadPool(THREAD_COUNT) for blog_url in blog_urls: pool.apply_async(parse_blog, args=(blog_url, )) pool.close() pool.join()
def __init__(self, n_thread, work_dir=None, debug=False): super().__init__(n_thread, work_dir=work_dir, debug=debug) self._pool = pool.ThreadPool(n_thread) self._failed_queues = {} self._queues = {} self._n_running = 0 self.times = {}