def _create_instance(self): """ Initialize and create instance. """ futures = [] with _ThreadPoolExecutor(max_workers=6) as executor: # Run configuration in parallel policy = executor.submit(self._init_policy) role = executor.submit(self._init_role) instance_profile = executor.submit(self._init_instance_profile) for method in (self._init_key_pair, self._init_security_group, self._init_block_device_mappings): futures.append(executor.submit(method)) # Wait that role, instance_profile and policy are completed # to attach them for future in _as_completed((policy, instance_profile)): role.result() futures.append( executor.submit(self._attach_role_policy if future.result( ) == 'policy' else self._attach_instance_profile_role)) # Wait completion for future in _as_completed(futures): future.result()
def wait(self): """ TODO """ self._pool.shutdown(wait=True) self._pool = _ThreadPoolExecutor(max_workers=self._n_procs)
def identify_async(movie, minimum_ng, box, roi=None): "Use the user settings to define the number of workers that are being used" settings = _io.load_user_settings() try: cpu_utilization = settings['Localize']['cpu_utilization'] if cpu_utilization >= 1: cpu_utilization = 1 except Exception as e: print(e) print('An Error occured. Setting cpu_utilization to 0.8' ) # TODO at some point re-write this cpu_utilization = 0.8 settings['Localize']['cpu_utilization'] = cpu_utilization _io.save_user_settings(settings) n_workers = max(1, int(cpu_utilization * _multiprocessing.cpu_count())) current = [0] executor = _ThreadPoolExecutor(n_workers) lock = _threading.Lock() f = [ executor.submit(_identify_worker, movie, current, minimum_ng, box, roi, lock) for _ in range(n_workers) ] executor.shutdown(wait=False) return current, f
def __call__(self, request, timeout=None, metadata=None, credentials=None, *, standalone_pool=None): """Invokes the underlying RPC. Args: request: The request value for the RPC. timeout: An optional duration of time in seconds to allow for the RPC. If None, the timeout is considered infinite. metadata: An optional :term:`metadata` to be transmitted to the service-side of the RPC. credentials: An optional CallCredentials for the RPC. Returns: An object that is both a Call for the RPC and an ASYNC iterator of response values. Drawing response values from the returned Call-iterator may raise RpcError indicating termination of the RPC with non-OK status. """ if standalone_pool is None: standalone_pool = self._standalone_pool if standalone_pool: stream_executor = _ThreadPoolExecutor(1) else: stream_executor = None return _utils.WrappedIterator( self._inner(request, timeout, metadata, credentials), self._loop, self._executor, stream_executor)
def __init__(self, executable: str = ZGOUBI_EXECUTABLE_NAME, path: str = None, n_procs: Optional[int] = None): """ `Zgoubi` is responsible for running the Zgoubi executable within Zgoubidoo. It will run Zgoubi as a subprocess and offers a variety of concurency and parallelisation features. The `Zgoubi` object is an interface to the Zgoubi executable. The executable can be found automatically or its name and path can be specified. The Zgoubi executable is called on an instance of `Input` specifying a list of paths containing Zgoubi input files. Multiple instances can thus be run in parallel. TODO details on concurrency Args: - executable: name of the Zgoubi executable - path: path to the Zgoubi executable - n_procs: maximum number of Zgoubi simulations to be started in parallel """ self._executable: str = executable self._n_procs: int = n_procs or multiprocessing.cpu_count() self._path: Optional[str] = path self._futures: Dict[str, _Future] = dict() self._pool: _ThreadPoolExecutor = _ThreadPoolExecutor( max_workers=self._n_procs)
def identify_async(movie, minimum_ng, box, roi=None): n_workers = max(1, int(0.75 * _multiprocessing.cpu_count())) current = [0] executor = _ThreadPoolExecutor(n_workers) lock = _threading.Lock() f = [executor.submit(_identify_worker, movie, current, minimum_ng, box, roi, lock) for _ in range(n_workers)] executor.shutdown(wait=False) return current, f
def _workers(self): """ Worker threads pool. Returns: concurrent.future.ThreadPoolExecutor """ return _ThreadPoolExecutor(max_workers=self._WORKERS_COUNT)
def distance_histogram(locs, info, bin_size, r_max): locs, size, x_index, y_index, block_starts, block_ends, K, L = get_index_blocks(locs, info, r_max) N = len(locs) n_threads = _multiprocessing.cpu_count() chunk = int(N / n_threads) starts = range(0, N, chunk) args = [(locs, bin_size, r_max, x_index, y_index, block_starts, block_ends, start, chunk) for start in starts] with _ThreadPoolExecutor() as executor: futures = [executor.submit(_distance_histogram, *_) for _ in args] results = [future.result() for future in futures] return _np.sum(results, axis=0)
def compute_local_density(locs, info, radius): locs, x_index, y_index, block_starts, block_ends = get_index_blocks(locs, info, radius) N = len(locs) n_threads = _multiprocessing.cpu_count() chunk = int(N / n_threads) starts = range(0, N, chunk) args = [(locs, radius, x_index, y_index, block_starts, block_ends, start, chunk) for start in starts] with _ThreadPoolExecutor() as executor: futures = [executor.submit(_local_density, *_) for _ in args] density = _np.sum([future.result() for future in futures], axis=0) locs = _lib.remove_from_rec(locs, 'density') return _lib.append_to_rec(locs, density, 'density')
def distance_matrix(sqadj, n_threads=4): """Compared the distance matrix from the squared adjacency.""" n = len(sqadj) d = _np.empty_like(sqadj) assignments = _distribute_assignments(n, n_threads) arglist = [(sqadj, d, a) for a in assignments] with _ThreadPoolExecutor(n_threads) as pool: res = list(pool.map(_distance_matrix_core, arglist)) _np.fill_diagonal(d, _np.inf) return d
def _create_instance(self): """ Initializes and creates instance. """ # Run configuration in parallel futures = [] with _ThreadPoolExecutor( max_workers=len(self._INIT_METHODS)) as executor: for method in self._INIT_METHODS: futures.append(executor.submit(getattr(self, method))) # Wait completion for future in _as_completed(futures): future.result()
def __init__(self): # Avoid double __exit__ / __del__ call self._activated = False # Handle SIGTERM like SIGINT (systemd stop services using SIGTERM) _signal.signal(_signal.SIGTERM, self._interrupt) # Get Systemd notify socket self._sd_notify_address = self._get_sd_notify_socket() # Get FPGA slots configuration self._fpga_slots = dict() for env_key in _environ: for env, key in (('ACCELIZE_DRM_DRIVER_', 'fpga_driver_name'), ('ACCELIZE_DRM_CRED_', 'cred_file_path'), ('ACCELIZE_DRM_CONF_', 'conf_file_path'), ('ACCELIZE_DRM_IMAGE_', 'fpga_image'), ('ACCELIZE_DRM_DISABLED_', 'drm_disabled')): if env_key.startswith(env): slot = int(env_key.rsplit('_', maxsplit=1)[1]) try: slot_dict = self._fpga_slots[slot] except KeyError: # Add slot to configuration self._fpga_slots[slot] = slot_dict = dict() slot_dict[key] = _environ[env_key] if not self._fpga_slots: # If no configuration passed by environment, activate default slot self._fpga_slots[self.DEFAULT_FPGA_SLOT_ID] = dict() # Initialize DRM manager self._drivers = [] self._drm_managers = [] self._lisenced_slots = [] futures = [] with _ThreadPoolExecutor() as executor: for fpga_slot_id in self._fpga_slots: futures.append(executor.submit( self._init_drm_manager, int(fpga_slot_id))) with self._handle_exception((RuntimeError, OSError, _DRMException)): for future in _as_completed(futures): future.result()
def __init__(self, num_threads=None): """ Initialize the executor. :param num_threads: number of worker threads in the pool. If None the number of threads will use multiprocessing.cpu_count(). If that's not implemented the number of threads defaults to 1. :type num_threads: int """ super().__init__() if num_threads is None: try: num_threads = multiprocessing.cpu_count() except NotImplementedError: num_threads = 1 self._executor = _ThreadPoolExecutor(num_threads)
def __call__(self, request_iterator, timeout=None, metadata=None, credentials=None, *, standalone_pool = None): """Invokes the underlying RPC on the client. Args: request_iterator: An iterator that yields request values for the RPC. timeout: An optional duration of time in seconds to allow for the RPC. if not specified the timeout is considered infinite. metadata: Optional :term:`metadata` to be transmitted to the service-side of the RPC. credentials: An optional CallCredentials for the RPC. Returns: An object that is both a Call for the RPC and an iterator of response values. Drawing response values from the returned Call-iterator may raise RpcError indicating termination of the RPC with non-OK status. """ if standalone_pool is None: standalone_pool = self._standalone_pool if standalone_pool: stream_executor = _ThreadPoolExecutor(1) else: stream_executor = None input_iterator = _utils.WrappedAsyncIterator(request_iterator, self._loop) grpc_iterator = self._inner( input_iterator, timeout, metadata, credentials ) r = _utils.WrappedIterator( grpc_iterator, self._loop, self._executor, stream_executor) # Make sure the input iterator exits, or the thread may block indefinitely def _end_callback(): input_iterator.cancel(False) grpc_iterator.add_callback(_end_callback) return r
def get_executor(): """ Get the shared ThreadPoolExecutor. Returns: ThreadPoolExecutor: the shared thread pool executor. Raises: TypeError: if ``set_thread_pool_size()`` was not called first. """ global _THREADPOOL_EXECUTOR if _THREADPOOL_EXECUTOR is None: if _MAX_WORKERS is None: raise TypeError("set_thread_pool_size() has to be called first.") _THREADPOOL_EXECUTOR = _ThreadPoolExecutor(max_workers=_MAX_WORKERS) return _THREADPOOL_EXECUTOR
def __exit__(self, exc_type, exc_val, exc_tb): if self._activated: self._activated = False self._sd_notify(b"STOPPING=1") # Deactivate DRM manager for all slots futures = [] with _ThreadPoolExecutor() as executor: for drm_manager in self._drm_managers: futures.append(executor.submit(drm_manager.deactivate)) try: with self._handle_exception(_DRMException): for future in _as_completed(futures): future.result() finally: self._drivers.clear() self._drm_managers.clear()
def __enter__(self): self._activated = True # Activate DRM manager for all slots futures = [] with _ThreadPoolExecutor() as executor: for drm_manager in self._drm_managers: futures.append(executor.submit(drm_manager.activate)) with self._handle_exception(_DRMException): for future in _as_completed(futures): future.result() # Notify systemd self._sd_notify(b"READY=1\nSTATUS=Licensing FPGA slot(s) %s" % ', '.join(self._lisenced_slots).encode()) return self
def _create_instance(self): """ Initialize and create instance. """ futures = [] with _ThreadPoolExecutor(max_workers=5) as executor: # Run configuration in parallel policy = executor.submit(self._init_policy) role = executor.submit(self._init_role) for method in (self._init_key_pair, self._init_security_group): futures.append(executor.submit(method)) # Wait that role and policy are completed to attach them for future in _as_completed((policy, role)): future.result() futures.append(executor.submit(self._attach_role_policy)) # Wait completion for future in _as_completed(futures): future.result()
def _auto_mount(): """mounts from configuration""" # Get configuration config = _cfg.Configuration() # Finds possibles storage to_mount = set() name = config['host']['host_type'] if name: to_mount.add(name) for section in config: if section.startswith('host.') or section.startswith('storage.'): name = section.split('.', 1)[1] if name: to_mount.add(name) if to_mount: # Tries to mount storage if _py[0] == 2: # On Python 2: Seem to have a deadlock on import if use of # ThreadPoolExecutor for storage_type in to_mount: try: mount(storage_type=storage_type, config=config) except (ImportError, _exc.AcceleratorException): continue return futures = [] with _ThreadPoolExecutor(max_workers=len(to_mount)) as executor: for storage_type in to_mount: try: storage = _Storage(storage_type=storage_type, config=config) except (ImportError, _exc.AcceleratorException): continue futures.append(executor.submit(storage.mount)) # Waits completion for future in _as_completed(futures): future.result()
def __init__(self, executable: str, results_type: ResultsType, path: str = None, n_procs: Optional[int] = None): """ Args: - executable: name of the executable - results_type: - path: path to the Zgoubi executable - n_procs: maximum number of Zgoubi simulations to be started in parallel """ self._executable: str = executable self._results_type: ResultsType = results_type self._n_procs: int = n_procs or multiprocessing.cpu_count() self._path: Optional[str] = path self._futures: Dict[str, _Future] = dict() self._pool: _ThreadPoolExecutor = _ThreadPoolExecutor( max_workers=self._n_procs)