def _create_instance(self):
"""
Initialize and create instance.
"""
futures = []
with _ThreadPoolExecutor(max_workers=6) as executor:
# Run configuration in parallel
policy = executor.submit(self._init_policy)
role = executor.submit(self._init_role)
instance_profile = executor.submit(self._init_instance_profile)
for method in (self._init_key_pair, self._init_security_group,
self._init_block_device_mappings):
futures.append(executor.submit(method))
# Wait that role, instance_profile and policy are completed
# to attach them
for future in _as_completed((policy, instance_profile)):
role.result()
futures.append(
executor.submit(self._attach_role_policy if future.result(
) == 'policy' else self._attach_instance_profile_role))
# Wait completion
for future in _as_completed(futures):
future.result()
def wait(self):
"""
TODO
"""
self._pool.shutdown(wait=True)
self._pool = _ThreadPoolExecutor(max_workers=self._n_procs)
def identify_async(movie, minimum_ng, box, roi=None):
"Use the user settings to define the number of workers that are being used"
settings = _io.load_user_settings()
try:
cpu_utilization = settings['Localize']['cpu_utilization']
if cpu_utilization >= 1:
cpu_utilization = 1
except Exception as e:
print(e)
print('An Error occured. Setting cpu_utilization to 0.8'
) # TODO at some point re-write this
cpu_utilization = 0.8
settings['Localize']['cpu_utilization'] = cpu_utilization
_io.save_user_settings(settings)
n_workers = max(1, int(cpu_utilization * _multiprocessing.cpu_count()))
current = [0]
executor = _ThreadPoolExecutor(n_workers)
lock = _threading.Lock()
f = [
executor.submit(_identify_worker, movie, current, minimum_ng, box, roi,
lock) for _ in range(n_workers)
]
executor.shutdown(wait=False)
return current, f
def __call__(self,
request,
timeout=None,
metadata=None,
credentials=None,
*,
standalone_pool=None):
"""Invokes the underlying RPC.
Args:
request: The request value for the RPC.
timeout: An optional duration of time in seconds to allow for the RPC.
If None, the timeout is considered infinite.
metadata: An optional :term:`metadata` to be transmitted to the
service-side of the RPC.
credentials: An optional CallCredentials for the RPC.
Returns:
An object that is both a Call for the RPC and an ASYNC iterator of response
values. Drawing response values from the returned Call-iterator may
raise RpcError indicating termination of the RPC with non-OK status.
"""
if standalone_pool is None:
standalone_pool = self._standalone_pool
if standalone_pool:
stream_executor = _ThreadPoolExecutor(1)
else:
stream_executor = None
return _utils.WrappedIterator(
self._inner(request, timeout, metadata, credentials), self._loop,
self._executor, stream_executor)
def __init__(self,
executable: str = ZGOUBI_EXECUTABLE_NAME,
path: str = None,
n_procs: Optional[int] = None):
"""
`Zgoubi` is responsible for running the Zgoubi executable within Zgoubidoo. It will run Zgoubi as a subprocess
and offers a variety of concurency and parallelisation features.
The `Zgoubi` object is an interface to the Zgoubi executable. The executable can be found automatically or its
name and path can be specified.
The Zgoubi executable is called on an instance of `Input` specifying a list of paths containing Zgoubi input
files. Multiple instances can thus be run in parallel.
TODO details on concurrency
Args:
- executable: name of the Zgoubi executable
- path: path to the Zgoubi executable
- n_procs: maximum number of Zgoubi simulations to be started in parallel
"""
self._executable: str = executable
self._n_procs: int = n_procs or multiprocessing.cpu_count()
self._path: Optional[str] = path
self._futures: Dict[str, _Future] = dict()
self._pool: _ThreadPoolExecutor = _ThreadPoolExecutor(
max_workers=self._n_procs)
def identify_async(movie, minimum_ng, box, roi=None):
n_workers = max(1, int(0.75 * _multiprocessing.cpu_count()))
current = [0]
executor = _ThreadPoolExecutor(n_workers)
lock = _threading.Lock()
f = [executor.submit(_identify_worker, movie, current, minimum_ng, box, roi, lock) for _ in range(n_workers)]
executor.shutdown(wait=False)
return current, f
def _workers(self):
"""
Worker threads pool.
Returns:
concurrent.future.ThreadPoolExecutor
"""
return _ThreadPoolExecutor(max_workers=self._WORKERS_COUNT)
def distance_histogram(locs, info, bin_size, r_max):
locs, size, x_index, y_index, block_starts, block_ends, K, L = get_index_blocks(locs, info, r_max)
N = len(locs)
n_threads = _multiprocessing.cpu_count()
chunk = int(N / n_threads)
starts = range(0, N, chunk)
args = [(locs, bin_size, r_max, x_index, y_index, block_starts, block_ends, start, chunk) for start in starts]
with _ThreadPoolExecutor() as executor:
futures = [executor.submit(_distance_histogram, *_) for _ in args]
results = [future.result() for future in futures]
return _np.sum(results, axis=0)
def compute_local_density(locs, info, radius):
locs, x_index, y_index, block_starts, block_ends = get_index_blocks(locs, info, radius)
N = len(locs)
n_threads = _multiprocessing.cpu_count()
chunk = int(N / n_threads)
starts = range(0, N, chunk)
args = [(locs, radius, x_index, y_index, block_starts, block_ends, start, chunk) for start in starts]
with _ThreadPoolExecutor() as executor:
futures = [executor.submit(_local_density, *_) for _ in args]
density = _np.sum([future.result() for future in futures], axis=0)
locs = _lib.remove_from_rec(locs, 'density')
return _lib.append_to_rec(locs, density, 'density')
def distance_matrix(sqadj, n_threads=4):
"""Compared the distance matrix from the squared adjacency."""
n = len(sqadj)
d = _np.empty_like(sqadj)
assignments = _distribute_assignments(n, n_threads)
arglist = [(sqadj, d, a) for a in assignments]
with _ThreadPoolExecutor(n_threads) as pool:
res = list(pool.map(_distance_matrix_core, arglist))
_np.fill_diagonal(d, _np.inf)
return d
def _create_instance(self):
"""
Initializes and creates instance.
"""
# Run configuration in parallel
futures = []
with _ThreadPoolExecutor(
max_workers=len(self._INIT_METHODS)) as executor:
for method in self._INIT_METHODS:
futures.append(executor.submit(getattr(self, method)))
# Wait completion
for future in _as_completed(futures):
future.result()
def __init__(self):
# Avoid double __exit__ / __del__ call
self._activated = False
# Handle SIGTERM like SIGINT (systemd stop services using SIGTERM)
_signal.signal(_signal.SIGTERM, self._interrupt)
# Get Systemd notify socket
self._sd_notify_address = self._get_sd_notify_socket()
# Get FPGA slots configuration
self._fpga_slots = dict()
for env_key in _environ:
for env, key in (('ACCELIZE_DRM_DRIVER_', 'fpga_driver_name'),
('ACCELIZE_DRM_CRED_', 'cred_file_path'),
('ACCELIZE_DRM_CONF_', 'conf_file_path'),
('ACCELIZE_DRM_IMAGE_', 'fpga_image'),
('ACCELIZE_DRM_DISABLED_', 'drm_disabled')):
if env_key.startswith(env):
slot = int(env_key.rsplit('_', maxsplit=1)[1])
try:
slot_dict = self._fpga_slots[slot]
except KeyError:
# Add slot to configuration
self._fpga_slots[slot] = slot_dict = dict()
slot_dict[key] = _environ[env_key]
if not self._fpga_slots:
# If no configuration passed by environment, activate default slot
self._fpga_slots[self.DEFAULT_FPGA_SLOT_ID] = dict()
# Initialize DRM manager
self._drivers = []
self._drm_managers = []
self._lisenced_slots = []
futures = []
with _ThreadPoolExecutor() as executor:
for fpga_slot_id in self._fpga_slots:
futures.append(executor.submit(
self._init_drm_manager, int(fpga_slot_id)))
with self._handle_exception((RuntimeError, OSError, _DRMException)):
for future in _as_completed(futures):
future.result()
def __init__(self, num_threads=None):
"""
Initialize the executor.
:param num_threads: number of worker threads in the pool. If None the number of threads
will use multiprocessing.cpu_count(). If that's not implemented the number
of threads defaults to 1.
:type num_threads: int
"""
super().__init__()
if num_threads is None:
try:
num_threads = multiprocessing.cpu_count()
except NotImplementedError:
num_threads = 1
self._executor = _ThreadPoolExecutor(num_threads)
def __call__(self,
request_iterator,
timeout=None,
metadata=None,
credentials=None,
*, standalone_pool = None):
"""Invokes the underlying RPC on the client.
Args:
request_iterator: An iterator that yields request values for the RPC.
timeout: An optional duration of time in seconds to allow for the RPC.
if not specified the timeout is considered infinite.
metadata: Optional :term:`metadata` to be transmitted to the
service-side of the RPC.
credentials: An optional CallCredentials for the RPC.
Returns:
An object that is both a Call for the RPC and an iterator of response
values. Drawing response values from the returned Call-iterator may
raise RpcError indicating termination of the RPC with non-OK status.
"""
if standalone_pool is None:
standalone_pool = self._standalone_pool
if standalone_pool:
stream_executor = _ThreadPoolExecutor(1)
else:
stream_executor = None
input_iterator = _utils.WrappedAsyncIterator(request_iterator,
self._loop)
grpc_iterator = self._inner(
input_iterator,
timeout,
metadata,
credentials
)
r = _utils.WrappedIterator(
grpc_iterator,
self._loop,
self._executor,
stream_executor)
# Make sure the input iterator exits, or the thread may block indefinitely
def _end_callback():
input_iterator.cancel(False)
grpc_iterator.add_callback(_end_callback)
return r
def get_executor():
"""
Get the shared ThreadPoolExecutor.
Returns:
ThreadPoolExecutor: the shared thread pool executor.
Raises:
TypeError: if ``set_thread_pool_size()`` was not called first.
"""
global _THREADPOOL_EXECUTOR
if _THREADPOOL_EXECUTOR is None:
if _MAX_WORKERS is None:
raise TypeError("set_thread_pool_size() has to be called first.")
_THREADPOOL_EXECUTOR = _ThreadPoolExecutor(max_workers=_MAX_WORKERS)
return _THREADPOOL_EXECUTOR
def __exit__(self, exc_type, exc_val, exc_tb):
if self._activated:
self._activated = False
self._sd_notify(b"STOPPING=1")
# Deactivate DRM manager for all slots
futures = []
with _ThreadPoolExecutor() as executor:
for drm_manager in self._drm_managers:
futures.append(executor.submit(drm_manager.deactivate))
try:
with self._handle_exception(_DRMException):
for future in _as_completed(futures):
future.result()
finally:
self._drivers.clear()
self._drm_managers.clear()
def __enter__(self):
self._activated = True
# Activate DRM manager for all slots
futures = []
with _ThreadPoolExecutor() as executor:
for drm_manager in self._drm_managers:
futures.append(executor.submit(drm_manager.activate))
with self._handle_exception(_DRMException):
for future in _as_completed(futures):
future.result()
# Notify systemd
self._sd_notify(b"READY=1\nSTATUS=Licensing FPGA slot(s) %s" %
', '.join(self._lisenced_slots).encode())
return self
def _create_instance(self):
"""
Initialize and create instance.
"""
futures = []
with _ThreadPoolExecutor(max_workers=5) as executor:
# Run configuration in parallel
policy = executor.submit(self._init_policy)
role = executor.submit(self._init_role)
for method in (self._init_key_pair, self._init_security_group):
futures.append(executor.submit(method))
# Wait that role and policy are completed to attach them
for future in _as_completed((policy, role)):
future.result()
futures.append(executor.submit(self._attach_role_policy))
# Wait completion
for future in _as_completed(futures):
future.result()
def _auto_mount():
"""mounts from configuration"""
# Get configuration
config = _cfg.Configuration()
# Finds possibles storage
to_mount = set()
name = config['host']['host_type']
if name:
to_mount.add(name)
for section in config:
if section.startswith('host.') or section.startswith('storage.'):
name = section.split('.', 1)[1]
if name:
to_mount.add(name)
if to_mount:
# Tries to mount storage
if _py[0] == 2:
# On Python 2: Seem to have a deadlock on import if use of
# ThreadPoolExecutor
for storage_type in to_mount:
try:
mount(storage_type=storage_type, config=config)
except (ImportError, _exc.AcceleratorException):
continue
return
futures = []
with _ThreadPoolExecutor(max_workers=len(to_mount)) as executor:
for storage_type in to_mount:
try:
storage = _Storage(storage_type=storage_type,
config=config)
except (ImportError, _exc.AcceleratorException):
continue
futures.append(executor.submit(storage.mount))
# Waits completion
for future in _as_completed(futures):
future.result()
def __init__(self,
executable: str,
results_type: ResultsType,
path: str = None,
n_procs: Optional[int] = None):
"""
Args:
- executable: name of the executable
- results_type:
- path: path to the Zgoubi executable
- n_procs: maximum number of Zgoubi simulations to be started in parallel
"""
self._executable: str = executable
self._results_type: ResultsType = results_type
self._n_procs: int = n_procs or multiprocessing.cpu_count()
self._path: Optional[str] = path
self._futures: Dict[str, _Future] = dict()
self._pool: _ThreadPoolExecutor = _ThreadPoolExecutor(
max_workers=self._n_procs)
