def function_executor(config=None,
runtime=None,
runtime_memory=None,
workers=None,
compute_backend=None,
region=None,
storage_backend=None,
storage_backend_region=None,
rabbitmq_monitor=None,
remote_invoker=None,
log_level=None):
"""
Generic function executor
"""
return FunctionExecutor(config=config,
runtime=runtime,
runtime_memory=runtime_memory,
workers=workers,
compute_backend=compute_backend,
compute_backend_region=region,
storage_backend=storage_backend,
storage_backend_region=storage_backend_region,
rabbitmq_monitor=rabbitmq_monitor,
remote_invoker=remote_invoker,
log_level=log_level)
def knative_executor(config=None,
runtime=None,
runtime_memory=None,
workers=None,
region=None,
storage_backend=None,
storage_backend_region=None,
rabbitmq_monitor=None,
remote_invoker=None,
log_level=None):
"""
Function executor for Knative
"""
compute_backend = 'knative'
return FunctionExecutor(config=config,
runtime=runtime,
runtime_memory=runtime_memory,
workers=workers,
compute_backend=compute_backend,
compute_backend_region=region,
storage_backend=storage_backend,
storage_backend_region=storage_backend_region,
rabbitmq_monitor=rabbitmq_monitor,
remote_invoker=remote_invoker,
log_level=log_level)
def benchmark(workers, memory, loopcount, matn):
iterable = [(loopcount, matn) for i in range(workers)]
exc = FunctionExecutor(runtime_memory=memory)
start_time = time.time()
worker_futures = exc.map(compute_flops, iterable)
results = exc.get_result()
end_time = time.time()
worker_stats = [f.stats for f in worker_futures]
total_time = end_time - start_time
print("Total time:", round(total_time, 3))
est_flops = workers * 2 * loopcount * matn**3
print('Estimated GFLOPS:', round(est_flops / 1e9 / total_time, 4))
res = {
'start_time': start_time,
'total_time': total_time,
'est_flops': est_flops,
'worker_stats': worker_stats,
'results': results
}
return res
def __init__(self,
processes=None,
initializer=None,
initargs=(),
maxtasksperchild=None,
context=None):
self._ctx = context or get_context()
#self._setup_queues()
self._taskqueue = queue.Queue()
self._cache = {}
self._state = RUN
self._maxtasksperchild = maxtasksperchild
self._initializer = initializer
self._initargs = initargs
if processes is not None and processes < 1:
raise ValueError("Number of processes must be at least 1")
if processes is not None:
if self._initargs:
self._executor = FunctionExecutor(workers=processes,
**self._initargs)
else:
self._executor = FunctionExecutor(workers=processes)
self._processes = processes
else:
if self._initargs:
self._executor = FunctionExecutor(**self._initargs)
else:
self._executor = FunctionExecutor()
self._processes = self._executor.invoker.workers
if initializer is not None and not callable(initializer):
raise TypeError('initializer must be a callable')
self._pool = []
def local_executor(config=None,
workers=None,
storage_backend=None,
storage_backend_region=None,
rabbitmq_monitor=None,
log_level=None):
"""
Localhost function executor
"""
compute_backend = 'localhost'
if storage_backend is None:
storage_backend = 'localhost'
return FunctionExecutor(config=config,
workers=workers,
compute_backend=compute_backend,
storage_backend=storage_backend,
storage_backend_region=storage_backend_region,
rabbitmq_monitor=rabbitmq_monitor,
log_level=log_level)
def docker_executor(config=None,
runtime=None,
workers=None,
storage_backend=None,
storage_backend_region=None,
rabbitmq_monitor=None,
log_level=None):
"""
Localhost function executor
"""
compute_backend = 'docker'
if storage_backend is None:
storage_backend = 'localhost'
return FunctionExecutor(config=config,
runtime=runtime,
workers=workers,
compute_backend=compute_backend,
storage_backend=storage_backend,
storage_backend_region=storage_backend_region,
rabbitmq_monitor=rabbitmq_monitor,
remote_invoker=True,
log_level=log_level)
def __init__(self, process_obj):
util._flush_std_streams()
self.returncode = None
self._executor = FunctionExecutor()
self._launch(process_obj)