def parallel_runs(data_list):
pool = multiprocessing.Pool(4)
prod_x = partial(read_process_store,
dask_df=dask_df,
symbol_unique_dates=symbol_unique_dates)
try:
results = pool.map(prod_x, data_list)
print(results)
except KeyError:
pass
def create_dask_client(args):
"""Create and install a Dask distributed client using args from a
Namespace, supporting the following attributes:
- .scheduler: Address of the distributed scheduler, or the
empty string to start one locally
"""
import dask
scheduler = getattr(args, 'scheduler', None)
num_workers = getattr(args, 'num_workers', 0)
num_threads_per_worker = getattr(args, 'num_threads_per_worker', 0)
if scheduler == 'multithreading':
import dask.threaded
from multiprocessing.pool import ThreadPool
if num_threads_per_worker <= 0:
num_workers = max(
1,
psutil.cpu_count(logical=False) + num_threads_per_worker)
else:
num_workers = num_threads_per_worker
print('Starting dask thread pool with %d thread(s)' % num_workers)
dask.config.set(pool=ThreadPool(num_workers))
dask.config.set(scheduler='threads')
return
if scheduler == 'multiprocessing':
import dask.multiprocessing
import multiprocessing
dask.config.set(scheduler='processes')
if num_workers <= 0:
num_workers = max(1, psutil.cpu_count(logical=False) + num_workers)
print('Starting dask multiprocessing pool with %d worker(s)' %
num_workers)
dask.config.set(pool=multiprocessing.Pool(
num_workers,
initializer=dask.multiprocessing.initialize_worker_process))
return
import dask.distributed
if not scheduler:
if num_workers <= 0:
num_workers = max(1, psutil.cpu_count(logical=False) + num_workers)
num_threads_per_worker = (num_threads_per_worker
if num_threads_per_worker >= 1 else None)
print('Creating dask LocalCluster with %d worker(s), %r thread(s) per '
'worker' % (num_workers, num_threads_per_worker))
scheduler = dask.distributed.LocalCluster(
ip='0.0.0.0', # Allow reaching the diagnostics port externally
scheduler_port=0, # Don't expose the scheduler port
n_workers=num_workers,
memory_limit=0,
threads_per_worker=num_threads_per_worker,
silence_logs=False)
return dask.distributed.Client(scheduler)