def ipyparallel_mapper(X,y,w_list,*args):
num_cores = 6
if w_list.shape[1]<16*num_cores:
return map_losses(X,y,w_list,np.arange(0,w_list.shape[1]))
weights_index = np.arange(0,w_list.shape[1])
weights_index_split = np.array_split(weights_index,num_cores)
mydict=dict(w_list=w_list)
dview.push(mydict)
rX = [ipp.Reference('X') for i in range(num_cores)]
ry = [ipp.Reference('y') for i in range(num_cores)]
rweights = [ipp.Reference('w_list') for i in range(num_cores)]
result = dview.map_sync(map_losses,rX,ry,rweights,weights_index_split)
loss = []
indx = []
for l in result:
loss.extend(l)
loss = np.hstack(loss)
return loss
def update_dtc_pop(pop, td):
'''
inputs a population of genes/alleles, the population size MU, and an optional argument of a rheobase value guess
outputs a population of genes/alleles, a population of individual object shells, ie a pickleable container for gene attributes.
Rationale, not every gene value will result in a model for which rheobase is found, in which case that gene is discarded, however to
compensate for losses in gene population size, more gene samples must be tested for a successful return from a rheobase search.
If the tests return are successful these new sampled individuals are appended to the population, and then their attributes are mapped onto
corresponding virtual model objects.
'''
import copy
import numpy as np
from deap import base
toolbox = base.Toolbox()
Individual = ipp.Reference('Individual')
pop = [toolbox.clone(i) for i in pop]
def transform(ind):
from neuronunit.optimization.data_transport_container import DataTC
dtc = DataTC()
print(dtc)
param_dict = {}
for i, j in enumerate(ind):
param_dict[td[i]] = str(j)
dtc.attrs = param_dict
dtc.evaluated = False
return dtc
if len(pop) > 0:
dtcpop = list(dview.map_sync(transform, pop))
else:
# In this case pop is not really a population but an individual
# but parsimony of naming variables
# suggests not to change the variable name to reflect this.
dtcpop = list(transform(pop))
return dtcpop
def test_can_list_kwarg(self):
"""kwargs in lists are canned"""
view = self.client[-1]
view['a'] = 128
rA = pmod.Reference('a')
ar = view.apply_async(lambda x=5: x, x=[rA])
r = ar.get(5)
self.assertEqual(r, [128])
def test_can_dict_kwarg(self):
"""kwargs in dicts are canned"""
view = self.client[-1]
view['a'] = 128
rA = pmod.Reference('a')
ar = view.apply_async(lambda x=5: x, dict(foo=rA))
r = ar.get(5)
self.assertEqual(r, dict(foo=128))
def test_run_newline(self):
"""test that run appends newline to files"""
with NamedTemporaryFile('w', delete=False) as f:
f.write("""def g():
return 5
""")
v = self.client[-1]
v.run(f.name, block=True)
self.assertEqual(v.apply_sync(lambda f: f(), pmod.Reference('g')), 5)
def test_push_numpy_nocopy(self):
import numpy
view = self.client[:]
a = numpy.arange(64)
view['A'] = a
@interactive
def check_writeable(x):
return x.flags.writeable
for flag in view.apply_sync(check_writeable, pmod.Reference('A')):
self.assertFalse(
flag, "array is writeable, push shouldn't have pickled it")
view.push(dict(B=a))
for flag in view.apply_sync(check_writeable, pmod.Reference('B')):
self.assertFalse(
flag, "array is writeable, push shouldn't have pickled it")
def test_apply_reference(self):
"""view.apply(<Reference>, *args) should work"""
v = self.client[:]
v.scatter('n', self.client.ids, flatten=True)
v.execute("f = lambda x: n*x")
rf = pmod.Reference('f')
result = v.apply_sync(rf, 5)
expected = [5 * id for id in self.client.ids]
self.assertEqual(result, expected)
def test_map_ref(self):
"""view.map works with references"""
view = self.client[:]
ranks = sorted(self.client.ids)
view.scatter('rank', ranks, flatten=True)
rrank = pmod.Reference('rank')
amr = view.map_async(lambda x: x * 2, [rrank] * len(view))
drank = amr.get(5)
self.assertEqual(drank, [r * 2 for r in ranks])
def test_map_reference(self):
"""view.map(<Reference>, *seqs) should work"""
v = self.client[:]
v.scatter('n', self.client.ids, flatten=True)
v.execute("f = lambda x,y: x*y")
rf = pmod.Reference('f')
nlist = list(range(10))
mlist = nlist[::-1]
expected = [m * n for m, n in zip(mlist, nlist)]
result = v.map_sync(rf, mlist, nlist)
self.assertEqual(result, expected)
def test_z_crash_mux(self):
"""test graceful handling of engine death (direct)"""
self.add_engines(1)
self.minimum_engines(2)
eid = self.client.ids[-1]
view = self.client[-2:]
view.scatter('should_crash', [False, True], flatten=True)
ar = view.apply_async(conditional_crash, ipp.Reference("should_crash"))
self.assertRaisesRemote(error.EngineError, ar.get, 10)
tic = time.perf_counter()
while eid in self.client.ids and time.perf_counter() - tic < 5:
time.sleep(0.05)
assert eid not in self.client.ids
def __next__(self):
it_name = '_%s_iter' % self.name
self.view.execute('%s = iter(%s)' % (it_name, self.name), block=True)
next_ref = ipyparallel.Reference(it_name + '.next')
while True:
try:
yield self.view.apply_sync(next_ref)
# This causes the StopIteration exception to be raised.
except ipyparallel.RemoteError as e:
if e.ename == 'StopIteration':
raise StopIteration
else:
raise e
def remote_iterator(view, name):
"""Return an iterator on an object living on a remote engine."""
view.execute(f'it{name}=iter({name})', block=True)
while True:
try:
result = view.apply_sync(next, ipp.Reference('it' + name))
# This causes the StopIteration exception to be raised.
except RemoteError as e:
if e.ename == 'StopIteration':
raise StopIteration
else:
raise e
else:
yield result
def remote_iterator(view, name):
"""Return an iterator on an object living on a remote engine."""
it_name = '_%s_iter' % name
view.execute('%s = iter(%s)' % (it_name,name), block=True)
ref = parallel.Reference(it_name)
while True:
try:
yield view.apply_sync(next, ref)
# This causes the StopIteration exception to be raised.
except parallel.RemoteError as e:
if e.ename == 'StopIteration':
raise StopIteration
else:
raise e
def import_list(ipp, subset, NDIM):
Individual = ipp.Reference('Individual')
from deap import base, creator, tools
import deap
import random
history = deap.tools.History()
toolbox = base.Toolbox()
import numpy as np
##
# Range of the genes:
##
BOUND_LOW = [np.min(i) for i in subset.values()]
BOUND_UP = [np.max(i) for i in subset.values()]
##
# number of objectives/error functions
##
#NDIM = numb_err_f
def uniform(low, up, size=None):
try:
return [random.uniform(a, b) for a, b in zip(low, up)]
except TypeError:
return [
random.uniform(a, b) for a, b in zip([low] * size, [up] * size)
]
# weights vector should compliment a numpy matrix of eigenvalues and other values
#creator.create("FitnessMin", base.Fitness, weights=(-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0))
weights = tuple([-1.0 for i in range(0, NDIM)])
creator.create("FitnessMin", base.Fitness, weights=weights)
creator.create("Individual", list, fitness=creator.FitnessMin)
toolbox.register("attr_float", uniform, BOUND_LOW, BOUND_UP, len(BOUND_UP))
toolbox.register("individual", tools.initIterate, creator.Individual,
toolbox.attr_float)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("select", tools.selNSGA2)
toolbox.register("mate",
tools.cxSimulatedBinaryBounded,
low=BOUND_LOW,
up=BOUND_UP,
eta=30.0)
toolbox.register("mutate",
tools.mutPolynomialBounded,
low=BOUND_LOW,
up=BOUND_UP,
eta=20.0,
indpb=1.0 / NDIM)
return toolbox, tools, history, creator, base
def pwordfreq(view, fnames):
"""Parallel word frequency counter.
view - An IPython DirectView
fnames - The filenames containing the split data.
"""
assert len(fnames) == len(view.targets)
view.scatter('fname', fnames, flatten=True)
ar = view.apply(wordfreq, ipp.Reference('fname'))
freqs_list = ar.get()
word_set = set()
for f in freqs_list:
word_set.update(f.keys())
freqs = dict(zip(word_set, repeat(0)))
for f in freqs_list:
for word, count in f.items():
freqs[word] += count
return freqs
def test_nested_getitem_setitem(self):
"""get and set with view['a.b']"""
view = self.client[-1]
view.execute('\n'.join([
'class A(object): pass',
'a = A()',
'a.b = 128',
]), block=True)
ra = pmod.Reference('a')
r = view.apply_sync(lambda x: x.b, ra)
self.assertEqual(r, 128)
self.assertEqual(view['a.b'], 128)
view['a.b'] = 0
r = view.apply_sync(lambda x: x.b, ra)
self.assertEqual(r, 0)
self.assertEqual(view['a.b'], 0)
def test_return_exceptions_postmortem(self):
self.minimum_engines(3)
dv = self.client[:]
bad_id = dv.targets[1]
dv.scatter("rank", dv.targets, flatten=True)
def fail_on_bad_id(rank, bad_id):
if rank == bad_id:
raise ValueError(f"{rank} is bad!")
return rank
ar = dv.apply_async(fail_on_bad_id, ipp.Reference('rank'), bad_id)
with raises_remote(ValueError):
ar.get()
result = ar.get(return_exceptions=True)
assert result[:1] == dv.targets[:1]
assert result[2:] == dv.targets[2:]
assert isinstance(result[1], ipp.RemoteError)
def test_wait_first_exception(self):
dv = self.client[:]
def fail(i):
print(i)
import time
if i == 0:
print(1 / i)
time.sleep(1)
return i
dv.scatter('rank', range(len(dv)), flatten=True, block=True)
amr = dv.apply_async(fail, ipp.Reference("rank"))
tic = time.perf_counter()
done, pending = amr.wait(timeout=5, return_when=ipp.FIRST_EXCEPTION)
assert done
assert len(done) == 1
first_done = done.pop()
assert first_done.msg_ids == amr.msg_ids[:1]
assert amr.wait(timeout=10) == True
done, pending = amr.wait(timeout=0, return_when=ipp.FIRST_EXCEPTION)
assert pending == set()
assert len(done) == len(amr)
view['u_hist'] = []
# set vector/scalar implementation details
impl = {}
impl['ic'] = 'vectorized'
impl['inner'] = 'scalar'
impl['bc'] = 'vectorized'
# execute some files so that the classes we need will be defined on the engines:
view.run('RectPartitioner.py')
view.run('wavesolver.py')
# setup remote partitioner
# note that Reference means that the argument passed to setup_partitioner will be the
# object named 'my_id' in the engine's namespace
view.apply_sync(setup_partitioner, ipp.Reference('my_id'), num_procs, grid,
partition)
# wait for initial communication to complete
view.execute('mpi.barrier()')
# setup remote solvers
view.apply_sync(
setup_solver,
I,
f,
c,
bc,
Lx,
Ly,
partitioner=ipp.Reference('partitioner'),
dt=0,
implementation=impl,
#create dataset
data = np.random.uniform(-10,10,size=(5000000,3))
df = pd.DataFrame(data,columns=('x1','x2','x3'))
df_split = np.array_split(df, 6, axis=0)
#load the dataframe chunk into the core
for i in dview.targets:
print('chunk:',i,'df shape:',df_split[i].shape)
rc[i]['df'] = df_split[i]
#create a reference to each dataframe
#this allows you do pass the reference instead of the data
rdf = [ipp.Reference('df') for i in range(num_cores)]
def map_nunique(df):
return df.nunique()
def reduce_nunique(x):
return pd.concat(x).groupby(level=0).sum()
%%time
reduce_nunique(dview.map_sync(map_nunique,rdf))
%%time
df.nunique()
def map_nearest_neighbors(df,qry,k=10):
euclid = np.sum((df.values-qry)**2,axis=1)
pub_url = root.apply_sync(lambda: com.pub_url) # noqa: F821
# gather the connection information into a dict
ar = view.apply_async(lambda: com.info) # noqa: F821
peers = ar.get_dict()
# this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators
# connect the engines to each other:
def connect(com, peers, tree, pub_url, root_id):
"""this function will be called on the engines"""
com.connect(peers, tree, pub_url, root_id)
view.apply_sync(connect, ipp.Reference('com'), peers, btree, pub_url, root_id)
# functions that can be used for reductions
# max and min builtins can be used as well
def add(a, b):
"""cumulative sum reduction"""
return a + b
def mul(a, b):
"""cumulative product reduction"""
return a * b
view['add'] = add
import time
import ipyparallel as parallel
# create client & view
rc = parallel.Client()
dv = rc[:]
v = rc.load_balanced_view()
# scatter 'id', so id=0,1,2 on engines 0,1,2
dv.scatter('id', rc.ids, flatten=True)
print("Engine IDs: ", dv['id'])
# create a Reference to `id`. This will be a different value on each engine
ref = parallel.Reference('id')
print("sleeping for `id` seconds on each engine")
tic = time.time()
ar = dv.apply(time.sleep, ref)
for i, r in enumerate(ar):
print("%i: %.3f" % (i, time.time() - tic))
def sleep_here(t):
import time
time.sleep(t)
return id, t
# one call per task
print("running with one call per task")
def test_reference_nameerror(self):
v = self.client[self.client.ids[0]]
r = pmod.Reference('elvis_has_left')
echo = lambda x: x
self.assertRaisesRemote(NameError, v.apply_sync, echo, r)
def test_eval_reference(self):
v = self.client[self.client.ids[0]]
v['g'] = list(range(5))
rg = pmod.Reference('g[0]')
echo = lambda x: x
self.assertEqual(v.apply_sync(echo, rg), 0)
# create the engine connectors
view.execute('com = EngineCommunicator()')
# gather the connection information into a single dict
ar = view.apply_async(lambda: com.info) # noqa: F821
peers = ar.get_dict()
# print peers
# this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators
# setup remote partitioner
# note that Reference means that the argument passed to setup_partitioner will be the
# object named 'com' in the engine's namespace
view.apply_sync(
setup_partitioner,
ipp.Reference('com'),
peers,
ipp.Reference('my_id'),
num_procs,
grid,
partition,
)
time.sleep(1)
# convenience lambda to call solver.solve:
_solve = lambda *args, **kwargs: solver.solve(*args, **kwargs)
if ns.scalar:
impl['inner'] = 'scalar'
# setup remote solvers
view.apply_sync(
setup_solver,
import time
import ipyparallel as ipp
# create client & view
rc = ipp.Client()
dv = rc[:]
v = rc.load_balanced_view()
# scatter 'id', so id=0,1,2 on engines 0,1,2
dv.scatter('id', rc.ids, flatten=True)
print("Engine IDs: ", dv['id'])
# create a Reference to `id`. This will be a different value on each engine
ref = ipp.Reference('id')
print("sleeping for `id` seconds on each engine")
tic = time.time()
ar = dv.apply(time.sleep, ref)
for i, r in enumerate(ar):
print("%i: %.3f" % (i, time.time() - tic))
def sleep_here(t):
import time
time.sleep(t)
return id, t
# one call per task
print("running with one call per task")
def test_remote_reference(self):
v = self.client[-1]
v['a'] = 123
ra = pmod.Reference('a')
b = v.apply_sync(lambda x: x, ra)
self.assertEqual(b, 123)