class IPClusterEnsemble(SurveyEnsemble):
"""
Parallelized suvey ensemble based on IPython parallal (ipcluster)
Args:
\*\*specs:
user specified values
Attributes:
Notes:
"""
def __init__(self, **specs):
SurveyEnsemble.__init__(self, **specs)
# access the cluster
self.rc = Client()
self.dview = self.rc[:]
self.dview.block = True
with self.dview.sync_imports(): import EXOSIMS,EXOSIMS.util.get_module
r1 = self.dview.execute("SurveySim = EXOSIMS.util.get_module.get_module('%s', 'SurveySimulation')"%specs['modules']['SurveySimulation'])
self.dview.push(dict(specs=specs))
r2 = self.dview.execute("sim = SurveySim(**specs)")
self.lview = self.rc.load_balanced_view()
def run_ensemble(self,run_one,N=10):
t1 = time.time()
async_res = []
for j in range(N):
ar = self.lview.apply_async(run_one)
async_res.append(ar)
print "Submitted tasks: ", len(async_res)
self.rc.wait(async_res)
t2 = time.time()
print "Completed in %d sec" %(t2-t1)
res = [ar.get() for ar in async_res]
return res
def parallel_combinatorialSelection(self,
target,
N=2,
n_jobs=None,
c=None):
'''
score_log:
key - name of feature removed
val - expected value of class1 withou the key feature
This works with N=1, where expected vals with each feature
is calculated.
'''
if N > self.num_features:
print('Error: N has to be smaller than num_features')
return
def getExp2(combo):
test_feature = self.data[list(combo)]
jp = JointProbability()
jp.fit(test_feature, target)
contingency = jp.contingency
E = np.sum(contingency[1] * contingency['cond_proba'])
return E
# Set up parallel calculation
if c is None:
c = Client(profile="default")
else:
c = c
feature_combo = list(combinations(self.feature_list_, N))
if n_jobs is None:
vals = c[:].map(getExp2, feature_combo)
else:
vals = c[:n_jobs].map(getExp2, feature_combo)
c.wait(vals)
expected_vals = dict(zip(feature_combo, vals.get()))
self.best_expected_value_ = np.max(vals.get())
best_ind = np.argmax(vals.get())
self.best_combination_ = feature_combo[best_ind]
return expected_vals
def connect_client(self):
"""connect a client with my Context, and track its sockets for cleanup"""
c = Client(profile='iptest', context=self.context)
c.wait = lambda *a, **kw: self.client_wait(c, *a, **kw)
for name in filter(lambda n: n.endswith('socket'), dir(c)):
s = getattr(c, name)
s.setsockopt(zmq.LINGER, 0)
self.sockets.append(s)
return c
def connect_client(self):
"""connect a client with my Context, and track its sockets for cleanup"""
c = Client(profile='iptest', context=self.context)
c.wait = lambda *a, **kw: self.client_wait(c, *a, **kw)
for name in filter(lambda n:n.endswith('socket'), dir(c)):
s = getattr(c, name)
s.setsockopt(zmq.LINGER, 0)
self.sockets.append(s)
return c
def par_value(n):
"""
Parallel option valuation
Parameters
==========
n: int
number of option valuations/strikes
"""
import numpy as np
from ipyparallel import Client
c = Client(profile="default")
view = c.load_balanced_view()
strikes = np.linspace(80, 20, n)
option_values = []
for strike in strikes:
values = view.apply_async(bsm_mcs_valuation, strike)
option_values.append(values)
c.wait(option_values)
return strikes, option_values
def client_wait(self, client, jobs=None, timeout=-1):
"""my wait wrapper, sets a default finite timeout to avoid hangs"""
if timeout < 0:
timeout = self.timeout
return Client.wait(client, jobs, timeout)
def client_wait(self, client, jobs=None, timeout=-1):
"""my wait wrapper, sets a default finite timeout to avoid hangs"""
if timeout < 0:
timeout = self.timeout
return Client.wait(client, jobs, timeout)