def importanceSampling_parallel4(generateSample,args_generateSample,likelihoodRatio,args_likelihoodRatio, N):
# not working
temp = 3
from IPython.parallel import Client
p = Client()[:]
p.use_dill()
print('This is the parallel output')
p.map_sync(generateSample,np.arange(6))
#sample = p.map_sync(generateSample,np.array((10,)))
#print(sample)
return(1)
def fit(self, X, y, sample_weight=None):
self._check_params()
self.evaluations_done = 0
X = pandas.DataFrame(X)
self._log("\n\nGridSearch started\n\n")
if self.ipc_profile is None:
while self.evaluations_done < self.generator.n_evaluations:
state_indices, state_dict = self.generator.generate_next_point()
value = estimate_classifier(params_dict=state_dict, base_estimator=self.base_estimator,
X=X, y=y, folds=self.folds, fold_checks=self.fold_checks,
score_function=self.score_function, sample_weight=sample_weight,
label=self.label,
scorer_needs_x=self.scorer_needs_x,
catch_exceptions=False)
self.generator.add_result(state_indices, value)
self.evaluations_done += 1
state_string = ", ".join([k + '=' + str(v) for k, v in state_dict.items()])
self._log(value, ": ", state_string)
else:
from IPython.parallel import Client
direct_view = Client(profile=self.ipc_profile).direct_view()
portion = len(direct_view)
print("There are {0} cores in cluster, the portion is equal {1}".format(len(direct_view), portion))
while self.evaluations_done < self.generator.n_evaluations:
state_indices_array, state_dict_array = self.generator.generate_batch_points(size=portion)
result = direct_view.map_sync(estimate_classifier, state_dict_array,
[self.base_estimator] * portion, [X]*portion, [y]*portion,
[self.folds] * portion, [self.fold_checks] * portion,
[self.score_function] * portion,
[sample_weight]*portion,
[self.label]*portion,
[self.scorer_needs_x] * portion)
assert len(result) == portion, "The length of result is very strange"
for state_indices, state_dict, score in zip(state_indices_array, state_dict_array, result):
params = ", ".join([k + '=' + str(v) for k, v in state_dict.items()])
if isinstance(score, Exception):
# returned exception
message = 'Fail during training on the node \nException ' + str(score) +\
'\nParameters:' + params
print(message)
self._log(message)
continue
self.generator.add_result(state_indices, score)
self._log(score, ": ", params)
self.evaluations_done += portion
print("%i evaluations done" % self.evaluations_done)
if self.refit:
self._fit_best_estimator(X, y, sample_weight=sample_weight)
def fit(self, X, y, sample_weight=None):
"""
Run fit with all sets of parameters.
:param X: array-like, shape = [n_samples, n_features]
Training vector, where n_samples is the number of samples and n_features is the number of features.
:param y: array-like, shape = [n_samples] or [n_samples, n_output], optional
:param sample_weight: array-like, shape = [n_samples], weight
"""
X, y, sample_weight = check_inputs(X, y, sample_weight=sample_weight, allow_none_weights=True)
if self.parallel_profile is None:
while self.evaluations_done < self.params_generator.n_evaluations:
state_indices, state_dict = self.params_generator.generate_next_point()
status, value = apply_scorer(self.scorer, state_dict, self.base_estimator, X, y, sample_weight)
assert status == 'success', 'Error during grid search ' + str(value)
self.params_generator.add_result(state_indices, value)
self.evaluations_done += 1
state_string = ", ".join([k + '=' + str(v) for k, v in state_dict.items()])
self._log('{}: {}'.format(value, state_string))
else:
from IPython.parallel import Client
direct_view = Client(profile=self.parallel_profile).direct_view()
portion = len(direct_view)
print("There are {0} cores in cluster, the portion is equal {1}".format(len(direct_view), portion))
while self.evaluations_done < self.params_generator.n_evaluations:
state_indices_array, state_dict_array = self.params_generator.generate_batch_points(size=portion)
result = direct_view.map_sync(apply_scorer, [self.scorer] * portion, state_dict_array,
[self.base_estimator] * portion,
[X] * portion, [y] * portion, [sample_weight] * portion)
assert len(result) == portion, "The length of result is very strange"
for state_indices, state_dict, (status, score) in zip(state_indices_array, state_dict_array, result):
params = ", ".join([k + '=' + str(v) for k, v in state_dict.items()])
if status != 'success':
message = 'Fail during training on the node \nException {exc}\n Parameters {params}'
self._log(message.format(exc=score, params=params), level=40)
else:
self.params_generator.add_result(state_indices, score)
self._log("{}: {}".format(score, params))
self.evaluations_done += portion
print("%i evaluations done" % self.evaluations_done)
return self
def fit(self, X, y, sample_weight=None):
"""
Run fit with all sets of parameters.
:param X: array-like, shape = [n_samples, n_features]
Training vector, where n_samples is the number of samples and n_features is the number of features.
:param y: array-like, shape = [n_samples] or [n_samples, n_output], optional
:param sample_weight: array-like, shape = [n_samples], weight
"""
X, y, sample_weight = check_inputs(X, y, sample_weight=sample_weight, allow_none_weights=True)
if self.parallel_profile is None:
while self.evaluations_done < self.params_generator.n_evaluations:
state_indices, state_dict = self.params_generator.generate_next_point()
status, value = apply_scorer(self.scorer, state_dict, self.base_estimator, X, y, sample_weight)
assert status == 'success', 'Error during grid search ' + str(value)
self.params_generator.add_result(state_indices, value)
self.evaluations_done += 1
state_string = ", ".join([k + '=' + str(v) for k, v in state_dict.items()])
self._log('{}: {}'.format(value, state_string))
else:
from IPython.parallel import Client
direct_view = Client(profile=self.parallel_profile).direct_view()
portion = len(direct_view)
print("There are {0} cores in cluster, the portion is equal {1}".format(len(direct_view), portion))
while self.evaluations_done < self.params_generator.n_evaluations:
state_indices_array, state_dict_array = self.params_generator.generate_batch_points(size=portion)
result = direct_view.map_sync(apply_scorer, [self.scorer] * portion, state_dict_array,
[self.base_estimator] * portion,
[X] * portion, [y] * portion, [sample_weight] * portion)
assert len(result) == portion, "The length of result is very strange"
for state_indices, state_dict, (status, score) in zip(state_indices_array, state_dict_array, result):
params = ", ".join([k + '=' + str(v) for k, v in state_dict.items()])
if status != 'success':
message = 'Fail during training on the node \nException {exc}\n Parameters {params}'
self._log(message.format(exc=score, params=params), level=40)
else:
self.params_generator.add_result(state_indices, score)
self._log("{}: {}".format(score, params))
self.evaluations_done += portion
print("%i evaluations done" % self.evaluations_done)
return self
f = pd.HDFStore(savefile, 'r+')
stats = f['stats']
stats[stat] = pd.np.NaN
if stat == 'rate':
fcomp = compute_rate_each
if stat == 'xcov':
fcomp = compute_xcov_each
if stat == 'acov':
fcomp = compute_acov_each
if stat == 'pcov':
fcomp = compute_pcov_each
if args['--serial']:
rvs = map(fcomp, fnames)
else:
from IPython.parallel import Client
ipypath = args['<ipython_dir>']
ipypath = join(ipypath, 'security', 'ipcontroller-client.json')
lview = Client(ipypath).load_balanced_view()
rvs = lview.map_sync(fcomp, fnames)
for i, s in rvs:
stats[stat][i] = s
f['stats'] = stats
print f['stats']
f.close()
f = pd.HDFStore(savefile, 'r+')
stats = f['stats']
stats[stat] = pd.np.NaN
if stat == 'rate':
fcomp = compute_rate_each
if stat == 'xcov':
fcomp = compute_xcov_each
if stat == 'acov':
fcomp = compute_acov_each
if stat == 'pcov':
fcomp = compute_pcov_each
if args['--serial']:
rvs = map(fcomp, fnames)
else:
from IPython.parallel import Client
ipypath = args['<ipython_dir>']
ipypath = join(ipypath,'security','ipcontroller-client.json')
lview = Client(ipypath).load_balanced_view()
rvs = lview.map_sync(fcomp, fnames)
for i, s in rvs:
stats[stat][i] = s
f['stats'] = stats
print f['stats']
f.close()
def run_each(fname):
import subprocess as sp
import os
NEURON = '/Applications/NEURON/nrn/x86_64/bin/nrniv'
return sp.call([NEURON, '-python', 'tcurve_network.py', fname])
if __name__ == '__main__':
from docopt import docopt
from os.path import join, exists
args = docopt(__doc__)
dpath = args['<data_dir>']
fnames = open(join(dpath,'files.txt')).read().split('\n')
fnames = [join(dpath, fname) for fname in fnames]
if args['--serial']:
r = map(run_each, fnames)
else:
from IPython.parallel import Client
ipypath = args['<ipython_dir>']
ipypath = join(ipypath,'security','ipcontroller-client.json')
# print ipypath + str(exists(ipypath))
lview = Client(ipypath).load_balanced_view()
r = lview.map_sync(run_each, fnames)
for x in r:
print x # check the result.
def run_each(fname):
import subprocess as sp
import os
NEURON = '/Applications/NEURON/nrn/x86_64/bin/nrniv'
return sp.call([NEURON, '-python', 'tcurve_network.py', fname])
if __name__ == '__main__':
from docopt import docopt
from os.path import join, exists
args = docopt(__doc__)
dpath = args['<data_dir>']
fnames = open(join(dpath, 'files.txt')).read().split('\n')
fnames = [join(dpath, fname) for fname in fnames]
if args['--serial']:
r = map(run_each, fnames)
else:
from IPython.parallel import Client
ipypath = args['<ipython_dir>']
ipypath = join(ipypath, 'security', 'ipcontroller-client.json')
# print ipypath + str(exists(ipypath))
lview = Client(ipypath).load_balanced_view()
r = lview.map_sync(run_each, fnames)
for x in r:
print x # check the result.
