def run_mcmc(self):
complete = self.chains.check_completness()
if not complete:
for i in range(self.nchains):
self.chains.chains.append(
Chain(self.chains.chains_filename, self.covfile, nchain=i, nsteps=self.nsteps))
pool = Pool(processes=self.nchains)
try:
# Without the .get(9999), you can't interrupt this with Ctrl+C.
pool.map_async(self.mcmc, self.chains.chains).get(999999)
pool.close()
pool.join()
# to skip lines after the progress bars
print '\n' * self.nchains
except KeyboardInterrupt:
pool.terminate()
self.likelihood = self.chains.chains_to_likelihood()
self.likelihood.stats(self.covfile)
# [self.results[i].append(self.likelihood.pdfs[i].mean) for i in range(self.chains.dim)]
# self.p = [self.likelihood.pdfs[i].mean for i in range(self.chains.dim)]
self.p = self.chains.best_row_params
self.simulation(self.spectrum.lambdas, *self.p)
# [self.results_err[i].append([self.likelihood.pdfs[i].error_high,self.likelihood.pdfs[i].error_low]) for i in range(self.chains.dim)]
# if(self.plot):
self.likelihood.triangle_plots()
self.plot_fit()
# if convergence_test :
self.chains.convergence_tests()
return self.likelihood
def gridSearch(options, use_datasets, numExamples, compute_mistakes=False, verbose=False, parallelize=False):
if MODEL_KEYWORD not in options:
print 'ERROR: must specify models for grid search under "%s" key.' % (MODEL_KEYWORD)
return
paramCombos = myProduct(options)
partialTestCombo = partial(testCombo, use_datasets=use_datasets, numExamples=numExamples, compute_mistakes=compute_mistakes, verbose=verbose)
if parallelize:
from pathos.multiprocessing import Pool
p = Pool(5)
try:
result = p.map_async(partialTestCombo, paramCombos)
result = result.get(999999999)
bestScore, bestParamsStr, bestCombo = max(result, key=lambda x:x[0])
sys.stdout = open("best.out", "w")
print 'Best score of %s was achieved by parameters:\n%s' % (bestScore, bestParamsStr)
except KeyboardInterrupt:
p.terminate()
print "You cancelled the program!"
sys.exit(1)
else:
bestScore, bestCombo, bestComboStr = float('-inf'), None, ''
for paramCombo in paramCombos:
score, paramsStr, _ = testCombo(paramCombo, use_datasets=use_datasets, numExamples=numExamples, compute_mistakes=compute_mistakes, verbose=verbose, parallelize=False)
if score > bestScore:
bestScore, bestCombo, bestComboStr = score, paramCombo, paramsStr
print 'Best score of %s was achieved by parameters:\n%s' % (bestScore, bestComboStr)
def main():
from dask.diagnostics import ProgressBar
ProgressBar().register()
p = Pool(8)
data_long = dd.read_csv('../NREL_Locations_clean.csv').compute()
results = p.map_async(make_request,
[gb for gb in data_long.groupby('VIN_ID')]).get()
print('total vehicles: ', np.sum(results))
class MyProblem(ea.Problem): # 继承Problem父类
def __init__(self, func, dim, lb, ub, njobs=0):
self.func = func
# 初始化name（函数名称，可以随意设置）
name = 'MyProblem'
# 初始化M（目标维数）
M = 1
# 初始化maxormins（目标最小最大化标记列表，1：最小化该目标；-1：最大化该目标）
maxormins = [1]
# 初始化Dim（决策变量维数）
Dim = dim
# 初始化varTypes（决策变量的类型，元素为0表示对应的变量是连续的；1表示是离散的）
varTypes = [0] * Dim
# 决策变量下界
# lb = [0] * Dim
# 决策变量上界
# ub = [1, 1, 3, 3]
# 决策变量下边界（0表示不包含该变量的下边界，1表示包含）
lbin = [1] * Dim
# 决策变量上边界（0表示不包含该变量的上边界，1表示包含）
ubin = [1] * Dim
# 多进程
self.njobs = njobs
if njobs > 1:
self.pool = Pool(njobs)
elif njobs == -1:
self.pool = Pool(int(cpu_count()))
else:
self.pool = None
# 可视化
self.count = 0
# 调用父类构造方法完成实例化
ea.Problem.__init__(self, name, M, maxormins, Dim, varTypes, lb, ub,
lbin, ubin)
def aimFunc(self, pop): # 目标函数
x = pop.Phen # 得到决策变量矩阵
# 使用多进程进行
if self.pool is not None:
results = self.pool.map_async(self.func, list(x))
results.wait()
# 计算目标函数值，赋值给pop种群对象的ObjV属性
results = np.array([results.get()]).T
else:
results = []
for i in range(x.shape[0]):
results.append(self.func(x[i, :]))
results = np.stack(results, axis=0)
results = np.expand_dims(results, 1)
# 计算目标函数值，赋值给pop种群对象的ObjV属性
pop.ObjV = results
# 打印计数信息，便于进行查看
self.count += 1
print("第%d代完成" % self.count)
def vw_train_and_test(self, options_list, data_file_paths):
def init_worker():
signal.signal(signal.SIGINT, signal.SIG_IGN)
def run_learner(options):
train_options = {
'final_regressor': os.path.join(self.work_dir, id_generator()),
'data': data_file_paths['train'],
'cache_file': data_file_paths['train'] + '.cache'
}
train_options.update(options)
test_options = {
'data': data_file_paths['test'],
'predictions': os.path.join(self.work_dir, id_generator()),
'cache_file': data_file_paths['test'] + '.cache'
}
test_options.update(options)
# TO DO: remove below if.
if 'kill_cache' in options:
del train_options['kill_cache']
del test_options['kill_cache']
vw_wrapper = VW_Wrapper(verbose=False)
vw_wrapper.train(train_options)
predictions = vw_wrapper.test(test_options)
os.remove(train_options['final_regressor'])
os.remove(test_options['predictions'])
return options, predictions
if len(options_list) > 1:
try:
if not os.path.isfile(data_file_paths['test'] +
'.cache') or not os.path.isfile(
data_file_paths['train'] + '.cache'):
run_learner(options_list[0])
pool = Pool(len(options_list), init_worker)
result_list = pool.map_async(run_learner,
options_list).get(99999999)
pool.close()
pool.join()
return result_list
except KeyboardInterrupt:
print ' Keyboard Interrupt, exiting...) '
pool.terminate()
pool.join()
sys.exit(0)
elif len(options_list) == 1:
return [run_learner(options_list[0])]
else:
return []
def gridSearch(options,
use_datasets,
numExamples,
compute_mistakes=False,
verbose=False,
parallelize=False):
if MODEL_KEYWORD not in options:
print 'ERROR: must specify models for grid search under "%s" key.' % (
MODEL_KEYWORD)
return
paramCombos = myProduct(options)
partialTestCombo = partial(testCombo,
use_datasets=use_datasets,
numExamples=numExamples,
compute_mistakes=compute_mistakes,
verbose=verbose)
if parallelize:
from pathos.multiprocessing import Pool
p = Pool(5)
try:
result = p.map_async(partialTestCombo, paramCombos)
result = result.get(999999999)
bestScore, bestParamsStr, bestCombo = max(result,
key=lambda x: x[0])
sys.stdout = open("best.out", "w")
print 'Best score of %s was achieved by parameters:\n%s' % (
bestScore, bestParamsStr)
except KeyboardInterrupt:
p.terminate()
print "You cancelled the program!"
sys.exit(1)
else:
bestScore, bestCombo, bestComboStr = float('-inf'), None, ''
for paramCombo in paramCombos:
score, paramsStr, _ = testCombo(paramCombo,
use_datasets=use_datasets,
numExamples=numExamples,
compute_mistakes=compute_mistakes,
verbose=verbose,
parallelize=False)
if score > bestScore:
bestScore, bestCombo, bestComboStr = score, paramCombo, paramsStr
print 'Best score of %s was achieved by parameters:\n%s' % (
bestScore, bestComboStr)
