def run_exp_racos_for_synthetic_problem_analysis(): # parameters sample_size = 10 # the instance number of sampling in an iteration budget = 500 # budget in online style positive_num = 2 # the set size of PosPop rand_probability = 0.99 # the probability of sample in model uncertain_bit = 1 # the dimension size that is sampled randomly adv_threshold = 10 # advance sample size opt_repeat = 10 dimension_size = 10 problem_name = 'sphere' problem_num = 200 start_index = 0 bias_region = 0.2 dimension = Dimension() dimension.set_dimension_size(dimension_size) dimension.set_regions([[-1.0, 1.0] for _ in range(dimension_size)], [0 for _ in range(dimension_size)]) log_buffer = [] # logging learner_path = './ExpLearner/SyntheticProbsLearner/' + problem_name + '/dimension' + str(dimension_size)\ + '/DirectionalModel/' + 'learner-' + problem_name + '-' + 'dim' + str(dimension_size) + '-'\ + 'bias' + str(bias_region) + '-' problem_path = './ExpLog/SyntheticProbsLog/' + problem_name + '/dimension' + str(dimension_size)\ + '/DirectionalModel/' + 'bias-' + problem_name + '-' + 'dim' + str(dimension_size) + '-'\ + 'bias' + str(bias_region) + '-' func = DistributedFunction(dimension, bias_region=[-0.5, 0.5]) target_bias = [0.1 for _ in range(dimension_size)] func.setBias(target_bias) if problem_name == 'ackley': prob_fct = func.DisAckley else: prob_fct = func.DisSphere relate_error_list = [] for prob_i in range(problem_num): print( start_index + prob_i, '++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++') log_buffer.append( str(start_index + prob_i) + '++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++') log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('optimization parameters') log_buffer.append('sample size: ' + str(sample_size)) log_buffer.append('budget: ' + str(budget)) log_buffer.append('positive num: ' + str(positive_num)) log_buffer.append('random probability: ' + str(rand_probability)) log_buffer.append('uncertain bits: ' + str(uncertain_bit)) log_buffer.append('advance num: ' + str(adv_threshold)) log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('problem parameters') log_buffer.append('dimension size: ' + str(dimension_size)) log_buffer.append('problem name: ' + problem_name) log_buffer.append('bias_region: ' + str(bias_region)) log_buffer.append('+++++++++++++++++++++++++++++++') problem_file = problem_path + str(start_index + prob_i) + '.txt' problem_str = fo.FileReader(problem_file)[0].split(',') problem_index = int(problem_str[0]) problem_bias = string2list(problem_str[1]) if problem_index != (start_index + prob_i): print('problem error!') exit(0) print('source bias: ', problem_bias) log_buffer.append('source bias: ' + list2string(problem_bias)) reduisal = np.array(target_bias) - np.array(problem_bias) this_distance = reduisal * reduisal.T learner_file = learner_path + str(start_index + prob_i) + '.pkl' log_buffer.append('learner file: ' + learner_file) print('learner file: ', learner_file) net = torch.load(learner_file) net_list = [net] opt_error_list = [] for i in range(opt_repeat): print('optimize ', i, '===================================================') log_buffer.append( 'optimize ' + str(i) + '===================================================') exp_racos = ExpRacosOptimization(dimension, net_list) start_t = time.time() exp_racos.exp_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) end_t = time.time() print('total budget is ', budget) log_buffer.append('total budget is ' + str(budget)) hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) optimal = exp_racos.get_optimal() opt_error = optimal.get_fitness() optimal_x = optimal.get_features() opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) print('optimal x: ', optimal_x) log_buffer.append('optimal nn structure: ' + list2string(optimal_x)) opt_mean = np.mean(np.array(opt_error_list)) relate_error_list.append([this_distance, opt_mean]) opt_std = np.std(np.array(opt_error_list)) print('--------------------------------------------------') print('optimization result: ', opt_mean, '#', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result: ' + str(opt_mean) + '#' + str(opt_std)) result_path = './Results/SyntheticProbs/' + problem_name + '/dimension' + str( dimension_size) + '/' relate_error_file = result_path + 'relate-error-' + problem_name + '-dim' + str(dimension_size) + '-bias'\ + str(bias_region) + '.txt' temp_buffer = [] for i in range(len(relate_error_list)): relate, error = relate_error_list[i] temp_buffer.append(str(relate) + ',' + str(error)) print('relate error logging: ', relate_error_file) log_buffer.append('relate error logging: ' + relate_error_file) fo.FileWriter(relate_error_file, temp_buffer, style='w') optimization_log_file = result_path + 'opt-log-' + problem_name + '-dim' + str(dimension_size) + '-bias'\ + str(bias_region) + '.txt' print('optimization logging: ', optimization_log_file) fo.FileWriter(optimization_log_file, log_buffer, style='w')
def run(type): dimension = Dimension() dimension.set_dimension_size(dimension_size) dimension.set_regions([[-1.0, 1.0] for _ in range(dimension_size)], [0 for _ in range(dimension_size)]) # problem define func = DistributedFunction(dimension, bias_region=[-bias_region, bias_region]) target_bias = [0.1 for _ in range(dimension_size)] func.setBias(target_bias) if problem_name == 'ackley': prob_fct = func.DisAckley elif problem_name == 'sphere': prob_fct = func.DisSphere elif problem_name == 'rosenbrock': prob_fct = func.DisRosenbrock else: print('Wrong function!') exit() opt_error_list = [] log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('Running: ' + type) log_buffer.append('+++++++++++++++++++++++++++++++') print('+++++++++++++++++++++++++++++++') print('Running: ' + type) print('+++++++++++++++++++++++++++++++') if type == 'ada': # pre=sorted(predictors,key=lambda a:a.dist) expert = Experts(predictors=predictors, eta=eta, bg=budget) for i in range(opt_repeat): print('optimize ', i, '===================================================') log_buffer.append( 'optimize ' + str(i) + '===================================================') start_t = time.time() if type == 'exp': exp_racos = ExpRacosOptimization(dimension, nets) opt_error = exp_racos.exp_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) elif type == 'ada': exp_racos = ExpAdaRacosOptimization(dimension, expert) opt_error = exp_racos.exp_ada_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) else: print('Wrong type!') return end_t = time.time() hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) opt_mean = np.mean(np.array(opt_error_list), axis=0) opt_std = np.std(np.array(opt_error_list), axis=0) print('--------------------------------------------------') print('optimization result for ' + str(opt_repeat) + ' times average: ', opt_mean, ', standard variance is: ', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result for ' + str(opt_repeat) + ' times average: ' + str(opt_mean) + ', standard variance is: ' + str(opt_std)) return opt_mean
def run_for_real_problem(problem_name, type): dtrain, dtest, dvalid = mlbp.get_train_test_data(problem_name) opt_error_list = [] gen_error_list = [] print(type, ' optimize ', problem_name, '===================================================') log_buffer.append(type + ' optimize ' + problem_name + '===================================================') for j in range(opt_repeat): print(j) log_buffer.append(str(j)) model = lgb.LGBMClassifier() start_t = time.time() def score_fun(x): ## here is the score function hyper_param = (sample_codec.sample_decode(x)) model.set_params(**hyper_param) bst = model.fit(dtrain[:, :-1], dtrain[:, -1]) pred = bst.predict(dvalid[:, :-1]) fitness = -f1_score(dvalid[:, -1], pred, average='macro') return fitness if type == 'racos': optimizer = RacosOptimization(dimension) optimizer.clear() optimizer.mix_opt(obj_fct=score_fun, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit) elif type == 'ave': optimizer = ExpRacosOptimization(dimension, nets) log = optimizer.exp_mix_opt(obj_fct=score_fun, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) for line in log: log_buffer.append(line) elif type == 'ada': optimizer = ExpAdaRacosOptimization(dimension, expert) optimizer.clear() log = optimizer.exp_ada_mix_opt(obj_fct=score_fun, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold, step=step) for line in log: log_buffer.append(line) else: print('Wrong type!') return end_t = time.time() print('total budget is ', budget) log_buffer.append('total budget is ' + str(budget)) hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) optimal = optimizer.get_optimal() opt_error = optimal.get_fitness() optimal_x = optimal.get_features() hyper_param = (sample_codec.sample_decode(optimal_x)) model = lgb.LGBMClassifier() model.set_params(**hyper_param) train = np.concatenate((dtrain, dvalid), axis=0) bst = model.fit(train[:, :-1], train[:, -1]) pred = bst.predict(dtest[:, :-1]) gen_error = -f1_score(dtest[:, -1], pred, average='macro') gen_error_list.append(gen_error) opt_error_list.append(opt_error) print('***********validation optimal value: ', opt_error) log_buffer.append('***********validation optimal value: ' + str(opt_error)) print('***********generalize optimal value: ', gen_error) log_buffer.append('***********generalize optimal value: ' + str(gen_error)) print('optimal x: ', optimal_x) # log_buffer.append('optimal nn structure: ' + list2string(optimal_x)) opt_mean = np.mean(np.array(opt_error_list)) opt_std = np.std(np.array(opt_error_list)) gen_mean = np.mean(np.array(gen_error_list)) gen_std = np.std(np.array(gen_error_list)) return -opt_mean, opt_std, -gen_mean, gen_std
def run(type): opt_error_list = [] log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('Running: ' + type) log_buffer.append('+++++++++++++++++++++++++++++++') print('+++++++++++++++++++++++++++++++') print('Running: ' + type) print('+++++++++++++++++++++++++++++++') if type == 'ada': # pre=sorted(predictors,key=lambda a:a.dist) expert = Experts(predictors=predictors, eta=eta, bg=budget) for i in range(opt_repeat): print('optimize ', i, '===================================================') log_buffer.append( 'optimize ' + str(i) + '===================================================') start_t = time.time() if type == 'ave': exp_racos = ExpRacosOptimization(dimension, nets) opt_error = exp_racos.exp_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) elif type == 'ada': exp_racos = ExpAdaRacosOptimization(dimension, expert) opt_error = exp_racos.exp_ada_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold, step=step) elif type == 'ground truth': exp_racos = ExpRacosOptimization(dimension, nets[:step]) exp_racos.exp_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) else: print('Wrong type!') return end_t = time.time() hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) optimal = exp_racos.get_optimal() opt_error = optimal.get_fitness() optimal_x = optimal.get_features() opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) print('optimal x: ', optimal_x) log_buffer.append('optimal nn structure: ' + list2string(optimal_x)) opt_mean = np.mean(np.array(opt_error_list), axis=0) opt_std = np.std(np.array(opt_error_list), axis=0) print('--------------------------------------------------') print('optimization result for ' + str(opt_repeat) + ' times average: ', opt_mean, ', standard variance is: ', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result for ' + str(opt_repeat) + ' times average: ' + str(opt_mean) + ', standard variance is: ' + str(opt_std)) return opt_mean, opt_std