def run_no_expert(): log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('Running: no experts, pure Racos') log_buffer.append('+++++++++++++++++++++++++++++++') print('+++++++++++++++++++++++++++++++') print('Running: no experts, pure Racos') print('+++++++++++++++++++++++++++++++') # optimization racos = RacosOptimization(dimension) opt_error_list = [] for i in range(opt_repeat): start_t = time.time() racos.mix_opt(prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit) end_t = time.time() optimal = racos.get_optimal() opt_error = optimal.get_fitness() hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ' hours ', minute, ' minutes ', second, ' seconds') print('optimal value: ', opt_error) opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) opt_mean = np.mean(np.array(opt_error_list)) opt_std = np.std(np.array(opt_error_list)) print('--------------------------------------------------') print('optimization result for ' + str(opt_repeat) + ' times average: ', opt_mean, ', standard variance is: ', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result for ' + str(opt_repeat) + ' times average: ' + str(opt_mean) + ', standard variance is: ' + str(opt_std)) return opt_mean, opt_std
def run_exp_racos_for_synthetic_problem_analysis(): # parameters sample_size = 10 # the instance number of sampling in an iteration budget = 500 # budget in online style positive_num = 2 # the set size of PosPop rand_probability = 0.99 # the probability of sample in model uncertain_bit = 1 # the dimension size that is sampled randomly adv_threshold = 10 # advance sample size opt_repeat = 10 dimension_size = 10 problem_name = 'sphere' problem_num = 200 start_index = 0 bias_region = 0.2 dimension = Dimension() dimension.set_dimension_size(dimension_size) dimension.set_regions([[-1.0, 1.0] for _ in range(dimension_size)], [0 for _ in range(dimension_size)]) log_buffer = [] # logging learner_path = './ExpLearner/SyntheticProbsLearner/' + problem_name + '/dimension' + str(dimension_size)\ + '/DirectionalModel/' + 'learner-' + problem_name + '-' + 'dim' + str(dimension_size) + '-'\ + 'bias' + str(bias_region) + '-' problem_path = './ExpLog/SyntheticProbsLog/' + problem_name + '/dimension' + str(dimension_size)\ + '/DirectionalModel/' + 'bias-' + problem_name + '-' + 'dim' + str(dimension_size) + '-'\ + 'bias' + str(bias_region) + '-' func = DistributedFunction(dimension, bias_region=[-0.5, 0.5]) target_bias = [0.1 for _ in range(dimension_size)] func.setBias(target_bias) if problem_name == 'ackley': prob_fct = func.DisAckley else: prob_fct = func.DisSphere relate_error_list = [] for prob_i in range(problem_num): print( start_index + prob_i, '++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++') log_buffer.append( str(start_index + prob_i) + '++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++') log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('optimization parameters') log_buffer.append('sample size: ' + str(sample_size)) log_buffer.append('budget: ' + str(budget)) log_buffer.append('positive num: ' + str(positive_num)) log_buffer.append('random probability: ' + str(rand_probability)) log_buffer.append('uncertain bits: ' + str(uncertain_bit)) log_buffer.append('advance num: ' + str(adv_threshold)) log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('problem parameters') log_buffer.append('dimension size: ' + str(dimension_size)) log_buffer.append('problem name: ' + problem_name) log_buffer.append('bias_region: ' + str(bias_region)) log_buffer.append('+++++++++++++++++++++++++++++++') problem_file = problem_path + str(start_index + prob_i) + '.txt' problem_str = fo.FileReader(problem_file)[0].split(',') problem_index = int(problem_str[0]) problem_bias = string2list(problem_str[1]) if problem_index != (start_index + prob_i): print('problem error!') exit(0) print('source bias: ', problem_bias) log_buffer.append('source bias: ' + list2string(problem_bias)) reduisal = np.array(target_bias) - np.array(problem_bias) this_distance = reduisal * reduisal.T learner_file = learner_path + str(start_index + prob_i) + '.pkl' log_buffer.append('learner file: ' + learner_file) print('learner file: ', learner_file) net = torch.load(learner_file) net_list = [net] opt_error_list = [] for i in range(opt_repeat): print('optimize ', i, '===================================================') log_buffer.append( 'optimize ' + str(i) + '===================================================') exp_racos = ExpRacosOptimization(dimension, net_list) start_t = time.time() exp_racos.exp_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) end_t = time.time() print('total budget is ', budget) log_buffer.append('total budget is ' + str(budget)) hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) optimal = exp_racos.get_optimal() opt_error = optimal.get_fitness() optimal_x = optimal.get_features() opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) print('optimal x: ', optimal_x) log_buffer.append('optimal nn structure: ' + list2string(optimal_x)) opt_mean = np.mean(np.array(opt_error_list)) relate_error_list.append([this_distance, opt_mean]) opt_std = np.std(np.array(opt_error_list)) print('--------------------------------------------------') print('optimization result: ', opt_mean, '#', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result: ' + str(opt_mean) + '#' + str(opt_std)) result_path = './Results/SyntheticProbs/' + problem_name + '/dimension' + str( dimension_size) + '/' relate_error_file = result_path + 'relate-error-' + problem_name + '-dim' + str(dimension_size) + '-bias'\ + str(bias_region) + '.txt' temp_buffer = [] for i in range(len(relate_error_list)): relate, error = relate_error_list[i] temp_buffer.append(str(relate) + ',' + str(error)) print('relate error logging: ', relate_error_file) log_buffer.append('relate error logging: ' + relate_error_file) fo.FileWriter(relate_error_file, temp_buffer, style='w') optimization_log_file = result_path + 'opt-log-' + problem_name + '-dim' + str(dimension_size) + '-bias'\ + str(bias_region) + '.txt' print('optimization logging: ', optimization_log_file) fo.FileWriter(optimization_log_file, log_buffer, style='w')
def run_for_real_problem(problem_name, type): dtrain, dtest, dvalid = mlbp.get_train_test_data(problem_name) opt_error_list = [] gen_error_list = [] print(type, ' optimize ', problem_name, '===================================================') log_buffer.append(type + ' optimize ' + problem_name + '===================================================') for j in range(opt_repeat): print(j) log_buffer.append(str(j)) model = lgb.LGBMClassifier() start_t = time.time() def score_fun(x): ## here is the score function hyper_param = (sample_codec.sample_decode(x)) model.set_params(**hyper_param) bst = model.fit(dtrain[:, :-1], dtrain[:, -1]) pred = bst.predict(dvalid[:, :-1]) fitness = -f1_score(dvalid[:, -1], pred, average='macro') return fitness if type == 'racos': optimizer = RacosOptimization(dimension) optimizer.clear() optimizer.mix_opt(obj_fct=score_fun, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit) elif type == 'ave': optimizer = ExpRacosOptimization(dimension, nets) log = optimizer.exp_mix_opt(obj_fct=score_fun, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) for line in log: log_buffer.append(line) elif type == 'ada': optimizer = ExpAdaRacosOptimization(dimension, expert) optimizer.clear() log = optimizer.exp_ada_mix_opt(obj_fct=score_fun, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold, step=step) for line in log: log_buffer.append(line) else: print('Wrong type!') return end_t = time.time() print('total budget is ', budget) log_buffer.append('total budget is ' + str(budget)) hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) optimal = optimizer.get_optimal() opt_error = optimal.get_fitness() optimal_x = optimal.get_features() hyper_param = (sample_codec.sample_decode(optimal_x)) model = lgb.LGBMClassifier() model.set_params(**hyper_param) train = np.concatenate((dtrain, dvalid), axis=0) bst = model.fit(train[:, :-1], train[:, -1]) pred = bst.predict(dtest[:, :-1]) gen_error = -f1_score(dtest[:, -1], pred, average='macro') gen_error_list.append(gen_error) opt_error_list.append(opt_error) print('***********validation optimal value: ', opt_error) log_buffer.append('***********validation optimal value: ' + str(opt_error)) print('***********generalize optimal value: ', gen_error) log_buffer.append('***********generalize optimal value: ' + str(gen_error)) print('optimal x: ', optimal_x) # log_buffer.append('optimal nn structure: ' + list2string(optimal_x)) opt_mean = np.mean(np.array(opt_error_list)) opt_std = np.std(np.array(opt_error_list)) gen_mean = np.mean(np.array(gen_error_list)) gen_std = np.std(np.array(gen_error_list)) return -opt_mean, opt_std, -gen_mean, gen_std
def synthetic_problems_sample(budget=500, problem_name='sphere', problem_size=5, max_bias=0.5, bias_step=0): sample_size = 10 # the instance number of sampling in an iteration positive_num = 2 # the set size of PosPop rand_probability = 0.99 # the probability of sample in model uncertain_bits = 2 # the dimension size that is sampled randomly start_index = 0 repeat_num = 10 exp_path = path + '/ExpLog/SyntheticProbsLog/' bias = 0 dimension_size = 10 dimension = Dimension() dimension.set_dimension_size(dimension_size) dimension.set_regions([[-1.0, 1.0] for _ in range(dimension_size)], [0 for _ in range(dimension_size)]) if bias_step > 0: problem_name += '_group-sample' for prob_i in range(problem_size): if bias_step > 0 and prob_i % (problem_size / max_bias * bias_step) == 0: bias += bias_step else: bias = max_bias # bias log format: 'index,bias_list: dim1 dim2 dim3...' bias_log = [] running_log = [] running_log.append('+++++++++++++++++++++++++++++++++') running_log.append('optimization setting: ') running_log.append('sample_size: ' + str(sample_size)) running_log.append('positive_num: ' + str(positive_num)) running_log.append('rand_probability: ' + str(rand_probability)) running_log.append('uncertain_bits: ' + str(uncertain_bits)) running_log.append('budget: ' + str(budget)) running_log.append('group sample step: ' + str(bias_step)) running_log.append('+++++++++++++++++++++++++++++++++') print(problem_name, ': ', start_index + prob_i, ' ==============================================') running_log.append(problem_name + ': ' + str(start_index + prob_i) + ' ==============================================') # problem setting func = DistributedFunction(dim=dimension, bias_region=[-bias, bias]) if 'ackley' in problem_name: prob = func.DisAckley elif 'sphere' in problem_name: prob = func.DisSphere elif 'rosenbrock' in problem_name: prob = func.DisRosenbrock else: print('Wrong function!') return # bias log bias_log.append(str(prob_i) + ',' + list2string(func.getBias())) print('function: ', problem_name, ', this bias: ', func.getBias()) running_log.append('function: ' + problem_name + ', this bias: ' + list2string(func.getBias())) # optimization setting optimizer = RacosOptimization(dimension) positive_set = [] negative_set = [] new_sample_set = [] label_set = [] for repeat_i in range(repeat_num): print('repeat ', repeat_i, ' ----------------------------------------') running_log.append('repeat ' + str(repeat_i) + ' ----------------------------------------') # optimization process start_t = time.time() optimizer.mix_opt(obj_fct=prob, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bits) end_t = time.time() hour, minute, second = time_formulate(start_t, end_t) # optimization results optimal = optimizer.get_optimal() print('optimal v: ', optimal.get_fitness(), ' - ', optimal.get_features()) running_log.append('optimal v: ' + str(optimal.get_fitness()) + ' - ' + list2string(optimal.get_features())) print('spent time: ', hour, ':', minute, ':', second) running_log.append('spent time: ' + str(hour) + ':' + str(minute) + ':' + str(second)) # log samples this_positive, this_negative, this_new, this_label = optimizer.get_log( ) print('sample number: ', len(this_positive), ':', len(this_label)) running_log.append('sample number: ' + str(len(this_positive)) + ':' + str(len(this_label))) positive_set.extend(this_positive) negative_set.extend(this_negative) new_sample_set.extend(this_new) label_set.extend(this_label) print('----------------------------------------------') print('sample finish!') print('all sample number: ', len(positive_set), '-', len(negative_set), '-', len(new_sample_set), \ '-', len(label_set)) running_log.append('----------------------------------------------') running_log.append('all sample number: ' + str(len(positive_set)) + '-' + str(len(negative_set)) + '-' + str(len(new_sample_set)) + '-' + str(len(label_set))) data_log_file = exp_path + str(problem_name) + '/dimension' + str(dimension_size) + '/DataLog/' + \ 'data-' + problem_name + '-' + 'dim' + str(dimension_size) + '-' + 'bias' \ + str(bias) + '-' + str(start_index + prob_i) + '.pkl' bias_log_file = exp_path + str(problem_name) + '/dimension' + str(dimension_size) + '/RecordLog/' + 'bias-' \ + problem_name + '-' + 'dim' + str(dimension_size) + '-' + 'bias' + str(bias) \ + '-' + str(start_index + prob_i) + '.txt' running_log_file = exp_path + str(problem_name) + '/dimension' + str(dimension_size) + '/RecordLog/' + \ 'running-' + problem_name + '-' + 'dim' + str(dimension_size) + '-' + 'bias' \ + str(bias) + '-' + str(start_index + prob_i) + '.txt' print('data logging: ', data_log_file) running_log.append('data log path: ' + data_log_file) save_log(positive_set, negative_set, new_sample_set, label_set, data_log_file) print('bias logging: ', bias_log_file) running_log.append('bias log path: ' + bias_log_file) fo.FileWriter(bias_log_file, bias_log, style='w') print('running logging: ', running_log_file) fo.FileWriter(running_log_file, running_log, style='w') return
def run(type): dimension = Dimension() dimension.set_dimension_size(dimension_size) dimension.set_regions([[-1.0, 1.0] for _ in range(dimension_size)], [0 for _ in range(dimension_size)]) # problem define func = DistributedFunction(dimension, bias_region=[-bias_region, bias_region]) target_bias = [0.1 for _ in range(dimension_size)] func.setBias(target_bias) if problem_name == 'ackley': prob_fct = func.DisAckley elif problem_name == 'sphere': prob_fct = func.DisSphere elif problem_name == 'rosenbrock': prob_fct = func.DisRosenbrock else: print('Wrong function!') exit() opt_error_list = [] log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('Running: ' + type) log_buffer.append('+++++++++++++++++++++++++++++++') print('+++++++++++++++++++++++++++++++') print('Running: ' + type) print('+++++++++++++++++++++++++++++++') if type == 'ada': # pre=sorted(predictors,key=lambda a:a.dist) expert = Experts(predictors=predictors, eta=eta, bg=budget) for i in range(opt_repeat): print('optimize ', i, '===================================================') log_buffer.append( 'optimize ' + str(i) + '===================================================') start_t = time.time() if type == 'exp': exp_racos = ExpRacosOptimization(dimension, nets) opt_error = exp_racos.exp_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) elif type == 'ada': exp_racos = ExpAdaRacosOptimization(dimension, expert) opt_error = exp_racos.exp_ada_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) else: print('Wrong type!') return end_t = time.time() hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) opt_mean = np.mean(np.array(opt_error_list), axis=0) opt_std = np.std(np.array(opt_error_list), axis=0) print('--------------------------------------------------') print('optimization result for ' + str(opt_repeat) + ' times average: ', opt_mean, ', standard variance is: ', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result for ' + str(opt_repeat) + ' times average: ' + str(opt_mean) + ', standard variance is: ' + str(opt_std)) return opt_mean
def run_racos(): # parameters sample_size = 10 # the instance number of sampling in an iteration budget = 500 # budget in online style positive_num = 2 # the set size of PosPop rand_probability = 0.99 # the probability of sample in model uncertain_bit = 1 # the dimension size that is sampled randomly bias_region = 0.5 repeat = 10 # dimension setting dimension_size = 10 dimension = Dimension() dimension.set_dimension_size(dimension_size) dimension.set_regions([[-1.0, 1.0] for _ in range(dimension_size)], [0 for _ in range(dimension_size)]) func = DistributedFunction(dim=dimension, bias_region=[-bias_region, bias_region]) if problem_name == 'rosenbrock': prob = func.DisRosenbrock else: prob = func.DisSphere # optimization racos = RacosOptimization(dimension) opt_error_list = [] for i in range(repeat): start_t = time.time() racos.mix_opt(prob, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit) end_t = time.time() optimal = racos.get_optimal() hour, minute, second = time_formulate(start_t, end_t) print('total budget is ', budget, '------------------------------') print('spending time: ', hour, ' hours ', minute, ' minutes ', second, ' seconds') print('optimal value: ', optimal.get_fitness()) opt_error = optimal.get_fitness() optimal_x = optimal.get_features() opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) print('optimal x: ', optimal_x) log_buffer.append('optimal nn structure: ' + list2string(optimal_x)) opt_mean = np.mean(np.array(opt_error_list)) opt_std = np.std(np.array(opt_error_list)) print('--------------------------------------------------') print('optimization result: ', opt_mean, '#', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result: ' + str(opt_mean) + '#' + str(opt_std)) return opt_mean
def learning_exp_ensemble(): random.seed(1) # training parameters epoch_size = 50 batch_size = 32 vali_rate = 0.1 learn_rate = 0.0005 categorical_size = 1 validation_switch = True # exp data parameters dim_size = 10 problem_name = 'sphere' start_index = 2000 bias_region = 0.5 problem_num = 2000 learner_path = path + '/ExpLearner/SyntheticProbsLearner/' data_path = path + '/ExpLog/SyntheticProbsLog/' log_buffer = [] log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('training parameter') log_buffer.append('epoch size: ' + str(epoch_size)) log_buffer.append('batch size: ' + str(batch_size)) log_buffer.append('validation rate: ' + str(vali_rate)) log_buffer.append('learning rate: ' + str(learn_rate)) log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('experience data parameter') log_buffer.append('dimension size: ' + str(dim_size)) log_buffer.append('problem name: ' + problem_name) log_buffer.append('problem index: ' + str(start_index + 1)) log_buffer.append('+++++++++++++++++++++++++++++++') log_name = learner_path + problem_name + '/dimension' + str(dim_size) + '/TrainingLog/' + 'learning-log-' \ + problem_name + '-' + 'dim' + str(dim_size) + '-' + 'bias' + str(bias_region) \ + '-' + str(start_index + 1) + '.txt' train_data = [] train_label = [] for prob_i in range(problem_num): data_file = data_path + problem_name + '/dimension' + str(dim_size) + '/LearningData/' + 'learning-data-' \ + problem_name + '-' + 'dim' + str(dim_size) + '-' + 'bias' + str(bias_region) + '-' \ + str(prob_i) + '.pkl' print('data loading: ', data_file) data_inf, bias, ori_data, blc_data = learning_data_load( file_path=data_file) train_data_, train_label_ = blc_data # test_data_, test_label_ = ori_data train_data.append(train_data_) train_label.append(train_label_) print('data transfer...') train_data = learning_data_transfer(instance_set=train_data) print('train data formulation: ', len(train_data), '*', len(train_data[0]), '*', len(train_data[0][0]), '*', len(train_data[0][0][0])) log_buffer.append('--' + 'train data formulation: ' + str(len(train_data)) + '*' + str(len(train_data[0])) + '*' + str(len(train_data[0][0])) + '*' + str(len(train_data[0][0][0])) + '--') print('train label size: ', len(train_label)) log_buffer.append('--' + 'train label size: ' + str(len(train_label))) # train new models print( 'training model...---------------------------------------------------------------------------' ) print('split train and validation data...') trainloader, validationloader = split_data([train_data, train_label], batch_size=batch_size, validation_rate=vali_rate) log_buffer.append('--split data: train data size: ' + str(len(trainloader)) + ' validation data size: ' + str(len(validationloader))) print( 'training net ================================================================' ) # data = mix_data(minority_data, majority_data[net_i]) net_start = time.time() net = ImageNet(middle_input_size=dim_size, output_size=categorical_size) net.cuda() # criterion = nn.CrossEntropyLoss() criterion = nn.BCELoss() optimizer = optim.Adam(net.parameters(), lr=learn_rate) log_buffer.append('criterion: BCELoss, optimizer: Adam') log_buffer.append('--net train--') for epoch in range(epoch_size): # loop over the dataset multiple times running_loss = 0.0 for i, data in enumerate(trainloader, 0): # get the inputs inputs, labels = data # wrap them in Variable inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda()) # zero the parameter gradients optimizer.zero_grad() # forward + backward + optimize outputs = net(inputs) labels = labels.float() # BCELoss used loss = criterion(outputs, labels) loss.backward() optimizer.step() # print statistics running_loss += loss.item() if i % 50 == 49: # print every 2000 mini-batches print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 50)) log_buffer.append('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 50)) running_loss = 0.0 # validation if validation_switch is True: all_predictions, all_labels = [], [] for data in validationloader: images, labels = data images = Variable(images.cuda()) outputs = net(images) outputs = outputs.cpu() # for BCELoss predicted = (torch.sign(outputs * 2 - 1) + 1) / 2 predicted = predicted.data.numpy() predicted = predicted.reshape( predicted.size).astype(int).tolist() all_predictions.extend(predicted) all_labels.extend(labels.numpy().tolist()) accuracy = accuracy_score(all_labels, all_predictions) recall = recall_score(all_labels, all_predictions) precision = precision_score(all_labels, all_predictions) print('accuracy: ', accuracy, ', recall rate: ', recall, ', precision rate: ', precision) log_buffer.append('accuracy: ' + str(accuracy) + ', recall rate: ' + str(recall) + ', precision rate: ' + str(precision)) net_end = time.time() hour, minute, second = time_formulate(net_start, net_end) print('train net time: ', hour, ':', minute, ':', second) log_buffer.append('train net time: ' + str(hour) + ':' + str(minute) + ':' + str(second)) net_file = learner_path + problem_name + '/dimension' + str(dim_size) + '/DirectionalModel/' + 'learner-' \ + problem_name + '-' + 'dim' + str(dim_size) + '-' + 'bias' + str(bias_region) \ + '-' + str(start_index + 1) + 'alldata.pkl' print('net saving...') torch.save(net, net_file) log_buffer.append('--net save: ' + net_file + '--') print('net saved!') fo.FileWriter(log_name, log_buffer, style='w') return
def run_for_synthetic_problem(): sample_size = 10 # the instance number of sampling in an iteration budget = 50 # budget in online style positive_num = 2 # the set size of PosPop rand_probability = 0.99 # the probability of sample in model uncertain_bit = 1 # the dimension size that is sampled randomly adv_threshold = 10 # advance sample size opt_repeat = 10 dimension_size = 10 problem_name = 'sphere' bias_region = 0.5 eta = 0.9 step = 100 dimension = Dimension() dimension.set_dimension_size(dimension_size) dimension.set_regions([[-1.0, 1.0] for _ in range(dimension_size)], [0 for _ in range(dimension_size)]) log_buffer = [] # problem define func = DistributedFunction(dimension, bias_region=[-0.5, 0.5]) target_bias = [0.2 for _ in range(dimension_size)] func.setBias(target_bias) if problem_name == 'ackley': prob_fct = func.DisAckley else: prob_fct = func.DisSphere log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('optimization parameters') log_buffer.append('sample size: ' + str(sample_size)) log_buffer.append('budget: ' + str(budget)) log_buffer.append('positive num: ' + str(positive_num)) log_buffer.append('random probability: ' + str(rand_probability)) log_buffer.append('uncertain bits: ' + str(uncertain_bit)) log_buffer.append('advance num: ' + str(adv_threshold)) log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('problem parameters') log_buffer.append('dimension size: ' + str(dimension_size)) log_buffer.append('problem name: ' + problem_name) log_buffer.append('bias: ' + list2string(target_bias)) log_buffer.append('+++++++++++++++++++++++++++++++') predictors, load_buffer = get_predicotrs() expert = Experts(predictors=predictors, eta=eta, step=step) log_buffer.extend(load_buffer) opt_error_list = [] for i in range(opt_repeat): print('optimize ', i, '===================================================') log_buffer.append( 'optimize ' + str(i) + '===================================================') exp_racos = ExpAdaRacosOptimization(dimension, expert) start_t = time.time() exp_racos.exp_ada_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) end_t = time.time() print('total budget is ', budget) log_buffer.append('total budget is ' + str(budget)) hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) optimal = exp_racos.get_optimal() opt_error = optimal.get_fitness() optimal_x = optimal.get_features() opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) print('optimal x: ', optimal_x) log_buffer.append('optimal nn structure: ' + list2string(optimal_x)) opt_mean = np.mean(np.array(opt_error_list)) opt_std = np.std(np.array(opt_error_list)) print('--------------------------------------------------') print('optimization result: ', opt_mean, '#', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result: ' + str(opt_mean) + '#' + str(opt_std)) result_path = path + '/Results/Ada/' + problem_name + '/dimension' + str( dimension_size) + '/' optimization_log_file = result_path + 'opt-log-' + problem_name + '-dim' + str(dimension_size) + '-bias' \ + str(bias_region) + '.txt' print('optimization logging: ', optimization_log_file) fo.FileWriter(optimization_log_file, log_buffer, style='w') return
def run(type): opt_error_list = [] log_buffer.append('+++++++++++++++++++++++++++++++') log_buffer.append('Running: ' + type) log_buffer.append('+++++++++++++++++++++++++++++++') print('+++++++++++++++++++++++++++++++') print('Running: ' + type) print('+++++++++++++++++++++++++++++++') if type == 'ada': # pre=sorted(predictors,key=lambda a:a.dist) expert = Experts(predictors=predictors, eta=eta, bg=budget) for i in range(opt_repeat): print('optimize ', i, '===================================================') log_buffer.append( 'optimize ' + str(i) + '===================================================') start_t = time.time() if type == 'ave': exp_racos = ExpRacosOptimization(dimension, nets) opt_error = exp_racos.exp_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) elif type == 'ada': exp_racos = ExpAdaRacosOptimization(dimension, expert) opt_error = exp_racos.exp_ada_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold, step=step) elif type == 'ground truth': exp_racos = ExpRacosOptimization(dimension, nets[:step]) exp_racos.exp_mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit, at=adv_threshold) else: print('Wrong type!') return end_t = time.time() hour, minute, second = time_formulate(start_t, end_t) print('spending time: ', hour, ':', minute, ':', second) log_buffer.append('spending time: ' + str(hour) + '+' + str(minute) + '+' + str(second)) optimal = exp_racos.get_optimal() opt_error = optimal.get_fitness() optimal_x = optimal.get_features() opt_error_list.append(opt_error) print('validation optimal value: ', opt_error) log_buffer.append('validation optimal value: ' + str(opt_error)) print('optimal x: ', optimal_x) log_buffer.append('optimal nn structure: ' + list2string(optimal_x)) opt_mean = np.mean(np.array(opt_error_list), axis=0) opt_std = np.std(np.array(opt_error_list), axis=0) print('--------------------------------------------------') print('optimization result for ' + str(opt_repeat) + ' times average: ', opt_mean, ', standard variance is: ', opt_std) log_buffer.append('--------------------------------------------------') log_buffer.append('optimization result for ' + str(opt_repeat) + ' times average: ' + str(opt_mean) + ', standard variance is: ' + str(opt_std)) return opt_mean, opt_std
def synthetic_problems_sample(prob_i): # bias log format: 'index,bias_list: dim1 dim2 dim3...' bias_log = [] running_log = [] running_log.append('+++++++++++++++++++++++++++++++++') running_log.append('optimization setting: ') running_log.append('sample_size: ' + str(sample_size)) running_log.append('positive_num: ' + str(positive_num)) running_log.append('rand_probability: ' + str(rand_probability)) running_log.append('uncertain_bits: ' + str(uncertain_bits)) running_log.append('budget: ' + str(budget)) running_log.append('+++++++++++++++++++++++++++++++++') print(problem_name, ': ', start_index + prob_i, ' ==============================================') running_log.append(problem_name + ': ' + str(start_index + prob_i) + ' ==============================================') # bias setting group_num = 10 group_size = problem_num / group_num bias_step = bias_region / group_num new_bias_region = int(prob_i / group_size) * bias_step # problem setting func = DistributedFunction(dim=dimension, bias_region=[-new_bias_region, new_bias_region]) if 'ackley' in problem_name: prob_fct = func.DisAckley elif 'sphere' in problem_name: prob_fct = func.DisSphere elif 'rosenbrock' in problem_name: prob_fct = func.DisRosenbrock else: print('Wrong Function!') exit() # bias log bias_log.append( str(prob_i + start_index) + ',' + list2string(func.getBias())) print('function: ', problem_name, ', this bias: ', func.getBias()) running_log.append('function: ' + problem_name + ', this bias: ' + list2string(func.getBias())) # optimization setting optimizer = RacosOptimization(dimension) positive_set = [] negative_set = [] new_sample_set = [] label_set = [] for repeat_i in range(repeat_num): print('repeat ', repeat_i, ' ----------------------------------------') running_log.append('repeat ' + str(repeat_i) + ' ----------------------------------------') # optimization process start_t = time.time() optimizer.mix_opt(obj_fct=prob_fct, ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bits) end_t = time.time() hour, minute, second = time_formulate(start_t, end_t) # optimization results optimal = optimizer.get_optimal() print('optimal v: ', optimal.get_fitness(), ' - ', optimal.get_features()) running_log.append('optimal v: ' + str(optimal.get_fitness()) + ' - ' + list2string(optimal.get_features())) print('spent time: ', hour, ':', minute, ':', second) running_log.append('spent time: ' + str(hour) + ':' + str(minute) + ':' + str(second)) # log samples this_positive, this_negative, this_new, this_label = optimizer.get_log( ) print('sample number: ', len(this_positive), ':', len(this_label)) running_log.append('sample number: ' + str(len(this_positive)) + ':' + str(len(this_label))) positive_set.extend(this_positive) negative_set.extend(this_negative) new_sample_set.extend(this_new) label_set.extend(this_label) print('----------------------------------------------') print('sample finish!') print('all sample number: ', len(positive_set), '-', len(negative_set), '-', len(new_sample_set), \ '-', len(label_set)) running_log.append('----------------------------------------------') running_log.append('all sample number: ' + str(len(positive_set)) + '-' + str(len(negative_set)) + '-' + str(len(new_sample_set)) + '-' + str(len(label_set))) data_log_file = exp_path + str(problem_name) + '/dimension' + str(dimension_size) + '/DataLog/' + \ 'data-' + problem_name + '-' + 'dim' + str(dimension_size) + '-' + 'bias' \ + str(bias_region) + '-' + str(start_index + prob_i) + '.pkl' bias_log_file = exp_path + str(problem_name) + '/dimension' + str(dimension_size) + '/RecordLog/' + 'bias-' \ + problem_name + '-' + 'dim' + str(dimension_size) + '-' + 'bias' + str(bias_region) \ + '-' + str(start_index + prob_i) + '.txt' running_log_file = exp_path + str(problem_name) + '/dimension' + str(dimension_size) + '/RecordLog/' + \ 'running-' + problem_name + '-' + 'dim' + str(dimension_size) + '-' + 'bias' \ + str(bias_region) + '-' + str(start_index + prob_i) + '.txt' print('data logging: ', data_log_file) running_log.append('data log path: ' + data_log_file) save_log(positive_set, negative_set, new_sample_set, label_set, data_log_file) print('bias logging: ', bias_log_file) running_log.append('bias log path: ' + bias_log_file) fo.FileWriter(bias_log_file, bias_log, style='w') print('running logging: ', running_log_file) fo.FileWriter(running_log_file, running_log, style='w') return