#return outputval return tuple #define the search space. objective = obj_func('./all-cnn_bi_msphere.py') real_space = ContinuousSpace([0.0, 4.0], 'real_space') * 5 integer_space = OrdinalSpace([0, 4], 'integer_space') * 5 discrete_space = NominalSpace(['0', '1', '2', '3', '4'], 'discrete_space') * 5 search_space = real_space * integer_space * discrete_space print('starting program...') #available_gpus = gp.getAvailable(limit=2) available_gpus = gp.getAvailable(limit=5) #try: #available_gpus.remove(0)#CHRIS gpu 0 and 5 are differen gpu types on duranium since they are faster, timing will be unreliable, so remove them from list #except: #pass #try: #available_gpus.remove(5) #except: #pass print(available_gpus) n_job = max(min(5, len(available_gpus)), 1) # use random forest as the surrogate model #CHRIS two surrogate models are needed model = RandomForest(levels=search_space.levels, n_estimators=100)
def _eval_gpu(self, x, gpu=0, runs=1): """ evaluate one solution """ with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 1\n') # TODO: sometimes the obj_func take a dictionary as input... time_,loss_, n_eval = x.time,x.loss, x.n_eval with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 2\n') # try: # ans = [self.obj_func(x.tolist()) for i in range(runs)] # except: #ans = [self.obj_func(x.to_dict(), gpu_no=gpu) for i in range(runs)] gpu_patch = gpu while True: with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 3\n') ans = self.obj_func(x.to_dict(), gpu_no=gpu_patch,eval_epochs=self.eval_epochs,save_name=self.save_name,data_augmentation=self.data_augmentation,use_validation=self.use_validation) with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 4\n') print("n_left,max_iter:") print(self.n_left,self.max_iter) print('_eval_gpu():') print(ans) time_ans,loss_ans,success= ans[0],ans[1],ans[2] with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 5\n') if success: with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 6a\n') break else: while True: with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 6b\n') print('gpu ' + str(gpu_patch) + ' failed to give answer, searching for new gpu') available_gpus_patch = gp.getAvailable(limit=5) with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 6b2\n') for i in range(len(self.ignore_gpu)): try: available_gpus_patch.remove(self.ignore_gpu[i]) except: pass with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 6b3\n') if len(available_gpus_patch) > 0: with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 6b4a\n') gpu_patch = available_gpus_patch[0] break else: with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 6b4b\n') print('no gpus available, waiting 60 seconds') time.sleep(60) with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 7\n') #TODO_CHRIS make this work when runs != 1 #time_ans = [] #loss_ans = [] #for i in range(len(ans)): # time_ans.append(ans[i][0]) # los_ans.append(ans[i][1]) #TODO_CHRIS apply S-metric to all solutions to get fitness #so here take average of loss and time time_loc = np.sum(time_ans) loss_loc = np.sum(loss_ans) #fitness = np.sum(ans)#CHRIS removed, because fitness will be determined later x.n_eval += runs x.time = time_loc / runs if time_ is None else (time_ * n_eval + time_loc) / x.n_eval x.loss = loss_loc / runs if loss_ is None else (loss_ * n_eval + loss_loc) / x.n_eval #self.eval_count += runs#CHRIS no double counting self.eval_hist_loss += [loss_ans] #CHRIS added time and loss history self.eval_hist_time += [time_ans] self.eval_hist_id += [x.index] * runs with open(self.save_name + '_thread_log.json', 'a') as outfile: outfile.write('thread ' + str(gpu) + ': step 3 gpu 7\n') return x, runs, time_loc, loss_loc, [x.index] * runs