def run(): print("starting") eqpy.OUT_put("1") result = eqpy.IN_get() eqpy.OUT_put("33" + 1) result = eqpy.IN_get() eqpy.OUT_put("FINAL")
def run(): # handshake to ensure working eqpy.OUT_put("Params") # initial parameter set telling us the number of times to run the loop initparams = eqpy.IN_get() (num_trials, num_points) = eval('{}'.format(initparams)) # going to have to run 1 more than number of requested loops to record the final loop's returned data points loops = num_trials + 1 for i in range(0, loops): if i > 0: params = eqpy.IN_get() expt_dir = os.getcwd() home_dir = os.path.join(os.path.split(expt_dir)[0], 'data') # received string of results, need to put it into the shared file params = params.split(";") res_file = os.path.join(home_dir, 'results.dat') resfile = open(res_file, 'a+') temp = resfile.readlines() resfile.close() for b in range(0, len(params)): # solved data set comes in form "answer 0 X-values; answer 0 X-values..." # split these up to replace each one in our shared file l = params[b].split() # first need to find the pending rows in shared file list1 = l[2:] X = [] X.append("P P " + ' '.join(map(str, list1))) # replacing every solved data set in our shared folder temp = [w.replace(str(X[0]), params[b]) for w in temp] #update the file with the newly entered solved pieces resfile = open(res_file, 'w') resfile.writelines(temp) resfile.close() #if we haven't met all of the required loops, we run the Simulator #if we have met all the required loops, we want to fill in the final set but not run the Simulator again if i < num_trials: print("running loop number %d of %d" % (i + 1, num_trials)) queue = main_controller( { 'config_file': 'config.json', 'grid_size': 20000, 'max_finished_jobs': 1000, 'chooser_args': "", 'results_file': 'results.dat', 'num_jobs': num_points, 'chooser_module': 'GPEIOptChooser2', 'grid_seed': 1 }, ['']) # Send out the pending list of data to be modeled eqpy.OUT_put(queue) #finished all of the loops, let EMEWS know to stop eqpy.OUT_put("DONE") #let the user know where to look for the results of this EMEWS project eqpy.OUT_put("Refer to results.dat file in data directory")
def run(): eqpy.OUT_put("Hello From Algorithm") params = eqpy.IN_get() print("Params: {}".format(params)) eqpy.OUT_put("1;2;3") result = eqpy.IN_get() eqpy.OUT_put("33;34;35" + 1) result = eqpy.IN_get() eqpy.OUT_put("DONE") eqpy.OUT_put("results")
def run(): eqpy.OUT_put('params') cfg_file = eqpy.IN_get() params = [{'a': 1}, {'a': 2}] eqpy.OUT_put(json.dumps(params)) r = eqpy.IN_get() print(r, flush=True) params = [{'a': 3}, {'a': 4}] eqpy.OUT_put(json.dumps(params)) r = eqpy.IN_get() print(r, flush=True) eqpy.OUT_put("DONE") eqpy.OUT_put("BYE")
def run(): # my swift-t MPI comm rank, and destination rank for cache_comm rank = eqpy.IN_get() #printf("AL Start on {}".format(rank)) param = eqpy.IN_get() for _ in range(10): op = [param] * 5 ps = ";".join(op) eqpy.OUT_put(ps) result = eqpy.IN_get() eqpy.OUT_put("DONE") eqpy.OUT_put("42") data = {'msg': 'put', 'rank': rank} cache_comm.send(data, dest=0, tag=1)
def eqpy_func(params): retvals = [] # unpack and send to out out_params = ";".join([str(p) for p in params]) eqpy.OUT_put(out_params) # get result and format for hyperopt result = eqpy.IN_get() split_result = result.split(",") return [{'loss': float(x), 'status': base.STATUS_OK} for x in split_result]
def queue_map(obj_func, pops): """ Note that the obj_func is a dummy pops: data that looks like: [[x1,x2],[x1,x2],...] """ if not pops: return [] eqpy.OUT_put(create_list_of_lists_string(pops)) result = eqpy.IN_get() split_result = result.split(';') return [(float(x), ) for x in split_result]
def init(): global cache_comm ranks_str = eqpy.IN_get() ranks = ranks_str.split(',')[1:] #print(ranks) if cache_comm == None: comm = MPI.COMM_WORLD group = comm.Get_group() cache_group = group.Incl([int(x) for x in ranks]) #printf("ME newgroup size is {}".format(cache_group.size)) cache_comm = comm.Create_group(cache_group, 1)
def run(): """ :param num_iter: number of generations :param num_pop: size of population :param seed: random seed :param csv_file_name: csv file name (e.g., "params_for_deap.csv") """ eqpy.OUT_put("Settings") settings_filename = eqpy.IN_get() load_settings(settings_filename) # parse settings # num_iter, num_pop, seed, creator.create("FitnessMin", base.Fitness, weights=(-1.0, )) creator.create("Individual", list, fitness=creator.FitnessMin) toolbox = base.Toolbox() toolbox.register("attr_float", random.random) # toolbox.register("individual", tools.initRepeat, creator.Individual, # toolbox.attr_float, n=2) toolbox.register("individual", tools.initIterate, creator.Individual, make_random_params) toolbox.register("population", tools.initRepeat, list, toolbox.individual) toolbox.register("evaluate", obj_func) toolbox.register("mate", cxUniform, indpb=mate_pb) toolbox.register("mutate", custom_mutate, indpb=mutate_pb) toolbox.register("select", tools.selTournament, tournsize=num_pop / 2) toolbox.register("map", queue_map) pop = toolbox.population(n=num_pop) hof = tools.HallOfFame(2) stats = tools.Statistics(lambda ind: ind.fitness.values) stats.register("avg", np.mean) stats.register("std", np.std) stats.register("min", np.min) stats.register("max", np.max) # num_iter-1 generations since the initial population is evaluated once first pop, log = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=mutate_pb, ngen=num_iter - 1, stats=stats, halloffame=hof, verbose=True) fitnesses = [str(p.fitness.values[0]) for p in pop] eqpy.OUT_put("FINAL") # return the final population eqpy.OUT_put("{0}\n{1}\n{2}".format(create_list_of_lists_string(pop), ';'.join(fitnesses), log))
def run(): # gets dummy params for this me params = eqpy.IN_get() print("Params: {}".format(params)) for _ in range(10): op = [] for _ in range(5): p = "{},{},{},{}".format(random.randint(1, 10), random.randint(1, 10), random.randint(1, 10), random.randint(1, 10)) op.append(p) ps = ";".join(op) eqpy.OUT_put(ps) # wait to get result back eqpy.IN_get() eqpy.OUT_put("DONE") eqpy.OUT_put("final result")
def queue_map(obj_func, pops): # Note that the obj_func is not used # sending data that looks like: # [[a,b,c,d],[e,f,g,h],...] if not pops: return [] eqpy.OUT_put(create_list_of_json_strings(pops)) result = eqpy.IN_get() split_result = result.split(';') # TODO determine if max'ing or min'ing and use -9999999 or 99999999 return [(float(x), ) if not math.isnan(float(x)) else (float(99999999), ) for x in split_result]
def run(): """run function for eqpy based run""" eqpy.OUT_put("") # params should be formatted as a dictionary hp_params = eqpy.IN_get() hp_dict = eval(hp_params) trials = base.Trials() rstate = None if 'seed' in hp_dict: rstate = np.random.RandomState(hp_dict['seed']) fmin(eqpy_func, hp_dict['space'], hp_dict['algo'], hp_dict['max_evals'], hp_dict['param_batch_size'], trials, rstate) eqpy.OUT_put("FINAL") eqpy.OUT_put(str(trials.argmin))
def measure_perf(conf_df): conf_colns = conf_df.columns.tolist() app_name = data.get_name(conf_colns) data_df = data.csv2df(app_name + "_time.csv", conf_colns) conf_perf_df = tool.df_intersection(data_df, conf_df, conf_colns) new_conf_df = tool.df_sub(conf_df, conf_perf_df, conf_colns) if (new_conf_df.shape[0] > 0): new_conf_df = new_conf_df.astype(int) eqpy.OUT_put(app_name) eqpy.OUT_put(data.df2string(new_conf_df)) result = eqpy.IN_get() time_df = data.string2df(result, ['run_time']) new_conf_perf_df = pd.concat([new_conf_df, time_df], axis=1) data.df2csv(new_conf_perf_df, app_name + "_time_new.csv") conf_perf_df = tool.df_union(conf_perf_df, new_conf_perf_df) conf_perf_df = data.get_exec_mach_df(data.get_runnable_df(conf_perf_df, conf_colns)) if (conf_df.shape[0] != conf_perf_df.shape[0]): print "Error: conf_df.shape[0] != conf_perf_df.shape[0]", conf_df.shape[0], conf_perf_df.shape[0] return conf_perf_df
def run(): ranks_str = eqpy.IN_get() ranks = ranks_str.split(',') # include only the al ranks task_ranks = ranks[2:] for r in task_ranks: eqpy.OUT_put(r) # include self and tasks in comm comm = init_comm(ranks[1:]) rank = comm.rank #printf("task cache rank: {}".format(rank)) while True: status = MPI.Status() data = comm.recv(source=MPI.ANY_SOURCE, status=status) msg = data['msg'] if msg == 'put': # this is its rank in the swift mpi communicator eqpy.OUT_put(data['rank']) elif msg == 'DONE': break
def init(): eqpy.OUT_put("Settings") settings_filename = eqpy.IN_get() load_settings(settings_filename)
def run(): """ :param num_iter: number of generations :param num_pop: size of population :param seed: random seed :param strategy: one of 'simple', 'mu_plus_lambda' :param ga parameters file name: ga parameters file name (e.g., "ga_params.json") :param param_file: name of file containing initial parameters """ eqpy.OUT_put("Params") params = eqpy.IN_get() # parse params printf("Parameters: {}".format(params)) (num_iter, num_pop, seed, strategy, mut_prob, ga_params_file, param_file, classifer_path, scaler_path) = eval('{}'.format(params)) random.seed(seed) ga_params = ga_utils.create_parameters(ga_params_file) create_transformer(ga_params, classifer_path, scaler_path) creator.create("FitnessMin", base.Fitness, weights=(-1.0, )) creator.create("Individual", list, fitness=creator.FitnessMin) toolbox = base.Toolbox() toolbox.register("individual", tools.initIterate, creator.Individual, make_random_params) toolbox.register("population", tools.initRepeat, list, toolbox.individual) toolbox.register("evaluate", obj_func) toolbox.register("mate", cxUniform, indpb=0.5) mutate_indpb = mut_prob toolbox.register("mutate", custom_mutate, indpb=mutate_indpb) toolbox.register("select", tools.selTournament, tournsize=3) toolbox.register("map", queue_map) pop = toolbox.population(n=num_pop) if param_file != "": update_init_pop(pop, param_file) hof = tools.HallOfFame(1) stats = tools.Statistics(lambda ind: ind.fitness.values) stats.register("avg", sts.mean) stats.register("std", sts.std) stats.register("min", sts.min) stats.register("max", sts.max) stats.register("ts", timestamp) # num_iter-1 generations since the initial population is evaluated once first mutpb = mut_prob start_time = time.time() if strategy == 'simple': pop, log = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=mutpb, ngen=num_iter - 1, stats=stats, halloffame=hof, verbose=True) elif strategy == 'mu_plus_lambda': mu = int(math.floor(float(num_pop) * 0.5)) lam = int(math.floor(float(num_pop) * 0.5)) if mu + lam < num_pop: mu += num_pop - (mu + lam) pop, log = algorithms.eaMuPlusLambda(pop, toolbox, mu=mu, lambda_=lam, cxpb=0.5, mutpb=mutpb, ngen=num_iter - 1, stats=stats, halloffame=hof, verbose=True) else: raise NameError('invalid strategy: {}'.format(strategy)) end_time = time.time() fitnesses = [str(p.fitness.values[0]) for p in pop] eqpy.OUT_put("DONE") # return the final population eqpy.OUT_put("{}\n{}\n{}\n{}\n{}".format(create_list_of_json_strings(pop), ';'.join(fitnesses), start_time, log, end_time))
def run(): start_time = time.time() print("run() start: {}".format(str(datetime.datetime.now()))) comm = MPI.COMM_WORLD # get MPI communicator object size = comm.size # total number of processes rank = comm.rank # rank of this process status = MPI.Status() # get MPI status object print("ME rank is {}".format(rank)) instance = problem.Problem() spaceDict = instance.space params = instance.params global problem_params problem_params = params starting_point = instance.starting_point # handshake to ensure working eqpy.OUT_put("Params") # initial parameter set telling us the number of times to run the loop initparams = eqpy.IN_get() (init_size, max_evals, num_workers, num_buffer, seed, max_threshold, n_jobs) = eval('{}'.format(initparams)) space = [spaceDict[key] for key in params] print(space) parDict = {} resultsList = [] parDict['kappa'] = 1.96 # can set to num cores parDict['n_jobs'] = n_jobs init_x = [] opt = Optimizer(space, base_estimator='RF', acq_optimizer='sampling', acq_func='LCB', acq_func_kwargs=parDict, random_state=seed) eval_counter = 0 askedDict = {} print( "Master starting with {} init_size, {} max_evals, {} num_workers, {} num_buffer, {} max_threshold" .format(init_size, max_evals, num_workers, num_buffer, max_threshold)) x = opt.ask(n_points=init_size) res, resstring = create_list_of_json_strings(x) print("Initial design is {}".format(resstring)) for r, xx in zip(res, x): askedDict[r] = xx eqpy.OUT_put(resstring) currently_out = init_size total_out = init_size results = [] group = comm.Get_group() # Assumes only one adlb_server # num_workers + 1 = num_turbine_workers newgroup = group.Excl([num_workers + 1]) #print("ME newgroup size is {}".format(newgroup.size)) newcomm = comm.Create_group(newgroup, 1) nrank = newcomm.rank #print("ME nrank is {}".format(nrank)) counter_threshold = 1 counter = 0 end_iter_time = 0 while eval_counter < max_evals: start_iter_time = time.time() print("\neval_counter = {}".format(eval_counter)) data = newcomm.recv(source=MPI.ANY_SOURCE, status=status) counter = counter + 1 xstring = data['x'] x = askedDict[xstring] y = data['cost'] if math.isnan(y): y = sys.float_info.max opt.tell(x, y) #source = status.Get_source() #tag = status.Get_tag() elapsed_time = float(time.time() - start_time) print('elapsed_time:%1.3f' % elapsed_time) results.append(str(data)) eval_counter = eval_counter + 1 currently_out = currently_out - 1 # if jobs are finishing within 16 seconds of # each other, then batch the point production if start_iter_time - end_iter_time < 16: counter_threshold = max_threshold if max_evals - eval_counter < counter_threshold: counter_threshold = max_evals - eval_counter if counter_threshold > currently_out: counter_threshold = currently_out else: counter_threshold = 1 print("counter_threshold: {}".format(counter_threshold)) print("currently_out:{}, total_out:{}".format(currently_out, total_out)) if currently_out < num_workers + num_buffer and total_out < max_evals and counter >= counter_threshold: n_points = counter if n_points + total_out > max_evals: n_points = max_evals - total_out ts = time.time() x = opt.ask(n_points=n_points) res, resstring = create_list_of_json_strings(x) for r, xx in zip(res, x): askedDict[r] = xx eqpy.OUT_put(resstring) print('point production elapsed_time:%1.3f' % float(time.time() - ts)) currently_out = currently_out + n_points total_out = total_out + n_points counter = 0 end_iter_time = start_iter_time print('Search finishing') eqpy.OUT_put("DONE") eqpy.OUT_put(";".join(results))
def run(): """ :param num_iterations: number of generations :param seed: random seed :param ga parameters file name: ga parameters file name (e.g., "ga_params.json") :param num_population population of ga algorithm """ eqpy.OUT_put("Params") parameters = eqpy.IN_get() # parse params printf("Parameters: {}".format(parameters)) (num_iterations, num_population, seed, ga_parameters_file) = eval('{}'.format(parameters)) random.seed(seed) ga_parameters = ga_utils.create_parameters(ga_parameters_file) global transformer transformer = Transformer(ga_parameters) # deap class creators creator.create("FitnessMin", base.Fitness, weights=(-1.0, )) creator.create("Individual", list, fitness=creator.FitnessMin) # deap method definitions toolbox = base.Toolbox() toolbox.register("individual", tools.initIterate, creator.Individual, make_random_parameters) toolbox.register("population", tools.initRepeat, list, toolbox.individual) toolbox.register("evaluate", obj_func) toolbox.register("mate", cxUniform, indpb=0.5) toolbox.register("mutate", custom_mutate, indpb=0.2) toolbox.register("select", tools.selTournament, tournsize=3) toolbox.register("map", queue_map) pop = toolbox.population(n=num_population) hof = tools.HallOfFame(1) stats = tools.Statistics(lambda ind: ind.fitness.values) stats.register("avg", np.mean) stats.register("std", np.std) stats.register("min", np.min) stats.register("max", np.max) stats.register("ts", timestamp) # num_iter-1 generations since the initial population is evaluated once first start_time = time.time() pop, log = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=0.2, ngen=num_iterations - 1, stats=stats, halloffame=hof, verbose=True) end_time = time.time() fitnesses = [str(p.fitness.values[0]) for p in pop] eqpy.OUT_put("DONE") # return the final population eqpy.OUT_put("{}\n{}\n{}\n{}\n{}".format(create_list_of_json_strings(pop), ';'.join(fitnesses), start_time, log, end_time))