def run():
print("starting")
eqpy.OUT_put("1")
result = eqpy.IN_get()
eqpy.OUT_put("33" + 1)
result = eqpy.IN_get()
eqpy.OUT_put("FINAL")
def run():
# handshake to ensure working
eqpy.OUT_put("Params")
# initial parameter set telling us the number of times to run the loop
initparams = eqpy.IN_get()
(num_trials, num_points) = eval('{}'.format(initparams))
# going to have to run 1 more than number of requested loops to record the final loop's returned data points
loops = num_trials + 1
for i in range(0, loops):
if i > 0:
params = eqpy.IN_get()
expt_dir = os.getcwd()
home_dir = os.path.join(os.path.split(expt_dir)[0], 'data')
# received string of results, need to put it into the shared file
params = params.split(";")
res_file = os.path.join(home_dir, 'results.dat')
resfile = open(res_file, 'a+')
temp = resfile.readlines()
resfile.close()
for b in range(0, len(params)):
# solved data set comes in form "answer 0 X-values; answer 0 X-values..."
# split these up to replace each one in our shared file
l = params[b].split()
# first need to find the pending rows in shared file
list1 = l[2:]
X = []
X.append("P P " + ' '.join(map(str, list1)))
# replacing every solved data set in our shared folder
temp = [w.replace(str(X[0]), params[b]) for w in temp]
#update the file with the newly entered solved pieces
resfile = open(res_file, 'w')
resfile.writelines(temp)
resfile.close()
#if we haven't met all of the required loops, we run the Simulator
#if we have met all the required loops, we want to fill in the final set but not run the Simulator again
if i < num_trials:
print("running loop number %d of %d" % (i + 1, num_trials))
queue = main_controller(
{
'config_file': 'config.json',
'grid_size': 20000,
'max_finished_jobs': 1000,
'chooser_args': "",
'results_file': 'results.dat',
'num_jobs': num_points,
'chooser_module': 'GPEIOptChooser2',
'grid_seed': 1
}, [''])
# Send out the pending list of data to be modeled
eqpy.OUT_put(queue)
#finished all of the loops, let EMEWS know to stop
eqpy.OUT_put("DONE")
#let the user know where to look for the results of this EMEWS project
eqpy.OUT_put("Refer to results.dat file in data directory")
def run():
eqpy.OUT_put("Hello From Algorithm")
params = eqpy.IN_get()
print("Params: {}".format(params))
eqpy.OUT_put("1;2;3")
result = eqpy.IN_get()
eqpy.OUT_put("33;34;35" + 1)
result = eqpy.IN_get()
eqpy.OUT_put("DONE")
eqpy.OUT_put("results")
def run():
eqpy.OUT_put('params')
cfg_file = eqpy.IN_get()
params = [{'a': 1}, {'a': 2}]
eqpy.OUT_put(json.dumps(params))
r = eqpy.IN_get()
print(r, flush=True)
params = [{'a': 3}, {'a': 4}]
eqpy.OUT_put(json.dumps(params))
r = eqpy.IN_get()
print(r, flush=True)
eqpy.OUT_put("DONE")
eqpy.OUT_put("BYE")
def run():
# my swift-t MPI comm rank, and destination rank for cache_comm
rank = eqpy.IN_get()
#printf("AL Start on {}".format(rank))
param = eqpy.IN_get()
for _ in range(10):
op = [param] * 5
ps = ";".join(op)
eqpy.OUT_put(ps)
result = eqpy.IN_get()
eqpy.OUT_put("DONE")
eqpy.OUT_put("42")
data = {'msg': 'put', 'rank': rank}
cache_comm.send(data, dest=0, tag=1)
def eqpy_func(params):
retvals = []
# unpack and send to out
out_params = ";".join([str(p) for p in params])
eqpy.OUT_put(out_params)
# get result and format for hyperopt
result = eqpy.IN_get()
split_result = result.split(",")
return [{'loss': float(x), 'status': base.STATUS_OK} for x in split_result]
def queue_map(obj_func, pops):
""" Note that the obj_func is a dummy
pops: data that looks like: [[x1,x2],[x1,x2],...]
"""
if not pops:
return []
eqpy.OUT_put(create_list_of_lists_string(pops))
result = eqpy.IN_get()
split_result = result.split(';')
return [(float(x), ) for x in split_result]
def init():
global cache_comm
ranks_str = eqpy.IN_get()
ranks = ranks_str.split(',')[1:]
#print(ranks)
if cache_comm == None:
comm = MPI.COMM_WORLD
group = comm.Get_group()
cache_group = group.Incl([int(x) for x in ranks])
#printf("ME newgroup size is {}".format(cache_group.size))
cache_comm = comm.Create_group(cache_group, 1)
def run():
"""
:param num_iter: number of generations
:param num_pop: size of population
:param seed: random seed
:param csv_file_name: csv file name (e.g., "params_for_deap.csv")
"""
eqpy.OUT_put("Settings")
settings_filename = eqpy.IN_get()
load_settings(settings_filename)
# parse settings # num_iter, num_pop, seed,
creator.create("FitnessMin", base.Fitness, weights=(-1.0, ))
creator.create("Individual", list, fitness=creator.FitnessMin)
toolbox = base.Toolbox()
toolbox.register("attr_float", random.random)
# toolbox.register("individual", tools.initRepeat, creator.Individual,
# toolbox.attr_float, n=2)
toolbox.register("individual", tools.initIterate, creator.Individual,
make_random_params)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("evaluate", obj_func)
toolbox.register("mate", cxUniform, indpb=mate_pb)
toolbox.register("mutate", custom_mutate, indpb=mutate_pb)
toolbox.register("select", tools.selTournament, tournsize=num_pop / 2)
toolbox.register("map", queue_map)
pop = toolbox.population(n=num_pop)
hof = tools.HallOfFame(2)
stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("avg", np.mean)
stats.register("std", np.std)
stats.register("min", np.min)
stats.register("max", np.max)
# num_iter-1 generations since the initial population is evaluated once first
pop, log = algorithms.eaSimple(pop,
toolbox,
cxpb=0.5,
mutpb=mutate_pb,
ngen=num_iter - 1,
stats=stats,
halloffame=hof,
verbose=True)
fitnesses = [str(p.fitness.values[0]) for p in pop]
eqpy.OUT_put("FINAL")
# return the final population
eqpy.OUT_put("{0}\n{1}\n{2}".format(create_list_of_lists_string(pop),
';'.join(fitnesses), log))
def run():
# gets dummy params for this me
params = eqpy.IN_get()
print("Params: {}".format(params))
for _ in range(10):
op = []
for _ in range(5):
p = "{},{},{},{}".format(random.randint(1, 10),
random.randint(1, 10),
random.randint(1, 10),
random.randint(1, 10))
op.append(p)
ps = ";".join(op)
eqpy.OUT_put(ps)
# wait to get result back
eqpy.IN_get()
eqpy.OUT_put("DONE")
eqpy.OUT_put("final result")
def queue_map(obj_func, pops):
# Note that the obj_func is not used
# sending data that looks like:
# [[a,b,c,d],[e,f,g,h],...]
if not pops:
return []
eqpy.OUT_put(create_list_of_json_strings(pops))
result = eqpy.IN_get()
split_result = result.split(';')
# TODO determine if max'ing or min'ing and use -9999999 or 99999999
return [(float(x), ) if not math.isnan(float(x)) else (float(99999999), )
for x in split_result]
def run():
"""run function for eqpy based run"""
eqpy.OUT_put("")
# params should be formatted as a dictionary
hp_params = eqpy.IN_get()
hp_dict = eval(hp_params)
trials = base.Trials()
rstate = None
if 'seed' in hp_dict:
rstate = np.random.RandomState(hp_dict['seed'])
fmin(eqpy_func, hp_dict['space'], hp_dict['algo'], hp_dict['max_evals'],
hp_dict['param_batch_size'], trials, rstate)
eqpy.OUT_put("FINAL")
eqpy.OUT_put(str(trials.argmin))
def measure_perf(conf_df):
conf_colns = conf_df.columns.tolist()
app_name = data.get_name(conf_colns)
data_df = data.csv2df(app_name + "_time.csv", conf_colns)
conf_perf_df = tool.df_intersection(data_df, conf_df, conf_colns)
new_conf_df = tool.df_sub(conf_df, conf_perf_df, conf_colns)
if (new_conf_df.shape[0] > 0):
new_conf_df = new_conf_df.astype(int)
eqpy.OUT_put(app_name)
eqpy.OUT_put(data.df2string(new_conf_df))
result = eqpy.IN_get()
time_df = data.string2df(result, ['run_time'])
new_conf_perf_df = pd.concat([new_conf_df, time_df], axis=1)
data.df2csv(new_conf_perf_df, app_name + "_time_new.csv")
conf_perf_df = tool.df_union(conf_perf_df, new_conf_perf_df)
conf_perf_df = data.get_exec_mach_df(data.get_runnable_df(conf_perf_df, conf_colns))
if (conf_df.shape[0] != conf_perf_df.shape[0]):
print "Error: conf_df.shape[0] != conf_perf_df.shape[0]", conf_df.shape[0], conf_perf_df.shape[0]
return conf_perf_df
def run():
ranks_str = eqpy.IN_get()
ranks = ranks_str.split(',')
# include only the al ranks
task_ranks = ranks[2:]
for r in task_ranks:
eqpy.OUT_put(r)
# include self and tasks in comm
comm = init_comm(ranks[1:])
rank = comm.rank
#printf("task cache rank: {}".format(rank))
while True:
status = MPI.Status()
data = comm.recv(source=MPI.ANY_SOURCE, status=status)
msg = data['msg']
if msg == 'put':
# this is its rank in the swift mpi communicator
eqpy.OUT_put(data['rank'])
elif msg == 'DONE':
break
def init():
eqpy.OUT_put("Settings")
settings_filename = eqpy.IN_get()
load_settings(settings_filename)
def run():
"""
:param num_iter: number of generations
:param num_pop: size of population
:param seed: random seed
:param strategy: one of 'simple', 'mu_plus_lambda'
:param ga parameters file name: ga parameters file name (e.g., "ga_params.json")
:param param_file: name of file containing initial parameters
"""
eqpy.OUT_put("Params")
params = eqpy.IN_get()
# parse params
printf("Parameters: {}".format(params))
(num_iter, num_pop, seed, strategy, mut_prob, ga_params_file, param_file,
classifer_path, scaler_path) = eval('{}'.format(params))
random.seed(seed)
ga_params = ga_utils.create_parameters(ga_params_file)
create_transformer(ga_params, classifer_path, scaler_path)
creator.create("FitnessMin", base.Fitness, weights=(-1.0, ))
creator.create("Individual", list, fitness=creator.FitnessMin)
toolbox = base.Toolbox()
toolbox.register("individual", tools.initIterate, creator.Individual,
make_random_params)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("evaluate", obj_func)
toolbox.register("mate", cxUniform, indpb=0.5)
mutate_indpb = mut_prob
toolbox.register("mutate", custom_mutate, indpb=mutate_indpb)
toolbox.register("select", tools.selTournament, tournsize=3)
toolbox.register("map", queue_map)
pop = toolbox.population(n=num_pop)
if param_file != "":
update_init_pop(pop, param_file)
hof = tools.HallOfFame(1)
stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("avg", sts.mean)
stats.register("std", sts.std)
stats.register("min", sts.min)
stats.register("max", sts.max)
stats.register("ts", timestamp)
# num_iter-1 generations since the initial population is evaluated once first
mutpb = mut_prob
start_time = time.time()
if strategy == 'simple':
pop, log = algorithms.eaSimple(pop,
toolbox,
cxpb=0.5,
mutpb=mutpb,
ngen=num_iter - 1,
stats=stats,
halloffame=hof,
verbose=True)
elif strategy == 'mu_plus_lambda':
mu = int(math.floor(float(num_pop) * 0.5))
lam = int(math.floor(float(num_pop) * 0.5))
if mu + lam < num_pop:
mu += num_pop - (mu + lam)
pop, log = algorithms.eaMuPlusLambda(pop,
toolbox,
mu=mu,
lambda_=lam,
cxpb=0.5,
mutpb=mutpb,
ngen=num_iter - 1,
stats=stats,
halloffame=hof,
verbose=True)
else:
raise NameError('invalid strategy: {}'.format(strategy))
end_time = time.time()
fitnesses = [str(p.fitness.values[0]) for p in pop]
eqpy.OUT_put("DONE")
# return the final population
eqpy.OUT_put("{}\n{}\n{}\n{}\n{}".format(create_list_of_json_strings(pop),
';'.join(fitnesses), start_time,
log, end_time))
def run():
start_time = time.time()
print("run() start: {}".format(str(datetime.datetime.now())))
comm = MPI.COMM_WORLD # get MPI communicator object
size = comm.size # total number of processes
rank = comm.rank # rank of this process
status = MPI.Status() # get MPI status object
print("ME rank is {}".format(rank))
instance = problem.Problem()
spaceDict = instance.space
params = instance.params
global problem_params
problem_params = params
starting_point = instance.starting_point
# handshake to ensure working
eqpy.OUT_put("Params")
# initial parameter set telling us the number of times to run the loop
initparams = eqpy.IN_get()
(init_size, max_evals, num_workers, num_buffer, seed, max_threshold,
n_jobs) = eval('{}'.format(initparams))
space = [spaceDict[key] for key in params]
print(space)
parDict = {}
resultsList = []
parDict['kappa'] = 1.96
# can set to num cores
parDict['n_jobs'] = n_jobs
init_x = []
opt = Optimizer(space,
base_estimator='RF',
acq_optimizer='sampling',
acq_func='LCB',
acq_func_kwargs=parDict,
random_state=seed)
eval_counter = 0
askedDict = {}
print(
"Master starting with {} init_size, {} max_evals, {} num_workers, {} num_buffer, {} max_threshold"
.format(init_size, max_evals, num_workers, num_buffer, max_threshold))
x = opt.ask(n_points=init_size)
res, resstring = create_list_of_json_strings(x)
print("Initial design is {}".format(resstring))
for r, xx in zip(res, x):
askedDict[r] = xx
eqpy.OUT_put(resstring)
currently_out = init_size
total_out = init_size
results = []
group = comm.Get_group()
# Assumes only one adlb_server
# num_workers + 1 = num_turbine_workers
newgroup = group.Excl([num_workers + 1])
#print("ME newgroup size is {}".format(newgroup.size))
newcomm = comm.Create_group(newgroup, 1)
nrank = newcomm.rank
#print("ME nrank is {}".format(nrank))
counter_threshold = 1
counter = 0
end_iter_time = 0
while eval_counter < max_evals:
start_iter_time = time.time()
print("\neval_counter = {}".format(eval_counter))
data = newcomm.recv(source=MPI.ANY_SOURCE, status=status)
counter = counter + 1
xstring = data['x']
x = askedDict[xstring]
y = data['cost']
if math.isnan(y):
y = sys.float_info.max
opt.tell(x, y)
#source = status.Get_source()
#tag = status.Get_tag()
elapsed_time = float(time.time() - start_time)
print('elapsed_time:%1.3f' % elapsed_time)
results.append(str(data))
eval_counter = eval_counter + 1
currently_out = currently_out - 1
# if jobs are finishing within 16 seconds of
# each other, then batch the point production
if start_iter_time - end_iter_time < 16:
counter_threshold = max_threshold
if max_evals - eval_counter < counter_threshold:
counter_threshold = max_evals - eval_counter
if counter_threshold > currently_out:
counter_threshold = currently_out
else:
counter_threshold = 1
print("counter_threshold: {}".format(counter_threshold))
print("currently_out:{}, total_out:{}".format(currently_out,
total_out))
if currently_out < num_workers + num_buffer and total_out < max_evals and counter >= counter_threshold:
n_points = counter
if n_points + total_out > max_evals:
n_points = max_evals - total_out
ts = time.time()
x = opt.ask(n_points=n_points)
res, resstring = create_list_of_json_strings(x)
for r, xx in zip(res, x):
askedDict[r] = xx
eqpy.OUT_put(resstring)
print('point production elapsed_time:%1.3f' %
float(time.time() - ts))
currently_out = currently_out + n_points
total_out = total_out + n_points
counter = 0
end_iter_time = start_iter_time
print('Search finishing')
eqpy.OUT_put("DONE")
eqpy.OUT_put(";".join(results))
def run():
"""
:param num_iterations: number of generations
:param seed: random seed
:param ga parameters file name: ga parameters file name (e.g., "ga_params.json")
:param num_population population of ga algorithm
"""
eqpy.OUT_put("Params")
parameters = eqpy.IN_get()
# parse params
printf("Parameters: {}".format(parameters))
(num_iterations, num_population, seed,
ga_parameters_file) = eval('{}'.format(parameters))
random.seed(seed)
ga_parameters = ga_utils.create_parameters(ga_parameters_file)
global transformer
transformer = Transformer(ga_parameters)
# deap class creators
creator.create("FitnessMin", base.Fitness, weights=(-1.0, ))
creator.create("Individual", list, fitness=creator.FitnessMin)
# deap method definitions
toolbox = base.Toolbox()
toolbox.register("individual", tools.initIterate, creator.Individual,
make_random_parameters)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("evaluate", obj_func)
toolbox.register("mate", cxUniform, indpb=0.5)
toolbox.register("mutate", custom_mutate, indpb=0.2)
toolbox.register("select", tools.selTournament, tournsize=3)
toolbox.register("map", queue_map)
pop = toolbox.population(n=num_population)
hof = tools.HallOfFame(1)
stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("avg", np.mean)
stats.register("std", np.std)
stats.register("min", np.min)
stats.register("max", np.max)
stats.register("ts", timestamp)
# num_iter-1 generations since the initial population is evaluated once first
start_time = time.time()
pop, log = algorithms.eaSimple(pop,
toolbox,
cxpb=0.5,
mutpb=0.2,
ngen=num_iterations - 1,
stats=stats,
halloffame=hof,
verbose=True)
end_time = time.time()
fitnesses = [str(p.fitness.values[0]) for p in pop]
eqpy.OUT_put("DONE")
# return the final population
eqpy.OUT_put("{}\n{}\n{}\n{}\n{}".format(create_list_of_json_strings(pop),
';'.join(fitnesses), start_time,
log, end_time))
