def work(params):
worker_id = params[0][0]
config = params[0][1]
server = jsonrpclib.Server(config["server"])
num_var = config["num_var"]
results = []
pset = conf_sets(num_var)
toolbox = getToolBox(config, pset)
d = './ReSpeciation/%s/time_%d_%d.txt' % (
config["problem"], config["n_problem"], config["set_specie"])
neatGPLS.ensure_dir(d)
time_r = open(d, 'a')
for sample_num in range(1, config["max_samples"] + 1):
# revisar si sigue existiendo o no la especie.
print("check specie - 1")
evospace_sample = check_specie(server, config["set_specie"])
if evospace_sample == []:
a = server.getSpecie()
while evospace_sample == []:
aux_specie = random.choice(a)
evospace_sample = check_specie(server, aux_specie)
d_intracluster = server.getIntraSpecie(config["set_specie"])
time_r.write(
'\n%s;%s;%s;%s;%s;%s;%s;%s' %
(config["set_specie"], sample_num, str(datetime.datetime.now()),
d_intracluster, 'NA', 'NA', 'NA', funcEval.cont_evalp))
gen_data, len_pop, flag_, fEval_, d_intracluster = evolve(
sample_num, config, toolbox, pset, evospace_sample)
time_r.write(
'\n%s;%s;%s;%s;%s;%s;%s' %
(config["set_specie"], sample_num, str(datetime.datetime.now()),
d_intracluster, len_pop, flag_, fEval_))
data_specie = {'id': config["set_specie"], 'b_key': 'True'}
server.setSpecieFree(data_specie)
# Inicia especiacion
print 'Going to counter'
re_sp = evo_specie.counter(toolbox, pset)
print 'Ending to counter'
if gen_data == []:
print 'No-Evolution'
results = []
else:
results.append([worker_id] + gen_data)
return results
def evalSymbReg_b(individual, points, toolbox, config, type):
points.flags['WRITEABLE'] = False
func = toolbox.compile(expr=individual)
if config["benchmark"]:
vector = vector_benchmarks(config["problem"], points)
data_x = copy.deepcopy(np.asarray(points)[:])
else:
vector = copy.deepcopy(points[config["num_var"]])
data_x = copy.deepcopy(np.asarray(points)[:config["num_var"]])
try:
vector_x = func(*data_x)
except TypeError:
print individual
print data_x
with np.errstate(divide='ignore', invalid='ignore'):
if isinstance(vector_x, np.ndarray):
for e in range(len(vector_x)):
if np.isnan(vector_x[e]) or np.isinf(vector_x[e]):
vector_x[e] = 0.
if config["funcion"] == 1:
d = './Results/%s/best_data_%d_%s.txt' % (config["problem"],
config["n_problem"], type)
neatGPLS.ensure_dir(d)
best_test = open(d, 'w')
for item in vector_x:
best_test.write("%s\n" % item)
result = np.sum((vector_x - vector)**2)
return np.sqrt(result / len(points[0])),
elif config["funcion"] == 2:
l = len(points[0])
ME = np.sum(abs((vector_x - vector))) / l
SD = np.sqrt(np.sum((vector_x - ME)**2) / (l))
try:
calc = 100 * (SD / ME)
return calc,
except ZeroDivisionError:
return 100,
version = 1
pop, log = neatGPLS.neat_GP_LS(pop, toolbox, cxpb, mutpb, ngen, neat_alg, neat_cx, neat_h,
neat_pelit,funcEval.LS_flag, LS_select, cont_evalf,
num_salto, SaveMatrix, GenMatrix, pset, n_corr, num_p,
params, direccion, problem, testing, version,
stats=mstats, halloffame=hof,verbose=True)
return pop, log, hof
def run(number,problem):
n = 1
while n <= number:
main(n,problem)
n += 1
if __name__ == "__main__":
problem = 'wastewater2'
number = 1000
d = './Timing/%s/timing_cxneat_%d.txt' % (problem, number)
neatGPLS.ensure_dir(d)
time_conc = open(d, 'a')
n = 1
while n < 11:
begin_p = time.time()
main(n, number, problem)
n += 1
end_p = time.time()
time_conc.write('\n%s;%s;%s;%s' % (n, begin_p, end_p, str(round(end_p - begin_p, 2))))
pop, log = neatGPLS.neat_GP_LS(pop, toolbox, cxpb, mutpb, ngen, neat_alg, neat_cx, neat_h,
neat_pelit, funcEval.LS_flag, LS_select, cont_evalf,
num_salto, SaveMatrix, GenMatrix, pset,n_corr, p,
params, direccion,problem,testing, version,
stats=mstats, halloffame=None, verbose=True)
return pop, log
def run(number, problem):
n = 1
while n <= number:
main(n, problem)
n += 1
if __name__ == "__main__":
problem = 'Housing'
number = 1702
d = './Timing/%s/timing_cxneat_%d.txt' % (problem, number)
neatGPLS.ensure_dir(d)
time_conc = open(d, 'a')
n = 1
while n < 31:
begin_p = time.time()
main(n, number, problem)
n += 1
end_p = time.time()
time_conc.write('\n%s;%s;%s;%s' % (n, begin_p, end_p, str(round(end_p - begin_p, 2))))
print num
direccion3 = "./Results/%s/bestind_LStr_%d_%d.txt" % (directory, index_,
num)
with open(direccion3) as spambase:
spamReader = csv.reader(spambase,
delimiter=';',
skipinitialspace=False)
cont = 0
for row in spamReader:
if row != []:
cont = cont + 1
with open(direccion3) as spambase:
spamReader = csv.reader(spambase,
delimiter=';',
skipinitialspace=False)
for row in spamReader:
if row != [] and row[0] == str(cont):
#data2.append([num, row[2]])
print row[2]
out.write('\n%d;%s;%s;%s' % (index_, num, min(data), row[2]))
d = './Engine3/data_min.txt'
ensure_dir(d)
out = open(d, 'a')
directory = "Engine3"
#for i in (1101):
i = 10121
get_data(directory, i, out)
import datetime
from shutil import copyfile
config = yaml.load(open("conf/conf.yaml"))
start = time.time()
start = time.time()
params = [(i, config) for i in range(1)]
num_p = config["n_problem"]
problem = config["problem"]
specie = 10
d = './Timing/%s/time_%d.txt' % (problem, num_p)
ensure_dir(d)
best = open(d, 'a')
c_f = './conf_record/%s/' % (problem)
ensure_dir(c_f)
copyfile('./conf/conf.yaml', ('./conf_record/%s/config_%s_%d_%d.yaml' %
(problem, problem, num_p, specie)))
# int(random.choice(b))
config["set_specie"] = specie
# config["neat_alg"] = False
# for ci in range(1, 2):
# config["n_corr"] = ci
ci = config["n_corr"]
with open("conf/conf.yaml", "w") as f:
yaml.dump(config, f)
def counter(toolbox, pset):
print 'Counter'
re_sp = 0
config = yaml.load(open("conf/conf.yaml"))
server = jsonrpclib.Server(config["server"])
free_pop = eval(server.getFreePopulation())
free_file = eval(server.getFreeFile())
print free_file, free_pop
if free_pop and free_file:
server.setFreeFile('False')
print 'Free and Free'
r = server.get_CounterSpecie()
print r
if scheck_(server, r, config["porcentage_flag"], config["porcentage"]):
print "speciation-required"
free_species = [] # List of free species
flag_check = True
rs_species = [] # List of species
for i in range(1, int(r)):
rs_species.append(int(server.getSpecieInfo(i)['specie']))
# Opening files to save data.
d = './ReSpeciacion/%s/rspecie_%d.txt' % (config["problem"],
config["n_problem"])
ensure_dir(d)
best = open(d, 'a')
d = './ReSpeciacion/%s/nspecie_%d.csv' % (config["problem"],
config["n_problem"])
ensure_dir(d)
n_specie = open(d, 'a')
d = './General/%s/datapop_%d_%d.txt' % (
config["problem"], config["n_problem"], config["set_specie"])
neatGPLS.ensure_dir(d)
datapop_ = open(d, 'a')
while flag_check:
flag_check = check_(server, r, free_species)
print("waiting - this worker will make speciation")
if not flag_check:
print 'ReSpeciacion'
sp_init = datetime.datetime.now()
pop = []
for sp in rs_species:
evospace_sample = server.getSample_specie(sp)
for cs in evospace_sample['sample']:
i = creator.Individual(
neat_gp.PrimitiveTree.from_string(
cs['chromosome'], pset))
if isinstance(cs['params'], list):
i.params_set(np.asarray(cs['params']))
elif isinstance(cs['params'], unicode):
i.params_set(
np.asarray([
float(elem) for elem in cs['params'].strip(
'[]').split(',')
]))
i.specie(int(cs['specie']))
pop.append(i)
print 'Flush population'
server.flushPopulation()
for ind in pop:
datapop_.write(
'\n%s;%s;%s;%s' %
(str(sp_init), ind, len(pop), ind.get_specie()))
server.initialize()
print 'Initialize population'
neat_alg = config["neat_alg"]
if neat_alg:
a, b, init_pop = speciation_init(config, server, pop,
config["neat_h"])
list_spe = calc_intracluster(pop)
for elem in list_spe:
specielist = {
'id': None,
'specie': str(elem[0]),
'intra_distance': str(elem[1]),
'flag_speciation': 'False',
'sp_event': 'True'
}
server.putSpecie(specielist)
n_specie.write('\n%s,%s' %
(str(elem[0]), str(elem[1])))
server.putZample(init_pop)
server.setFreePopulation('True')
num_specie = server.get_CounterSpecie()
print "numero de especies creadas: ", num_specie
print 'ReSpeciacion- Done'
re_sp = 1
best.write(
'\n%s;%s;%s;%s' % (str(datetime.datetime.now()),
str(sp_init), len(pop), num_specie))
server.setFreePopulation('True')
server.setFreeFile('True')
contSpecie.cont_specie = 0
return re_sp
else:
print 'No Free'
if free_pop and free_file is False:
while free_file is False:
print "waiting"
time.sleep(5)
try:
free_file = eval(server.getFreeFile())
except TypeError:
free_file = False
re_sp = 0
free_pop = eval(server.getFreePopulation())
while free_pop is False:
re_sp = 1
print "still waiting", free_pop
try:
time.sleep(1)
free_pop = eval(server.getFreePopulation())
except TypeError:
time.sleep(5)
free_pop = eval(server.getFreePopulation())
return re_sp
verbose=True)
# Regresa el mejor de las generaciones
return pop, log, hof, train_f, test_f
if __name__ == "__main__":
config = yaml.load(open("conf/conf.yaml"))
problem = config["problem"]
num_var = config["num_var"]
database_name = config["db_name"]
number = config["n_problem"]
c_f = './conf_record/%s/' % (problem)
neatGPLS.ensure_dir(c_f)
copyfile('./conf/conf.yaml', ('./conf_record/%s/config_%s_%d.yaml' %
(problem, problem, number)))
c_m = './matrix_complete/%s/Matrix_c_%d.csv' % (problem, number)
neatGPLS.ensure_dir(c_m)
matrix_complete = open(c_m, 'a')
c_s = './matrix_complete/%s/stats_train_%d.csv' % (problem, number)
neatGPLS.ensure_dir(c_s)
matrix_stats = open(c_s, 'a')
c_s = './matrix_complete/%s/stats_test_%d.csv' % (problem, number)
neatGPLS.ensure_dir(c_s)
matrix_stats_t = open(c_s, 'a')
def evolve(sample_num, config, toolbox, pset, evospace_sample):
start = time.time()
problem = config["problem"]
direccion = "./data_corridas/%s/train_%d_%d.txt"
n_corr = sample_num # config["n_corr"]
n_prob = config["n_problem"]
name_database = config["db_name"]
print config["set_specie"]
server = jsonrpclib.Server(config["server"])
# evospace_sample = server.getSample(config["population_size"])
pop = []
for cs in evospace_sample['sample']:
i = creator.Individual(
neat_gp.PrimitiveTree.from_string(cs['chromosome'], pset))
if isinstance(cs['params'], list):
i.params_set(np.asarray(cs['params']))
elif isinstance(cs['params'], unicode):
i.params_set(
np.asarray([
float(elem)
for elem in cs['params'].strip('f[]').split(',')
]))
i.specie(int(cs['specie']))
pop.append(i)
if len(pop) < 2:
a = server.getSpecie()
aux_specie = random.choice(a)
print("check specie - 2 - %d" % aux_specie)
evospace_sample2 = check_specie_aux(server, aux_specie)
if evospace_sample2 == []:
while evospace_sample2 == []:
aux_specie = random.choice(a)
evospace_sample2 = check_specie_aux(server, aux_specie)
for cs in evospace_sample2['sample']:
i = creator.Individual(
neat_gp.PrimitiveTree.from_string(cs['chromosome'], pset))
if isinstance(cs['params'], list):
i.params_set(np.asarray(cs['params']))
elif isinstance(cs['params'], unicode):
i.params_set(
np.asarray([
float(elem)
for elem in cs['params'].strip('[]').split(',')
]))
i.specie(int(cs['specie']))
pop.append(i)
data_specie = {'id': aux_specie, 'b_key': 'True'}
server.setSpecieFree(data_specie)
#Tomar una especie aleatoria
print 'Especie: %d --- Numero de individuos: %d' % (len(pop),
config["set_specie"])
cxpb = config["cxpb"]
mutpb = config["mutpb"]
ngen = config["worker_generations"]
params = config["params"]
neat_cx = config["neat_cx"]
neat_alg = config["neat_alg"]
neat_pelit = config["neat_pelit"]
neat_h = config["neat_h"]
funcEval.LS_flag = config["ls_flag"]
LS_select = config["ls_select"]
funcEval.cont_evalp = 0
num_salto = config["num_salto"]
cont_evalf = config["cont_evalf"]
SaveMatrix = config["save_matrix"]
GenMatrix = config["gen_matrix"]
version = 3
testing = True
data_(n_corr, n_prob, problem, name_database, toolbox, config)
begin = time.time()
print "inicio del proceso"
pop, log, funcEval_ = neatGPLS.neat_GP_LS(pop,
toolbox,
cxpb,
mutpb,
ngen,
neat_alg,
neat_cx,
neat_h,
neat_pelit,
funcEval.LS_flag,
LS_select,
cont_evalf,
num_salto,
SaveMatrix,
GenMatrix,
pset,
n_corr,
n_prob,
params,
direccion,
problem,
testing,
version=version,
benchmark_flag=False,
beta=0.5,
random_speciation=True,
set_specie=config["set_specie"],
stats=None,
halloffame=None,
verbose=True)
putback = time.time()
d_intraspecie = intracluster(pop)
d_intracluster = server.getIntraSpecie(config["set_specie"])
resp_flag = 0
id_ = "specie:%s" % config["set_specie"]
print 'calculation intra-cluster'
if d_intraspecie > (1.5 * float(d_intracluster)):
specielist = {
'id': id_,
'specie': config["set_specie"],
'intra_distance': str(d_intracluster),
'flag_speciation': 'True',
'sp_event': 'False'
}
server.putSpecie(specielist)
resp_flag = 1
sample = [{
"specie":
str(config["set_specie"]),
"chromosome":
str(ind),
"id":
None,
"fitness": {
"DefaultContext": [
ind.fitness.values[0].item() if isinstance(
ind.fitness.values[0], np.float64) else
ind.fitness.values[0]
]
},
"params":
str([x for x in ind.get_params()]) if funcEval.LS_flag else None
} for ind in pop]
evospace_sample = {
'sample_id': 'None',
'sample_specie': str(config["set_specie"]),
'sample': sample
}
server.putZample(evospace_sample)
d = './Data/%s/dintra_%d_%s.txt' % (problem, n_prob, config["set_specie"])
neatGPLS.ensure_dir(d)
dintr = open(d, 'a')
dintr.write('\n%s;%s;%s;%s' %
(n_corr, d_intraspecie, d_intracluster, resp_flag))
best_ind = tools.selBest(pop, 1)[0]
best = [
len(best_ind), sample_num,
round(time.time() - start, 2),
round(begin - start, 2),
round(putback - begin, 2),
round(time.time() - putback, 2), best_ind
], len(pop), resp_flag, funcEval, d_intracluster
return best
