def sequencer_mutation(oa, sequencer_rate=0.5):
''' Swap two elements from a randomly selected column if not successful swapping return False '''
if oa.get_fitness_value() < 0.1:
return False
ar = matrix(oa.array)
prob = random.random()
if prob < sequencer_rate:
# Randomly select a column
c = random.randint(0, ar.shape[1] - 1)
# Randomly select two indexes in the column
idx1 = random.randint(0, oa.runs - 1)
idx2 = random.randint(0, oa.runs - 1)
# Swap values at indexes and return the new oa if it is more fit else return False
temp = ar[:, c].tolist()
temp_st = temp[idx1]
temp[idx1] = temp[idx2]
temp[idx2] = temp_st
ar[:, c] = matrix(temp)
tempoa = OA(oa.string, ar)
if tempoa.get_fitness_value() < oa.get_fitness_value():
return tempoa
return False
def sequencer_mutation(oa, sequencer_rate=0.5):
''' Swap two elements from a randomly selected column if not successful swapping return False '''
if oa.get_fitness_value() < 0.1:
return False
ar = matrix(oa.array)
prob = random.random()
if prob < sequencer_rate:
# Randomly select a column
c = random.randint(0, ar.shape[1] - 1)
# Randomly select two indexes in the column
idx1 = random.randint(0, oa.runs - 1)
idx2 = random.randint(0, oa.runs - 1)
# Swap values at indexes and return the new oa if it is more fit else return False
temp = ar[:, c].tolist()
temp_st = temp[idx1]
temp[idx1] = temp[idx2]
temp[idx2] = temp_st
ar[:, c] = matrix(temp)
tempoa = OA(oa.string, ar)
if tempoa.get_fitness_value() < oa.get_fitness_value():
return tempoa
return False
def __init__(self):
config = ConfigParser()
config.read("OSM_2_OA.cfg")
self.__createLogger()
pgString = "postgres://"
pgString += "%s:%s" % (config.get("OA", "user"), config.get(
"OA", "pwd"))
pgString += "@%s:%s" % (config.get(
"OA", "dbServer"), config.get("OA", "port"))
pgString += "/%s" % (config.get("OA", "database"))
sqlFile = os.path.join(os.getcwd(), "OSM_2_OA.sql")
self.oa = OA(pgString, sqlFile)
self.osm = OSM(config.get("OSM", "XAPI"), config.get("OSM", "bbox"),
config.get("OSM", "maxSize"))
class OSM_2_OA:
"""main class to synchronize data from OSM to OA
does the orchestration"""
def __init__(self):
config = ConfigParser()
config.read("OSM_2_OA.cfg")
self.__createLogger()
pgString = "postgres://"
pgString += "%s:%s" % (config.get("OA", "user"), config.get(
"OA", "pwd"))
pgString += "@%s:%s" % (config.get(
"OA", "dbServer"), config.get("OA", "port"))
pgString += "/%s" % (config.get("OA", "database"))
sqlFile = os.path.join(os.getcwd(), "OSM_2_OA.sql")
self.oa = OA(pgString, sqlFile)
self.osm = OSM(config.get("OSM", "XAPI"), config.get("OSM", "bbox"),
config.get("OSM", "maxSize"))
def __createLogger(self):
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s | %(name)-13s | %(levelname)-6s | %(message)s',
filename=os.path.join(os.getcwd(), self.__class__.__name__) +
".log")
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter(
'%(asctime)s | %(name)-13s | %(levelname)-6s | %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.info("Started")
def execute(self):
dateOfLastSynchro = self.oa.getDateOfLastUpdate()
deltas = self.osm.getDelta(dateOfLastSynchro)
##debug
#deltas=[]
#for f in os.listdir(tempfile.gettempdir()):
# if f.find(".osm")>-1:
# deltas.append(os.path.join(tempfile.gettempdir(),f))
for delta in deltas:
converter = OSMDelta_2_AddressRowConverter(delta)
rows = converter.convert()
self.oa.insert(rows)
pass
def mutation_t2(oa, mutation_rate2=0.6):
''' Create a new OA with all old plus a new factor with some randomly chosen levels'''
prob = random.random()
if prob < mutation_rate2:
lev = find_levels(oa.runs)
lev_selected = lev[random.randint(0, len(lev) - 1)]
new_col = []
for i in range(lev_selected):
for j in range(int(oa.runs / lev_selected)):
new_col.append(i)
new_col = matrix(new_col).reshape(oa.runs, 1)
# here finding factors of new array then finding string representation then again finding the factors Improve it later
temp_fact = oa.factors[:]
temp_fact.append(lev_selected)
temp = str(oa.runs)
for i in list(remove_duplicates(temp_fact)):
temp += ',' + str(i) + '^' + str(temp_fact.count(i))
new_oa = OA(temp, concatenate((oa.array, new_col), axis=1))
return new_oa
return oa
def brkga_mutation(runsize, num_col=4):
'''Here we will create random mutant solutions of randomly chosen levels and number of factors,
Also no mutation rate is required as we always have to generate a particular number of mutant,
solutions based on the size of population. Call this function recursively to generate more than one mutant'''
lev = find_levels(runsize)
new_matrix = []
temp_fact = []
for l in range(num_col):
lev_selected = lev[random.randint(0, len(lev) - 1)]
temp_fact.append(lev_selected)
new_col = []
for i in range(lev_selected):
for j in range(int(runsize / lev_selected)):
new_col.append(i)
random.shuffle(new_col)
new_matrix.append(new_col)
new_matrix = matrix(new_matrix).reshape(num_col, runsize)
new_matrix = matrix.transpose(new_matrix)
temp = str(runsize)
for i in list(remove_duplicates(temp_fact)):
temp += ',' + str(i) + '^' + str(temp_fact.count(i))
new_oa = OA(temp, new_matrix)
return new_oa
class CollisionDetection(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.shutdown_flag = threading.Event()
self.OA = OA()
self.OA.setSensitivity(10)
def run(self):
print('Thread #%s started' % self.ident)
while not self.shutdown_flag.is_set():
result = self.OA.algoritme()
if result == 1:
print ("Thumper robot heeft een botsing gedetecteerd")
# ... Clean shutdown code here ...
print('Thread #%s stopped' % self.ident)
def select_runsize(self, runsize):
''' Scans the data directory for OA's of selected runsize and loads them as members of initial population'''
if (runsize == ""):
return
self.initial_oa_population = []
load_data = [
i for i, word in enumerate(self.all_files)
if re.search('^' + str(runsize) + ',' + '.*', word)
]
self.set_in_console("Loading ...")
for i in load_data:
print self.all_files[i]
file1 = open(join(self.data_path, self.all_files[i]), 'r')
ar = ''
for j in file1.readlines():
ar += j.rstrip(',\n') + ';'
ar = ar.rstrip(';')
self.initial_oa_population.append(
OA(self.all_files[i].split('.')[0], matrix(ar)))
self.initial_population_size = len(load_data)
for i in self.initial_oa_population:
print i.get_fitness_value()
self.set_in_console("Loaded " + str(self.initial_population_size) +
" OA's of run size " + str(runsize))
class OSM_2_OA:
"""main class to synchronize data from OSM to OA
does the orchestration"""
def __init__(self):
config = ConfigParser()
config.read("OSM_2_OA.cfg")
self.__createLogger()
pgString = "postgres://"
pgString += "%s:%s" % (config.get("OA", "user"), config.get("OA", "pwd"))
pgString += "@%s:%s" % (config.get("OA", "dbServer"), config.get("OA", "port"))
pgString += "/%s" % (config.get("OA", "database"))
sqlFile = os.path.join(os.getcwd(), "OSM_2_OA.sql")
self.oa = OA(pgString, sqlFile)
self.osm = OSM(config.get("OSM", "XAPI"), config.get("OSM", "bbox"), config.get("OSM", "maxSize"))
def __createLogger(self):
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s | %(name)-13s | %(levelname)-6s | %(message)s",
filename=os.path.join(os.getcwd(), self.__class__.__name__) + ".log",
)
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter("%(asctime)s | %(name)-13s | %(levelname)-6s | %(message)s")
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger("").addHandler(console)
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.info("Started")
def execute(self):
dateOfLastSynchro = self.oa.getDateOfLastUpdate()
deltas = self.osm.getDelta(dateOfLastSynchro)
##debug
# deltas=[]
# for f in os.listdir(tempfile.gettempdir()):
# if f.find(".osm")>-1:
# deltas.append(os.path.join(tempfile.gettempdir(),f))
for delta in deltas:
converter = OSMDelta_2_AddressRowConverter(delta)
rows = converter.convert()
self.oa.insert(rows)
pass
def one_point_crossover(oa, crossover_rate=0.7):
prob = random.random()
if prob < crossover_rate:
orthogonal_array = [oa[0].array, oa[1].array]
run_size = oa[0].runs
factors = [len(oa[0].factors), len(oa[1].factors)]
index_of_crossover = [
random.randint(0, factors[0] - 1),
random.randint(0, factors[1] - 1)
]
[left_1, right_1] = [
orthogonal_array[0][:, :index_of_crossover[0]],
orthogonal_array[0][:, index_of_crossover[0]:]
]
[left_2, right_2] = [
orthogonal_array[1][:, :index_of_crossover[1]],
orthogonal_array[1][:, index_of_crossover[1]:]
]
[left_fact_1, right_fact_1] = [
oa[0].factors[:index_of_crossover[0]],
oa[0].factors[index_of_crossover[0]:]
]
[left_fact_2, right_fact_2] = [
oa[1].factors[:index_of_crossover[1]],
oa[1].factors[index_of_crossover[1]:]
]
[fact_1,
fact_2] = [left_fact_1 + right_fact_2, left_fact_2 + right_fact_1]
temp = [str(run_size), str(run_size)]
for i in list(remove_duplicates(fact_1)):
temp[0] += ',' + str(i) + '^' + str(fact_1.count(i))
for i in list(remove_duplicates(fact_2)):
temp[1] += ',' + str(i) + '^' + str(fact_2.count(i))
oa_left = OA(temp[0], concatenate([left_1, right_2], 1))
oa_right = OA(temp[1], concatenate([left_2, right_1], 1))
return [oa_left, oa_right]
return [oa[0], oa[1]]
def __init__(self):
config = ConfigParser()
config.read("OSM_2_OA.cfg")
self.__createLogger()
pgString = "postgres://"
pgString += "%s:%s" % (config.get("OA", "user"), config.get("OA", "pwd"))
pgString += "@%s:%s" % (config.get("OA", "dbServer"), config.get("OA", "port"))
pgString += "/%s" % (config.get("OA", "database"))
sqlFile = os.path.join(os.getcwd(), "OSM_2_OA.sql")
self.oa = OA(pgString, sqlFile)
self.osm = OSM(config.get("OSM", "XAPI"), config.get("OSM", "bbox"), config.get("OSM", "maxSize"))
def brkga_crossover(oa, crossover_rate=0.7):
# In Brkga only one offspring is generated from one crossover
elite = oa[0].array
non_elite = oa[1].array
fact = []
ret = []
# or[0].array belongs to elite population
if oa[0].num_factors > oa[1].num_factors:
total = oa[0].num_factors
idx = 1
else:
total = oa[1].num_factors
idx = 0
for i in range(total):
prob = random.random()
if i < oa[idx].num_factors:
if prob < crossover_rate:
fact.append(oa[0].factors[i])
if i == 0:
child = elite[:, i]
else:
child = concatenate([child, elite[:, i]], 1)
else:
fact.append(oa[1].factors[i])
if i == 0:
child = non_elite[:, i]
else:
child = concatenate([child, non_elite[:, i]], 1)
else:
if idx == 1 and prob < crossover_rate:
fact.append(oa[0].factors[i])
child = concatenate([child, elite[:, i]], 1)
elif idx == 0 and prob >= crossover_rate:
fact.append(oa[1].factors[i])
child = concatenate([child, non_elite[:, i]], 1)
temp = str(oa[0].runs)
for i in list(remove_duplicates(fact)):
temp += ',' + str(i) + '^' + str(fact.count(i))
ret.append(OA(temp, child))
return ret
def run_seq_mut():
# Take file path w.r.t. Orthogonal Arrays dir as input takes only dump oa file format
got_input = False
while(not got_input):
try:
path = raw_input('Enter path of array w.r.t. root OA dir: ')
oa_name = raw_input('Enter oa name: ')
count = raw_input('Enter number of times to apply seq mutation: ')
count = int(count)
path += oa_name + '.csv'
path = join(util.module_path(), path)
if not isfile(path):
raise ValueError
got_input = True
except ValueError:
print 'Invalid Input try again ...'
# Now load the OA
oa_file = open(path, 'r')
ar = ''
for j in oa_file.readlines():
ar += j.rstrip(',\n') + ';'
ar = ar.rstrip(';')
oa = OA(oa_name.split('[')[0], matrix(ar))
# Apply sequencer mutation
cont_flag = True
while(cont_flag):
for i in range(count):
seq_oa = sequencer_mutation(oa, 1)
if seq_oa:
print 'HIT ' + str(seq_oa.get_fitness_value())
oa = seq_oa
cont = raw_input('Enter N to stop else continue again: ')
if cont == "N":
cont_flag = False
util.dump_oa_to_file(oa)
def check_strength():
######## Write code to calculate the lower bound also #########
# Take Input
got_input = False
while (not got_input):
try:
num_run = raw_input('Enter number of runs: ')
num_run = int(num_run)
got_input = True
except ValueError:
print 'Invalid Input try again ...'
# Load run_size oa files
all_files = [f for f in listdir(data_path) if isfile(join(data_path, f))]
all_files.sort(key=lambda x: int(x.split(',')[0]))
elite_oa = []
load_data = [
i for i, word in enumerate(all_files)
if re.search('^' + str(num_run) + ',' + '.*', word)
]
for i in load_data:
print all_files[i]
file1 = open(join(data_path, all_files[i]), 'r')
ar = ''
for j in file1.readlines():
ar += j.rstrip(',\n') + ';'
ar = ar.rstrip(';')
elite_oa.append(OA(all_files[i].split('.')[0], matrix(ar)))
S_e = len(load_data)
for i in elite_oa:
print i.get_fitness_value()
print "Other Jn's: "
for j in range(2, i.array.shape[1]):
print " " + str(Jn(i, j))
def __init__(self): threading.Thread.__init__(self) self.shutdown_flag = threading.Event() self.OA = OA() self.OA.setSensitivity(10)
def get_input_and_run_ga():
# Take Input
got_input = False
while (not got_input):
try:
num_run = raw_input('Enter number of runs: ')
num_run = int(num_run)
num_crossover_rate = raw_input('Enter crossover rate: ')
num_crossover_rate = float(num_crossover_rate)
frac_mutants = raw_input('Enter mutant fraction: ')
frac_mutants = float(frac_mutants)
frac_elites = raw_input('Enter elite fraction: ')
frac_elites = float(frac_elites)
max_gen = raw_input('Enter maximum allowed generation: ')
max_gen = int(max_gen)
max_col_mut = raw_input(
'Enter maximum allowed columns for mutation: ')
max_col_mut = int(max_col_mut)
tour_sel_size = raw_input('Enter tournament selection size: ')
tour_sel_size = int(tour_sel_size)
min_col_accept = raw_input(
'Enter minimum number of columns for acceptance: ')
min_col_accept = int(min_col_accept)
sequencer_rate = raw_input('Enter sequencer mutation rate: ')
sequencer_rate = float(sequencer_rate)
initial_seq_mult = raw_input('Enter initial sequencer count: ')
initial_seq_mult = int(initial_seq_mult)
growth_rate = raw_input(
'Enter sequencer mutation count growth rate: ')
growth_rate = float(growth_rate)
seq_mult_max = raw_input(
'Enter maximum sequencer multiplicator value: ')
seq_mult_max = int(seq_mult_max)
if num_run == 0:
raise ValueError
if num_crossover_rate > 1.0:
raise ValueError
if frac_mutants >= 1.0 or frac_elites >= 1.0:
raise ValueError
got_input = True
except ValueError:
print 'Invalid Input try again ...'
# Load run_size oa files
all_files = [f for f in listdir(data_path) if isfile(join(data_path, f))]
all_files.sort(key=lambda x: int(x.split(',')[0]))
elite_oa = []
load_data = [
i for i, word in enumerate(all_files)
if re.search('^' + str(num_run) + ',' + '.*', word)
]
for i in load_data:
print all_files[i]
file1 = open(join(data_path, all_files[i]), 'r')
ar = ''
for j in file1.readlines():
ar += j.rstrip(',\n') + ';'
ar = ar.rstrip(';')
elite_oa.append(OA(all_files[i].split('.')[0], matrix(ar)))
S_e = len(load_data)
for i in elite_oa:
print i.get_fitness_value()
# Run BRKGA
# S = Total population, S_e = Elite population, S_m = mutant population
S = int(S_e / frac_elites)
S_m = int(S * frac_mutants)
non_elite_oa = []
# Fill the non_elite_population with randomly generated oa's
for i in range(S - S_e - S_m):
if max_col_mut > 5:
temp_count = random.randint(7, max_col_mut)
else:
temp_count = max_col_mut
non_elite_oa.append(brkga_mutation(num_run, temp_count))
# ---- Generation 0 Completed ---- #
goto_next_gen = True
current_gen = 1
while (goto_next_gen):
print " Undergoing generation " + str(current_gen)
# Generate mutants
print " Mutants introduced in population "
for i in range(S_m):
temp = brkga_mutation(num_run, max_col_mut)
non_elite_oa.append(temp)
print temp.string + ' ' + str(temp.get_fitness_value())
# Apply sequencer mutation
seq_mult = int(initial_seq_mult * pow((1 + growth_rate), current_gen))
sel_seq_oa_idx = tournament_selection(non_elite_oa, tour_sel_size, 1)
if seq_mult > seq_mult_max:
seq_mult = seq_mult_max
for i in range(seq_mult):
seq_oa = sequencer_mutation(non_elite_oa[sel_seq_oa_idx],
sequencer_rate)
if seq_oa:
print 'HIT with sequencer mutation'
del non_elite_oa[sel_seq_oa_idx]
non_elite_oa.insert(sel_seq_oa_idx, seq_oa)
# Perform Crossover
print " Successful crossovers "
for i in range(2 * (S - S_m - S_e)):
oa_l = brkga_crossover([
random.choice(elite_oa),
tournament_selection(non_elite_oa, tour_sel_size)
], num_crossover_rate)
for gen_oa in oa_l:
if is_not_subset(
elite_oa,
gen_oa) and gen_oa.array.shape[1] >= min_col_accept:
eq = all_equal_set(non_elite_oa, gen_oa)
if not eq:
non_elite_oa.append(gen_oa)
print gen_oa.string + ' ' + str(
gen_oa.get_fitness_value())
else:
for j in eq:
if gen_oa.get_fitness_value(
) < non_elite_oa[j].get_fitness_value():
del non_elite_oa[j]
non_elite_oa.insert(j, gen_oa)
print gen_oa.string + ' ' + str(
gen_oa.get_fitness_value())
non_elite_oa.sort(key=lambda x: x.get_fitness_value())
non_elite_oa = non_elite_oa[:S - S_e - S_m]
current_gen += 1
print " Final members "
for i in non_elite_oa:
if i.get_fitness_value() < 0.1:
util.dump_oa_to_file(i)
print i.string + ' ' + str(i.get_fitness_value())
if current_gen == max_gen:
cont = raw_input(
'Continue with next generation (Enter N for No): ')
if cont == "N":
goto_next_gen = False
else:
max_gen += 100
for i in non_elite_oa:
util.dump_oa_to_file(i)
