def create_non_homogeneus_cloud(self, symbolClasses):
indicator = 1.0
ratio = global_v.N_LEARNING/(global_v.N_LEARNING+global_v.N_TEST)
for cl in symbolClasses:
for i in range(0, int(indicator)):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color)
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
cl.characteristicsValues[:],
global_v.HOMO_STD_DEV)
scope = int((global_v.N_LEARNING + global_v.N_TEST)/int(indicator))
for j in range (0, scope):
cloud_point = SymbolClass(cl.name, cl.color)
cloud_point.characteristicsValues = self.__generate_distortion(
distortedClass.characteristicsValues[:],
global_v.NON_HOMO_STD_DEV)
# store result
if(j < ratio * scope):
cl.learning_set.append(cloud_point)
else:
cl.test_set.append(cloud_point)
# Info about number of created points
console.write_non_homo(cl.name, int(indicator), "Symbol Class", "Groups")
console.write_point_text_number(">> Number of distorted classes per symbol", len(cl.learning_set) + len(cl.test_set))
console.write_point_text_number(">> Number of learning set points",
len(cl.learning_set))
console.write_point_text_number(">> Number of test set points",
len(cl.test_set))
indicator += 0.5
def create_cluster_assessment_cloud(self, k, symbolClasses):
for cl in symbolClasses:
for i in range(0, k):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color)
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
cl.characteristicsValues[:],
3.5)
scope = int((global_v.N_LEARNING)/k)
for j in range (0, scope):
cloud_point = SymbolClass(cl.name, cl.color)
cloud_point.characteristicsValues = self.__generate_distortion(
distortedClass.characteristicsValues[:],
1.0)
cl.learning_set.append(cloud_point)
# Info about number of created points
console.write_non_homo(cl.name, int(k), "Symbol Class", "Groups")
def create_k_clouds(self, k ,symbolClasses):
count_in_cloud = int(global_v.N_LEARNING/k)
for cl in symbolClasses:
for i in range(0, k):
for j in range(0, count_in_cloud):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color)
values = []
for v in range(0,len(cl.characteristicsValues)):
values.append(cl.characteristicsValues[v] + (cl.characteristicsValues[v]*i*0.2) )
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
values,
global_v.HOMO_STD_DEV)
cl.learning_set.append(distortedClass)
for j in range(0, int(count_in_cloud/2)):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color)
values = []
for v in range(0,len(cl.characteristicsValues)):
values.append(cl.characteristicsValues[v] + (cl.characteristicsValues[v]*i*0.2) )
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
values,
global_v.HOMO_STD_DEV)
cl.test_set.append(distortedClass)
def load_foreign_xls():
startRow = global_v.XLS_START_ROW
maxColumns = global_v.XLS_MAX_COL
foreignClasses = []
print("Opening file:", global_v.FOREIGN_FILE_PATH)
path = ""
wb = open_workbook(global_v.FOREIGN_FILE_PATH)
for s in wb.sheets():
for row in range(startRow, s.nrows):
characteristics = []
for col in range(s.ncols):
if maxColumns > 0 and col == maxColumns + 1:
break
currentValue = float(str(s.cell(row,col).value).replace(',','.'))
if col == 0:
continue
characteristics.append(currentValue)
foreignClass = SymbolClass('foreign', ColorChooser().getForeignColor)
foreignClass.characteristicsValues = characteristics
foreignClasses.append(foreignClass)
return foreignClasses
def create_homogeneous_foreign(nativeClasses, characteristics):
sys.stdout.flush()
# create n Foreign classes
foreignClasses = []
for i in range(0, global_v.CLASS_NUM * (global_v.N_LEARNING)):
name = "Foreign" + str(i)
foreignClass = SymbolClass(name,
ColorChooser().getForeignColor(),
type=SymbolType.FOREIGN)
# Keep generating characteristics for given Foreign class
# while the values are not valid
while True:
foreignCharacteristics = []
# Generate characteristics
for j in range(0, len(characteristics)):
foreignCharacteristic = RandomGenerator().generateRandom(
characteristics[j].interval.lowerBound,
characteristics[j].interval.upperBound)
#foreignCharacteristic += random.gauss(0, global_v.HOMO_STD_DEV)
foreignCharacteristics.append(foreignCharacteristic)
# Check if it is foreign 'enough'
if __isForeign(foreignCharacteristics, nativeClasses):
foreignClass.characteristicsValues = foreignCharacteristics
break
# Add to all classes
foreignClasses.append(foreignClass)
print(" >> Generated:", global_v.CLASS_NUM * (global_v.N_LEARNING),
" Homogeneous Foreign classes")
return foreignClasses
def create_non_homogeneous_foreign(nativeClasses):
sys.stdout.flush()
foreignClasses = []
for i in range(0, len(nativeClasses)):
# take two centers
i_c1 = i
i_c2 = i + 1
if (i_c2 == len(nativeClasses)):
i_c2 = 0
center1 = nativeClasses[i_c1].characteristicsValues
center2 = nativeClasses[i_c2].characteristicsValues
# find the midpoint between two centers
midpoint = []
for p in range(0, len(center1)):
midpoint.append((center1[p] + center2[p]) / 2)
# create a cloud around midpoint
for j in range(0, global_v.N_LEARNING):
foreignClass = SymbolClass("foreign",
ColorChooser().getForeignColor(),
SymbolType.FOREIGN)
foreignClass.characteristicsValues = __generate_distortion(
midpoint[:], global_v.NON_HOMO_STD_DEV)
foreignClasses.append(foreignClass)
print(" >> Generated:", len(foreignClasses),
"Non Homogeneous Foreign classes")
sys.stdout.flush()
return foreignClasses
def load_native_xls():
startRow = global_v.XLS_START_ROW
maxColumns = global_v.XLS_MAX_COL
symbolClasses = []
print("Opening file:", global_v.NATIVE_FILE_PATH)
wb = open_workbook(global_v.NATIVE_FILE_PATH)
for s in wb.sheets():
for row in range(startRow, s.nrows):
characteristics = []
for col in range(s.ncols):
if maxColumns > 0 and col == maxColumns + 1:
break
currentValue = float(str(s.cell(row,col).value).replace(',','.'))
# create new symbol class
if (
(row != startRow and col == 0 and
currentValue != symbolClasses[len(symbolClasses)-1].name) or
(row == startRow and col == 0)
):
symbolClass = SymbolClass(currentValue, ColorChooser().get_color())
symbolClasses.append(symbolClass)
if col == 0:
continue
characteristics.append(currentValue)
distortedClass = SymbolClass(symbolClass.name, symbolClass.color)
distortedClass.characteristicsValues = characteristics
random.choice((symbolClass.learning_set,symbolClass.test_set)).append(distortedClass)
#symbolClass.learning_set.append(distortedClass)
#symbolClasses.append(symbolClass)
return symbolClasses
def create_homogeneous_foreign(nativeClasses, characteristics):
sys.stdout.flush()
# create n Foreign classes
foreignClasses = []
for i in range(0,global_v.CLASS_NUM * (global_v.N_LEARNING)):
name = "Foreign" + str(i)
foreignClass = SymbolClass(name, ColorChooser().getForeignColor(),
type = SymbolType.FOREIGN)
# Keep generating characteristics for given Foreign class
# while the values are not valid
while True:
foreignCharacteristics = []
# Generate characteristics
for j in range(0,len(characteristics)):
foreignCharacteristic = RandomGenerator().generateRandom(
characteristics[j].interval.lowerBound,
characteristics[j].interval.upperBound)
#foreignCharacteristic += random.gauss(0, global_v.HOMO_STD_DEV)
foreignCharacteristics.append(foreignCharacteristic)
# Check if it is foreign 'enough'
if __isForeign(foreignCharacteristics, nativeClasses):
foreignClass.characteristicsValues = foreignCharacteristics
break
# Add to all classes
foreignClasses.append(foreignClass)
print(" >> Generated:",global_v.CLASS_NUM * (global_v.N_LEARNING)," Homogeneous Foreign classes")
return foreignClasses
def generate_symbol_classes(symbolClasses, characteristics):
for i in range(0,global_v.CLASS_NUM):
newSymbol = SymbolClass(i, ColorChooser().get_color(), type = SymbolType.NATIVE_BASE)
loopCounter = 20000 # to control number of iterations
while True:
# Generate random characteristics for new symbol
newCharacteristics =[]
for j in range(0,len(characteristics)):
newCharacteristics.append(
RandomGenerator().generateRandom(characteristics[j].interval.lowerBound,
characteristics[j].interval.upperBound))
# Check generated numbers
found = False
for c in range(0,i):
eucl = euclidian_distance(symbolClasses[c].characteristicsValues[:], newCharacteristics[:])
if (eucl < global_v.EUCL_MIN_D or eucl > global_v.EUCL_MAX_D) and loopCounter > 0:
found = True
if(not found):
break
loopCounter -= 1
# Save symbol
newSymbol.characteristicsValues = newCharacteristics
symbolClasses.append(newSymbol)
# INFO
f = open(os.path.join("..","log",global_v.DIR_NAME,"NATIVE_SYMBOLS.txt"), 'w')
for symbolClass in symbolClasses:
console.write_symbol_classes(f,symbolClass.name,symbolClass.characteristicsValues,text="Symbol Class:", )
f.close()
def load_native_xls():
startRow = global_v.XLS_START_ROW
maxColumns = global_v.XLS_MAX_COL
symbolClasses = []
logger.log("Opening file: " + str(global_v.NATIVE_FILE_PATH))
wb = open_workbook(global_v.NATIVE_FILE_PATH)
for s in wb.sheets():
for row in range(startRow, s.nrows):
characteristics = []
for col in range(s.ncols):
if maxColumns > 0 and col == maxColumns + 1:
break
currentValue = float(
str(s.cell(row, col).value).replace(',', '.'))
# create new symbol class
if ((row != startRow and col == 0 and currentValue !=
symbolClasses[len(symbolClasses) - 1].name)
or (row == startRow and col == 0)):
symbolClass = SymbolClass(int(currentValue),
ColorChooser().get_color())
symbolClasses.append(symbolClass)
if col == 0:
continue
characteristics.append(currentValue)
distortedClass = SymbolClass(symbolClass.name, symbolClass.color)
distortedClass.characteristicsValues = characteristics
random.choice((symbolClass.learning_set,
symbolClass.test_set)).append(distortedClass)
return symbolClasses
def load_txt(filepath):
symbolClasses = []
filepath = 'native1.txt'
f = open(filepath)
lines = f.readlines()
class_num = lines[0].split()[1]
n_learning = lines[1].split()[1]
n_test = lines[2].split()[1]
prev_name = 0
for i in range(3, len(lines)):
line = lines[i].split()
name = line[0]
if prev_name != name or i == 3:
symbolClass = SymbolClass(name, ColorChooser().get_color())
for c in range(1, len(line)):
symbolClass.characteristicsValues.append(float(line[c]))
symbolClasses.append(symbolClass)
else:
symbol = SymbolClass(symbolClasses[int(name)].name,
symbolClasses[int(name)].color)
for c in range(1, len(line)):
symbol.characteristicsValues.append(float(line[c]))
if ((i - 3) %
(int(n_learning) + int(n_test) + 1) < int(n_learning)):
symbolClasses[int(name)].learning_set.append(symbol)
else:
symbolClasses[int(name)].test_set.append(symbol)
prev_name = name
return symbolClasses
def create_non_homogeneous_foreign(nativeClasses):
sys.stdout.flush()
foreignClasses = []
for i in range(0, len(nativeClasses)):
# take two centers
i_c1 = i
i_c2 = i + 1
if(i_c2 == len(nativeClasses)):
i_c2 = 0
center1 = nativeClasses[i_c1].characteristicsValues
center2 = nativeClasses[i_c2].characteristicsValues
# find the midpoint between two centers
midpoint = []
for p in range(0,len(center1)):
midpoint.append((center1[p] + center2[p]) / 2)
# create a cloud around midpoint
for j in range(0,global_v.N_LEARNING):
foreignClass = SymbolClass("foreign", ColorChooser().getForeignColor(), SymbolType.FOREIGN)
foreignClass.characteristicsValues = __generate_distortion(midpoint[:] ,global_v.NON_HOMO_STD_DEV)
foreignClasses.append(foreignClass)
print(" >> Generated:", len(foreignClasses) ,"Non Homogeneous Foreign classes")
sys.stdout.flush()
return foreignClasses
def plot_clusters(learningClusters, testClusters):
symbolLearn = SymbolClass("0", ColorChooser().get_color())
symbolLearn.clusters = learningClusters
symbolTest = SymbolClass("0", ColorChooser().get_color())
symbolTest.clusters = testClusters
symbols = [symbolLearn]
Plot3D().renderPlot(symbols)
symbols = [symbolTest]
Plot3D().renderPlot(symbols)
def create_clone_foreign(nativeClasses):
foreignClasses = []
for nc in nativeClasses:
for dc in nc.learning_set:
name = "Foreign"
foreignClass = SymbolClass(name, ColorChooser().getForeignColor(),
type = SymbolType.FOREIGN)
foreignClass.characteristicsValues = dc.characteristicsValues
foreignClasses.append(foreignClass)
return foreignClasses
def create_clone_foreign(nativeClasses):
foreignClasses = []
for nc in nativeClasses:
for dc in nc.learning_set:
name = "Foreign"
foreignClass = SymbolClass(name,
ColorChooser().getForeignColor(),
type=SymbolType.FOREIGN)
foreignClass.characteristicsValues = dc.characteristicsValues
foreignClasses.append(foreignClass)
return foreignClasses
def serialize_chosen_elements(nativeElements):
logger.log_header("Choosing Native elements")
chosenNativeElements = SymbolClass("", ColorChooser().get_color())
m_filename = "["
# Go through all symbols classes and choose the classes we want
for i in range(0, len(nativeElements)):
if (nativeElements[i].name in global_v.NATIVE_CLASSES
or len(global_v.NATIVE_CLASSES) == 0):
chosenNativeElements.learning_set += nativeElements[i].learning_set
chosenNativeElements.test_set += nativeElements[i].test_set
chosenNativeElements.name += str(nativeElements[i].name) + ", "
m_filename += str(nativeElements[i].name) + ", "
chosenNativeElements.name = chosenNativeElements.name.rstrip(", ")
m_filename = m_filename.rstrip(", ")
m_filename += "]"
for learning_element in chosenNativeElements.learning_set:
element_str = str(learning_element.characteristicsValues)
element_str = element_str.strip("[]")
element_str = element_str.rstrip("]")
logger.log(element_str,
filename="training" + "_" + m_filename + ".txt",
styles=[logger.LogStyle.NONE, logger.LogStyle.FILE_ONLY],
text_indent="")
for test_element in chosenNativeElements.test_set:
element_str = str(test_element.characteristicsValues)
element_str = element_str.strip("[")
element_str = element_str.rstrip("]")
logger.log(element_str,
filename="test" + "_" + m_filename + ".txt",
styles=[logger.LogStyle.NONE, logger.LogStyle.FILE_ONLY],
text_indent="")
# Log
logger.log(str(chosenNativeElements))
return chosenNativeElements
def generate_symbol_classes(symbolClasses, characteristics):
for i in range(0, global_v.CLASS_NUM):
newSymbol = SymbolClass(i,
ColorChooser().get_color(),
type=SymbolType.NATIVE_BASE)
loopCounter = 20000 # to control number of iterations
while True:
# Generate random characteristics for new symbol
newCharacteristics = []
for j in range(0, len(characteristics)):
newCharacteristics.append(RandomGenerator().generateRandom(
characteristics[j].interval.lowerBound,
characteristics[j].interval.upperBound))
# Check generated numbers
found = False
for c in range(0, i):
eucl = euclidian_distance(
symbolClasses[c].characteristicsValues[:],
newCharacteristics[:])
if (eucl < global_v.EUCL_MIN_D
or eucl > global_v.EUCL_MAX_D) and loopCounter > 0:
found = True
if (not found):
break
loopCounter -= 1
# Save symbol
newSymbol.characteristicsValues = newCharacteristics
symbolClasses.append(newSymbol)
# INFO
f = open(
os.path.join("..", "log", logger.LOG_CURRENT_DIR_PATH,
"NATIVE_SYMBOLS.txt"), 'w')
for symbolClass in symbolClasses:
console.write_symbol_classes(
f,
symbolClass.name,
symbolClass.characteristicsValues,
text="Symbol Class:",
)
f.close()
def create_homogeneus_cloud(self, symbolClasses):
for cl in symbolClasses:
# Learning set
for i in range(0, global_v.N_LEARNING + global_v.N_TEST):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color, type = SymbolType.NATIVE_LEARNING)
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
cl.characteristicsValues[:],
global_v.HOMO_STD_DEV)
# store result
if(i < global_v.N_LEARNING):
cl.learning_set.append(distortedClass)
else:
cl.test_set.append(distortedClass)
print(cl.name)
console.write_point_text_number(">> Number of learning set points",
len(cl.learning_set))
console.write_point_text_number(">> Number of test set points",
len(cl.test_set))
def create_homogeneus_cloud(self, symbolClasses):
for cl in symbolClasses:
# Learning set
for i in range(0, global_v.N_LEARNING + global_v.N_TEST):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color, type = SymbolType.NATIVE_LEARNING)
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
cl.characteristicsValues[:],
global_v.HOMO_STD_DEV)
# store result
if(i < global_v.N_LEARNING):
cl.learning_set.append(distortedClass)
else:
cl.test_set.append(distortedClass)
print(cl.name)
console.write_point_text_number(">> Number of learning set points",
len(cl.learning_set))
console.write_point_text_number(">> Number of test set points",
len(cl.test_set))
def load_foreign_xls():
startRow = global_v.XLS_START_ROW
maxColumns = global_v.XLS_MAX_COL
foreignClasses = []
skip = False
number = -1
logger.log("Opening file: " + str(global_v.FOREIGN_FILE_PATH))
logger.log("Including classes: " + str(global_v.FOREIGN_CLASSES))
path = ""
wb = open_workbook(global_v.FOREIGN_FILE_PATH)
for s in wb.sheets():
for row in range(startRow, s.nrows):
characteristics = []
for col in range(s.ncols):
if maxColumns > 0 and col == maxColumns + 1:
break
if col == 0:
number = int(s.cell(row, col).value)
if (number in global_v.FOREIGN_CLASSES
or len(global_v.FOREIGN_CLASSES) == 0):
skip = False
else:
skip = True
continue
currentValue = float(
str(s.cell(row, col).value).replace(',', '.'))
characteristics.append(currentValue)
if not skip:
foreignClass = SymbolClass('foreign: ' + str(number),
ColorChooser().getForeignColor)
foreignClass.characteristicsValues = characteristics
foreignClasses.append(foreignClass)
return foreignClasses
def create_non_homogeneus_cloud(self, symbolClasses):
indicator = 1.0
ratio = global_v.N_LEARNING/(global_v.N_LEARNING+global_v.N_TEST)
for cl in symbolClasses:
for i in range(0, int(indicator)):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color)
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
cl.characteristicsValues[:],
global_v.HOMO_STD_DEV)
scope = int((global_v.N_LEARNING + global_v.N_TEST)/int(indicator))
for j in range (0, scope):
cloud_point = SymbolClass(cl.name, cl.color)
cloud_point.characteristicsValues = self.__generate_distortion(
distortedClass.characteristicsValues[:],
global_v.NON_HOMO_STD_DEV)
# store result
if(j < ratio * scope):
cl.learning_set.append(cloud_point)
else:
cl.test_set.append(cloud_point)
# Info about number of created points
console.write_non_homo(cl.name, int(indicator), "Symbol Class", "Groups")
console.write_point_text_number(">> Number of distorted classes per symbol", len(cl.learning_set) + len(cl.test_set))
console.write_point_text_number(">> Number of learning set points",
len(cl.learning_set))
console.write_point_text_number(">> Number of test set points",
len(cl.test_set))
indicator += 0.5
def create_k_clouds(self, k ,symbolClasses):
count_in_cloud = int(global_v.N_LEARNING/k)
for cl in symbolClasses:
for i in range(0, k):
for j in range(0, count_in_cloud):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color)
values = []
for v in range(0,len(cl.characteristicsValues)):
values.append(cl.characteristicsValues[v] + (cl.characteristicsValues[v]*i*0.2) )
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
values,
global_v.HOMO_STD_DEV)
cl.learning_set.append(distortedClass)
for j in range(0, int(count_in_cloud/2)):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color)
values = []
for v in range(0,len(cl.characterpisticsValues)):
values.append(cl.characteristicsValues[v] + (cl.characteristicsValues[v]*i*0.2) )
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
values,
global_v.HOMO_STD_DEV)
cl.test_set.append(distortedClass)
def plot_clusters(learningClusters, testClusters):
symbolLearn = SymbolClass("0", ColorChooser().get_color())
symbolLearn.clusters = learningClusters
symbolTest = SymbolClass("0", ColorChooser().get_color())
symbolTest.clusters = testClusters
symbols = [symbolLearn]
Plot3D().renderPlot(symbols)
symbols = [symbolTest]
Plot3D().renderPlot(symbols)
def create_cluster_assessment_cloud(self, k, symbolClasses):
for cl in symbolClasses:
for i in range(0, k):
# instance of new symbol
distortedClass = SymbolClass(cl.name, cl.color)
# randomize position around a given class(based on position)
distortedClass.characteristicsValues = self.__generate_distortion(
cl.characteristicsValues[:],
global_v.HOMO_CENTER_STD_DEV)
scope = int((global_v.N_LEARNING)/k)
for j in range (0, scope):
cloud_point = SymbolClass(cl.name, cl.color)
cloud_point.characteristicsValues = self.__generate_distortion(
distortedClass.characteristicsValues[:],
global_v.HOMO_STD_DEV)
cl.learning_set.append(cloud_point)
def deserialize_native():
nativeSymbols = SymbolClass("Native", ColorChooser().get_color())
# Training
f = open(global_v.NATIVE_TRAINING_FILE)
lines = f.readlines()
for l in range(2, len(lines)):
line = lines[l].split(", ")
features = []
symbol = SymbolClass("Native", nativeSymbols.color)
for i in range(0, len(line)):
features.append(float(line[i]))
symbol.characteristicsValues = features
nativeSymbols.learning_set.append(symbol)
# Testing
if global_v.NATIVE_TESTING_FILE:
f2 = open(global_v.NATIVE_TESTING_FILE)
lines = f2.readlines()
for l in range(2, len(lines)):
line = lines[l].split(", ")
features = []
symbol = SymbolClass("Native", nativeSymbols.color)
for i in range(0, len(line)):
features.append(float(line[i]))
symbol.characteristicsValues = features
nativeSymbols.test_set.append(symbol)
return nativeSymbols
f.close()
f2.close()
