def merge_result():
y_test = readData(save_work_file, 'y_test_n0')
reject = readData(save_work_file, 'reject_n0')
for i in range(1, count_thread):
y_test = np.concatenate(
(y_test, readData(save_work_file, 'y_test_n' + str(i))), axis=0)
reject = np.concatenate(
(reject, readData(save_work_file, 'reject_n' + str(i))), axis=0)
saveData(save_work_file, 'y_test', y_test)
saveData(save_work_file, 'reject', reject)
def merging(n_files = count_thread):
points = np.zeros( (0, count_pair_points, count_shifts, count_pixels, count_pixels) )
rej = np.zeros( (0, count_pair_points, count_shifts))
res = np.zeros( (0, count_pair_points, count_shifts))
for i in range(n_files):
points = np.concatenate( (points,readData(os.path.join(path,'points_05_09_2018_'+str(0)), 'points_' + str(0))), axis = 0 )
rej = np.concatenate( (rej, readData(os.path.join(path,'reject_05_09_2018_'+str(0)), 'reject_' + str(0))), axis = 0)
res = np.concatenate( (res, readData(os.path.join(path,'results_05_09_2018_'+str(0)), 'results_' + str(0))), axis = 0)
points = np.expand_dims(points, axis = -1)
landscapes = X_data.copy()
landscapes = np.expand_dims(landscapes, axis=-1)
input_data = np.concatenate( (landscapes, points), axis = -1)
divided_train_test(input_data, res, rej)
def get_and_show_reject():
reject = readData('reject.hdf5', 'reject')
print(reject.shape)
rej_test = reject.reshape((reject.shape[0] * reject.shape[1] * reject.shape[2], -1))[count_train:]
rej_test = rej_test.reshape(-1)
print(rej_test.shape)
plt.hist(rej_test, bins=[-3, -2, -1, 0, 1, 2, 3])
plt.title("histogram")
plt.show()
return rej_test
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Aug 6 19:16:52 2018
@author: andrey
"""
# %% когда y_test = 0; reject <0, если y_test = 1, reject > 0
from myUtils import readData, saveData
import numpy as np
import os
work_file = '/home/andrey/datasetsNN/landScapes/landScape_3000_32/mix_fixed/dataset_30_07_2018'
save_work_file = '/home/andrey/datasetsNN/landScapes/landScape_3000_32/mix_fixed/dataset_07_08_2018'
y_test = readData(work_file, 'y_test')
x_test = readData(work_file, 'x_test')
count_points = 2
count_thread = 4
nThread = np.arange(count_thread)
count_el_one_thread = len(x_test) // count_thread
if (len(x_test) % count_thread != 0):
print('WARNING, len(x_test)%count_thread != 0 ')
print(x_test.shape)
print(y_test.shape)
# %%
def separate_landscape_and_points(matrix):
print(matrix.shape)
mod_matrix = np.moveaxis(matrix, 3, 0)
"""
# %%
import numpy as np
from myUtils import readData, saveData
count_landscapes = 3000
count_tuples = 15 # количество комбанаций в данном случае трех точек на одном ландшафте
count_shifts = 8
count_pixels = 32
count_points = 3
count_train = 300000
# %%
first_dim = count_landscapes * count_tuples * count_shifts
# %%
data = readData('dataset.hdf5', 'input')
print(data.shape)
# %%
data = np.reshape(data, (first_dim, 4, count_pixels, count_pixels))
print(data.shape)
np.moveaxis
data = np.moveaxis(data, 1, -2) # recheck
print(data.shape)
# %%
result = readData('result.hdf5', 'result')
print(result.shape)
# %%
result = np.reshape(result, (first_dim, 3))
print(result.shape)
# %%
distAll = getDistance_arrays(ind_all, matrixData)
vectorAll = hist_dist(distAll)
distFP = getDistance_arrays(ind_FP, matrixData)
distFN = getDistance_arrays(ind_FN, matrixData)
vectorFN = hist_dist(distFN)
vectorFP = hist_dist(distFP)
relFP = vectorFP[0] / vectorAll[0]
print('relFP= ', relFP)
relFN = vectorFN[0] / vectorAll[0]
print('relFN= ', relFN)
# %%
if __name__ == "__main__":
x_test = readData(path_to_dataset, name_x_test)
y_test = readData(path_to_dataset, name_y_test)
rej_test = readData(path_to_dataset, name_rej_test)
print('x_test shape: ', x_test.shape)
print('y_test shape: ', y_test.shape)
print('rej_test shape: ', rej_test.shape)
model = load_model(path_to_model)
print(model.summary())
# для graphCNN
# p_test = getPoints(x_test)
# x_test = x_test[:,:,:,:1] # 360000*32*32*4 ->360000*32*32*1 (убираем матрицы точек)
res_predict = model.predict([x_test, ])
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed May 16 14:40:15 2018
@author: andrey
"""
# %%
from myUtils import readData
import numpy as np
data = readData('ready3PointTestData.hdf5', 'testData')
result = readData('ready3PointTestResult.hdf5', 'testResult')
reject = readData('reject.hdf5', 'reject')
reject = reject.reshape(
(reject.shape[0] * reject.shape[1] * reject.shape[2], -1))[300000:]
print(reject.shape)
# %%
print(reject.shape)
print(result.shape)
# %%
ind_clear = np.where(((reject < 0) & (result == 1)))[0]
print(ind_clear[:10])
ind_clear = np.unique(ind_clear)
print(len(ind_clear))
print(ind_clear)
# %%
#
clData = np.empty(
(len(ind_clear), data.shape[1], data.shape[2], data.shape[3]))
import math
from IterativeAlgorithmWithAnswer import calculateResult as calcRes_2points
from myUtils import readData, saveData
from multiprocessing import Pool
count_thr = 15
# %%
count_landscapes = 3000
count_tuples = 5 # количество комбанаций в данном случае трех точек на одном ландшафте
count_shifts = 8
count_pixels = 32
count_points = 3
path = 'Landscapes_3000_32x32_clear.hdf5'
name_dataset = 'Landscapes'
X_data = readData(path, name_dataset)
print(X_data.shape)
# %%
def norm2(p1, p2):
return math.sqrt((p2[0] - p1[0]) * (p2[0] - p1[0]) + (p2[1] - p1[1]) *
(p2[1] - p1[1]))
def random_points():
p1 = (random.randint(0, count_pixels - 1),
random.randint(0, count_pixels - 1))
p2 = p1
while (norm2(p1, p2) < math.sqrt(8)):
p2 = (random.randint(0, count_pixels - 1),
from numpy import linalg as LA
from IterativeAlgorithmWithAnswer import calculateResult
import os
from myUtils import saveData
from multiprocessing import Pool
count_landscapes = 270
count_pair_points = 15
count_shifts = 8
count_pixels = 32
shift = 2 # cдвиг от границы при рандомной генерации точек
min_reject = 1e-03 # минимальный доступимый отклонение.
count_thread = 20
count_train = 2700 # count unequal landscapes
path = ''
save_path = 'train/'
X_data = readData(os.path.join(path,'Landscapes_05_09_2018.hdf5'), 'Landscapes')
print(X_data.shape)
#%%
def random_points():
p1 = np.random.randint(shift, count_pixels - shift - 1, 2)
p2 = p1
while (LA.norm(p1-p2) < math.sqrt(8)): # чтобы точки не были близки друг к другу
p2 = np.random.randint(shift, count_pixels - shift - 1, 2)
return p1, p2
#point_data = np.zeros((count_landscapes, count_pair_points, count_shifts, count_pixels, count_pixels))
#result = np.zeros((count_landscapes, count_pair_points, count_shifts))
#
#
plt.scatter(data0[:, 0], data0[:, 1], color='red', marker='o')
plt.scatter(data1[:, 0], data1[:, 1], color='blue', marker='x')
x1_line_point = [-7, 0, 7] # для линии раздленеия
x2_line_point = x1_line_point
x2_line_point[0] = (-w[0] - w[1] * x1_line_point[0]) / w[2]
x2_line_point[1] = (-w[0] - w[1] * x1_line_point[1]) / w[2]
x2_line_point[2] = (-w[0] - w[1] * x1_line_point[2]) / w[2]
print(x2_line_point)
plt.plot(x1_line_point, x2_line_point)
# plt.plot([-10,10], [-25.23456990138067, 22.200020415768552])
plt.show()
#%%
if __name__ == "__main__":
data0 = readData(os.path.join(path_save, LS), 'data0')
data1 = readData(os.path.join(path_save, LS), 'data1')
label0 = np.full(len(data0), 0) # -1 для adaline, у остальных - 0
label1 = np.full(len(data0), 1)
data, label = merge_and_mix_train_data(data0, data1, label0, label1)
count_train = len(data) // 2
x_train, y_train, x_test, y_test = data[:
count_train], label[:count_train], data[
count_train:], label[
count_train:]
w = train(x_train, y_train)
evaluate(x_test, y_test, w)
evaluate(data, label, w)
show_results(data0, data1, w)
#%%
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Apr 4 19:46:16 2018
@author: andrey
"""
# %%
work_dir = '/home/andrey/datasetsNN/landScapes/landScape_3000_32/mix_fixed/'
from myUtils import readData
x_train = readData(work_dir + 'dataset_30_07_2018', 'x_train')
x_test = readData(work_dir + 'dataset_30_07_2018', 'x_test')
y_train = readData(work_dir + 'dataset_30_07_2018', 'y_train')
y_test = readData(work_dir + 'dataset_30_07_2018', 'y_test')
# %%
import keras
import math
import keras.initializers as ki
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K
batch_size = 256
epochs = 100
def getModel(input_shape=(32, 32, 2)):
model = Sequential()
Points = np.zeros((len(data), 6))
for i in range(len(data)):
matrixes = np.reshape(data[i], (4, 32, 32))
p = np.zeros((6))
indp = 0
indm = 1
while (indp < 5):
p[indp], p[indp + 1] = getIndexNonZeroInMatrix(matrixes[indm])
indp += 2
indm += 1
Points[i] = p
return Points
# %%
x_train = readData('clearTrainData.hdf5', 'input')
x_test = readData('clearTestData.hdf5', 'input')
y_train = readData('clearTrainResult.hdf5', 'result')
y_test = readData('clearTestResult.hdf5', 'result')
p_train = getPoints(x_train)
p_test = getPoints(x_test)
x_train = x_train[:, :, :, :1] # 360000*32*32*4 ->360000*32*32*1
x_test = x_test[:, :, :, :1] # 360000*32*32*4 ->360000*32*32*1
print("x")
print(x_train.shape)
print(x_test.shape)
print("y")