def k_fold(k, label, epochs, params, load_best_weigth, verbose, TensorB,
name_of_best_weight, base_model):
flag = params['agumentation']
data = ld.load_data(
label, phase="aug_evaluation") if flag == True else ld.load_data(
label, phase="evaluation")
results = []
size = len(data['x']) // k
tmp_idx = np.arange(data['x'].shape[0])
np.random.shuffle(tmp_idx)
x = data['x'][tmp_idx]
y = data['y'][tmp_idx]
np.save("x.npy", x)
np.save("y.npy", y)
acc_vec = []
for i in range(k):
x_test = x[i * size:(i + 1) * size]
y_test = y[i * size:(i + 1) * size]
x_train = np.append(x[0:i * size], x[(i + 1) * size:], axis=0)
y_train = np.append(y[0:i * size], y[(i + 1) * size:], axis=0)
tmp = random.sample(range(len(x_train)), 232)
x_val = []
y_val = []
for j in tmp:
x_val.append(x_train[j])
y_val.append(y_train[j])
x_val = np.array(x_val)
y_val = np.array(y_val)
x_train = np.delete(x_train, tmp, axis=0)
y_train = np.delete(y_train, tmp, axis=0)
##########fixing data#########
data = ld.fix_data(flag, x_train, y_train, x_val, y_val, x_test,
y_test)
model = DTL(params=params,
base_model=base_model,
label=label,
data=data)
model.train(epochs, load_best_weigth, verbose, TensorB,
name_of_best_weight + str(i) + ".h5", "k_fold")
results.append(model.evaluate())
print(results[-1])
model.clear()
del model
return results
def prepare_data(self, phase, params):
d_a = load_data(label="a", phase="search")
d_h = load_data(label="h", phase="search")
d_v = load_data(label="v", phase="search")
y = {"y_val", "y_train", "y_test"}
x = {"x_val", "x_train", "x_test", "x_val_128", "x_train_128"}
data = {}
for t in y:
if phase == 1:
data[t] = np.array([[d_a[t][i][0], d_v[t][i][0], d_h[t][i][0]]
for i in range(len(d_a[t]))])
else:
data[t] = [d_a[t], d_v[t], d_h[t]]
for t in x:
data[t] = d_a[t]
if params['agumentation']:
data["x_val"] = ld.normalize(data["x_val"])
data["x_test"] = ld.normalize(data["x_test"])
elif params["scale"]:
data["x_val"] = ld.normalize(data["x_val"])
data["x_test"] = ld.normalize(data["x_test"])
data["x_train"] = ld.normalize(data["x_train"])
return data
def __init__(self,
data,
batch_size,
list_IDs=None,
labels=None,
number_of_classes=2,
shuffle=True):
self.batch_size = batch_size
self.__data = data
preprocess_config = {
'shift_range': 5,
'rotate_range': 360.0,
'flip_ud': True,
'flip_lr': True,
'scale_range': 1.25,
}
self.data_aug = ld.DataAug(**preprocess_config)
help="verbose") opt = parser.parse_args() #Parameters Params = json.load(open(opt.fname, 'r')) Num_epoch = Params['Num_epoch'] Batch_size = Params['Batch_size'] Model_name = Params['Model']['Name'] Model_flatten = Params['Model']['Flatten'] Other_cmd = '' #=================================================================================== # Load Data #=================================================================================== (x_train, y_train), (x_test, y_test) = LoadData.Load_dataset(Params['Dataset']) y_test_org = y_test.copy() #show one plot # import matplotlib.pyplot as plt # plt.imshow(x_train[0]) # plt.show() # check colorful or not #print(x_train.shape) #print(np.shape(x_train.shape)) # scaling x_train = PreProcessing.PrePro_1(x_train) x_test = PreProcessing.PrePro_1(x_test) # (x_train,x_test) = PreProcessing.color_preprocessing_CIFAR10(x_train,x_test)
def k_fold(k, epochs, params1, params2, load_best_weigth, verbose, TensorB,
name_of_best_weight, base_model):
data_tmp = [None, None, None]
flag = None
if params2['agumentation']:
data_tmp[0] = ld.load_data('a', phase="aug_evaluation")
data_tmp[1] = ld.load_data('v', phase="aug_evaluation")
data_tmp[2] = ld.load_data('h', phase="aug_evaluation")
flag = True
else:
data_tmp[0] = ld.load_data('a', phase="evaluation")
data_tmp[1] = ld.load_data('v', phase="evaluation")
data_tmp[2] = ld.load_data('h', phase="evaluation")
flag = False
data = {}
data['x'] = data_tmp[0]['x']
data['y'] = [data_tmp[0]['y'], data_tmp[1]['y'], data_tmp[2]['y']]
size = len(data['x']) // k
tmp_idx = np.arange(data['x'].shape[0])
np.random.shuffle(tmp_idx)
x = data['x'][tmp_idx]
y = [
data['y'][0][tmp_idx], data['y'][1][tmp_idx], data['y'][2][tmp_idx]
]
acc_vec = []
np.save("x.npy", x)
np.save("y.npy", y)
for i in range(k):
x_test = x[i * size:(i + 1) * size]
y_test = [
y[0][i * size:(i + 1) * size], y[1][i * size:(i + 1) * size],
y[2][i * size:(i + 1) * size]
]
x_train = np.append(x[0:i * size], x[(i + 1) * size:], axis=0)
y_train = [
np.append(y[0][0:i * size], y[0][(i + 1) * size:], axis=0),
np.append(y[1][0:i * size], y[1][(i + 1) * size:], axis=0),
np.append(y[2][0:i * size], y[2][(i + 1) * size:], axis=0)
]
y_test1 = np.array([[y[0][i][0], y[1][i][0], y[2][i][0]]
for i in range(i * size, (i + 1) * size)])
y_train1 = [[y[0][i][0], y[1][i][0], y[2][i][0]]
for i in range(i * size)
] + [[y[0][i][0], y[1][i][0], y[2][i][0]]
for i in range((i + 1) * size, len(y[2]))]
y_train1 = np.array(y_train1)
tmp = random.sample(range(len(x_train)), 232)
x_val = []
y_val = [[], [], []]
y_val1 = []
for j in tmp:
x_val.append(x_train[j])
y_val[0].append(y_train[0][j])
y_val[1].append(y_train[1][j])
y_val[2].append(y_train[2][j])
y_val1.append(
[y_train[0][j][0], y_train[1][j][0], y_train[2][j][0]])
y_val1 = np.array(y_val1)
x_val = np.array(x_val)
x_train = np.delete(x_train, tmp, axis=0)
y_train[0] = np.delete(y_train[0], tmp, axis=0)
y_train[1] = np.delete(y_train[1], tmp, axis=0)
y_train[2] = np.delete(y_train[2], tmp, axis=0)
y_train1 = np.delete(y_train1, tmp, axis=0)
for j in range(3):
y_val[j] = np.array(y_val[j])
##########fixing data#########
data2 = ld.fix_data(flag, x_train, y_train, x_val, y_val, x_test,
y_test)
data1 = ld.fix_data(flag, x_train, y_train1, x_val, y_val1, x_test,
y_test1)
model = DMTL(params=params1,
base_model=base_model,
label="",
loss='binary_crossentropy',
second_model=None,
phase=1,
data=data1)
model.train(1,
False,
verbose,
TensorB,
name_of_best_weight + str(i) + ".h5",
"k_fold",
save_m=True)
epochs = [1, 1, 1]
labels = ['a', 'v', 'h']
for l in range(3):
second_model = load_model("model_" + name_of_best_weight +
str(i) + ".h5",
custom_objects={
"a": Meric.a,
"v": Meric.v,
"h": Meric.h,
"loss_a": Meric.loss_a,
"loss_v": Meric.loss_v,
"loss_h": Meric.loss_h
})
model = DMTL(params=params2,
base_model=base_model,
label=labels[l],
loss='binary_crossentropy',
second_model=second_model,
phase=2,
data=data2)
model.train(epochs[l],
load_best_weigth,
verbose,
TensorB,
name_of_best_weight + str(i) + ".h5",
"k_fold",
save_m=False)
ans = model.evaluate()
acc_vec.append(ans)
print(ans)
model.clear()
del model
return acc_vec
def __init__(self, params, base_model, label, data=None):
default_params = {
"agumentation": False,
"scale": False,
"dense_activation": "relu",
"regularizition": 0.0,
"dropout": 0.0,
"optimizer": "adam",
"number_of_dense": 1,
"balancer": "None",
"batch_size": 32
}
default_params.update(params)
Model = base_model
params = default_params
if data == None:
data = load_data(label=label, phase="search")
self.batch_size = params["batch_size"]
if params['agumentation']:
data["x_val"] = ld.normalize(data["x_val"])
data["x_test"] = ld.normalize(data["x_test"])
elif params["scale"]:
data["x_val"] = ld.normalize(data["x_val"])
data["x_test"] = ld.normalize(data["x_test"])
data["x_train"] = ld.normalize(data["x_train"])
regularization = not (params["regularizition"] == 0.0)
dropout = not (params["dropout"] == 0.0)
self.agumnetation = params["agumentation"]
############ Creating CNN ##############
optimizer = params["optimizer"]
inp = Input((64, 64, 1))
con = concatenate([inp, inp, inp])
model = Model(include_top=False, weights='imagenet', input_tensor=con)
x = Flatten()(model.layers[-1].output)
for i in range(params["number_of_dense"]):
if regularization:
x = Dense(params["nn"],
activation=params["dense_activation"],
kernel_regularizer=l2(params["regularizition"]))(x)
else:
x = Dense(params["nn"],
activation=params["dense_activation"])(x)
if dropout:
x = Dropout(params["dropout"])(x)
x = Dense(1, activation="sigmoid", name="classification")(x)
model = tf.keras.Model(model.input, x)
model.compile(optimizer=optimizer,
metrics=["accuracy"],
loss=params["loss"])
self.__model = model
self.__data = data
self.balancer = params["balancer"]
self.__number_of_dense = params["number_of_dense"]
self.details = [
list(params.keys())[i] + ":" + str(list(params.values())[i])
for i in range(len(params))
]
def main():
# User variables
#multiple = 20
#inputfilename = "F:\Documents\simple-gps-points.csv"
#outputimage = "Whitby.png"
#picklefilename = "array.pickle3"
# whitby
#topleft = [54.493873, -0.63386]
#bottomright = [54.468342, -0.602446]
# scale=50000
#largestside = 6000
# system variables
Vars = LoadVariables.OpenFile()
Vars.filename = sys.argv[1]
Vars.ReadFile()
if Vars.ErrorCode == 0:
inputfilename = Vars.inputfilename
outputimage = Vars.outputfilename
topleft = Vars.topleft
bottomright = Vars.bottomright
largestside = Vars.largestside
else:
print "There is a problem with the ini file"
quit(Vars.ErrorCode)
starttime = time.time()
print time.ctime()
mapdata = LoadData.LoadData()
mapdata.inputfilename = inputfilename
mapdata.topleft = topleft
mapdata.bottomright = bottomright
mapdata.largestside = largestside
mapdata.DatabaseCheck()
# this is the bit where we search in the database for the coords..
# we need to convert the GPS coords the the inifile gives us
# into the modified coords in the database
mapdata.MakeArray()
maparray = mapdata.maparray
mapwidth = mapdata.mapwidth
mapheight = mapdata.mapheight
print "Mapheight:", mapheight
print "Mapwidth:", mapwidth
# Make the map pretty
newarray = copy.deepcopy(maparray)
maxvalue = newarray.max()
count = 0
rowcount = 0
GetColourdict = ColourRules.getcolour()
GetColourdict.init()
colourdict = GetColourdict.colourdict
print "emprettify"
for row in maparray:
colcount = 0
for pixel in row:
if pixel != 0:
colourvalue, halfcolourvalue = ColourRules.ReturnColour(
pixel, maxvalue)
# place the half colours
colour = colourdict[colourvalue]
halfcolour = colourdict[halfcolourvalue]
newarray[rowcount][colcount] = colour
if colourvalue > 75:
try:
if rowcount != 0:
if newarray[rowcount -
1][colcount] == colourdict[0]:
newarray[rowcount - 1][colcount] = halfcolour
if rowcount != (mapheight - 1):
if newarray[rowcount +
1][colcount] != colourdict[0]:
newarray[rowcount + 1][colcount] = halfcolour
if colcount != 0:
if newarray[rowcount][colcount +
1] != colourdict[0]:
newarray[rowcount][colcount + 1] = halfcolour
if colcount != (mapwidth - 1):
if newarray[rowcount][colcount -
1] == colourdict[0]:
newarray[rowcount][colcount - 1] = halfcolour
except:
print traceback.print_exc()
print colcount, rowcount
pass
else:
if newarray[rowcount][colcount] == 0:
newarray[rowcount][colcount] = colourdict[0]
colcount += 1
rowcount += 1
print time.ctime(),
print "creating image"
# imageMap=Image.new("RGB", (mapwidth,mapheight), (0,0,0) )
imageMap = Image.frombuffer('RGBA', (mapwidth, mapheight), newarray, 'raw',
'RGBA', 0, 1).transpose(Image.FLIP_TOP_BOTTOM)
imageMap.save(outputimage)
image2 = imageMap.resize((mapwidth / 2, mapheight / 2), Image.ANTIALIAS)
quality_val = 100
image2.save("map7-1.png", 'PNG', quality=quality_val)
# imageMap.show()
endtime = time.time()
print time.ctime()
print "Elapsed time:%s", endtime - starttime