def trainModel(model, X_train, y_train):
Console.info("Create validation sets, training set, testing set...")
# Split images dataset
k = int(len(X_train) / 6) # Decides split count
hist_test = X_train[:k]
hand_test = y_train[:k]
hist_valid = X_train[k: 2 * k]
hand_valid = y_train[k: 2 * k]
hist_train = X_train[2 * k:]
hand_train = y_train[2 * k:]
Console.info("Training network...")
history = model.fit(
hist_train,
hand_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=2,
validation_data=(hist_valid, hand_valid),
callbacks=loadCallBack(),
)
Console.info("Save model to disck...")
# Path to save model
PATHE_SAVE_MODEL = os.path.join(__location__, "model")
# Save weights after every epoch
if not os.path.exists(PATHE_SAVE_MODEL):
os.makedirs(PATHE_SAVE_MODEL)
# serialize model to YAML
model_yaml = model.to_yaml()
with open(
os.path.join(PATHE_SAVE_MODEL, "model_hands_not_hands.yaml"), "w"
) as yaml_file:
yaml_file.write(model_yaml)
# serialize weights to HDF5
model.save_weights(os.path.join(PATHE_SAVE_MODEL, "model_hands_not_hands.h5"))
# save image of build model
# save image of build model
plot_model(
model,
to_file=os.path.join(PATHE_SAVE_MODEL, "model_hands_not_hands.png"),
show_shapes=True
)
print("OK")
# evaluate the network
Console.info("Evaluating network...")
score = model.evaluate([hist_test], [hand_test], batch_size=BATCH_SIZE, verbose=1)
Console.log("Test loss:", score[0])
Console.log("Test Acc:", score[1])
# list all data in history
Console.info("Save model history graphics...")
print(history.history.keys())
def getFiles():
rta = []
# Read csv
df = pd.read_csv(
os.path.join(__location__, "dataset", "histogram-dataset.csv"))
# file names on train_dir
files = os.listdir(train_dir)
# filter image files
files = [f for f in files if fnmatch.fnmatch(f, "*.png")]
# Sort randomly
np.random.shuffle(files)
for file_name in files:
# Cut list of file
if CUT_DATASET <= 0 or len(rta) < CUT_DATASET:
# print(file_name)
# Get row with id equil image's name
csv_row = df[df.id == int(file_name[:-4])]
# Get lower color
lower = csv_row.lower.tolist()
# Get upper color
upper = csv_row.upper.tolist()
if (lower and upper) and (not (lower[0] != lower[0]
and upper[0] != upper[0])):
rta.append((file_name, lower[0], upper[0]))
else:
Console.log("Not data for", file_name)
return rta
def loadModel(dir_model_backup):
Console.info("Get model and weights for", dir_model_backup)
# load YAML and create model
yaml_file = open(os.path.join(__location__, "model", dir_model_backup + ".yaml"), "r")
loaded_model_yaml = yaml_file.read()
yaml_file.close()
model = model_from_yaml(loaded_model_yaml)
# load weights into new model
model.load_weights(os.path.join(__location__, "model", dir_model_backup + ".h5"))
Console.log("Loaded model from disk")
return model
def processeImg(files, y_lower_upper):
total_file = len(files)
Console.log("Process", total_file, "images")
x_files = []
for i in range(total_file):
min_color = int(y_lower_upper[0][i])
max_color = int(y_lower_upper[1][i])
min_color = min_color if min_color > 0 else 0
max_color = max_color if max_color < 255 else 255
x_files.append((files[i], min_color, max_color))
# Usado en caso de usar multiples core
output = multiprocessing.Queue()
num_processes = multiprocessing.cpu_count()
if platform.system() == "Linux" and num_processes > 1:
processes = []
lot_size = int(total_file / num_processes)
for x in range(1, num_processes + 1):
if x < num_processes:
lote = x_files[(x - 1) * lot_size: ((x - 1) * lot_size) + lot_size]
else:
lote = x_files[(x - 1) * lot_size:]
processes.append(Process(target=mpProcessImg, args=(lote, output)))
if len(processes) > 0:
Console.info("Fix colors of the images...")
for p in processes:
p.start()
result = []
for x in range(num_processes):
result.append(output.get(True))
for p in processes:
p.join()
X_values = []
for x in result:
X_values = X_values + x
updateProgress(1, total_file, total_file, "")
Console.log("Image processed:", len(X_values))
else:
Console.info("We can not divide the load into different processors")
# X_values = mpGetHistogramFormFiles(files)
exit(0)
return X_values
def writeFile(dataset, X_img, y_img):
Console.log("Saving", dataset, "data...")
file_name = "img-for-autoencoder-" + dataset + ".hdf5"
path_to_save = os.path.join(__location__, "dataset", file_name)
with h5py.File(os.path.join(path_to_save), "w") as f:
f.create_dataset(
"x_img",
data=X_img,
dtype=np.float16,
compression="gzip",
compression_opts=5,
)
f.create_dataset("y_img", data=y_img, dtype=np.uint8)
f.close()
def getHistogramFormFiles(files=[]):
total_file = len(files)
Console.log("Process", total_file, "images")
# Usado en caso de usar multiples core
output = multiprocessing.Queue()
num_processes = multiprocessing.cpu_count()
if platform.system() == "Linux" and num_processes > 1:
processes = []
lot_size = int(total_file / num_processes)
for x in range(1, num_processes + 1):
if x < num_processes:
lote = files[(x - 1) * lot_size: ((x - 1) * lot_size) + lot_size]
else:
lote = files[(x - 1) * lot_size:]
processes.append(Process(target=mpGetHistogramFormFiles, args=(lote, output)))
if len(processes) > 0:
Console.info("Get histogram of the images...")
for p in processes:
p.start()
result = []
for x in range(num_processes):
result.append(output.get(True))
for p in processes:
p.join()
X_values = []
for x in result:
X_values = X_values + x
updateProgress(1, total_file, total_file, "")
Console.log("Image processed:", len(X_values))
else:
Console.info("We can not divide the load into different processors")
# X_values = mpGetHistogramFormFiles(files)
exit(0)
return X_values
Console.info("No podemos dividir la cargan en distintos procesadores")
exit(0)
if __name__ == "__main__":
if MAKE_HANDS_FROM_HUMAN:
makeHandsHuman()
TRAIN_DIR = ["dataset_hands", "hands"]
files = getFiles(TRAIN_DIR)
(X_train, y_train) = progressFiles(TRAIN_DIR, files, hands_valid=1)
TRAIN_DIR = ["dataset_hands", "not_hands"]
files = getFiles(TRAIN_DIR)
(X2_train, y2_train) = progressFiles(TRAIN_DIR, files, hands_valid=0)
X_train = X_train + X2_train
y_train = y_train + y2_train
# Sort randomly
random_id = np.random.choice(len(X_train),
size=len(y_train),
replace=False)
X_train = np.asarray(X_train)[random_id]
y_train = np.asarray(y_train)[random_id]
saveDataSet(X_train, y_train)
hist, valid = openDataSet()
Console.log("Dataset", len(hist[0]), len(valid))
Console.log("Test loss:", score[0])
Console.log("Test Acc:", score[1])
# list all data in history
Console.info("Save model history graphics...")
print(history.history.keys())
# Como vamos a usar multi procesos uno por core.
# Los procesos hijos cargan el mismo código.
# Este if permite que solo se ejecute lo que sigue si es llamado
# como proceso raíz.
if __name__ == "__main__":
if args["predict"] == None or args["predict"] == "True":
(X_train, y_train) = openDataSet()
Console.log("Dataset file count", len(y_train))
# Create model
model = makerModel()
if args["train"] == "True":
trainModel(model, X_train, y_train)
if args["evaluate"] == "True":
Console.info("Evaluating model...")
score = model.evaluate([X_train], [y_train], batch_size=BATCH_SIZE, verbose=1)
Console.log("Test loss:", score[0])
Console.log("Test Acc:", score[1])
if args["predict"] != None and args["predict"] != "False":
if args["predict"] != "True":
for x in range(num_processes):
result.append(output.get(True))
for p in processes:
p.join()
X_img = []
y_img = []
for mp_X_img, mp_y_img in result:
X_img = X_img + mp_X_img
y_img = y_img + mp_y_img
updateProgress(1, total_file, total_file, "")
return X_img, y_img
else:
Console.info("No podemos dividir la cargan en distintos procesadores")
exit(0)
if __name__ == "__main__":
# Make two folder for hands and not_hands, with histogram values on csv file
if MAKE_HANDS_FROM_HUMAN:
makeHandsHuman()
files = getFiles()
X_img, y_img = progressFiles(files)
saveDataSet(X_img, y_img)
X_img, y_img = openDataSet("training")
Console.log("Dataset", len(X_img), len(y_img))
