def load_data_and_run(model, input_shape, TBCallBack):
train_data_loader = ImageDataGenerator(samplewise_center=True,
samplewise_std_normalization=True)
train_data = train_data_loader.flow_from_directory(
r"E:\Gitlab\MarkerTrainer\data_train",
target_size=(input_shape, input_shape),
batch_size=128,
color_mode="grayscale",
class_mode='binary')
validation_data_loader = ImageDataGenerator(
samplewise_center=True, samplewise_std_normalization=True)
validation_data = validation_data_loader.flow_from_directory(
r"E:\Gitlab\MarkerTrainer\data_validate",
target_size=(input_shape, input_shape),
batch_size=128,
color_mode="grayscale",
class_mode='binary')
model_name = os.path.join(r"E:\Gitlab\MarkerTrainer\models", unique_name())
checkpoint = ModelCheckpoint(model_name,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [TBCallBack, checkpoint]
model.fit_generator(train_data,
steps_per_epoch=100,
epochs=20,
validation_data=validation_data,
validation_steps=100,
callbacks=callbacks_list)
model.save(os.path.join(r'E:\Gitlab\MarkerTrainer\models\\',
unique_name()))
def load_data_and_run(model, input_shape, TBCallBack):
load_dotenv()
train_path = os.getenv("train_path")
train_csv_path = os.getenv("train_csv_path")
validate_path = os.getenv("validate_path")
validate_csv_path = os.getenv("validate_csv_path")
# Dynamicly generate model path.
project_root = os.path.realpath(__file__)
model_path = os.path.join(os.path.dirname(project_root), "models")
generate_csv(train_path, train_csv_path)
train_data = DataSequence(train_csv_path, 128, mode="Train")
generate_csv(validate_path, validate_csv_path)
validation_data = DataSequence(validate_csv_path, 128)
model_name = os.path.join(model_path, unique_name())
checkpoint = ModelCheckpoint(model_name,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [TBCallBack, checkpoint]
model.fit_generator(train_data,
steps_per_epoch=256,
epochs=500,
validation_data=validation_data,
validation_steps=256,
callbacks=callbacks_list)
model.save(os.path.join(model_path, unique_name()))
def list_random(bg_list, overlay_list, output_path, samples):
"""
Output the combinatino of BG and OVERLAY into a TARGET folder. in random draw manner.
:param bg_folder:
:param overlay_folder:
:param output_path:
:return:
"""
from random import randint
# Generate this number of samples.
i = 0
pbar = tqdm(samples)
while i < samples:
# Randomly draw a bg
bg_image = bg_list[randint(0, len(bg_list)) - 1]
# Randomly draw an overlay
overlay_image = overlay_list[randint(0, len(overlay_list)) - 1]
# Attempt overlay.
try:
# Generate new image name.
merged_image = os.path.join(output_path, unique_name() + ".png")
randomly(bg_image, overlay_image, merged_image)
i = i + 1 # only increase counter if successfully generated one
pbar.update(i)
except: #FileNotFoundError or OSError or IOError
logger.info("Bad image found during overlay: " + bg_image)
continue
pbar.close()
def subfolder(bg_folder, overlay_folder, output_root_path):
"""
Wrapper function that output to a UNIQUE subfolder instead of directly into the folder. For output into a SPECIFIC output folder, use the folder_foreground instead.
:param bg_folder:
:param overlay_folder:
:param output_root_path:
:return:
"""
# Generate and make the directory
unique_path = os.path.join(output_root_path, "Overlay_" + unique_name())
os.makedirs(unique_path)
folder(bg_folder, overlay_folder, unique_path)
return unique_path
def ImageAugmentator(image_path, out_path, aug_seq, iterations):
'''
Augment ONE image over ITERATIONS and output to the path, with the augmentation sequence given.
:param image_path:
:param out_path:
:param aug_seq:
:param iterations:
:return:
'''
# Duplicate the images x times.
for x in range(0, iterations):
new_file_name = os.path.join(out_path, unique_name() + ".png")
shutil.copyfile(image_path, new_file_name)
folder(out_path, out_path, aug_seq, 1)
def folder_random(bg_folder, overlay_folder, output_root_path, samples=100):
"""
Wrapper function that output to a UNIQUE subfolder instead of directly into the folder. For output into a SPECIFIC output folder, use the folder_foreground instead.
:param bg_folder:
:param overlay_folder:
:param output_root_path:
:return:
"""
# Generate and make the directory
unique_path = os.path.join(output_root_path, "Overlay_" + unique_name())
os.makedirs(unique_path)
bg_list = recursive_list(bg_folder)
overlay_list = recursive_list(overlay_folder)
list_random(bg_list, overlay_list, unique_path, samples)
return unique_path
def crop_filelist(filelist, output_folder, width, height, iterations):
"""
crop the background images and generate the cropped version of them that are only 500x500
:param image_folder: folder contain downloads.
:param width: width of the area will be cropped out.
:param height: height of the area will be cropped out.
:return:
"""
# Duplicate input file lists X iterations into the output destination
updated_filelist = duplicates_into_folders(filelist, output_folder,
iterations)
# For all the files, try to convert and export to that address
for file in tqdm(updated_filelist):
try:
image_path = file
image = randomly(image_path, width, height)
except OSError:
logger.info("Found a bad file. Ignoring: " + file)
continue
bg_cropped_path = os.path.join(output_folder, unique_name() + ".png")
if image is None:
continue
else:
try:
# Generate a RGB image from the cropped image. This FORCE the image to be RGB even if it was originally GRAY scale!
rgbimg = Image.new("RGBA", image.size)
# Paste the image in.
rgbimg.paste(image)
# Save the file.
rgbimg.save(bg_cropped_path, "PNG")
logger.info("Saved " + bg_cropped_path)
# Delete the original
os.remove(file)
except OSError:
logger.info("Found a bad file. Ignoring: " + file + " from " +
image_path)
continue
def save_images(images_aug_collection, out_path):
"""
Take an augmented image collection an a path, then iterate through the collection to save them.
:param images_aug_collection: the data bundle that has been properly augmented
:param out_path: the output ROOT path where all images will reside.
:return:
"""
for image in tqdm(images_aug_collection):
# import matplotlib.pyplot as plt
# plt.imshow(image, aspect="auto")
# plt.show()
# Generate the time stamp required.
filename = os.path.join(out_path, unique_name() + ".png")
logger.info("Saving" + filename)
# Saving the file.
imageio.imwrite(filename, image)
def subfolder(input_path, output_path, aug_sequence, iterations,
aug_description):
"""
A wrapped version of FolderAugmentator that create a UNQIUE subfolder.
:param input_path: input path that contain the list of files.
:param output_path: output root path, a
:param aug_sequence: the augmentation seuqence that will be applied.
:param iterations: number of the time to augment these input folders.
:param aug_description: additional text string to be part of the folder name.
:return: the path of the created SUBFOLDER
"""
# Set and create the path of the augmented background.
augmentation_folder = os.path.join(output_path,
unique_name() + aug_description)
os.makedirs(augmentation_folder)
# Augment from input folder into the output folder.
folder(input_path, augmentation_folder, aug_sequence, iterations)
# return the path.
return augmentation_folder
def folder(bg_folder, overlay_folder, output_path):
"""
Output the combinatino of BG and OVERLAY into a TARGET folder after all possible combination.
:param bg_folder:
:param overlay_folder:
:param output_path:
:return:
"""
from PythonUtils.folder import recursive_list
bg_list = recursive_list(bg_folder)
overlay_list = recursive_list(overlay_folder)
for bg_image in tqdm(bg_list):
for overlay_image in overlay_list:
try:
# Generate new image name.
merged_image = os.path.join(output_path,
unique_name() + ".png")
randomly(bg_image, overlay_image, merged_image)
except FileNotFoundError:
logger.info(bg_image)
continue
def crop_folder_bg(input_image_root_folder, output_root_folder, width, height,
iterations):
"""
A batch function for cropping the background images and generate the cropped version of them that are only 500x500
:param input_image_root_folder: folder contain downloads.
:param width: width of the area will be cropped out.
:param height: height of the area will be cropped out.
:return:
"""
# Generate the temp name that will be used to store the crop results.
crop_folder_name = unique_name() + "_" + str(width) + "x" + str(height)
# Combine folder and root to form path name.
output_subfolder = os.path.join(output_root_folder, crop_folder_name)
# Make DIR if it does not already exist.
if not os.path.exists(output_subfolder):
os.makedirs(output_subfolder)
crop_folder(input_image_root_folder, output_subfolder, width, height,
iterations)
return output_subfolder