def build_material_dataset(self):
self.data_dir = self.config.dirs.data
self.train = "train_{}".format(self.image_size)
self.test = "test_{}".format(self.image_size)
self.test_vis = "test_vis"
self.test_vis_big ="test_vis_big"
self.valid = "valid_{}".format(self.image_size)
self.train_dataset = os.path.join(self.data_dir, self.train)
self.valid_dataset = os.path.join(self.data_dir, self.valid)
self.img_location = os.path.join(self.data_dir, self.test, "imgs/")
self.img_location_vis = os.path.join(self.data_dir, self.test_vis, "imgs/")
self.img_location_vis_big = os.path.join(self.data_dir, self.test_vis_big, "imgs/")
self.tag_location = os.path.join(self.data_dir, self.test, "labels/")
self.tag_location_vis = os.path.join(self.data_dir, self.test_vis, "labels/")
self.tag_location_vis_big = os.path.join(self.data_dir, self.test_vis_big, "labels/")
if not os.path.exists(self.data_dir):
self.logger.info("Dataset is not present. Download is started.")
download_data_material(self.data_dir)
self.data_dir_normal = self.config.dirs.data_normal
self.data_dir_anomalous = self.config.dirs.data_anomalous
# Up until this part only the raw dataset existence is checked and downloaded if not
self.dataset_name = None
# this is to list all the folders
self.dir_names = listdir_nohidden(self.data_dir)
self.test_size_per_img = (
None
) # This will be the number of patches that will be extracted from each test image
# Normal images for the train and validation dataset
normal_imgs = self.data_dir_normal
# Anormal images and the tag infor regarding the anomaly for test set
anorm_imgs = self.data_dir_anomalous + "/images/"
anorm_tag_imgs = self.data_dir_anomalous + "/gt/"
norm_img_names = [normal_imgs + x for x in listdir_nohidden(normal_imgs)]
anorm_img_names = [anorm_imgs + x for x in listdir_nohidden(anorm_imgs)]
anorm_tag_names = [anorm_tag_imgs + x for x in listdir_nohidden(anorm_tag_imgs)]
self.norm_img_array = self.create_image_array(norm_img_names, save=False)
self.anorm_img_array = self.create_image_array(anorm_img_names, save=False)
self.anorm_tag_array = self.create_image_array(anorm_tag_names, save=False)
self.image_tag_list = list(zip(self.anorm_img_array, self.anorm_tag_array))
if not self.config.data_loader.validation:
self.populate_train_material()
else:
self.populate_train_valid_material()
if self.config.data_loader.mode == "anomaly":
self.populate_test_material()
if self.config.data_loader.mode == "visualization":
self.populate_test_material_vis()
if self.config.data_loader.mode == "visualization_big":
self.populate_test_material_vis_big()
def check_training_process(args):
# Check if the process is running or not
try:
p = subprocess.run('ps aux | grep "[p]ython run.py"',
shell=True,
check=True)
except subprocess.CalledProcessError as e:
print("TrainerChecker: Process is not working")
p = None
if p is not None:
# process is running, leave it alone
pass
else:
# Process is not running, so we need to check the images
parameter_dir = os.path.join("Logs", args.experiment, "parameters")
with working_directory(parameter_dir):
f = open("parameters.txt", "r")
lines = f.readlines()
print("Getting the info")
epochs = int(lines[21].split(":")[1].strip("\n "))
print(epochs)
epochs_per_image = int(lines[22].split(":")[1].strip("\n "))
print(epochs_per_image)
# This is the number of images we need to find
num_finished = epochs // epochs_per_image + 2
print("We need {} images".format(num_finished))
image_dir = os.path.join("Logs", args.experiment, "generated")
with working_directory(image_dir):
image_num = len(listdir_nohidden("."))
if image_num != num_finished:
program_command = "python run.py -c {} -e {}".format(
args.config, args.experiment)
subprocess.run(program_command, shell=True)
def get_test_dataset(self):
"""
:param size: size of the image
:return: numpy array of images and corresponding labels
"""
img_list = listdir_nohidden(self.img_location)
img_names = tf.constant([os.path.join(self.img_location, x) for x in img_list])
tag_list = listdir_nohidden(self.tag_location)
tag_list_merged = [os.path.join(self.tag_location, x) for x in tag_list]
labels = []
for label in tag_list_merged:
im2arr = io.imread(label)
labels.append(1) if np.sum(im2arr) > 5100 else labels.append(0)
labels_f = tf.constant(labels)
self.logger.info("Test Dataset is Loaded")
return [img_names, labels_f]
def get_valid_dataset(self):
"""
:param size: size of the image
:return: numpy array of images and corresponding labels
"""
img_list = listdir_nohidden(self.valid_dataset)
img_names = tf.constant([os.path.join(self.train_dataset, x) for x in img_list])
self.logger.info("Validation Dataset is Loaded")
return img_names
def create_gif(folder_name, output_name, num):
files = listdir_nohidden(folder_name)
files = [x for x in files if x.startswith(str(num))]
num_epochs = len(files)
images = []
for filename in files:
images.append(imageio.imread(folder_name + "/" + filename))
imageio.mimsave(output_name + "_" + str(num_epochs) + ".gif",
images,
duration=0.5)