def build_material_dataset(self): self.data_dir = self.config.dirs.data self.train = "train_{}".format(self.image_size) self.test = "test_{}".format(self.image_size) self.test_vis = "test_vis" self.test_vis_big ="test_vis_big" self.valid = "valid_{}".format(self.image_size) self.train_dataset = os.path.join(self.data_dir, self.train) self.valid_dataset = os.path.join(self.data_dir, self.valid) self.img_location = os.path.join(self.data_dir, self.test, "imgs/") self.img_location_vis = os.path.join(self.data_dir, self.test_vis, "imgs/") self.img_location_vis_big = os.path.join(self.data_dir, self.test_vis_big, "imgs/") self.tag_location = os.path.join(self.data_dir, self.test, "labels/") self.tag_location_vis = os.path.join(self.data_dir, self.test_vis, "labels/") self.tag_location_vis_big = os.path.join(self.data_dir, self.test_vis_big, "labels/") if not os.path.exists(self.data_dir): self.logger.info("Dataset is not present. Download is started.") download_data_material(self.data_dir) self.data_dir_normal = self.config.dirs.data_normal self.data_dir_anomalous = self.config.dirs.data_anomalous # Up until this part only the raw dataset existence is checked and downloaded if not self.dataset_name = None # this is to list all the folders self.dir_names = listdir_nohidden(self.data_dir) self.test_size_per_img = ( None ) # This will be the number of patches that will be extracted from each test image # Normal images for the train and validation dataset normal_imgs = self.data_dir_normal # Anormal images and the tag infor regarding the anomaly for test set anorm_imgs = self.data_dir_anomalous + "/images/" anorm_tag_imgs = self.data_dir_anomalous + "/gt/" norm_img_names = [normal_imgs + x for x in listdir_nohidden(normal_imgs)] anorm_img_names = [anorm_imgs + x for x in listdir_nohidden(anorm_imgs)] anorm_tag_names = [anorm_tag_imgs + x for x in listdir_nohidden(anorm_tag_imgs)] self.norm_img_array = self.create_image_array(norm_img_names, save=False) self.anorm_img_array = self.create_image_array(anorm_img_names, save=False) self.anorm_tag_array = self.create_image_array(anorm_tag_names, save=False) self.image_tag_list = list(zip(self.anorm_img_array, self.anorm_tag_array)) if not self.config.data_loader.validation: self.populate_train_material() else: self.populate_train_valid_material() if self.config.data_loader.mode == "anomaly": self.populate_test_material() if self.config.data_loader.mode == "visualization": self.populate_test_material_vis() if self.config.data_loader.mode == "visualization_big": self.populate_test_material_vis_big()
def check_training_process(args): # Check if the process is running or not try: p = subprocess.run('ps aux | grep "[p]ython run.py"', shell=True, check=True) except subprocess.CalledProcessError as e: print("TrainerChecker: Process is not working") p = None if p is not None: # process is running, leave it alone pass else: # Process is not running, so we need to check the images parameter_dir = os.path.join("Logs", args.experiment, "parameters") with working_directory(parameter_dir): f = open("parameters.txt", "r") lines = f.readlines() print("Getting the info") epochs = int(lines[21].split(":")[1].strip("\n ")) print(epochs) epochs_per_image = int(lines[22].split(":")[1].strip("\n ")) print(epochs_per_image) # This is the number of images we need to find num_finished = epochs // epochs_per_image + 2 print("We need {} images".format(num_finished)) image_dir = os.path.join("Logs", args.experiment, "generated") with working_directory(image_dir): image_num = len(listdir_nohidden(".")) if image_num != num_finished: program_command = "python run.py -c {} -e {}".format( args.config, args.experiment) subprocess.run(program_command, shell=True)
def get_test_dataset(self): """ :param size: size of the image :return: numpy array of images and corresponding labels """ img_list = listdir_nohidden(self.img_location) img_names = tf.constant([os.path.join(self.img_location, x) for x in img_list]) tag_list = listdir_nohidden(self.tag_location) tag_list_merged = [os.path.join(self.tag_location, x) for x in tag_list] labels = [] for label in tag_list_merged: im2arr = io.imread(label) labels.append(1) if np.sum(im2arr) > 5100 else labels.append(0) labels_f = tf.constant(labels) self.logger.info("Test Dataset is Loaded") return [img_names, labels_f]
def get_valid_dataset(self): """ :param size: size of the image :return: numpy array of images and corresponding labels """ img_list = listdir_nohidden(self.valid_dataset) img_names = tf.constant([os.path.join(self.train_dataset, x) for x in img_list]) self.logger.info("Validation Dataset is Loaded") return img_names
def create_gif(folder_name, output_name, num): files = listdir_nohidden(folder_name) files = [x for x in files if x.startswith(str(num))] num_epochs = len(files) images = [] for filename in files: images.append(imageio.imread(folder_name + "/" + filename)) imageio.mimsave(output_name + "_" + str(num_epochs) + ".gif", images, duration=0.5)