def make_models(variables_params): vars = [] for params in variables_params: var = make_variable(**params) log_message(var.summary(), logging.WARN) vars += [var] return vars
def load_models(file_name, variables_names): log_message('Restore old models ...', logging.DEBUG) vars = [] for name in variables_names: var = os.path.join(file_name, name+'.h5') variable = load_model(var) vars += [variable] log_message(variable.summary(), logging.WARN) return vars
def log(file_name, message, printed=True): header = file_name.split('\\')[-1].split('_')[0] fieldnames = [*message] if printed: msg = header for k, v in message.items(): msg += '\n {}: {}'.format(k, v) log_message(msg, logging.DEBUG) with open(file_name+'.csv', mode='a') as csv_file: writer = csv.DictWriter(csv_file, fieldnames=fieldnames) if message['Epoch']==0: writer.writeheader() writer.writerow(message)
def create_if_not_exist(directories): """ dirs - a list of directories to create if these directories are not found :param directories: """ if not isinstance(directories, list): directories = [directories] for directory in directories: try: if not os.path.exists(directory): os.makedirs(directory) except Exception as err: log_message("Creating directory {}, error: {}".format(directory, err), logging.ERROR)
def create_variables(variables_params, model_name, restore=None): variables_names = [variables['name'] for variables in variables_params] variables = None if restore: try: variables = load_models( restore, [model_name + '_' + var for var in variables_names]) except Exception as e: print(str(e)) print() log_message('Faild to restore old models !', logging.ERROR) variables = variables or create_models(variables_params) return variables
def clone_model(old_model, new_name, restore=None): if restore: log_message('Restore old models ...', logging.DEBUG) variable_path = os.path.join(restore, new_name + '.hdf5') variable = load_model(variable_path, compile=False) else: temp_layers = tf.keras.models.clone_model(old_model).layers temp_layers.append(tf.keras.layers.Flatten()) temp_layers.append( tf.keras.layers.Dense(units=1, activation='linear', name=new_name + '_outputs')) temp_layers = tf.keras.Sequential(temp_layers) variable = tf.keras.Model(name=new_name, inputs=temp_layers.inputs, outputs=temp_layers.outputs) return variable
def compute_unsupervised_metrics(ground_truth_data, representation_function, random_state, num_train, batch_size=16): """Computes unsupervised scores based on covariance and mutual information. Args: ground_truth_data: GroundTruthData to be sampled from. representation_function: Function that takes observations as input and outputs a dim_representation sized representation for each observation. random_state: Numpy random state used for randomness. artifact_dir: Optional path to directory where artifacts can be saved. num_train: Number of points used for training. batch_size: Batch size for sampling. Returns: Dictionary with scores. """ scores = {} log_message('Generating training set ...', logging.INFO) mus_train, _ = generate_batch_factor_code(ground_truth_data, representation_function, num_train, random_state, batch_size) num_codes = mus_train.shape[0] cov_mus = np.cov(mus_train) assert num_codes == cov_mus.shape[0] # Gaussian total correlation. scores["gaussian_total_correlation"] = gaussian_total_correlation(cov_mus) # Gaussian Wasserstein correlation. scores[ "gaussian_wasserstein_correlation"] = gaussian_wasserstein_correlation( cov_mus) scores["gaussian_wasserstein_correlation_norm"] = ( scores["gaussian_wasserstein_correlation"] / np.sum(np.diag(cov_mus))) # Compute average mutual information between different factors. mus_discrete = discrete_entropy(mus_train).reshape(-1, 1) mutual_info_matrix = discrete_mutual_info(mus_discrete, mus_discrete) np.fill_diagonal(mutual_info_matrix, 0) mutual_info_score = np.sum(mutual_info_matrix) / (num_codes**2 - num_codes) scores["mutual_info_score"] = mutual_info_score return scores
def stop(self, loss_val): if self.prev_loss_val != np.Infinity: if np.abs(np.abs(self.prev_loss_val) - np.abs(loss_val)) >= self.min_delta: self.patience_cnt = 0 self.prev_loss_val = loss_val else: self.patience_cnt += 1 msg = '\nPervious {}: {}'.format(self.name, self.prev_loss_val) msg += '\nCurrent {}: {}'.format(self.name, loss_val) msg += '\n{} Patience count-out: {}'.format(self.name, self.patience_cnt) log_message(msg, logging.DEBUG) if (self.patience_cnt > self.patience): log_message('{} is out of patience'.format(self.name), logging.CRITICAL) return True else: return False
def inspect_log(file_name): try: with open(file_name, 'r', encoding="utf-8", errors="ignore") as scraped: reader = csv.reader(scraped, delimiter=',') last_row = [0] for row in reader: if row: # avoid blank lines last_row = row if last_row[0]==0: log_message('No former training found ... ', logging.ERROR) else: log_message('Found Record for {} Epochs'.format(int(last_row[0])), logging.INFO) return int(last_row[0]) except: log_message('No former training found ... ', logging.ERROR) return 0
def fit(self, train_dataset, test_dataset, instance_names=['image'], epochs=10, learning_rate=1e-3, random_latent=None, recoding_dir='./recoding', gray_plot=True, generate_epoch=5, save_epoch=5, metric_epoch=10, gt_epoch=10, gt_data=None): assert isinstance(train_dataset, Iterable), 'dataset must be iterable' assert isinstance(test_dataset, Iterable), 'dataset must be iterable' self.dir_setup(recoding_dir) # generate random latent latent_shape = [50, self.latent_dim] if random_latent is None: random_latent = tf.random.normal(shape=latent_shape) if generate_epoch: generated = self.generate_sample(model=self.get_varibale, inputs_shape=self.inputs_shape, latent_shape=latent_shape, eps=random_latent) plot_and_save_generated(generated=generated, epoch=0, path=self.image_gen_dir, gray=gray_plot) self.optimizer = RAdamOptimizer(learning_rate) file_Name = os.path.join(self.csv_log_dir, 'TRAIN_' + self.model_name + '.csv') start_epoch = inspect_log(file_Name) early_stopper = EarlyStopping(name='on-Test dataset ELBO monitor', patience=5, min_delta=1e-6) epochs_pbar = tqdm(iterable=range(start_epoch, start_epoch + epochs), position=0, desc='Epochs Progress') for epoch in epochs_pbar: # training dataset tr_start_time = time.time() loss_tr = defaultdict() loss_tr['Epoch'] = epoch log_message('Training ... ', logging.INFO) for i, data_train in enumerate(train_dataset): data_train = self.cast_batch(data_train) total_loss = self.train_step(input=data_train, names=instance_names) tr_losses = self.evaluate_step(input=data_train, names=instance_names) loss_tr = self.reduce_sum_dict(tr_losses, loss_tr) epochs_pbar.set_description( 'Epochs Progress, Training Iterations {}'.format(i)) tr_end_time = time.time() loss_tr['Elapsed'] = '{:06f}'.format(tr_end_time - tr_start_time) # testing dataset val_start_time = time.time() loss_val = defaultdict() loss_val['Epoch'] = epoch log_message('Testing ... ', logging.INFO) tbar = tqdm(iterable=range(100), position=0, desc='Testing ...') for i, data_test in enumerate(test_dataset): data_test = self.cast_batch(data_test) val_losses = self.evaluate_step(input=data_test, names=instance_names) loss_val = self.reduce_sum_dict(val_losses, loss_val) montiored_loss = loss_val['Total'] tbar.update(i % 100) val_end_time = time.time() loss_val['Elapsed'] = '{:06f}'.format(val_end_time - val_start_time) if metric_epoch is not None and epoch % metric_epoch == 0: # testing dataset met_start_time = time.time() met_values = defaultdict() met_values['Epoch'] = epoch log_message('Evaluating Mertics ... ', logging.INFO) tbar = tqdm(iterable=range(100), position=0, desc='Evaluating ...') for i, data_test in enumerate(test_dataset): data_test = self.cast_batch(data_test) inputs = { 'X': data_test[instance_names[0]], 'y': self.feedforward(data_test[instance_names[0]]) } met_computed = compute_metrics(inputs) met_values = self.reduce_sum_dict(met_computed, met_values) tbar.update(i % 100) met_end_time = time.time() met_values['Elapsed'] = '{:06f}'.format(met_end_time - met_start_time) if epoch % gt_epoch == 0 and gt_data is not None: # testing dataset gt_start_time = time.time() gt_values = defaultdict() gt_values['Epoch'] = epoch log_message('Evaluating ground truth data ... ', logging.INFO) tbar = tqdm(iterable=range(100), position=0, desc='gt Evaluating ...') def rep_func(x): return self.feedforward(x)['latent'] us_scores = compute_unsupervised_metrics( ground_truth_data=gt_data, representation_function=rep_func, random_state=np.random.RandomState(0), num_train=10000, batch_size=32) s_scores = compute_supervised_metrics( ground_truth_data=gt_data, representation_function=rep_func, random_state=np.random.RandomState(0), num_train=10000, num_test=2000, continuous_factors=False, batch_size=32) ############################# display.clear_output(wait=False) log_message( "==================================================================", logging.INFO) file_Name = os.path.join(self.csv_log_dir, 'TRAIN_' + self.model_name) log(file_name=file_Name, message=dict(loss_tr), printed=True) log_message( "==================================================================", logging.INFO) log_message( "==================================================================", logging.INFO) file_Name = os.path.join(self.csv_log_dir, 'TEST_' + self.model_name) log(file_name=file_Name, message=dict(loss_val), printed=True) log_message( "==================================================================", logging.INFO) if epoch % metric_epoch: log_message( "==================================================================", logging.INFO) file_Name = os.path.join(self.csv_log_dir, 'Metrics_' + self.model_name) log(file_name=file_Name, message=dict(met_values), printed=True) log_message( "==================================================================", logging.INFO) if epoch % gt_epoch and gt_data is not None: gt_metrics = {**s_scores, **us_scores} log_message( "==================================================================", logging.INFO) file_Name = os.path.join(self.csv_log_dir, 'GroundTMetrics_' + self.model_name) log(file_name=file_Name, message=dict(gt_metrics), printed=True) log_message( "==================================================================", logging.INFO) if generate_epoch is not None and epoch % generate_epoch == 0: generated = self.generate_sample( model=self.get_varibale, inputs_shape=self.inputs_shape, latent_shape=latent_shape, eps=random_latent) plot_and_save_generated(generated=generated, epoch=epoch, path=self.image_gen_dir, gray=gray_plot, save=epoch % generate_epoch == 0) if epoch % save_epoch == 0: log_message('Saving Status in Epoch {}'.format(epoch), logging.CRITICAL) self.save_status() # Early stopping if (early_stopper.stop(montiored_loss)): log_message( 'Aborting Training after {} epoch because no progress ... ' .format(epoch), logging.WARN) break
def __init__(self, image_lists, image_data_generator, category, image_dir, image_shape=(256, 256, 3), color_mode='rgb', class_mode='categorical', batch_size=32, episode_len=20, episode_shift=10, shuffle=True, seed=None, data_format=None, save_to_dir=None, save_prefix='', save_format='jpeg', dtype=np.float32 ): if data_format is None: data_format = tf.keras.backend.image_data_format() classes = list(image_lists.keys()) self.category = category self.batch_size = batch_size self.num_class = len(classes) self.image_lists = image_lists self.image_dir = image_dir self.episode_len = episode_len self.episode_shift = episode_shift how_many_files = 0 for label_name in classes: for _ in self.image_lists[label_name][category]: how_many_files += 1 self.samples = how_many_files self.class2id = dict(zip(classes, range(len(classes)))) self.id2class = dict((v, k) for k, v in self.class2id.items()) self.classes = np.zeros((self.samples,), dtype='int32') self.image_data_generator = image_data_generator #self.target_size = tuple(target_size) if color_mode not in {'rgb', 'grayscale'}: raise ValueError('Invalid color mode:', color_mode, '; expected "rgb" or "grayscale".') self.color_mode = color_mode self.data_format = data_format self.image_shape = image_shape if self.data_format == 'jpeg': self.read_image = lambda x: tf.image.resize(tf.image.decode_jpeg(tf.io.read_file(x), channels=3), self.image_shape[:2]) else: self.read_image = lambda x: tf.image.resize(tf.image.decode_png(tf.io.read_file(x), channels=3), self.image_shape[:2]) if (class_mode not in {'categorical', 'binary', 'sparse', 'episode', 'episode_flat', None}) and (not hasattr(class_mode, '__call__')): raise ValueError('Invalid class_mode:', class_mode, '; expected one of "categorical", ' '"binary", "sparse", "episode", or None.') self.class_mode = class_mode self.dtype = dtype self.save_to_dir = save_to_dir self.save_prefix = save_prefix self.save_format = save_format i = 0 self.filenames = [] for label_name in classes: for j, _ in enumerate(self.image_lists[label_name][category]): self.classes[i] = self.class2id[label_name] img_path = get_file_path(self.image_lists, label_name, j, self.image_dir, self.category) self.filenames.append(img_path) i += 1 log_message("Found {} {} files".format(len(self.filenames), category), logging.INFO) Iterator.__init__(self, self.samples, self.batch_size, shuffle, seed)
def create_image_lists(image_dir, validation_pct, valid_imgae_formats, max_num_images_per_class=2**27 - 1, sequenced=None, verbose=1): """Builds a list of training images from the file system. Analyzes the sub folders in the image directory, splits them into stable training, testing, and validation sets, and returns a data structure describing the lists of images for each label and their paths. # Arguments image_dir: string path to a folder containing subfolders of images. validation_pct: integer percentage of images reserved for validation. # Returns dictionary of label subfolder, with images split into training and validation sets within each label. """ if not os.path.isdir(image_dir): raise ValueError("Image directory {} not found.".format(image_dir)) image_lists = {} sub_dirs = [x[0] for x in os.walk(image_dir)] sub_dirs_without_root = sub_dirs[1:] # first element is root directory sub_dirs_without_root = sorted(sub_dirs_without_root, key=lambda x: str(x.split(os.sep)[-1])) for sub_dir in sub_dirs_without_root: file_list = [] dir_name = os.path.basename(sub_dir) if dir_name == image_dir: continue if verbose == 1: log_message("Looking for images in '{}'".format(dir_name), logging.DEBUG) if isinstance(valid_imgae_formats, str): valid_imgae_formats = [valid_imgae_formats] for extension in valid_imgae_formats: file_glob = os.path.join(image_dir, dir_name, '*.' + extension) file_list.extend(glob.glob(file_glob)) if not file_list: msg = 'No files found' if verbose == 1: log_message(msg, logging.WARN) warnings.warn(msg) continue else: if verbose == 1: log_message('{} file found'.format(len(file_list)), logging.INFO) if len(file_list) < 20: msg = 'Folder has less than 20 images, which may cause issues.' if verbose == 1: log_message(msg, logging.WARN) warnings.warn(msg) elif len(file_list) > max_num_images_per_class: msg='WARNING: Folder {} has more than {} images. Some '\ 'images will never be selected.' \ .format(dir_name, max_num_images_per_class) log_message(msg, logging.WARN) warnings.warn(msg) label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower()) training_images = [] validation_images = [] if sequenced is True: #Sequenced in the case of try: file_list = sorted( file_list, key=lambda x: int(x.split(os.sep)[-1].split('.')[0])) except: msg = 'WARNING: Sorting folder {} has failed!' \ .format(dir_name) log_message(msg, logging.WARN) warnings.warn(msg) for file_name in file_list: base_name = os.path.basename(file_name) if sequenced is True: hash_pct = int( (int(file_name.split(os.sep)[-1].split('.')[0]) / len(file_list)) * 100) else: # Get the hash of the file name and perform variant assignment. hash_name = hashlib.sha1(as_bytes(base_name)).hexdigest() hash_pct = ((int(hash_name, 16) % (max_num_images_per_class + 1)) * (100.0 / max_num_images_per_class)) if hash_pct < validation_pct: validation_images.append(base_name) else: training_images.append(base_name) image_lists[label_name] = { 'dir': dir_name, 'training': training_images, 'validation': validation_images, } return image_lists