def test_synthetic_random_data(self): ds = DataGenerator.get_random_dataset(height=32, width=32, num_classes=5, data_type=tf.float32) assert (DataGenerator.evaluate_size_dataset(ds) == 1) assert (isinstance(ds, tf.data.Dataset))
def predict(model): input_files = fnmatch.filter( listdir(FLAGS.filePath), FLAGS.filePattern + FLAGS.predictionSetExtension + '.*.' + FLAGS.extension) if len(input_files) == 0: sys.exit("File not found: " + filename_pattern) for file in input_files: print('Number of predictions: ', end='') predict_data_generator = DataGenerator( data_shape, num_class, FLAGS.filePath, file, FLAGS.testBatchSize, useOddSample=True, ) prediction = model.predict_generator( predict_data_generator, predict_data_generator.num_steps(), max_queue_size=FLAGS.queueSize, ) np.save( path.join( FLAGS.filePath, file.replace(FLAGS.predictionSetExtension, FLAGS.predictionExtension).replace( '.' + FLAGS.extension, '')), prediction)
def test_malformed_directory(self): imagenet_original_data_path = '/localdata/datasets/imagenet-raw-data' output_directory = '/tmp/temporary_imagenet_dataset_directory' split = 'val' with self.assertRaises(NameError): # if the path imagenet_original_data_path + split doesn't exist the function will throw an error DataGenerator.build_imagenet_tf_record( imagenet_original_data_path, split, output_directory=output_directory)
def main(): model = create_model(trainable=TRAINABLE) # if TRAINABLE: # model.load_weights(WEIGHTS) train_datagen = DataGenerator(file_path=cfg.TRAIN.DATA_PATH, config_path=cfg.TRAIN.ANNOTATION_PATH) val_generator = DataGenerator(file_path=cfg.TEST.DATA_PATH, config_path=cfg.TEST.ANNOTATION_PATH, debug=False) validation_datagen = Validation(generator=val_generator) learning_rate = cfg.TRAIN.LEARNING_RATE if TRAINABLE: learning_rate /= 10 optimizer = tf.keras.optimizers.SGD(lr=learning_rate, decay=cfg.TRAIN.LR_DECAY, momentum=0.9, nesterov=False) model.compile(loss=detect_loss(), optimizer=optimizer, metrics=[]) checkpoint = tf.keras.callbacks.ModelCheckpoint("model-{val_iou:.2f}.h5", monitor="val_iou", verbose=1, save_best_only=True, save_weights_only=True, mode="max") stop = tf.keras.callbacks.EarlyStopping(monitor="val_iou", patience=cfg.TRAIN.PATIENCE, mode="max") reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor="val_iou", factor=0.6, patience=5, min_lr=1e-6, verbose=1, mode="max") # Define the Keras TensorBoard callback. logdir = "logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S") tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=logdir) model.fit_generator(generator=train_datagen, epochs=cfg.TRAIN.EPOCHS, callbacks=[ tensorboard_callback, validation_datagen, checkpoint, reduce_lr, stop ], shuffle=True, verbose=1)
def test_fetch_from_directory(self): dirpath = '/tmp/temporary_dataset_directory' split_name = 'tmp_train' label_name1 = 'tmp_label1' label_name2 = 'tmp_label2' full_dir_to_create_1 = os.path.join(dirpath, split_name, label_name1) full_dir_to_create_2 = os.path.join(dirpath, split_name, label_name2) image_path_1 = os.path.join(full_dir_to_create_1, 'temporary_image_1.jpg') image_path_2 = os.path.join(full_dir_to_create_2, 'temporary_image_2.jpg') if not (os.path.exists(dirpath)): os.makedirs(full_dir_to_create_1) os.makedirs(full_dir_to_create_2) image_1 = np.ones((1, 1, 3), np.uint8) cv2.imwrite(image_path_1, image_1) image_2 = np.ones((1, 1, 3), np.uint8) cv2.imwrite(image_path_2, image_2) ds, img_shape, num_examples, num_classes = DataGenerator.get_dataset_from_directory( ds_path=dirpath, split=split_name) assert (num_classes == 2) assert (num_examples == 2) assert (tuple(img_shape) == (1, 1, 3)) assert (isinstance(ds, tf.data.Dataset)) for image, label in tfds.as_numpy(ds): assert (np.all(image == 1)) assert (isinstance(label, np.int64)) shutil.rmtree(dirpath)
def predict(model_name, model=None, row_start=None, row_end=None, custom_objects=None): if 'batch_size' not in config: config['batch_size'] = default_batch_size if 'max_queue_size' not in config: config['max_queue_size'] = default_max_queue_size if 'does_use_multiprocessing' not in config: config['does_use_multiprocessing'] = default_does_use_multiprocessing if 'worker_number' not in config: config['worker_number'] = default_worker_number if 'verbose' not in config: config['verbose'] = default_verbose if model is None: if custom_objects is None: custom_objects = custom_metrics model = load_model(model_name=model_name, custom_objects=custom_objects, does_compile=True) if model is None: raise NoTrainedModelException(model_name) rolling_window_size = get_rolling_window_size(model_name) generator = DataGenerator( dataset_name=DATASET_NAME_PREDICT, rolling_window_size=rolling_window_size, row_start=row_start, row_end=row_end, max_batch_size=config['batch_size'], does_shuffle=False, # NOT shuffle! ) snpr = generator.get_sample_number_per_row() if config['batch_size'] % snpr != 0: logging.warning( 'predict: batch_size(%d) cannot divide %d. ' 'Some inputs will be ignored.', config['batch_size'], snpr) result = model.predict_generator( generator=generator, max_queue_size=config['max_queue_size'], use_multiprocessing=config['does_use_multiprocessing'], workers=config['worker_number'], verbose=config['verbose'], ) return result
def test_build_imagenet_validation(self): imagenet_original_data_path = '/localdata/datasets/imagenet-raw-data' output_directory = '/tmp/temporary_imagenet_dataset_directory' DataGenerator.build_imagenet_tf_record( imagenet_original_data_path, 'validation', output_directory=output_directory) ds, img_shape, num_examples, num_classes = DataGenerator.get_imagenet( output_directory, 'validation') self.assertTrue(isinstance(ds, tf.data.Dataset)) self.assertEqual(img_shape, (224, 224, 3)) self.assertEqual(num_examples, 50000) self.assertEqual(num_classes, 1000) shutil.rmtree(output_directory)
def evaluation(session, model, mode='validation', percent_limit=None, save_path=None): def log(msg, add_trailing=True): LOG(msg, add_trailing) dataset = DataGenerator(mode) target_formulas = [] predicted_formulas = [] pp_hist = [] last_log_percentage = 0 log_percentage_every = 10 for epoch, percentage, images, formulas, _ in dataset.generator(1, percent_limit): target = dataset.decode_formulas(formulas) prediction, pp = model.predict(sess=session, images=images) prediction = dataset.decode_formulas(prediction) target_formulas += target predicted_formulas += prediction pp_hist += [pp] max_per = 1 if percent_limit is None else percent_limit percentage = int(100 * (percentage/max_per)) if percentage >= last_log_percentage + log_percentage_every: idx = random.randint(0, len(prediction) - 1) last_log_percentage += log_percentage_every log('Evaluation prediction progress completion: {}%\ntrue -> {}\npred -> {}'. format(percentage, '' if len(target) <= idx else target[idx], prediction[idx])) if save_path is not None: with open(save_path, 'w+') as f: f.write('\n'.join(predicted_formulas)) if len(target_formulas) != len(predicted_formulas): log("number of formulas doesn't match", False) return None bleu_score = bleu_eval(target_formulas, predicted_formulas) edit_distance_score = edit_distance_eval(target_formulas, predicted_formulas) pp_mean = np.mean(np.array(pp_hist)) log('Bleu score: {0:2.3f} %'.format(100 * bleu_score)) log('Edit distance score: {0:2.3f} %'.format(100 * edit_distance_score)) log('Perplexity: {0:2.3f}'.format(pp_mean)) return bleu_score, edit_distance_score
def test_load_imagenet_train(self): imagenet_tf_record_data_path = '/localdata/datasets/imagenet-data' # if the path doesn't exist the function will throw an error ds, img_shape, num_examples, num_classes = DataGenerator.get_imagenet( imagenet_tf_record_data_path, 'train') self.assertTrue(isinstance(ds, tf.data.Dataset)) self.assertEqual(img_shape, (224, 224, 3)) self.assertEqual(num_examples, 1281167) self.assertEqual(num_classes, 1000)
def test_mnist(self): dirpath = '/tmp/mnist_test_sample_directory' if not (os.path.exists(dirpath)): os.makedirs(dirpath) ds, img_shape, num_examples, num_classes = DataGenerator.get_dataset_from_name( ds_name='mnist', ds_path=dirpath, split='train[0:3]') assert (num_classes == 10) assert (num_examples == 3) assert (tuple(img_shape) == (28, 28, 1)) assert (isinstance(ds, tf.data.Dataset)) ds, img_shape, num_examples, num_classes = DataGenerator.get_dataset_from_name( ds_name='mnist', ds_path=dirpath, split='test[0:3]') assert (num_classes == 10) assert (num_examples == 3) assert (tuple(img_shape) == (28, 28, 1)) assert (isinstance(ds, tf.data.Dataset))
def main(): print("------- get configs -------") config = get_configs("./py/configs.json") print("------- generate dataset -------") data = DataGenerator(config) X_train, y_train = data.get_train_data() X_test, y_test = data.get_test_data() word_index = data.get_word_index() print("------- create model -------") model = BiGRUAttention(config, word_index) print("------- model training and predicting-------") trainer = CBTrainer(model, config) trainer.fit(X_train, y_train) y_pred = trainer.predict(X_test) trainer.score(X_test, y_test) return y_pred
def test(model): print('Number of test samples: ', end='') test_data_generator = DataGenerator( data_shape, num_class, FLAGS.filePath, FLAGS.filePattern + FLAGS.testSetExtension + '.*.' + FLAGS.extension, FLAGS.testBatchSize, useOddSample=True, percent=FLAGS.testPercent, shuffle=True, ) score = model.evaluate_generator( test_data_generator, test_data_generator.num_steps(), max_queue_size=FLAGS.queueSize, ) print('Test loss: ', score[0]) print('Test accuracy:', score[1])
def main(): # Data train_class = DataGenerator(number_of_examples= samples, number_of_features= features, batch_size = batch, shuffle = to_shuffle) test_class = DataGenerator(number_of_examples=samples, number_of_features= features, batch_size = batch, shuffle = to_shuffle) train_data, train_target = train_class.generate_data() test_data, test_target = test_class.generate_data() # Layers input_layer = Linear(in_features= features, out_features = 25) first_hidden_layer = Linear(in_features = 25, out_features = 25) second_hidden_layer = Linear(in_features= 25, out_features = 25) output_layer = Linear(in_features= 25, out_features = 2) layers = [input_layer, Relu(), first_hidden_layer, Relu(), second_hidden_layer, Relu(), output_layer, Tanh()] # Model model = Sequential(layers) # Optimizer optimizer = StochasticGD(parameters_list=model.param(), learning_rate = 0.01) # Loss loss = Mse() # Initializing Network class rn = RunNetwork(epochs = number_of_epochs, train_class = train_class,\ test_class = test_class, model = model, loss = loss, optimizer= optimizer, samples = samples) # Training the rn.run_network()
def initialize(mode='train', is_evaluation=False): def log(msg, trailing_line=False): LOG(msg, trailing_line) dataset = DataGenerator(mode) start_token, pad_token = dataset.data.start_token(), dataset.data.pad_token() model_name = FLAGS.model_name model_path = os.path.join(config.save_path, model_name + '.ckpt') secondary_model_path = os.path.join(config.secondary_path, config.save_path, model_name + '.ckpt') logger.set_model_name(model_name + ('-test' if is_evaluation else '')) log(config, True) log(h_params, True) gpu_is_available = tf.test.is_gpu_available() log('GPU is available: ' + str(gpu_is_available)) log('Start building graph') tf.reset_default_graph() model = ImageToLatexModel(start_token, pad_token) log('Graph building finished!', True) # params = tf.trainable_variables() # print('Trainable params:') # for p in params: # print(p) sess = tf.Session() init_opt = tf.global_variables_initializer() saver = tf.train.Saver(keep_checkpoint_every_n_hours=1) restored = False if FLAGS.load_from_previous or is_evaluation: def load(path, name): nonlocal restored, saver, sess if restored: return try: saver.restore(sess, path) step = sess.run([model.step])[0] log('Model restored from last session in {}, current step: {}'.format(name, step)) # run_eval() restored = True except Exception as e: log('Can\'t load from previous session in {}, error: {}'.format(name, e)) load(model_path, 'model path') load(secondary_model_path, 'secondary model path') if not restored: sess.run(init_opt) return model, sess, dataset, saver, model_path, secondary_model_path
def __init__(self, # splitter related label_list: list, is_sliced_data: bool, is_create_negative_sample: bool, negative_sample_scale: int, # data generator related sample_spaces: int, # model related input_size: int, hidden_size: int, num_layers: int, dropout: float, learning_rate: float, embedding_scale: int, # train related epochs: int, train_batch: int, eval_batch: int): self.epochs = epochs self.train_batch = train_batch self.eval_batch = eval_batch splitter = Splitter(label_list, is_sliced_data, is_create_negative_sample, negative_sample_scale) self.nested_label2id = splitter.label2id self.generator = DataGenerator(splitter, sample_spaces) self.model = RnnRelatedModel( input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, output_size=self.generator.get_label_size(), # TODO: this should be dynamically generated loss_weight=None, embedding_scale=embedding_scale ) self.optimizer = torch.optim.Adadelta(self.model.parameters(), lr=learning_rate) self.name = "trainer_" + self.generator.name
def generate_data(): """ Route to regenerate the data collection. """ # We could also run it on a separate thread # from threading import Thread # t = Thread(daemon=True) # t.run(DataGenerator().generate()) logger.info( f'CALLED FROM: {GENERATE_DATA} ENDPOINT: Generating the data collection.' ) DataGenerator().generate() return jsonify("The data collection has been generated")
def test_model(model_name, model=None, row_start=None, row_end=None, custom_objects=None): if 'batch_size' not in config: config['batch_size'] = default_batch_size if 'does_shuffle' not in config: config['does_shuffle'] = default_does_shuffle if 'max_queue_size' not in config: config['max_queue_size'] = default_max_queue_size if 'does_use_multiprocessing' not in config: config['does_use_multiprocessing'] = default_does_use_multiprocessing if 'worker_number' not in config: config['worker_number'] = default_worker_number if 'verbose' not in config: config['verbose'] = default_verbose if model is None: if custom_objects is None: custom_objects = custom_metrics model = load_model(model_name=model_name, custom_objects=custom_objects, does_compile=True) if model is None: raise NoTrainedModelException(model_name) rolling_window_size = get_rolling_window_size(model_name) generator = DataGenerator( dataset_name=DATASET_NAME_TEST, rolling_window_size=rolling_window_size, row_start=row_start, row_end=row_end, max_batch_size=config['batch_size'], does_shuffle=config['does_shuffle'], ) result_list = model.evaluate_generator( generator=generator, max_queue_size=config['max_queue_size'], use_multiprocessing=config['does_use_multiprocessing'], workers=config['worker_number'], verbose=config['verbose'], ) result_map = dict(zip(model.metrics_names, result_list)) return result_map
import random import matplotlib.pyplot as plt import numpy as np from data.data_generator import DataGenerator gen = DataGenerator('train') for ep, per, images, formulas, l in gen.generator(1): for i in range(1): idx = random.randint(0, images.shape[0] - 1) img = images[idx] formula = np.expand_dims(formulas[idx], axis=0) formula = gen.decode_formulas(formula)[0] print(formula) plt.imshow(img, cmap='gray') plt.title(formula) plt.show()
def get_dataset(dataset_name: str, dataset_path: str, split: str, img_datatype: tf.dtypes.DType, micro_batch_size: int, shuffle: bool = False, accelerator_side_preprocess: bool = True, eight_bit_transfer: Optional[EightBitTransfer] = None, apply_preprocessing: bool = True, pipeline_num_parallel: int = 48, seed: Optional[int] = None): logging.info(f'dataset_name = {dataset_name}') if popdist.getNumInstances() == 1: logging.info(f'Since the training is run in a single process, setting dataset pipeline threading ' f'and prefetching buffer size to tf.data.AUTOTUNE.') pipeline_num_parallel = prefetch_size = tf.data.AUTOTUNE else: prefetch_size = PREFETCH_BUFFER_SIZE logging.info(f'Setting number of threads for the dataset pipeline to {pipeline_num_parallel}, ' f'and the prefetching buffer size to {prefetch_size}.') ds, img_shape, dataset_size, num_classes = DataGenerator.get_dataset_from_name( ds_name=dataset_name, ds_path=dataset_path, split=split) preprocess_fn = None if apply_preprocessing: if dataset_name == 'cifar10': ds, preprocess_fn = DataTransformer.cifar_preprocess( ds, buffer_size=dataset_size, img_type=img_datatype, is_training=(split == 'train'), accelerator_side_preprocess=accelerator_side_preprocess, pipeline_num_parallel=pipeline_num_parallel, ) elif dataset_name == 'imagenet': ds, preprocess_fn = DataTransformer.imagenet_preprocessing( ds, img_type=img_datatype, is_training=(split == 'train'), accelerator_side_preprocess=accelerator_side_preprocess, pipeline_num_parallel=pipeline_num_parallel, seed=seed ) if shuffle: # Shuffle the input files ds = ds.shuffle(buffer_size=IMAGENET_SHUFFLE_BUFFER_SIZE) else: ds = DataTransformer.cache_shuffle(ds, buffer_size=dataset_size, shuffle=(split == 'train')) ds = DataTransformer.normalization(ds, img_type=img_datatype) preprocess_fn = None if eight_bit_transfer is not None: ds = ds.map(lambda x, y: (eight_bit_transfer.compress(x), y), num_parallel_calls=pipeline_num_parallel) ds = ds.batch(batch_size=micro_batch_size, drop_remainder=True) ds = ds.repeat().prefetch(prefetch_size) cpu_memory_usage = psutil.virtual_memory().percent if cpu_memory_usage > 100: logging.warning(f'cpu_memory_usage is {cpu_memory_usage} > 100% so your program is likely to crash') return ds, img_shape, dataset_size, num_classes, preprocess_fn
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Thu Apr 25 14:47:12 2019 @author: mohamed """ import tensorflow as tf from trainers.trainer import Trainer from models.model import DCGAN from data.data_generator import DataGenerator from utils import config config = config.process_config("config.json") data = DataGenerator(config) c = tf.ConfigProto() c.gpu_options.allow_growth = True sess = tf.Session(config=c) model = DCGAN(config) trainer = Trainer(config , model , data, sess) trainer.train()
return dataset, dataset_size if __name__ == '__main__': set_seed() print('process id ', os.getpid()) opt = TrainOptions().parse() opt_val, opt_test = get_val_test_opts(opt) test_index = opt.n_fold - 1 if opt.test_index is None else opt.test_index test_index = opt.n_fold if 'test' not in opt.test_mode else test_index val_indices = [x for x in range(opt.n_fold) if x != test_index ] if opt.test_mode != 'test' else [test_index] models = [] data_generator = DataGenerator(opt.dataroot) for val_index in val_indices: data_generator.build_dataset(val_index, test_index, opt.test_mode) dataset, dataset_size = create_data_loader(opt) dataset_val, dataset_size_val = create_data_loader(opt_val) dataset_test, dataset_size_test = create_data_loader(opt_test) model_suffix = 'val%d' % val_index if 'val' in opt.test_mode else '' model_suffix += 'test%d' % test_index if 'test' in opt.test_mode else '' model = create_model(opt, model_suffix) model.setup(opt) visualizer = Visualizer(opt) total_steps, best_epochs, val_losses_best = 0, 0, 0 for epoch in range(opt.epoch_count, opt.niter + opt.niter_decay + 1): epoch_start_time = time.time()
def test_check_size(self): ds = tf.data.Dataset.from_tensor_slices([1, 2, 3]) assert (DataGenerator.evaluate_size_dataset(ds) == 3)
def test_mnist_wrong_dir(self): with self.assertRaises(NameError): _ = DataGenerator.get_dataset_from_name(ds_name='mnist', ds_path='foo')
def test_unsupported_dataset(self): with self.assertRaises(NameError): _ = DataGenerator.get_dataset_from_name(ds_name='foo')
class Trainer(object): def __init__(self, # splitter related label_list: list, is_sliced_data: bool, is_create_negative_sample: bool, negative_sample_scale: int, # data generator related sample_spaces: int, # model related input_size: int, hidden_size: int, num_layers: int, dropout: float, learning_rate: float, embedding_scale: int, # train related epochs: int, train_batch: int, eval_batch: int): self.epochs = epochs self.train_batch = train_batch self.eval_batch = eval_batch splitter = Splitter(label_list, is_sliced_data, is_create_negative_sample, negative_sample_scale) self.nested_label2id = splitter.label2id self.generator = DataGenerator(splitter, sample_spaces) self.model = RnnRelatedModel( input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, output_size=self.generator.get_label_size(), # TODO: this should be dynamically generated loss_weight=None, embedding_scale=embedding_scale ) self.optimizer = torch.optim.Adadelta(self.model.parameters(), lr=learning_rate) self.name = "trainer_" + self.generator.name def train(self): data_gen = self.generator model = self.model optimizer = self.optimizer total_batch = data_gen.get_total_batch(self.train_batch, "train") print("---------- Start Training ----------") for epoch in range(self.epochs): for idx, batch_data in enumerate(data_gen.generate(batch_size=self.train_batch, mode="train")): batch_input, batch_label = batch_data optimizer.zero_grad() preds, loss = model(batch_input, batch_label, "train") loss.backward() optimizer.step() if idx % 10 == 0: print(f"loss for batch {idx} / {total_batch}: {loss.data}") self.save_model(epoch + 1) def evaluate(self): self.restore_model() data_gen = self.generator model = self.model model.eval() preds, labels = [], [] total_batch = data_gen.get_total_batch(self.eval_batch, "eval") print("---------- Start Evaluating ----------") for idx, batch_data in enumerate(data_gen.generate(batch_size=self.eval_batch, mode="eval")): batch_input, batch_label = batch_data pred, _ = model(batch_input, batch_label, "eval") preds.extend(pred.detach().numpy()) labels.extend(batch_label) if idx % 10 == 0: print(f"eval batch: {idx} / {total_batch}") metrics = eval_result(preds, labels, self.nested_label2id, True) return metrics def predict(self): self.restore_model() def save_model(self, epoch: int): torch.save({'epoch': epoch, 'state_dict': self.model.state_dict(), 'optimizer': self.optimizer.state_dict()}, './result/{}.epoch-{}.ckpt'.format(self.name, epoch)) def restore_model(self, epoch: int = -1): if epoch == -1: model_file_paths = glob.glob('result/{}.epoch-*'.format(self.name)) try: epoch = max([int(re.findall('{}\.epoch-(\d*)\.ckpt'.format(self.name), path)[0]) for path in model_file_paths]) except: print("---------- Error: Model not saved. ----------") exit(-1) ckpt_path = './result/{}.epoch-{}.ckpt'.format(self.name, epoch) model_ckpt = torch.load(ckpt_path) self.model.load_state_dict(model_ckpt['state_dict']) self.optimizer.load_state_dict(model_ckpt['optimizer'])
dropout_output = self.dropout(dense_output) # (batch_size, output_size) output = self.output_layer(dropout_output) if mode == "train": loss = self.loss(output, batch_label) return output, loss return output, None if __name__ == '__main__': splitter = Splitter(label_list=[0, 2, 7, 17], is_slice_data=False, is_create_negative_sample=True) splitter.split() generator = DataGenerator(splitter, 10) model = RnnRelatedModel(input_size=12, hidden_size=64, num_layers=1, dropout=0.2, output_size=splitter.get_label_size()) optimizer = torch.optim.Adadelta(model.parameters(), lr=0.9) for idx, data in enumerate(generator.generate(batch_size=20)): optimizer.zero_grad() preds, loss = model(data[0], data[1], "train") print(loss.data) loss.backward() optimizer.step()
def test(self, test_batch_num=50): print('Start to test') print('#' * 20) self.ckpt.restore(self.ckpt_manager.latest_checkpoint) if self.ckpt_manager.latest_checkpoint: print("Restored from {}".format( self.ckpt_manager.latest_checkpoint)) else: print("Initializing from scratch.") dg = DataGenerator(task_version=self.task_version, action='test') # todo: should be test for action psycollection = {'coh': [], 'perc': []} for batch_index in range(20): descs, test_masks, test_inputs, test_outputs = dg.get_valid_test_datasets( ) test_logits, _ = self.ei_rnn(test_inputs, [self.init_state], training=False) acc_test_logits = test_logits.numpy() acc_test_outputs = tf.transpose(test_outputs, perm=[0, 2, 1]).numpy() test_acc = self.get_accuracy(acc_test_logits, acc_test_outputs, test_masks) test_logits = tf.transpose(test_logits, perm=[0, 2, 1]) test_loss = self.loss_fun(test_outputs, test_logits, test_masks) test_loss += sum(self.ei_rnn.losses) print('test loss:', test_loss.numpy()) print('test acc:', test_acc) tmp_data = self.get_psychometric_data(descs, test_logits.numpy()) psycollection['coh'] += tmp_data['coh'] psycollection['perc'] += tmp_data['perc'] with self.test_summary_writer.as_default(): tf.summary.scalar('loss', test_loss, step=batch_index) tf.summary.scalar('acc', test_acc, step=batch_index) curve_image = plot.plot_dots(psycollection['coh'], psycollection['perc']) tf.summary.image('psycollection', curve_image, step=batch_index) cm_image = plot.plot_confusion_matrix(self.get_w_rec_m()) tf.summary.image('M_rec', cm_image, step=batch_index) win_image = plot.plot_confusion_matrix( funs.rectify( self.rnn_cell.W_in.numpy()[:, :int(UNITS_SIZE * EI_RATIO)]), False) tf.summary.image('M_in', win_image, step=batch_index) wout_image = plot.plot_confusion_matrix( self.get_w_out_m()[:, :int(UNITS_SIZE * EI_RATIO)], False) tf.summary.image('M_out', wout_image, step=batch_index)
def train(self): self.ckpt.restore(self.ckpt_manager.latest_checkpoint) if self.ckpt_manager.latest_checkpoint: print("Restored from {}".format( self.ckpt_manager.latest_checkpoint)) else: print("Initializing from scratch.") dg = DataGenerator(task_version=self.task_version, action='train') validation_batch_num = self.batch_num // 10 over_all_performace = [] print('Start to train') print('#' * 20) for epoch_i in range(self.epoch_num): print('Epoch ' + str(epoch_i)) # Train # train_loss_all = 0 for batch_i in range(self.batch_num): decs, masks, inputs, outputs = next(dg) with tf.GradientTape() as tape: logits, _ = self.ei_rnn(inputs, [self.init_state], training=True) logits = tf.transpose(logits, perm=[0, 2, 1]) train_loss = self.loss_fun(outputs, logits, masks) train_loss += sum(self.ei_rnn.losses) train_loss_all += train_loss.numpy() all_weights = self.ei_rnn.trainable_weights + [self.init_state] grads = tape.gradient(train_loss, all_weights) grads, _ = tf.clip_by_global_norm(grads, self.grad_clip) self.optimizer.apply_gradients( zip(grads, self.ei_rnn.trainable_weights)) train_loss_all = train_loss_all / self.batch_num print('train loss:', train_loss_all) with self.train_summary_writer.as_default(): tf.summary.scalar('loss', train_loss_all, step=epoch_i) # Validation # validation_loss_all = 0 validation_acc_all = 0 for v_batch_i in range(validation_batch_num): _, v_masks, v_inputs, v_outputs = dg.get_valid_test_datasets( ) # dg.get_valid_test_datasets() v_logits, _ = self.ei_rnn(v_inputs, [self.init_state]) acc_v_logits = v_logits.numpy() acc_v_outputs = tf.transpose(v_outputs, perm=[0, 2, 1]).numpy() validation_acc = self.get_accuracy(acc_v_logits, acc_v_outputs, v_masks) validation_acc_all += validation_acc v_logits = tf.transpose(v_logits, perm=[0, 2, 1]) validation_loss = self.loss_fun(v_outputs, v_logits, v_masks) validation_loss += sum(self.ei_rnn.losses) validation_loss_all += validation_loss.numpy() validation_loss_all = validation_loss_all / validation_batch_num validation_acc_all = validation_acc_all / validation_batch_num over_all_performace.append(validation_acc_all) print('validation loss:', validation_loss_all) print('validation acc:', validation_acc_all) with self.validation_summary_writer.as_default(): tf.summary.scalar('loss', validation_loss_all, step=epoch_i) tf.summary.scalar('acc', validation_acc_all, step=epoch_i) cm_image = plot.plot_confusion_matrix(self.get_w_rec_m()) tf.summary.image('M_rec', cm_image, step=epoch_i) win_image = plot.plot_confusion_matrix( funs.rectify( self.rnn_cell.W_in.numpy()[:, :int(UNITS_SIZE * EI_RATIO)]), False) tf.summary.image('M_in', win_image, step=epoch_i) wout_image = plot.plot_confusion_matrix( self.get_w_out_m()[:, :int(UNITS_SIZE * EI_RATIO)], False) tf.summary.image('M_out', wout_image, step=epoch_i) print('spr: ', funs.spectral_radius(self.get_w_rec_m().T)) if epoch_i > PERFORMANCE_CHECK_REGION and \ np.mean(over_all_performace[-PERFORMANCE_CHECK_REGION:]) > PERFORMANCE_LEVEL: print( 'Overall performance level is satisfied, training is terminated\n' ) break # Save Model self.ckpt.step.assign_add(1) self.ckpt_manager.save() # self.reset_all_weights() # todo: may uncomment # print('Remove all weights below ' + str(SGD_p['mini_w_threshold'])) # print('\n') print('Training is done') print('#' * 20) print('\n') # Test # self.test()
def main(): model_serial = define_model() # Save model model_yaml = model_serial.to_yaml() model_yaml_file = open( path.join(FLAGS.modelPath, FLAGS.sessionName + '.model.yaml'), "w") model_yaml_file.write(model_yaml) model_yaml_file.close() #keras.utils.plot_model(model_serial, to_file=path.join(FLAGS.modelPath, FLAGS.sessionName + '.model.png'), show_shapes= True) # load weights if FLAGS.startEpoch > 0: model_serial.load_weights( path.join( FLAGS.modelPath, FLAGS.sessionName + '.weights.{:02d}.hdf5'.format(FLAGS.startEpoch))) print('Loaded epoch', FLAGS.startEpoch) if (FLAGS.testOnly == True or FLAGS.predictOnly) and FLAGS.startEpoch == 0: model_serial.load_weights( path.join(FLAGS.modelPath, FLAGS.sessionName + '.weights.hdf5')) model = model_serial model.summary() model.compile( loss='categorical_crossentropy', optimizer=keras.optimizers.nadam(lr=FLAGS.learningRate), metrics=['accuracy'], ) if FLAGS.randomSeed > 0: random.seed(FLAGS.randomSeed) if FLAGS.testOnly == True: test(model) sys.exit() if FLAGS.predictOnly == True: predict(model) sys.exit() trainingParameterText = 'BatchSize : {:02d}; learningRate : {:.8f}; L2 : {:.8f}; Dropout : {:.4f}'.format( FLAGS.batchSize, FLAGS.learningRate, FLAGS.L2, FLAGS.dropout) print(trainingParameterText) print('_________________________________________________________________') print('Number of training samples: ', end='') training_data_generator = DataGenerator( data_shape, num_class, FLAGS.filePath, FLAGS.filePattern + FLAGS.trainingSetExtension + '.*.' + FLAGS.extension, FLAGS.batchSize, percent=FLAGS.trainingPercent, queue_size=FLAGS.queueSize, shuffle=True, #class_weight=class_weight, ) print('Number of validation samples: ', end='') validation_data_generator = DataGenerator( data_shape, num_class, FLAGS.filePath, FLAGS.filePattern + FLAGS.validationSetExtension + '.*.' + FLAGS.extension, FLAGS.testBatchSize, percent=FLAGS.validationPercent, shuffle=True, #class_weight=class_weight, ) print('Number of test samples: ', end='') test_data_generator = DataGenerator( data_shape, num_class, FLAGS.filePath, FLAGS.filePattern + FLAGS.testSetExtension + '.*.' + FLAGS.extension, FLAGS.testBatchSize, percent=FLAGS.testPercent, shuffle=True, ) print('_________________________________________________________________') callback_list = [] callback_list.append( RecordEpoch( best_epoch_dict, weigh_filepath=path.join( FLAGS.modelPath, FLAGS.sessionName + '.weights.{epoch:02d}.hdf5'), csv_log_filepath=path.join( FLAGS.modelPath, FLAGS.sessionName + '.' + FLAGS.logExtension, ), csv_log_header=trainingParameterText, patience=FLAGS.earlyStoppingPatience, sleep_after_epoch=sleep_after_epoch, )) history = model.fit_generator( training_data_generator, training_data_generator.num_steps(), epochs=FLAGS.maxEpoch, initial_epoch=FLAGS.startEpoch, verbose=1, callbacks=callback_list, #class_weight=class_weight, validation_data=validation_data_generator, validation_steps=validation_data_generator.num_steps(), max_queue_size=FLAGS.queueSize, ) training_data_generator.terminate() print('Best epoch:', best_epoch_dict['epoch_index']) print('Validation accuracy:', best_epoch_dict['val_acc']) model.load_weights( path.join( FLAGS.modelPath, FLAGS.sessionName + '.weights.{:02d}.hdf5'.format(best_epoch_dict['epoch_index']))) print('Loaded weights of this epoch') test(model)
from data.data_generator import DataGenerator from evaluation.synthetic_experiment import SynthExperiment dataGen = DataGenerator('./config/data.ini', seed=42) #Experiment(dataGen, config='./config/class_bias_experiment.ini').replot_results(exclude_methods=['bac_mace'], recompute=True) #Experiment(dataGen, config='./config/acc_bias_experiment.ini').replot_results(exclude_methods=['bac_mace'], recompute=True) #Experiment(dataGen, config='./config/short_bias_experiment.ini').replot_results(exclude_methods=['bac_mace'], recompute=True) # Experiment(dataGen, config='./config/group_ratio_experiment.ini').replot_results(exclude_methods=['bac_mace'], recompute=True) SynthExperiment(dataGen, config='./config/crowd_size_experiment.ini').replot_results() # Experiment(dataGen, config='./config/doc_length_experiment.ini').replot_results()