def test_synthetic_random_data(self):
ds = DataGenerator.get_random_dataset(height=32,
width=32,
num_classes=5,
data_type=tf.float32)
assert (DataGenerator.evaluate_size_dataset(ds) == 1)
assert (isinstance(ds, tf.data.Dataset))
def predict(model):
input_files = fnmatch.filter(
listdir(FLAGS.filePath), FLAGS.filePattern +
FLAGS.predictionSetExtension + '.*.' + FLAGS.extension)
if len(input_files) == 0:
sys.exit("File not found: " + filename_pattern)
for file in input_files:
print('Number of predictions: ', end='')
predict_data_generator = DataGenerator(
data_shape,
num_class,
FLAGS.filePath,
file,
FLAGS.testBatchSize,
useOddSample=True,
)
prediction = model.predict_generator(
predict_data_generator,
predict_data_generator.num_steps(),
max_queue_size=FLAGS.queueSize,
)
np.save(
path.join(
FLAGS.filePath,
file.replace(FLAGS.predictionSetExtension,
FLAGS.predictionExtension).replace(
'.' + FLAGS.extension, '')), prediction)
def test_malformed_directory(self):
imagenet_original_data_path = '/localdata/datasets/imagenet-raw-data'
output_directory = '/tmp/temporary_imagenet_dataset_directory'
split = 'val'
with self.assertRaises(NameError):
# if the path imagenet_original_data_path + split doesn't exist the function will throw an error
DataGenerator.build_imagenet_tf_record(
imagenet_original_data_path,
split,
output_directory=output_directory)
def main():
model = create_model(trainable=TRAINABLE)
# if TRAINABLE:
# model.load_weights(WEIGHTS)
train_datagen = DataGenerator(file_path=cfg.TRAIN.DATA_PATH,
config_path=cfg.TRAIN.ANNOTATION_PATH)
val_generator = DataGenerator(file_path=cfg.TEST.DATA_PATH,
config_path=cfg.TEST.ANNOTATION_PATH,
debug=False)
validation_datagen = Validation(generator=val_generator)
learning_rate = cfg.TRAIN.LEARNING_RATE
if TRAINABLE:
learning_rate /= 10
optimizer = tf.keras.optimizers.SGD(lr=learning_rate,
decay=cfg.TRAIN.LR_DECAY,
momentum=0.9,
nesterov=False)
model.compile(loss=detect_loss(), optimizer=optimizer, metrics=[])
checkpoint = tf.keras.callbacks.ModelCheckpoint("model-{val_iou:.2f}.h5",
monitor="val_iou",
verbose=1,
save_best_only=True,
save_weights_only=True,
mode="max")
stop = tf.keras.callbacks.EarlyStopping(monitor="val_iou",
patience=cfg.TRAIN.PATIENCE,
mode="max")
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor="val_iou",
factor=0.6,
patience=5,
min_lr=1e-6,
verbose=1,
mode="max")
# Define the Keras TensorBoard callback.
logdir = "logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=logdir)
model.fit_generator(generator=train_datagen,
epochs=cfg.TRAIN.EPOCHS,
callbacks=[
tensorboard_callback, validation_datagen,
checkpoint, reduce_lr, stop
],
shuffle=True,
verbose=1)
def test_fetch_from_directory(self):
dirpath = '/tmp/temporary_dataset_directory'
split_name = 'tmp_train'
label_name1 = 'tmp_label1'
label_name2 = 'tmp_label2'
full_dir_to_create_1 = os.path.join(dirpath, split_name, label_name1)
full_dir_to_create_2 = os.path.join(dirpath, split_name, label_name2)
image_path_1 = os.path.join(full_dir_to_create_1,
'temporary_image_1.jpg')
image_path_2 = os.path.join(full_dir_to_create_2,
'temporary_image_2.jpg')
if not (os.path.exists(dirpath)):
os.makedirs(full_dir_to_create_1)
os.makedirs(full_dir_to_create_2)
image_1 = np.ones((1, 1, 3), np.uint8)
cv2.imwrite(image_path_1, image_1)
image_2 = np.ones((1, 1, 3), np.uint8)
cv2.imwrite(image_path_2, image_2)
ds, img_shape, num_examples, num_classes = DataGenerator.get_dataset_from_directory(
ds_path=dirpath, split=split_name)
assert (num_classes == 2)
assert (num_examples == 2)
assert (tuple(img_shape) == (1, 1, 3))
assert (isinstance(ds, tf.data.Dataset))
for image, label in tfds.as_numpy(ds):
assert (np.all(image == 1))
assert (isinstance(label, np.int64))
shutil.rmtree(dirpath)
def predict(model_name,
model=None,
row_start=None,
row_end=None,
custom_objects=None):
if 'batch_size' not in config:
config['batch_size'] = default_batch_size
if 'max_queue_size' not in config:
config['max_queue_size'] = default_max_queue_size
if 'does_use_multiprocessing' not in config:
config['does_use_multiprocessing'] = default_does_use_multiprocessing
if 'worker_number' not in config:
config['worker_number'] = default_worker_number
if 'verbose' not in config:
config['verbose'] = default_verbose
if model is None:
if custom_objects is None:
custom_objects = custom_metrics
model = load_model(model_name=model_name,
custom_objects=custom_objects,
does_compile=True)
if model is None:
raise NoTrainedModelException(model_name)
rolling_window_size = get_rolling_window_size(model_name)
generator = DataGenerator(
dataset_name=DATASET_NAME_PREDICT,
rolling_window_size=rolling_window_size,
row_start=row_start,
row_end=row_end,
max_batch_size=config['batch_size'],
does_shuffle=False, # NOT shuffle!
)
snpr = generator.get_sample_number_per_row()
if config['batch_size'] % snpr != 0:
logging.warning(
'predict: batch_size(%d) cannot divide %d. '
'Some inputs will be ignored.', config['batch_size'], snpr)
result = model.predict_generator(
generator=generator,
max_queue_size=config['max_queue_size'],
use_multiprocessing=config['does_use_multiprocessing'],
workers=config['worker_number'],
verbose=config['verbose'],
)
return result
def test_build_imagenet_validation(self):
imagenet_original_data_path = '/localdata/datasets/imagenet-raw-data'
output_directory = '/tmp/temporary_imagenet_dataset_directory'
DataGenerator.build_imagenet_tf_record(
imagenet_original_data_path,
'validation',
output_directory=output_directory)
ds, img_shape, num_examples, num_classes = DataGenerator.get_imagenet(
output_directory, 'validation')
self.assertTrue(isinstance(ds, tf.data.Dataset))
self.assertEqual(img_shape, (224, 224, 3))
self.assertEqual(num_examples, 50000)
self.assertEqual(num_classes, 1000)
shutil.rmtree(output_directory)
def evaluation(session, model, mode='validation', percent_limit=None, save_path=None):
def log(msg, add_trailing=True):
LOG(msg, add_trailing)
dataset = DataGenerator(mode)
target_formulas = []
predicted_formulas = []
pp_hist = []
last_log_percentage = 0
log_percentage_every = 10
for epoch, percentage, images, formulas, _ in dataset.generator(1, percent_limit):
target = dataset.decode_formulas(formulas)
prediction, pp = model.predict(sess=session, images=images)
prediction = dataset.decode_formulas(prediction)
target_formulas += target
predicted_formulas += prediction
pp_hist += [pp]
max_per = 1 if percent_limit is None else percent_limit
percentage = int(100 * (percentage/max_per))
if percentage >= last_log_percentage + log_percentage_every:
idx = random.randint(0, len(prediction) - 1)
last_log_percentage += log_percentage_every
log('Evaluation prediction progress completion: {}%\ntrue -> {}\npred -> {}'.
format(percentage, '' if len(target) <= idx else target[idx], prediction[idx]))
if save_path is not None:
with open(save_path, 'w+') as f:
f.write('\n'.join(predicted_formulas))
if len(target_formulas) != len(predicted_formulas):
log("number of formulas doesn't match", False)
return None
bleu_score = bleu_eval(target_formulas, predicted_formulas)
edit_distance_score = edit_distance_eval(target_formulas, predicted_formulas)
pp_mean = np.mean(np.array(pp_hist))
log('Bleu score: {0:2.3f} %'.format(100 * bleu_score))
log('Edit distance score: {0:2.3f} %'.format(100 * edit_distance_score))
log('Perplexity: {0:2.3f}'.format(pp_mean))
return bleu_score, edit_distance_score
def test_load_imagenet_train(self):
imagenet_tf_record_data_path = '/localdata/datasets/imagenet-data'
# if the path doesn't exist the function will throw an error
ds, img_shape, num_examples, num_classes = DataGenerator.get_imagenet(
imagenet_tf_record_data_path, 'train')
self.assertTrue(isinstance(ds, tf.data.Dataset))
self.assertEqual(img_shape, (224, 224, 3))
self.assertEqual(num_examples, 1281167)
self.assertEqual(num_classes, 1000)
def test_mnist(self):
dirpath = '/tmp/mnist_test_sample_directory'
if not (os.path.exists(dirpath)):
os.makedirs(dirpath)
ds, img_shape, num_examples, num_classes = DataGenerator.get_dataset_from_name(
ds_name='mnist', ds_path=dirpath, split='train[0:3]')
assert (num_classes == 10)
assert (num_examples == 3)
assert (tuple(img_shape) == (28, 28, 1))
assert (isinstance(ds, tf.data.Dataset))
ds, img_shape, num_examples, num_classes = DataGenerator.get_dataset_from_name(
ds_name='mnist', ds_path=dirpath, split='test[0:3]')
assert (num_classes == 10)
assert (num_examples == 3)
assert (tuple(img_shape) == (28, 28, 1))
assert (isinstance(ds, tf.data.Dataset))
def main():
print("------- get configs -------")
config = get_configs("./py/configs.json")
print("------- generate dataset -------")
data = DataGenerator(config)
X_train, y_train = data.get_train_data()
X_test, y_test = data.get_test_data()
word_index = data.get_word_index()
print("------- create model -------")
model = BiGRUAttention(config, word_index)
print("------- model training and predicting-------")
trainer = CBTrainer(model, config)
trainer.fit(X_train, y_train)
y_pred = trainer.predict(X_test)
trainer.score(X_test, y_test)
return y_pred
def test(model):
print('Number of test samples: ', end='')
test_data_generator = DataGenerator(
data_shape,
num_class,
FLAGS.filePath,
FLAGS.filePattern + FLAGS.testSetExtension + '.*.' + FLAGS.extension,
FLAGS.testBatchSize,
useOddSample=True,
percent=FLAGS.testPercent,
shuffle=True,
)
score = model.evaluate_generator(
test_data_generator,
test_data_generator.num_steps(),
max_queue_size=FLAGS.queueSize,
)
print('Test loss: ', score[0])
print('Test accuracy:', score[1])
def main():
# Data
train_class = DataGenerator(number_of_examples= samples, number_of_features= features, batch_size = batch, shuffle = to_shuffle)
test_class = DataGenerator(number_of_examples=samples, number_of_features= features, batch_size = batch, shuffle = to_shuffle)
train_data, train_target = train_class.generate_data()
test_data, test_target = test_class.generate_data()
# Layers
input_layer = Linear(in_features= features, out_features = 25)
first_hidden_layer = Linear(in_features = 25, out_features = 25)
second_hidden_layer = Linear(in_features= 25, out_features = 25)
output_layer = Linear(in_features= 25, out_features = 2)
layers = [input_layer, Relu(), first_hidden_layer, Relu(), second_hidden_layer, Relu(), output_layer, Tanh()]
# Model
model = Sequential(layers)
# Optimizer
optimizer = StochasticGD(parameters_list=model.param(), learning_rate = 0.01)
# Loss
loss = Mse()
# Initializing Network class
rn = RunNetwork(epochs = number_of_epochs, train_class = train_class,\
test_class = test_class, model = model, loss = loss, optimizer= optimizer, samples = samples)
# Training the
rn.run_network()
def initialize(mode='train', is_evaluation=False):
def log(msg, trailing_line=False):
LOG(msg, trailing_line)
dataset = DataGenerator(mode)
start_token, pad_token = dataset.data.start_token(), dataset.data.pad_token()
model_name = FLAGS.model_name
model_path = os.path.join(config.save_path, model_name + '.ckpt')
secondary_model_path = os.path.join(config.secondary_path, config.save_path, model_name + '.ckpt')
logger.set_model_name(model_name + ('-test' if is_evaluation else ''))
log(config, True)
log(h_params, True)
gpu_is_available = tf.test.is_gpu_available()
log('GPU is available: ' + str(gpu_is_available))
log('Start building graph')
tf.reset_default_graph()
model = ImageToLatexModel(start_token, pad_token)
log('Graph building finished!', True)
# params = tf.trainable_variables()
# print('Trainable params:')
# for p in params:
# print(p)
sess = tf.Session()
init_opt = tf.global_variables_initializer()
saver = tf.train.Saver(keep_checkpoint_every_n_hours=1)
restored = False
if FLAGS.load_from_previous or is_evaluation:
def load(path, name):
nonlocal restored, saver, sess
if restored:
return
try:
saver.restore(sess, path)
step = sess.run([model.step])[0]
log('Model restored from last session in {}, current step: {}'.format(name, step))
# run_eval()
restored = True
except Exception as e:
log('Can\'t load from previous session in {}, error: {}'.format(name, e))
load(model_path, 'model path')
load(secondary_model_path, 'secondary model path')
if not restored:
sess.run(init_opt)
return model, sess, dataset, saver, model_path, secondary_model_path
def __init__(self,
# splitter related
label_list: list,
is_sliced_data: bool,
is_create_negative_sample: bool,
negative_sample_scale: int,
# data generator related
sample_spaces: int,
# model related
input_size: int,
hidden_size: int,
num_layers: int,
dropout: float,
learning_rate: float,
embedding_scale: int,
# train related
epochs: int,
train_batch: int,
eval_batch: int):
self.epochs = epochs
self.train_batch = train_batch
self.eval_batch = eval_batch
splitter = Splitter(label_list, is_sliced_data, is_create_negative_sample, negative_sample_scale)
self.nested_label2id = splitter.label2id
self.generator = DataGenerator(splitter, sample_spaces)
self.model = RnnRelatedModel(
input_size=input_size,
hidden_size=hidden_size,
num_layers=num_layers,
dropout=dropout,
output_size=self.generator.get_label_size(),
# TODO: this should be dynamically generated
loss_weight=None,
embedding_scale=embedding_scale
)
self.optimizer = torch.optim.Adadelta(self.model.parameters(), lr=learning_rate)
self.name = "trainer_" + self.generator.name
def generate_data():
"""
Route to regenerate the data collection.
"""
# We could also run it on a separate thread
# from threading import Thread
# t = Thread(daemon=True)
# t.run(DataGenerator().generate())
logger.info(
f'CALLED FROM: {GENERATE_DATA} ENDPOINT: Generating the data collection.'
)
DataGenerator().generate()
return jsonify("The data collection has been generated")
def test_model(model_name,
model=None,
row_start=None,
row_end=None,
custom_objects=None):
if 'batch_size' not in config:
config['batch_size'] = default_batch_size
if 'does_shuffle' not in config:
config['does_shuffle'] = default_does_shuffle
if 'max_queue_size' not in config:
config['max_queue_size'] = default_max_queue_size
if 'does_use_multiprocessing' not in config:
config['does_use_multiprocessing'] = default_does_use_multiprocessing
if 'worker_number' not in config:
config['worker_number'] = default_worker_number
if 'verbose' not in config:
config['verbose'] = default_verbose
if model is None:
if custom_objects is None:
custom_objects = custom_metrics
model = load_model(model_name=model_name,
custom_objects=custom_objects,
does_compile=True)
if model is None:
raise NoTrainedModelException(model_name)
rolling_window_size = get_rolling_window_size(model_name)
generator = DataGenerator(
dataset_name=DATASET_NAME_TEST,
rolling_window_size=rolling_window_size,
row_start=row_start,
row_end=row_end,
max_batch_size=config['batch_size'],
does_shuffle=config['does_shuffle'],
)
result_list = model.evaluate_generator(
generator=generator,
max_queue_size=config['max_queue_size'],
use_multiprocessing=config['does_use_multiprocessing'],
workers=config['worker_number'],
verbose=config['verbose'],
)
result_map = dict(zip(model.metrics_names, result_list))
return result_map
import random
import matplotlib.pyplot as plt
import numpy as np
from data.data_generator import DataGenerator
gen = DataGenerator('train')
for ep, per, images, formulas, l in gen.generator(1):
for i in range(1):
idx = random.randint(0, images.shape[0] - 1)
img = images[idx]
formula = np.expand_dims(formulas[idx], axis=0)
formula = gen.decode_formulas(formula)[0]
print(formula)
plt.imshow(img, cmap='gray')
plt.title(formula)
plt.show()
def get_dataset(dataset_name: str,
dataset_path: str,
split: str,
img_datatype: tf.dtypes.DType,
micro_batch_size: int,
shuffle: bool = False,
accelerator_side_preprocess: bool = True,
eight_bit_transfer: Optional[EightBitTransfer] = None,
apply_preprocessing: bool = True,
pipeline_num_parallel: int = 48,
seed: Optional[int] = None):
logging.info(f'dataset_name = {dataset_name}')
if popdist.getNumInstances() == 1:
logging.info(f'Since the training is run in a single process, setting dataset pipeline threading '
f'and prefetching buffer size to tf.data.AUTOTUNE.')
pipeline_num_parallel = prefetch_size = tf.data.AUTOTUNE
else:
prefetch_size = PREFETCH_BUFFER_SIZE
logging.info(f'Setting number of threads for the dataset pipeline to {pipeline_num_parallel}, '
f'and the prefetching buffer size to {prefetch_size}.')
ds, img_shape, dataset_size, num_classes = DataGenerator.get_dataset_from_name(
ds_name=dataset_name, ds_path=dataset_path, split=split)
preprocess_fn = None
if apply_preprocessing:
if dataset_name == 'cifar10':
ds, preprocess_fn = DataTransformer.cifar_preprocess(
ds,
buffer_size=dataset_size,
img_type=img_datatype,
is_training=(split == 'train'),
accelerator_side_preprocess=accelerator_side_preprocess,
pipeline_num_parallel=pipeline_num_parallel,
)
elif dataset_name == 'imagenet':
ds, preprocess_fn = DataTransformer.imagenet_preprocessing(
ds,
img_type=img_datatype,
is_training=(split == 'train'),
accelerator_side_preprocess=accelerator_side_preprocess,
pipeline_num_parallel=pipeline_num_parallel,
seed=seed
)
if shuffle:
# Shuffle the input files
ds = ds.shuffle(buffer_size=IMAGENET_SHUFFLE_BUFFER_SIZE)
else:
ds = DataTransformer.cache_shuffle(ds, buffer_size=dataset_size, shuffle=(split == 'train'))
ds = DataTransformer.normalization(ds, img_type=img_datatype)
preprocess_fn = None
if eight_bit_transfer is not None:
ds = ds.map(lambda x, y: (eight_bit_transfer.compress(x), y), num_parallel_calls=pipeline_num_parallel)
ds = ds.batch(batch_size=micro_batch_size, drop_remainder=True)
ds = ds.repeat().prefetch(prefetch_size)
cpu_memory_usage = psutil.virtual_memory().percent
if cpu_memory_usage > 100:
logging.warning(f'cpu_memory_usage is {cpu_memory_usage} > 100% so your program is likely to crash')
return ds, img_shape, dataset_size, num_classes, preprocess_fn
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 25 14:47:12 2019
@author: mohamed
"""
import tensorflow as tf
from trainers.trainer import Trainer
from models.model import DCGAN
from data.data_generator import DataGenerator
from utils import config
config = config.process_config("config.json")
data = DataGenerator(config)
c = tf.ConfigProto()
c.gpu_options.allow_growth = True
sess = tf.Session(config=c)
model = DCGAN(config)
trainer = Trainer(config , model , data, sess)
trainer.train()
return dataset, dataset_size
if __name__ == '__main__':
set_seed()
print('process id ', os.getpid())
opt = TrainOptions().parse()
opt_val, opt_test = get_val_test_opts(opt)
test_index = opt.n_fold - 1 if opt.test_index is None else opt.test_index
test_index = opt.n_fold if 'test' not in opt.test_mode else test_index
val_indices = [x for x in range(opt.n_fold) if x != test_index
] if opt.test_mode != 'test' else [test_index]
models = []
data_generator = DataGenerator(opt.dataroot)
for val_index in val_indices:
data_generator.build_dataset(val_index, test_index, opt.test_mode)
dataset, dataset_size = create_data_loader(opt)
dataset_val, dataset_size_val = create_data_loader(opt_val)
dataset_test, dataset_size_test = create_data_loader(opt_test)
model_suffix = 'val%d' % val_index if 'val' in opt.test_mode else ''
model_suffix += 'test%d' % test_index if 'test' in opt.test_mode else ''
model = create_model(opt, model_suffix)
model.setup(opt)
visualizer = Visualizer(opt)
total_steps, best_epochs, val_losses_best = 0, 0, 0
for epoch in range(opt.epoch_count, opt.niter + opt.niter_decay + 1):
epoch_start_time = time.time()
def test_check_size(self):
ds = tf.data.Dataset.from_tensor_slices([1, 2, 3])
assert (DataGenerator.evaluate_size_dataset(ds) == 3)
def test_mnist_wrong_dir(self):
with self.assertRaises(NameError):
_ = DataGenerator.get_dataset_from_name(ds_name='mnist',
ds_path='foo')
def test_unsupported_dataset(self):
with self.assertRaises(NameError):
_ = DataGenerator.get_dataset_from_name(ds_name='foo')
class Trainer(object):
def __init__(self,
# splitter related
label_list: list,
is_sliced_data: bool,
is_create_negative_sample: bool,
negative_sample_scale: int,
# data generator related
sample_spaces: int,
# model related
input_size: int,
hidden_size: int,
num_layers: int,
dropout: float,
learning_rate: float,
embedding_scale: int,
# train related
epochs: int,
train_batch: int,
eval_batch: int):
self.epochs = epochs
self.train_batch = train_batch
self.eval_batch = eval_batch
splitter = Splitter(label_list, is_sliced_data, is_create_negative_sample, negative_sample_scale)
self.nested_label2id = splitter.label2id
self.generator = DataGenerator(splitter, sample_spaces)
self.model = RnnRelatedModel(
input_size=input_size,
hidden_size=hidden_size,
num_layers=num_layers,
dropout=dropout,
output_size=self.generator.get_label_size(),
# TODO: this should be dynamically generated
loss_weight=None,
embedding_scale=embedding_scale
)
self.optimizer = torch.optim.Adadelta(self.model.parameters(), lr=learning_rate)
self.name = "trainer_" + self.generator.name
def train(self):
data_gen = self.generator
model = self.model
optimizer = self.optimizer
total_batch = data_gen.get_total_batch(self.train_batch, "train")
print("---------- Start Training ----------")
for epoch in range(self.epochs):
for idx, batch_data in enumerate(data_gen.generate(batch_size=self.train_batch, mode="train")):
batch_input, batch_label = batch_data
optimizer.zero_grad()
preds, loss = model(batch_input, batch_label, "train")
loss.backward()
optimizer.step()
if idx % 10 == 0:
print(f"loss for batch {idx} / {total_batch}: {loss.data}")
self.save_model(epoch + 1)
def evaluate(self):
self.restore_model()
data_gen = self.generator
model = self.model
model.eval()
preds, labels = [], []
total_batch = data_gen.get_total_batch(self.eval_batch, "eval")
print("---------- Start Evaluating ----------")
for idx, batch_data in enumerate(data_gen.generate(batch_size=self.eval_batch, mode="eval")):
batch_input, batch_label = batch_data
pred, _ = model(batch_input, batch_label, "eval")
preds.extend(pred.detach().numpy())
labels.extend(batch_label)
if idx % 10 == 0:
print(f"eval batch: {idx} / {total_batch}")
metrics = eval_result(preds, labels, self.nested_label2id, True)
return metrics
def predict(self):
self.restore_model()
def save_model(self, epoch: int):
torch.save({'epoch': epoch,
'state_dict': self.model.state_dict(),
'optimizer': self.optimizer.state_dict()},
'./result/{}.epoch-{}.ckpt'.format(self.name, epoch))
def restore_model(self, epoch: int = -1):
if epoch == -1:
model_file_paths = glob.glob('result/{}.epoch-*'.format(self.name))
try:
epoch = max([int(re.findall('{}\.epoch-(\d*)\.ckpt'.format(self.name), path)[0]) for path in model_file_paths])
except:
print("---------- Error: Model not saved. ----------")
exit(-1)
ckpt_path = './result/{}.epoch-{}.ckpt'.format(self.name, epoch)
model_ckpt = torch.load(ckpt_path)
self.model.load_state_dict(model_ckpt['state_dict'])
self.optimizer.load_state_dict(model_ckpt['optimizer'])
dropout_output = self.dropout(dense_output)
# (batch_size, output_size)
output = self.output_layer(dropout_output)
if mode == "train":
loss = self.loss(output, batch_label)
return output, loss
return output, None
if __name__ == '__main__':
splitter = Splitter(label_list=[0, 2, 7, 17],
is_slice_data=False,
is_create_negative_sample=True)
splitter.split()
generator = DataGenerator(splitter, 10)
model = RnnRelatedModel(input_size=12,
hidden_size=64,
num_layers=1,
dropout=0.2,
output_size=splitter.get_label_size())
optimizer = torch.optim.Adadelta(model.parameters(), lr=0.9)
for idx, data in enumerate(generator.generate(batch_size=20)):
optimizer.zero_grad()
preds, loss = model(data[0], data[1], "train")
print(loss.data)
loss.backward()
optimizer.step()
def test(self, test_batch_num=50):
print('Start to test')
print('#' * 20)
self.ckpt.restore(self.ckpt_manager.latest_checkpoint)
if self.ckpt_manager.latest_checkpoint:
print("Restored from {}".format(
self.ckpt_manager.latest_checkpoint))
else:
print("Initializing from scratch.")
dg = DataGenerator(task_version=self.task_version,
action='test') # todo: should be test for action
psycollection = {'coh': [], 'perc': []}
for batch_index in range(20):
descs, test_masks, test_inputs, test_outputs = dg.get_valid_test_datasets(
)
test_logits, _ = self.ei_rnn(test_inputs, [self.init_state],
training=False)
acc_test_logits = test_logits.numpy()
acc_test_outputs = tf.transpose(test_outputs, perm=[0, 2,
1]).numpy()
test_acc = self.get_accuracy(acc_test_logits, acc_test_outputs,
test_masks)
test_logits = tf.transpose(test_logits, perm=[0, 2, 1])
test_loss = self.loss_fun(test_outputs, test_logits, test_masks)
test_loss += sum(self.ei_rnn.losses)
print('test loss:', test_loss.numpy())
print('test acc:', test_acc)
tmp_data = self.get_psychometric_data(descs, test_logits.numpy())
psycollection['coh'] += tmp_data['coh']
psycollection['perc'] += tmp_data['perc']
with self.test_summary_writer.as_default():
tf.summary.scalar('loss', test_loss, step=batch_index)
tf.summary.scalar('acc', test_acc, step=batch_index)
curve_image = plot.plot_dots(psycollection['coh'],
psycollection['perc'])
tf.summary.image('psycollection',
curve_image,
step=batch_index)
cm_image = plot.plot_confusion_matrix(self.get_w_rec_m())
tf.summary.image('M_rec', cm_image, step=batch_index)
win_image = plot.plot_confusion_matrix(
funs.rectify(
self.rnn_cell.W_in.numpy()[:, :int(UNITS_SIZE *
EI_RATIO)]), False)
tf.summary.image('M_in', win_image, step=batch_index)
wout_image = plot.plot_confusion_matrix(
self.get_w_out_m()[:, :int(UNITS_SIZE * EI_RATIO)], False)
tf.summary.image('M_out', wout_image, step=batch_index)
def train(self):
self.ckpt.restore(self.ckpt_manager.latest_checkpoint)
if self.ckpt_manager.latest_checkpoint:
print("Restored from {}".format(
self.ckpt_manager.latest_checkpoint))
else:
print("Initializing from scratch.")
dg = DataGenerator(task_version=self.task_version, action='train')
validation_batch_num = self.batch_num // 10
over_all_performace = []
print('Start to train')
print('#' * 20)
for epoch_i in range(self.epoch_num):
print('Epoch ' + str(epoch_i))
# Train
#
train_loss_all = 0
for batch_i in range(self.batch_num):
decs, masks, inputs, outputs = next(dg)
with tf.GradientTape() as tape:
logits, _ = self.ei_rnn(inputs, [self.init_state],
training=True)
logits = tf.transpose(logits, perm=[0, 2, 1])
train_loss = self.loss_fun(outputs, logits, masks)
train_loss += sum(self.ei_rnn.losses)
train_loss_all += train_loss.numpy()
all_weights = self.ei_rnn.trainable_weights + [self.init_state]
grads = tape.gradient(train_loss, all_weights)
grads, _ = tf.clip_by_global_norm(grads, self.grad_clip)
self.optimizer.apply_gradients(
zip(grads, self.ei_rnn.trainable_weights))
train_loss_all = train_loss_all / self.batch_num
print('train loss:', train_loss_all)
with self.train_summary_writer.as_default():
tf.summary.scalar('loss', train_loss_all, step=epoch_i)
# Validation
#
validation_loss_all = 0
validation_acc_all = 0
for v_batch_i in range(validation_batch_num):
_, v_masks, v_inputs, v_outputs = dg.get_valid_test_datasets(
) # dg.get_valid_test_datasets()
v_logits, _ = self.ei_rnn(v_inputs, [self.init_state])
acc_v_logits = v_logits.numpy()
acc_v_outputs = tf.transpose(v_outputs, perm=[0, 2, 1]).numpy()
validation_acc = self.get_accuracy(acc_v_logits, acc_v_outputs,
v_masks)
validation_acc_all += validation_acc
v_logits = tf.transpose(v_logits, perm=[0, 2, 1])
validation_loss = self.loss_fun(v_outputs, v_logits, v_masks)
validation_loss += sum(self.ei_rnn.losses)
validation_loss_all += validation_loss.numpy()
validation_loss_all = validation_loss_all / validation_batch_num
validation_acc_all = validation_acc_all / validation_batch_num
over_all_performace.append(validation_acc_all)
print('validation loss:', validation_loss_all)
print('validation acc:', validation_acc_all)
with self.validation_summary_writer.as_default():
tf.summary.scalar('loss', validation_loss_all, step=epoch_i)
tf.summary.scalar('acc', validation_acc_all, step=epoch_i)
cm_image = plot.plot_confusion_matrix(self.get_w_rec_m())
tf.summary.image('M_rec', cm_image, step=epoch_i)
win_image = plot.plot_confusion_matrix(
funs.rectify(
self.rnn_cell.W_in.numpy()[:, :int(UNITS_SIZE *
EI_RATIO)]), False)
tf.summary.image('M_in', win_image, step=epoch_i)
wout_image = plot.plot_confusion_matrix(
self.get_w_out_m()[:, :int(UNITS_SIZE * EI_RATIO)], False)
tf.summary.image('M_out', wout_image, step=epoch_i)
print('spr: ', funs.spectral_radius(self.get_w_rec_m().T))
if epoch_i > PERFORMANCE_CHECK_REGION and \
np.mean(over_all_performace[-PERFORMANCE_CHECK_REGION:]) > PERFORMANCE_LEVEL:
print(
'Overall performance level is satisfied, training is terminated\n'
)
break
# Save Model
self.ckpt.step.assign_add(1)
self.ckpt_manager.save()
# self.reset_all_weights() # todo: may uncomment
# print('Remove all weights below ' + str(SGD_p['mini_w_threshold']))
# print('\n')
print('Training is done')
print('#' * 20)
print('\n')
# Test
#
self.test()
def main():
model_serial = define_model()
# Save model
model_yaml = model_serial.to_yaml()
model_yaml_file = open(
path.join(FLAGS.modelPath, FLAGS.sessionName + '.model.yaml'), "w")
model_yaml_file.write(model_yaml)
model_yaml_file.close()
#keras.utils.plot_model(model_serial, to_file=path.join(FLAGS.modelPath, FLAGS.sessionName + '.model.png'), show_shapes= True)
# load weights
if FLAGS.startEpoch > 0:
model_serial.load_weights(
path.join(
FLAGS.modelPath, FLAGS.sessionName +
'.weights.{:02d}.hdf5'.format(FLAGS.startEpoch)))
print('Loaded epoch', FLAGS.startEpoch)
if (FLAGS.testOnly == True or FLAGS.predictOnly) and FLAGS.startEpoch == 0:
model_serial.load_weights(
path.join(FLAGS.modelPath, FLAGS.sessionName + '.weights.hdf5'))
model = model_serial
model.summary()
model.compile(
loss='categorical_crossentropy',
optimizer=keras.optimizers.nadam(lr=FLAGS.learningRate),
metrics=['accuracy'],
)
if FLAGS.randomSeed > 0:
random.seed(FLAGS.randomSeed)
if FLAGS.testOnly == True:
test(model)
sys.exit()
if FLAGS.predictOnly == True:
predict(model)
sys.exit()
trainingParameterText = 'BatchSize : {:02d}; learningRate : {:.8f}; L2 : {:.8f}; Dropout : {:.4f}'.format(
FLAGS.batchSize, FLAGS.learningRate, FLAGS.L2, FLAGS.dropout)
print(trainingParameterText)
print('_________________________________________________________________')
print('Number of training samples: ', end='')
training_data_generator = DataGenerator(
data_shape,
num_class,
FLAGS.filePath,
FLAGS.filePattern + FLAGS.trainingSetExtension + '.*.' +
FLAGS.extension,
FLAGS.batchSize,
percent=FLAGS.trainingPercent,
queue_size=FLAGS.queueSize,
shuffle=True,
#class_weight=class_weight,
)
print('Number of validation samples: ', end='')
validation_data_generator = DataGenerator(
data_shape,
num_class,
FLAGS.filePath,
FLAGS.filePattern + FLAGS.validationSetExtension + '.*.' +
FLAGS.extension,
FLAGS.testBatchSize,
percent=FLAGS.validationPercent,
shuffle=True,
#class_weight=class_weight,
)
print('Number of test samples: ', end='')
test_data_generator = DataGenerator(
data_shape,
num_class,
FLAGS.filePath,
FLAGS.filePattern + FLAGS.testSetExtension + '.*.' + FLAGS.extension,
FLAGS.testBatchSize,
percent=FLAGS.testPercent,
shuffle=True,
)
print('_________________________________________________________________')
callback_list = []
callback_list.append(
RecordEpoch(
best_epoch_dict,
weigh_filepath=path.join(
FLAGS.modelPath,
FLAGS.sessionName + '.weights.{epoch:02d}.hdf5'),
csv_log_filepath=path.join(
FLAGS.modelPath,
FLAGS.sessionName + '.' + FLAGS.logExtension,
),
csv_log_header=trainingParameterText,
patience=FLAGS.earlyStoppingPatience,
sleep_after_epoch=sleep_after_epoch,
))
history = model.fit_generator(
training_data_generator,
training_data_generator.num_steps(),
epochs=FLAGS.maxEpoch,
initial_epoch=FLAGS.startEpoch,
verbose=1,
callbacks=callback_list,
#class_weight=class_weight,
validation_data=validation_data_generator,
validation_steps=validation_data_generator.num_steps(),
max_queue_size=FLAGS.queueSize,
)
training_data_generator.terminate()
print('Best epoch:', best_epoch_dict['epoch_index'])
print('Validation accuracy:', best_epoch_dict['val_acc'])
model.load_weights(
path.join(
FLAGS.modelPath, FLAGS.sessionName +
'.weights.{:02d}.hdf5'.format(best_epoch_dict['epoch_index'])))
print('Loaded weights of this epoch')
test(model)
from data.data_generator import DataGenerator
from evaluation.synthetic_experiment import SynthExperiment
dataGen = DataGenerator('./config/data.ini', seed=42)
#Experiment(dataGen, config='./config/class_bias_experiment.ini').replot_results(exclude_methods=['bac_mace'], recompute=True)
#Experiment(dataGen, config='./config/acc_bias_experiment.ini').replot_results(exclude_methods=['bac_mace'], recompute=True)
#Experiment(dataGen, config='./config/short_bias_experiment.ini').replot_results(exclude_methods=['bac_mace'], recompute=True)
# Experiment(dataGen, config='./config/group_ratio_experiment.ini').replot_results(exclude_methods=['bac_mace'], recompute=True)
SynthExperiment(dataGen,
config='./config/crowd_size_experiment.ini').replot_results()
# Experiment(dataGen, config='./config/doc_length_experiment.ini').replot_results()
