def preprocess(opt):
print('Begin preprocessing')
train_dataset = data.DataSet(opt.train_data, display_freq=opt.display_freq)
train_dataset.max_dict = opt.dict_size
train_dataset.build_dict()
print('Save training data')
torch.save(train_dataset, opt.train_data + '.prep.train.pt')
val_dataset = data.DataSet(opt.val_data, display_freq=opt.display_freq)
val_dataset.change_dict(train_dataset.dictionary)
print('Save validation data')
torch.save(val_dataset, opt.val_data + '.prep.val.pt')
print('Preprocessing done')
def train_wrapper(model):
'''Data loader'''
train_set = data.DataSet(FLAGS.root_dir,
FLAGS.dataset,
'train',
FLAGS.batch_size,
FLAGS.n_label,
data_aug=False,
shuffle=True)
valid_set = data.DataSet(FLAGS.root_dir,
FLAGS.dataset,
'val',
FLAGS.batch_size,
FLAGS.n_label,
data_aug=False,
shuffle=False)
'''create a tf session for training and validation
TODO: to run your model, you may call model.train(), model.save(), model.valid()'''
best_accuracy = 0
acc_train = []
acc_valid = []
for step in range(1, FLAGS.max_iteration + 1):
if not train_set.has_next_batch():
train_set.init_epoch()
batch_x, batch_y = train_set.next_batch()
if len(batch_x) == FLAGS.batch_size:
loss, acc = model.train(batch_x, batch_y)
if step == 1 or step % 10 == 0:
print("Step " + str(step) + ", Minibatch Loss= " + \
"{:.4f}".format(loss) + ", Training Accuracy= " + "{:.3f}".format(acc))
if step % FLAGS.valid_iteration == 0:
acc_train.append(acc)
tot_acc = 0.0
tot_input = 0
while valid_set.has_next_batch():
valid_ims, valid_labels = valid_set.next_batch()
loss, acc = model.valid(valid_ims, valid_labels)
tot_acc += acc * len(valid_ims)
tot_input += len(valid_ims)
acc = tot_acc / tot_input
valid_set.init_epoch()
print("Current Accuracy= " + "{:.3f}".format(acc))
acc_valid.append(acc)
if acc > best_accuracy:
model.save(step)
best_accuracy = acc
print("Optimization Finished!")
def dump():
dataset = data.DataSet()
dataset.classes
experiment = '2'
dataset.dumpNumpyFiles(seq_len_limit=20, experiment=experiment)
return (experiment, dataset)
def test_save_load(dataset_type, repeat):
""" Test initialising a dataset subclass with a random number of input and
output dimensions and number of points in the training and test sets, and
check that it can be saved and loaded, and that the saved and loaded
dataset objects are equivalent """
set_random_seed_from_args("test_save_load", dataset_type, repeat)
dataset = get_dataset_and_name_from_type(dataset_type)
dataset_name = repr(dataset)
dataset.save(dataset_name, output_dir)
dataset_loaded = data.DataSet().load(dataset_name, output_dir)
assert np.all(dataset.train.x == dataset_loaded.train.x)
assert np.all(dataset.train.y == dataset_loaded.train.y)
assert np.all(dataset.train.n == dataset_loaded.train.n)
assert np.all(dataset.test.x == dataset_loaded.test.x)
assert np.all(dataset.test.y == dataset_loaded.test.y)
assert np.all(dataset.test.n == dataset_loaded.test.n)
def preprocess():
data_info = data.DataSet(FLAGS.positive_data_file,
FLAGS.negative_data_file)
x_text, y = data_info.x_text, data_info.y
# Build vocabulary
max_document_length = max([len(x.split(" ")) for x in x_text])
print('max_document_length', max_document_length)
# 根据所有已分词好的文本建立好一个词典，然后找出每个词在词典中对应的索引，不足长度或者不存在的词补0
vocab_processor = learn.preprocessing.VocabularyProcessor(
max_document_length)
x = np.array(list(vocab_processor.fit_transform(x_text)))
tf.logging.info("Shape of X :{}".format(str(x.shape)))
with open(os.path.join(FLAGS.model_dir, 'vocab_processor.pkl'), 'wb') as f:
pickle.dump(vocab_processor, f)
# Random shuffle data
np.random.seed(10)
shuffle_indices = np.random.permutation(np.arange(len(y)))
x_shuffled = x[shuffle_indices]
y_shuffled = y[shuffle_indices]
# Split train/test set
dev_sample_index = -1 * int(0.1 * float(len(y)))
x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[
dev_sample_index:]
y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[
dev_sample_index:]
# Init model config
model_config = ModelConfig(embedding_dim=FLAGS.embedding_dim,
filter_sizes=FLAGS.filter_sizes,
num_filters=FLAGS.num_filters,
dropout_rate=FLAGS.drop_rate,
l2_reg_lambda=FLAGS.l2_reg_lambda,
max_seq_length=max_document_length,
vocab_size=len(vocab_processor.vocabulary_),
label_size=2)
tf.logging.info("Vocabulary size: {:d}".format(
len(vocab_processor.vocabulary_)))
tf.logging.info("Train/dev split: {:d}/{:d}".format(
len(y_train), len(y_dev)))
tf.logging.info("*******Init Model CONFIG*************")
tf.logging.info(model_config.to_string())
return x_train, y_train, vocab_processor, x_dev, y_dev, model_config
def main(data_set_name, data_directory, log_directory, results_directory,
splitting_method, splitting_fraction,
latent_size, hidden_sizes, reconstruction_distribution,
number_of_reconstruction_classes,
number_of_warm_up_epochs,
number_of_epochs, batch_size, learning_rate,
reset_training):
# Data
data_set = data.DataSet(data_set_name, data_directory)
training_set, validation_set, test_set = data_set.split(
splitting_method, splitting_fraction)
feature_size = data_set.number_of_features
# Modeling
model = modeling.VariationalAutoEncoder(
feature_size, latent_size, hidden_sizes,
reconstruction_distribution,
number_of_reconstruction_classes,
)
log_directory = os.path.join(log_directory, data_set_name, model.name)
model.train(training_set, validation_set,
number_of_epochs, batch_size, learning_rate,
# number_of_warm_up_epochs,
log_directory, reset_training)
# Analysis
results_directory = os.path.join(results_directory, data_set_name,
model.name)
analysis.analyseModel(log_directory, results_directory)
reconstructed_test_set, latent_set, test_metrics = model.evaluate(test_set, batch_size, log_directory)
analysis.analyseResults(test_set, reconstructed_test_set, latent_set,
results_directory)
def test_wrapper(model):
'''Test your code.'''
test_set = data.DataSet(FLAGS.root_dir,
FLAGS.dataset,
'test',
FLAGS.batch_size,
FLAGS.n_label,
data_aug=False,
shuffle=False)
'''TODO: Your code here.'''
model.load()
tot_acc = 0.0
tot_input = 0
while test_set.has_next_batch():
test_ims, test_labels = test_set.next_batch()
_, acc = model.valid(test_ims, test_labels)
tot_acc += acc * len(test_ims)
tot_input += len(test_ims)
acc = tot_acc / tot_input
print("Test Accuracy= " + "{:.3f}".format(acc))
print("Test Finished!")
def train(self):
saver = tf.train.Saver()
with tf.Session() as sess:
BEST_LOSS = 999999.0
sess.run(tf.global_variables_initializer())
for e in range(self.EPOCH):
if len(self.INPUT) < self.BATCH_SIZE:
self.BATCH_SIZE = len(self.INPUT)
TRAIN_DATA = dataset.DataSet(self.INPUT, self.LABEL)
BATCH_TOTAL = math.ceil(len(self.INPUT) / self.BATCH_SIZE)
print('Total Batch = {}, Batch Size = {}'.format(
BATCH_TOTAL, self.BATCH_SIZE))
for b in range(BATCH_TOTAL):
feed = {
self.MODEL.inputs: self.INPUT,
self.MODEL.labels: self.LABEL,
self.MODEL.learning_rate: self.LEARNING_RATE,
self.MODEL.is_training: True
}
TRAIN_LOSS, _, TRAIN_ACC = sess.run([
self.MODEL.cost, self.MODEL.optimizer,
self.MODEL.accuracy
],
feed_dict=feed)
if BEST_LOSS > TRAIN_LOSS:
BEST_LOSS = TRAIN_LOSS
if b % (BATCH_TOTAL // 5) == 0:
print(
'Epoch [{0:2d}/{1:2d}] ---------------------------------- \n'
.format(e + 1, self.EPOCH) +
'Batch [{0:4d}/{1:4d}]: Loss = {2:4.2f}, best Loss = {3:4.2f}'
.format((b +
1), BATCH_TOTAL, TRAIN_LOSS, BEST_LOSS))
# VALIDATION
valid_acc_list = []
pred_class_all = []
pred_class_all = []
total_vali_loss = []
target_class_all = []
VALID_DATA = dataset.DataSet(self.VAL_INPUT, self.VAL_LABEL)
BATCH_VALID_TOTAL = math.ceil(
len(self.VAL_INPUT) / self.BATCH_SIZE)
for v in range(BATCH_VALID_TOTAL):
BATCH_VALID_IMAGES, BATCH_VALID_LABELS = VALID_DATA.next_batch(
self.BATCH_SIZE)
feed = {
self.MODEL.inputs: self.VAL_INPUT,
self.MODEL.labels: self.VAL_LABEL,
self.MODEL.is_training: False
}
VAL_LOSS, VAL_ACC, VALID_CLASS = sess.run([
self.MODEL.cost, self.MODEL.accuracy,
self.MODEL.predicted
],
feed_dict=feed)
valid_acc_list.append(VAL_ACC)
pred_class_all.extend(VALID_CLASS)
total_vali_loss.append(VAL_LOSS)
target_class_all.extend(BATCH_VALID_LABELS)
if v == BATCH_VALID_TOTAL - 1:
valid_acc_avg = np.mean(valid_acc_list)
print("Valid Accuracy Average: ", valid_acc_avg)
print('[Weight] Update.')
saver.save(sess, "checkpoint/porto_pilsa.ckpt")
help='是否一起训练embeddding')
parser.add_argument('--mode',
type=str,
default='test',
choices=['train', 'test', 'demo', 'export'],
help='train:训练模型；test：测试模型；demo：使用模型')
args = parser.parse_args()
config = config.model_config()
# for arg in vars(args):
# setattr(config,arg,getattr(args,arg))
if args.mode == 'train':
train_data = data.DataSet(config.word2id_path, config.embedding_path)
val_data = data.DataSet(config.word2id_path, config.embedding_path)
train_data.read_tag_file(config.train_data)
val_data.read_tag_file(config.val_data)
config.embedding_matrix = train_data.embeddding
model = model.BiLSTM_CRF(config)
model.build()
model.train(train_data, val_data)
if args.mode == 'test':
test_data = data.DataSet(config.word2id_path, config.embedding_path)
test_data.read_tag_file(config.test_data)
config.embedding_matrix = test_data.embeddding
model = model.BiLSTM_CRF(config)
model.build()
error_list = model.test(test_data)
def __init__(self):
self.data_set = data.DataSet()
def load_data(self):
t_data, te_data, t_label, te_label = dataset.read_CIFAR10_subset()
train_set = dataset.DataSet(t_data, t_label)
test_set = dataset.DataSet(te_data, te_label)
return train_set, test_set
label = int(image_dir)
for image_file in os.listdir(os.path.join(image_root_dir, image_dir)):
image_raw_data = tf.gfile.FastGFile(os.path.join(image_root_dir, image_dir, image_file), 'rb').read()
# 灰度图像可以收敛 奇怪
img_data = tf.image.rgb_to_grayscale(tf.image.decode_jpeg(image_raw_data)).eval() / 255
img_data = tf.image.decode_jpeg(image_raw_data).eval() / 255
image_array[i] = img_data
label_array[i, label] = 1
i += 1
print("i=", i)
image_array = tf.cast(image_array, tf.float32).eval()
label_array = tf.cast(label_array, tf.float32).eval()
return image_array, label_array
# 读取图片训练集
images, labels = read_images_and_labels("E:/singleCaptcha")
train = data.DataSet(images, labels)
# 测试随机洗牌是否准确
# for i in range(100000):
# train_image, train_label = train.next_batch(20)
# weight initialization
# tf.truncated_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None, name=None)
# 生产正态分布,均值为0 标准差为0.1
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
# 声明一个变量
return tf.Variable(initial)
def runGraph(gdef,
batch_size,
num_loops,
input_name,
outputs,
dtype=np.float32,
input_data=None):
#set up graph
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
tf.reset_default_graph()
g = tf.Graph()
outlist = []
with g.as_default():
#input
if not input_data:
input_data = np.random.uniform(size=(batch_size * 100, 3, 224,
224)).astype(dtype)
dataset = tf.data.Dataset.from_tensor_slices(input_data)
dataset = dataset.batch(batch_size)
iterator = dataset.make_initializable_iterator()
next_image = iterator.get_next()
elif isinstance(input_data, str):
#scan input path
filelist = sorted([
os.path.join(input_data, x) for x in os.listdir(input_data)
if x.endswith('.h5')
])
#instantiate reader:
h5ir = data.DataSet(filelist, dtype=dtype)
if dtype == np.float32:
tftype = tf.float32
elif dtype == np.float16:
tftype = tf.float16
else:
raise ValueError(
"Error, type {dt} not supported.".format(dt=dtype))
#create dataset
dataset = tf.data.Dataset.from_generator(
h5ir.next,
output_types=(tftype, tf.int32, tf.float32, tf.float32,
tf.int32),
output_shapes=((3, 224, 224), (1), (1), (1), (1)))
dataset = dataset.prefetch(batch_size)
dataset = dataset.batch(batch_size, drop_remainder=True)
iterator = dataset.make_initializable_iterator()
next_element = iterator.get_next()
next_image = next_element[0]
out = tf.import_graph_def(graph_def=gdef,
input_map={input_name: next_image},
return_elements=outputs)
out = out[0].outputs[0]
outlist.append(out)
outlist.append(next_element[1])
outlist.append(next_element[3])
outlist.append(next_element[4])
with tf.Session(graph=g,
config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
predictions = []
labels = []
weights = []
psr = []
#loop over epochs
for epoch in range(num_loops):
sess.run(iterator.initializer)
#loop over dataset
while True:
try:
vals = sess.run(outlist)
predictions.append(vals[0][:, 1])
labels.append(vals[1][:, 0])
weights.append(vals[2][:, 0])
psr.append(vals[3][:, 0])
except tf.errors.OutOfRangeError:
print("Epoch {} done.".format(epoch))
break
predictions = np.concatenate(predictions, axis=0)
labels = np.concatenate(labels, axis=0)
weights = np.concatenate(weights, axis=0)
psr = np.concatenate(psr, axis=0)
return predictions, labels, weights, psr
def train(rank, world_size, options, gpus):
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = gpus[rank]
if options.DDP:
os.environ['MASTER_ADDR'] = 'localhost'
os.environ['MASTER_PORT'] = '77777'
torch.distributed.init_process_group(backend="nccl",
rank=rank,
world_size=world_size)
is_first = rank == 0
else:
is_first = True
# initialize data
train_labaled_dataset = data.DataSet(source_file_name='labeled_train.h5')
train_labaled_sampler = torch.utils.data.RandomSampler(
train_labaled_dataset,
replacement=True,
num_samples=options.epoch_samples)
train_labaled_loader = data.FastDataLoader(dataset=train_labaled_dataset,
batch_size=options.batch_size,
sampler=train_labaled_sampler,
num_workers=2,
drop_last=True,
prefetch_factor=4)
train_dataset = train_labaled_dataset
train_loader = train_labaled_loader
test_dataset = data.DataSet(source_file_name='test_data.h5')
# initialize model
if options.fine_tune_last:
#options.max_lr = 1e-4 if options.max_lr >= 1e-3 else options.max_lr
datetime_regex = re.compile(r'[a-zA-Z_]_(\d{4}.*)$')
logs = [(foldername, datetime_regex.search(foldername).groups()[0])
for foldername in os.listdir(options.log_path)]
logs = [(foldername,
datetime.strptime(timestamp, '%Y_%b_%d_%p_%I_%M_%S'))
for foldername, timestamp in logs]
logs.sort(key=lambda item: item[1], reverse=True)
last_log = logs[0][0]
# load the pickle object
with open(f'{options.log_path}/{last_log}/model.pickle',
'rb') as filein:
model = pickle.load(filein)
print(f'loading check point {last_log}/check_point.pth')
# load last saved state dicts and setup loss function
epoch_start = model.load_check_point(
f'{options.log_path}/{last_log}/check_point.pth')
# change log_path
log_path = f'{options.log_path}/{last_log}'
elif options.model_pickle_path:
with open(options.model_pickle_path, 'rb') as filein:
model = pickle.load(filein)
epoch_start = 0
else:
model = candidate.single_task(options=options,
input_shape=(patch_size, patch_size))
epoch_start = 0
model.update_rules(options)
model.update_scheduler(options, epoch_start)
if is_first:
# initialize new folder for logs, only do this when folder not exist
if not options.fine_tune_last:
time = datetime.now().strftime('%Y_%b_%d_%p_%I_%M_%S')
log_path = f'{options.log_path}/{model.name}_{time}'
os.mkdir(log_path)
[
os.mkdir(f'{log_path}/{foldername}')
for foldername in ['models', 'test_images']
]
shutil.copytree(os.getcwd(), f'{log_path}/code_used')
with open(f'{log_path}/train_epoch_log.csv', 'w') as fileout:
fileout.write(','.join(
['epoch'] + [f'train_{name}'
for name in model.loss_names] +
[f'test_{name}' for name in model.loss_names]) + '\n')
# dump model and options
with open(f'{log_path}/model.pickle', 'wb') as fileout:
pickle.dump(model, fileout)
with open(f'{log_path}/options.pickle', 'wb') as fileout:
pickle.dump(options, fileout)
print(model)
print('-' * line_width)
print(
f'{len(train_dataset):_} images, {options.epoch_samples:_} samples per epoch, '
f'{len(train_dataset) // options.epoch_samples} epochs for full coverage'
)
print(f'{model.name} parameters: {model.compute_params()}')
print(f'foldername: {log_path}')
max_batch = len(train_loader)
current_lr = options.max_lr
epoch_start += 1
epoch_max = epoch_start + options.max_epoch
# start training
for epoch_current in range(epoch_start, epoch_max):
start_time = datetime.now()
model.set_to_train()
print('-' * line_width) if is_first else None
# do batches
for batch_index, input_data in enumerate(train_loader, 1):
# this should not be changed, modification of loss and input should be done in model methods
model.set_input(input_data)
current_lr = model.train(batch_index)
# dynamic alignment by length
line = f'[{epoch_current:>{len(str(epoch_max))}}/{epoch_max-1}]' + \
f'[{batch_index:>{len(str(max_batch))}}/{max_batch}] ' + \
f'current learning rate: {current_lr:5e}'
print(f'\r{options.schedule}{line}', end=' ',
flush=True) if is_first else None
# exit training, do test
if is_first:
model.set_to_test()
validation = []
for image_index, input_data in enumerate(test_dataset):
input_data = torch.unsqueeze(input_data, 0)
model.set_input(input_data)
# prep the data
prediction, target = model.test()
# flush image to disk
validation.append([test_dataset.name, *prediction, target])
validation = pd.DataFrame(columns=[
'name', *[f'prediction_{number}' for number in range(1, 10)],
'target'
],
data=validation)
validation.to_csv(
f'{log_path}/test_images/epoch_{epoch_current}.csv',
index=False)
# we're done with doing test, write to log file
train_loss = [
f'{value:.5f}' for value in model.gather_loss('train')
]
test_loss = [f'{value:.5f}' for value in model.gather_loss('test')]
with open(f'{log_path}/train_epoch_log.csv', 'a') as fileout:
fileout.write(','.join([str(epoch_current)] + train_loss +
test_loss) + '\n')
# save model state_dict and check_point
model.save_inference(
f'{log_path}/models/epoch_{epoch_current}.pth')
model.save_check_point(f'{log_path}/check_point.pth',
epoch=epoch_current)
# use df.to_string() for convenience
epoch_df = pd.DataFrame(columns=model.loss_names,
data=[train_loss, test_loss])
epoch_df.index = ['train', 'test']
time_used = datetime.now() - start_time
time_str = f'epoch time: {time_used.seconds // 60 :>02}:{time_used.seconds % 60 :>02}'
print(time_str, epoch_df.to_string(), sep='\n')
# exit validation phase
# exit training
if options.DDP:
torch.distributed.destroy_process_group()
return None
from datetime import datetime
import data
import model
from benchmark import timeit
# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
# Load config file
# ! Specify base_dir
with open('/workspaces/ps_project/config.yaml') as file:
config = yaml.safe_load(file)
# Load train, test and validation dataset
train_dataset = data.DataSet(batch_size=config['batch_size'],
mode='train',
path=config["base_dir"] + 'data/',
patch_size=config["patch_size"],
wavelengths=config["wavelengths"])
train_ds = train_dataset.load_data()
print("Loading test dataset...")
validation_dataset = data.DataSet(batch_size=config["batch_size"],
mode='validation',
path=config["base_dir"] + 'data/',
patch_size=config["patch_size"],
wavelengths=config["wavelengths"])
validation_ds = validation_dataset.load_data()
print("Loading validation dataset...")
test_dataset = data.DataSet(batch_size=config["batch_size"],
mode='test',
# Loading config gile
# ! Specify base_dir
with open('/workspaces/ps_project/config.yaml') as file:
config = yaml.safe_load(file)
# Create dataset from images in sample/
prepare = sample_prepare.SamplePrepare(dest_path=config["base_dir"] +
'sample/',
src_path=config["base_dir"] + 'sample/',
size=config["patch_size"])
prepare.prepare_sample()
# Create tf object for dataset
sample_dataset = data.DataSet(batch_size=config["batch_size"],
mode='sample',
path=config["base_dir"] + 'sample/',
patch_size=config["patch_size"],
wavelengths=config["wavelengths"])
sample_ds = sample_dataset.load_data()
print("\nDataset created and loaded")
# Model setup
checkpoint_dir_u = config["base_dir"] + 'models/unet/train/'
checkpoint_path_u = config["base_dir"] + 'models/unet/train/cp-{epoch:04d}.ckpt'
latest_checkpoint_u = tf.train.latest_checkpoint(checkpoint_dir_u)
checkpoint_dir_r = config["base_dir"] + 'models/resnet/train/'
checkpoint_path_r = config[
"base_dir"] + 'models/resnet/train/cp-{epoch:04d}.ckpt'
latest_checkpoint_r = tf.train.latest_checkpoint(checkpoint_dir_r)
}
# Build the model
print "Compiling model"
start = time.time()
NFE = nfe.NeuralFoldingEngine(num_gpus=ARGS.num_gpus,
use_profiles=ARGS.profiles)
elapsed = time.time() - start
print " compiles in " + str(elapsed) + " s\n"
# Load data
CATH_FILE = "data/cath_L200_class123.txt"
CATH_SETS_FILE = "data/cath_L200_class123_sets.json"
dataset = data.DataSet(cath_file=CATH_FILE,
cath_sets_file=CATH_SETS_FILE,
max_length=ARGS.length,
num_gpus=ARGS.num_gpus,
placeholder_sets=NFE.placeholder_sets)
# Load auxilliary sequence data
prefix = "data/"
if ARGS.enrich:
seq_file = prefix + "sequences_cath_L200.txt"
weights_file = prefix + "sequence_weights_cath_L200.txt"
dataset.load_alignments(seq_file, weights_file)
if ARGS.profiles:
profile_file = prefix + "profiles_cath_L200.json"
profile_dict = dataset.load_profiles(profile_file)
# Build experiment paths
base_folder = time.strftime("log/%y%b%d_%I%M%p/", time.localtime())
def run_training():
print "==================\n================\nLooking for Directory: {}".format(FLAGS.train)
data_set = data.DataSet(FLAGS.train)
if data_set.length() < FLAGS.batch_size:
FLAGS.batch_size = data_set.length()
if not FLAGS.val:
print "missing --val flag"
exit(1)
val_data_set = data.DataSet(FLAGS.val)
rgb = tf.placeholder("float", [FLAGS.batch_size, 224, 224, 3], name="rgb")
variables_to_restore = {}
step_var = tf.Variable(0, trainable=False)
variables_to_restore["step"] = step_var
inc_step = step_var.assign_add(1)
loss, rgb_dist, summary_image = training_model(rgb, variables_to_restore)
tf.scalar_summary("loss", loss)
tf.scalar_summary("learning_rate", FLAGS.learning_rate)
#avg_loss = define_avg("avg_loss", step_var, loss)
#avg_rgb_dist = define_avg("avg_rgb_dist", step_var, rgb_dist)
# Create a saver for writing training checkpoints.
print_variables_to_restore(variables_to_restore)
saver = tf.train.Saver(var_list=variables_to_restore, max_to_keep=3)
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter('log', flush_secs=5)
opt = tf.train.MomentumOptimizer(FLAGS.learning_rate, 0.0)
train_op = opt.minimize(loss)
# Important step for batch norm see batch_norm.py
with tf.control_dependencies([train_op]):
train_op2 = tf.group(*tf.get_collection("update_assignments"))
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
sess.run(tf.initialize_all_variables())
train_csv = TextData('train-loss.csv')
val_csv = TextData('val-loss.csv')
maybe_restore_checkpoint(sess, saver)
print "begin training"
print "learning rate", FLAGS.learning_rate
print "batch size", FLAGS.batch_size
if FLAGS.fixed:
feed_dict = { rgb: data_set.next_batch(FLAGS.batch_size) }
while True:
step = sess.run(inc_step)
start_time = time.time()
if not FLAGS.fixed:
feed_dict = { rgb: data_set.next_batch(FLAGS.batch_size) }
_, loss_value, rgb_dist_val = sess.run([train_op2, loss, rgb_dist], feed_dict=feed_dict)
duration = time.time() - start_time
# Print status to stdout.
print('Step %d: rgb_dist = %.4f / loss = %.4f (%.1f sec)' % (step, rgb_dist_val, loss_value, duration))
train_csv.write(step, loss_value, rgb_dist_val)
if step % 50 == 0:
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
# Save a checkpoint
saver.save(sess, './checkpoint', global_step=step)
# now test the validation
if step % 100 == 0:
print "running validation..."
count = 0
summary_image_index = 0
val_loss = []
val_rgb_dist = []
while count < 100:
feed_dict = { rgb: val_data_set.next_batch(FLAGS.batch_size) }
loss_value, rgb_dist_value, summary_image_value = sess.run([loss, rgb_dist, summary_image], feed_dict=feed_dict)
for i in range(0, FLAGS.batch_size):
img = summary_image_value[i]
summary_image_index+=1
if summary_image_index <= 4:
fn = "val-imgs/val-%06d-%d.jpg" % (step, summary_image_index)
imsave(fn, img)
val_loss.append(loss_value)
val_rgb_dist.append(rgb_dist_value)
count += FLAGS.batch_size
val_avg_loss = np.mean(val_loss)
val_avg_rgb_dist = np.mean(val_rgb_dist)
print('VALIDATION SET: count = %d / rgb_dist = %.4f / loss = %.4f' % (count, val_avg_rgb_dist, val_avg_loss))
val_csv.write(step, val_avg_loss, val_avg_rgb_dist)
parser.add_argument('--model',
type=str,
default='./model/penn-lm.best.pt',
help='Path to trained model')
parser.add_argument('--cuda', action='store_true', help='Use GPU')
parser.add_argument('--batch_size',
type=int,
default=64,
help='Evaluate batch size if use gpu')
opt = parser.parse_args()
# assert not os.path.exists(opt.model) or not os.path.exists(opt.eval_data)
model = torch.load(opt.model)
model.eval()
eval_dataset = data.DataSet(opt.eval_data)
eval_dataset.change_dict(model.dictionary)
if opt.cuda:
model.cuda()
eval_dataset.set_batch_size(opt.batch_size)
print('Start Evaluation ...')
loss = evaluate(model, eval_dataset, opt.cuda)
print('Evaluation Result')
print('Loss : %f, Perplexity : %f' % (loss, math.exp(loss)))
def main(input_file_or_name, data_directory = "data",
log_directory = "log", results_directory = "results",
temporary_log_directory = None,
map_features = False, feature_selection = [], example_filter = [],
preprocessing_methods = [], noisy_preprocessing_methods = [],
split_data_set = True,
splitting_method = "default", splitting_fraction = 0.9,
model_type = "VAE", latent_size = 50, hidden_sizes = [500],
number_of_importance_samples = [5],
number_of_monte_carlo_samples = [10],
inference_architecture = "MLP",
latent_distribution = "gaussian",
number_of_classes = None,
parameterise_latent_posterior = False,
generative_architecture = "MLP",
reconstruction_distribution = "poisson",
number_of_reconstruction_classes = 0,
prior_probabilities_method = "uniform",
number_of_warm_up_epochs = 0,
kl_weight = 1,
proportion_of_free_KL_nats = 0.0,
batch_normalisation = True,
dropout_keep_probabilities = [],
count_sum = True,
number_of_epochs = 200, plotting_interval_during_training = None,
batch_size = 100, learning_rate = 1e-4,
run_id = None, new_run = False,
prediction_method = None, prediction_training_set_name = "training",
prediction_decomposition_method = None,
prediction_decomposition_dimensionality = None,
decomposition_methods = ["PCA"], highlight_feature_indices = [],
reset_training = False, skip_modelling = False,
model_versions = ["all"],
analyse = True, evaluation_set_name = "test", analyse_data = False,
analyses = ["default"], analysis_level = "normal", fast_analysis = False,
export_options = []):
# Setup
model_versions = parseModelVersions(model_versions)
## Analyses
if fast_analysis:
analyse = True
analyses = ["simple"]
analysis_level = "limited"
## Distributions
reconstruction_distribution = parseDistribution(
reconstruction_distribution)
latent_distribution = parseDistribution(latent_distribution)
## Model configuration validation
if not skip_modelling:
if run_id:
run_id = checkRunID(run_id)
model_valid, model_errors = validateModelParameters(
model_type, latent_distribution,
reconstruction_distribution, number_of_reconstruction_classes,
parameterise_latent_posterior
)
if not model_valid:
print("Model configuration is invalid:")
for model_error in model_errors:
print(" ", model_error)
print()
if analyse_data:
print("Skipping modelling.")
print("")
skip_modelling = True
else:
print("Modelling cancelled.")
return
## Binarisation
binarise_values = False
if reconstruction_distribution == "bernoulli":
if noisy_preprocessing_methods:
if noisy_preprocessing_methods[-1] != "binarise":
noisy_preprocessing_methods.append("binarise")
print("Appended binarisation method to noisy preprocessing,",
"because of the Bernoulli distribution.\n")
else:
binarise_values = True
## Data sets
if not split_data_set or analyse_data or evaluation_set_name == "full" \
or prediction_training_set_name == "full":
full_data_set_needed = True
else:
full_data_set_needed = False
# Data
print(title("Data"))
data_set = data.DataSet(
input_file_or_name,
directory = data_directory,
map_features = map_features,
feature_selection = feature_selection,
example_filter = example_filter,
preprocessing_methods = preprocessing_methods,
binarise_values = binarise_values,
noisy_preprocessing_methods = noisy_preprocessing_methods
)
if full_data_set_needed:
data_set.load()
if split_data_set:
training_set, validation_set, test_set = data_set.split(
splitting_method, splitting_fraction)
all_data_sets = [data_set, training_set, validation_set, test_set]
else:
splitting_method = None
training_set = data_set
validation_set = None
test_set = data_set
all_data_sets = [data_set]
evaluation_set_name = "full"
prediction_training_set_name = "full"
## Setup of log and results directories
log_directory = data.directory(log_directory, data_set,
splitting_method, splitting_fraction)
data_results_directory = data.directory(results_directory, data_set,
splitting_method, splitting_fraction, preprocessing = False)
results_directory = data.directory(results_directory, data_set,
splitting_method, splitting_fraction)
if temporary_log_directory:
main_temporary_log_directory = temporary_log_directory
temporary_log_directory = data.directory(temporary_log_directory,
data_set, splitting_method, splitting_fraction)
## Data analysis
if analyse and analyse_data:
print(subtitle("Analysing data"))
analysis.analyseData(
data_sets = all_data_sets,
decomposition_methods = decomposition_methods,
highlight_feature_indices = highlight_feature_indices,
analyses = analyses,
analysis_level = analysis_level,
export_options = export_options,
results_directory = data_results_directory
)
print()
## Full data set clean up
if not full_data_set_needed:
data_set.clear()
# Modelling
if skip_modelling:
print("Modelling skipped.")
return
print(title("Modelling"))
# Set the number of features for the model
feature_size = training_set.number_of_features
# Parse numbers of samples
number_of_monte_carlo_samples = parseSampleLists(
number_of_monte_carlo_samples)
number_of_importance_samples = parseSampleLists(
number_of_importance_samples)
# Use analytical KL term for single-Gaussian-VAE
if "VAE" in model_type:
if latent_distribution == "gaussian":
analytical_kl_term = True
else:
analytical_kl_term = False
# Change latent distribution to Gaussian mixture if not already set
if model_type == "GMVAE" and latent_distribution != "gaussian mixture":
latent_distribution = "gaussian mixture"
print("The latent distribution was changed to",
"a Gaussian-mixture model, because of the model chosen.\n")
# Set the number of classes if not already set
if not number_of_classes:
if training_set.has_labels:
number_of_classes = training_set.number_of_classes \
- training_set.number_of_excluded_classes
elif "mixture" in latent_distribution:
raise ValueError(
"For a mixture model and a data set without labels, "
"the number of classes has to be set."
)
else:
number_of_classes = 1
print(subtitle("Model setup"))
if model_type == "VAE":
model = VariationalAutoencoder(
feature_size = feature_size,
latent_size = latent_size,
hidden_sizes = hidden_sizes,
number_of_monte_carlo_samples =number_of_monte_carlo_samples,
number_of_importance_samples = number_of_importance_samples,
analytical_kl_term = analytical_kl_term,
inference_architecture = inference_architecture,
latent_distribution = latent_distribution,
number_of_latent_clusters = number_of_classes,
parameterise_latent_posterior = parameterise_latent_posterior,
generative_architecture = generative_architecture,
reconstruction_distribution = reconstruction_distribution,
number_of_reconstruction_classes = number_of_reconstruction_classes,
batch_normalisation = batch_normalisation,
dropout_keep_probabilities = dropout_keep_probabilities,
count_sum = count_sum,
number_of_warm_up_epochs = number_of_warm_up_epochs,
kl_weight = kl_weight,
log_directory = log_directory,
results_directory = results_directory
)
elif model_type == "GMVAE":
if prior_probabilities_method == "uniform":
prior_probabilities = None
elif prior_probabilities_method == "infer":
prior_probabilities = training_set.class_probabilities
elif prior_probabilities_method == "literature":
prior_probabilities = training_set.literature_probabilities
else:
prior_probabilities = None
if not prior_probabilities:
prior_probabilities_method = "uniform"
prior_probabilities_values = None
else:
prior_probabilities_values = list(prior_probabilities.values())
prior_probabilities = {
"method": prior_probabilities_method,
"values": prior_probabilities_values
}
model = GaussianMixtureVariationalAutoencoder(
feature_size = feature_size,
latent_size = latent_size,
hidden_sizes = hidden_sizes,
number_of_monte_carlo_samples = number_of_monte_carlo_samples,
number_of_importance_samples = number_of_importance_samples,
analytical_kl_term = analytical_kl_term,
prior_probabilities = prior_probabilities,
number_of_latent_clusters = number_of_classes,
proportion_of_free_KL_nats = proportion_of_free_KL_nats,
reconstruction_distribution = reconstruction_distribution,
number_of_reconstruction_classes = number_of_reconstruction_classes,
batch_normalisation = batch_normalisation,
dropout_keep_probabilities = dropout_keep_probabilities,
count_sum = count_sum,
number_of_warm_up_epochs = number_of_warm_up_epochs,
kl_weight = kl_weight,
log_directory = log_directory,
results_directory = results_directory
)
else:
raise ValueError("Model type not found: `{}`.".format(model_type))
print(model.description)
print()
print(model.parameters)
print()
## Training
print(subtitle("Model training"))
status, run_id = model.train(
training_set,
validation_set,
number_of_epochs = number_of_epochs,
batch_size = batch_size,
learning_rate = learning_rate,
plotting_interval = plotting_interval_during_training,
run_id = run_id,
new_run = new_run,
reset_training = reset_training,
temporary_log_directory = temporary_log_directory
)
# Remove temporary directories created and emptied during training
if temporary_log_directory and os.path.exists(main_temporary_log_directory):
removeEmptyDirectories(main_temporary_log_directory)
if not status["completed"]:
print(status["message"])
return
status_filename = "status"
if "epochs trained" in status:
status_filename += "-" + status["epochs trained"]
status_path = os.path.join(
model.logDirectory(run_id = run_id),
status_filename + ".log"
)
with open(status_path, "w") as status_file:
for status_field, status_value in status.items():
if status_value:
status_file.write(
status_field + ": " + str(status_value) + "\n"
)
print()
# Evaluation, prediction, and analysis
## Setup
if analyse:
if prediction_method:
predict_labels_using_model = False
elif "GM" in model.type:
predict_labels_using_model = True
prediction_method = "model"
else:
predict_labels_using_model = False
else:
predict_labels_using_model = False
evaluation_title_parts = ["evaluation"]
if analyse:
if prediction_method:
evaluation_title_parts.append("prediction")
evaluation_title_parts.append("analysis")
evaluation_title = enumerateListOfStrings(evaluation_title_parts)
print(title(evaluation_title.capitalize()))
### Set selection
for data_subset in all_data_sets:
clear_subset = True
if data_subset.kind == evaluation_set_name:
evaluation_set = data_subset
clear_subset = False
if prediction_method \
and data_subset.kind == prediction_training_set_name:
prediction_training_set = data_subset
clear_subset = False
if clear_subset:
data_subset.clear()
### Evaluation set
evaluation_subset_indices = analysis.evaluationSubsetIndices(
evaluation_set)
print("Evaluation set: {} set.".format(evaluation_set.kind))
### Prediction method
if prediction_method:
prediction_method = properString(
prediction_method,
PREDICTION_METHOD_NAMES
)
prediction_method_specifications = PREDICTION_METHOD_SPECIFICATIONS\
.get(prediction_method, {})
prediction_method_inference = prediction_method_specifications.get(
"inference", None)
prediction_method_fixed_number_of_clusters \
= prediction_method_specifications.get(
"fixed number of clusters", None)
prediction_method_cluster_kind = prediction_method_specifications.get(
"cluster kind", None)
if prediction_method_fixed_number_of_clusters:
number_of_clusters = number_of_classes
else:
number_of_clusters = None
if prediction_method_inference \
and prediction_method_inference == "transductive":
prediction_training_set = None
prediction_training_set_name = None
else:
prediction_training_set_name = prediction_training_set.kind
prediction_details = {
"method": prediction_method,
"number_of_classes": number_of_clusters,
"training_set_name": prediction_training_set_name,
"decomposition_method": prediction_decomposition_method,
"decomposition_dimensionality":
prediction_decomposition_dimensionality
}
print("Prediction method: {}.".format(prediction_method))
if number_of_clusters:
print("Number of clusters: {}.".format(number_of_clusters))
if prediction_training_set:
print("Prediction training set: {} set.".format(
prediction_training_set.kind))
prediction_id_parts = []
if prediction_decomposition_method:
prediction_decomposition_method = properString(
prediction_decomposition_method,
DECOMPOSITION_METHOD_NAMES
)
if not prediction_decomposition_dimensionality:
prediction_decomposition_dimensionality \
= DEFAULT_DECOMPOSITION_DIMENSIONALITY
prediction_id_parts += [
prediction_decomposition_method,
prediction_decomposition_dimensionality
]
prediction_details.update({
"decomposition_method": prediction_decomposition_method,
"decomposition_dimensionality":
prediction_decomposition_dimensionality
})
print("Decomposition method before prediction: {}-d {}.".format(
prediction_decomposition_dimensionality,
prediction_decomposition_method
))
prediction_id_parts.append(prediction_method)
if number_of_clusters:
prediction_id_parts.append(number_of_clusters)
if prediction_training_set \
and prediction_training_set.kind != "training":
prediction_id_parts.append(prediction_training_set.kind)
prediction_id = "_".join(map(
lambda s: normaliseString(str(s)).replace("_", ""),
prediction_id_parts
))
prediction_details["id"] = prediction_id
else:
prediction_details = {}
### Model parameter sets
model_parameter_set_names = []
if "end_of_training" in model_versions:
model_parameter_set_names.append("end of training")
if "best_model" in model_versions \
and betterModelExists(model, run_id = run_id):
model_parameter_set_names.append("best model")
if "early_stopping" in model_versions \
and modelStoppedEarly(model, run_id = run_id):
model_parameter_set_names.append("early stopping")
print("Model parameter sets: {}.".format(enumerateListOfStrings(
model_parameter_set_names)))
print()
## Model analysis
if analyse:
print(subtitle("Model analysis"))
analysis.analyseModel(
model = model,
run_id = run_id,
analyses = analyses,
analysis_level = analysis_level,
export_options = export_options,
results_directory = results_directory
)
## Results evaluation, prediction, and analysis
for model_parameter_set_name in model_parameter_set_names:
if model_parameter_set_name == "best model":
use_best_model = True
else:
use_best_model = False
if model_parameter_set_name == "early stopping":
use_early_stopping_model = True
else:
use_early_stopping_model = False
model_parameter_set_name = model_parameter_set_name.capitalize()
print(subtitle(model_parameter_set_name))
# Evaluation
model_parameter_set_name = model_parameter_set_name.replace(" ", "-")
print(heading("{} evaluation".format(model_parameter_set_name)))
if "VAE" in model.type:
transformed_evaluation_set, reconstructed_evaluation_set,\
latent_evaluation_sets = model.evaluate(
evaluation_set = evaluation_set,
evaluation_subset_indices = evaluation_subset_indices,
batch_size = batch_size,
predict_labels = predict_labels_using_model,
run_id = run_id,
use_best_model = use_best_model,
use_early_stopping_model = use_early_stopping_model
)
else:
transformed_evaluation_set, reconstructed_evaluation_set = \
model.evaluate(
evaluation_set = evaluation_set,
evaluation_subset_indices = evaluation_subset_indices,
batch_size = batch_size,
run_id = run_id,
use_best_model = use_best_model,
use_early_stopping_model = use_early_stopping_model
)
latent_evaluation_sets = None
print()
# Prediction
if analyse and "VAE" in model.type and prediction_method \
and not transformed_evaluation_set.has_predictions:
print(heading("{} prediction".format(model_parameter_set_name)))
latent_prediction_evaluation_set = latent_evaluation_sets["z"]
if prediction_method_inference \
and prediction_method_inference == "inductive":
latent_prediction_training_sets = model.evaluate(
evaluation_set = prediction_training_set,
batch_size = batch_size,
run_id = run_id,
use_best_model = use_best_model,
use_early_stopping_model = use_early_stopping_model,
output_versions = "latent",
log_results = False
)
latent_prediction_training_set \
= latent_prediction_training_sets["z"]
print()
else:
latent_prediction_training_set = None
if prediction_decomposition_method:
if latent_prediction_training_set:
latent_prediction_training_set, \
latent_prediction_evaluation_set \
= data.decomposeDataSubsets(
latent_prediction_training_set,
latent_prediction_evaluation_set,
method = prediction_decomposition_method,
number_of_components =
prediction_decomposition_dimensionality,
random = True
)
else:
latent_prediction_evaluation_set \
= data.decomposeDataSubsets(
latent_prediction_evaluation_set,
method = prediction_decomposition_method,
number_of_components =
prediction_decomposition_dimensionality,
random = True
)
print()
cluster_ids, predicted_labels, predicted_superset_labels \
= predict(
latent_prediction_training_set,
latent_prediction_evaluation_set,
prediction_method,
number_of_clusters
)
transformed_evaluation_set.updatePredictions(
predicted_cluster_ids = cluster_ids,
predicted_labels = predicted_labels,
predicted_superset_labels = predicted_superset_labels
)
reconstructed_evaluation_set.updatePredictions(
predicted_cluster_ids = cluster_ids,
predicted_labels = predicted_labels,
predicted_superset_labels = predicted_superset_labels
)
for variable in latent_evaluation_sets:
latent_evaluation_sets[variable].updatePredictions(
predicted_cluster_ids = cluster_ids,
predicted_labels = predicted_labels,
predicted_superset_labels = predicted_superset_labels
)
print()
# Analysis
if analyse:
print(heading("{} results analysis".format(model_parameter_set_name)))
analysis.analyseResults(
evaluation_set = transformed_evaluation_set,
reconstructed_evaluation_set = reconstructed_evaluation_set,
latent_evaluation_sets = latent_evaluation_sets,
model = model,
run_id = run_id,
decomposition_methods = decomposition_methods,
evaluation_subset_indices = evaluation_subset_indices,
highlight_feature_indices = highlight_feature_indices,
prediction_details = prediction_details,
best_model = use_best_model,
early_stopping = use_early_stopping_model,
analyses = analyses, analysis_level = analysis_level,
export_options = export_options,
results_directory = results_directory
)
# Clean up
if transformed_evaluation_set.version == "original":
transformed_evaluation_set.resetPredictions()
def train(self):
start_time = time.time()
mode = [key for (key, value) in self.IS_MODEL.items() if value == True]
if len(mode) > 1 or len(mode) == 0:
print("You can't select more than one mode.")
return
x_train, y_train, x_valid, y_valid, input_size = 0, 0, 0, 0, (0, 0)
channel = 3
class_names = []
mode = mode[0]
network_model = self.NETWORK_MODEL[mode]
#_____________________________________________________________load dataset(Train, Validation) __________________________________________________________________________#
x_train, y_train, x_valid, y_valid, class_names = self._loadDataSetClassification(
)
# x_train => type : np.array(uint8) list, ex : [array([[[68,68,68]...,array(...]
# y_train => type : tuple, ex : ('4','2','0',...)
self.DATASET_CROP_COUNT = 1
class_num = len(class_names)
train_image_num = len(x_train)
#__________________________________________________________________________ create model __________________________________________________________________________#
model = build_model.build_model(
mode, network_model,
(self.DATASET_CROP_SIZE, self.DATASET_CROP_SIZE), self.BATCH_SIZE,
train_image_num, class_num, self.PATH_PRETRAIN,
self.HYPERP_LEARNING_RATE, self.HYPERP_DECAY_FACTOR, channel,
channel)
#__________________________________________________________________________ Training __________________________________________________________________________#
print('Starting at Epoch = {}, learning rate = {}'.format(
self.HYPERP_EPOCH, self.HYPERP_LEARNING_RATE))
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
deviceList = ['/gpu:0', '/gpu:1']
GPU_index = 0
with tf.Session(config=config) as sess:
with tf.device(deviceList[GPU_index]):
# Initialize all variables
#tf.global_variables_initializer().run()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
best_loss = 999999.0
totallCnt = 0
loss_list = []
# checkpoint & pretrain checkpoit load
if not os.path.exists(self.PATH_CHECKPOINT):
os.makedirs(self.PATH_CHECKPOINT)
if not os.path.exists(self.PATH_CHECKPOINT_SAVE):
os.makedirs(self.PATH_CHECKPOINT_SAVE)
if mode == 'CLASSIFICATION':
model.restore_fn(sess)
elif self.CHECKPOINT_CONTINUE == True:
model.restore_fn(sess, self.PATH_CHECKPOINT_SAVE)
# Data generate - Validation
input_valid_images = x_valid.copy()
label_valid_images = y_valid
input_valid_count = len(input_valid_images)
for e in range(self.HYPERP_EPOCH_START, self.HYPERP_EPOCH):
if self.TRAINING_STOP:
break
input_images = x_train.copy()
label_images = y_train
input_count = len(input_images)
label_count = 0
start = time.time()
# Data Augmentation
dataset.dataAugmentation_run_thread(
mode=mode,
dict_rotation=self.AUG_ROTATION,
dict_blur_gaussian=self.AUG_BLUR_GAUSSIAN,
dict_noise_gaussian=self.AUG_NOISE_GAUSSIAN,
input_images=input_images,
label_images=label_images)
print(
'Data generator : Input image count = {}, Label image count = {}'
.format(input_count, label_count))
if input_count < self.BATCH_SIZE:
self.BATCH_SIZE = input_count
# Dataset object 생성
train_data = dataset.DataSet(input_images, label_images)
# train_data => type : data.DataSet Object
if input_valid_count > 0:
valid_data = dataset.DataSet(input_valid_images,
label_valid_images)
batch_total = math.ceil(input_count / self.BATCH_SIZE)
print('Total Batch = {}, Batch size = {}'.format(
batch_total, self.BATCH_SIZE))
for b in range(batch_total):
if self.TRAINING_STOP:
print('Train Stop...!!!')
break
# Batch 추출
batch_images, batch_labels = train_data.next_batch(
self.BATCH_SIZE)
batch_labels_num = batch_labels.copy()
## labels to one hot encoding ##
batch_one_hot_list = []
for i in range(len(batch_labels)):
a = np.zeros(class_num)
a[batch_labels[i]] = 1
batch_one_hot_list.append(list(a))
batch_labels = batch_one_hot_list
## apply cutmix ##
if self.apply_cutmix == True:
#c_size = np.ceil(96*(e+1)/self.HYPERP_EPOCH) increasing window
c_size = 35 + e
batch_images, batch_labels = self.cutmix(
batch_images,
batch_labels,
class_num,
c_size,
rate=1)
# Train Step
loss, predicted_class, cur_learning_rate, acc = model.train_step(
sess, batch_images, batch_labels)
acc = acc[1]
if best_loss > loss:
best_loss = loss
if b % (batch_total // 5) == 0:
self._print_train_info(
e,
self.HYPERP_EPOCH,
b,
batch_total,
Loss=loss,
BestLoss=best_loss,
CurrentLearningRate=cur_learning_rate)
print('chart=0,{0:},{1:}'.format(totallCnt, loss))
loss_list.append(loss)
totallCnt += 1
#acc = model.train_step_accuracy(sess, batch_images, batch_labels)
#print('accuracy:{}'.format(acc))
if b == batch_total - 1:
# save model checkpoint
checkpoint_save_path = "%s/%04d-%04d" % (
self.PATH_CHECKPOINT_SAVE, e, b)
if (e + 1) % 10 == 0:
model.save_checkpoint(sess,
checkpoint_save_path,
mode)
model.save_checkpoint(
sess, self.PATH_CHECKPOINT_SAVE, mode)
print('Saved the checkpoint.(path:{})'.format(
checkpoint_save_path))
## input image save
#model.save_image(utils, checkpoint_save_path, batch_images, batch_labels_num, output_image, class_names, predicted_class, 'Train') #hcw, cutmix
#print('[Weight] Update.')
#loss_list.append(loss)
#self._SaveTrainInfoTextFile(checkpoint_save_path, e, loss, best_loss) @ hcw, cutmix
# validation # hcw, cutmix
valid_acc_list = []
batch_valid_total = math.ceil(input_valid_count /
self.BATCH_SIZE)
for v in range(batch_valid_total):
if self.TRAINING_STOP:
break
batch_valid_images, batch_valid_labels = valid_data.next_batch(
self.BATCH_SIZE)
###################################
batch_one_hot_list = []
for i in range(len(batch_valid_labels)):
a = np.zeros(class_num)
a[batch_valid_labels[i]] = 1
batch_one_hot_list.append(a)
batch_valid_labels = batch_one_hot_list
######################################
loss, predicted_valid_class, acc = model.validation_step(
sess, batch_valid_images, batch_valid_labels)
#print(acc[1])
valid_acc_list.append(acc[1])
if v == batch_valid_total - 1:
valid_acc_avg = np.mean(valid_acc_list)
#model.save_image(utils, checkpoint_save_path, batch_valid_images, batch_valid_labels, output_valid_image, class_names, predicted_valid_class, 'Validation') #hcw, cutmix
print('[Weight] Update.')
print("Validation Accuracy : ")
print(valid_acc_avg)
# txt file save
#self._SaveTrainInfoTextFile(self.PATH_CHECKPOINT_SAVE, e, 0.0, best_loss) # hcw, cutmix
#self._SaveLossTextFile(self.PATH_CHECKPOINT_SAVE, loss_list) # hcw, cutmix
end_time = time.time()
print('Train Time :', end_time - start_time)
time.sleep(1)
print('\n')
