def __init__(self, record_path, n_cls, img_shape):
self.path = record_path
self._ncls = n_cls
self.img_shape = img_shape
self.desired_size = img_shape[0]
self.encoder = AnchorBoxes(img_size=self.desired_size, ncls=self._ncls)
self.augmentor = Augmentor()
def train(self):
print("Training..")
gpu_opts = tf.GPUOptions(per_process_gpu_memory_fraction=0.985)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_opts)) as sess:
# Prepare for writing TensorBoard summaries
if self.tb_log_freq and self.name:
if not os.path.exists(self.summary_path) and self.tb_log_freq:
os.makedirs(self.summary_path)
self.summarywriter = tf.train.SummaryWriter(self.summary_path, sess.graph)
if self.latest_checkpoint:
self.checkpoint_saver.restore(sess, self.latest_checkpoint)
else:
tf.initialize_all_variables().run()
# prepare summary operations and summary writer
summaries = tf.merge_all_summaries()
self.val_summaries = self.setup_validation_summaries()
combined_time = 0.0 # total time for each print
swap_amount = None
augmentor = Augmentor()
for t_feed_dict, extra in self.model.next_train_feed():
# NOTE is this slower than enumerate()?
i = self.model.global_step.eval()
if i in self.model.swap_schedule:
swap_amount = self.model.swap_schedule[i]
print(" setting swap amount to {:.4f}".format(swap_amount))
if swap_amount > 0.0:
t_feed_dict[self.model.ts_go] = augmentor.run(
t_feed_dict[self.model.ts_go], extra['t_len'],
swap_amount, skip_left=1)
if self.valid_freq and i % self.valid_freq == 0:
self.validate(sess)
fetches = { 'loss': self.model.loss,
'ys': self.model.ys,
'train_op': self.model.train_op,
'accuracy': self.model.accuracy,
'summaries': summaries }
res, elapsed_it = self.perform_iteration(sess, fetches,
t_feed_dict)
combined_time += elapsed_it
if self.visualize and i % self.visualize == 0:
self.visualize_ys(res['ys'], t_feed_dict)
if self.summarywriter and i % self.tb_log_freq == 0:
self.summarywriter.add_summary(res['summaries'], i)
if self.checkpoint_saver and i and i % self.save_freq == 0:
self.checkpoint_saver.save(sess, self.checkpoint_file_path,
self.model.global_step)
if self.log_freq and i % self.log_freq == 0:
out = "Iteration {:d}\tLoss {:f}\tAcc: {:f}\tElapsed: {:f} ({:f})"
print(out.format(i, np.mean(res['loss']), res['accuracy'],
combined_time, (combined_time/self.log_freq)))
combined_time = 0.0
if i >= self.iterations:
print('Reached max iteration: {:d}'.format(i))
break
raw_data_dir = 'generated/hard_sammple_minning'
target_data_dir = 'generated_augment/hard_sammple_minning'
# initalize augmentors
dislocate = Dislocate(r_scale=(30, 100))
resize = Resize(dsize=(30, None))
affine = Affine(r_degree=(-20, 20), direction='x')
# clear
if os.path.exists(target_data_dir):
shutil.rmtree(target_data_dir)
os.mkdir(target_data_dir)
# collect raw data
files = os.listdir(raw_data_dir)
aug = Augmentor()
for idx, file in enumerate(files):
# if idx > 1:
# continue
print("{}/{}".format(idx + 1, len(files)))
if file.endswith('txt'):
shutil.copy(os.path.join(raw_data_dir, file),
os.path.join(target_data_dir, file))
continue
raw_data_path = os.path.join(raw_data_dir, file)
img = misc.imread(raw_data_path)
# perform augmentors
img = resize(img)
def main(model_type, model_name, logger):
"""
train model until the maximum number of steps reached
:param model_type: model type
:param model_name: model name
:param logger: logger
:return: None
"""
with tf.Graph().as_default() as g:
with tf.Session() as sess:
# Create a new model or reload existing checkpoint
model = create_model(sess, model_type, model_name, logger)
# Create a log writer object
log_writer = tf.summary.FileWriter(model.model_dir,
graph=sess.graph)
valid_loss = []
train_loss = []
# initial saver for
# 1. save every 3 epcohs
saver = tf.train.Saver(max_to_keep=3)
# 2. the best loss
best_saver = tf.train.Saver(max_to_keep=1)
# CSV files for train and test set
root_path = "data/"
train_csv = "train_data.csv"
valid_csv = "valid_data.csv"
train_path = os.path.join(root_path, train_csv)
valid_path = os.path.join(root_path, valid_csv)
# initial batchizer
train_batchizer = Batchizer(train_path, config["batch_size"])
valid_batchizer = Batchizer(valid_path, config["batch_size"])
# init augmentor only once for both train and validation set
ag = Augmentor('data/noisy_videos/', config)
train_batches = train_batchizer.batches(
ag,
config["output_dim"],
num_c=config["input_channel"],
zero_mean=True)
valid_batches = valid_batchizer.batches(
ag,
config["output_dim"],
num_c=config["input_channel"],
zero_mean=True)
while model.global_step.eval() < config["total_steps"]:
# get the learning rate from config file
lr_idx = int(model.global_step.eval() / config["decay_step"])
lr_idx = min(lr_idx, len(config["learning_rate"]) - 1)
lr = config["learning_rate"][lr_idx]
# train phase
with tqdm(total=config["validate_every"], unit="batch") as t:
for x, y, _ in train_batches:
if x is None:
continue
batch_loss, summary = model.train(
sess, x, y, config["keep_prob"], lr)
train_loss.append(batch_loss)
t.set_description_str(
"batch_loss:{0:2.8f}, ".format(batch_loss))
log_writer.add_summary(summary,
model.global_step.eval())
t.update(1)
if model.global_step.eval(
) % config["validate_every"] == 0:
break
# validation phase
valid_counter = 0
pred_result = []
with tqdm(total=config["validate_for"], unit="batch") as t:
for x, y, img in valid_batches:
if x is None:
continue
batch_loss, _, pred = model.eval(sess, x, y)
valid_loss.append(batch_loss)
t.set_description_str(
"batch_loss:{0:2.8f}".format(batch_loss))
valid_counter += 1
# select a random image from current batch and add it for visualization
# do it with a little chance! to reduce the size of output
if np.random.rand() > 0.95:
r = np.random.randint(0, high=len(x))
pred_result.append([y[r], pred[r], img[r]])
t.update(1)
if valid_counter == config["validate_for"]:
break
# print the results of validation dataset
print_predictions(pred_result, logger)
train_mean_loss = np.mean(train_loss)
valid_mean_loss = np.mean(valid_loss)
logger.log(
'Step:{0:6}: avg train loss:{1:2.8f}, avg validation loss:{2:2.8f}'
.format(model.global_step.eval(), train_mean_loss,
valid_mean_loss))
# save a checkpoint with the best loss value
if valid_mean_loss < logger.best_loss:
logger.save_best_loss(valid_mean_loss)
best_path = os.path.join(model.model_dir, "best_loss/")
check_dir(best_path)
save_path = best_saver.save(sess,
best_path,
global_step=model.global_step)
logger.log("model saved with best loss {0} at {1}".format(
valid_mean_loss, save_path))
# save_every and validate_every should be dividable, otherwise this step will jump
if model.global_step.eval() % config["save_every"] == 0:
save_path = saver.save(sess,
model.model_dir,
global_step=model.global_step)
logger.log("model saved at {}".format(save_path))
summary = tf.Summary()
summary.value.add(tag="train_loss",
simple_value=train_mean_loss)
summary.value.add(tag="valid_loss",
simple_value=valid_mean_loss)
log_writer.add_summary(summary, model.global_step.eval())
train_loss = []
valid_loss = []
logger.log('Training is done.')
import nibabel as nib
import numpy as np
from augmentor import Augmentor
from utils import to_channels
if __name__ == "__main__":
img1 = nib.load('test_img.nii.gz').get_fdata()
seg1 = nib.load('test_img_seg.nii.gz').get_fdata()
# use the same image multiple times to simulate a batch
img1 = img1.reshape(img1.shape + (1, ))
img = np.stack([img1, img1, img1, img1, img1], axis=0)
# seg1 = seg1.reshape(seg1.shape + (1,))
seg1 = to_channels(seg1)
seg = np.stack([seg1, seg1, seg1, seg1, seg1], axis=0)
augmentor = Augmentor()
# augmentor.add_elastic_deformation(8, [3, 4, 5, 6])
# augmentor.add_elastic_deformation(3, [3, 4, 5, 6])
# augmentor.add_affine_warp(30)
# augmentor.add_uniaxial_swirl(3, 300)
# augmentor.add_uniaxial_rotation(20)
# augmentor.add_shifts([20, 20, 20])
# augmentor.add_bezier_lut([0.3, 0.6], [0.3, 0.6], 0.45, degree=2)
# augmentor.add_bezier_lut([0.3, 0.6], [0.3, 0.6], 0.15, degree=2)
# augmentor.add_uniaxial_rotation(20)
# augmentor.add_sequence().add_uniaxial_rotation(20).add_shifts(
# [20, 20, 20]).add_elastic_deformation(3, [3, 4, 5]).end_sequence()
# augmentor.add_linear_gradient([3, 3, 3])
augmentor.summary()
def train(self):
print("Training..")
gpu_opts = tf.GPUOptions(per_process_gpu_memory_fraction=0.985)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_opts)) as sess:
# Prepare for writing TensorBoard summaries
if self.tb_log_freq and self.name:
if not os.path.exists(self.summary_path) and self.tb_log_freq:
os.makedirs(self.summary_path)
self.summarywriter = tf.train.SummaryWriter(
self.summary_path, sess.graph)
if self.latest_checkpoint:
self.checkpoint_saver.restore(sess, self.latest_checkpoint)
else:
tf.initialize_all_variables().run()
# prepare summary operations and summary writer
summaries = tf.merge_all_summaries()
self.val_summaries = self.setup_validation_summaries()
combined_time = 0.0 # total time for each print
swap_amount = None
augmentor = Augmentor()
for t_feed_dict, extra in self.model.next_train_feed():
# NOTE is this slower than enumerate()?
i = self.model.global_step.eval()
if i in self.model.swap_schedule:
swap_amount = self.model.swap_schedule[i]
print(
" setting swap amount to {:.4f}".format(swap_amount))
if swap_amount > 0.0:
t_feed_dict[self.model.ts_go] = augmentor.run(
t_feed_dict[self.model.ts_go],
extra['t_len'],
swap_amount,
skip_left=1)
if self.valid_freq and i % self.valid_freq == 0:
self.validate(sess)
fetches = {
'loss': self.model.loss,
'ys': self.model.ys,
'train_op': self.model.train_op,
'accuracy': self.model.accuracy,
'summaries': summaries
}
res, elapsed_it = self.perform_iteration(
sess, fetches, t_feed_dict)
combined_time += elapsed_it
if self.visualize and i % self.visualize == 0:
self.visualize_ys(res['ys'], t_feed_dict)
if self.summarywriter and i % self.tb_log_freq == 0:
self.summarywriter.add_summary(res['summaries'], i)
if self.checkpoint_saver and i and i % self.save_freq == 0:
self.checkpoint_saver.save(sess, self.checkpoint_file_path,
self.model.global_step)
if self.log_freq and i % self.log_freq == 0:
out = "Iteration {:d}\tLoss {:f}\tAcc: {:f}\tElapsed: {:f} ({:f})"
print(
out.format(i, np.mean(res['loss']), res['accuracy'],
combined_time,
(combined_time / self.log_freq)))
combined_time = 0.0
if i >= self.iterations:
print('Reached max iteration: {:d}'.format(i))
break
from flask import Flask, request
from pydispatch import dispatcher
from augmentor import Augmentor
from data_store import DataStore
from raw_digest import RawDataDigester
app = Flask(__name__)
@app.route('/api/add_raw_data', methods=['POST'])
def add_raw_data():
content = request.json
dispatcher.send("raw_data", data=content)
return '', 204
if __name__ == '__main__':
r = RawDataDigester()
a = Augmentor()
#d = DataStore()
app.run(host='0.0.0.0', debug=True)
def set_augmentor(self, aug):
if isinstance(aug, Augmentor):
self.augment = aug
else:
self.augment = Augmentor()
from augmentor import Augmentor expr_fname = './regex_table.csv' utter_fname = './texts.txt' u = Augmentor(expr_fname, utter_fname) vr = u.calc_variations(100, do_print=True)