def __init__(self, record_path, n_cls, img_shape): self.path = record_path self._ncls = n_cls self.img_shape = img_shape self.desired_size = img_shape[0] self.encoder = AnchorBoxes(img_size=self.desired_size, ncls=self._ncls) self.augmentor = Augmentor()
def train(self): print("Training..") gpu_opts = tf.GPUOptions(per_process_gpu_memory_fraction=0.985) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_opts)) as sess: # Prepare for writing TensorBoard summaries if self.tb_log_freq and self.name: if not os.path.exists(self.summary_path) and self.tb_log_freq: os.makedirs(self.summary_path) self.summarywriter = tf.train.SummaryWriter(self.summary_path, sess.graph) if self.latest_checkpoint: self.checkpoint_saver.restore(sess, self.latest_checkpoint) else: tf.initialize_all_variables().run() # prepare summary operations and summary writer summaries = tf.merge_all_summaries() self.val_summaries = self.setup_validation_summaries() combined_time = 0.0 # total time for each print swap_amount = None augmentor = Augmentor() for t_feed_dict, extra in self.model.next_train_feed(): # NOTE is this slower than enumerate()? i = self.model.global_step.eval() if i in self.model.swap_schedule: swap_amount = self.model.swap_schedule[i] print(" setting swap amount to {:.4f}".format(swap_amount)) if swap_amount > 0.0: t_feed_dict[self.model.ts_go] = augmentor.run( t_feed_dict[self.model.ts_go], extra['t_len'], swap_amount, skip_left=1) if self.valid_freq and i % self.valid_freq == 0: self.validate(sess) fetches = { 'loss': self.model.loss, 'ys': self.model.ys, 'train_op': self.model.train_op, 'accuracy': self.model.accuracy, 'summaries': summaries } res, elapsed_it = self.perform_iteration(sess, fetches, t_feed_dict) combined_time += elapsed_it if self.visualize and i % self.visualize == 0: self.visualize_ys(res['ys'], t_feed_dict) if self.summarywriter and i % self.tb_log_freq == 0: self.summarywriter.add_summary(res['summaries'], i) if self.checkpoint_saver and i and i % self.save_freq == 0: self.checkpoint_saver.save(sess, self.checkpoint_file_path, self.model.global_step) if self.log_freq and i % self.log_freq == 0: out = "Iteration {:d}\tLoss {:f}\tAcc: {:f}\tElapsed: {:f} ({:f})" print(out.format(i, np.mean(res['loss']), res['accuracy'], combined_time, (combined_time/self.log_freq))) combined_time = 0.0 if i >= self.iterations: print('Reached max iteration: {:d}'.format(i)) break
raw_data_dir = 'generated/hard_sammple_minning' target_data_dir = 'generated_augment/hard_sammple_minning' # initalize augmentors dislocate = Dislocate(r_scale=(30, 100)) resize = Resize(dsize=(30, None)) affine = Affine(r_degree=(-20, 20), direction='x') # clear if os.path.exists(target_data_dir): shutil.rmtree(target_data_dir) os.mkdir(target_data_dir) # collect raw data files = os.listdir(raw_data_dir) aug = Augmentor() for idx, file in enumerate(files): # if idx > 1: # continue print("{}/{}".format(idx + 1, len(files))) if file.endswith('txt'): shutil.copy(os.path.join(raw_data_dir, file), os.path.join(target_data_dir, file)) continue raw_data_path = os.path.join(raw_data_dir, file) img = misc.imread(raw_data_path) # perform augmentors img = resize(img)
def main(model_type, model_name, logger): """ train model until the maximum number of steps reached :param model_type: model type :param model_name: model name :param logger: logger :return: None """ with tf.Graph().as_default() as g: with tf.Session() as sess: # Create a new model or reload existing checkpoint model = create_model(sess, model_type, model_name, logger) # Create a log writer object log_writer = tf.summary.FileWriter(model.model_dir, graph=sess.graph) valid_loss = [] train_loss = [] # initial saver for # 1. save every 3 epcohs saver = tf.train.Saver(max_to_keep=3) # 2. the best loss best_saver = tf.train.Saver(max_to_keep=1) # CSV files for train and test set root_path = "data/" train_csv = "train_data.csv" valid_csv = "valid_data.csv" train_path = os.path.join(root_path, train_csv) valid_path = os.path.join(root_path, valid_csv) # initial batchizer train_batchizer = Batchizer(train_path, config["batch_size"]) valid_batchizer = Batchizer(valid_path, config["batch_size"]) # init augmentor only once for both train and validation set ag = Augmentor('data/noisy_videos/', config) train_batches = train_batchizer.batches( ag, config["output_dim"], num_c=config["input_channel"], zero_mean=True) valid_batches = valid_batchizer.batches( ag, config["output_dim"], num_c=config["input_channel"], zero_mean=True) while model.global_step.eval() < config["total_steps"]: # get the learning rate from config file lr_idx = int(model.global_step.eval() / config["decay_step"]) lr_idx = min(lr_idx, len(config["learning_rate"]) - 1) lr = config["learning_rate"][lr_idx] # train phase with tqdm(total=config["validate_every"], unit="batch") as t: for x, y, _ in train_batches: if x is None: continue batch_loss, summary = model.train( sess, x, y, config["keep_prob"], lr) train_loss.append(batch_loss) t.set_description_str( "batch_loss:{0:2.8f}, ".format(batch_loss)) log_writer.add_summary(summary, model.global_step.eval()) t.update(1) if model.global_step.eval( ) % config["validate_every"] == 0: break # validation phase valid_counter = 0 pred_result = [] with tqdm(total=config["validate_for"], unit="batch") as t: for x, y, img in valid_batches: if x is None: continue batch_loss, _, pred = model.eval(sess, x, y) valid_loss.append(batch_loss) t.set_description_str( "batch_loss:{0:2.8f}".format(batch_loss)) valid_counter += 1 # select a random image from current batch and add it for visualization # do it with a little chance! to reduce the size of output if np.random.rand() > 0.95: r = np.random.randint(0, high=len(x)) pred_result.append([y[r], pred[r], img[r]]) t.update(1) if valid_counter == config["validate_for"]: break # print the results of validation dataset print_predictions(pred_result, logger) train_mean_loss = np.mean(train_loss) valid_mean_loss = np.mean(valid_loss) logger.log( 'Step:{0:6}: avg train loss:{1:2.8f}, avg validation loss:{2:2.8f}' .format(model.global_step.eval(), train_mean_loss, valid_mean_loss)) # save a checkpoint with the best loss value if valid_mean_loss < logger.best_loss: logger.save_best_loss(valid_mean_loss) best_path = os.path.join(model.model_dir, "best_loss/") check_dir(best_path) save_path = best_saver.save(sess, best_path, global_step=model.global_step) logger.log("model saved with best loss {0} at {1}".format( valid_mean_loss, save_path)) # save_every and validate_every should be dividable, otherwise this step will jump if model.global_step.eval() % config["save_every"] == 0: save_path = saver.save(sess, model.model_dir, global_step=model.global_step) logger.log("model saved at {}".format(save_path)) summary = tf.Summary() summary.value.add(tag="train_loss", simple_value=train_mean_loss) summary.value.add(tag="valid_loss", simple_value=valid_mean_loss) log_writer.add_summary(summary, model.global_step.eval()) train_loss = [] valid_loss = [] logger.log('Training is done.')
import nibabel as nib import numpy as np from augmentor import Augmentor from utils import to_channels if __name__ == "__main__": img1 = nib.load('test_img.nii.gz').get_fdata() seg1 = nib.load('test_img_seg.nii.gz').get_fdata() # use the same image multiple times to simulate a batch img1 = img1.reshape(img1.shape + (1, )) img = np.stack([img1, img1, img1, img1, img1], axis=0) # seg1 = seg1.reshape(seg1.shape + (1,)) seg1 = to_channels(seg1) seg = np.stack([seg1, seg1, seg1, seg1, seg1], axis=0) augmentor = Augmentor() # augmentor.add_elastic_deformation(8, [3, 4, 5, 6]) # augmentor.add_elastic_deformation(3, [3, 4, 5, 6]) # augmentor.add_affine_warp(30) # augmentor.add_uniaxial_swirl(3, 300) # augmentor.add_uniaxial_rotation(20) # augmentor.add_shifts([20, 20, 20]) # augmentor.add_bezier_lut([0.3, 0.6], [0.3, 0.6], 0.45, degree=2) # augmentor.add_bezier_lut([0.3, 0.6], [0.3, 0.6], 0.15, degree=2) # augmentor.add_uniaxial_rotation(20) # augmentor.add_sequence().add_uniaxial_rotation(20).add_shifts( # [20, 20, 20]).add_elastic_deformation(3, [3, 4, 5]).end_sequence() # augmentor.add_linear_gradient([3, 3, 3]) augmentor.summary()
def train(self): print("Training..") gpu_opts = tf.GPUOptions(per_process_gpu_memory_fraction=0.985) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_opts)) as sess: # Prepare for writing TensorBoard summaries if self.tb_log_freq and self.name: if not os.path.exists(self.summary_path) and self.tb_log_freq: os.makedirs(self.summary_path) self.summarywriter = tf.train.SummaryWriter( self.summary_path, sess.graph) if self.latest_checkpoint: self.checkpoint_saver.restore(sess, self.latest_checkpoint) else: tf.initialize_all_variables().run() # prepare summary operations and summary writer summaries = tf.merge_all_summaries() self.val_summaries = self.setup_validation_summaries() combined_time = 0.0 # total time for each print swap_amount = None augmentor = Augmentor() for t_feed_dict, extra in self.model.next_train_feed(): # NOTE is this slower than enumerate()? i = self.model.global_step.eval() if i in self.model.swap_schedule: swap_amount = self.model.swap_schedule[i] print( " setting swap amount to {:.4f}".format(swap_amount)) if swap_amount > 0.0: t_feed_dict[self.model.ts_go] = augmentor.run( t_feed_dict[self.model.ts_go], extra['t_len'], swap_amount, skip_left=1) if self.valid_freq and i % self.valid_freq == 0: self.validate(sess) fetches = { 'loss': self.model.loss, 'ys': self.model.ys, 'train_op': self.model.train_op, 'accuracy': self.model.accuracy, 'summaries': summaries } res, elapsed_it = self.perform_iteration( sess, fetches, t_feed_dict) combined_time += elapsed_it if self.visualize and i % self.visualize == 0: self.visualize_ys(res['ys'], t_feed_dict) if self.summarywriter and i % self.tb_log_freq == 0: self.summarywriter.add_summary(res['summaries'], i) if self.checkpoint_saver and i and i % self.save_freq == 0: self.checkpoint_saver.save(sess, self.checkpoint_file_path, self.model.global_step) if self.log_freq and i % self.log_freq == 0: out = "Iteration {:d}\tLoss {:f}\tAcc: {:f}\tElapsed: {:f} ({:f})" print( out.format(i, np.mean(res['loss']), res['accuracy'], combined_time, (combined_time / self.log_freq))) combined_time = 0.0 if i >= self.iterations: print('Reached max iteration: {:d}'.format(i)) break
from flask import Flask, request from pydispatch import dispatcher from augmentor import Augmentor from data_store import DataStore from raw_digest import RawDataDigester app = Flask(__name__) @app.route('/api/add_raw_data', methods=['POST']) def add_raw_data(): content = request.json dispatcher.send("raw_data", data=content) return '', 204 if __name__ == '__main__': r = RawDataDigester() a = Augmentor() #d = DataStore() app.run(host='0.0.0.0', debug=True)
def set_augmentor(self, aug): if isinstance(aug, Augmentor): self.augment = aug else: self.augment = Augmentor()
from augmentor import Augmentor expr_fname = './regex_table.csv' utter_fname = './texts.txt' u = Augmentor(expr_fname, utter_fname) vr = u.calc_variations(100, do_print=True)