def run_configuration(from_date, to_date, out_from_date, out_to_date): with tf.Graph().as_default(), tf.Session() as session: config = BaseConfig() std = 1.0 / np.sqrt(config.num_hidden) # std = 0.01 initializer = tf.random_normal_initializer(0.0, std, None) max_epoch = 200 limit = 500 data_reader_in = reader.DataReader(config.batch_size, from_date, to_date) data_reader_out = reader.DataReader(config.batch_size, out_from_date, out_to_date, limit) with tf.variable_scope("model", reuse=None, initializer=initializer): model = NNModel(True, config) tf.initialize_all_variables().run() best_out = 2.0 best_sum_out = 1e+6 in_rss = 2.0 best_iteration = 0 # cost, naive_cost, total_seq = run_epoch(session, data_reader_in, data_reader_in, model, config, tf.no_op()) # t_cost, t_naive_cost, t_total_seq = run_epoch(session, data_reader_out, data_reader_in, model, config, tf.no_op()) # print ("first is ", (cost/naive_cost), "first test", (t_cost/t_naive_cost)) for i in range(max_epoch): in_rss, sum_in, total_seq, sum_naive = run_epoch( session, data_reader_in, data_reader_in, model, config, model.train_op, data_reader_in.mean_out()) out_rss, sum_out, total_seq_out, sum_naive = run_epoch( session, data_reader_out, data_reader_in, model, config, tf.no_op(), data_reader_in.mean_out()) # if i % 10 == 0: # print ("in", (cost/naive_cost), "out", (t_cost/t_naive_cost), "i", i) if (sum_out < best_sum_out): best_out = out_rss best_sum_out = sum_out best_iteration = i # print ("test model ", cost, "ncost ", naive_cost, "relative", cost / naive_cost, "i", i) last_out, last_sum_out, total_seq_out, sum_naive = run_epoch( session, data_reader_out, data_reader_in, model, config, tf.no_op(), data_reader_in.mean_out()) print("best_iteration ", best_iteration, "total_sql", total_seq_out) return best_out, best_sum_out, last_out, last_sum_out, in_rss, total_seq_out, sum_naive
def docs_from_dataset(folder_path, file_name, column_types, used_columns, sent2features): dataset = reader.DataReader(folder_path, fileids=file_name, columntypes=column_types) y = [el[1] for el in dataset.get_ne()] x = dataset.get_tags(tags=used_columns) x_sent_base, y_sent = [], [] index = 0 for sent in dataset.sents(): length = len(sent) if length == 0: continue x_sent_base.append(x[index:index + length]) y_sent.append(y[index:index + length]) index += length x_sent = [sent2features(s) for s in x_sent_base] x_docs, y_docs = [], [] index = 0 for doc in dataset.docs(): length = len(doc) if length == 0: continue x_docs.append(x_sent[index:index + length]) y_docs.append(y_sent[index:index + length]) index += length return x_docs, y_docs
def test_context(train_progm, avg_cost, train_exe, dev_count, data_input_names, util_input_names, sum_cost, token_num): # Context to do validation. test_program = train_progm.clone() with fluid.program_guard(test_program): test_program = fluid.io.get_inference_program([avg_cost]) val_data = reader.DataReader( src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, fpattern=args.val_file_pattern, token_delimiter=args.token_delimiter, use_token_batch=args.use_token_batch, batch_size=args.batch_size * (1 if args.use_token_batch else dev_count), pool_size=args.pool_size, sort_type=args.sort_type, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], # count start and end tokens out max_length=ModelHyperParams.max_length - 2, clip_last_batch=False, shuffle=False, shuffle_batch=False) test_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, main_program=test_program, share_vars_from=train_exe) def test(exe=test_exe): test_total_cost = 0 test_total_token = 0 test_data = read_multiple( reader=val_data.batch_generator, count=dev_count if args.use_token_batch else 1) for batch_id, data in enumerate(test_data()): feed_list = [] for place_id, data_buffer in enumerate( split_data(data, num_part=dev_count)): data_input_dict, util_input_dict, _ = prepare_batch_input( data_buffer, data_input_names, util_input_names, ModelHyperParams.eos_idx, ModelHyperParams.eos_idx, ModelHyperParams.n_head, ModelHyperParams.d_model) feed_list.append( dict(data_input_dict.items() + util_input_dict.items())) outs = exe.run(feed=feed_list, fetch_list=[sum_cost.name, token_num.name]) sum_cost_val, token_num_val = np.array(outs[0]), np.array(outs[1]) test_total_cost += sum_cost_val.sum() test_total_token += token_num_val.sum() test_avg_cost = test_total_cost / test_total_token test_ppl = np.exp([min(test_avg_cost, 100)]) return test_avg_cost, test_ppl return test
def __init__(self, cfg, limit=1000): self.data_reader = reader.DataReader() self.limit = limit self.generator = datagen.ImageGenerator( self.data_reader, batch_size=1, input_size=cfg['input_size'], jitter=True, mode='val', # TODO add arguments )
def __init__(self, data_dir=None, is_training=True, learning_rate=0.0002, beta1=0.9, reconstr_weight=0.85, smooth_weight=0.05, ssim_weight=0.15, icp_weight=0.0, batch_size=4, img_height=128, img_width=416, seq_length=3, legacy_mode=False): self.data_dir = data_dir self.is_training = is_training self.learning_rate = learning_rate self.reconstr_weight = reconstr_weight self.smooth_weight = smooth_weight self.ssim_weight = ssim_weight self.icp_weight = icp_weight self.beta1 = beta1 self.batch_size = batch_size self.img_height = img_height self.img_width = img_width self.seq_length = seq_length self.legacy_mode = legacy_mode logging.info('data_dir: %s', data_dir) logging.info('learning_rate: %s', learning_rate) logging.info('beta1: %s', beta1) logging.info('smooth_weight: %s', smooth_weight) logging.info('ssim_weight: %s', ssim_weight) logging.info('icp_weight: %s', icp_weight) logging.info('batch_size: %s', batch_size) logging.info('img_height: %s', img_height) logging.info('img_width: %s', img_width) logging.info('seq_length: %s', seq_length) logging.info('legacy_mode: %s', legacy_mode) if self.is_training: self.reader = reader.DataReader(self.data_dir, self.batch_size, self.img_height, self.img_width, self.seq_length, NUM_SCALES) self.build_train_graph() else: self.build_depth_test_graph() self.build_egomotion_test_graph() # At this point, the model is ready. Print some info on model params. util.count_parameters()
def infer(args, inferencer=fast_infer): place = fluid.CUDAPlace(0) if InferTaskConfig.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) test_data = reader.DataReader(src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, fpattern=args.test_file_pattern, batch_size=args.batch_size, use_token_batch=False, pool_size=args.pool_size, sort_type=reader.SortType.NONE, shuffle=False, shuffle_batch=False, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], max_length=ModelHyperParams.max_length, clip_last_batch=False) trg_idx2word = test_data.load_dict(dict_path=args.trg_vocab_fpath, reverse=True) inferencer(test_data, trg_idx2word)
def __init__(self, data_dir=None, file_extension='png', is_training=True, learning_rate=0.0002, beta1=0.9, reconstr_weight=0.85, smooth_weight=0.05, object_depth_weight=0.0, object_depth_threshold=0.01, exclude_object_mask=True, stop_egomotion_gradient=True, ssim_weight=0.15, batch_size=4, img_height=128, img_width=416, seq_length=3, architecture=nets.RESNET, imagenet_norm=True, weight_reg=0.05, exhaustive_mode=False, random_scale_crop=False, flipping_mode=reader.FLIP_RANDOM, random_color=True, depth_upsampling=True, depth_normalization=True, compute_minimum_loss=True, use_skip=True, use_axis_angle=False, joint_encoder=True, build_sum=True, shuffle=True, input_file='train', handle_motion=False, equal_weighting=False, same_trans_rot_scaling=True, residual_deformer=True, seg_align_type='null', use_rigid_residual_flow=True, region_deformer_scaling=1.0): self.data_dir = data_dir self.file_extension = file_extension self.is_training = is_training self.learning_rate = learning_rate self.reconstr_weight = reconstr_weight self.smooth_weight = smooth_weight self.ssim_weight = ssim_weight self.object_depth_weight = object_depth_weight self.object_depth_threshold = object_depth_threshold self.exclude_object_mask = exclude_object_mask self.beta1 = beta1 self.batch_size = batch_size self.img_height = img_height self.img_width = img_width self.seq_length = seq_length self.architecture = architecture self.imagenet_norm = imagenet_norm self.weight_reg = weight_reg self.exhaustive_mode = exhaustive_mode self.random_scale_crop = random_scale_crop self.flipping_mode = flipping_mode self.random_color = random_color self.depth_upsampling = depth_upsampling self.depth_normalization = depth_normalization self.compute_minimum_loss = compute_minimum_loss self.use_skip = use_skip self.joint_encoder = joint_encoder self.build_sum = build_sum self.shuffle = shuffle self.input_file = input_file self.handle_motion = handle_motion self.equal_weighting = equal_weighting self.same_trans_rot_scaling = same_trans_rot_scaling self.residual_deformer = residual_deformer self.seg_align_type = seg_align_type self.use_rigid_residual_flow = use_rigid_residual_flow self.region_deformer_scaling = region_deformer_scaling self.stop_egomotion_gradient = stop_egomotion_gradient self.use_axis_angle = use_axis_angle self.trans_params_size = 32 # parameters of the bicubic function logging.info('data_dir: %s', data_dir) logging.info('file_extension: %s', file_extension) logging.info('is_training: %s', is_training) logging.info('learning_rate: %s', learning_rate) logging.info('reconstr_weight: %s', reconstr_weight) logging.info('smooth_weight: %s', smooth_weight) logging.info('ssim_weight: %s', ssim_weight) logging.info('beta1: %s', beta1) logging.info('batch_size: %s', batch_size) logging.info('img_height: %s', img_height) logging.info('img_width: %s', img_width) logging.info('seq_length: %s', seq_length) logging.info('architecture: %s', architecture) logging.info('imagenet_norm: %s', imagenet_norm) logging.info('weight_reg: %s', weight_reg) logging.info('exhaustive_mode: %s', exhaustive_mode) logging.info('random_scale_crop: %s', random_scale_crop) logging.info('flipping_mode: %s', flipping_mode) logging.info('random_color: %s', random_color) logging.info('depth_upsampling: %s', depth_upsampling) logging.info('depth_normalization: %s', depth_normalization) logging.info('compute_minimum_loss: %s', compute_minimum_loss) logging.info('use_skip: %s', use_skip) logging.info('joint_encoder: %s', joint_encoder) logging.info('build_sum: %s', build_sum) logging.info('shuffle: %s', shuffle) logging.info('input_file: %s', input_file) logging.info('handle_motion: %s', handle_motion) logging.info('equal_weighting: %s', equal_weighting) if self.is_training: self.reader = reader.DataReader(self.data_dir, self.batch_size, self.img_height, self.img_width, self.seq_length, NUM_SCALES, self.file_extension, self.random_scale_crop, self.flipping_mode, self.random_color, self.imagenet_norm, self.shuffle, self.input_file, self.seg_align_type) self.build_train_graph() else: self.build_depth_test_graph() self.build_egomotion_test_graph() # At this point, the model is ready. Print some info on model params. util.count_parameters()
def _build_loss(self): """Builds the loss tensor, to be minimized by the optimizer.""" self.reader = reader.DataReader( self.data_dir, self.batch_size, self.img_height, self.img_width, SEQ_LENGTH, 1, # num_scales self.file_extension, self.random_scale_crop, reader.FLIP_RANDOM, self.random_color, self.imagenet_norm, self.shuffle, self.input_file, queue_size=self.queue_size) (self.image_stack, self.image_stack_norm, self.seg_stack, self.intrinsic_mat, _) = self.reader.read_data() if self.learn_intrinsics: self.intrinsic_mat = None if self.intrinsic_mat is None and not self.learn_intrinsics: raise RuntimeError( 'Could not read intrinsic matrix. Turn ' 'learn_intrinsics on to learn it instead of loading ' 'it.') self.export('self.image_stack', self.image_stack) object_masks = [] for i in range(self.batch_size): object_ids = tf.unique(tf.reshape(self.seg_stack[i], [-1]))[0] object_masks_i = [] for j in range(SEQ_LENGTH): current_seg = self.seg_stack[i, :, :, j * 3] # (H, W) def process_obj_mask(obj_id): """Create a mask for obj_id, skipping the background mask.""" mask = tf.logical_and( tf.equal(current_seg, obj_id), # pylint: disable=cell-var-from-loop tf.not_equal(tf.cast(0, tf.uint8), obj_id)) # Leave out vert small masks, that are most often errors. size = tf.reduce_sum(tf.to_int32(mask)) mask = tf.logical_and(mask, tf.greater(size, MIN_OBJECT_AREA)) if not self.boxify: return mask # Complete the mask to its bounding box. binary_obj_masks_y = tf.reduce_any(mask, axis=1, keepdims=True) binary_obj_masks_x = tf.reduce_any(mask, axis=0, keepdims=True) return tf.logical_and(binary_obj_masks_y, binary_obj_masks_x) object_mask = tf.map_fn( # (N, H, W) process_obj_mask, object_ids, dtype=tf.bool) object_mask = tf.reduce_any(object_mask, axis=0) object_masks_i.append(object_mask) object_masks.append(tf.stack(object_masks_i, axis=-1)) self.seg_stack = tf.to_float(tf.stack(object_masks, axis=0)) tf.summary.image('Masks', self.seg_stack) with tf.variable_scope(DEPTH_SCOPE): # Organized by ...[i][scale]. Note that the order is flipped in # variables in build_loss() below. self.disp = {} self.depth = {} # Parabolic rampup of he noise over LAYER_NORM_NOISE_RAMPUP_STEPS steps. # We stop at 0.5 because this is the value above which the multiplicative # noise we use can become negative. Further experimentation is needed to # find if non-negativity is indeed needed. noise_stddev = 0.5 * tf.square( tf.minimum( tf.to_float(self.global_step) / float(LAYER_NORM_NOISE_RAMPUP_STEPS), 1.0)) def _normalizer_fn(x, is_train, name='bn'): return randomized_layer_normalization.normalize( x, is_train=is_train, name=name, stddev=noise_stddev) with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): for i in range(SEQ_LENGTH): image = self.image_stack_norm[:, :, :, 3 * i:3 * (i + 1)] self.depth[ i] = depth_prediction_net.depth_prediction_resnet18unet( image, True, self.weight_reg, _normalizer_fn) self.disp[i] = 1.0 / self.depth[i] with tf.name_scope('compute_loss'): self.reconstr_loss = 0 self.smooth_loss = 0 self.ssim_loss = 0 self.depth_consistency_loss = 0 # Smoothness. if self.smooth_weight > 0: for i in range(SEQ_LENGTH): disp_smoothing = self.disp[i] # Perform depth normalization, dividing by the mean. mean_disp = tf.reduce_mean(disp_smoothing, axis=[1, 2, 3], keep_dims=True) disp_input = disp_smoothing / mean_disp self.smooth_loss += _depth_smoothness( disp_input, self.image_stack[:, :, :, 3 * i:3 * (i + 1)]) self.rot_loss = 0.0 self.trans_loss = 0.0 def add_result_to_loss_and_summaries(endpoints, i, j): tf.summary.image( 'valid_mask%d%d' % (i, j), tf.expand_dims(endpoints['depth_proximity_weight'], -1)) self.depth_consistency_loss += endpoints['depth_error'] self.reconstr_loss += endpoints['rgb_error'] self.ssim_loss += 0.5 * endpoints['ssim_error'] self.rot_loss += endpoints['rotation_error'] self.trans_loss += endpoints['translation_error'] self.motion_smoothing = 0.0 with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): for i in range(SEQ_LENGTH - 1): j = i + 1 depth_i = self.depth[i][:, :, :, 0] depth_j = self.depth[j][:, :, :, 0] image_j = self.image_stack[:, :, :, 3 * j:3 * (j + 1)] image_i = self.image_stack[:, :, :, i * 3:(i + 1) * 3] # We select a pair of consecutive images (and their respective # predicted depth maps). Now we have the network predict a motion # field that connects the two. We feed the pair of images into the # network, once in forward order and then in reverse order. The # results are fed into the loss calculation. The following losses are # calculated: # - RGB and SSIM photometric consistency. # - Cycle consistency of rotations and translations for every pixel. # - L1 smoothness of the disparity and the motion field. # - Depth consistency rot, trans, trans_res, mat = motion_prediction_net.motion_field_net( images=tf.concat([image_i, image_j], axis=-1), weight_reg=self.weight_reg) inv_rot, inv_trans, inv_trans_res, inv_mat = ( motion_prediction_net.motion_field_net( images=tf.concat([image_j, image_i], axis=-1), weight_reg=self.weight_reg)) if self.learn_intrinsics: intrinsic_mat = 0.5 * (mat + inv_mat) else: intrinsic_mat = self.intrinsic_mat[:, 0, :, :] def dilate(x): # Dilation by n pixels is roughtly max pooling by 2 * n + 1. p = self.foreground_dilation * 2 + 1 return tf.nn.max_pool(x, [1, p, p, 1], [1] * 4, 'SAME') trans += trans_res * \ dilate(self.seg_stack[:, :, :, j:j + 1]) inv_trans += inv_trans_res * \ dilate(self.seg_stack[:, :, :, i:i + 1]) tf.summary.image('trans%d%d' % (i, i + 1), trans) tf.summary.image('trans%d%d' % (i + 1, i), inv_trans) tf.summary.image('trans_res%d%d' % (i + 1, i), inv_trans_res) tf.summary.image('trans_res%d%d' % (i, i + 1), trans_res) self.motion_smoothing += _smoothness(trans) self.motion_smoothing += _smoothness(inv_trans) tf.summary.scalar( 'trans_stdev', tf.sqrt(0.5 * tf.reduce_mean( tf.square(trans) + tf.square(inv_trans)))) transformed_depth_j = transform_depth_map.using_motion_vector( depth_j, trans, rot, intrinsic_mat) add_result_to_loss_and_summaries( consistency_losses.rgbd_and_motion_consistency_loss( transformed_depth_j, image_j, depth_i, image_i, rot, trans, inv_rot, inv_trans), i, j) transformed_depth_i = transform_depth_map.using_motion_vector( depth_i, inv_trans, inv_rot, intrinsic_mat) add_result_to_loss_and_summaries( consistency_losses.rgbd_and_motion_consistency_loss( transformed_depth_i, image_i, depth_j, image_j, inv_rot, inv_trans, rot, trans), j, i) # Build the total loss as composed of L1 reconstruction, SSIM, smoothing # and object size constraint loss as appropriate. self.reconstr_loss *= self.reconstr_weight self.export('self.reconstr_loss', self.reconstr_loss) self.total_loss = self.reconstr_loss if self.smooth_weight > 0: self.smooth_loss *= self.smooth_weight self.total_loss += self.smooth_loss self.export('self.smooth_loss', self.smooth_loss) if self.ssim_weight > 0: self.ssim_loss *= self.ssim_weight self.total_loss += self.ssim_loss self.export('self.ssim_loss', self.ssim_loss) if self.motion_smoothing_weight > 0: self.motion_smoothing *= self.motion_smoothing_weight self.total_loss += self.motion_smoothing self.export('self.motion_sm_loss', self.motion_smoothing) if self.depth_consistency_loss_weight: self.depth_consistency_loss *= self.depth_consistency_loss_weight self.total_loss += self.depth_consistency_loss self.export('self.depth_consistency_loss', self.depth_consistency_loss) self.rot_loss *= self.rotation_consistency_weight self.trans_loss *= self.translation_consistency_weight self.export('rot_loss', self.rot_loss) self.export('trans_loss', self.trans_loss) self.total_loss += self.rot_loss self.total_loss += self.trans_loss self.export('self.total_loss', self.total_loss)
def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, token_num, predict): # Initialize the parameters. if TrainTaskConfig.ckpt_path: fluid.io.load_persistables(exe, TrainTaskConfig.ckpt_path) lr_scheduler.current_steps = TrainTaskConfig.start_step else: logging.info("init fluid.framework.default_startup_program") exe.run(fluid.framework.default_startup_program()) logging.info("begin reader") train_data = reader.DataReader( src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, fpattern=args.train_file_pattern, token_delimiter=args.token_delimiter, use_token_batch=args.use_token_batch, batch_size=args.batch_size * (1 if args.use_token_batch else dev_count), pool_size=args.pool_size, sort_type=args.sort_type, shuffle=args.shuffle, shuffle_batch=args.shuffle_batch, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], # count start and end tokens out max_length=ModelHyperParams.max_length - 2, clip_last_batch=False) logging.info("begin read multiple") train_data = read_multiple(reader=train_data.batch_generator, count=dev_count if args.use_token_batch else 1) build_strategy = fluid.BuildStrategy() # Since the token number differs among devices, customize gradient scale to # use token average cost among multi-devices. and the gradient scale is # `1 / token_number` for average cost. build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized train_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, loss_name=sum_cost.name, main_program=train_progm, build_strategy=build_strategy) data_input_names = encoder_data_input_fields + decoder_data_input_fields[: -1] + label_data_input_fields util_input_names = encoder_util_input_fields + decoder_util_input_fields if args.val_file_pattern is not None: test = test_context(train_progm, avg_cost, train_exe, dev_count, data_input_names, util_input_names, sum_cost, token_num) # the best cross-entropy value with label smoothing loss_normalizer = -((1. - TrainTaskConfig.label_smooth_eps) * np.log( (1. - TrainTaskConfig.label_smooth_eps)) + TrainTaskConfig.label_smooth_eps * np.log(TrainTaskConfig.label_smooth_eps / (ModelHyperParams.trg_vocab_size - 1) + 1e-20)) logging.info("begin train:") init = False for pass_id in xrange(TrainTaskConfig.pass_num): pass_start_time = time.time() logging.info("pass_id:{0}".format(pass_id)) avg_batch_time = time.time() for batch_id, data in enumerate(train_data()): logging.info("batch_id:{0} data_len:{1}".format( batch_id, len(data))) feed_list = [] total_num_token = 0 if args.local: lr_rate = lr_scheduler.update_learning_rate() for place_id, data_buffer in enumerate( split_data(data, num_part=dev_count)): data_input_dict, util_input_dict, num_token = prepare_batch_input( data_buffer, data_input_names, util_input_names, ModelHyperParams.eos_idx, ModelHyperParams.eos_idx, ModelHyperParams.n_head, ModelHyperParams.d_model) total_num_token += num_token feed_kv_pairs = data_input_dict.items( ) + util_input_dict.items() if args.local: feed_kv_pairs += { lr_scheduler.learning_rate.name: lr_rate }.items() feed_list.append(dict(feed_kv_pairs)) if not init: for pos_enc_param_name in pos_enc_param_names: pos_enc = position_encoding_init( ModelHyperParams.max_length + 1, ModelHyperParams.d_model) feed_list[place_id][pos_enc_param_name] = pos_enc for feed_dict in feed_list: feed_dict[sum_cost.name + "@GRAD"] = 1. / total_num_token #outs = train_exe.run(fetch_list=[sum_cost.name, token_num.name], # feed=feed_list) outs = train_exe.run( fetch_list=[sum_cost.name, token_num.name] if batch_id % 100 == 0 else [], feed=feed_list) if batch_id % 100 == 0 and batch_id > 0: sum_cost_val, token_num_val = np.array(outs[0]), np.array( outs[1]) total_sum_cost = sum_cost_val.sum( ) # sum the cost from multi-devices total_token_num = token_num_val.sum() total_avg_cost = total_sum_cost / total_token_num logging.info( "epoch: %d, batch: %d, avg loss: %f, normalized loss: %f," " ppl: %f" % (pass_id, batch_id, total_avg_cost, total_avg_cost - loss_normalizer, np.exp([min(total_avg_cost, 100)]))) logging.info("speed: {0} batch/s".format( 100.0 / (time.time() - avg_batch_time))) """ if batch_id > 0 and batch_id % 1000 == 0: fluid.io.save_persistables( exe, os.path.join(TrainTaskConfig.ckpt_dir, "latest.checkpoint")) """ init = True if batch_id % 100 == 0 and batch_id > 0: avg_batch_time = time.time() time_consumed = time.time() - pass_start_time # Validate and save the model for inference. if args.val_file_pattern is not None: val_avg_cost, val_ppl = test() logging.info( "epoch: %d, val avg loss: %f, val normalized loss: %f, val ppl: %f," " consumed %fs" % (pass_id, val_avg_cost, val_avg_cost - loss_normalizer, val_ppl, time_consumed)) else: logging.info("epoch: %d, consumed %fs" % (pass_id, time_consumed)) fluid.io.save_persistables( exe, os.path.join(TrainTaskConfig.ckpt_dir, "pass_" + str(pass_id) + ".checkpoint")) fluid.io.save_inference_model( os.path.join(TrainTaskConfig.model_dir, "pass_" + str(pass_id) + ".infer.model"), data_input_names[:-2] + util_input_names, [predict], exe) if args.enable_ce: # For CE print("kpis\ttrain_cost_card%d\t%f" % (dev_count, total_avg_cost)) print("kpis\ttest_cost_card%d\t%f" % (dev_count, val_avg_cost)) print("kpis\ttrain_duration_card%d\t%f" % (dev_count, time_consumed))
def train_loop(exe, train_progm): def read_multiple(reader, count=dev_count if args.use_token_batch else 1, clip_last=True): """ Stack data from reader for multi-devices. """ def __impl__(): res = [] for item in reader(): res.append(item) if len(res) == count: yield res res = [] if len(res) == count: yield res elif not clip_last: data = [] for item in res: data += item if len(data) > count: inst_num_per_part = len(data) // count yield [ data[inst_num_per_part * i:inst_num_per_part * (i + 1)] for i in range(count) ] return __impl__ def split_data(data, num_part=dev_count): """ Split data for each device. """ if len(data) == num_part: return data data = data[0] inst_num_per_part = len(data) // num_part return [ data[inst_num_per_part * i:inst_num_per_part * (i + 1)] for i in range(num_part) ] # Initialize the parameters. if TrainTaskConfig.ckpt_path: fluid.io.load_persistables(exe, TrainTaskConfig.ckpt_path) lr_scheduler.current_steps = TrainTaskConfig.start_step else: print "init fluid.framework.default_startup_program" exe.run(fluid.framework.default_startup_program()) train_data = reader.DataReader( src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, fpattern=args.train_file_pattern, use_token_batch=args.use_token_batch, batch_size=args.batch_size * (1 if args.use_token_batch else dev_count), pool_size=args.pool_size, sort_type=args.sort_type, shuffle=args.shuffle, shuffle_batch=args.shuffle_batch, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], clip_last_batch=False) train_data = read_multiple(reader=train_data.batch_generator) build_strategy = fluid.BuildStrategy() # Since the token number differs among devices, customize gradient scale to # use token average cost among multi-devices. and the gradient scale is # `1 / token_number` for average cost. build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized #''' train_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, loss_name=sum_cost.name, main_program=train_progm, build_strategy=build_strategy) #''' def test_context(): # Context to do validation. test_program = train_progm.clone() with fluid.program_guard(test_program): test_program = fluid.io.get_inference_program([avg_cost]) val_data = reader.DataReader( src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, fpattern=args.val_file_pattern, use_token_batch=args.use_token_batch, batch_size=args.batch_size * (1 if args.use_token_batch else dev_count), pool_size=args.pool_size, sort_type=args.sort_type, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], clip_last_batch=False, shuffle=False, shuffle_batch=False) test_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, main_program=test_program, share_vars_from=train_exe) def test(exe=test_exe): test_total_cost = 0 test_total_token = 0 test_data = read_multiple(reader=val_data.batch_generator) for batch_id, data in enumerate(test_data()): feed_list = [] for place_id, data_buffer in enumerate(split_data(data)): data_input_dict, util_input_dict, _ = prepare_batch_input( data_buffer, data_input_names, util_input_names, ModelHyperParams.eos_idx, ModelHyperParams.eos_idx, ModelHyperParams.n_head, ModelHyperParams.d_model) feed_list.append( dict(data_input_dict.items() + util_input_dict.items())) outs = exe.run(feed=feed_list, fetch_list=[sum_cost.name, token_num.name]) sum_cost_val, token_num_val = np.array(outs[0]), np.array( outs[1]) test_total_cost += sum_cost_val.sum() test_total_token += token_num_val.sum() test_avg_cost = test_total_cost / test_total_token test_ppl = np.exp([min(test_avg_cost, 100)]) return test_avg_cost, test_ppl return test if args.val_file_pattern is not None: test = test_context() data_input_names = encoder_data_input_fields + decoder_data_input_fields[: -1] + label_data_input_fields util_input_names = encoder_util_input_fields + decoder_util_input_fields init = False for pass_id in xrange(TrainTaskConfig.pass_num): pass_start_time = time.time() for batch_id, data in enumerate(train_data()): feed_list = [] total_num_token = 0 #lr_rate = lr_scheduler.update_learning_rate() for place_id, data_buffer in enumerate(split_data(data)): data_input_dict, util_input_dict, num_token = prepare_batch_input( data_buffer, data_input_names, util_input_names, ModelHyperParams.eos_idx, ModelHyperParams.eos_idx, ModelHyperParams.n_head, ModelHyperParams.d_model) total_num_token += num_token feed_list.append( dict(data_input_dict.items() + util_input_dict.items())) if not init: for pos_enc_param_name in pos_enc_param_names: pos_enc = position_encoding_init( ModelHyperParams.max_length + 1, ModelHyperParams.d_model) feed_list[place_id][pos_enc_param_name] = pos_enc for feed_dict in feed_list: feed_dict[ sum_cost.name + "@GRAD"] = 1. / total_num_token if TrainTaskConfig.use_avg_cost else np.asarray( [1.], dtype="float32") outs = train_exe.run( fetch_list=[sum_cost.name, token_num.name], feed=feed_list) #outs = exe.run(train_progm,fetch_list=[sum_cost.name, token_num.name],feed=feed_list[0]) sum_cost_val, token_num_val = np.array(outs[0]), np.array( outs[1]) total_sum_cost = sum_cost_val.sum( ) # sum the cost from multi-devices total_token_num = token_num_val.sum() total_avg_cost = total_sum_cost / total_token_num print( "epoch: %d, batch: %d, sum loss: %f, avg loss: %f, ppl: %f" % (pass_id, batch_id, total_sum_cost, total_avg_cost, np.exp([min(total_avg_cost, 100)]))) init = True # Validate and save the model for inference. print("epoch: %d, " % pass_id + ("val avg loss: %f, val ppl: %f, " % test() if args.val_file_pattern is not None else "") + "consumed %fs" % (time.time() - pass_start_time)) fluid.io.save_persistables( exe, os.path.join(TrainTaskConfig.ckpt_dir, "pass_" + str(pass_id) + ".checkpoint")) fluid.io.save_inference_model( os.path.join(TrainTaskConfig.model_dir, "pass_" + str(pass_id) + ".infer.model"), data_input_names[:-2] + util_input_names, [predict], exe)
def profile(args): print args if args.device == 'CPU': TrainTaskConfig.use_gpu = False if not TrainTaskConfig.use_gpu: place = fluid.CPUPlace() dev_count = multiprocessing.cpu_count() else: place = fluid.CUDAPlace(0) dev_count = fluid.core.get_cuda_device_count() exe = fluid.Executor(place) sum_cost, avg_cost, predict, token_num = transformer( ModelHyperParams.src_vocab_size, ModelHyperParams.trg_vocab_size, ModelHyperParams.max_length + 1, ModelHyperParams.n_layer, ModelHyperParams.n_head, ModelHyperParams.d_key, ModelHyperParams.d_value, ModelHyperParams.d_model, ModelHyperParams.d_inner_hid, ModelHyperParams.dropout, ModelHyperParams.weight_sharing, TrainTaskConfig.label_smooth_eps) lr_scheduler = LearningRateScheduler(ModelHyperParams.d_model, TrainTaskConfig.warmup_steps, TrainTaskConfig.learning_rate) optimizer = fluid.optimizer.Adam(learning_rate=lr_scheduler.learning_rate, beta1=TrainTaskConfig.beta1, beta2=TrainTaskConfig.beta2, epsilon=TrainTaskConfig.eps) optimizer.minimize(sum_cost) # Initialize the parameters. if TrainTaskConfig.ckpt_path: fluid.io.load_persistables(exe, TrainTaskConfig.ckpt_path) lr_scheduler.current_steps = TrainTaskConfig.start_step else: exe.run(fluid.framework.default_startup_program()) # Disable all sorts for they will be done in the 1st batch. train_data = reader.DataReader( src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, fpattern=args.train_file_pattern, use_token_batch=args.use_token_batch, batch_size=args.batch_size * (1 if args.use_token_batch else dev_count), pool_size=args.pool_size, sort_type='none', shuffle=False, shuffle_batch=False, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], # count start and end tokens out max_length=ModelHyperParams.max_length - 2, clip_last_batch=False) train_data = read_multiple(reader=train_data.batch_generator, count=dev_count if args.use_token_batch else 1) if dev_count > 1: build_strategy = fluid.BuildStrategy() build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized train_exe = fluid.ParallelExecutor( use_cuda=TrainTaskConfig.use_gpu, loss_name=sum_cost.name, main_program=fluid.default_main_program(), build_strategy=build_strategy) print("Warming up ...") train_loop(exe if dev_count == 1 else train_exe, fluid.default_main_program(), False, 3, train_data, dev_count, sum_cost, avg_cost, lr_scheduler, token_num, predict) print("\nProfiling ...") if dev_count == 1: with profiler.profiler('All', 'total', '/tmp/profile_file'): total_time, exec_time = train_loop(exe, fluid.default_main_program(), True, args.num_iters, train_data, dev_count, sum_cost, avg_cost, lr_scheduler, token_num, predict) else: total_time, exec_time = train_loop(train_exe, fluid.default_main_program(), True, args.num_iters, train_data, dev_count, sum_cost, avg_cost, lr_scheduler, token_num, predict) print("Elapsed time: total %f s, in executor %f s" % (total_time, exec_time))
def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, token_num, predict): # Initialize the parameters. if TrainTaskConfig.ckpt_path: fluid.io.load_persistables(exe, TrainTaskConfig.ckpt_path) lr_scheduler.current_steps = TrainTaskConfig.start_step else: print "init fluid.framework.default_startup_program" exe.run(fluid.framework.default_startup_program()) train_data = reader.DataReader( src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, fpattern=args.train_file_pattern, use_token_batch=args.use_token_batch, batch_size=args.batch_size * (1 if args.use_token_batch else dev_count), pool_size=args.pool_size, sort_type=args.sort_type, shuffle=args.shuffle, shuffle_batch=args.shuffle_batch, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], # count start and end tokens out max_length=ModelHyperParams.max_length - 2, clip_last_batch=False) train_data = read_multiple(reader=train_data.batch_generator, count=dev_count if args.use_token_batch else 1) build_strategy = fluid.BuildStrategy() # Since the token number differs among devices, customize gradient scale to # use token average cost among multi-devices. and the gradient scale is # `1 / token_number` for average cost. build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized train_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, loss_name=sum_cost.name, main_program=train_progm, build_strategy=build_strategy) data_input_names = encoder_data_input_fields + decoder_data_input_fields[: -1] + label_data_input_fields util_input_names = encoder_util_input_fields + decoder_util_input_fields if args.val_file_pattern is not None: test = test_context(train_progm, avg_cost, train_exe, dev_count, data_input_names, util_input_names, sum_cost, token_num) init = False for pass_id in xrange(TrainTaskConfig.pass_num): pass_start_time = time.time() for batch_id, data in enumerate(train_data()): feed_list = [] total_num_token = 0 for place_id, data_buffer in enumerate( split_data(data, num_part=dev_count)): data_input_dict, util_input_dict, num_token = prepare_batch_input( data_buffer, data_input_names, util_input_names, ModelHyperParams.eos_idx, ModelHyperParams.eos_idx, ModelHyperParams.n_head, ModelHyperParams.d_model) total_num_token += num_token feed_kv_pairs = data_input_dict.items( ) + util_input_dict.items() if args.local: lr_rate = lr_scheduler.update_learning_rate() feed_kv_pairs += { lr_scheduler.learning_rate.name: lr_rate }.items() feed_list.append(dict(feed_kv_pairs)) if not init: for pos_enc_param_name in pos_enc_param_names: pos_enc = position_encoding_init( ModelHyperParams.max_length + 1, ModelHyperParams.d_model) feed_list[place_id][pos_enc_param_name] = pos_enc for feed_dict in feed_list: feed_dict[sum_cost.name + "@GRAD"] = 1. / total_num_token outs = train_exe.run(fetch_list=[sum_cost.name, token_num.name], feed=feed_list) sum_cost_val, token_num_val = np.array(outs[0]), np.array(outs[1]) total_sum_cost = sum_cost_val.sum( ) # sum the cost from multi-devices total_token_num = token_num_val.sum() total_avg_cost = total_sum_cost / total_token_num print("epoch: %d, batch: %d, sum loss: %f, avg loss: %f, ppl: %f" % (pass_id, batch_id, total_sum_cost, total_avg_cost, np.exp([min(total_avg_cost, 100)]))) init = True # Validate and save the model for inference. print("epoch: %d, " % pass_id + ("val avg loss: %f, val ppl: %f, " % test() if args.val_file_pattern is not None else "") + "consumed %fs" % (time.time() - pass_start_time)) fluid.io.save_persistables( exe, os.path.join(TrainTaskConfig.ckpt_dir, "pass_" + str(pass_id) + ".checkpoint")) fluid.io.save_inference_model( os.path.join(TrainTaskConfig.model_dir, "pass_" + str(pass_id) + ".infer.model"), data_input_names[:-2] + util_input_names, [predict], exe)
def infer(args): place = fluid.CUDAPlace(0) if InferTaskConfig.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) encoder_program = fluid.Program() with fluid.program_guard(main_program=encoder_program): enc_output = encoder(ModelHyperParams.src_vocab_size, ModelHyperParams.max_length + 1, ModelHyperParams.n_layer, ModelHyperParams.n_head, ModelHyperParams.d_key, ModelHyperParams.d_value, ModelHyperParams.d_model, ModelHyperParams.d_inner_hid, ModelHyperParams.dropout) decoder_program = fluid.Program() with fluid.program_guard(main_program=decoder_program): predict = decoder(ModelHyperParams.trg_vocab_size, ModelHyperParams.max_length + 1, ModelHyperParams.n_layer, ModelHyperParams.n_head, ModelHyperParams.d_key, ModelHyperParams.d_value, ModelHyperParams.d_model, ModelHyperParams.d_inner_hid, ModelHyperParams.dropout) # Load model parameters of encoder and decoder separately from the saved # transformer model. encoder_var_names = [] for op in encoder_program.block(0).ops: encoder_var_names += op.input_arg_names encoder_param_names = filter( lambda var_name: isinstance( encoder_program.block(0).var(var_name), fluid.framework.Parameter), encoder_var_names) encoder_params = map(encoder_program.block(0).var, encoder_param_names) decoder_var_names = [] for op in decoder_program.block(0).ops: decoder_var_names += op.input_arg_names decoder_param_names = filter( lambda var_name: isinstance( decoder_program.block(0).var(var_name), fluid.framework.Parameter), decoder_var_names) decoder_params = map(decoder_program.block(0).var, decoder_param_names) fluid.io.load_vars(exe, InferTaskConfig.model_path, vars=encoder_params) fluid.io.load_vars(exe, InferTaskConfig.model_path, vars=decoder_params) # This is used here to set dropout to the test mode. encoder_program = fluid.io.get_inference_program( target_vars=[enc_output], main_program=encoder_program) decoder_program = fluid.io.get_inference_program( target_vars=[predict], main_program=decoder_program) test_data = reader.DataReader(src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, fpattern=args.test_file_pattern, batch_size=args.batch_size, use_token_batch=False, pool_size=args.pool_size, sort_type=reader.SortType.NONE, shuffle=False, shuffle_batch=False, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], clip_last_batch=False) trg_idx2word = test_data.load_dict(dict_path=args.trg_vocab_fpath, reverse=True) def post_process_seq(seq, bos_idx=ModelHyperParams.bos_idx, eos_idx=ModelHyperParams.eos_idx, output_bos=InferTaskConfig.output_bos, output_eos=InferTaskConfig.output_eos): """ Post-process the beam-search decoded sequence. Truncate from the first <eos> and remove the <bos> and <eos> tokens currently. """ eos_pos = len(seq) - 1 for i, idx in enumerate(seq): if idx == eos_idx: eos_pos = i break seq = seq[:eos_pos + 1] return filter( lambda idx: (output_bos or idx != bos_idx) and \ (output_eos or idx != eos_idx), seq) for batch_id, data in enumerate(test_data.batch_generator()): batch_seqs, batch_scores = translate_batch( exe, [item[0] for item in data], encoder_program, encoder_data_input_fields + encoder_util_input_fields, [enc_output.name], decoder_program, decoder_data_input_fields[:-1] + decoder_util_input_fields + (decoder_data_input_fields[-1], ), [predict.name], InferTaskConfig.beam_size, InferTaskConfig.max_length, InferTaskConfig.n_best, len(data), ModelHyperParams.n_head, ModelHyperParams.d_model, ModelHyperParams.eos_idx, # Use eos_idx to pad. ModelHyperParams.eos_idx, # Use eos_idx to pad. ModelHyperParams.bos_idx, ModelHyperParams.eos_idx, ModelHyperParams.unk_idx, output_unk=InferTaskConfig.output_unk) for i in range(len(batch_seqs)): # Post-process the beam-search decoded sequences. seqs = map(post_process_seq, batch_seqs[i]) scores = batch_scores[i] for seq in seqs: print(" ".join([trg_idx2word[idx] for idx in seq]))
def prepare_data_generator(args, is_test, count, pyreader): """ Data generator wrapper for DataReader. If use py_reader, set the data provider for py_reader """ data_reader = reader.DataReader( fpattern=args.val_file_pattern if is_test else args.train_file_pattern, src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, token_delimiter=args.token_delimiter, use_token_batch=args.use_token_batch, batch_size=args.batch_size * (1 if args.use_token_batch else count), pool_size=args.pool_size, sort_type=args.sort_type, shuffle=args.shuffle, shuffle_batch=args.shuffle_batch, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], # count start and end tokens out max_length=ModelHyperParams.max_length - 2, clip_last_batch=False).batch_generator def stack(data_reader, count, clip_last=True): def __impl__(): res = [] for item in data_reader(): res.append(item) if len(res) == count: yield res res = [] if len(res) == count: yield res elif not clip_last: data = [] for item in res: data += item if len(data) > count: inst_num_per_part = len(data) // count yield [ data[inst_num_per_part * i:inst_num_per_part * (i + 1)] for i in range(count) ] return __impl__ def split(data_reader, count): def __impl__(): for item in data_reader(): inst_num_per_part = len(item) // count for i in range(count): yield item[inst_num_per_part * i:inst_num_per_part * (i + 1)] return __impl__ if not args.use_token_batch: # to make data on each device have similar token number data_reader = split(data_reader, count) if args.use_py_reader: pyreader.decorate_tensor_provider( py_reader_provider_wrapper(data_reader)) data_reader = None else: # Data generator for multi-devices data_reader = stack(data_reader, count) return data_reader
def prepare_data_generator(args, is_test, count, pyreader, py_reader_provider_wrapper, place=None): """ Data generator wrapper for DataReader. If use py_reader, set the data provider for py_reader """ # NOTE: If num_trainers > 1, the shuffle_seed must be set, because # the order of batch data generated by reader # must be the same in the respective processes. shuffle_seed = 1 if num_trainers > 1 else None data_reader = reader.DataReader( fpattern=args.val_file_pattern if is_test else args.train_file_pattern, src_vocab_fpath=args.src_vocab_fpath, trg_vocab_fpath=args.trg_vocab_fpath, token_delimiter=args.token_delimiter, use_token_batch=args.use_token_batch, batch_size=args.batch_size * (1 if args.use_token_batch else count), pool_size=args.pool_size, sort_type=args.sort_type, shuffle=args.shuffle, shuffle_seed=shuffle_seed, shuffle_batch=args.shuffle_batch, start_mark=args.special_token[0], end_mark=args.special_token[1], unk_mark=args.special_token[2], # count start and end tokens out max_length=ModelHyperParams.max_length - 2, clip_last_batch=False).batch_generator def stack(data_reader, count, clip_last=True): def __impl__(): res = [] for item in data_reader(): res.append(item) if len(res) == count: yield res res = [] if len(res) == count: yield res elif not clip_last: data = [] for item in res: data += item if len(data) > count: inst_num_per_part = len(data) // count yield [ data[inst_num_per_part * i:inst_num_per_part * (i + 1)] for i in range(count) ] return __impl__ def split(data_reader, count): def __impl__(): for item in data_reader(): inst_num_per_part = len(item) // count for i in range(count): yield item[inst_num_per_part * i:inst_num_per_part * (i + 1)] return __impl__ if not args.use_token_batch: # to make data on each device have similar token number data_reader = split(data_reader, count) if args.use_py_reader: train_reader = py_reader_provider_wrapper(data_reader, place) if num_trainers > 1: assert shuffle_seed is not None train_reader = fluid.contrib.reader.distributed_batch_reader( train_reader) pyreader.decorate_tensor_provider(train_reader) data_reader = None else: # Data generator for multi-devices data_reader = stack(data_reader, count) return data_reader
def __init__(self, data_dir=None, file_extension='png', is_training=True, learning_rate=0.0002, beta1=0.9, reconstr_weight=0.85, smooth_weight=0.05, ssim_weight=0.15, icp_weight=0.0, batch_size=4, img_height=128, img_width=416, seq_length=3, architecture=nets.RESNET, imagenet_norm=True, weight_reg=0.05, exhaustive_mode=False, random_scale_crop=False, flipping_mode=reader.FLIP_RANDOM, random_color=True, depth_upsampling=True, depth_normalization=True, compute_minimum_loss=True, use_skip=True, joint_encoder=True, build_sum=True, shuffle=True, input_file='train', handle_motion=False, equal_weighting=False, size_constraint_weight=0.0, train_global_scale_var=True): self.data_dir = data_dir self.file_extension = file_extension self.is_training = is_training self.learning_rate = learning_rate self.reconstr_weight = reconstr_weight self.smooth_weight = smooth_weight self.ssim_weight = ssim_weight self.icp_weight = icp_weight self.beta1 = beta1 self.batch_size = batch_size self.img_height = img_height self.img_width = img_width self.seq_length = seq_length self.architecture = architecture self.imagenet_norm = imagenet_norm self.weight_reg = weight_reg self.exhaustive_mode = exhaustive_mode self.random_scale_crop = random_scale_crop self.flipping_mode = flipping_mode self.random_color = random_color self.depth_upsampling = depth_upsampling self.depth_normalization = depth_normalization self.compute_minimum_loss = compute_minimum_loss self.use_skip = use_skip self.joint_encoder = joint_encoder self.build_sum = build_sum self.shuffle = shuffle self.input_file = input_file self.handle_motion = handle_motion self.equal_weighting = equal_weighting self.size_constraint_weight = size_constraint_weight self.train_global_scale_var = train_global_scale_var logging.info('data_dir: %s', data_dir) logging.info('file_extension: %s', file_extension) logging.info('is_training: %s', is_training) logging.info('learning_rate: %s', learning_rate) logging.info('reconstr_weight: %s', reconstr_weight) logging.info('smooth_weight: %s', smooth_weight) logging.info('ssim_weight: %s', ssim_weight) logging.info('icp_weight: %s', icp_weight) logging.info('size_constraint_weight: %s', size_constraint_weight) logging.info('beta1: %s', beta1) logging.info('batch_size: %s', batch_size) logging.info('img_height: %s', img_height) logging.info('img_width: %s', img_width) logging.info('seq_length: %s', seq_length) logging.info('architecture: %s', architecture) logging.info('imagenet_norm: %s', imagenet_norm) logging.info('weight_reg: %s', weight_reg) logging.info('exhaustive_mode: %s', exhaustive_mode) logging.info('random_scale_crop: %s', random_scale_crop) logging.info('flipping_mode: %s', flipping_mode) logging.info('random_color: %s', random_color) logging.info('depth_upsampling: %s', depth_upsampling) logging.info('depth_normalization: %s', depth_normalization) logging.info('compute_minimum_loss: %s', compute_minimum_loss) logging.info('use_skip: %s', use_skip) logging.info('joint_encoder: %s', joint_encoder) logging.info('build_sum: %s', build_sum) logging.info('shuffle: %s', shuffle) logging.info('input_file: %s', input_file) logging.info('handle_motion: %s', handle_motion) logging.info('equal_weighting: %s', equal_weighting) logging.info('train_global_scale_var: %s', train_global_scale_var) if self.size_constraint_weight > 0 or not is_training: self.global_scale_var = tf.Variable( 0.1, name='global_scale_var', trainable=self.is_training and train_global_scale_var, dtype=tf.float32, constraint=lambda x: tf.clip_by_value(x, 0, np.infty)) if self.is_training: self.reader = reader.DataReader(self.data_dir, self.batch_size, self.img_height, self.img_width, self.seq_length, NUM_SCALES, self.file_extension, self.random_scale_crop, self.flipping_mode, self.random_color, self.imagenet_norm, self.shuffle, self.input_file) self.build_train_graph() else: self.build_depth_test_graph() self.build_egomotion_test_graph() self.build_objectmotion_test_graph() if self.handle_motion: self.build_objectmotion_test_graph() # At this point, the model is ready. Print some info on model params. util.count_parameters()
def main(_argv): assert not ((FLAGS.overwrite) and (FLAGS.from_ckpt is not None)) input_size = list(map(int, FLAGS.input_size)) # (width, height) input_shape = (input_size[1], input_size[0], 3) loss_weights = list(map(float, FLAGS.loss_weights)) # Load data logging.info("Loading data") data_reader = reader.DataReader() # Define batch generators logging.info("Creating batch generators") traingen = datagen.BatchGenerator( data_reader, batch_size=FLAGS.batch_size, input_size=input_size, jitter=FLAGS.jitter, mode='train', # TODO Add arguments to BatchGenerator ) valgen = datagen.BatchGenerator( data_reader, batch_size=FLAGS.batch_size, input_size=input_size, jitter=False, mode='val', # TODO Add arguments to BatchGenerator ) # Prepare network logging.info("Constructing model") model = config.MODELS[FLAGS.model]( input_shape=input_shape) # TODO Add arguments to constructor # Setup and compile model model.compile( optimizer='adam', loss='mse', # TODO change accordingly ) # logging.info("Compiled model with loss weights:%s"%str(loss_weights)) # TODO Comment in if have loss weights model.summary() if FLAGS.from_ckpt is not None: logging.info("Loading weights from %s" % FLAGS.from_ckpt) model.load_weights(FLAGS.from_ckpt) logging.info("Genrating callbacks") train_callbacks = callbacks.get(directory=FLAGS.save_to, overwrite=FLAGS.overwrite) cfg = config.save( FLAGS.save_to, model=FLAGS.model, input_size=input_size, jitter=FLAGS.jitter, batch_size=FLAGS.batch_size, num_epochs=FLAGS.num_epochs, # TODO Add arguments to config ) logging.info("Saving config : %s" % str(cfg)) logging.info("Starting training") model.fit(traingen, epochs=FLAGS.num_epochs, verbose=1, validation_data=valgen, callbacks=train_callbacks, workers=8, max_queue_size=3)