def run_for_epochs(batches):
for batch in batches:
d_indices, q_indices, a_indices, entity_counts = data_utils.pad_batch(batch)
current_step = tf.train.global_step(sess, global_step)
if current_step % 5 == 0:
train_loss, train_accuracy = \
train_step((d_indices, q_indices, a_indices, entity_counts), current_step, print_bool=True)
dev_loss, dev_accuracy = \
dev_step((d_indices_dev, q_indices_dev, a_indices_dev, entity_counts_dev), current_step, writer=dev_summary_writer)
else:
train_loss, train_accuracy = \
train_step((d_indices, q_indices, a_indices, entity_counts), current_step, print_bool=False)
if current_step % 5 == 0:
path = saver.save(sess, checkpoint_prefix, global_step=global_step)
print("Saved model checkpoint to {}".format(path))
return (train_loss, train_accuracy)
def train(
train_sparse_depth_path,
train_ground_truth_path,
# Validation data
val_sparse_depth_path=None,
val_ground_truth_path=None,
# Dataloader settings
n_batch=settings.N_BATCH,
n_height=settings.N_HEIGHT,
n_width=settings.N_WIDTH,
min_dataset_depth=settings.MIN_DATASET_DEPTH,
max_dataset_depth=settings.MAX_DATASET_DEPTH,
crop_type=settings.CROP_TYPE,
augmentation_random_horizontal_crop=False,
augmentation_random_vertical_crop=False,
augmentation_random_horizontal_flip=False,
# Network architecture
network_type=settings.NETWORK_TYPE_SCAFFNET,
activation_func=settings.ACTIVATION_FUNC,
n_filter_output=settings.N_FILTER_OUTPUT_SCAFFNET,
# Spatial pyramid pooling
pool_kernel_sizes_spp=settings.POOL_KERNEL_SIZES_SPP,
n_convolution_spp=settings.N_CONVOLUTION_SPP,
n_filter_spp=settings.N_FILTER_SPP,
# Depth prediction settings
min_predict_depth=settings.MIN_PREDICT_DEPTH,
max_predict_depth=settings.MAX_PREDICT_DEPTH,
# Training settings
learning_rates=settings.LEARNING_RATES,
learning_schedule=settings.LEARNING_SCHEDULE,
n_epoch=settings.N_EPOCH,
loss_func=settings.LOSS_FUNC_SCAFFNET,
# Depth evaluation settings
min_evaluate_depth=settings.MIN_EVALUATE_DEPTH,
max_evaluate_depth=settings.MAX_EVALUATE_DEPTH,
# Checkpoint settings
n_checkpoint=settings.N_CHECKPOINT,
n_summary=settings.N_SUMMARY,
checkpoint_path=settings.CHECKPOINT_PATH,
restore_path=settings.RESTORE_PATH,
# Hardware settings
n_thread=settings.N_THREAD):
model_path = os.path.join(checkpoint_path, 'model.ckpt')
event_path = os.path.join(checkpoint_path, 'events')
log_path = os.path.join(checkpoint_path, 'results.txt')
# Load sparse depth, validity map and ground truth paths from file for training
train_sparse_depth_paths = data_utils.read_paths(train_sparse_depth_path)
train_ground_truth_paths = data_utils.read_paths(train_ground_truth_path)
n_train_sample = len(train_sparse_depth_paths)
assert n_train_sample == len(train_ground_truth_paths)
n_train_step = n_epoch * np.ceil(n_train_sample / n_batch).astype(np.int32)
# Load sparse depth, validity map and ground truth paths from file for validation
val_sparse_depth_paths = data_utils.read_paths(val_sparse_depth_path)
val_ground_truth_paths = data_utils.read_paths(val_ground_truth_path)
n_val_sample = len(val_sparse_depth_paths)
assert n_val_sample == len(val_ground_truth_paths)
# Pad validation paths based on batch size
val_sparse_depth_paths = data_utils.pad_batch(val_sparse_depth_paths,
n_batch)
# Load validation ground truth and do center crop
val_ground_truths = []
for idx in range(len(val_ground_truth_paths)):
# Load ground truth and validity map
ground_truth, validity_map = \
data_utils.load_depth_with_validity_map(val_ground_truth_paths[idx])
# Get crop start and end positions
if crop_type == 'center':
start_height = int(float(ground_truth.shape[0] - n_height))
elif crop_type == 'bottom':
start_height = ground_truth.shape[0] - n_height
else:
start_height = 0
end_height = n_height + start_height
start_width = int(float(ground_truth.shape[1] - n_width) / 2.0)
end_width = n_width + start_width
# Concatenate ground truth and validity map together
ground_truth = np.concatenate([
np.expand_dims(ground_truth, axis=-1),
np.expand_dims(validity_map, axis=-1)
],
axis=-1)
# Crop ground truth
val_ground_truths.append(ground_truth[start_height:end_height,
start_width:end_width, :])
val_ground_truth_paths = data_utils.pad_batch(val_ground_truth_paths,
n_batch)
with tf.Graph().as_default():
# Set up current training step
global_step = tf.Variable(0, trainable=False)
# Initialize optimizer with learning schedule
learning_schedule_steps = [
np.int32((float(v) / n_epoch) * n_train_step)
for v in learning_schedule
]
learning_rate_schedule = tf.train.piecewise_constant(
global_step, learning_schedule_steps, learning_rates)
optimizer = tf.train.AdamOptimizer(learning_rate_schedule)
# Initialize dataloader
dataloader = ScaffNetDataloader(shape=[n_batch, n_height, n_width, 2],
name='scaffnet_dataloader',
is_training=True,
n_thread=n_thread,
prefetch_size=(2 * n_thread))
# Fetch the input from dataloader
input_depth = dataloader.next_element[0]
ground_truth = dataloader.next_element[1]
# Build computation graph
model = ScaffNetModel(input_depth,
ground_truth,
is_training=True,
network_type=network_type,
activation_func=activation_func,
n_filter_output=n_filter_output,
pool_kernel_sizes_spp=pool_kernel_sizes_spp,
n_convolution_spp=n_convolution_spp,
n_filter_spp=n_filter_spp,
min_dataset_depth=min_dataset_depth,
max_dataset_depth=max_dataset_depth,
min_predict_depth=min_predict_depth,
max_predict_depth=max_predict_depth,
loss_func=loss_func)
# Compute loss and gradients
loss = model.loss
gradients = optimizer.compute_gradients(loss)
gradients = optimizer.apply_gradients(gradients,
global_step=global_step)
model_summary = tf.summary.merge_all()
# Count trainable parameters
n_parameter = 0
for variable in tf.trainable_variables():
n_parameter += np.array(variable.get_shape().as_list()).prod()
# Log settings
log('Dataloader settings:', log_path)
log(
'n_batch=%d n_height=%d n_width=%d' %
(n_batch, n_height, n_width), log_path)
log(
'min_dataset_depth=%.2f max_dataset_depth=%.2f' %
(min_dataset_depth, max_dataset_depth), log_path)
log('crop_type=%s' % (crop_type), log_path)
log(
'random_horizontal_crop=%s random_vertical_crop=%s' %
(augmentation_random_horizontal_crop,
augmentation_random_vertical_crop), log_path)
log(
'random_horizontal_flip=%s' %
(augmentation_random_horizontal_flip), log_path)
log('', log_path)
log('Network settings:', log_path)
log('network_type=%s n_parameter=%d' % (network_type, n_parameter),
log_path)
log('activation_func=%s' % (activation_func), log_path)
log(
'n_filter_output=%s' %
(str(n_filter_output) if n_filter_output > 0 else 'upsample'),
log_path)
log('', log_path)
log('Spatial pyramid pooling settings:', log_path)
log(
'pool_kernel_sizes_spp=[%s]' %
(', '.join([str(i) for i in pool_kernel_sizes_spp])), log_path)
log(
'n_convolution_spp=%d n_filter_spp=%d' %
(n_convolution_spp, n_filter_spp), log_path)
log('', log_path)
log('Depth prediction settings:', log_path)
log(
'min_predict_depth=%.2f max_predict_depth=%.2f' %
(min_predict_depth, max_predict_depth), log_path)
log('', log_path)
log('Training settings:', log_path)
log(
'n_sample=%d n_epoch=%d n_step=%d' %
(n_train_sample, n_epoch, n_train_step), log_path)
log('loss_func=%s' % (loss_func), log_path)
log(
'learning_schedule=[%s, %d (%d)]' % (', '.join(
'{} ({}) : {:.1E}'.format(s, v, r)
for s, v, r in zip([0] + learning_schedule, [0] +
learning_schedule_steps, learning_rates)),
n_epoch, n_train_step),
log_path)
log('', log_path)
log('Depth evaluation settings:', log_path)
log(
'min_evaluate_depth=%.2f max_evaluate_depth=%.2f' %
(min_evaluate_depth, max_evaluate_depth), log_path)
log('', log_path)
log('Checkpoint settings:', log_path)
log('checkpoint_path=%s' % (checkpoint_path), log_path)
log(
'restore_path=%s' %
('None' if restore_path == '' else restore_path), log_path)
log('', log_path)
# Initialize Tensorflow session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
# Initialize saver for storing and restoring checkpoints
train_summary_writer = tf.summary.FileWriter(event_path + '-train',
session.graph)
val_summary_writer = tf.summary.FileWriter(event_path + '-val')
train_saver = tf.train.Saver(max_to_keep=50)
# Initialize all variables
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
# If given, load the weights from the restore path
if restore_path != '':
train_saver.restore(session, restore_path)
# Begin training
log('Begin training...', log_path)
start_step = global_step.eval(session=session)
time_start = time.time()
train_step = start_step
step = 0
do_center_crop = True if crop_type == 'center' else False
do_bottom_crop = True if crop_type == 'bottom' else False
# Shuffle data for current epoch
train_sparse_depth_paths_epoch, \
train_ground_truth_paths_epoch = data_utils.make_epoch(
input_arr=[train_sparse_depth_paths, train_ground_truth_paths],
n_batch=n_batch)
# Feed input paths into dataloader for training
dataloader.initialize(
session,
sparse_depth_paths=train_sparse_depth_paths_epoch,
ground_truth_paths=train_ground_truth_paths_epoch,
do_center_crop=do_center_crop,
do_bottom_crop=do_bottom_crop,
random_horizontal_crop=augmentation_random_horizontal_crop,
random_vertical_crop=augmentation_random_vertical_crop,
random_horizontal_flip=augmentation_random_horizontal_flip)
while train_step < n_train_step:
try:
if train_step % n_summary == 0:
# Compute loss and training summary
_, loss_value, train_summary = session.run(
[gradients, loss, model_summary])
# Write training results to summary
train_summary_writer.add_summary(train_summary,
global_step=train_step)
else:
# Compute loss
_, loss_value = session.run([gradients, loss])
if train_step and (train_step % n_checkpoint) == 0:
time_elapse = (time.time() -
time_start) / 3600 * train_step / (
train_step - start_step + 1)
time_remain = (n_train_step / train_step - 1) * time_elapse
checkpoint_log = \
'batch: {:>6}/{:>6} time elapsed: {:.2f}h time left: {:.2f}h \n' + \
'loss: {:.5f}'
log(
checkpoint_log.format(train_step, n_train_step,
time_elapse, time_remain,
loss_value), log_path)
# Feed input paths into dataloader for validation
dataloader.initialize(
session,
sparse_depth_paths=val_sparse_depth_paths,
ground_truth_paths=val_ground_truth_paths,
do_center_crop=do_center_crop,
do_bottom_crop=do_bottom_crop,
random_horizontal_crop=False,
random_vertical_crop=False,
random_horizontal_flip=False)
# Run model on validation samples
val_output_depths = run(model,
session,
n_sample=n_val_sample,
summary=model_summary,
summary_writer=val_summary_writer,
step=train_step,
verbose=False)
# Run validation metrics
eval_utils.evaluate(val_output_depths,
val_ground_truths,
train_step,
log_path=log_path,
min_evaluate_depth=min_evaluate_depth,
max_evaluate_depth=max_evaluate_depth)
# Switch back to training
current_sample = n_batch * (step + 1)
dataloader.initialize(
session,
sparse_depth_paths=train_sparse_depth_paths_epoch[
current_sample:],
ground_truth_paths=train_ground_truth_paths_epoch[
current_sample:],
do_center_crop=do_center_crop,
do_bottom_crop=do_bottom_crop,
random_horizontal_crop=
augmentation_random_horizontal_crop,
random_vertical_crop=augmentation_random_vertical_crop,
random_horizontal_flip=
augmentation_random_horizontal_flip)
train_saver.save(session,
model_path,
global_step=train_step)
train_step += 1
step += 1
except tf.errors.OutOfRangeError:
step = 0
# Shuffle data for next epoch
train_sparse_depth_paths_epoch, \
train_ground_truth_paths_epoch = data_utils.make_epoch(
input_arr=[train_sparse_depth_paths, train_ground_truth_paths],
n_batch=n_batch)
# Feed input paths into dataloader for training
dataloader.initialize(
session,
sparse_depth_paths=train_sparse_depth_paths_epoch,
ground_truth_paths=train_ground_truth_paths_epoch,
do_center_crop=do_center_crop,
do_bottom_crop=do_bottom_crop,
random_horizontal_crop=augmentation_random_horizontal_crop,
random_vertical_crop=augmentation_random_vertical_crop,
random_horizontal_flip=augmentation_random_horizontal_flip)
train_saver.save(session, model_path, global_step=n_train_step)
'''
Read input paths and load ground truth (if available)
'''
log_path = os.path.join(args.output_path, 'results.txt')
if not os.path.exists(args.output_path):
os.makedirs(args.output_path)
# Load sparse depth and validity map paths from file for evaluation
sparse_depth_paths = data_utils.read_paths(args.sparse_depth_path)
sparse_depth_paths = sparse_depth_paths[args.start_idx:args.end_idx]
n_sample = len(sparse_depth_paths)
# Pad all paths based on batch size
sparse_depth_paths = data_utils.pad_batch(sparse_depth_paths, args.n_batch)
n_step = n_sample // args.n_batch
ground_truth_available = True if args.ground_truth_path != '' else False
ground_truths = []
if ground_truth_available:
ground_truth_paths = data_utils.read_paths(args.ground_truth_path)
ground_truth_paths = ground_truth_paths[args.start_idx:args.end_idx]
assert n_sample == len(ground_truth_paths)
# Load ground truth
for idx in range(n_sample):
image_paths = data_utils.read_paths(args.image_path)
image_paths = image_paths[args.start_idx:args.end_idx]
input_depth_paths = data_utils.read_paths(args.input_depth_path)
input_depth_paths = input_depth_paths[args.start_idx:args.end_idx]
sparse_depth_paths = data_utils.read_paths(args.sparse_depth_path)
sparse_depth_paths = sparse_depth_paths[args.start_idx:args.end_idx]
n_sample = len(image_paths)
assert n_sample == len(input_depth_paths)
assert n_sample == len(sparse_depth_paths)
# Pad all paths based on batch size
image_paths = data_utils.pad_batch(image_paths, args.n_batch)
input_depth_paths = data_utils.pad_batch(input_depth_paths, args.n_batch)
sparse_depth_paths = data_utils.pad_batch(sparse_depth_paths, args.n_batch)
n_step = n_sample // args.n_batch
ground_truth_available = True if args.ground_truth_path != '' else False
ground_truths = []
if ground_truth_available:
ground_truth_paths = data_utils.read_paths(args.ground_truth_path)
ground_truth_paths = ground_truth_paths[args.start_idx:args.end_idx]
assert n_sample == len(ground_truth_paths)
# Load ground truth
def evaluate(im0_path,
restore_path,
output_path,
n_batch=settings.N_BATCH,
n_height=settings.N_HEIGHT,
n_width=settings.N_WIDTH,
n_channel=settings.N_CHANNEL,
n_pyramid=settings.N_PYRAMID,
max_disparity=settings.MAX_DISPARITY,
n_gpu=settings.N_GPU,
n_thread=settings.N_THREAD):
"""Test function."""
# Create dataloader for computation graph
dataloader = DataLoader(shape=[n_batch, n_height, n_width, n_channel],
name='dataloader',
n_thread=n_thread,
prefetch_size=n_thread,
normalize=True,
random_flip=False,
random_gamma=False,
gamma_range=[0.8, 1.2],
random_brightness=False,
brightness_range=[0.5, 2.0],
random_color=False,
color_range=[0.8, 1.2])
# Build model
model = MonoDispNet(dataloader.next_element[0],
dataloader.next_element[1],
n_pyramid=n_pyramid,
max_disparity=max_disparity)
# Start a Tensorflow session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
# Initialize saver that will be used for restore
train_saver = tf.train.Saver()
# Initialize all variables
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
# Restore weights from checkpoint
log('Restoring from: %s' % restore_path)
train_saver.restore(session, restore_path)
# Load the files for evaluation
im0_paths = data_utils.read_paths(im0_path)
n_sample = len(im0_paths)
im0_paths = data_utils.pad_batch(im0_paths, n_batch)
n_step = len(im0_paths) // n_batch
log('Evaluating %d files...' % n_sample)
dataloader.initialize(session,
im0_paths=im0_paths,
im1_paths=im0_paths,
augment=False)
d_arr = np.zeros((n_step * n_batch, n_height, n_width), dtype=np.float32)
start_time = time.time()
for step in range(n_step):
batch_start = step * n_batch
batch_end = step * n_batch + n_batch
d = session.run(model.model0[0])
d_arr[batch_start:batch_end, :, :] = d[:, :, :, 0]
end_time = time.time()
log('Total time: %.1f ms Average time per image: %.1f ms' %
(1000 * (end_time - start_time),
(1000 * (end_time - start_time) / n_sample)))
d_arr = d_arr[0:n_sample, :, :]
output_path = os.path.join(output_path, 'disparities.npy')
log('Storing predictions to %s' % output_path)
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
np.save(output_path, d_arr)
args.image_path))[args.start_idx:args.end_idx]
iz_paths = sorted(data_utils.read_paths(
args.interp_depth_path))[args.start_idx:args.end_idx]
vm_paths = sorted(data_utils.read_paths(
args.validity_map_path))[args.start_idx:args.end_idx]
assert (len(im_paths) == len(iz_paths))
assert (len(im_paths) == len(vm_paths))
n_sample = len(im_paths)
if args.ground_truth_path != '':
gt_paths = sorted(data_utils.read_paths(
args.ground_truth_path))[args.start_idx:args.end_idx]
assert (len(im_paths) == len(gt_paths))
# Pad all paths based on batch
im_paths = data_utils.pad_batch(im_paths, args.n_batch)
iz_paths = data_utils.pad_batch(iz_paths, args.n_batch)
vm_paths = data_utils.pad_batch(vm_paths, args.n_batch)
n_step = len(im_paths) // args.n_batch
gt_arr = []
if args.ground_truth_path != '':
# Load ground truth
for idx in range(n_sample):
sys.stdout.write('Loading {}/{} groundtruth depth maps \r'.format(
idx + 1, n_sample))
sys.stdout.flush()
gt, vm = data_utils.load_depth_with_validity_map(gt_paths[idx])
gt = np.concatenate(
[np.expand_dims(gt, axis=-1),
def train(train_image_path,
train_input_depth_path,
train_sparse_depth_path,
train_intrinsics_path,
# Validation data filepaths
val_image_path=None,
val_input_depth_path=None,
val_sparse_depth_path=None,
val_ground_truth_path=None,
# Dataloader settings
n_batch=settings.N_BATCH,
n_height=settings.N_HEIGHT,
n_width=settings.N_WIDTH,
crop_type=settings.CROP_TYPE,
augmentation_random_horizontal_crop=False,
augmentation_random_vertical_crop=False,
# Network settings
network_type=settings.NETWORK_TYPE_FUSIONNET,
image_filter_pct=settings.IMAGE_FILTER_PCT,
depth_filter_pct=settings.DEPTH_FILTER_PCT,
activation_func=settings.ACTIVATION_FUNC,
# Depth prediction settings
min_predict_depth=settings.MIN_PREDICT_DEPTH,
max_predict_depth=settings.MAX_PREDICT_DEPTH,
min_scale_depth=settings.MIN_SCALE_DEPTH,
max_scale_depth=settings.MAX_SCALE_DEPTH,
min_residual_depth=settings.MIN_RESIDUAL_DEPTH,
max_residual_depth=settings.MAX_RESIDUAL_DEPTH,
# Training settings
n_epoch=settings.N_EPOCH,
learning_rates=settings.LEARNING_RATES,
learning_schedule=settings.LEARNING_SCHEDULE,
# Loss function settings
validity_map_color=settings.VALIDITY_MAP_COLOR,
w_color=settings.W_COLOR,
w_structure=settings.W_STRUCTURE,
w_sparse_depth=settings.W_SPARSE_DEPTH,
w_smoothness=settings.W_SMOOTHNESS,
w_prior_depth=settings.W_PRIOR_DEPTH,
residual_threshold_prior_depth=settings.RESIDUAL_THRESHOLD_PRIOR_DEPTH,
rotation_param=settings.ROTATION_PARAM,
# Depth evaluation settings
min_evaluate_depth=settings.MIN_EVALUATE_DEPTH,
max_evaluate_depth=settings.MAX_EVALUATE_DEPTH,
# Checkpoint settings
n_checkpoint=settings.N_CHECKPOINT,
n_summary=settings.N_SUMMARY,
checkpoint_path=settings.CHECKPOINT_PATH,
restore_path=settings.RESTORE_PATH,
# Hardware settings
n_thread=settings.N_THREAD):
model_path = os.path.join(checkpoint_path, 'model.ckpt')
event_path = os.path.join(checkpoint_path, 'events')
log_path = os.path.join(checkpoint_path, 'results.txt')
# Load image, input depth, sparse depth, instrinsics paths from file
train_image_paths = data_utils.read_paths(train_image_path)
train_input_depth_paths = data_utils.read_paths(train_input_depth_path)
train_sparse_depth_paths = data_utils.read_paths(train_sparse_depth_path)
train_intrinsics_paths = data_utils.read_paths(train_intrinsics_path)
n_train_sample = len(train_image_paths)
assert n_train_sample == len(train_input_depth_paths)
assert n_train_sample == len(train_sparse_depth_paths)
assert n_train_sample == len(train_intrinsics_paths)
n_train_step = n_epoch * np.ceil(n_train_sample / n_batch).astype(np.int32)
# Load image, input depth, and sparse depth paths from file for validation
val_image_paths = data_utils.read_paths(val_image_path)
val_input_depth_paths = data_utils.read_paths(val_input_depth_path)
val_sparse_depth_paths = data_utils.read_paths(val_sparse_depth_path)
val_ground_truth_paths = data_utils.read_paths(val_ground_truth_path)
n_val_sample = len(val_image_paths)
assert n_val_sample == len(val_input_depth_paths)
assert n_val_sample == len(val_sparse_depth_paths)
assert n_val_sample == len(val_ground_truth_paths)
val_image_paths = data_utils.pad_batch(val_image_paths, n_batch)
val_input_depth_paths = data_utils.pad_batch(val_input_depth_paths, n_batch)
val_sparse_depth_paths = data_utils.pad_batch(val_sparse_depth_paths, n_batch)
# Load validation ground truth and do center crop
val_ground_truths = []
for idx in range(len(val_ground_truth_paths)):
ground_truth, validity_map_ground_truth = \
data_utils.load_depth_with_validity_map(val_ground_truth_paths[idx])
ground_truth = np.concatenate([
np.expand_dims(ground_truth, axis=-1),
np.expand_dims(validity_map_ground_truth, axis=-1)],
axis=-1)
# Get start and end of crop
if crop_type == 'center':
start_height = int(float(ground_truth.shape[0] - n_height))
elif crop_type == 'bottom':
start_height = ground_truth.shape[0] - n_height
else:
start_height = 0
end_height = n_height + start_height
start_width = int(float(ground_truth.shape[1] - n_width) / 2.0)
end_width = n_width + start_width
ground_truth = \
ground_truth[start_height:end_height, start_width:end_width, :]
val_ground_truths.append(ground_truth)
val_ground_truth_paths = data_utils.pad_batch(val_ground_truth_paths, n_batch)
with tf.Graph().as_default():
# Set up current training step
global_step = tf.Variable(0, trainable=False)
# Initialize optimizer with learning schedule
learning_schedule_steps = [
np.int32((float(v) / n_epoch) * n_train_step) for v in learning_schedule
]
learning_rate_schedule = tf.train.piecewise_constant(
global_step,
learning_schedule_steps,
learning_rates)
optimizer = tf.train.AdamOptimizer(learning_rate_schedule)
# Initialize dataloader
dataloader = FusionNetDataloader(
shape=[n_batch, n_height, n_width, 3],
name='fusionnet_dataloader',
is_training=True,
n_thread=n_thread,
prefetch_size=2 * n_thread)
# Fetch the input from dataloader
image0 = dataloader.next_element[0]
image1 = dataloader.next_element[1]
image2 = dataloader.next_element[2]
input_depth = dataloader.next_element[3]
intrinsics = dataloader.next_element[4]
# Build computation graph
model = FusionNetModel(
image0=image0,
image1=image1,
image2=image2,
input_depth=input_depth,
intrinsics=intrinsics,
is_training=True,
network_type=network_type,
image_filter_pct=image_filter_pct,
depth_filter_pct=depth_filter_pct,
activation_func=activation_func,
min_predict_depth=min_predict_depth,
max_predict_depth=max_predict_depth,
min_scale_depth=min_scale_depth,
max_scale_depth=max_scale_depth,
min_residual_depth=min_residual_depth,
max_residual_depth=max_residual_depth,
validity_map_color=validity_map_color,
w_color=w_color,
w_structure=w_structure,
w_sparse_depth=w_sparse_depth,
w_smoothness=w_smoothness,
w_prior_depth=w_prior_depth,
residual_threshold_prior_depth=residual_threshold_prior_depth,
rotation_param=rotation_param)
loss = model.loss
gradients = optimizer.compute_gradients(loss)
gradients = optimizer.apply_gradients(gradients, global_step=global_step)
model_summary = tf.summary.merge_all()
# Count trainable parameters
n_parameter = 0
for variable in tf.trainable_variables():
n_parameter += np.array(variable.get_shape().as_list()).prod()
# Log settings
log('Dataloader settings:', log_path)
log('n_batch=%d n_height=%d n_width=%d' %
(n_batch, n_height, n_width), log_path)
log('crop_type=%s' %
(crop_type), log_path)
log('random_horizontal_crop=%s random_vertical_crop=%s' %
(augmentation_random_horizontal_crop, augmentation_random_vertical_crop), log_path)
log('', log_path)
log('Network settings:', log_path)
log('network_type=%s n_parameter=%d' %
(network_type, n_parameter), log_path)
log('image_filter_pct=%.2f depth_filter_pct=%.2f' %
(image_filter_pct, depth_filter_pct), log_path)
log('activation_func=%s' %
(activation_func), log_path)
log('', log_path)
log('Depth prediction settings:', log_path)
log('min_predict_depth=%.2f max_predict_depth=%.2f' %
(min_predict_depth, max_predict_depth), log_path)
log('min_scale_depth=%.2f max_scale_depth=%.2f' %
(min_scale_depth, max_scale_depth), log_path)
log('min_residual_depth=%.2f max_residual_depth=%.2f' %
(min_residual_depth, max_residual_depth), log_path)
log('', log_path)
log('Training settings:', log_path)
log('n_sample=%d n_epoch=%d n_step=%d' %
(n_train_sample, n_epoch, n_train_step), log_path)
log('learning_schedule=[%s, %d (%d)]' %
(', '.join('{} ({}) : {:.1E}'.format(s, v, r)
for s, v, r in zip(
[0] + learning_schedule, [0] + learning_schedule_steps, learning_rates)),
n_epoch,
n_train_step), log_path)
log('validity_map_color=%s' %
(validity_map_color), log_path)
log('w_color=%.2f w_structure=%.2f w_sparse_depth=%.2f' %
(w_color, w_structure, w_sparse_depth), log_path)
log('w_smoothness=%.3f w_prior_depth=%.2f' %
(w_smoothness, w_prior_depth), log_path)
log('residual_threshold_prior_depth=%.2f' %
(residual_threshold_prior_depth), log_path)
log('rotation_param=%s' %
(rotation_param), log_path)
log('', log_path)
log('Depth evaluation settings:', log_path)
log('min_evaluate_depth=%.2f max_evaluate_depth=%.2f' %
(min_evaluate_depth, max_evaluate_depth), log_path)
log('', log_path)
log('Checkpoint settings:', log_path)
log('checkpoint_path=%s' %
(checkpoint_path), log_path)
log('restore_path=%s' %
('None' if restore_path == '' else restore_path), log_path)
log('', log_path)
# Initialize Tensorflow session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
# Initialize saver for storing and restoring checkpoints
train_summary_writer = tf.summary.FileWriter(event_path + '-train', session.graph)
val_summary_writer = tf.summary.FileWriter(event_path + '-val')
train_saver = tf.train.Saver(max_to_keep=50)
# Initialize all variables
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
# If given, load the weights from the restore path
if restore_path != '':
import tensorflow.contrib.slim as slim
vars_to_restore_fusionnet = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES,
scope=network_type)
init_assign_op_fusionnet, init_feed_dict_fusionnet = slim.assign_from_checkpoint(
restore_path,
vars_to_restore_fusionnet,
ignore_missing_vars=True)
session.run(init_assign_op_fusionnet, init_feed_dict_fusionnet)
vars_to_restore_posenet = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES,
scope='posenet')
init_assign_op_posenet, init_feed_dict_posenet = slim.assign_from_checkpoint(
restore_path,
vars_to_restore_posenet,
ignore_missing_vars=True)
session.run(init_assign_op_posenet, init_feed_dict_posenet)
# Begin training
log('Begin training...', log_path)
start_step = global_step.eval(session=session)
time_start = time.time()
train_step = start_step
step = 0
do_center_crop = True if crop_type == 'center' else False
do_bottom_crop = True if crop_type == 'bottom' else False
# Shuffle data for current epoch
train_image_paths_epoch, \
train_input_depth_paths_epoch, \
train_sparse_depth_paths_epoch, \
train_intrinsics_paths_epoch = data_utils.make_epoch(
input_arr=[
train_image_paths,
train_input_depth_paths,
train_sparse_depth_paths,
train_intrinsics_paths],
n_batch=n_batch)
# Feed input paths into dataloader for training
dataloader.initialize(
session,
image_paths=train_image_paths_epoch,
input_depth_paths=train_input_depth_paths_epoch,
sparse_depth_paths=train_sparse_depth_paths_epoch,
intrinsics_paths=train_intrinsics_paths_epoch,
do_center_crop=do_center_crop,
do_bottom_crop=do_bottom_crop,
random_horizontal_crop=augmentation_random_horizontal_crop,
random_vertical_crop=augmentation_random_vertical_crop)
while train_step < n_train_step:
try:
if train_step % n_summary == 0:
# Compute loss and training summary
_, loss_value, train_summary = session.run([gradients, loss, model_summary])
# Write training summary
train_summary_writer.add_summary(train_summary, global_step=train_step)
else:
# Compute loss
_, loss_value = session.run([gradients, loss])
if train_step and (train_step % n_checkpoint) == 0:
time_elapse = (time.time() - time_start) / 3600 * train_step / (train_step - start_step + 1)
time_remain = (n_train_step / train_step - 1) * time_elapse
checkpoint_log = \
'batch: {:>6}/{:>6} time elapsed: {:.2f}h time left: {:.2f}h \n' + \
'loss: {:.5f}'
log(checkpoint_log.format(
train_step,
n_train_step,
time_elapse,
time_remain,
loss_value), log_path)
# Feed input paths into dataloader for validation
dataloader.initialize(
session,
image_paths=val_image_paths,
input_depth_paths=val_input_depth_paths,
sparse_depth_paths=val_sparse_depth_paths,
intrinsics_paths=train_intrinsics_paths[0:len(val_image_paths)],
do_center_crop=do_center_crop,
do_bottom_crop=do_bottom_crop,
random_horizontal_crop=False,
random_vertical_crop=False)
# Run model on validation samples
val_output_depths = run(
model,
session,
n_sample=n_val_sample,
summary=model_summary,
summary_writer=val_summary_writer,
step=train_step,
verbose=False)
eval_utils.evaluate(
val_output_depths,
val_ground_truths,
train_step,
log_path=log_path,
min_evaluate_depth=min_evaluate_depth,
max_evaluate_depth=max_evaluate_depth)
# Switch back to training
current_sample = n_batch * (step + 1)
dataloader.initialize(
session,
image_paths=train_image_paths_epoch[current_sample:],
input_depth_paths=train_input_depth_paths_epoch[current_sample:],
sparse_depth_paths=train_sparse_depth_paths_epoch[current_sample:],
intrinsics_paths=train_intrinsics_paths_epoch[current_sample:],
do_center_crop=do_center_crop,
do_bottom_crop=do_bottom_crop,
random_horizontal_crop=augmentation_random_horizontal_crop,
random_vertical_crop=augmentation_random_vertical_crop)
train_saver.save(session, model_path, global_step=train_step)
train_step += 1
step += 1
except tf.errors.OutOfRangeError:
step = 0
# Shuffle data for next epoch
train_image_paths_epoch, \
train_input_depth_paths_epoch, \
train_sparse_depth_paths_epoch, \
train_intrinsics_paths_epoch, = data_utils.make_epoch(
input_arr=[
train_image_paths,
train_input_depth_paths,
train_sparse_depth_paths,
train_intrinsics_paths],
n_batch=n_batch)
# Feed input paths into dataloader for training
dataloader.initialize(
session,
image_paths=train_image_paths_epoch,
input_depth_paths=train_input_depth_paths_epoch,
sparse_depth_paths=train_sparse_depth_paths_epoch,
intrinsics_paths=train_intrinsics_paths_epoch,
do_center_crop=do_center_crop,
do_bottom_crop=do_bottom_crop,
random_horizontal_crop=augmentation_random_horizontal_crop,
random_vertical_crop=augmentation_random_vertical_crop)
train_saver.save(session, model_path, global_step=n_train_step)
vocab_size=10000
batch_size=32
num_epochs=5
max_global_norm=10
max_entities = len(pickle.load(open("entity.p", "rb")))
print(max_entities)
# Load Data
# =================================================================================
data_path = "/Users/kellyzhang/Documents/ReadingComprehension/DeepMindDataset/cnn/questions"
batches = data_utils.make_batches(num_epochs=num_epochs, batch_size=batch_size, shuffle=False, dataset="train", data_path=data_path, max_words=vocab_size, max_examples=500)
dev_batch = data_utils.load_data(dataset="validation", data_path=data_path, max_words=None, max_examples=32)
d_indices_dev, q_indices_dev, a_indices_dev, entity_counts_dev = \
data_utils.pad_batch(np.array(dev_batch), train=False)
# Helper Functions
# =================================================================================
def train_step(data, current_step, writer=None, print_bool=False):
"""
Single training step
"""
d_indices, q_indices, a_indices, entity_counts = data
feed_dict = {
stan_reader.input_d: d_indices,
stan_reader.input_q: q_indices,
stan_reader.input_a: a_indices,
stan_reader.input_m: entity_counts
}
# print("hidden states d")
batch_size=2,
shuffle=False,
dataset="train",
data_path=data_path,
max_words=10000,
max_examples=100)
d_indices_dev, q_indices_dev, a_indices_dev, entity_counts_dev = data_utils.load_data(
dataset="validation",
data_path=data_path,
max_words=None,
max_examples=100)
count = 0
for batch in batches:
d_indices, q_indices, a_indices, entity_counts = data_utils.pad_batch(
batch)
print(d_indices)
print(q_indices)
print(a_indices)
print(entity_counts)
count += 1
if count > 3:
break
# Starting interactive Session
sess = tf.InteractiveSession()
# MODEL
stan_reader = StanfordReader(max_entities=len(