def evaluate(im0_path,
restore_path,
output_path,
n_batch=settings.N_BATCH,
n_height=settings.N_HEIGHT,
n_width=settings.N_WIDTH,
n_channel=settings.N_CHANNEL,
n_pyramid=settings.N_PYRAMID,
max_disparity=settings.MAX_DISPARITY,
n_gpu=settings.N_GPU,
n_thread=settings.N_THREAD):
"""Test function."""
# Create dataloader for computation graph
dataloader = DataLoader(shape=[n_batch, n_height, n_width, n_channel],
name='dataloader',
n_thread=n_thread,
prefetch_size=n_thread,
normalize=True,
random_flip=False,
random_gamma=False,
gamma_range=[0.8, 1.2],
random_brightness=False,
brightness_range=[0.5, 2.0],
random_color=False,
color_range=[0.8, 1.2])
# Build model
model = MonoDispNet(dataloader.next_element[0],
dataloader.next_element[1],
n_pyramid=n_pyramid,
max_disparity=max_disparity)
# Start a Tensorflow session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
# Initialize saver that will be used for restore
train_saver = tf.train.Saver()
# Initialize all variables
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
# Restore weights from checkpoint
log('Restoring from: %s' % restore_path)
train_saver.restore(session, restore_path)
# Load the files for evaluation
im0_paths = data_utils.read_paths(im0_path)
n_sample = len(im0_paths)
im0_paths = data_utils.pad_batch(im0_paths, n_batch)
n_step = len(im0_paths) // n_batch
log('Evaluating %d files...' % n_sample)
dataloader.initialize(session,
im0_paths=im0_paths,
im1_paths=im0_paths,
augment=False)
d_arr = np.zeros((n_step * n_batch, n_height, n_width), dtype=np.float32)
start_time = time.time()
for step in range(n_step):
batch_start = step * n_batch
batch_end = step * n_batch + n_batch
d = session.run(model.model0[0])
d_arr[batch_start:batch_end, :, :] = d[:, :, :, 0]
end_time = time.time()
log('Total time: %.1f ms Average time per image: %.1f ms' %
(1000 * (end_time - start_time),
(1000 * (end_time - start_time) / n_sample)))
d_arr = d_arr[0:n_sample, :, :]
output_path = os.path.join(output_path, 'disparities.npy')
log('Storing predictions to %s' % output_path)
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
np.save(output_path, d_arr)
def train(trn_im0_path,
trn_im1_path,
n_epoch=settings.N_EPOCH,
n_batch=settings.N_BATCH,
n_height=settings.N_HEIGHT,
n_width=settings.N_WIDTH,
n_channel=settings.N_CHANNEL,
learning_rates=settings.LEARNING_RATES,
learning_bounds=settings.LEARNING_BOUNDS,
n_pyramid=settings.N_PYRAMID,
max_disparity=settings.MAX_DISPARITY,
w_ph=settings.W_PH,
w_st=settings.W_ST,
w_sm=settings.W_SM,
w_bc=settings.W_BC,
w_ar=settings.W_AR,
n_checkpoint=settings.N_CHECKPOINT,
n_summary=settings.N_SUMMARY,
checkpoint_path=settings.CHECKPOINT_PATH,
restore_path=settings.RESTORE_PATH,
n_gpu=settings.N_GPU,
n_thread=settings.N_THREAD):
event_path = os.path.join(checkpoint_path, 'event')
model_path = os.path.join(checkpoint_path, 'model.ckpt')
log_path = os.path.join(checkpoint_path, 'results.txt')
# Load image paths from paths file for training and validation
trn_im0_paths = data_utils.read_paths(trn_im0_path)
trn_im1_paths = data_utils.read_paths(trn_im1_path)
n_trn_sample = len(trn_im0_paths)
n_trn_step = n_epoch * np.ceil(n_trn_sample / n_batch).astype(np.int32)
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
# Initialize optimizer
boundaries = [np.int32(b * n_trn_step) for b in learning_bounds]
learning_rate = tf.train.piecewise_constant(global_step, boundaries,
learning_rates)
optimizer = tf.train.AdamOptimizer(learning_rate)
# Initialize dataloader
dataloader = DataLoader(shape=[n_batch, n_height, n_width, n_channel],
name='dataloader',
n_thread=n_thread,
prefetch_size=8,
normalize=True,
random_flip=True,
random_gamma=True,
gamma_range=[0.8, 1.2],
random_brightness=True,
brightness_range=[0.5, 2.0],
random_color=True,
color_range=[0.8, 1.2])
# Split data into towers for each GPU
im0_split = tf.split(dataloader.next_element[0], n_gpu, 0)
im1_split = tf.split(dataloader.next_element[1], n_gpu, 0)
# Build computation graph
tower_gradients = []
tower_losses = []
with tf.variable_scope(tf.get_variable_scope()):
for i in range(n_gpu):
with tf.device('/gpu:%d' % i):
params = []
model = MonoDispNet(im0_split[i],
im1_split[i],
n_pyramid=n_pyramid,
w_ph=w_ph,
w_st=w_st,
w_sm=w_sm,
w_bc=w_bc,
w_ar=w_ar,
max_disparity=max_disparity,
reuse_variables=tf.AUTO_REUSE,
model_index=i)
loss = model.total_loss
tower_losses.append(loss)
tower_gradients.append(optimizer.compute_gradients(loss))
# Set up gradient computations
avg_gradients = average_gradients(tower_gradients)
gradients = optimizer.apply_gradients(avg_gradients,
global_step=global_step)
total_loss = tf.reduce_mean(tower_losses)
tf.summary.scalar('learning_rate', learning_rate, ['model_0'])
tf.summary.scalar('total_loss', total_loss, ['model_0'])
trn_summary = tf.summary.merge_all('model_0')
# Count trainable parameters
n_parameter = 0
for variable in tf.trainable_variables():
n_parameter += np.array(variable.get_shape().as_list()).prod()
# Log network parameters
log('Network Parameters:', log_path)
log(
'n_batch=%d n_height=%d n_width=%d n_channel=%d ' %
(n_batch, n_height, n_width, n_channel), log_path)
log('n_pyramid=%d max_disparity=%.3f' % (n_pyramid, max_disparity),
log_path)
log(
'n_sample=%d n_epoch=%d n_step=%d n_param=%d' %
(n_trn_sample, n_epoch, n_trn_step, n_parameter), log_path)
log(
'learning_rates=[%s]' % ', '.join('{:.6f}'.format(r)
for r in learning_rates),
log_path)
log(
'boundaries=[%s]' %
', '.join('{:.2f}:{}'.format(l, b)
for l, b in zip(learning_bounds, boundaries)), log_path)
log(
'w_ph=%.3f w_st=%.3f w_sm=%.3f w_bc=%.3f w_ar=%.3f' %
(w_ph, w_st, w_sm, w_bc, w_ar), log_path)
log(
'Restoring from: %s' %
('None' if restore_path == '' else restore_path), log_path)
# Initialize Tensorflow session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
# Initialize saver for storing and restoring checkpoints
summary_writer = tf.summary.FileWriter(model_path, session.graph)
train_saver = tf.train.Saver()
# Initialize all variables
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
# If given, load the weights from the restore path
if restore_path != '':
train_saver.restore(session, restore_path)
# Begin training
log('Begin training...')
start_step = global_step.eval(session=session)
time_start = time.time()
trn_step = start_step
while trn_step < n_trn_step:
trn_im0_paths_epoch, trn_im1_paths_epoch = data_utils.shuffle_and_drop(
[trn_im0_paths, trn_im1_paths], n_batch)
dataloader.initialize(session,
im0_paths=trn_im0_paths_epoch,
im1_paths=trn_im1_paths_epoch,
augment=True)
while trn_step < n_trn_step:
try:
_, loss_value = session.run([gradients, total_loss])
if trn_step % n_summary == 0:
summary = session.run(trn_summary)
summary_writer.add_summary(summary,
global_step=trn_step)
if trn_step and trn_step % n_checkpoint == 0:
time_elapse = (time.time() -
time_start) / 3600 * trn_step / (
trn_step - start_step + 1)
time_remain = (n_trn_step / trn_step -
1.0) * time_elapse
checkpoint_log = 'batch {:>6} loss: {:.5f} time elapsed: {:.2f}h time left: {:.2f}h'
log(
checkpoint_log.format(trn_step, loss_value,
time_elapse, time_remain),
log_path)
train_saver.save(session,
model_path,
global_step=trn_step)
trn_step += 1
except tf.errors.OutOfRangeError:
break
train_saver.save(session, model_path, global_step=n_trn_step)
im_filter_pct=args.im_filter_pct,
sz_filter_pct=args.sz_filter_pct,
min_predict_z=args.min_predict_z,
max_predict_z=args.max_predict_z)
# Initialize Tensorflow session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
# Load from checkpoint
train_saver = tf.train.Saver()
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
train_saver.restore(session, args.restore_path)
log('Evaluating {}'.format(args.restore_path), log_path)
# Load image, dense depth, sparse depth, intrinsics, and ground-truth
dataloader.initialize(session,
image_paths=im_paths,
interp_depth_paths=iz_paths,
validity_map_paths=vm_paths)
z_arr = np.zeros([n_step * args.n_batch, args.n_height, args.n_width, 1])
step = 0
while True:
try:
sys.stdout.write('Processed {}/{} examples \r'.format(
step * args.n_batch, n_sample))
sys.stdout.flush()
batch_start = step * args.n_batch
"domain": domain,
"canonical_url": canonical_url,
"mozrank": mozrank,
"subdomain_mozrank": subdomain_mozrank,
"da": da,
"pa": pa,
"equity_links": equity_links,
"links": links,
"status_code": status_code,
"moz_time_last_crawled": moz_time_last_crawled
}
return result
for chunk in domain_chunks:
log('--- Starting request ---')
metrics = None
try:
metrics = client.urlMetrics(chunk)
except MozscapeError as e:
log('ERROR! : %s' % (e))
continue
results = []
for idx, domain in enumerate(chunk):
metric = metrics[idx]
result = get_result(metric)
log_domain(result)
results.append(result)
help='Percentage (or number) of test instances.',
required=True)
parser.add_argument('--seed',
type=int,
help='PRNG seed (default: 0).',
required=False,
default=0)
parser.add_argument('--out',
type=str,
help='Output file (.json).',
required=True)
args = parser.parse_args()
X, Y, _, Npages, Nloads = load_features(args.features)
log('Seed is {}'.format(args.seed))
n = len(X)
# Get training/test set size
if args.train > 1:
train_size = int(args.train)
else:
train_size = int(args.train * n)
if args.test > 1:
test_size = int(args.test)
else:
test_size = int(args.test * n)
log('Training set size: {}. Test set size: {}.'.format(
train_size, test_size))
if train_size + test_size != n:
import numpy as np
from data_utils import load_dataset, log
from attacks import ATTACKS
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Extract feature vectors')
parser.add_argument('--traces', type=str, help='Traces.', required=True)
parser.add_argument('--out',
type=str,
help='Output directory.',
required=True)
args = parser.parse_args()
X, Y, W, Npages, Nloads = load_dataset(args.traces)
attack = ATTACKS["CUMUL"]()
log('Extracting features')
X_f = attack.extract_features(X, Y)
log('Length of a feature vector: {}'.format(len(X_f)))
if not os.path.isdir(args.out):
log('Creating directory {}'.format(args.out))
os.makedirs(args.out)
log('Storing features into {}'.format(args.out))
for x, w in zip(X_f, W):
fname = os.path.join(args.out, w) + '.features'
np.savetxt(fname, x, delimiter=',')
def train(
train_image_path,
train_interp_depth_path,
train_validity_map_path,
train_intrinsics_path,
# Batch parameters
n_batch=settings.N_BATCH,
n_height=settings.N_HEIGHT,
n_width=settings.N_WIDTH,
n_channel=settings.N_CHANNEL,
# Training settings
n_epoch=settings.N_EPOCH,
learning_rates=settings.LEARNING_RATES,
learning_bounds=settings.LEARNING_BOUNDS,
# Weights on loss function
w_ph=settings.W_PH,
w_co=settings.W_CO,
w_st=settings.W_ST,
w_sm=settings.W_SM,
w_sz=settings.W_SZ,
w_pc=settings.W_PC,
# Network settings
occ_threshold=settings.OCC_THRESHOLD,
occ_ksize=settings.OCC_KSIZE,
net_type=settings.NET_TYPE,
im_filter_pct=settings.IM_FILTER_PCT,
sz_filter_pct=settings.SZ_FILTER_PCT,
min_predict_z=settings.MIN_Z,
max_predict_z=settings.MAX_Z,
# Pose parameterization
rot_param=settings.ROT_PARAM,
pose_norm=settings.POSE_NORM,
# Model checkpoints and hardware
n_checkpoint=settings.N_CHECKPOINT,
n_summary=settings.N_SUMMARY,
checkpoint_path=settings.CHECKPOINT_PATH,
restore_path=settings.RESTORE_PATH,
# Hardware settings
n_thread=settings.N_THREAD):
model_path = os.path.join(checkpoint_path, 'model.ckpt')
log_path = os.path.join(checkpoint_path, 'results.txt')
# Load image, instrinsics paths from file for training
train_im_paths = data_utils.read_paths(train_image_path)
train_iz_paths = data_utils.read_paths(train_interp_depth_path)
train_vm_paths = data_utils.read_paths(train_validity_map_path)
train_kin_paths = data_utils.read_paths(train_intrinsics_path)
assert (len(train_im_paths) == len(train_iz_paths))
assert (len(train_im_paths) == len(train_vm_paths))
assert (len(train_im_paths) == len(train_kin_paths))
n_train_sample = len(train_im_paths)
n_train_step = n_epoch * np.ceil(n_train_sample / n_batch).astype(np.int32)
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
# Initialize optimizer
boundaries = [
np.int32((float(v) / n_epoch) * n_train_step)
for v in learning_bounds
]
learning_rate = tf.train.piecewise_constant(global_step, boundaries,
learning_rates)
optimizer = tf.train.AdamOptimizer(learning_rate)
# Initialize dataloader
dataloader = DataLoader(shape=[n_batch, n_height, n_width, n_channel],
normalize=True,
name='dataloader',
n_thread=n_thread,
prefetch_size=2 * n_thread)
# Fetch the input from dataloader
im0 = dataloader.next_element[0]
im1 = dataloader.next_element[1]
im2 = dataloader.next_element[2]
sz0 = dataloader.next_element[3]
kin = dataloader.next_element[4]
# Build computation graph
model = VOICEDModel(im0,
im1,
im2,
sz0,
kin,
is_training=True,
occ_threshold=occ_threshold,
occ_ksize=occ_ksize,
net_type=net_type,
im_filter_pct=im_filter_pct,
sz_filter_pct=sz_filter_pct,
min_predict_z=min_predict_z,
max_predict_z=max_predict_z,
rot_param=rot_param,
pose_norm=pose_norm,
w_ph=w_ph,
w_co=w_co,
w_st=w_st,
w_sm=w_sm,
w_sz=w_sz,
w_pc=w_pc)
loss = model.loss
gradients = optimizer.compute_gradients(loss)
gradients = optimizer.apply_gradients(gradients,
global_step=global_step)
model_summary = tf.summary.merge_all()
# Count trainable parameters
n_parameter = 0
for variable in tf.trainable_variables():
n_parameter += np.array(variable.get_shape().as_list()).prod()
# Log network parameters
log('Network Parameters:', log_path)
log(
'n_batch=%d n_height=%d n_width=%d n_channel=%d' %
(n_batch, n_height, n_width, n_channel), log_path)
log(
'n_sample=%d n_epoch=%d n_step=%d n_param=%d ' %
(n_train_sample, n_epoch, n_train_step, n_parameter), log_path)
log(
'net_type=%s im_filter_pct=%.3f sz_filter_pct=%.3f' %
(net_type, im_filter_pct, sz_filter_pct), log_path)
log('occ_threshold=%.2f occ_ksize=%d' % (occ_threshold, occ_ksize),
log_path)
log('rot_param=%s pose_norm=%s' % (rot_param, pose_norm), log_path)
log(
'min_predict_z=%.3f max_predict_z=%.3f' %
(min_predict_z, max_predict_z), log_path)
log(
'learning_rates=[%s]' % ', '.join('{:.6f}'.format(r)
for r in learning_rates),
log_path)
log(
'boundaries=[%s]' %
', '.join('{}:{}'.format(l, v)
for l, v in zip(learning_bounds, boundaries)), log_path)
log(
'w_ph=%.3f w_co=%.3f w_st=%.3f w_sm=%.3f w_sz=%.3f w_pc=%.3f'
% (w_ph, w_co, w_st, w_sm, w_sz, w_pc), log_path)
log(
'Restoring from: %s' %
('None' if restore_path == '' else restore_path), log_path)
# Initialize Tensorflow session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
# Initialize saver for storing and restoring checkpoints
summary_writer = tf.summary.FileWriter(model_path + '-train',
session.graph)
train_saver = tf.train.Saver()
# Initialize all variables
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
# If given, load the weights from the restore path
if restore_path != '':
train_saver.restore(session, restore_path)
# Begin training
log('Begin training...', log_path)
start_step = global_step.eval(session=session)
time_start = time.time()
train_step = start_step
step = 0
train_im_paths_epoch, train_iz_paths_epoch, \
train_vm_paths_epoch, train_kin_paths_epoch = data_utils.make_epoch(
input_arr=[
train_im_paths, train_iz_paths, train_vm_paths, train_kin_paths],
n_batch=n_batch)
dataloader.initialize(session,
image_paths=train_im_paths_epoch,
interp_depth_paths=train_iz_paths_epoch,
validity_map_paths=train_vm_paths_epoch,
intrinsics_paths=train_kin_paths_epoch,
do_crop=True)
while train_step < n_train_step:
try:
if train_step % n_summary == 0:
_, loss_value, summary = session.run(
[gradients, loss, model_summary])
summary_writer.add_summary(summary, global_step=train_step)
else:
_, loss_value = session.run([gradients, loss])
if train_step and (train_step % n_checkpoint) == 0:
time_elapse = (time.time() -
time_start) / 3600 * train_step / (
train_step - start_step + 1)
time_remain = (n_train_step / train_step - 1) * time_elapse
checkpoint_log = 'batch {:>6} loss: {:.5f} time elapsed: {:.2f}h time left: {:.2f}h'
log(
checkpoint_log.format(train_step, loss_value,
time_elapse, time_remain),
log_path)
train_saver.save(session,
model_path,
global_step=train_step)
train_step += 1
step += 1
except tf.errors.OutOfRangeError:
step = 0
train_im_paths_epoch, train_iz_paths_epoch, \
train_vm_paths_epoch, train_kin_paths_epoch = data_utils.make_epoch(
input_arr=[
train_im_paths, train_iz_paths, train_vm_paths, train_kin_paths],
n_batch=n_batch)
dataloader.initialize(session,
image_paths=train_im_paths_epoch,
interp_depth_paths=train_iz_paths_epoch,
validity_map_paths=train_vm_paths_epoch,
intrinsics_paths=train_kin_paths_epoch,
do_crop=True)
train_saver.save(session, model_path, global_step=n_train_step)