def evaluate(im0_path, restore_path, output_path, n_batch=settings.N_BATCH, n_height=settings.N_HEIGHT, n_width=settings.N_WIDTH, n_channel=settings.N_CHANNEL, n_pyramid=settings.N_PYRAMID, max_disparity=settings.MAX_DISPARITY, n_gpu=settings.N_GPU, n_thread=settings.N_THREAD): """Test function.""" # Create dataloader for computation graph dataloader = DataLoader(shape=[n_batch, n_height, n_width, n_channel], name='dataloader', n_thread=n_thread, prefetch_size=n_thread, normalize=True, random_flip=False, random_gamma=False, gamma_range=[0.8, 1.2], random_brightness=False, brightness_range=[0.5, 2.0], random_color=False, color_range=[0.8, 1.2]) # Build model model = MonoDispNet(dataloader.next_element[0], dataloader.next_element[1], n_pyramid=n_pyramid, max_disparity=max_disparity) # Start a Tensorflow session config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True session = tf.Session(config=config) # Initialize saver that will be used for restore train_saver = tf.train.Saver() # Initialize all variables session.run(tf.global_variables_initializer()) session.run(tf.local_variables_initializer()) # Restore weights from checkpoint log('Restoring from: %s' % restore_path) train_saver.restore(session, restore_path) # Load the files for evaluation im0_paths = data_utils.read_paths(im0_path) n_sample = len(im0_paths) im0_paths = data_utils.pad_batch(im0_paths, n_batch) n_step = len(im0_paths) // n_batch log('Evaluating %d files...' % n_sample) dataloader.initialize(session, im0_paths=im0_paths, im1_paths=im0_paths, augment=False) d_arr = np.zeros((n_step * n_batch, n_height, n_width), dtype=np.float32) start_time = time.time() for step in range(n_step): batch_start = step * n_batch batch_end = step * n_batch + n_batch d = session.run(model.model0[0]) d_arr[batch_start:batch_end, :, :] = d[:, :, :, 0] end_time = time.time() log('Total time: %.1f ms Average time per image: %.1f ms' % (1000 * (end_time - start_time), (1000 * (end_time - start_time) / n_sample))) d_arr = d_arr[0:n_sample, :, :] output_path = os.path.join(output_path, 'disparities.npy') log('Storing predictions to %s' % output_path) if not os.path.exists(os.path.dirname(output_path)): os.makedirs(os.path.dirname(output_path)) np.save(output_path, d_arr)
def train(trn_im0_path, trn_im1_path, n_epoch=settings.N_EPOCH, n_batch=settings.N_BATCH, n_height=settings.N_HEIGHT, n_width=settings.N_WIDTH, n_channel=settings.N_CHANNEL, learning_rates=settings.LEARNING_RATES, learning_bounds=settings.LEARNING_BOUNDS, n_pyramid=settings.N_PYRAMID, max_disparity=settings.MAX_DISPARITY, w_ph=settings.W_PH, w_st=settings.W_ST, w_sm=settings.W_SM, w_bc=settings.W_BC, w_ar=settings.W_AR, n_checkpoint=settings.N_CHECKPOINT, n_summary=settings.N_SUMMARY, checkpoint_path=settings.CHECKPOINT_PATH, restore_path=settings.RESTORE_PATH, n_gpu=settings.N_GPU, n_thread=settings.N_THREAD): event_path = os.path.join(checkpoint_path, 'event') model_path = os.path.join(checkpoint_path, 'model.ckpt') log_path = os.path.join(checkpoint_path, 'results.txt') # Load image paths from paths file for training and validation trn_im0_paths = data_utils.read_paths(trn_im0_path) trn_im1_paths = data_utils.read_paths(trn_im1_path) n_trn_sample = len(trn_im0_paths) n_trn_step = n_epoch * np.ceil(n_trn_sample / n_batch).astype(np.int32) with tf.Graph().as_default(): global_step = tf.Variable(0, trainable=False) # Initialize optimizer boundaries = [np.int32(b * n_trn_step) for b in learning_bounds] learning_rate = tf.train.piecewise_constant(global_step, boundaries, learning_rates) optimizer = tf.train.AdamOptimizer(learning_rate) # Initialize dataloader dataloader = DataLoader(shape=[n_batch, n_height, n_width, n_channel], name='dataloader', n_thread=n_thread, prefetch_size=8, normalize=True, random_flip=True, random_gamma=True, gamma_range=[0.8, 1.2], random_brightness=True, brightness_range=[0.5, 2.0], random_color=True, color_range=[0.8, 1.2]) # Split data into towers for each GPU im0_split = tf.split(dataloader.next_element[0], n_gpu, 0) im1_split = tf.split(dataloader.next_element[1], n_gpu, 0) # Build computation graph tower_gradients = [] tower_losses = [] with tf.variable_scope(tf.get_variable_scope()): for i in range(n_gpu): with tf.device('/gpu:%d' % i): params = [] model = MonoDispNet(im0_split[i], im1_split[i], n_pyramid=n_pyramid, w_ph=w_ph, w_st=w_st, w_sm=w_sm, w_bc=w_bc, w_ar=w_ar, max_disparity=max_disparity, reuse_variables=tf.AUTO_REUSE, model_index=i) loss = model.total_loss tower_losses.append(loss) tower_gradients.append(optimizer.compute_gradients(loss)) # Set up gradient computations avg_gradients = average_gradients(tower_gradients) gradients = optimizer.apply_gradients(avg_gradients, global_step=global_step) total_loss = tf.reduce_mean(tower_losses) tf.summary.scalar('learning_rate', learning_rate, ['model_0']) tf.summary.scalar('total_loss', total_loss, ['model_0']) trn_summary = tf.summary.merge_all('model_0') # Count trainable parameters n_parameter = 0 for variable in tf.trainable_variables(): n_parameter += np.array(variable.get_shape().as_list()).prod() # Log network parameters log('Network Parameters:', log_path) log( 'n_batch=%d n_height=%d n_width=%d n_channel=%d ' % (n_batch, n_height, n_width, n_channel), log_path) log('n_pyramid=%d max_disparity=%.3f' % (n_pyramid, max_disparity), log_path) log( 'n_sample=%d n_epoch=%d n_step=%d n_param=%d' % (n_trn_sample, n_epoch, n_trn_step, n_parameter), log_path) log( 'learning_rates=[%s]' % ', '.join('{:.6f}'.format(r) for r in learning_rates), log_path) log( 'boundaries=[%s]' % ', '.join('{:.2f}:{}'.format(l, b) for l, b in zip(learning_bounds, boundaries)), log_path) log( 'w_ph=%.3f w_st=%.3f w_sm=%.3f w_bc=%.3f w_ar=%.3f' % (w_ph, w_st, w_sm, w_bc, w_ar), log_path) log( 'Restoring from: %s' % ('None' if restore_path == '' else restore_path), log_path) # Initialize Tensorflow session config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True session = tf.Session(config=config) # Initialize saver for storing and restoring checkpoints summary_writer = tf.summary.FileWriter(model_path, session.graph) train_saver = tf.train.Saver() # Initialize all variables session.run(tf.global_variables_initializer()) session.run(tf.local_variables_initializer()) # If given, load the weights from the restore path if restore_path != '': train_saver.restore(session, restore_path) # Begin training log('Begin training...') start_step = global_step.eval(session=session) time_start = time.time() trn_step = start_step while trn_step < n_trn_step: trn_im0_paths_epoch, trn_im1_paths_epoch = data_utils.shuffle_and_drop( [trn_im0_paths, trn_im1_paths], n_batch) dataloader.initialize(session, im0_paths=trn_im0_paths_epoch, im1_paths=trn_im1_paths_epoch, augment=True) while trn_step < n_trn_step: try: _, loss_value = session.run([gradients, total_loss]) if trn_step % n_summary == 0: summary = session.run(trn_summary) summary_writer.add_summary(summary, global_step=trn_step) if trn_step and trn_step % n_checkpoint == 0: time_elapse = (time.time() - time_start) / 3600 * trn_step / ( trn_step - start_step + 1) time_remain = (n_trn_step / trn_step - 1.0) * time_elapse checkpoint_log = 'batch {:>6} loss: {:.5f} time elapsed: {:.2f}h time left: {:.2f}h' log( checkpoint_log.format(trn_step, loss_value, time_elapse, time_remain), log_path) train_saver.save(session, model_path, global_step=trn_step) trn_step += 1 except tf.errors.OutOfRangeError: break train_saver.save(session, model_path, global_step=n_trn_step)
im_filter_pct=args.im_filter_pct, sz_filter_pct=args.sz_filter_pct, min_predict_z=args.min_predict_z, max_predict_z=args.max_predict_z) # Initialize Tensorflow session config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True session = tf.Session(config=config) # Load from checkpoint train_saver = tf.train.Saver() session.run(tf.global_variables_initializer()) session.run(tf.local_variables_initializer()) train_saver.restore(session, args.restore_path) log('Evaluating {}'.format(args.restore_path), log_path) # Load image, dense depth, sparse depth, intrinsics, and ground-truth dataloader.initialize(session, image_paths=im_paths, interp_depth_paths=iz_paths, validity_map_paths=vm_paths) z_arr = np.zeros([n_step * args.n_batch, args.n_height, args.n_width, 1]) step = 0 while True: try: sys.stdout.write('Processed {}/{} examples \r'.format( step * args.n_batch, n_sample)) sys.stdout.flush() batch_start = step * args.n_batch
"domain": domain, "canonical_url": canonical_url, "mozrank": mozrank, "subdomain_mozrank": subdomain_mozrank, "da": da, "pa": pa, "equity_links": equity_links, "links": links, "status_code": status_code, "moz_time_last_crawled": moz_time_last_crawled } return result for chunk in domain_chunks: log('--- Starting request ---') metrics = None try: metrics = client.urlMetrics(chunk) except MozscapeError as e: log('ERROR! : %s' % (e)) continue results = [] for idx, domain in enumerate(chunk): metric = metrics[idx] result = get_result(metric) log_domain(result) results.append(result)
help='Percentage (or number) of test instances.', required=True) parser.add_argument('--seed', type=int, help='PRNG seed (default: 0).', required=False, default=0) parser.add_argument('--out', type=str, help='Output file (.json).', required=True) args = parser.parse_args() X, Y, _, Npages, Nloads = load_features(args.features) log('Seed is {}'.format(args.seed)) n = len(X) # Get training/test set size if args.train > 1: train_size = int(args.train) else: train_size = int(args.train * n) if args.test > 1: test_size = int(args.test) else: test_size = int(args.test * n) log('Training set size: {}. Test set size: {}.'.format( train_size, test_size)) if train_size + test_size != n:
import numpy as np from data_utils import load_dataset, log from attacks import ATTACKS if __name__ == '__main__': parser = argparse.ArgumentParser(description='Extract feature vectors') parser.add_argument('--traces', type=str, help='Traces.', required=True) parser.add_argument('--out', type=str, help='Output directory.', required=True) args = parser.parse_args() X, Y, W, Npages, Nloads = load_dataset(args.traces) attack = ATTACKS["CUMUL"]() log('Extracting features') X_f = attack.extract_features(X, Y) log('Length of a feature vector: {}'.format(len(X_f))) if not os.path.isdir(args.out): log('Creating directory {}'.format(args.out)) os.makedirs(args.out) log('Storing features into {}'.format(args.out)) for x, w in zip(X_f, W): fname = os.path.join(args.out, w) + '.features' np.savetxt(fname, x, delimiter=',')
def train( train_image_path, train_interp_depth_path, train_validity_map_path, train_intrinsics_path, # Batch parameters n_batch=settings.N_BATCH, n_height=settings.N_HEIGHT, n_width=settings.N_WIDTH, n_channel=settings.N_CHANNEL, # Training settings n_epoch=settings.N_EPOCH, learning_rates=settings.LEARNING_RATES, learning_bounds=settings.LEARNING_BOUNDS, # Weights on loss function w_ph=settings.W_PH, w_co=settings.W_CO, w_st=settings.W_ST, w_sm=settings.W_SM, w_sz=settings.W_SZ, w_pc=settings.W_PC, # Network settings occ_threshold=settings.OCC_THRESHOLD, occ_ksize=settings.OCC_KSIZE, net_type=settings.NET_TYPE, im_filter_pct=settings.IM_FILTER_PCT, sz_filter_pct=settings.SZ_FILTER_PCT, min_predict_z=settings.MIN_Z, max_predict_z=settings.MAX_Z, # Pose parameterization rot_param=settings.ROT_PARAM, pose_norm=settings.POSE_NORM, # Model checkpoints and hardware n_checkpoint=settings.N_CHECKPOINT, n_summary=settings.N_SUMMARY, checkpoint_path=settings.CHECKPOINT_PATH, restore_path=settings.RESTORE_PATH, # Hardware settings n_thread=settings.N_THREAD): model_path = os.path.join(checkpoint_path, 'model.ckpt') log_path = os.path.join(checkpoint_path, 'results.txt') # Load image, instrinsics paths from file for training train_im_paths = data_utils.read_paths(train_image_path) train_iz_paths = data_utils.read_paths(train_interp_depth_path) train_vm_paths = data_utils.read_paths(train_validity_map_path) train_kin_paths = data_utils.read_paths(train_intrinsics_path) assert (len(train_im_paths) == len(train_iz_paths)) assert (len(train_im_paths) == len(train_vm_paths)) assert (len(train_im_paths) == len(train_kin_paths)) n_train_sample = len(train_im_paths) n_train_step = n_epoch * np.ceil(n_train_sample / n_batch).astype(np.int32) with tf.Graph().as_default(): global_step = tf.Variable(0, trainable=False) # Initialize optimizer boundaries = [ np.int32((float(v) / n_epoch) * n_train_step) for v in learning_bounds ] learning_rate = tf.train.piecewise_constant(global_step, boundaries, learning_rates) optimizer = tf.train.AdamOptimizer(learning_rate) # Initialize dataloader dataloader = DataLoader(shape=[n_batch, n_height, n_width, n_channel], normalize=True, name='dataloader', n_thread=n_thread, prefetch_size=2 * n_thread) # Fetch the input from dataloader im0 = dataloader.next_element[0] im1 = dataloader.next_element[1] im2 = dataloader.next_element[2] sz0 = dataloader.next_element[3] kin = dataloader.next_element[4] # Build computation graph model = VOICEDModel(im0, im1, im2, sz0, kin, is_training=True, occ_threshold=occ_threshold, occ_ksize=occ_ksize, net_type=net_type, im_filter_pct=im_filter_pct, sz_filter_pct=sz_filter_pct, min_predict_z=min_predict_z, max_predict_z=max_predict_z, rot_param=rot_param, pose_norm=pose_norm, w_ph=w_ph, w_co=w_co, w_st=w_st, w_sm=w_sm, w_sz=w_sz, w_pc=w_pc) loss = model.loss gradients = optimizer.compute_gradients(loss) gradients = optimizer.apply_gradients(gradients, global_step=global_step) model_summary = tf.summary.merge_all() # Count trainable parameters n_parameter = 0 for variable in tf.trainable_variables(): n_parameter += np.array(variable.get_shape().as_list()).prod() # Log network parameters log('Network Parameters:', log_path) log( 'n_batch=%d n_height=%d n_width=%d n_channel=%d' % (n_batch, n_height, n_width, n_channel), log_path) log( 'n_sample=%d n_epoch=%d n_step=%d n_param=%d ' % (n_train_sample, n_epoch, n_train_step, n_parameter), log_path) log( 'net_type=%s im_filter_pct=%.3f sz_filter_pct=%.3f' % (net_type, im_filter_pct, sz_filter_pct), log_path) log('occ_threshold=%.2f occ_ksize=%d' % (occ_threshold, occ_ksize), log_path) log('rot_param=%s pose_norm=%s' % (rot_param, pose_norm), log_path) log( 'min_predict_z=%.3f max_predict_z=%.3f' % (min_predict_z, max_predict_z), log_path) log( 'learning_rates=[%s]' % ', '.join('{:.6f}'.format(r) for r in learning_rates), log_path) log( 'boundaries=[%s]' % ', '.join('{}:{}'.format(l, v) for l, v in zip(learning_bounds, boundaries)), log_path) log( 'w_ph=%.3f w_co=%.3f w_st=%.3f w_sm=%.3f w_sz=%.3f w_pc=%.3f' % (w_ph, w_co, w_st, w_sm, w_sz, w_pc), log_path) log( 'Restoring from: %s' % ('None' if restore_path == '' else restore_path), log_path) # Initialize Tensorflow session config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True session = tf.Session(config=config) # Initialize saver for storing and restoring checkpoints summary_writer = tf.summary.FileWriter(model_path + '-train', session.graph) train_saver = tf.train.Saver() # Initialize all variables session.run(tf.global_variables_initializer()) session.run(tf.local_variables_initializer()) # If given, load the weights from the restore path if restore_path != '': train_saver.restore(session, restore_path) # Begin training log('Begin training...', log_path) start_step = global_step.eval(session=session) time_start = time.time() train_step = start_step step = 0 train_im_paths_epoch, train_iz_paths_epoch, \ train_vm_paths_epoch, train_kin_paths_epoch = data_utils.make_epoch( input_arr=[ train_im_paths, train_iz_paths, train_vm_paths, train_kin_paths], n_batch=n_batch) dataloader.initialize(session, image_paths=train_im_paths_epoch, interp_depth_paths=train_iz_paths_epoch, validity_map_paths=train_vm_paths_epoch, intrinsics_paths=train_kin_paths_epoch, do_crop=True) while train_step < n_train_step: try: if train_step % n_summary == 0: _, loss_value, summary = session.run( [gradients, loss, model_summary]) summary_writer.add_summary(summary, global_step=train_step) else: _, loss_value = session.run([gradients, loss]) if train_step and (train_step % n_checkpoint) == 0: time_elapse = (time.time() - time_start) / 3600 * train_step / ( train_step - start_step + 1) time_remain = (n_train_step / train_step - 1) * time_elapse checkpoint_log = 'batch {:>6} loss: {:.5f} time elapsed: {:.2f}h time left: {:.2f}h' log( checkpoint_log.format(train_step, loss_value, time_elapse, time_remain), log_path) train_saver.save(session, model_path, global_step=train_step) train_step += 1 step += 1 except tf.errors.OutOfRangeError: step = 0 train_im_paths_epoch, train_iz_paths_epoch, \ train_vm_paths_epoch, train_kin_paths_epoch = data_utils.make_epoch( input_arr=[ train_im_paths, train_iz_paths, train_vm_paths, train_kin_paths], n_batch=n_batch) dataloader.initialize(session, image_paths=train_im_paths_epoch, interp_depth_paths=train_iz_paths_epoch, validity_map_paths=train_vm_paths_epoch, intrinsics_paths=train_kin_paths_epoch, do_crop=True) train_saver.save(session, model_path, global_step=n_train_step)