def run_training( cfg ):
# set up logging
tf.logging.set_verbosity( tf.logging.INFO )
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = utils.setup_input( cfg, is_training=False, use_filename_queue=True )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = setup_model( inputs, cfg, is_training=False )
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics( inputs, model, cfg )
# execute training
start_time = time.time()
utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() }
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=True )
training_runners[ 'threads' ] = data_prefetch_init_fn( training_runners[ 'sess' ], training_runners[ 'coord' ] )
try:
# This just returns the imput as output. It is for testing data
# input only.
for step in xrange( inputs[ 'max_steps' ] ):
input_batch, target_batch, data_idx = training_runners['sess'].run( [
model['input_batch'], model['target_batch'], model[ 'data_idxs' ] ] )
if training_runners['coord'].should_stop():
break
finally:
utils.request_data_loading_end( training_runners )
utils.end_data_loading_and_sess( training_runners )
# else: # Use tf.slim
# train_log_dir = os.path.join( cfg['log_dir'], 'slim-train' )
# # When ready to use a model, use the code below
# train( model[ 'train_op' ],
# train_log_dir,
# get_data_prefetch_threads_init_fn( inputs, cfg ),
# global_step=model[ 'global_step' ],
# number_of_steps=inputs[ 'max_steps' ],
# init_fn=model[ 'init_fn' ],
# save_summaries_secs=300,
# save_interval_secs=600,
# saver=model[ 'saver_op' ] )
end_train_time = time.time() - start_time
print('time to train %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time/(60*60)))
print('avg time per epoch: %.3f hrs' % ( (end_train_time/(60*60)) / cfg['num_epochs']) )
def generate_cfg(task):
repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
CONFIG_DIR = os.path.join(repo_dir, 'experiments/final', task)
############## Load Configs ##############
import utils
import data.load_ops as load_ops
from general_utils import RuntimeDeterminedEnviromentVars
cfg = utils.load_config( CONFIG_DIR, nopause=True )
RuntimeDeterminedEnviromentVars.register_dict( cfg )
cfg['batch_size'] = 1
if 'batch_size' in cfg['encoder_kwargs']:
cfg['encoder_kwargs']['batch_size'] = 1
cfg['model_path'] = os.path.join( repo_dir, 'temp', task, 'model.permanent-ckpt' )
cfg['root_dir'] = repo_dir
return cfg
def generate_cfg(task):
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{}'.format(
task)
############## Load Configs ##############
import utils
import data.load_ops as load_ops
from general_utils import RuntimeDeterminedEnviromentVars
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
root_dir = cfg['root_dir']
cfg['batch_size'] = 1
if 'batch_size' in cfg['encoder_kwargs']:
cfg['encoder_kwargs']['batch_size'] = 1
cfg['model_path'] = os.path.join('/home/ubuntu/temp', task,
'model.permanent-ckpt')
return cfg
def run_training( cfg, cfg_dir ):
# set up logging
tf.logging.set_verbosity( tf.logging.INFO )
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = utils.setup_input_transfer( cfg, is_training=True )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model_chained_transfer( inputs, cfg, is_training=True )
# execute training
start_time = time.time()
utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=True )
train_log_dir = os.path.join( cfg['log_dir'], 'slim-train' )
permanent_checkpoint_dir = os.path.join( cfg['log_dir'], 'checkpoints' )
session_config = tf.ConfigProto()
session_config.gpu_options.allow_growth = True
# When ready to use a model, use the code below
train( model[ 'train_op' ],
train_log_dir,
utils.get_data_prefetch_threads_init_fn_transfer( inputs, cfg, is_training=True ),
train_step_fn=model[ 'train_step_fn' ],
train_step_kwargs=model[ 'train_step_kwargs' ],
global_step=model[ 'global_step' ],
number_of_steps=inputs[ 'max_steps' ],
number_of_epochs=cfg['num_epochs'],
init_fn=model[ 'init_fn' ],
save_checkpoint_every=inputs['max_steps'] // (cfg['num_epochs'] * 2),
cfg_dir=cfg_dir,
#RuntimeDeterminedEnviromentVars.steps_per_epoch,
permanent_checkpoint_dir=permanent_checkpoint_dir,
save_summaries_secs=cfg['summary_save_every_secs'],
save_interval_secs=cfg['checkpoint_save_every_secs'],
saver=model[ 'saver_op' ],
return_accuracy= 'return_accuracy' in cfg and cfg['return_accuracy'],
session_config=session_config )
end_train_time = time.time() - start_time
print('time to train %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time/(60*60)))
print('avg time per epoch: %.3f hrs' % ( (end_train_time/(60*60)) / cfg['num_epochs']) )
def get_cfg(nopause=False):
cfg = {}
cfg['is_discriminative'] = True
# params
cfg['num_epochs'] = 30
cfg['model_path'] = None
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
# model
cfg['model_type'] = architectures.BasicFF
cfg['weight_decay'] = 1e-5
cfg['instance_noise_sigma'] = 0.1
cfg['instance_noise_anneal_fn'] = tf.train.inverse_time_decay
cfg['instance_noise_anneal_fn_kwargs'] = {
'decay_rate': 0.2,
'decay_steps': 1000
}
batch_size = 32
cfg['encoder'] = resnet_encoder_fully_convolutional_16x16x8
cfg['hidden_size'] = 1024
cfg['encoder_kwargs'] = {
'resnet_build_fn': resnet_v1_50_16x16,
'weight_decay': cfg['weight_decay'],
'flatten': True,
'batch_size': batch_size
}
cfg['return_accuracy'] = True
# learning
cfg['initial_learning_rate'] = 1e-4
cfg['optimizer'] = tf.train.AdamOptimizer
cfg['optimizer_kwargs'] = {}
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(80000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# inputs
cfg['input_dim'] = (256, 256) # (1024, 1024)
cfg['input_num_channels'] = 3
cfg['input_dtype'] = tf.float32
cfg['input_domain_name'] = 'rgb'
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image
cfg['input_preprocessing_fn_kwargs'] = {
'new_dims': cfg['input_dim'],
'new_scale': [-1, 1]
}
cfg['single_filename_to_multiple'] = True
# outputs
cfg['target_dim'] = 63 # (1024, 1024)
cfg['target_dtype'] = tf.float32
cfg['target_from_filenames'] = load_ops.class_places_workspace_and_home
cfg['mask_by_target_func'] = True
# Post processing
cfg['metric_net'] = encoder_multilayers_fc_bn
cfg['metric_kwargs'] = {
'hidden_size': 2048,
'layer_num': 2,
'output_size': cfg['target_dim']
}
# input pipeline
cfg['preprocess_fn'] = load_and_specify_preprocessors
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = batch_size
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 600
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def run_to_task(args):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
tf.logging.set_verbosity(tf.logging.ERROR)
img = load_raw_image_center_crop(args.im_name)
img = skimage.img_as_float(img)
scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(args.im_name)
task = args.task
if task not in list_of_tasks:
raise ValueError('Task not supported')
cfg = generate_cfg(task)
# Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
segment25d segment2d room_layout'.split()
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
if task == 'jigsaw':
img = cfg['input_preprocessing_fn'](
img,
target=cfg['target_dict'][random.randint(0, 99)],
**cfg['input_preprocessing_fn_kwargs'])
else:
img = cfg['input_preprocessing_fn'](
img, **cfg['input_preprocessing_fn_kwargs'])
img = img[np.newaxis, :]
if task == 'class_places' or task == 'class_1000':
synset = get_synset(task)
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
predicted, representation = training_runners['sess'].run(
[m.decoder_output, m.encoder_endpoints[args.encoder_layer]],
feed_dict={m.input_images: img})
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
if args.compress_rep:
representation = tf.keras.layers.AveragePooling2D(
pool_size=(2, 2))(tf.constant(representation))
representation = training_runners['sess'].run(representation)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(representation))
if args.store_pred:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}_pred.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(predicted))
if task == 'segment2d' or task == 'segment25d':
segmentation_pca(predicted, args.store_name)
return
if task == 'colorization':
single_img_colorize(predicted, img, args.store_name)
return
if task == 'curvature':
curvature_single_image(predicted, args.store_name)
return
just_rescale = [
'autoencoder', 'denoise', 'edge2d', 'edge3d', 'keypoint2d',
'keypoint3d', 'reshade', 'rgb2sfnorm'
]
if task in just_rescale:
simple_rescale_img(predicted, args.store_name)
return
just_clip = ['rgb2depth', 'rgb2mist']
if task in just_clip:
depth_single_image(predicted, args.store_name)
return
if task == 'inpainting_whole':
inpainting_bbox(predicted, args.store_name)
return
if task == 'segmentsemantic':
semseg_single_image(predicted, img, args.store_name)
return
if task in ['class_1000', 'class_places']:
classification(predicted, synset, args.store_name)
return
if task == 'vanishing_point':
_ = plot_vanishing_point_smoothed(np.squeeze(predicted),
(np.squeeze(img) + 1.) / 2.,
args.store_name, [])
return
if task == 'room_layout':
mean = np.array([
0.006072743318127848, 0.010272365569691076, -3.135909774145468,
1.5603802322235532, 5.6228218371102496e-05, -1.5669352793761442,
5.622875878174759, 4.082800262277375, 2.7713941642895956
])
std = np.array([
0.8669452525283652, 0.687915294956501, 2.080513632043758,
0.19627420479282623, 0.014680602791251812, 0.4183827359302299,
3.991778013006544, 2.703495278378409, 1.2269185938626304
])
predicted = predicted * std + mean
plot_room_layout(np.squeeze(predicted), (np.squeeze(img) + 1.) / 2.,
args.store_name, [],
cube_only=True)
return
if task == 'jigsaw':
predicted = np.argmax(predicted, axis=1)
perm = cfg['target_dict'][predicted[0]]
show_jigsaw((np.squeeze(img) + 1.) / 2., perm, args.store_name)
return
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
print("Done: {}".format(config_name))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def get_cfg(nopause=False):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
import pdb
global INPUT_TASK
INPUT_TASK = INPUT_TASK.split(',')
representation_task = INPUT_TASK
cfg['multiple_input_tasks'] = len(INPUT_TASK)
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
cfg['retrain_decoder'] = ("<RETRAIN_DECODER>" == "True")
cfg['unlock_decoder'] = ("<UNLOCK_DECODER>" == "True")
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = [
'/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(CFG_DIR, t)
for t in INPUT_TASK
]
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = [
general_utils.load_config(inp_cfg, nopause=True)
for inp_cfg in cfg['config_dir_input']
]
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'],
nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath(
os.path.join('/home/ubuntu/s3',
'meta/val_image_split_{}.npy'.format(VAL_DATA_USED)))
cfg['val_list_of_fileinfos'] = os.path.abspath(
os.path.join('/home/ubuntu/s3', 'meta/test_image_split_0.npy'))
# Define where the extracted representations are stored
cfg['train_representations_file'] = [
os.path.join(
input_cfg[i]['log_root'], t,
'{task}_{train_split}_representations.pkl'.format(
task=t, train_split='train' if transitive else 'val'))
for i, t in enumerate(representation_task)
]
cfg['val_representations_file'] = [
os.path.join(
input_cfg[i]['log_root'], t,
'{task}_{train_split}_representations.pkl'.format(
task=t, train_split='val' if transitive else 'test'))
for i, t in enumerate(representation_task)
]
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
if TARGET_TASK in ['ego_motion', 'fix_pose', 'point_match', 'non_fixated_pose'] \
and VAL_DATA_USED not in ['16k','30k']:
target_cfg['train_filenames'] = target_cfg['train_filenames'].replace(
'val_split_image_info_', 'val_split_image_info_multi_in_')
cfg['train_list_of_fileinfos'] = cfg[
'train_list_of_fileinfos'].replace('val_image_split_',
'val_image_split_multi_in_')
target_cfg['train_filenames'] = os.path.abspath(
os.path.join('/home/ubuntu/s3',
'meta/val_split_image_info_{}.pkl'.format(VAL_DATA_USED)))
if TARGET_TASK == 'ego_motion':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_fixated_trips_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'fix_pose':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_fixated_pairs_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'point_match':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir, 'assets/aws_data/val_point_match_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'non_fixated_pose':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_nonfixated_pairs_info_{}.pkl'.
format(VAL_DATA_USED)))
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
if VAL_DATA_USED == '1k':
cfg['num_epochs'] = 200
elif VAL_DATA_USED == '2k':
cfg['num_epochs'] = 100
elif VAL_DATA_USED == '4k':
cfg['num_epochs'] = 50
elif VAL_DATA_USED == '8k':
cfg['num_epochs'] = 50
elif VAL_DATA_USED == '5h':
cfg['num_epochs'] = 400
elif VAL_DATA_USED == '1h':
cfg['num_epochs'] = 1000
elif VAL_DATA_USED == '16k':
cfg['num_epochs'] = 30
else:
cfg['num_epochs'] = 15
#cfg['num_epochs'] = 15 if not cfg['finetune_decoder'] else 7 if not cfg['finetune_decoder'] else 7
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = [16, 16, 8 * len(INPUT_TASK)]
cfg['representation_dtype'] = tf.float32
def create_bn_decay_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchnorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
return tf.train.piecewise_constant(global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(6682),
np.int32(6700)
],
values=[0.2, 0.5, 0.9, 0.999])
def create_clipping_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchrenorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
rmax = tf.train.piecewise_constant(global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000)
],
values=[1., 1.5, 2., 2.5, 3.])
rmin = 1. / rmax
dmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000),
np.int32(4500),
np.int32(5000),
],
values=[0., 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 5.0])
return {'rmax': rmax, 'rmin': rmin, 'dmax': dmax}
## SETUP MODEL
# Transfer
cfg['encoder'] = transfer_models.transfer_two_stream_with_bn_ends_l2l0_no_image
if INPUT_TASK == 'pixels':
raise NotImplementedError(
"Cannot transfer from 'pixels' when using no_image!")
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int(
"<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'side_encoder_func': transfer_models.side_encoder_l2,
'output_channels': cfg['representation_dim'][-1],
'kernel_size': [KERNEL_SIZE, KERNEL_SIZE],
'stride': 1,
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.8, #0.95
'weight_decay': cfg['weight_decay'],
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
# 'flatten_output': (TARGET_TASK in FLATTEN_TASKS)
}
cfg['encoder_kwargs']['renorm'] = True
cfg['encoder_kwargs']['renorm_momentum'] = 0.9
cfg['encoder_kwargs']['renorm_clipping'] = create_clipping_schedule
# Type of decoder to use
cfg['replace_target_decoder'] = transfer_models.__dict__[
REPLACE_TARGET_DECODER_FUNC_NAME]
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(5000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
cfg['log_root'] = config_dir.replace('task-taxonomy-331b/', 's3/')
if cfg['finetune_decoder'] and cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_ft')
elif cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_no_ft')
elif cfg['finetune_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'ft')
elif cfg['unlock_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'scratch')
cfg['log_dir'] = os.path.join(cfg['log_root'], 'logs')
# Set the model path that we will restore from
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(
target_cfg['log_root'], TARGET_TASK)
if cfg['retrain_decoder']:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], '..', 'logs', 'slim-train'))
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = '../s3/meta'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def get_cfg(nopause=False):
cfg = {}
cfg['is_discriminative'] = True
# params
cfg['num_epochs'] = 30
cfg['model_path'] = None
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
# model
cfg['model_type'] = architectures.Siamese
cfg['weight_decay'] = 2e-6
cfg['instance_noise_sigma'] = 0.1
cfg['instance_noise_anneal_fn'] = tf.train.inverse_time_decay
cfg['instance_noise_anneal_fn_kwargs'] = {
'decay_rate': 0.2,
'decay_steps': 1000
}
batch_size = 4
cfg['encoder'] = resnet_encoder_fully_convolutional_16x16x8
cfg['hidden_size'] = 1024
cfg['encoder_kwargs'] = {
'resnet_build_fn': resnet_v1_50_16x16,
'weight_decay': cfg['weight_decay'],
'flatten': True,
'batch_size': batch_size
}
cfg['return_accuracy'] = True
# learning
cfg['initial_learning_rate'] = 1e-4
cfg['optimizer'] = tf.train.AdamOptimizer
cfg['optimizer_kwargs'] = {}
cfg['discriminator_learning_args'] = {
'initial_learning_rate': 1e-5,
'optimizer': tf.train.GradientDescentOptimizer,
'optimizer_kwargs': {}
}
#cfg['clip_norm'] = 1
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(80000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# inputs
cfg['input_dim'] = (256, 256) # (1024, 1024)
cfg['input_num_channels'] = 3
cfg['input_dtype'] = tf.float32
cfg['num_input'] = 9
cfg['input_domain_name'] = 'rgb'
cfg['input_preprocessing_fn'] = load_ops.generate_jigsaw_input
# cfg['input_preprocessing_fn'] = load_ops.generate_jigsaw_input_for_representation_extraction
cfg['input_preprocessing_fn_kwargs'] = {
'new_dims': cfg['input_dim'],
'new_scale': [-1, 1]
}
cfg['single_filename_to_multiple'] = True
num_jigsaw_class = 100
# outputs
cfg['target_dim'] = 1 # (1024, 1024)
cfg['target_dtype'] = tf.int64
cfg['target_from_filenames'] = load_ops.jigsaw_rand_index
cfg['find_target_in_config'] = True
cfg['target_dict'] = np.load(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../../../lib/data', 'jigsaw_max_hamming_set.npy'))
# masks
cfg['metric_net'] = encoder_multilayers_fc_bn
cfg['metric_kwargs'] = {
'hidden_size': 2048,
'layer_num': 3,
'output_size': num_jigsaw_class,
'batch_norm_decay': 0.9,
'batch_norm_epsilon': 1e-5,
'batch_norm_scale': True,
'batch_norm_center': True
}
# input pipeline
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_input_depends_on_target
cfg['randomize'] = False
cfg['num_read_threads'] = 300
cfg['batch_size'] = batch_size
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 600
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def run_training(cfg):
# set up logging
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = utils.setup_input(cfg,
is_training=False,
use_filename_queue=True)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = setup_model(inputs, cfg, is_training=False)
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics(inputs, model, cfg)
# execute training
start_time = time.time()
utils.print_start_info(cfg, inputs['max_steps'], is_training=False)
# start session and restore model
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
if cfg['model_path'] is None:
print('Please specify a checkpoint directory')
return
cfg['randomize'] = False
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
utils.print_start_info(cfg,
inputs['max_steps'],
is_training=is_training)
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=True)
training_runners['threads'] = data_prefetch_init_fn(
training_runners['sess'], training_runners['coord'])
representations, input_batch, target_batch, data_idx = training_runners[
'sess'].run([
model['model'].encoder_output, inputs['input_batch'],
inputs['target_batch'], inputs['data_idxs'],
inputs['mask_batch']
])
print('Got first batch representation with size:%s' %
(representations.shape))
for step in xrange(inputs['max_steps'] - 1):
encoder_output, input_batch, target_batch, data_idx = training_runners[
'sess'].run([
model['model'].encoder_output, inputs['input_batch'],
inputs['target_batch'], inputs['data_idxs'],
inputs['mask_batch']
])
representations = np.append(representations,
encoder_output,
axis=0)
if training_runners['coord'].should_stop():
break
print(
'The size of representations is %s while we expect it to run for %d steps with batchsize %d'
% (representations.shape, inputs['max_steps'], cfg['batch_size']))
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
end_train_time = time.time() - start_time
print('time to train %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
print('avg time per epoch: %.3f hrs' %
((end_train_time / (60 * 60)) / cfg['num_epochs']))
def run_to_task():
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
task = args.task
if task not in list_of_tasks:
raise ValueError('Task not supported')
cfg = utils.generate_cfg(task)
# Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
segment25d segment2d room_layout'.split()
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
############## Single Image ##############
if args.imgs_list:
with open(args.imgs_list) as imgs_list:
all_prediction = []
all_representation = []
for line in imgs_list:
filename = args.dir_name + line.strip().split(',')[0] # FIXME
img = prepare_image(task, filename, cfg)
predicted, representation = training_runners['sess'].run(
[m.decoder_output, m.encoder_output],
feed_dict={m.input_images: img})
utils.tasks(
task,
args,
predicted,
os.path.join(args.store_name +
line.split(os.path.sep)[-1].strip() + '.jpg'),
img=img)
all_prediction.append(np.squeeze(predicted))
all_representation.append(np.squeeze(representation))
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.array(all_representation))
if args.store_pred:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}_pred.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.array(all_prediction))
else:
img = prepare_image(task, args.im_name, cfg)
predicted, representation = training_runners['sess'].run(
[m.decoder_output, m.encoder_output],
feed_dict={m.input_images: img})
utils.tasks(task, args, predicted, representation, img)
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(representation))
if args.store_pred:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}_pred.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(predicted))
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# print("Done: {}".format(config_name))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def get_cfg( nopause=False ):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
cfg['retrain_decoder'] = ("<RETRAIN_DECODER>" == "True")
cfg['unlock_decoder'] = ("<UNLOCK_DECODER>" == "True")
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR,
INPUT_TASK)
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR,
TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = general_utils.load_config(cfg['config_dir_input'], nopause=True)
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'], nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/val_image_split_0.npy') )
cfg['val_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/test_image_split_0.npy') )
# Define where the extracted representations are stored
cfg['train_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='train' if transitive else 'val'))
cfg['val_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='val' if transitive else 'test'))
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 15 #if not cfg['finetune_decoder'] else 7
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = (16, 16, 8)
cfg['representation_dtype'] = tf.float32
def create_bn_decay_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchnorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
return tf.train.piecewise_constant(
global_step,
boundaries=[np.int32(0), np.int32(1500), np.int32(6682), np.int32(6700)],
values=[0.2, 0.5, 0.9, 0.999]
)
def create_clipping_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchrenorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
rmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000)],
values=[
1.,
1.5,
2.,
2.5,
3.]
)
rmin = 1./rmax
dmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000),
np.int32(4500),
np.int32(5000),
],
values=[
0.,
0.5,
1.0,
1.5,
2.0,
2.5,
3.0,
3.5,
5.0]
)
return {'rmax': rmax, 'rmin': rmin, 'dmax': dmax}
## SETUP MODEL
# Transfer
cfg['encoder'] = passthrough
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int("<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs' ] = {
'boundaries': [np.int64(0), np.int64(5000)], # need to be int64 since global step is...
'values': [cfg['initial_learning_rate'], cfg['initial_learning_rate']/10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_transitive_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_transitive_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
cfg['log_root'] = log_root
if cfg['finetune_decoder'] and cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_ft')
elif cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_no_ft')
elif cfg['finetune_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'ft')
elif cfg['unlock_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'scratch')
cfg['log_dir'] = os.path.join(cfg['log_root'], 'logs')
# Set the model path that we will restore from
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(target_cfg['log_root'], TARGET_TASK)
# if cfg['retrain_decoder']:
# cfg['model_path'] = tf.train.latest_checkpoint(
# os.path.join(
# cfg['log_root'],
# '..',
# 'logs',
# 'slim-train'
# )
# )
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def get_cfg( nopause=False ):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = True
### -----CHANGE HERE--------------------------
# cfg['representations_file'] = '/home/ubuntu/s3/model_log/{task}/{task}_representations.pkl'.format(task=INPUT_TASK)
cfg['train_representations_file'] = '/home/ubuntu/s3/model_log/{subdir}/{task}_{train_split}_representations.pkl'.format(
subdir=representation_dir, task=representation_task, train_split='val')
cfg['val_representations_file'] = '/home/ubuntu/s3/model_log/{subdir}/{task}_{train_split}_representations.pkl'.format(
subdir=representation_dir, task=representation_task, train_split='test')
cfg['input_config_dir'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(INPUT_TASK)
cfg['config_dir_target1'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(INTERMEDIATE_TASK)
cfg['config_dir_target2'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(TARGET_TASK)
cfg['task1'] = INTERMEDIATE_TASK
cfg['task2'] = TARGET_TASK
# Where the target model is saved
cfg['model_path'] = None # '/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(TARGET_TASK)
# cfg['model_path'] = '/home/ubuntu/s3/model_log/{}/logs/slim-train'.format(TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
# input_cfg = general_utils.load_config(cfg['input_config_dir'], nopause=True)
cfg['target_cfgs'] = [
general_utils.load_config(cfg['config_dir_target1'], nopause=True),
general_utils.load_config(cfg['config_dir_target2'], nopause=True),
]
cfg['model_paths'] = [
'/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(cfg['task1']),
'/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(cfg['task2']),
]
target_cfg = cfg['target_cfgs'][-1]
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
cfg['target_model_type'] = target_cfg['model_type']
# kludge
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/val_image_split_0.npy') )
cfg['val_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/test_image_split_0.npy') )
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# params
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 4
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.ChainedTransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = 1024
cfg['representation_dtype'] = tf.float32
## SETUP MODEL
# Transfer
# cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout # encoder_multilayers_fc_bn
cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout_normalize_input # encoder_multilayers_fc_bn
if INPUT_TASK == 'random':
cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout_normalize_input
# cfg['encoder'] = encoder_multilayers_fc
cfg['hidden_size'] = int("<HIDDEN_SIZE>")
cfg['encoder_kwargs'] = {
'output_size': cfg['representation_dim'],
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.9, #0.95
'weight_decay': cfg['weight_decay'],
'layer_num': 3
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/model_log/transfer_transitive'.format(cfg['hidden_size'])
else:
log_root = '/home/ubuntu/model_log/transfer_transitive'.format(cfg['hidden_size'])
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/model_log/transfer'.format(cfg['hidden_size'])
else:
log_root = '/home/ubuntu/model_log/transfer'.format(cfg['hidden_size'])
cfg['log_root'] = log_root
cfg['log_dir'] = os.path.join(log_root, task_name, 'logs')
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_chained_transfer
cfg['preprocess_fn'] = load_and_specify_preprocessors_multiple
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def generate_cfg(config_dir, vid_id):
target_task, is_transfer, is_high_order, config_name = parse_config_dir(config_dir)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{cfg_dir}'.format(cfg_dir=config_dir)
############## Load Configs ##############
import utils
import data.load_ops as load_ops
from general_utils import RuntimeDeterminedEnviromentVars
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
cfg = utils.load_config( CONFIG_DIR, nopause=True )
RuntimeDeterminedEnviromentVars.register_dict( cfg )
root_dir = cfg['root_dir']
split_file = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/video{vid_id}_info.pkl'.format(vid_id=vid_id)) )
cfg['dataset_dir'] = '/home/ubuntu'
if cfg['input_preprocessing_fn'] == load_ops.resize_rescale_image:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
if 'train_list_of_fileinfos' in cfg:
if type(cfg['train_representations_file']) is not list:
task = config_name.split('__')[0]
split_file_ = os.path.join(
cfg['input_cfg']['log_root'], task,
'{task}_vid{vid_id}_representations.pkl'.format( task=task, vid_id=vid_id ))
else:
split_file_ = []
for fname in cfg['train_representations_file']:
split_file_.append(fname.replace('val', 'vid{vid_id}'.format(vid_id = vid_id)))
cfg['train_representations_file'] = split_file_
cfg['val_representations_file'] = split_file_
cfg['test_representations_file'] = split_file_
split_file_ = os.path.join(root_dir, 'assets/aws_data/video{vid_id}_fname.npy'.format(vid_id=vid_id))
cfg['train_list_of_fileinfos'] = split_file_
cfg['val_list_of_fileinfos'] = split_file_
cfg['test_list_of_fileinfos'] = split_file_
cfg['num_epochs'] = 2
cfg['randomize'] = False
root_dir = cfg['root_dir']
if target_task != 'segment2d' and target_task != 'segment25d':
cfg['num_read_threads'] = 1
else:
cfg['num_read_threads'] = 1
print(cfg['log_root'])
if is_transfer:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(
cfg['log_root'],
'logs',
'slim-train'
))
# Try latest checkpoint by time
if cfg['model_path'] is None:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(
cfg['log_root'],
'logs',
'slim-train',
'time'
))
# Try to get one saved manually
if cfg['model_path'] is None:
cfg['model_path'] = os.path.join(cfg['log_root'], task, "model.permanent-ckpt")
else:
cfg['model_path'] = os.path.join(
cfg['log_root'],
config_name,
'model.permanent-ckpt'
)
print( cfg['model_path'])
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_representation_extraction
return cfg, is_transfer, target_task, config_name
def run_to_task():
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
imgs = args.im_name.split(',')
if args.task == 'ego_motion' and len(imgs) != 3:
raise ValueError('Wrong number of images, expecting 3 but got {}'.format(len(imgs)))
if args.task != 'ego_motion' and len(imgs) != 2:
raise ValueError('Wrong number of images, expecting 2 but got {}'.format(len(imgs)))
task = args.task
if task not in list_of_tasks:
raise ValueError('Task not supported')
cfg = generate_cfg(task)
input_img = np.empty((len(imgs),256,256,3), dtype=np.float32)
for i,imname in enumerate(imgs):
img = load_raw_image_center_crop( imname )
img = skimage.img_as_float(img)
scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(imname)
img = cfg[ 'input_preprocessing_fn' ]( img, **cfg['input_preprocessing_fn_kwargs'] )
input_img[i,:,:,:] = img
input_img = input_img[np.newaxis, :]
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() }
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn( cfg, is_training=False, use_filename_queue=False )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model( inputs, cfg, is_training=False )
m = model[ 'model' ]
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
predicted, representation = training_runners['sess'].run(
[ m.decoder_output, m.encoder_output ], feed_dict={m.input_images: input_img} )
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(representation))
if args.store_pred:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}_pred.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(predicted))
if task == 'ego_motion':
ego_motion(predicted, args.store_name)
return
if task == 'fix_pose':
cam_pose(predicted, args.store_name, is_fixated=True)
return
if task == 'non_fixated_pose':
cam_pose(predicted, args.store_name, is_fixated=False)
return
if task == 'point_match':
prediction = np.argmax(predicted, axis=1)
print('the prediction (1 stands for match, 0 for unmatch)is: ', prediction)
return
############## Clean Up ##############
training_runners[ 'coord' ].request_stop()
training_runners[ 'coord' ].join()
print("Done: {}".format(config_name))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def get_cfg( nopause=False ):
cfg = {}
# params
cfg['num_epochs'] = 30
cfg['model_path'] = None
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
# model
cfg['model_type'] = architectures.SegmentationEncoderDecoder
cfg['weight_decay'] = 2e-6
cfg['instance_noise_sigma'] = 0.1
cfg['instance_noise_anneal_fn'] = tf.train.inverse_time_decay
cfg['instance_noise_anneal_fn_kwargs'] = {
'decay_rate': 0.2,
'decay_steps': 1000
}
cfg['encoder'] = resnet_encoder_fully_convolutional_16x16x8
cfg['hidden_size'] = 1024
cfg['encoder_kwargs'] = {
'resnet_build_fn' : resnet_v1_50_16x16,
'weight_decay': cfg['weight_decay']
}
cfg['decoder'] = decoder_fc_15_layer_256_resolution_fully_convolutional_16x16x4
cfg['decoder_kwargs'] = {
'activation_fn': leaky_relu( 0.2 ),
'weight_decay': cfg['weight_decay']
}
# learning
cfg['initial_learning_rate'] = 1e-4
cfg[ 'optimizer' ] = tf.train.AdamOptimizer
cfg[ 'optimizer_kwargs' ] = {}
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs' ] = {
'boundaries': [np.int64(0), np.int64(80000)], # need to be int64 since global step is...
'values': [cfg['initial_learning_rate'], cfg['initial_learning_rate']/10]
}
# inputs
cfg['input_dim'] = (256, 256) # (1024, 1024)
cfg['input_num_channels'] = 3
cfg['input_dtype'] = tf.float32
cfg['input_domain_name'] = 'rgb'
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image
cfg['input_preprocessing_fn_kwargs'] = {
'new_dims': cfg['input_dim'],
'new_scale': [-1, 1]
}
# outputs
cfg['output_dim'] = (256,256)
cfg['num_pixels'] = 300
cfg['only_target_discriminative'] = True
cfg['target_num_channels'] = 64
cfg['target_dim'] = (cfg['num_pixels'], 3) # (1024, 1024)
cfg['target_dtype'] = tf.int32
cfg['target_domain_name'] = 'segment25d'
cfg['target_from_filenames'] = load_ops.segment_pixel_sample
cfg['target_from_filenames_kwargs'] = {
'new_dims': cfg['output_dim'],
'num_pixels': cfg['num_pixels'],
'domain': cfg['target_domain_name']
}
cfg['return_accuracy'] = False
# input pipeline
cfg['preprocess_fn'] = load_and_specify_preprocessors
cfg['randomize'] = False
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 600
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def run_training(cfg, cfg_dir, args):
if args.stat_type == "mean":
statistic = MeanMeter(cfg)
elif args.stat_type == 'median':
statistic = MedianMeter(cfg)
elif args.stat_type == 'marginal':
statistic = DiscreteDistributionMeter(cfg, args.not_one_hot)
elif args.stat_type == 'dense_marginal':
statistic = DenseDiscreteDistributionMeter(cfg)
elif args.stat_type == 'moments':
statistic = MomentsMeter(cfg)
else:
raise NotImplementedError("No average defined for type: {}".format(
args.stat_type))
# set up logging
tf.logging.set_verbosity(tf.logging.ERROR)
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = utils.setup_input(cfg, is_training=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# execute training
start_time = time.time()
max_steps = get_max_steps(inputs['max_steps'], args.data_split)
utils.print_start_info(cfg, max_steps, is_training=False)
data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False)
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
prefetch_threads = threading.Thread(
target=data_prefetch_threads_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
target_batch = training_runners['sess'].run(inputs['target_batch'])
# training_runners[ 'threads' ] = data_prefetch_init_fn( training_runners[ 'sess' ], training_runners[ 'coord' ] )
try:
# This just returns the imput as output. It is for testing data
# input only.
start_time = time.time()
batch_time = time.time()
k = int(args.print_every)
for step in range(max_steps):
target_batch, mask_batch = training_runners['sess'].run(
[inputs['target_batch'], inputs['mask_batch']])
target_batch = map_to_img(target_batch.mean(axis=0), cfg)
if len(mask_batch.shape) > 1:
mask_batch = mask_batch.mean(axis=0)
else:
mask_batch = 1
statistic.update(target_batch, mask_batch)
if (step + 1) % k == 0:
print('Step %d/%d: %.2f s/step ' %
(step + 1, max_steps,
(time.time() - batch_time) / k))
batch_time = time.time()
# print(statistic.get())
# break
if training_runners['coord'].should_stop():
break
end_train_time = time.time() - start_time
print('time to train %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
print('avg time per epoch: %.3f hrs' %
((end_train_time / (60 * 60)) / cfg['num_epochs']))
if args.stat_type == 'moments':
save_moments(statistic, cfg, args)
else:
save_data(statistic, cfg, args)
finally:
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
def run_training(cfg, cfg_dir):
# set up logging
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = utils.setup_input_transfer(cfg, is_training=True)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model(inputs, cfg, is_training=True)
# execute training
start_time = time.time()
utils.print_start_info(cfg, inputs['max_steps'], is_training=True)
if cfg['model_type'] == 'empty': # Can't use tf slim because not trainable variables
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=True)
training_runners['threads'] = data_prefetch_init_fn(
training_runners['sess'], training_runners['coord'])
try:
# This just returns the imput as output. It is for testing data
# input only.
for step in xrange(inputs['max_steps']):
input_batch, target_batch, data_idx = training_runners[
'sess'].run([
model['input_batch'], model['target_batch'],
model['data_idxs']
])
if training_runners['coord'].should_stop():
break
finally:
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
else: # Use tf.slim
train_log_dir = os.path.join(cfg['log_dir'], 'slim-train')
permanent_checkpoint_dir = os.path.join(cfg['log_dir'],
'checkpoints')
session_config = tf.ConfigProto()
session_config.gpu_options.allow_growth = True
#max_to_keep = cfg['num_epochs'] * 2
max_to_keep = 10
if 'max_ckpts_to_keep' in cfg:
max_to_keep = cfg['max_ckpts_to_keep']
# When ready to use a model, use the code below
train(
model['train_op'],
train_log_dir,
utils.get_data_prefetch_threads_init_fn_transfer(
inputs, cfg, is_training=True),
train_step_fn=model['train_step_fn'],
train_step_kwargs=model['train_step_kwargs'],
global_step=model['global_step'],
number_of_steps=inputs['max_steps'],
number_of_epochs=cfg['num_epochs'],
init_fn=model['init_fn'],
save_checkpoint_every=max(inputs['max_steps'] // (max_to_keep),
500),
cfg_dir=cfg_dir,
#RuntimeDeterminedEnviromentVars.steps_per_epoch,
permanent_checkpoint_dir=permanent_checkpoint_dir,
save_summaries_secs=cfg['summary_save_every_secs'],
save_interval_secs=cfg['checkpoint_save_every_secs'],
saver=model['saver_op'],
return_accuracy='return_accuracy' in cfg
and cfg['return_accuracy'],
session_config=session_config)
end_train_time = time.time() - start_time
print('time to train %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
print('avg time per epoch: %.3f hrs' %
((end_train_time / (60 * 60)) / cfg['num_epochs']))
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(
inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
'''
# encoder (extract features)
predicted, representation = training_runners['sess'].run(
[ m.decoder_output, m.encoder_output ], feed_dict={m.input_images: img} )
'''
representation = training_runners['sess'].run(
def get_cfg( nopause=False ):
cfg = {}
# params
cfg['num_epochs'] = 30
cfg['model_path'] = None
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
# model
cfg['model_type'] = architectures.EncoderDecoderWithCGAN
cfg['n_g_steps_before_d'] = 1
cfg['n_d_steps_after_g'] = 1
cfg['init_g_steps'] = 25000
cfg['l_norm_weight_prop'] = 0.996
cfg['weight_decay'] = 2e-6
cfg['instance_noise_sigma'] = 0.1
cfg['instance_noise_anneal_fn'] = tf.train.inverse_time_decay
cfg['instance_noise_anneal_fn_kwargs'] = {
'decay_rate': 0.2,
'decay_steps': 1000
}
cfg['encoder'] = resnet_encoder_fully_convolutional_16x16x8
cfg['hidden_size'] = 1024
cfg['encoder_kwargs'] = {
'resnet_build_fn' : resnet_v1_50_16x16,
'weight_decay': cfg['weight_decay']
}
cfg['decoder'] = decoder_fc_15_layer_256_resolution_fully_convolutional_16x16x4
cfg['decoder_kwargs'] = {
'activation_fn': leaky_relu( 0.2 ),
'weight_decay': cfg['weight_decay']
}
cfg['discriminator'] = pix2pix_discriminator
cfg['discriminator_kwargs'] = {
'n_layers': 5,
'stride': 4,
'n_channel_multiplier': 64,
'weight_decay': 10.*cfg['weight_decay']
}
# loss
cfg['gan_loss_kwargs'] = {
'real_label': 0.9, # Positive labels 1 -> 0.9
'fake_label': 0.0
}
# learning
cfg['initial_learning_rate'] = 1e-4
cfg[ 'optimizer' ] = tf.train.AdamOptimizer
cfg[ 'optimizer_kwargs' ] = {}
cfg[ 'discriminator_learning_args' ] = {
'initial_learning_rate':1e-5,
'optimizer': tf.train.GradientDescentOptimizer,
'optimizer_kwargs': {}
}
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs' ] = {
'boundaries': [np.int64(0), np.int64(80000)], # need to be int64 since global step is...
'values': [cfg['initial_learning_rate'], cfg['initial_learning_rate']/10]
}
# inputs
cfg['input_dim'] = (256, 256) # (1024, 1024)
cfg['input_num_channels'] = 3
cfg['input_dtype'] = tf.float32
cfg['input_domain_name'] = 'rgb'
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image
cfg['input_preprocessing_fn_kwargs'] = {
'new_dims': cfg['input_dim'],
'new_scale': [-1, 1]
}
# outputs
cfg['target_num_channels'] = 1
cfg['target_dim'] = (256, 256) # (1024, 1024)
cfg['target_dtype'] = tf.float32
cfg['target_domain_name'] = 'keypoint'
cfg['target_preprocessing_fn'] = load_ops.resize_rescale_image
cfg['target_preprocessing_fn_kwargs'] = {
'new_dims': cfg['target_dim'],
'new_scale': [-1, 1]
}
# masks
cfg['depth_mask'] = True
# input pipeline
cfg['preprocess_fn'] = load_and_specify_preprocessors
cfg['randomize'] = False
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 600
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def deep_attribution():
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
tf.logging.set_verbosity(tf.logging.ERROR)
imlist_file_path = os.path.join(prj_dir, args.explain_result_root,
args.dataset, 'imlist.txt')
explain_result_root = os.path.join(prj_dir, args.explain_result_root,
args.dataset)
with open(imlist_file_path) as f:
lines = []
line = f.readline().strip()
while len(line) != 0:
lines += [os.path.join(prj_dir, 'dataset', args.dataset, line)]
line = f.readline().strip()
explain_methods = ['saliency', 'grad*input', 'elrp']
for task in list_of_tasks:
if not os.path.exists(os.path.join(explain_result_root, task)):
os.mkdir(os.path.join(explain_result_root, task))
cfg = generate_cfg(task)
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
sess = tf.Session()
training_runners = {'sess': sess, 'coord': tf.train.Coordinator()}
with DeepExplain(session=sess, graph=sess.graph) as de:
############## Set Up Inputs ##############
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
encoder_endpoints = model['model'].encoder_endpoints
endpoints = encoder_endpoints
print('There are {} images in {}'.format(len(lines), imlist_file_path))
img = load_raw_image_center_crop(lines[0])
img = skimage.img_as_float(img)
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
segment25d segment2d room_layout'.split()
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
img = cfg['input_preprocessing_fn'](
img, **cfg['input_preprocessing_fn_kwargs'])
imgs = np.zeros([args.imlist_size, 1] + list(img.shape), float)
for line_i in range(args.imlist_size):
img = load_raw_image_center_crop(lines[line_i])
img = skimage.img_as_float(img)
if task not in list_of_tasks:
raise ValueError('Task not supported')
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
if task == 'jigsaw':
img = cfg['input_preprocessing_fn'](
img,
target=cfg['target_dict'][random.randint(0, 99)],
**cfg['input_preprocessing_fn_kwargs'])
else:
img = cfg['input_preprocessing_fn'](
img, **cfg['input_preprocessing_fn_kwargs'])
imgs[line_i, :] = img[np.newaxis, :]
elrp = np.zeros([args.imlist_size] + list(img.shape), float)
saliency = np.zeros([args.imlist_size] + list(img.shape), float)
gradXinput = np.zeros([args.imlist_size] + list(img.shape), float)
for im_i in range(args.imlist_size):
with DeepExplain(session=sess) as de:
representation = training_runners['sess'].run(
[endpoints['encoder_output']],
feed_dict={m.input_images: imgs[im_i]})
attributions = {
explain_method:
de.explain(explain_method, endpoints['encoder_output'],
m.input_images, imgs[im_i])
for explain_method in explain_methods
}
elrp[im_i] = attributions['elrp']
saliency[im_i] = attributions['saliency']
gradXinput[im_i] = attributions['grad*input']
print('{} images done.'.format(im_i))
if ((im_i + 1) % 5) == 0:
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
sess = tf.Session()
training_runners = {
'sess': sess,
'coord': tf.train.Coordinator()
}
with DeepExplain(session=sess, graph=sess.graph) as de:
############## Set Up Inputs ##############
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(
inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables(
)
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
encoder_endpoints = model['model'].encoder_endpoints
endpoints = encoder_endpoints
np.save(os.path.join(explain_result_root, task, 'elrp.npy'), elrp)
np.save(os.path.join(explain_result_root, task, 'saliency.npy'),
saliency)
np.save(os.path.join(explain_result_root, task, 'gradXinput.npy'),
gradXinput)
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
print('Task {} Done!'.format(task))
print('All Done.')
return
def run_rand_baseline( args, cfg, given_task ):
# set up logging
tf.logging.set_verbosity( tf.logging.INFO )
with tf.Graph().as_default() as g:
# create ops and placeholders
tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input( cfg, is_training=False, use_filename_queue=False )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model( inputs, cfg, is_training=False )
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics( inputs, model, cfg )
# execute training
start_time = time.time()
utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
# start session and restore model
training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() }
try:
utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False )
#training_runners[ 'threads' ] = data_prefetch_init_fn( training_runners[ 'sess' ], training_runners[ 'coord' ] )
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=( training_runners[ 'sess' ], training_runners[ 'coord' ] ))
prefetch_threads.start()
# run one example so that we can calculate some statistics about the representations
targets = training_runners['sess'].run( inputs[ 'target_batch' ] )
# run the remaining examples
for step in range( inputs[ 'max_steps' ] - 1 ):
#for step in range( 10 ):
if step % 100 == 0:
print( 'Step {0} of {1}'.format( step, inputs[ 'max_steps' ] - 1 ))
target = training_runners['sess'].run( inputs[ 'target_batch' ] )
targets = np.append( targets, target, axis=0)
if training_runners['coord'].should_stop():
break
rand_idx = [random.randint(0, targets.shape[0] - 1) for i in range(targets.shape[0])]
rand_target = [targets[i] for i in rand_idx]
rand_target = np.vstack(rand_target)
counter = 0
sum = 0
for step in range( inputs[ 'max_steps' ] - 1 ):
#for step in range( 10 ):
if step % 100 == 0:
print( 'Step {0} of {1}'.format( step, inputs[ 'max_steps' ] - 1 ))
tar = targets[step*cfg['batch_size']:(step+1)*cfg['batch_size']]
rand = rand_target[step*cfg['batch_size']:(step+1)*cfg['batch_size']]
losses = training_runners['sess'].run( model['model'].losses, feed_dict={
inputs['target_batch']: tar, model['model'].final_output:rand})
sum += losses[0]
counter += 1
if training_runners['coord'].should_stop():
break
print(sum)
print(counter)
print('random_baseline has loss: {loss}'.format(loss=sum/counter))
end_train_time = time.time() - start_time
finally:
utils.request_data_loading_end( training_runners )
utils.end_data_loading_and_sess( training_runners )
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
from data.load_ops import rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
from data.task_data_loading import load_and_specify_preprocessors_for_input_depends_on_target
import lib.data.load_ops as load_ops
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
cfg, is_transfer, task, config_name = generate_cfg(args.config, args.vid,
args)
if task == 'class_places' or task == 'class_1000':
synset = get_synset(task)
if task == 'jigsaw':
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_input_depends_on_target
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
############## Start dataloading workers ##############
if is_transfer:
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer
setup_input_fn = utils.setup_input_transfer
else:
setup_input_fn = utils.setup_input
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=IN_TRAIN_MODE)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
data_prefetch_init_fn = get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=False)
prefetch_threads = threading.Thread(target=data_prefetch_init_fn,
args=(training_runners['sess'],
training_runners['coord']))
prefetch_threads.start()
list_of_fname = np.load(
'/home/ubuntu/task-taxonomy-331b/assets/aws_data/video{}_fname.npy'.
format(args.vid))
import errno
try:
os.mkdir('/home/ubuntu/{}'.format(task))
os.mkdir('/home/ubuntu/{}/vid1'.format(task))
os.mkdir('/home/ubuntu/{}/vid2'.format(task))
os.mkdir('/home/ubuntu/{}/vid3'.format(task))
os.mkdir('/home/ubuntu/{}/vid4'.format(task))
except OSError as e:
if e.errno != errno.EEXIST:
raise
curr_comp = np.zeros((3, 64))
curr_fit_img = np.zeros((256, 256, 3))
embeddings = []
curr_vp = []
curr_layout = []
############## Run First Batch ##############
def rescale_l_for_display(batch, rescale=True):
'''
Prepares network output for display by optionally rescaling from [-1,1],
and by setting some pixels to the min/max of 0/1. This prevents matplotlib
from rescaling the images.
'''
if rescale:
display_batch = [
rescale_image(im.copy(),
new_scale=[0, 100],
current_scale=[-1, 1]) for im in batch
]
else:
display_batch = batch.copy()
for im in display_batch:
im[0, 0,
0] = 1.0 # Adjust some values so that matplotlib doesn't rescale
im[0, 1, 0] = 0.0 # Now adjust the min
return display_batch
for step_num in range(inputs['max_steps'] - 1):
#for step_num in range(20):
#if step_num > 0 and step_num % 20 == 0:
print(step_num)
if is_transfer:
(input_batch, target_batch, data_idx,
predicted) = training_runners['sess'].run([
m.input_images, m.target_images, model['data_idxs'],
m.decoder.decoder_output
])
else:
(input_batch, target_batch, data_idx,
predicted) = training_runners['sess'].run([
m.input_images, m.targets, model['data_idxs'],
m.decoder_output
])
if task == 'segment2d' or task == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
k_embed = 8
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
embeddings.append(embedding_flattened)
if len(embeddings) > k_embed:
embeddings.pop(0)
pca = PCA(n_components=3)
pca.fit(np.vstack(embeddings))
min_order = None
min_dist = float('inf')
copy_of_comp = np.copy(pca.components_)
for order in itertools.permutations([0, 1, 2]):
#reordered = pca.components_[list(order), :]
#dist = np.linalg.norm(curr_comp-reordered)
pca.components_ = copy_of_comp[order, :]
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (
lower_dim.max() - lower_dim.min())
dist = np.linalg.norm(lower_dim - curr_fit_img)
if dist < min_dist:
min_order = order
min_dist = dist
pca.components_ = copy_of_comp[min_order, :]
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() -
lower_dim.min())
curr_fit_img = np.copy(lower_dim)
x[i] = lower_dim
predicted = x
if task == 'curvature':
std = [31.922, 21.658]
mean = [123.572, 120.1]
predicted = (predicted * std) + mean
predicted[:, 0, 0, :] = 0.
predicted[:, 1, 0, :] = 1.
predicted = np.squeeze(
np.clip(predicted.astype(int) / 255., 0., 1.)[:, :, :, 0])
if task == 'colorization':
maxs = np.amax(predicted, axis=-1)
softmax = np.exp(predicted - np.expand_dims(maxs, axis=-1))
sums = np.sum(softmax, axis=-1)
softmax = softmax / np.expand_dims(sums, -1)
kernel = np.load(
'/home/ubuntu/task-taxonomy-331b/lib/data/pts_in_hull.npy')
gen_target_no_temp = np.dot(softmax, kernel)
images_resized = np.zeros([0, 256, 256, 2], dtype=np.float32)
for image in range(gen_target_no_temp.shape[0]):
temp = scipy.ndimage.zoom(np.squeeze(
gen_target_no_temp[image]), (4, 4, 1),
mode='nearest')
images_resized = np.append(images_resized,
np.expand_dims(temp, axis=0),
axis=0)
inp_rescale = rescale_l_for_display(input_batch)
output_lab_no_temp = np.concatenate((inp_rescale, images_resized),
axis=3).astype(np.float64)
for i in range(input_batch.shape[0]):
output_lab_no_temp[i, :, :, :] = skimage.color.lab2rgb(
output_lab_no_temp[i, :, :, :])
predicted = output_lab_no_temp
just_rescale = [
'autoencoder', 'denoise', 'edge2d', 'edge3d', 'keypoint2d',
'keypoint3d', 'reshade', 'rgb2sfnorm', 'impainting_whole'
]
if task in just_rescale:
predicted = (predicted + 1.) / 2.
predicted = np.clip(predicted, 0., 1.)
predicted[:, 0, 0, :] = 0.
predicted[:, 1, 0, :] = 1.
just_clip = ['rgb2depth', 'rgb2mist']
if task in just_clip:
predicted = np.exp(predicted * np.log(2.0**16.0)) - 1.0
predicted = np.log(predicted) / 11.09
predicted = (predicted - 0.64) / 0.18
predicted = (predicted + 1.) / 2
predicted[:, 0, 0, :] = 0.
predicted[:, 1, 0, :] = 1.
if task == 'segmentsemantic_rb':
label = np.argmax(predicted, axis=-1)
COLORS = ('white', 'red', 'blue', 'yellow', 'magenta', 'green',
'indigo', 'darkorange', 'cyan', 'pink', 'yellowgreen',
'black', 'darkgreen', 'brown', 'gray', 'purple',
'darkviolet')
rgb = (input_batch + 1.) / 2.
preds = [
color.label2rgb(np.squeeze(x),
np.squeeze(y),
colors=COLORS,
kind='overlay')[np.newaxis, :, :, :]
for x, y in zip(label, rgb)
]
predicted = np.vstack(preds)
if task in ['class_1000', 'class_places']:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
sorted_pred = np.argsort(predict_output)[::-1]
top_5_pred = [synset[sorted_pred[i]] for i in range(5)]
to_print_pred = "Top 5 prediction: \n {}\n {}\n {}\n {} \n {}".format(
*top_5_pred)
img = Image.new('RGBA', (400, 200), (255, 255, 255))
d = ImageDraw.Draw(img)
fnt = ImageFont.truetype(
'/usr/share/fonts/truetype/dejavu/DejaVuSerifCondensed.ttf',
25)
d.text((20, 5), to_print_pred, fill=(255, 0, 0), font=fnt)
img.save(to_store_name, 'PNG')
elif task == 'vanishing_point_well_defined':
counter = 0
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
curr_vp.append(
plot_vanishing_point_smoothed(
predict_output, (input_batch[counter] + 1.) / 2.,
to_store_name, curr_vp))
if len(curr_vp) > 5:
curr_vp.pop(0)
counter += 1
#scipy.misc.toimage(result, cmin=0.0, cmax=1.0).save(to_store_name)
elif task == 'room_layout':
mean = np.array([
0.006072743318127848, 0.010272365569691076, -3.135909774145468,
1.5603802322235532, 5.6228218371102496e-05,
-1.5669352793761442, 5.622875878174759, 4.082800262277375,
2.7713941642895956
])
std = np.array([
0.8669452525283652, 0.687915294956501, 2.080513632043758,
0.19627420479282623, 0.014680602791251812, 0.4183827359302299,
3.991778013006544, 2.703495278378409, 1.2269185938626304
])
predicted = predicted * std + mean
counter = 0
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
plot_room_layout(predict_output,
(input_batch[counter] + 1.) / 2.,
to_store_name,
curr_layout,
cube_only=True)
curr_layout.append(predict_output)
if len(curr_layout) > 5:
curr_layout.pop(0)
#scipy.misc.toimage(result, cmin=0.0, cmax=1.0).save(to_store_name)
counter += 1
elif task == 'segmentsemantic_rb':
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
process_semseg_frame(predict_output, to_store_name)
elif task == 'jigsaw':
predicted = np.argmax(predicted, axis=1)
counter = 0
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
perm = cfg['target_dict'][predict_output]
show_jigsaw((input_batch[counter] + 1.) / 2., perm,
to_store_name)
counter += 1
else:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
scipy.misc.toimage(np.squeeze(predict_output),
cmin=0.0,
cmax=1.0).save(to_store_name)
# subprocess.call('tar -czvf /home/ubuntu/{c}_{vid_id}.tar.gz /home/ubuntu/{t}/vid{vid_id}'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
# subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -vcodec libx264 -crf 15 {c}_{vid_id}.mp4'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
subprocess.call(
'ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -ss 00:01:54 -t 00:00:40 -c:v libvpx-vp9 -crf 10 -b:v 128k {c}_{vid_id}.webm'
.format(c=config_name, t=task, vid_id=args.vid),
shell=True)
# subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -vcodec libx264 -crf 15 -pix_fmt yuv420p {c}_{vid_id}.mp4'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
subprocess.call(
'sudo mkdir -p /home/ubuntu/s3/video_new/{t}'.format(t=task),
shell=True)
#subprocess.call('sudo mkdir -p /home/ubuntu/s3/video_new_all/{t}'.format(t=task), shell=True)
# subprocess.call('aws s3 cp /home/ubuntu/{c}_{vid_id}.tar.gz s3://task-preprocessing-512-oregon/video_new_all/{t}/'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
subprocess.call(
'aws s3 cp {c}_{vid_id}.webm s3://task-preprocessing-512-oregon/video_new/{t}/'
.format(c=config_name, t=task, vid_id=args.vid),
shell=True)
# subprocess.call('aws s3 cp /home/ubuntu/{c}_{vid_id}.tar.gz s3://taskonomy-unpacked-oregon/video_tar_all/{t}/'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
# subprocess.call('aws s3 cp {c}_{vid_id}.mp4 s3://taskonomy-unpacked-oregon/video_all/{t}/'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
print("Done: {}".format(config_name))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
from importlib import reload
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
# for arch in ['regular', 'shallow', 'dilated_shallow', 'dilated_regular']:
arch = args.arch
data_amount = args.data
if args.second_order:
global TRANSFER_TYPE
TRANSFER_TYPE = 'second_order'
if not args.no_regenerate_data:
#if False:
all_outputs = {}
pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format(
arch=arch, hs=args.hs, task_to=task_to)
subprocess.call(
"aws s3 cp s3://task-preprocessing-512-oregon/visualizations/transfer_viz/viz_{}.pkl {}"
.format(task_to, pickle_dir),
shell=True)
import os
if os.path.isfile(pickle_dir):
with open(pickle_dir, 'rb') as fp:
all_outputs = pickle.load(fp)
if args.second_order:
import itertools
with open(
'/home/ubuntu/task-taxonomy-331b/tools/ranked_first_order_transfers.pkl',
'rb') as fp:
data = pickle.load(fp)
list_of_src_tasks = list(
itertools.combinations(data[task_to][:5], 2))
list_of_src_tasks = [
'{}__{}'.format(x[0], x[1]) for x in list_of_src_tasks
]
with open(
'/home/ubuntu/task-taxonomy-331b/tools/second_order_should_flip.pkl',
'rb') as fp:
to_flip_dict = pickle.load(fp)
if args.find_all_src:
config_dir_root = '/home/ubuntu/task-taxonomy-331b/experiments/second_order/{arch}/{data}'.format(
arch=arch, data=data_amount)
all_configs = os.listdir(config_dir_root)
list_of_src_tasks = []
for i in all_configs:
if i.split('__')[-3] == task_to:
list_of_src_tasks.append(i)
list_of_src_tasks = [
'__'.join(x.split('__')[:2]) for x in list_of_src_tasks
]
rank_combo = {}
for task_from in list_of_src_tasks:
first, sec = task_from.split('__')
rank_combo[task_from] = (data[task_to].index(first),
data[task_to].index(sec))
global list_of_src_tasks
for i, task_from in enumerate(list_of_src_tasks):
if args.data == '16k':
if task_from in all_outputs:
print("{} already exists....\n\n\n".format(task_from))
continue
else:
if '{}_{}'.format(task_from, args.data) in all_outputs:
print("{} already exists....\n\n\n".format(task_from))
continue
print("Doing from {task_from} to {task_to}".format(
task_from=task_from, task_to=task_to))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
if task_from == "FULL" or task_from == "FULL_IMAGE":
task = '{f}__{t}__{hs}__unlocked'.format(f="FULL",
t=task_to,
hs=args.hs)
transfer_src = 'full_order' if task_from == "FULL" else 'full_order_image'
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=transfer_src,
arch=arch,
data=data_amount,
TASK=task)
elif task_from == "FULL_select" or task_from == "FULL_select_IMAGE":
task = '{f}__{t}__{hs}__unlocked'.format(f="FULL_select",
t=task_to,
hs=args.hs)
transfer_src = 'full_order_selected' if task_from == "FULL_select" else 'full_order_selected_image'
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=transfer_src,
arch=arch,
data=data_amount,
TASK=task)
else:
task = '{f}__{t}__{hs}__unlocked'.format(f=task_from,
t=task_to,
hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=TRANSFER_TYPE,
arch=arch,
data=data_amount,
TASK=task)
print(CONFIG_DIR)
############## Load Configs ##############
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
if args.second_order and not args.find_all_src and not to_flip_dict[
task]:
cfg['val_representations_file'] = cfg[
'val_representations_file'][::-1]
cfg['num_epochs'] = 1
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train'
#'time'
))
# print(cfg['model_path'])
if cfg['model_path'] is None and task == 'random':
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train',
'time'))
if cfg['model_path'] is None:
continue
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input_transfer(
cfg, is_training=ON_TEST_SET, use_filename_queue=False
) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=True)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn_transfer(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False)
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
############## Run First Batch ##############
(
input_batch,
representation_batch,
target_batch,
data_idx,
encoder_output,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.input_representations, m.decoder.targets,
model['data_idxs'], m.encoder_output, m.decoder.decoder_output,
m.total_loss
])
if task_to == 'segment2d' or task_to == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
pca = PCA(n_components=3)
pca.fit(embedding_flattened)
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (
lower_dim.max() - lower_dim.min())
x[i] = lower_dim
predicted = x
if task_to == 'segmentsemantic_rb':
predicted = np.argmax(predicted, axis=-1)
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
to_store = {
'data_idx': data_idx,
'output': predicted,
'loss': loss
}
if args.second_order and args.find_all_src:
store_key = "{}_{}".format(task_from[:20],
rank_combo[task_from])
all_outputs[store_key] = to_store
elif args.data != '16k':
store_key = "{}_{}".format(task_from, args.data)
all_outputs[store_key] = to_store
else:
all_outputs[task_from] = to_store
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
#print(sys.modules.keys())
#del sys.modules[ 'config' ]
sys.path = remove_dups(sys.path)
print('Current Directory: ', os.getcwd())
pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format(
arch=arch, hs=args.hs, task_to=task_to)
with open(pickle_dir, 'wb') as fp:
pickle.dump(all_outputs, fp)
subprocess.call(
"aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/transfer_viz/viz_{}.pkl"
.format(pickle_dir, task_to),
shell=True)
# Run jupyter nb
print('Running Jupyter Notebooks...')
#os.makedirs("/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/transfer_{hs}_{arch}".format(hs=args.hs, arch=arch), exist_ok=True)
notebook_path = '/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/Visual_{task_to}'.format(
task_to=task_to)
if args.second_order and not args.find_all_src:
notebook_path = '{}-Copy1'.format(notebook_path)
subprocess.call("jupyter nbconvert \
--execute {notebook_path}.ipynb \
--to html \
--ExecutePreprocessor.kernel_name=python3 \
--ExecutePreprocessor.timeout=1200 ".format(
notebook_path=notebook_path, arch=arch, hs=args.hs, task_to=task_to),
shell=True)
subprocess.call(
"aws s3 cp {}.html s3://task-preprocessing-512-oregon/visualizations/{}/"
.format(notebook_path, TRANSFER_TYPE),
shell=True)
def generate_cfg(config_dir, vid_id, args):
target_task, is_transfer, is_high_order, config_name = parse_config_dir(
config_dir)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{cfg_dir}'.format(
cfg_dir=config_dir)
############## Load Configs ##############
import utils
import data.load_ops as load_ops
from general_utils import RuntimeDeterminedEnviromentVars
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
root_dir = cfg['root_dir']
split_file = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/video{vid_id}_info.pkl'.format(vid_id=vid_id)))
cfg['dataset_dir'] = '/home/ubuntu'
os.system('sudo rm /home/ubuntu/temp/*')
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
room_layout segment25d segment2d \
segmentsemantic_rb class_1000 class_places'.split()
if target_task in low_sat_tasks and args.low_sat:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat_2
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
if 'train_list_of_fileinfos' in cfg:
if type(cfg['train_representations_file']) is not list:
task = config_name.split('__')[0]
split_file_ = os.path.join(
cfg['input_cfg']['log_root'], task,
'{task}_vid{vid_id}_representations.pkl'.format(task=task,
vid_id=vid_id))
else:
split_file_ = []
for fname in cfg['train_representations_file']:
split_file_.append(
fname.replace('val', 'vid{vid_id}'.format(vid_id=vid_id)))
cfg['train_representations_file'] = split_file_
cfg['val_representations_file'] = split_file_
cfg['test_representations_file'] = split_file_
split_file_ = os.path.join(
root_dir,
'assets/aws_data/video{vid_id}_fname.npy'.format(vid_id=vid_id))
cfg['train_list_of_fileinfos'] = split_file_
cfg['val_list_of_fileinfos'] = split_file_
cfg['test_list_of_fileinfos'] = split_file_
cfg['num_epochs'] = 2
cfg['randomize'] = False
root_dir = cfg['root_dir']
if target_task != 'segment2d' and target_task != 'segment25d':
cfg['num_read_threads'] = 1
else:
cfg['num_read_threads'] = 1
print(cfg['log_root'])
if is_transfer:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train'))
# Try latest checkpoint by time
if cfg['model_path'] is None:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train', 'time'))
# Try to get one saved manually
if cfg['model_path'] is None:
cfg['model_path'] = os.path.join(cfg['log_root'], task,
"model.permanent-ckpt")
else:
if target_task != 'vanishing_point_well_defined' and target_task != 'jigsaw':
cfg['model_path'] = os.path.join(cfg['dataset_dir'],
"model_log_final", target_task,
"logs/model.permanent-ckpt")
import tempfile
import subprocess
dirs, fname = os.path.split(cfg['model_path'])
dst_dir = dirs.replace(cfg['dataset_dir'],
"s3://taskonomy-unpacked-oregon")
tmp_path = "/home/ubuntu/temp"
tmp_fname = os.path.join(tmp_path, fname)
aws_cp_command = "aws s3 cp {}.data-00000-of-00001 {}".format(
os.path.join(dst_dir, fname), tmp_path)
subprocess.call(aws_cp_command, shell=True)
aws_cp_command = "aws s3 cp {}.meta {}".format(
os.path.join(dst_dir, fname), tmp_path)
subprocess.call(aws_cp_command, shell=True)
aws_cp_command = "aws s3 cp {}.index {}".format(
os.path.join(dst_dir, fname), tmp_path)
subprocess.call(aws_cp_command, shell=True)
cfg['model_path'] = tmp_fname
else:
cfg['model_path'] = os.path.join(cfg['log_root'], target_task,
'model.permanent-ckpt')
print(cfg['model_path'])
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_representation_extraction
return cfg, is_transfer, target_task, config_name
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
import pdb
global synset
synset_1000 = [" ".join(i.split(" ")[1:]) for i in synset]
select = np.asarray([ 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1.,
1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,
0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 1.,
1., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0.,
0., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0.,
0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 0., 0., 1.,
0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 1., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 1., 1., 0., 0., 1., 0., 1.,
0., 1., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0.,
0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,
0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 1.,
0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0.,
0., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0.])
with open('/home/ubuntu/task-taxonomy-331b/lib/data/places_class_names.txt', 'r') as fp:
synset_places = [x.rstrip()[4:-1] for x,y in zip(fp.readlines(), select) if y == 1.]
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
if args.task is not 'NONE':
args.idx = list_of_tasks.index(args.task)
for idx, task in enumerate(list_of_tasks):
if idx != args.idx and args.idx != -1:
continue
if task == 'class_places':
synset = synset_places
elif task == 'class_1000':
synset = synset_1000
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() }
# task = '{f}__{t}__{hs}'.format(f=task_from, t=task_to, hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{TASK}'.format(TASK=task)
############## Load Configs ##############
cfg = utils.load_config( CONFIG_DIR, nopause=True )
RuntimeDeterminedEnviromentVars.register_dict( cfg )
split_file = os.path.join('/home/ubuntu/task-taxonomy-331b/assets/aws_data/', 'video2_info.pkl')
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
cfg['num_epochs'] = 2
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
print(cfg['log_root'])
cfg['model_path'] = os.path.join(
cfg['log_root'],
task,
'model.permanent-ckpt'
)
print( cfg['model_path'])
if cfg['model_path'] is None:
continue
cfg['dataset_dir'] = '/home/ubuntu'
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_representation_extraction
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input( cfg, is_training=ON_TEST_SET, use_filename_queue=False ) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model( inputs, cfg, is_training=IN_TRAIN_MODE )
m = model[ 'model' ]
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False )
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=( training_runners[ 'sess' ], training_runners[ 'coord' ] ))
prefetch_threads.start()
list_of_fname = np.load('/home/ubuntu/task-taxonomy-331b/assets/aws_data/video2_fname.npy')
import errno
try:
os.mkdir('/home/ubuntu/{}'.format(task))
os.mkdir('/home/ubuntu/{}/vid1'.format(task))
os.mkdir('/home/ubuntu/{}/vid2'.format(task))
os.mkdir('/home/ubuntu/{}/vid3'.format(task))
os.mkdir('/home/ubuntu/{}/vid4'.format(task))
except OSError as e:
if e.errno != errno.EEXIST:
raise
curr_comp = np.zeros((3,64))
curr_fit_img = np.zeros((256,256,3))
embeddings = []
############## Run First Batch ##############
for step_num in range(inputs['max_steps'] - 1):
#for step_num in range(1):
#if step_num > 0 and step_num % 20 == 0:
print(step_num)
if not hasattr(m, 'masks'):
(
input_batch, target_batch,
data_idx,
predicted, loss,
) = training_runners['sess'].run( [
m.input_images, m.targets,
model[ 'data_idxs' ],
m.decoder_output, m.total_loss] )
mask_batch = 1.
else:
(
input_batch, target_batch, mask_batch,
data_idx,
predicted, loss,
) = training_runners['sess'].run( [
m.input_images, m.targets, m.masks,
model[ 'data_idxs' ],
m.decoder_output, m.total_loss] )
if task == 'segment2d' or task == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32,256,256,3), dtype='float')
k_embed = 8
# for i in range(predicted.shape[0]):
# embedding_flattened = np.squeeze(predicted[i]).reshape((-1,64))
# pca = PCA(n_components=3)
# pca.fit(embedding_flattened)
# min_order = None
# min_dist = float('inf')
# for order in itertools.permutations([0,1,2]):
# reordered = pca.components_[list(order), :]
# dist = np.linalg.norm(curr_comp-reordered)
# if dist < min_dist:
# min_order = list(order)
# min_dist = dist
# print(min_order)
# pca.components_ = pca.components_[min_order, :]
# curr_comp = pca.components_
# lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
# lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
# x[i] = lower_dim
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape((-1,64))
embeddings.append(embedding_flattened)
if len(embeddings) > k_embed:
embeddings.pop(0)
pca = PCA(n_components=3)
pca.fit(np.vstack(embeddings))
min_order = None
min_dist = float('inf')
copy_of_comp = np.copy(pca.components_)
for order in itertools.permutations([0,1,2]):
#reordered = pca.components_[list(order), :]
#dist = np.linalg.norm(curr_comp-reordered)
pca.components_ = copy_of_comp[order, :]
lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
dist = np.linalg.norm(lower_dim - curr_fit_img)
if dist < min_dist:
min_order = order
min_dist = dist
pca.components_ = copy_of_comp[min_order, :]
lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
curr_fit_img = np.copy(lower_dim)
x[i] = lower_dim
predicted = x
if task == 'curvature':
std = [31.922, 21.658]
mean = [123.572, 120.1]
predicted = (predicted * std) + mean
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
predicted = np.squeeze(np.clip(predicted.astype(int) / 255., 0., 1. )[:,:,:,0])
just_rescale = ['autoencoder', 'denoise', 'edge2d',
'edge3d', 'keypoint2d', 'keypoint3d',
'reshade', 'rgb2sfnorm']
if task in just_rescale:
predicted = (predicted + 1.) / 2.
predicted = np.clip(predicted, 0., 1.)
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
just_clip = ['rgb2depth', 'rgb2mist']
if task in just_clip:
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
if task == 'segmentsemantic_rb':
label = np.argmax(predicted, axis=-1)
COLORS = ('white','red', 'blue', 'yellow', 'magenta',
'green', 'indigo', 'darkorange', 'cyan', 'pink',
'yellowgreen', 'black', 'darkgreen', 'brown', 'gray',
'purple', 'darkviolet')
rgb = (input_batch + 1.) / 2.
preds = [color.label2rgb(np.squeeze(x), np.squeeze(y), colors=COLORS, kind='overlay')[np.newaxis,:,:,:] for x,y in zip(label, rgb)]
predicted = np.vstack(preds)
if task in ['class_1000', 'class_places']:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode('utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
sorted_pred = np.argsort(predict_output)[::-1]
top_5_pred = [synset[sorted_pred[i]] for i in range(5)]
to_print_pred = "Top 5 prediction: \n {}\n {}\n {}\n {} \n {}".format(*top_5_pred)
img = Image.new('RGBA', (400, 200), (255, 255, 255))
d = ImageDraw.Draw(img)
fnt = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSerifCondensed.ttf', 25)
d.text((20, 5), to_print_pred, fill=(255, 0, 0), font=fnt)
img.save(to_store_name, 'PNG')
else:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode('utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
scipy.misc.toimage(np.squeeze(predict_output), cmin=0.0, cmax=1.0).save(to_store_name)
subprocess.call('tar -czvf /home/ubuntu/{t}.tar.gz /home/ubuntu/{t}'.format(t=task), shell=True)
subprocess.call('aws s3 cp /home/ubuntu/{t}.tar.gz s3://task-preprocessing-512-oregon/video2/'.format(t=task), shell=True)
subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid2/020%04d.png -vcodec libx264 -crf 15 -pix_fmt yuv420p {t}_2.mp4'.format(t=task), shell=True)
subprocess.call('aws s3 cp {t}_2.mp4 s3://task-preprocessing-512-oregon/video2/'.format(t=task), shell=True)
############## Clean Up ##############
training_runners[ 'coord' ].request_stop()
training_runners[ 'coord' ].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
print("Done: {}".format(task))
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
tf.logging.set_verbosity(tf.logging.ERROR)
all_outputs = {}
pickle_dir = 'viz_output_single_task.pkl'
import os
if os.path.isfile(pickle_dir):
with open(pickle_dir, 'rb') as fp:
all_outputs = pickle.load(fp)
for task in list_of_tasks:
if task in all_outputs:
print("{} already exists....\n\n\n".format(task))
continue
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
# task = '{f}__{t}__{hs}'.format(f=task_from, t=task_to, hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{TASK}'.format(
TASK=task)
############## Load Configs ##############
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
split_file = cfg['test_filenames'] if ON_TEST_SET else cfg[
'val_filenames']
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
cfg['num_epochs'] = 1
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
print(cfg['log_root'])
if task == 'jigsaw':
continue
cfg['model_path'] = os.path.join(cfg['log_root'], task,
'model.permanent-ckpt')
print(cfg['model_path'])
if cfg['model_path'] is None:
continue
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input(
cfg, is_training=ON_TEST_SET, use_filename_queue=False
) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=IN_TRAIN_MODE)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False)
prefetch_threads = threading.Thread(target=data_prefetch_init_fn,
args=(training_runners['sess'],
training_runners['coord']))
prefetch_threads.start()
############## Run First Batch ##############
if not hasattr(m, 'masks'):
(
input_batch,
target_batch,
data_idx,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.targets, model['data_idxs'],
m.decoder_output, m.total_loss
])
mask_batch = 1.
else:
(
input_batch,
target_batch,
mask_batch,
data_idx,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.targets, m.masks, model['data_idxs'],
m.decoder_output, m.total_loss
])
if task == 'segment2d' or task == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
pca = PCA(n_components=3)
pca.fit(embedding_flattened)
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() -
lower_dim.min())
x[i] = lower_dim
predicted = x
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
to_store = {
'input': input_batch,
'target': target_batch,
'mask': mask_batch,
'data_idx': data_idx,
'output': predicted
}
all_outputs[task] = to_store
print("Done: {}".format(task))
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
try:
del sys.modules['config']
except:
pass
sys.path = remove_dups(sys.path)
print("FINISHED: {}\n\n\n\n\n\n".format(task))
pickle_dir = 'viz_output_single_task.pkl'
with open(pickle_dir, 'wb') as fp:
pickle.dump(all_outputs, fp)
try:
subprocess.call(
"aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/"
.format(pickle_dir),
shell=True)
except:
subprocess.call(
"sudo cp {} /home/ubuntu/s3/visualizations/".format(
pickle_dir),
shell=True)
return
def run_to_task():
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
args = parser.parse_args()
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
tf.logging.set_verbosity(tf.logging.ERROR)
image_list = glob.glob(args.img_dir +"/*.png")
image_list.sort()
#image_list=image_list[:5]
print(len(image_list))
for task in tqdm(list_of_tasks):
print("Task is ", task)
if task not in list_of_tasks:
raise ValueError('Task not supported')
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
segment25d segment2d room_layout'.split()
cfg = generate_cfg(task)
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() }
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn( cfg, is_training=False, use_filename_queue=False )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model( inputs, cfg, is_training=False )
m = model[ 'model' ]
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
#Prints all the class variables and functions
print(dir(m))
for img_name in tqdm(image_list):
filename = img_name.split('/')[-1].split('.')[0]
img = load_raw_image_center_crop( img_name )
img = skimage.img_as_float(img)
scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(img_name)
# Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value
if task == 'jigsaw' :
img = cfg[ 'input_preprocessing_fn' ]( img, target=cfg['target_dict'][random.randint(0,99)],
**cfg['input_preprocessing_fn_kwargs'] )
else:
img = cfg[ 'input_preprocessing_fn' ]( img, **cfg['input_preprocessing_fn_kwargs'] )
img = img[np.newaxis,:]
if task in fc_task_list:
predicted, representation, decoder_features, encoder_features = training_runners['sess'].run(
[ m.decoder_output, m.encoder_output, m.metric_endpoints, m.encoder_endpoints ], feed_dict={m.input_images: img} )
else:
predicted, representation, decoder_features, encoder_features = training_runners['sess'].run(
[ m.decoder_output, m.encoder_output, m.decoder_endpoints, m.encoder_endpoints ], feed_dict={m.input_images: img} )
#np.save(save_path,value)
#for name,value in decoder_features.items():
# print (name)
## CKD : Uncomment below for loop
for name,value in encoder_features.items():
if name in encoder_save_list or name in feedforward_encoder_save_list:
#print (name)
name = name.replace('/', '_')
save_path = os.path.join(args.save_dir,filename+"_"+task + "_" + name + ".npy")
np.save(save_path,value)
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(representation))
if args.store_pred:
save_path = os.path.join(args.save_dir,filename+"_"+task + "_" + "prediction" + ".npy")
with open(save_path, 'wb') as fp:
np.save(fp, np.squeeze(predicted))
#if task == 'segment2d' or task == 'segment25d':
# segmentation_pca(predicted, args.store_name)
#return
#if task == 'colorization':
# single_img_colorize(predicted, img , args.store_name)
#return
#if task == 'curvature':
# curvature_single_image(predicted, args.store_name)
#return
#just_rescale = ['autoencoder', 'denoise', 'edge2d',
# 'edge3d', 'keypoint2d', 'keypoint3d',
# 'reshade', 'rgb2sfnorm' ]
#if task in just_rescale:
# print(args.store_name)
#simple_rescale_img(predicted, args.store_name)
#return
#just_clip = ['rgb2depth', 'rgb2mist']
#if task in just_clip:
# depth_single_image(predicted, args.store_name)
# #return
#if task == 'inpainting_whole':
# inpainting_bbox(predicted, args.store_name)
#return
#if task == 'segmentsemantic':
# semseg_single_image( predicted, img, args.store_name)
#return
#if task in ['class_1000', 'class_places']:
# print("The shape of predicted is ---------------", predicted.shape)
#classification(predicted, synset, args.store_name)
#return
#if task == 'vanishing_point':
# _ = plot_vanishing_point_smoothed(np.squeeze(predicted), (np.squeeze(img) + 1. )/2., args.store_name, [])
#return
#if task == 'room_layout':
# mean = np.array([0.006072743318127848, 0.010272365569691076, -3.135909774145468,
# 1.5603802322235532, 5.6228218371102496e-05, -1.5669352793761442,
# 5.622875878174759, 4.082800262277375, 2.7713941642895956])
# std = np.array([0.8669452525283652, 0.687915294956501, 2.080513632043758,
# 0.19627420479282623, 0.014680602791251812, 0.4183827359302299,
# 3.991778013006544, 2.703495278378409, 1.2269185938626304])
# predicted = predicted * std + mean
# plot_room_layout(np.squeeze(predicted), (np.squeeze(img) + 1. )/2., args.store_name, [], cube_only=True)
#return
#if task == 'jigsaw':
# predicted = np.argmax(predicted, axis=1)
# perm = cfg[ 'target_dict' ][ predicted[0] ]
# show_jigsaw((np.squeeze(img) + 1. )/2., perm, args.store_name)
#return
############## Clean Up ##############
training_runners[ 'coord' ].request_stop()
training_runners[ 'coord' ].join()
print("Done: {}".format(task))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def get_cfg(nopause=False):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, INPUT_TASK)
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = general_utils.load_config(cfg['config_dir_input'],
nopause=True)
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'],
nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/val_image_split_0.npy'))
cfg['val_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/test_image_split_0.npy'))
# Define where the extracted representations are stored
cfg['train_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='train' if transitive else 'val'))
cfg['val_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='val' if transitive else 'test'))
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(
target_cfg['log_root'], TARGET_TASK)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 15
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = (16, 16, 8)
cfg['representation_dtype'] = tf.float32
## SETUP MODEL
# Transfer
cfg['encoder'] = transfer_two_stream_with_bn_ends
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int(
"<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'side_encoder_func': side_encoder,
'output_channels': cfg['representation_dim'][-1],
'kernel_size': [KERNEL_SIZE, KERNEL_SIZE],
'stride': 1,
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.8, #0.95
'weight_decay': cfg['weight_decay'],
'num_layers': int(NUM_LAYERS),
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
# 'flatten_output': (TARGET_TASK in FLATTEN_TASKS)
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(5000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_transitive_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_transitive_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
cfg['log_root'] = log_root
cfg['log_dir'] = os.path.join(log_root, 'logs')
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def run_extract_representations(args, cfg, save_dir, given_task):
transfer = (cfg['model_type'] == architectures.TransferNet)
if transfer:
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer
setup_input_fn = utils.setup_input_transfer
else:
setup_input_fn = utils.setup_input
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn
# set up logging
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as g:
# create ops and placeholders
tf.logging.set_verbosity(tf.logging.INFO)
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
# set up metrics to evaluate
# names_to_values, names_to_updates = setup_metrics( inputs, model, cfg )
# execute training
start_time = time.time()
utils.print_start_info(cfg, inputs['max_steps'], is_training=False)
# start session and restore model
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
try:
if cfg['model_path'] is None:
print('Please specify a checkpoint directory')
return
utils.print_start_info(cfg, inputs['max_steps'], is_training=False)
data_prefetch_init_fn = get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=False)
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
# run one example so that we can calculate some statistics about the representations
filenames = []
representations, data_idx = training_runners['sess'].run(
[inputs['input_batch'], inputs['data_idxs']])
filenames.extend(data_idx)
if type(representations) == list:
representations = representations[0]
representations = representations.reshape(
(-1, np.prod(cfg['input_dim'])))
print('Got first batch representation with size: {0}'.format(
representations.shape))
# run the remaining examples
for step in range(inputs['max_steps'] - 1):
#for step in range( 10 ):
if step % 100 == 0:
print('Step {0} of {1}'.format(step,
inputs['max_steps'] - 1))
# This is just for GAN, for the LEO meeting
encoder_output, data_idx = training_runners['sess'].run(
[inputs['input_batch'], inputs['data_idxs']])
if type(encoder_output) == list:
encoder_output = encoder_output[0]
representations = np.append(representations,
encoder_output.reshape(
(-1,
np.prod(cfg['input_dim']))),
axis=0)
filenames.extend(data_idx)
if training_runners['coord'].should_stop():
break
print(
'The size of representations is %s while we expect it to run for %d steps with batchsize %d'
% (representations.shape, inputs['max_steps'],
cfg['batch_size']))
end_train_time = time.time() - start_time
save_path = os.path.join(
save_dir, '{task}_{split}_representations.pkl'.format(
task='pixels', split=args.data_split))
with open(save_path, 'wb') as f:
pickle.dump(
{
'file_indexes': filenames,
'representations': representations
}, f)
copy_to = None
if args.out_dir:
os.system("sudo cp {fp} {out}/".format(fp=save_path,
out=args.out_dir))
copy_to = args.out_dir
else:
if transfer:
os.system(
"sudo cp {fp} /home/ubuntu/s3/model_log/representations_transfer/"
.format(fp=save_path))
copy_to = '/home/ubuntu/s3/model_log/representations_transfer/'
else:
os.system(
"sudo cp {fp} /home/ubuntu/s3/model_log/representations/"
.format(fp=save_path))
copy_to = "/home/ubuntu/s3/model_log/representations/"
print('saved representations to {0}'.format(save_path))
print('copied representations to {0}'.format(copy_to))
print('time to extract %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
finally:
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
def run_val_test(cfg):
# set up logging
tf.logging.set_verbosity(tf.logging.INFO)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
# create ops and placeholders
inputs = utils.setup_input(cfg, is_training=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model(inputs, cfg, is_training=False)
# full_path = tf.train.latest_checkpoint(checkpoint_dir)
# step = full_path.split('-')[-1]
# model_path = os.path.join('/home/ubuntu/s3/model_log', cfg['task_name'], 'model.permanent-ckpt')
model_path = os.path.join('/home/ubuntu/s3/model_log_final',
cfg['task_name'], 'model.permanent-ckpt')
model['saver_op'].restore(training_runners['sess'], model_path)
m = model['model']
# execute training
start_time = time.time()
utils.print_start_info(cfg, inputs['max_steps'], is_training=False)
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=False)
prefetch_threads = threading.Thread(target=data_prefetch_init_fn,
args=(training_runners['sess'],
training_runners['coord']))
prefetch_threads.start()
print("Dataloading workers dispatched....")
return_accuracy = 'return_accuracy' in cfg and cfg['return_accuracy'],
losses_mean = AverageMeter()
accuracy_mean = AverageMeter()
for step in range(inputs['max_steps']):
#print(step)
if return_accuracy:
(data_idx, loss, accuracy) = training_runners['sess'].run(
[model['data_idxs'], m.losses[0], m.accuracy])
losses_mean.update(loss)
accuracy_mean.update(accuracy)
if step % 100 == 0:
print(
'Step: {step} with Current Losses mean: {loss}; with accuracy: {accur}'
.format(step=step,
loss=losses_mean.avg,
accur=accuracy_mean.avg))
else:
(data_idx, loss) = training_runners['sess'].run(
[model['data_idxs'], m.losses[0]])
losses_mean.update(loss)
if step % 100 == 0:
print('Step: {step} with Current Losses mean: {loss}'.format(
step=step, loss=losses_mean.avg))
if return_accuracy:
print('Final Losses mean: {loss}; with accuracy: {accur}'.format(
loss=losses_mean.avg, accur=accuracy_mean.avg))
else:
print('Final Losses mean: {loss}'.format(loss=losses_mean.avg))
end_train_time = time.time() - start_time
print('time to train %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
print('avg time per epoch: %.3f hrs' % ((end_train_time /
(60 * 60)) / cfg['num_epochs']))