def generate_cfg(task):
repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
CONFIG_DIR = os.path.join(repo_dir, 'experiments/final', task)
############## Load Configs ##############
import utils
import data.load_ops as load_ops
from general_utils import RuntimeDeterminedEnviromentVars
cfg = utils.load_config( CONFIG_DIR, nopause=True )
RuntimeDeterminedEnviromentVars.register_dict( cfg )
cfg['batch_size'] = 1
if 'batch_size' in cfg['encoder_kwargs']:
cfg['encoder_kwargs']['batch_size'] = 1
cfg['model_path'] = os.path.join( repo_dir, 'temp', task, 'model.permanent-ckpt' )
cfg['root_dir'] = repo_dir
return cfg
def generate_cfg(task):
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{}'.format(
task)
############## Load Configs ##############
import utils
import data.load_ops as load_ops
from general_utils import RuntimeDeterminedEnviromentVars
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
root_dir = cfg['root_dir']
cfg['batch_size'] = 1
if 'batch_size' in cfg['encoder_kwargs']:
cfg['encoder_kwargs']['batch_size'] = 1
cfg['model_path'] = os.path.join('/home/ubuntu/temp', task,
'model.permanent-ckpt')
return cfg
def get_cfg(nopause=False):
cfg = {}
cfg['is_discriminative'] = True
# params
cfg['num_epochs'] = 30
cfg['model_path'] = None
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
# model
cfg['model_type'] = architectures.BasicFF
cfg['weight_decay'] = 1e-5
cfg['instance_noise_sigma'] = 0.1
cfg['instance_noise_anneal_fn'] = tf.train.inverse_time_decay
cfg['instance_noise_anneal_fn_kwargs'] = {
'decay_rate': 0.2,
'decay_steps': 1000
}
batch_size = 32
cfg['encoder'] = resnet_encoder_fully_convolutional_16x16x8
cfg['hidden_size'] = 1024
cfg['encoder_kwargs'] = {
'resnet_build_fn': resnet_v1_50_16x16,
'weight_decay': cfg['weight_decay'],
'flatten': True,
'batch_size': batch_size
}
cfg['return_accuracy'] = True
# learning
cfg['initial_learning_rate'] = 1e-4
cfg['optimizer'] = tf.train.AdamOptimizer
cfg['optimizer_kwargs'] = {}
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(80000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# inputs
cfg['input_dim'] = (256, 256) # (1024, 1024)
cfg['input_num_channels'] = 3
cfg['input_dtype'] = tf.float32
cfg['input_domain_name'] = 'rgb'
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image
cfg['input_preprocessing_fn_kwargs'] = {
'new_dims': cfg['input_dim'],
'new_scale': [-1, 1]
}
cfg['single_filename_to_multiple'] = True
# outputs
cfg['target_dim'] = 63 # (1024, 1024)
cfg['target_dtype'] = tf.float32
cfg['target_from_filenames'] = load_ops.class_places_workspace_and_home
cfg['mask_by_target_func'] = True
# Post processing
cfg['metric_net'] = encoder_multilayers_fc_bn
cfg['metric_kwargs'] = {
'hidden_size': 2048,
'layer_num': 2,
'output_size': cfg['target_dim']
}
# input pipeline
cfg['preprocess_fn'] = load_and_specify_preprocessors
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = batch_size
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 600
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
import pdb
global synset
synset_1000 = [" ".join(i.split(" ")[1:]) for i in synset]
select = np.asarray([ 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1.,
1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,
0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 1.,
1., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0.,
0., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0.,
0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 0., 0., 1.,
0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 1., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 1., 1., 0., 0., 1., 0., 1.,
0., 1., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0.,
0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,
0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 1.,
0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0.,
0., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0.])
with open('/home/ubuntu/task-taxonomy-331b/lib/data/places_class_names.txt', 'r') as fp:
synset_places = [x.rstrip()[4:-1] for x,y in zip(fp.readlines(), select) if y == 1.]
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
if args.task is not 'NONE':
args.idx = list_of_tasks.index(args.task)
for idx, task in enumerate(list_of_tasks):
if idx != args.idx and args.idx != -1:
continue
if task == 'class_places':
synset = synset_places
elif task == 'class_1000':
synset = synset_1000
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() }
# task = '{f}__{t}__{hs}'.format(f=task_from, t=task_to, hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{TASK}'.format(TASK=task)
############## Load Configs ##############
cfg = utils.load_config( CONFIG_DIR, nopause=True )
RuntimeDeterminedEnviromentVars.register_dict( cfg )
split_file = os.path.join('/home/ubuntu/task-taxonomy-331b/assets/aws_data/', 'video2_info.pkl')
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
cfg['num_epochs'] = 2
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
print(cfg['log_root'])
cfg['model_path'] = os.path.join(
cfg['log_root'],
task,
'model.permanent-ckpt'
)
print( cfg['model_path'])
if cfg['model_path'] is None:
continue
cfg['dataset_dir'] = '/home/ubuntu'
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_representation_extraction
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input( cfg, is_training=ON_TEST_SET, use_filename_queue=False ) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model( inputs, cfg, is_training=IN_TRAIN_MODE )
m = model[ 'model' ]
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False )
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=( training_runners[ 'sess' ], training_runners[ 'coord' ] ))
prefetch_threads.start()
list_of_fname = np.load('/home/ubuntu/task-taxonomy-331b/assets/aws_data/video2_fname.npy')
import errno
try:
os.mkdir('/home/ubuntu/{}'.format(task))
os.mkdir('/home/ubuntu/{}/vid1'.format(task))
os.mkdir('/home/ubuntu/{}/vid2'.format(task))
os.mkdir('/home/ubuntu/{}/vid3'.format(task))
os.mkdir('/home/ubuntu/{}/vid4'.format(task))
except OSError as e:
if e.errno != errno.EEXIST:
raise
curr_comp = np.zeros((3,64))
curr_fit_img = np.zeros((256,256,3))
embeddings = []
############## Run First Batch ##############
for step_num in range(inputs['max_steps'] - 1):
#for step_num in range(1):
#if step_num > 0 and step_num % 20 == 0:
print(step_num)
if not hasattr(m, 'masks'):
(
input_batch, target_batch,
data_idx,
predicted, loss,
) = training_runners['sess'].run( [
m.input_images, m.targets,
model[ 'data_idxs' ],
m.decoder_output, m.total_loss] )
mask_batch = 1.
else:
(
input_batch, target_batch, mask_batch,
data_idx,
predicted, loss,
) = training_runners['sess'].run( [
m.input_images, m.targets, m.masks,
model[ 'data_idxs' ],
m.decoder_output, m.total_loss] )
if task == 'segment2d' or task == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32,256,256,3), dtype='float')
k_embed = 8
# for i in range(predicted.shape[0]):
# embedding_flattened = np.squeeze(predicted[i]).reshape((-1,64))
# pca = PCA(n_components=3)
# pca.fit(embedding_flattened)
# min_order = None
# min_dist = float('inf')
# for order in itertools.permutations([0,1,2]):
# reordered = pca.components_[list(order), :]
# dist = np.linalg.norm(curr_comp-reordered)
# if dist < min_dist:
# min_order = list(order)
# min_dist = dist
# print(min_order)
# pca.components_ = pca.components_[min_order, :]
# curr_comp = pca.components_
# lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
# lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
# x[i] = lower_dim
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape((-1,64))
embeddings.append(embedding_flattened)
if len(embeddings) > k_embed:
embeddings.pop(0)
pca = PCA(n_components=3)
pca.fit(np.vstack(embeddings))
min_order = None
min_dist = float('inf')
copy_of_comp = np.copy(pca.components_)
for order in itertools.permutations([0,1,2]):
#reordered = pca.components_[list(order), :]
#dist = np.linalg.norm(curr_comp-reordered)
pca.components_ = copy_of_comp[order, :]
lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
dist = np.linalg.norm(lower_dim - curr_fit_img)
if dist < min_dist:
min_order = order
min_dist = dist
pca.components_ = copy_of_comp[min_order, :]
lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
curr_fit_img = np.copy(lower_dim)
x[i] = lower_dim
predicted = x
if task == 'curvature':
std = [31.922, 21.658]
mean = [123.572, 120.1]
predicted = (predicted * std) + mean
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
predicted = np.squeeze(np.clip(predicted.astype(int) / 255., 0., 1. )[:,:,:,0])
just_rescale = ['autoencoder', 'denoise', 'edge2d',
'edge3d', 'keypoint2d', 'keypoint3d',
'reshade', 'rgb2sfnorm']
if task in just_rescale:
predicted = (predicted + 1.) / 2.
predicted = np.clip(predicted, 0., 1.)
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
just_clip = ['rgb2depth', 'rgb2mist']
if task in just_clip:
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
if task == 'segmentsemantic_rb':
label = np.argmax(predicted, axis=-1)
COLORS = ('white','red', 'blue', 'yellow', 'magenta',
'green', 'indigo', 'darkorange', 'cyan', 'pink',
'yellowgreen', 'black', 'darkgreen', 'brown', 'gray',
'purple', 'darkviolet')
rgb = (input_batch + 1.) / 2.
preds = [color.label2rgb(np.squeeze(x), np.squeeze(y), colors=COLORS, kind='overlay')[np.newaxis,:,:,:] for x,y in zip(label, rgb)]
predicted = np.vstack(preds)
if task in ['class_1000', 'class_places']:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode('utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
sorted_pred = np.argsort(predict_output)[::-1]
top_5_pred = [synset[sorted_pred[i]] for i in range(5)]
to_print_pred = "Top 5 prediction: \n {}\n {}\n {}\n {} \n {}".format(*top_5_pred)
img = Image.new('RGBA', (400, 200), (255, 255, 255))
d = ImageDraw.Draw(img)
fnt = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSerifCondensed.ttf', 25)
d.text((20, 5), to_print_pred, fill=(255, 0, 0), font=fnt)
img.save(to_store_name, 'PNG')
else:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode('utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
scipy.misc.toimage(np.squeeze(predict_output), cmin=0.0, cmax=1.0).save(to_store_name)
subprocess.call('tar -czvf /home/ubuntu/{t}.tar.gz /home/ubuntu/{t}'.format(t=task), shell=True)
subprocess.call('aws s3 cp /home/ubuntu/{t}.tar.gz s3://task-preprocessing-512-oregon/video2/'.format(t=task), shell=True)
subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid2/020%04d.png -vcodec libx264 -crf 15 -pix_fmt yuv420p {t}_2.mp4'.format(t=task), shell=True)
subprocess.call('aws s3 cp {t}_2.mp4 s3://task-preprocessing-512-oregon/video2/'.format(t=task), shell=True)
############## Clean Up ##############
training_runners[ 'coord' ].request_stop()
training_runners[ 'coord' ].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
print("Done: {}".format(task))
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
from importlib import reload
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
# for arch in ['regular', 'shallow', 'dilated_shallow', 'dilated_regular']:
arch = args.arch
data_amount = args.data
if args.second_order:
global TRANSFER_TYPE
TRANSFER_TYPE = 'second_order'
if not args.no_regenerate_data:
#if False:
all_outputs = {}
pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format(
arch=arch, hs=args.hs, task_to=task_to)
subprocess.call(
"aws s3 cp s3://task-preprocessing-512-oregon/visualizations/transfer_viz/viz_{}.pkl {}"
.format(task_to, pickle_dir),
shell=True)
import os
if os.path.isfile(pickle_dir):
with open(pickle_dir, 'rb') as fp:
all_outputs = pickle.load(fp)
if args.second_order:
import itertools
with open(
'/home/ubuntu/task-taxonomy-331b/tools/ranked_first_order_transfers.pkl',
'rb') as fp:
data = pickle.load(fp)
list_of_src_tasks = list(
itertools.combinations(data[task_to][:5], 2))
list_of_src_tasks = [
'{}__{}'.format(x[0], x[1]) for x in list_of_src_tasks
]
with open(
'/home/ubuntu/task-taxonomy-331b/tools/second_order_should_flip.pkl',
'rb') as fp:
to_flip_dict = pickle.load(fp)
if args.find_all_src:
config_dir_root = '/home/ubuntu/task-taxonomy-331b/experiments/second_order/{arch}/{data}'.format(
arch=arch, data=data_amount)
all_configs = os.listdir(config_dir_root)
list_of_src_tasks = []
for i in all_configs:
if i.split('__')[-3] == task_to:
list_of_src_tasks.append(i)
list_of_src_tasks = [
'__'.join(x.split('__')[:2]) for x in list_of_src_tasks
]
rank_combo = {}
for task_from in list_of_src_tasks:
first, sec = task_from.split('__')
rank_combo[task_from] = (data[task_to].index(first),
data[task_to].index(sec))
global list_of_src_tasks
for i, task_from in enumerate(list_of_src_tasks):
if args.data == '16k':
if task_from in all_outputs:
print("{} already exists....\n\n\n".format(task_from))
continue
else:
if '{}_{}'.format(task_from, args.data) in all_outputs:
print("{} already exists....\n\n\n".format(task_from))
continue
print("Doing from {task_from} to {task_to}".format(
task_from=task_from, task_to=task_to))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
if task_from == "FULL" or task_from == "FULL_IMAGE":
task = '{f}__{t}__{hs}__unlocked'.format(f="FULL",
t=task_to,
hs=args.hs)
transfer_src = 'full_order' if task_from == "FULL" else 'full_order_image'
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=transfer_src,
arch=arch,
data=data_amount,
TASK=task)
elif task_from == "FULL_select" or task_from == "FULL_select_IMAGE":
task = '{f}__{t}__{hs}__unlocked'.format(f="FULL_select",
t=task_to,
hs=args.hs)
transfer_src = 'full_order_selected' if task_from == "FULL_select" else 'full_order_selected_image'
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=transfer_src,
arch=arch,
data=data_amount,
TASK=task)
else:
task = '{f}__{t}__{hs}__unlocked'.format(f=task_from,
t=task_to,
hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=TRANSFER_TYPE,
arch=arch,
data=data_amount,
TASK=task)
print(CONFIG_DIR)
############## Load Configs ##############
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
if args.second_order and not args.find_all_src and not to_flip_dict[
task]:
cfg['val_representations_file'] = cfg[
'val_representations_file'][::-1]
cfg['num_epochs'] = 1
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train'
#'time'
))
# print(cfg['model_path'])
if cfg['model_path'] is None and task == 'random':
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train',
'time'))
if cfg['model_path'] is None:
continue
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input_transfer(
cfg, is_training=ON_TEST_SET, use_filename_queue=False
) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=True)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn_transfer(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False)
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
############## Run First Batch ##############
(
input_batch,
representation_batch,
target_batch,
data_idx,
encoder_output,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.input_representations, m.decoder.targets,
model['data_idxs'], m.encoder_output, m.decoder.decoder_output,
m.total_loss
])
if task_to == 'segment2d' or task_to == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
pca = PCA(n_components=3)
pca.fit(embedding_flattened)
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (
lower_dim.max() - lower_dim.min())
x[i] = lower_dim
predicted = x
if task_to == 'segmentsemantic_rb':
predicted = np.argmax(predicted, axis=-1)
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
to_store = {
'data_idx': data_idx,
'output': predicted,
'loss': loss
}
if args.second_order and args.find_all_src:
store_key = "{}_{}".format(task_from[:20],
rank_combo[task_from])
all_outputs[store_key] = to_store
elif args.data != '16k':
store_key = "{}_{}".format(task_from, args.data)
all_outputs[store_key] = to_store
else:
all_outputs[task_from] = to_store
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
#print(sys.modules.keys())
#del sys.modules[ 'config' ]
sys.path = remove_dups(sys.path)
print('Current Directory: ', os.getcwd())
pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format(
arch=arch, hs=args.hs, task_to=task_to)
with open(pickle_dir, 'wb') as fp:
pickle.dump(all_outputs, fp)
subprocess.call(
"aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/transfer_viz/viz_{}.pkl"
.format(pickle_dir, task_to),
shell=True)
# Run jupyter nb
print('Running Jupyter Notebooks...')
#os.makedirs("/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/transfer_{hs}_{arch}".format(hs=args.hs, arch=arch), exist_ok=True)
notebook_path = '/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/Visual_{task_to}'.format(
task_to=task_to)
if args.second_order and not args.find_all_src:
notebook_path = '{}-Copy1'.format(notebook_path)
subprocess.call("jupyter nbconvert \
--execute {notebook_path}.ipynb \
--to html \
--ExecutePreprocessor.kernel_name=python3 \
--ExecutePreprocessor.timeout=1200 ".format(
notebook_path=notebook_path, arch=arch, hs=args.hs, task_to=task_to),
shell=True)
subprocess.call(
"aws s3 cp {}.html s3://task-preprocessing-512-oregon/visualizations/{}/"
.format(notebook_path, TRANSFER_TYPE),
shell=True)
def get_cfg(nopause=False):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
import pdb
global INPUT_TASK
INPUT_TASK = INPUT_TASK.split(',')
representation_task = INPUT_TASK
cfg['multiple_input_tasks'] = len(INPUT_TASK)
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
cfg['retrain_decoder'] = ("<RETRAIN_DECODER>" == "True")
cfg['unlock_decoder'] = ("<UNLOCK_DECODER>" == "True")
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = [
'/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(CFG_DIR, t)
for t in INPUT_TASK
]
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = [
general_utils.load_config(inp_cfg, nopause=True)
for inp_cfg in cfg['config_dir_input']
]
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'],
nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath(
os.path.join('/home/ubuntu/s3',
'meta/val_image_split_{}.npy'.format(VAL_DATA_USED)))
cfg['val_list_of_fileinfos'] = os.path.abspath(
os.path.join('/home/ubuntu/s3', 'meta/test_image_split_0.npy'))
# Define where the extracted representations are stored
cfg['train_representations_file'] = [
os.path.join(
input_cfg[i]['log_root'], t,
'{task}_{train_split}_representations.pkl'.format(
task=t, train_split='train' if transitive else 'val'))
for i, t in enumerate(representation_task)
]
cfg['val_representations_file'] = [
os.path.join(
input_cfg[i]['log_root'], t,
'{task}_{train_split}_representations.pkl'.format(
task=t, train_split='val' if transitive else 'test'))
for i, t in enumerate(representation_task)
]
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
if TARGET_TASK in ['ego_motion', 'fix_pose', 'point_match', 'non_fixated_pose'] \
and VAL_DATA_USED not in ['16k','30k']:
target_cfg['train_filenames'] = target_cfg['train_filenames'].replace(
'val_split_image_info_', 'val_split_image_info_multi_in_')
cfg['train_list_of_fileinfos'] = cfg[
'train_list_of_fileinfos'].replace('val_image_split_',
'val_image_split_multi_in_')
target_cfg['train_filenames'] = os.path.abspath(
os.path.join('/home/ubuntu/s3',
'meta/val_split_image_info_{}.pkl'.format(VAL_DATA_USED)))
if TARGET_TASK == 'ego_motion':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_fixated_trips_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'fix_pose':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_fixated_pairs_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'point_match':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir, 'assets/aws_data/val_point_match_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'non_fixated_pose':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_nonfixated_pairs_info_{}.pkl'.
format(VAL_DATA_USED)))
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
if VAL_DATA_USED == '1k':
cfg['num_epochs'] = 200
elif VAL_DATA_USED == '2k':
cfg['num_epochs'] = 100
elif VAL_DATA_USED == '4k':
cfg['num_epochs'] = 50
elif VAL_DATA_USED == '8k':
cfg['num_epochs'] = 50
elif VAL_DATA_USED == '5h':
cfg['num_epochs'] = 400
elif VAL_DATA_USED == '1h':
cfg['num_epochs'] = 1000
elif VAL_DATA_USED == '16k':
cfg['num_epochs'] = 30
else:
cfg['num_epochs'] = 15
#cfg['num_epochs'] = 15 if not cfg['finetune_decoder'] else 7 if not cfg['finetune_decoder'] else 7
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = [16, 16, 8 * len(INPUT_TASK)]
cfg['representation_dtype'] = tf.float32
def create_bn_decay_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchnorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
return tf.train.piecewise_constant(global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(6682),
np.int32(6700)
],
values=[0.2, 0.5, 0.9, 0.999])
def create_clipping_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchrenorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
rmax = tf.train.piecewise_constant(global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000)
],
values=[1., 1.5, 2., 2.5, 3.])
rmin = 1. / rmax
dmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000),
np.int32(4500),
np.int32(5000),
],
values=[0., 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 5.0])
return {'rmax': rmax, 'rmin': rmin, 'dmax': dmax}
## SETUP MODEL
# Transfer
cfg['encoder'] = transfer_models.transfer_two_stream_with_bn_ends_l2l0_no_image
if INPUT_TASK == 'pixels':
raise NotImplementedError(
"Cannot transfer from 'pixels' when using no_image!")
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int(
"<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'side_encoder_func': transfer_models.side_encoder_l2,
'output_channels': cfg['representation_dim'][-1],
'kernel_size': [KERNEL_SIZE, KERNEL_SIZE],
'stride': 1,
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.8, #0.95
'weight_decay': cfg['weight_decay'],
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
# 'flatten_output': (TARGET_TASK in FLATTEN_TASKS)
}
cfg['encoder_kwargs']['renorm'] = True
cfg['encoder_kwargs']['renorm_momentum'] = 0.9
cfg['encoder_kwargs']['renorm_clipping'] = create_clipping_schedule
# Type of decoder to use
cfg['replace_target_decoder'] = transfer_models.__dict__[
REPLACE_TARGET_DECODER_FUNC_NAME]
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(5000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
cfg['log_root'] = config_dir.replace('task-taxonomy-331b/', 's3/')
if cfg['finetune_decoder'] and cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_ft')
elif cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_no_ft')
elif cfg['finetune_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'ft')
elif cfg['unlock_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'scratch')
cfg['log_dir'] = os.path.join(cfg['log_root'], 'logs')
# Set the model path that we will restore from
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(
target_cfg['log_root'], TARGET_TASK)
if cfg['retrain_decoder']:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], '..', 'logs', 'slim-train'))
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = '../s3/meta'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def get_cfg(nopause=False):
cfg = {}
cfg['is_discriminative'] = True
# params
cfg['num_epochs'] = 30
cfg['model_path'] = None
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
# model
cfg['model_type'] = architectures.Siamese
cfg['weight_decay'] = 2e-6
cfg['instance_noise_sigma'] = 0.1
cfg['instance_noise_anneal_fn'] = tf.train.inverse_time_decay
cfg['instance_noise_anneal_fn_kwargs'] = {
'decay_rate': 0.2,
'decay_steps': 1000
}
batch_size = 4
cfg['encoder'] = resnet_encoder_fully_convolutional_16x16x8
cfg['hidden_size'] = 1024
cfg['encoder_kwargs'] = {
'resnet_build_fn': resnet_v1_50_16x16,
'weight_decay': cfg['weight_decay'],
'flatten': True,
'batch_size': batch_size
}
cfg['return_accuracy'] = True
# learning
cfg['initial_learning_rate'] = 1e-4
cfg['optimizer'] = tf.train.AdamOptimizer
cfg['optimizer_kwargs'] = {}
cfg['discriminator_learning_args'] = {
'initial_learning_rate': 1e-5,
'optimizer': tf.train.GradientDescentOptimizer,
'optimizer_kwargs': {}
}
#cfg['clip_norm'] = 1
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(80000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# inputs
cfg['input_dim'] = (256, 256) # (1024, 1024)
cfg['input_num_channels'] = 3
cfg['input_dtype'] = tf.float32
cfg['num_input'] = 9
cfg['input_domain_name'] = 'rgb'
cfg['input_preprocessing_fn'] = load_ops.generate_jigsaw_input
# cfg['input_preprocessing_fn'] = load_ops.generate_jigsaw_input_for_representation_extraction
cfg['input_preprocessing_fn_kwargs'] = {
'new_dims': cfg['input_dim'],
'new_scale': [-1, 1]
}
cfg['single_filename_to_multiple'] = True
num_jigsaw_class = 100
# outputs
cfg['target_dim'] = 1 # (1024, 1024)
cfg['target_dtype'] = tf.int64
cfg['target_from_filenames'] = load_ops.jigsaw_rand_index
cfg['find_target_in_config'] = True
cfg['target_dict'] = np.load(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../../../lib/data', 'jigsaw_max_hamming_set.npy'))
# masks
cfg['metric_net'] = encoder_multilayers_fc_bn
cfg['metric_kwargs'] = {
'hidden_size': 2048,
'layer_num': 3,
'output_size': num_jigsaw_class,
'batch_norm_decay': 0.9,
'batch_norm_epsilon': 1e-5,
'batch_norm_scale': True,
'batch_norm_center': True
}
# input pipeline
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_input_depends_on_target
cfg['randomize'] = False
cfg['num_read_threads'] = 300
cfg['batch_size'] = batch_size
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 600
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def get_cfg( nopause=False ):
cfg = {}
# params
cfg['num_epochs'] = 30
cfg['model_path'] = None
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
# model
cfg['model_type'] = architectures.EncoderDecoderWithCGAN
cfg['n_g_steps_before_d'] = 1
cfg['n_d_steps_after_g'] = 1
cfg['init_g_steps'] = 25000
cfg['l_norm_weight_prop'] = 0.996
cfg['weight_decay'] = 2e-6
cfg['instance_noise_sigma'] = 0.1
cfg['instance_noise_anneal_fn'] = tf.train.inverse_time_decay
cfg['instance_noise_anneal_fn_kwargs'] = {
'decay_rate': 0.2,
'decay_steps': 1000
}
cfg['encoder'] = resnet_encoder_fully_convolutional_16x16x8
cfg['hidden_size'] = 1024
cfg['encoder_kwargs'] = {
'resnet_build_fn' : resnet_v1_50_16x16,
'weight_decay': cfg['weight_decay']
}
cfg['decoder'] = decoder_fc_15_layer_256_resolution_fully_convolutional_16x16x4
cfg['decoder_kwargs'] = {
'activation_fn': leaky_relu( 0.2 ),
'weight_decay': cfg['weight_decay']
}
cfg['discriminator'] = pix2pix_discriminator
cfg['discriminator_kwargs'] = {
'n_layers': 5,
'stride': 4,
'n_channel_multiplier': 64,
'weight_decay': 10.*cfg['weight_decay']
}
# loss
cfg['gan_loss_kwargs'] = {
'real_label': 0.9, # Positive labels 1 -> 0.9
'fake_label': 0.0
}
# learning
cfg['initial_learning_rate'] = 1e-4
cfg[ 'optimizer' ] = tf.train.AdamOptimizer
cfg[ 'optimizer_kwargs' ] = {}
cfg[ 'discriminator_learning_args' ] = {
'initial_learning_rate':1e-5,
'optimizer': tf.train.GradientDescentOptimizer,
'optimizer_kwargs': {}
}
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs' ] = {
'boundaries': [np.int64(0), np.int64(80000)], # need to be int64 since global step is...
'values': [cfg['initial_learning_rate'], cfg['initial_learning_rate']/10]
}
# inputs
cfg['input_dim'] = (256, 256) # (1024, 1024)
cfg['input_num_channels'] = 3
cfg['input_dtype'] = tf.float32
cfg['input_domain_name'] = 'rgb'
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image
cfg['input_preprocessing_fn_kwargs'] = {
'new_dims': cfg['input_dim'],
'new_scale': [-1, 1]
}
# outputs
cfg['target_num_channels'] = 1
cfg['target_dim'] = (256, 256) # (1024, 1024)
cfg['target_dtype'] = tf.float32
cfg['target_domain_name'] = 'keypoint'
cfg['target_preprocessing_fn'] = load_ops.resize_rescale_image
cfg['target_preprocessing_fn_kwargs'] = {
'new_dims': cfg['target_dim'],
'new_scale': [-1, 1]
}
# masks
cfg['depth_mask'] = True
# input pipeline
cfg['preprocess_fn'] = load_and_specify_preprocessors
cfg['randomize'] = False
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 600
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def get_cfg(nopause=False):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, INPUT_TASK)
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = general_utils.load_config(cfg['config_dir_input'],
nopause=True)
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'],
nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/val_image_split_0.npy'))
cfg['val_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/test_image_split_0.npy'))
# Define where the extracted representations are stored
cfg['train_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='train' if transitive else 'val'))
cfg['val_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='val' if transitive else 'test'))
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(
target_cfg['log_root'], TARGET_TASK)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 15
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = (16, 16, 8)
cfg['representation_dtype'] = tf.float32
## SETUP MODEL
# Transfer
cfg['encoder'] = transfer_two_stream_with_bn_ends
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int(
"<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'side_encoder_func': side_encoder,
'output_channels': cfg['representation_dim'][-1],
'kernel_size': [KERNEL_SIZE, KERNEL_SIZE],
'stride': 1,
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.8, #0.95
'weight_decay': cfg['weight_decay'],
'num_layers': int(NUM_LAYERS),
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
# 'flatten_output': (TARGET_TASK in FLATTEN_TASKS)
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(5000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_transitive_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_transitive_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
cfg['log_root'] = log_root
cfg['log_dir'] = os.path.join(log_root, 'logs')
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
tf.logging.set_verbosity(tf.logging.ERROR)
all_outputs = {}
pickle_dir = 'viz_output_single_task.pkl'
import os
if os.path.isfile(pickle_dir):
with open(pickle_dir, 'rb') as fp:
all_outputs = pickle.load(fp)
for task in list_of_tasks:
if task in all_outputs:
print("{} already exists....\n\n\n".format(task))
continue
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
# task = '{f}__{t}__{hs}'.format(f=task_from, t=task_to, hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{TASK}'.format(
TASK=task)
############## Load Configs ##############
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
split_file = cfg['test_filenames'] if ON_TEST_SET else cfg[
'val_filenames']
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
cfg['num_epochs'] = 1
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
print(cfg['log_root'])
if task == 'jigsaw':
continue
cfg['model_path'] = os.path.join(cfg['log_root'], task,
'model.permanent-ckpt')
print(cfg['model_path'])
if cfg['model_path'] is None:
continue
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input(
cfg, is_training=ON_TEST_SET, use_filename_queue=False
) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=IN_TRAIN_MODE)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False)
prefetch_threads = threading.Thread(target=data_prefetch_init_fn,
args=(training_runners['sess'],
training_runners['coord']))
prefetch_threads.start()
############## Run First Batch ##############
if not hasattr(m, 'masks'):
(
input_batch,
target_batch,
data_idx,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.targets, model['data_idxs'],
m.decoder_output, m.total_loss
])
mask_batch = 1.
else:
(
input_batch,
target_batch,
mask_batch,
data_idx,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.targets, m.masks, model['data_idxs'],
m.decoder_output, m.total_loss
])
if task == 'segment2d' or task == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
pca = PCA(n_components=3)
pca.fit(embedding_flattened)
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() -
lower_dim.min())
x[i] = lower_dim
predicted = x
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
to_store = {
'input': input_batch,
'target': target_batch,
'mask': mask_batch,
'data_idx': data_idx,
'output': predicted
}
all_outputs[task] = to_store
print("Done: {}".format(task))
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
try:
del sys.modules['config']
except:
pass
sys.path = remove_dups(sys.path)
print("FINISHED: {}\n\n\n\n\n\n".format(task))
pickle_dir = 'viz_output_single_task.pkl'
with open(pickle_dir, 'wb') as fp:
pickle.dump(all_outputs, fp)
try:
subprocess.call(
"aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/"
.format(pickle_dir),
shell=True)
except:
subprocess.call(
"sudo cp {} /home/ubuntu/s3/visualizations/".format(
pickle_dir),
shell=True)
return
def generate_cfg(config_dir, vid_id, args):
target_task, is_transfer, is_high_order, config_name = parse_config_dir(
config_dir)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{cfg_dir}'.format(
cfg_dir=config_dir)
############## Load Configs ##############
import utils
import data.load_ops as load_ops
from general_utils import RuntimeDeterminedEnviromentVars
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
root_dir = cfg['root_dir']
split_file = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/video{vid_id}_info.pkl'.format(vid_id=vid_id)))
cfg['dataset_dir'] = '/home/ubuntu'
os.system('sudo rm /home/ubuntu/temp/*')
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
room_layout segment25d segment2d \
segmentsemantic_rb class_1000 class_places'.split()
if target_task in low_sat_tasks and args.low_sat:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat_2
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
if 'train_list_of_fileinfos' in cfg:
if type(cfg['train_representations_file']) is not list:
task = config_name.split('__')[0]
split_file_ = os.path.join(
cfg['input_cfg']['log_root'], task,
'{task}_vid{vid_id}_representations.pkl'.format(task=task,
vid_id=vid_id))
else:
split_file_ = []
for fname in cfg['train_representations_file']:
split_file_.append(
fname.replace('val', 'vid{vid_id}'.format(vid_id=vid_id)))
cfg['train_representations_file'] = split_file_
cfg['val_representations_file'] = split_file_
cfg['test_representations_file'] = split_file_
split_file_ = os.path.join(
root_dir,
'assets/aws_data/video{vid_id}_fname.npy'.format(vid_id=vid_id))
cfg['train_list_of_fileinfos'] = split_file_
cfg['val_list_of_fileinfos'] = split_file_
cfg['test_list_of_fileinfos'] = split_file_
cfg['num_epochs'] = 2
cfg['randomize'] = False
root_dir = cfg['root_dir']
if target_task != 'segment2d' and target_task != 'segment25d':
cfg['num_read_threads'] = 1
else:
cfg['num_read_threads'] = 1
print(cfg['log_root'])
if is_transfer:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train'))
# Try latest checkpoint by time
if cfg['model_path'] is None:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train', 'time'))
# Try to get one saved manually
if cfg['model_path'] is None:
cfg['model_path'] = os.path.join(cfg['log_root'], task,
"model.permanent-ckpt")
else:
if target_task != 'vanishing_point_well_defined' and target_task != 'jigsaw':
cfg['model_path'] = os.path.join(cfg['dataset_dir'],
"model_log_final", target_task,
"logs/model.permanent-ckpt")
import tempfile
import subprocess
dirs, fname = os.path.split(cfg['model_path'])
dst_dir = dirs.replace(cfg['dataset_dir'],
"s3://taskonomy-unpacked-oregon")
tmp_path = "/home/ubuntu/temp"
tmp_fname = os.path.join(tmp_path, fname)
aws_cp_command = "aws s3 cp {}.data-00000-of-00001 {}".format(
os.path.join(dst_dir, fname), tmp_path)
subprocess.call(aws_cp_command, shell=True)
aws_cp_command = "aws s3 cp {}.meta {}".format(
os.path.join(dst_dir, fname), tmp_path)
subprocess.call(aws_cp_command, shell=True)
aws_cp_command = "aws s3 cp {}.index {}".format(
os.path.join(dst_dir, fname), tmp_path)
subprocess.call(aws_cp_command, shell=True)
cfg['model_path'] = tmp_fname
else:
cfg['model_path'] = os.path.join(cfg['log_root'], target_task,
'model.permanent-ckpt')
print(cfg['model_path'])
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_representation_extraction
return cfg, is_transfer, target_task, config_name
def get_cfg( nopause=False ):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
cfg['retrain_decoder'] = ("<RETRAIN_DECODER>" == "True")
cfg['unlock_decoder'] = ("<UNLOCK_DECODER>" == "True")
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR,
INPUT_TASK)
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR,
TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = general_utils.load_config(cfg['config_dir_input'], nopause=True)
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'], nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/val_image_split_0.npy') )
cfg['val_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/test_image_split_0.npy') )
# Define where the extracted representations are stored
cfg['train_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='train' if transitive else 'val'))
cfg['val_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='val' if transitive else 'test'))
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 15 #if not cfg['finetune_decoder'] else 7
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = (16, 16, 8)
cfg['representation_dtype'] = tf.float32
def create_bn_decay_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchnorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
return tf.train.piecewise_constant(
global_step,
boundaries=[np.int32(0), np.int32(1500), np.int32(6682), np.int32(6700)],
values=[0.2, 0.5, 0.9, 0.999]
)
def create_clipping_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchrenorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
rmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000)],
values=[
1.,
1.5,
2.,
2.5,
3.]
)
rmin = 1./rmax
dmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000),
np.int32(4500),
np.int32(5000),
],
values=[
0.,
0.5,
1.0,
1.5,
2.0,
2.5,
3.0,
3.5,
5.0]
)
return {'rmax': rmax, 'rmin': rmin, 'dmax': dmax}
## SETUP MODEL
# Transfer
cfg['encoder'] = passthrough
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int("<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs' ] = {
'boundaries': [np.int64(0), np.int64(5000)], # need to be int64 since global step is...
'values': [cfg['initial_learning_rate'], cfg['initial_learning_rate']/10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_transitive_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_transitive_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
cfg['log_root'] = log_root
if cfg['finetune_decoder'] and cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_ft')
elif cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_no_ft')
elif cfg['finetune_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'ft')
elif cfg['unlock_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'scratch')
cfg['log_dir'] = os.path.join(cfg['log_root'], 'logs')
# Set the model path that we will restore from
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(target_cfg['log_root'], TARGET_TASK)
# if cfg['retrain_decoder']:
# cfg['model_path'] = tf.train.latest_checkpoint(
# os.path.join(
# cfg['log_root'],
# '..',
# 'logs',
# 'slim-train'
# )
# )
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def get_cfg( nopause=False ):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = True
### -----CHANGE HERE--------------------------
# cfg['representations_file'] = '/home/ubuntu/s3/model_log/{task}/{task}_representations.pkl'.format(task=INPUT_TASK)
cfg['train_representations_file'] = '/home/ubuntu/s3/model_log/{subdir}/{task}_{train_split}_representations.pkl'.format(
subdir=representation_dir, task=representation_task, train_split='val')
cfg['val_representations_file'] = '/home/ubuntu/s3/model_log/{subdir}/{task}_{train_split}_representations.pkl'.format(
subdir=representation_dir, task=representation_task, train_split='test')
cfg['input_config_dir'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(INPUT_TASK)
cfg['config_dir_target1'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(INTERMEDIATE_TASK)
cfg['config_dir_target2'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(TARGET_TASK)
cfg['task1'] = INTERMEDIATE_TASK
cfg['task2'] = TARGET_TASK
# Where the target model is saved
cfg['model_path'] = None # '/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(TARGET_TASK)
# cfg['model_path'] = '/home/ubuntu/s3/model_log/{}/logs/slim-train'.format(TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
# input_cfg = general_utils.load_config(cfg['input_config_dir'], nopause=True)
cfg['target_cfgs'] = [
general_utils.load_config(cfg['config_dir_target1'], nopause=True),
general_utils.load_config(cfg['config_dir_target2'], nopause=True),
]
cfg['model_paths'] = [
'/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(cfg['task1']),
'/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(cfg['task2']),
]
target_cfg = cfg['target_cfgs'][-1]
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
cfg['target_model_type'] = target_cfg['model_type']
# kludge
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/val_image_split_0.npy') )
cfg['val_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/test_image_split_0.npy') )
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# params
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 4
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.ChainedTransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = 1024
cfg['representation_dtype'] = tf.float32
## SETUP MODEL
# Transfer
# cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout # encoder_multilayers_fc_bn
cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout_normalize_input # encoder_multilayers_fc_bn
if INPUT_TASK == 'random':
cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout_normalize_input
# cfg['encoder'] = encoder_multilayers_fc
cfg['hidden_size'] = int("<HIDDEN_SIZE>")
cfg['encoder_kwargs'] = {
'output_size': cfg['representation_dim'],
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.9, #0.95
'weight_decay': cfg['weight_decay'],
'layer_num': 3
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/model_log/transfer_transitive'.format(cfg['hidden_size'])
else:
log_root = '/home/ubuntu/model_log/transfer_transitive'.format(cfg['hidden_size'])
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/model_log/transfer'.format(cfg['hidden_size'])
else:
log_root = '/home/ubuntu/model_log/transfer'.format(cfg['hidden_size'])
cfg['log_root'] = log_root
cfg['log_dir'] = os.path.join(log_root, task_name, 'logs')
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_chained_transfer
cfg['preprocess_fn'] = load_and_specify_preprocessors_multiple
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def generate_cfg(config_dir, vid_id):
target_task, is_transfer, is_high_order, config_name = parse_config_dir(config_dir)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{cfg_dir}'.format(cfg_dir=config_dir)
############## Load Configs ##############
import utils
import data.load_ops as load_ops
from general_utils import RuntimeDeterminedEnviromentVars
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
cfg = utils.load_config( CONFIG_DIR, nopause=True )
RuntimeDeterminedEnviromentVars.register_dict( cfg )
root_dir = cfg['root_dir']
split_file = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/video{vid_id}_info.pkl'.format(vid_id=vid_id)) )
cfg['dataset_dir'] = '/home/ubuntu'
if cfg['input_preprocessing_fn'] == load_ops.resize_rescale_image:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
if 'train_list_of_fileinfos' in cfg:
if type(cfg['train_representations_file']) is not list:
task = config_name.split('__')[0]
split_file_ = os.path.join(
cfg['input_cfg']['log_root'], task,
'{task}_vid{vid_id}_representations.pkl'.format( task=task, vid_id=vid_id ))
else:
split_file_ = []
for fname in cfg['train_representations_file']:
split_file_.append(fname.replace('val', 'vid{vid_id}'.format(vid_id = vid_id)))
cfg['train_representations_file'] = split_file_
cfg['val_representations_file'] = split_file_
cfg['test_representations_file'] = split_file_
split_file_ = os.path.join(root_dir, 'assets/aws_data/video{vid_id}_fname.npy'.format(vid_id=vid_id))
cfg['train_list_of_fileinfos'] = split_file_
cfg['val_list_of_fileinfos'] = split_file_
cfg['test_list_of_fileinfos'] = split_file_
cfg['num_epochs'] = 2
cfg['randomize'] = False
root_dir = cfg['root_dir']
if target_task != 'segment2d' and target_task != 'segment25d':
cfg['num_read_threads'] = 1
else:
cfg['num_read_threads'] = 1
print(cfg['log_root'])
if is_transfer:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(
cfg['log_root'],
'logs',
'slim-train'
))
# Try latest checkpoint by time
if cfg['model_path'] is None:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(
cfg['log_root'],
'logs',
'slim-train',
'time'
))
# Try to get one saved manually
if cfg['model_path'] is None:
cfg['model_path'] = os.path.join(cfg['log_root'], task, "model.permanent-ckpt")
else:
cfg['model_path'] = os.path.join(
cfg['log_root'],
config_name,
'model.permanent-ckpt'
)
print( cfg['model_path'])
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_representation_extraction
return cfg, is_transfer, target_task, config_name
def get_cfg( nopause=False ):
cfg = {}
# params
cfg['num_epochs'] = 30
cfg['model_path'] = None
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
# model
cfg['model_type'] = architectures.SegmentationEncoderDecoder
cfg['weight_decay'] = 2e-6
cfg['instance_noise_sigma'] = 0.1
cfg['instance_noise_anneal_fn'] = tf.train.inverse_time_decay
cfg['instance_noise_anneal_fn_kwargs'] = {
'decay_rate': 0.2,
'decay_steps': 1000
}
cfg['encoder'] = resnet_encoder_fully_convolutional_16x16x8
cfg['hidden_size'] = 1024
cfg['encoder_kwargs'] = {
'resnet_build_fn' : resnet_v1_50_16x16,
'weight_decay': cfg['weight_decay']
}
cfg['decoder'] = decoder_fc_15_layer_256_resolution_fully_convolutional_16x16x4
cfg['decoder_kwargs'] = {
'activation_fn': leaky_relu( 0.2 ),
'weight_decay': cfg['weight_decay']
}
# learning
cfg['initial_learning_rate'] = 1e-4
cfg[ 'optimizer' ] = tf.train.AdamOptimizer
cfg[ 'optimizer_kwargs' ] = {}
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs' ] = {
'boundaries': [np.int64(0), np.int64(80000)], # need to be int64 since global step is...
'values': [cfg['initial_learning_rate'], cfg['initial_learning_rate']/10]
}
# inputs
cfg['input_dim'] = (256, 256) # (1024, 1024)
cfg['input_num_channels'] = 3
cfg['input_dtype'] = tf.float32
cfg['input_domain_name'] = 'rgb'
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image
cfg['input_preprocessing_fn_kwargs'] = {
'new_dims': cfg['input_dim'],
'new_scale': [-1, 1]
}
# outputs
cfg['output_dim'] = (256,256)
cfg['num_pixels'] = 300
cfg['only_target_discriminative'] = True
cfg['target_num_channels'] = 64
cfg['target_dim'] = (cfg['num_pixels'], 3) # (1024, 1024)
cfg['target_dtype'] = tf.int32
cfg['target_domain_name'] = 'segment25d'
cfg['target_from_filenames'] = load_ops.segment_pixel_sample
cfg['target_from_filenames_kwargs'] = {
'new_dims': cfg['output_dim'],
'num_pixels': cfg['num_pixels'],
'domain': cfg['target_domain_name']
}
cfg['return_accuracy'] = False
# input pipeline
cfg['preprocess_fn'] = load_and_specify_preprocessors
cfg['randomize'] = False
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 600
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def get_cfg(nopause=False):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
# representation_task = INPUT_TASK
# transitive = False
# if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
# representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
# representation_dir = 'representations_transfer_1024'
### -----CHANGE HERE--------------------------
cfg['multiple_input_tasks'] = INPUT_TASKS
cfg['train_representations_template'] = '/home/ubuntu/s3/model_log/{subdir}/{task}__{target_task}__{hs}_train_representations.pkl'.format(
subdir=REPRESENTATION_DIR,
task='{task}',
target_task=INTERMEDIARY_TASK,
hs=HIDDEN_SIZE)
cfg['val_representations_template'] = '/home/ubuntu/s3/model_log/{subdir}/{task}__{target_task}__{hs}_val_representations.pkl'.format(
subdir=REPRESENTATION_DIR,
task='{task}',
target_task=INTERMEDIARY_TASK,
hs=HIDDEN_SIZE)
cfg['train_representations_file'] = [
make_representation_path_task_specific(
cfg['train_representations_template'].format(task=task)) if task
== INTERMEDIARY_TASK else cfg['train_representations_template'].format(
task=task) for task in INPUT_TASKS
]
cfg['val_representations_file'] = [
make_representation_path_task_specific(
cfg['val_representations_template'].format(task=task)) if task
== INTERMEDIARY_TASK else cfg['val_representations_template'].format(
task=task) for task in INPUT_TASKS
]
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(
FINAL_TASK)
# Where the target model is saved
cfg['model_path'] = '/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(
FINAL_TASK)
# cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/transfers/{source}__{target}__1024'.format(
# source=INTERMEDIARY_TASK, target=FINAL_TASK)
### -------------------------------------------
# input_cfg = general_utils.load_config(cfg['input_config_dir'], nopause=True)
target_cfg = general_utils.load_config(cfg['config_dir_target'],
nopause=True)
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
cfg['target_model_type'] = target_cfg['model_type']
# kludge
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/val_image_split_0.npy'))
cfg['val_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/test_image_split_0.npy'))
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
# target_cfg['train_filenames'] = str(target_cfg['train_filenames'])
# target_cfg['val_filenames'] = str(target_cfg['val_filenames'])
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# params
cfg['root_dir'] = root_dir
cfg['num_epochs'] = 6
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = 1024
cfg['representation_dtype'] = tf.float32
## SETUP MODEL
# Transfer
cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout # encoder_multilayers_fc_bn
# cfg['encoder'] = encoder_multilayers_fc
cfg['hidden_size'] = int("1024")
cfg['encoder_kwargs'] = {
'output_size': cfg['representation_dim'],
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.9, #0.95
'weight_decay': cfg['weight_decay'],
'layer_num': 3
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/model_log/transfer_projection'.format(
cfg['hidden_size'])
else:
log_root = '/home/ubuntu/model_log/transfer_projection'.format(
cfg['hidden_size'])
cfg['log_root'] = log_root
cfg['log_dir'] = os.path.join(log_root, task_name, 'logs')
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg