def write_spearmint_config():
WorkHOME = os.environ['WorkHOME']
with open('{}/pythia_space/pars_to_tune.txt'.format(WorkHOME),
'r') as parsList:
pars = parsList.read().splitlines()
tune_config = load_config('tune_config.json')
pars_info = load_config('{}/utils/all_pars_dict.json'.format(WorkHOME))
config = {}
config['language'] = 'PYTHON'
config['main-file'] = 'Spearmint-Pythia-Tune.py'
config['experiment-name'] = tune_config['spearmint_expt_name']
config['likelihood'] = 'NOISELESS'
var_info = {}
for par in pars:
par_info = {
'type': 'FLOAT',
'size': 1,
'min': pars_info[par]['min'],
'max': pars_info[par]['max']
}
var_info[par] = par_info
config['variables'] = var_info
with open('{}/spearmint_space/config.json'.format(WorkHOME),
'w') as spearmint_config:
json.dump(config, spearmint_config, sort_keys=True, indent=4)
def get_cfg(nopause=False):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
import pdb
global INPUT_TASK
INPUT_TASK = INPUT_TASK.split(',')
representation_task = INPUT_TASK
cfg['multiple_input_tasks'] = len(INPUT_TASK)
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
cfg['retrain_decoder'] = ("<RETRAIN_DECODER>" == "True")
cfg['unlock_decoder'] = ("<UNLOCK_DECODER>" == "True")
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = [
'/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(CFG_DIR, t)
for t in INPUT_TASK
]
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = [
general_utils.load_config(inp_cfg, nopause=True)
for inp_cfg in cfg['config_dir_input']
]
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'],
nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath(
os.path.join('/home/ubuntu/s3',
'meta/val_image_split_{}.npy'.format(VAL_DATA_USED)))
cfg['val_list_of_fileinfos'] = os.path.abspath(
os.path.join('/home/ubuntu/s3', 'meta/test_image_split_0.npy'))
# Define where the extracted representations are stored
cfg['train_representations_file'] = [
os.path.join(
input_cfg[i]['log_root'], t,
'{task}_{train_split}_representations.pkl'.format(
task=t, train_split='train' if transitive else 'val'))
for i, t in enumerate(representation_task)
]
cfg['val_representations_file'] = [
os.path.join(
input_cfg[i]['log_root'], t,
'{task}_{train_split}_representations.pkl'.format(
task=t, train_split='val' if transitive else 'test'))
for i, t in enumerate(representation_task)
]
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
if TARGET_TASK in ['ego_motion', 'fix_pose', 'point_match', 'non_fixated_pose'] \
and VAL_DATA_USED not in ['16k','30k']:
target_cfg['train_filenames'] = target_cfg['train_filenames'].replace(
'val_split_image_info_', 'val_split_image_info_multi_in_')
cfg['train_list_of_fileinfos'] = cfg[
'train_list_of_fileinfos'].replace('val_image_split_',
'val_image_split_multi_in_')
target_cfg['train_filenames'] = os.path.abspath(
os.path.join('/home/ubuntu/s3',
'meta/val_split_image_info_{}.pkl'.format(VAL_DATA_USED)))
if TARGET_TASK == 'ego_motion':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_fixated_trips_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'fix_pose':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_fixated_pairs_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'point_match':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir, 'assets/aws_data/val_point_match_info_{}.pkl'.format(
VAL_DATA_USED)))
if TARGET_TASK == 'non_fixated_pose':
target_cfg['train_filenames'] = os.path.abspath(
os.path.join(
root_dir,
'assets/aws_data/val_camera_nonfixated_pairs_info_{}.pkl'.
format(VAL_DATA_USED)))
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
if VAL_DATA_USED == '1k':
cfg['num_epochs'] = 200
elif VAL_DATA_USED == '2k':
cfg['num_epochs'] = 100
elif VAL_DATA_USED == '4k':
cfg['num_epochs'] = 50
elif VAL_DATA_USED == '8k':
cfg['num_epochs'] = 50
elif VAL_DATA_USED == '5h':
cfg['num_epochs'] = 400
elif VAL_DATA_USED == '1h':
cfg['num_epochs'] = 1000
elif VAL_DATA_USED == '16k':
cfg['num_epochs'] = 30
else:
cfg['num_epochs'] = 15
#cfg['num_epochs'] = 15 if not cfg['finetune_decoder'] else 7 if not cfg['finetune_decoder'] else 7
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = [16, 16, 8 * len(INPUT_TASK)]
cfg['representation_dtype'] = tf.float32
def create_bn_decay_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchnorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
return tf.train.piecewise_constant(global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(6682),
np.int32(6700)
],
values=[0.2, 0.5, 0.9, 0.999])
def create_clipping_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchrenorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
rmax = tf.train.piecewise_constant(global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000)
],
values=[1., 1.5, 2., 2.5, 3.])
rmin = 1. / rmax
dmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000),
np.int32(4500),
np.int32(5000),
],
values=[0., 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 5.0])
return {'rmax': rmax, 'rmin': rmin, 'dmax': dmax}
## SETUP MODEL
# Transfer
cfg['encoder'] = transfer_models.transfer_two_stream_with_bn_ends_l2l0_no_image
if INPUT_TASK == 'pixels':
raise NotImplementedError(
"Cannot transfer from 'pixels' when using no_image!")
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int(
"<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'side_encoder_func': transfer_models.side_encoder_l2,
'output_channels': cfg['representation_dim'][-1],
'kernel_size': [KERNEL_SIZE, KERNEL_SIZE],
'stride': 1,
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.8, #0.95
'weight_decay': cfg['weight_decay'],
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
# 'flatten_output': (TARGET_TASK in FLATTEN_TASKS)
}
cfg['encoder_kwargs']['renorm'] = True
cfg['encoder_kwargs']['renorm_momentum'] = 0.9
cfg['encoder_kwargs']['renorm_clipping'] = create_clipping_schedule
# Type of decoder to use
cfg['replace_target_decoder'] = transfer_models.__dict__[
REPLACE_TARGET_DECODER_FUNC_NAME]
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(5000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
cfg['log_root'] = config_dir.replace('task-taxonomy-331b/', 's3/')
if cfg['finetune_decoder'] and cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_ft')
elif cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_no_ft')
elif cfg['finetune_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'ft')
elif cfg['unlock_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'scratch')
cfg['log_dir'] = os.path.join(cfg['log_root'], 'logs')
# Set the model path that we will restore from
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(
target_cfg['log_root'], TARGET_TASK)
if cfg['retrain_decoder']:
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], '..', 'logs', 'slim-train'))
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = '../s3/meta'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def get_cfg(nopause=False):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, INPUT_TASK)
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR, TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = general_utils.load_config(cfg['config_dir_input'],
nopause=True)
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'],
nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/val_image_split_0.npy'))
cfg['val_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/test_image_split_0.npy'))
# Define where the extracted representations are stored
cfg['train_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='train' if transitive else 'val'))
cfg['val_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='val' if transitive else 'test'))
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(
target_cfg['log_root'], TARGET_TASK)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 15
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = (16, 16, 8)
cfg['representation_dtype'] = tf.float32
## SETUP MODEL
# Transfer
cfg['encoder'] = transfer_two_stream_with_bn_ends
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int(
"<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'side_encoder_func': side_encoder,
'output_channels': cfg['representation_dim'][-1],
'kernel_size': [KERNEL_SIZE, KERNEL_SIZE],
'stride': 1,
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.8, #0.95
'weight_decay': cfg['weight_decay'],
'num_layers': int(NUM_LAYERS),
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
# 'flatten_output': (TARGET_TASK in FLATTEN_TASKS)
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs'] = {
'boundaries':
[np.int64(0),
np.int64(5000)], # need to be int64 since global step is...
'values':
[cfg['initial_learning_rate'], cfg['initial_learning_rate'] / 10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_transitive_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_transitive_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_{}/regular/{}/{}'.format(
cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
cfg['log_root'] = log_root
cfg['log_dir'] = os.path.join(log_root, 'logs')
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
def get_cfg( nopause=False ):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = False
if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
representation_dir = 'representations_transfer_1024'
transitive = True
cfg['finetune_decoder'] = ("<FINETUNE_DECODER>" == "True")
cfg['retrain_decoder'] = ("<RETRAIN_DECODER>" == "True")
cfg['unlock_decoder'] = ("<UNLOCK_DECODER>" == "True")
### -----CHANGE HERE--------------------------
cfg['config_dir_input'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR,
INPUT_TASK)
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/{}/{}'.format(
CFG_DIR,
TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
input_cfg = general_utils.load_config(cfg['config_dir_input'], nopause=True)
cfg['input_cfg'] = input_cfg
# Replace loading info with the version from the target config
target_cfg = general_utils.load_config(cfg['config_dir_target'], nopause=True)
cfg['target_cfg'] = target_cfg
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
general_utils.update_keys(cfg, "mask_by_target_func", target_cfg)
general_utils.update_keys(cfg, "depth_mask", target_cfg)
general_utils.update_keys(cfg, "mask_fn", target_cfg)
general_utils.update_keys(cfg, "find_target_in_config", target_cfg)
# For segmentation
general_utils.update_keys(cfg, "num_pixels", target_cfg)
general_utils.update_keys(cfg, "only_target_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_pixels", target_cfg)
# kludge where we list all of the files
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/val_image_split_0.npy') )
cfg['val_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/test_image_split_0.npy') )
# Define where the extracted representations are stored
cfg['train_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='train' if transitive else 'val'))
cfg['val_representations_file'] = os.path.join(
input_cfg['log_root'], representation_task,
'{task}_{train_split}_representations.pkl'.format(
task=representation_task,
train_split='val' if transitive else 'test'))
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# Params for training
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 15 #if not cfg['finetune_decoder'] else 7
cfg['max_ckpts_to_keep'] = cfg['num_epochs']
cfg['target_model_type'] = target_cfg['model_type']
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = (16, 16, 8)
cfg['representation_dtype'] = tf.float32
def create_bn_decay_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchnorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
return tf.train.piecewise_constant(
global_step,
boundaries=[np.int32(0), np.int32(1500), np.int32(6682), np.int32(6700)],
values=[0.2, 0.5, 0.9, 0.999]
)
def create_clipping_schedule():
# We stick it in a callable function so that the variables will be created on the main graph
with tf.variable_scope("batchrenorm_decay", reuse=False) as sc:
global_step = tf.Variable(0, name='counter')
inc = tf.assign_add(global_step, 1, name='bn_increment')
rmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000)],
values=[
1.,
1.5,
2.,
2.5,
3.]
)
rmin = 1./rmax
dmax = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int32(0),
np.int32(1500),
np.int32(2000),
np.int32(2500),
np.int32(3000),
np.int32(3500),
np.int32(4000),
np.int32(4500),
np.int32(5000),
],
values=[
0.,
0.5,
1.0,
1.5,
2.0,
2.5,
3.0,
3.5,
5.0]
)
return {'rmax': rmax, 'rmin': rmin, 'dmax': dmax}
## SETUP MODEL
# Transfer
cfg['encoder'] = passthrough
# if INPUT_TASK == 'random':
# cfg['encoder'] = transfer_multilayer_conv_with_bn_ends
cfg['hidden_size'] = int("<HIDDEN_SIZE>") # This will be the number of interior channels
cfg['encoder_kwargs'] = {
'flatten_output': 'flatten' in target_cfg['encoder_kwargs']
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
def pwc(initial_lr, **kwargs):
global_step = kwargs['global_step']
del kwargs['global_step']
return tf.train.piecewise_constant(global_step, **kwargs)
cfg['learning_rate_schedule'] = pwc
cfg['learning_rate_schedule_kwargs' ] = {
'boundaries': [np.int64(0), np.int64(5000)], # need to be int64 since global step is...
'values': [cfg['initial_learning_rate'], cfg['initial_learning_rate']/10]
}
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_transitive_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_transitive_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/experiment_models/pix_stream_transfer_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
else:
log_root = '/home/ubuntu/experiment_models/pix_stream_transfer_{}/empty/{}/{}'.format(cfg['hidden_size'], INPUT_TASK, TARGET_TASK)
cfg['log_root'] = log_root
if cfg['finetune_decoder'] and cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_ft')
elif cfg['retrain_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'rt_no_ft')
elif cfg['finetune_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'ft')
elif cfg['unlock_decoder']:
cfg['log_root'] = os.path.join(cfg['log_root'], 'scratch')
cfg['log_dir'] = os.path.join(cfg['log_root'], 'logs')
# Set the model path that we will restore from
# Where the target decoder is stored
cfg['model_path'] = '{}/{}/model.permanent-ckpt'.format(target_cfg['log_root'], TARGET_TASK)
# if cfg['retrain_decoder']:
# cfg['model_path'] = tf.train.latest_checkpoint(
# os.path.join(
# cfg['log_root'],
# '..',
# 'logs',
# 'slim-train'
# )
# )
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 600
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def get_cfg( nopause=False ):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
representation_task = INPUT_TASK
representation_dir = 'representations'
transitive = True
### -----CHANGE HERE--------------------------
# cfg['representations_file'] = '/home/ubuntu/s3/model_log/{task}/{task}_representations.pkl'.format(task=INPUT_TASK)
cfg['train_representations_file'] = '/home/ubuntu/s3/model_log/{subdir}/{task}_{train_split}_representations.pkl'.format(
subdir=representation_dir, task=representation_task, train_split='val')
cfg['val_representations_file'] = '/home/ubuntu/s3/model_log/{subdir}/{task}_{train_split}_representations.pkl'.format(
subdir=representation_dir, task=representation_task, train_split='test')
cfg['input_config_dir'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(INPUT_TASK)
cfg['config_dir_target1'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(INTERMEDIATE_TASK)
cfg['config_dir_target2'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(TARGET_TASK)
cfg['task1'] = INTERMEDIATE_TASK
cfg['task2'] = TARGET_TASK
# Where the target model is saved
cfg['model_path'] = None # '/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(TARGET_TASK)
# cfg['model_path'] = '/home/ubuntu/s3/model_log/{}/logs/slim-train'.format(TARGET_TASK)
### -------------------------------------------
# Automatically populate data loading variables
# input_cfg = general_utils.load_config(cfg['input_config_dir'], nopause=True)
cfg['target_cfgs'] = [
general_utils.load_config(cfg['config_dir_target1'], nopause=True),
general_utils.load_config(cfg['config_dir_target2'], nopause=True),
]
cfg['model_paths'] = [
'/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(cfg['task1']),
'/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(cfg['task2']),
]
target_cfg = cfg['target_cfgs'][-1]
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
cfg['target_model_type'] = target_cfg['model_type']
# kludge
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/val_image_split_0.npy') )
cfg['val_list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/test_image_split_0.npy') )
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# params
cfg['root_dir'] = root_dir
# cfg['num_epochs'] = 30 if INPUT_TASK == 'random' else 4
cfg['num_epochs'] = 4
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.ChainedTransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = 1024
cfg['representation_dtype'] = tf.float32
## SETUP MODEL
# Transfer
# cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout # encoder_multilayers_fc_bn
cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout_normalize_input # encoder_multilayers_fc_bn
if INPUT_TASK == 'random':
cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout_normalize_input
# cfg['encoder'] = encoder_multilayers_fc
cfg['hidden_size'] = int("<HIDDEN_SIZE>")
cfg['encoder_kwargs'] = {
'output_size': cfg['representation_dim'],
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.9, #0.95
'weight_decay': cfg['weight_decay'],
'layer_num': 3
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath( __file__ ))
task_name = os.path.basename( config_dir )
if transitive:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/model_log/transfer_transitive'.format(cfg['hidden_size'])
else:
log_root = '/home/ubuntu/model_log/transfer_transitive'.format(cfg['hidden_size'])
else:
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/model_log/transfer'.format(cfg['hidden_size'])
else:
log_root = '/home/ubuntu/model_log/transfer'.format(cfg['hidden_size'])
cfg['log_root'] = log_root
cfg['log_dir'] = os.path.join(log_root, task_name, 'logs')
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_chained_transfer
cfg['preprocess_fn'] = load_and_specify_preprocessors_multiple
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict( cfg ) # These will be loaded at runtime
print_cfg( cfg, nopause=nopause )
return cfg
def get_cfg(nopause=False):
root_dir = '/home/ubuntu/task-taxonomy-331b'
cfg = {}
# representation_task = INPUT_TASK
# transitive = False
# if PRE_INPUT_TASK and PRE_INPUT_TASK != "PRE_INPUT_TASK":
# representation_task = PRE_INPUT_TASK + '__' + INPUT_TASK + '__' + '1024'
# representation_dir = 'representations_transfer_1024'
### -----CHANGE HERE--------------------------
cfg['multiple_input_tasks'] = INPUT_TASKS
cfg['train_representations_template'] = '/home/ubuntu/s3/model_log/{subdir}/{task}__{target_task}__{hs}_train_representations.pkl'.format(
subdir=REPRESENTATION_DIR,
task='{task}',
target_task=INTERMEDIARY_TASK,
hs=HIDDEN_SIZE)
cfg['val_representations_template'] = '/home/ubuntu/s3/model_log/{subdir}/{task}__{target_task}__{hs}_val_representations.pkl'.format(
subdir=REPRESENTATION_DIR,
task='{task}',
target_task=INTERMEDIARY_TASK,
hs=HIDDEN_SIZE)
cfg['train_representations_file'] = [
make_representation_path_task_specific(
cfg['train_representations_template'].format(task=task)) if task
== INTERMEDIARY_TASK else cfg['train_representations_template'].format(
task=task) for task in INPUT_TASKS
]
cfg['val_representations_file'] = [
make_representation_path_task_specific(
cfg['val_representations_template'].format(task=task)) if task
== INTERMEDIARY_TASK else cfg['val_representations_template'].format(
task=task) for task in INPUT_TASKS
]
cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/aws_second/{}'.format(
FINAL_TASK)
# Where the target model is saved
cfg['model_path'] = '/home/ubuntu/s3/model_log/{}/model.permanent-ckpt'.format(
FINAL_TASK)
# cfg['config_dir_target'] = '/home/ubuntu/task-taxonomy-331b/experiments/transfers/{source}__{target}__1024'.format(
# source=INTERMEDIARY_TASK, target=FINAL_TASK)
### -------------------------------------------
# input_cfg = general_utils.load_config(cfg['input_config_dir'], nopause=True)
target_cfg = general_utils.load_config(cfg['config_dir_target'],
nopause=True)
general_utils.update_keys(cfg, "input", target_cfg)
general_utils.update_keys(cfg, "target", target_cfg)
general_utils.update_keys(cfg, "is_discriminative", target_cfg)
general_utils.update_keys(cfg, "num_input", target_cfg)
general_utils.update_keys(cfg, "single_filename_to_multiple", target_cfg)
general_utils.update_keys(cfg, "preprocess_fn", target_cfg)
cfg['target_model_type'] = target_cfg['model_type']
# kludge
# cfg['list_of_fileinfos'] = os.path.abspath( os.path.join( root_dir, 'assets/aws_data/train_image_split_0.npy') )
cfg['train_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/val_image_split_0.npy'))
cfg['val_list_of_fileinfos'] = os.path.abspath(
os.path.join(root_dir, 'assets/aws_data/test_image_split_0.npy'))
# Now use 'val' for training and 'test' for validation... :(
tmp = target_cfg['train_filenames']
# target_cfg['train_filenames'] = str(target_cfg['train_filenames'])
# target_cfg['val_filenames'] = str(target_cfg['val_filenames'])
target_cfg['train_filenames'] = str(target_cfg['val_filenames'])
target_cfg['val_filenames'] = str(target_cfg['test_filenames'])
target_cfg['test_filenames'] = None # str(tmp)
general_utils.update_keys(cfg, "train_filenames", target_cfg)
general_utils.update_keys(cfg, "val_filenames", target_cfg)
general_utils.update_keys(cfg, "test_filenames", target_cfg)
general_utils.update_keys(cfg, "dataset_dir", target_cfg)
# params
cfg['root_dir'] = root_dir
cfg['num_epochs'] = 6
cfg['weight_decay'] = 1e-6 # 1e-7, 1
cfg['model_type'] = architectures.TransferNet
## DATA LOADING
# representations
cfg['representation_dim'] = 1024
cfg['representation_dtype'] = tf.float32
## SETUP MODEL
# Transfer
cfg['encoder'] = encoder_multilayers_fc_bn_res_no_dropout # encoder_multilayers_fc_bn
# cfg['encoder'] = encoder_multilayers_fc
cfg['hidden_size'] = int("1024")
cfg['encoder_kwargs'] = {
'output_size': cfg['representation_dim'],
'batch_norm_epsilon': 1e-5,
'batch_norm_decay': 0.9, #0.95
'weight_decay': cfg['weight_decay'],
'layer_num': 3
}
# learning
general_utils.update_keys(cfg, "initial_learning_rate", target_cfg)
general_utils.update_keys(cfg, "optimizer", target_cfg)
general_utils.update_keys(cfg, "clip_norm", target_cfg)
# cfg['initial_learning_rate'] = 1e-4 # 1e-6, 1e-1
# cfg[ 'optimizer' ] = tf.train.AdamOptimizer
# cfg[ 'optimizer_kwargs' ] = {}
## LOCATIONS
# logging
config_dir = os.path.dirname(os.path.realpath(__file__))
task_name = os.path.basename(config_dir)
if SAVE_TO_S3:
log_root = '/home/ubuntu/s3/model_log/transfer_projection'.format(
cfg['hidden_size'])
else:
log_root = '/home/ubuntu/model_log/transfer_projection'.format(
cfg['hidden_size'])
cfg['log_root'] = log_root
cfg['log_dir'] = os.path.join(log_root, task_name, 'logs')
# input pipeline
data_dir = '/home/ubuntu/s3'
cfg['meta_file_dir'] = 'assets/aws_data'
cfg['create_input_placeholders_and_ops_fn'] = create_input_placeholders_and_ops_transfer
cfg['randomize'] = True
cfg['num_read_threads'] = 300
cfg['batch_size'] = 32
cfg['inputs_queue_capacity'] = 4096
# Checkpoints and summaries
cfg['summary_save_every_secs'] = 300
cfg['checkpoint_save_every_secs'] = 3000
RuntimeDeterminedEnviromentVars.register_dict(
cfg) # These will be loaded at runtime
print_cfg(cfg, nopause=nopause)
return cfg
