def test_model_guest_creation(api_client, admin_user, model_name):
model_data = setup_model(model_name, admin_user)
response = api_client.post('/collab/{}/'.format(model_name),
data=model_data,
HTTP_ACCEPT="application/json")
assert_response(response, status.HTTP_401_UNAUTHORIZED)
def load_model(f, nn_layers=None, from_npz=False):
"""
Loads neural network architecture specified in MPPI code, and loads weights and biases
:param f: path of either .npz file or torch model
:type f: str
:param nn_layers: list consisting of number of nodes for each layer in network
:param from_npz: if True, load .npz file which contains weights and biases of neural network
:return: torch model
"""
# setup model architecture
model = setup_model(layers=nn_layers, verbose=False)
# load torch model
if not from_npz:
model.load_state_dict(
torch.load(f)
) # add arg map_location=torch.device('cpu') to torch.load if using CPU
else:
# load npz file
model = npz_to_torch_model(f, model)
# load model onto GPU
model.to(device)
return model
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', '-g', type=int, default=0,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--model-setup', required=True,
help='Model setup dictionary.')
parser.add_argument('--lsh', action='store_true', default=False,
help='If true, uses locally sensitive hashing \
(with k=10 NN) for NN search.')
args = parser.parse_args()
model, train, test, vocab, setup = setup_model(args)
if setup['dataset'] == 'snli':
converter = convert_snli_seq
use_snli = True
else:
converter = convert_seq
use_snli = False
with open(os.path.join(setup['save_path'], 'calib.json')) as f:
calibration_idx = json.load(f)
calibration = [train[i] for i in calibration_idx]
train = [x for i, x in enumerate(train) if i not in calibration_idx]
sm = ScaledModel(model)
optim = chainer.optimizers.Adam()
optim.setup(sm.temperature)
calib_iter = chainer.iterators.SerialIterator(
calibration, setup['batchsize'], repeat=False)
eceloss = ECELoss()
num_epochs = 50
for i in range(num_epochs):
calib_iter.reset()
all_logits = []
all_labels = []
for i, batch in enumerate(calib_iter):
batch = converter(batch, device=args.gpu, with_label=True)
logits = sm.predict(batch['xs'])
labels = F.concat(batch['ys'], axis=0)
all_logits.append(logits.data)
all_labels.append(labels.data)
all_logits = F.concat(all_logits, axis=0)
all_labels = F.concat(all_labels, axis=0)
print(sm.temperature.temperature.data[0],
eceloss(all_logits, all_labels))
logits = sm.temperature(all_logits)
loss = F.softmax_cross_entropy(logits, all_labels)
sm.temperature.zerograds()
loss.backward()
optim.update()
def test_model_creation(admin_api_client, admin_user, model_name):
model_data = setup_model(model_name, admin_user)
response = admin_api_client.post('/collab/{}/'.format(model_name),
data=model_data,
HTTP_ACCEPT='application/json')
assert_response(response, status.HTTP_201_CREATED)
projects_created = [response.data]
response = admin_api_client.get('/collab/{}/'.format(model_name),
HTTP_ACCEPT="application/json")
assert_eq(response.data, projects_created)
def test_model_creation(admin_api_client, admin_user, model_name):
model_data = setup_model(model_name, admin_user)
response = admin_api_client.post('/collab/{}/'.format(model_name),
data=model_data,
HTTP_ACCEPT='application/json')
# Manually handle matches, where API does not allow creation or
# modification of objects, as they're read only
if model_name == "matches":
assert_response(response, status.HTTP_405_METHOD_NOT_ALLOWED)
return
assert_response(response, status.HTTP_201_CREATED)
projects_created = [response.data]
response = admin_api_client.get('/collab/{}/'.format(model_name),
HTTP_ACCEPT="application/json")
assert_response(response, status.HTTP_200_OK, projects_created)
# Command Line ardguments
ap.add_argument('data_dir', nargs='*', action="store", default="./flowers/")
ap.add_argument('--gpu', dest="gpu", action="store", default="gpu")
ap.add_argument('--save_dir', dest="save_dir", action="store", default="./checkpoint.pth")
ap.add_argument('--learning_rate', dest="learning_rate", action="store", default=0.003)
ap.add_argument('--dropout', dest = "dropout", action = "store", default = 0.5)
ap.add_argument('--epochs', dest="epochs", action="store", type=int, default=12)
ap.add_argument('--arch', dest="arch", action="store", default="vgg16", type = str)
pa = ap.parse_args()
where = pa.data_dir
path = pa.save_dir
lr = pa.learning_rate
structure = pa.arch
dropout = pa.dropout
power = pa.gpu
epochs = pa.epochs
trainloader, validationloader, testloader = utils.load_data(where)
model, optimizer, criterion = utils.setup_model(structure,dropout,lr,power)
utils.train_network(model, optimizer, criterion, epochs, 12, trainloader, power)
utils.save_checkpoint(path,structure,dropout,lr)
def run_training(cfg, cfg_dir):
# set up logging
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = utils.setup_input_transfer(cfg, is_training=True)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model(inputs, cfg, is_training=True)
# execute training
start_time = time.time()
utils.print_start_info(cfg, inputs['max_steps'], is_training=True)
if cfg['model_type'] == 'empty': # Can't use tf slim because not trainable variables
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=True)
training_runners['threads'] = data_prefetch_init_fn(
training_runners['sess'], training_runners['coord'])
try:
# This just returns the imput as output. It is for testing data
# input only.
for step in xrange(inputs['max_steps']):
input_batch, target_batch, data_idx = training_runners[
'sess'].run([
model['input_batch'], model['target_batch'],
model['data_idxs']
])
if training_runners['coord'].should_stop():
break
finally:
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
else: # Use tf.slim
train_log_dir = os.path.join(cfg['log_dir'], 'slim-train')
permanent_checkpoint_dir = os.path.join(cfg['log_dir'],
'checkpoints')
session_config = tf.ConfigProto()
session_config.gpu_options.allow_growth = True
#max_to_keep = cfg['num_epochs'] * 2
max_to_keep = 10
if 'max_ckpts_to_keep' in cfg:
max_to_keep = cfg['max_ckpts_to_keep']
# When ready to use a model, use the code below
train(
model['train_op'],
train_log_dir,
utils.get_data_prefetch_threads_init_fn_transfer(
inputs, cfg, is_training=True),
train_step_fn=model['train_step_fn'],
train_step_kwargs=model['train_step_kwargs'],
global_step=model['global_step'],
number_of_steps=inputs['max_steps'],
number_of_epochs=cfg['num_epochs'],
init_fn=model['init_fn'],
save_checkpoint_every=max(inputs['max_steps'] // (max_to_keep),
500),
cfg_dir=cfg_dir,
#RuntimeDeterminedEnviromentVars.steps_per_epoch,
permanent_checkpoint_dir=permanent_checkpoint_dir,
save_summaries_secs=cfg['summary_save_every_secs'],
save_interval_secs=cfg['checkpoint_save_every_secs'],
saver=model['saver_op'],
return_accuracy='return_accuracy' in cfg
and cfg['return_accuracy'],
session_config=session_config)
end_train_time = time.time() - start_time
print('time to train %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
print('avg time per epoch: %.3f hrs' %
((end_train_time / (60 * 60)) / cfg['num_epochs']))
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
tf.logging.set_verbosity(tf.logging.ERROR)
all_outputs = {}
pickle_dir = 'viz_output_single_task.pkl'
import os
if os.path.isfile(pickle_dir):
with open(pickle_dir, 'rb') as fp:
all_outputs = pickle.load(fp)
for task in list_of_tasks:
if task in all_outputs:
print("{} already exists....\n\n\n".format(task))
continue
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
# task = '{f}__{t}__{hs}'.format(f=task_from, t=task_to, hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{TASK}'.format(
TASK=task)
############## Load Configs ##############
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
split_file = cfg['test_filenames'] if ON_TEST_SET else cfg[
'val_filenames']
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
cfg['num_epochs'] = 1
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
print(cfg['log_root'])
if task == 'jigsaw':
continue
cfg['model_path'] = os.path.join(cfg['log_root'], task,
'model.permanent-ckpt')
print(cfg['model_path'])
if cfg['model_path'] is None:
continue
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input(
cfg, is_training=ON_TEST_SET, use_filename_queue=False
) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=IN_TRAIN_MODE)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False)
prefetch_threads = threading.Thread(target=data_prefetch_init_fn,
args=(training_runners['sess'],
training_runners['coord']))
prefetch_threads.start()
############## Run First Batch ##############
if not hasattr(m, 'masks'):
(
input_batch,
target_batch,
data_idx,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.targets, model['data_idxs'],
m.decoder_output, m.total_loss
])
mask_batch = 1.
else:
(
input_batch,
target_batch,
mask_batch,
data_idx,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.targets, m.masks, model['data_idxs'],
m.decoder_output, m.total_loss
])
if task == 'segment2d' or task == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
pca = PCA(n_components=3)
pca.fit(embedding_flattened)
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() -
lower_dim.min())
x[i] = lower_dim
predicted = x
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
to_store = {
'input': input_batch,
'target': target_batch,
'mask': mask_batch,
'data_idx': data_idx,
'output': predicted
}
all_outputs[task] = to_store
print("Done: {}".format(task))
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
try:
del sys.modules['config']
except:
pass
sys.path = remove_dups(sys.path)
print("FINISHED: {}\n\n\n\n\n\n".format(task))
pickle_dir = 'viz_output_single_task.pkl'
with open(pickle_dir, 'wb') as fp:
pickle.dump(all_outputs, fp)
try:
subprocess.call(
"aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/"
.format(pickle_dir),
shell=True)
except:
subprocess.call(
"sudo cp {} /home/ubuntu/s3/visualizations/".format(
pickle_dir),
shell=True)
return
def run_val_test(cfg):
# set up logging
tf.logging.set_verbosity(tf.logging.INFO)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
# create ops and placeholders
inputs = utils.setup_input(cfg, is_training=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model(inputs, cfg, is_training=False)
# full_path = tf.train.latest_checkpoint(checkpoint_dir)
# step = full_path.split('-')[-1]
# model_path = os.path.join('/home/ubuntu/s3/model_log', cfg['task_name'], 'model.permanent-ckpt')
model_path = os.path.join('/home/ubuntu/s3/model_log_final',
cfg['task_name'], 'model.permanent-ckpt')
model['saver_op'].restore(training_runners['sess'], model_path)
m = model['model']
# execute training
start_time = time.time()
utils.print_start_info(cfg, inputs['max_steps'], is_training=False)
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=False)
prefetch_threads = threading.Thread(target=data_prefetch_init_fn,
args=(training_runners['sess'],
training_runners['coord']))
prefetch_threads.start()
print("Dataloading workers dispatched....")
return_accuracy = 'return_accuracy' in cfg and cfg['return_accuracy'],
losses_mean = AverageMeter()
accuracy_mean = AverageMeter()
for step in range(inputs['max_steps']):
#print(step)
if return_accuracy:
(data_idx, loss, accuracy) = training_runners['sess'].run(
[model['data_idxs'], m.losses[0], m.accuracy])
losses_mean.update(loss)
accuracy_mean.update(accuracy)
if step % 100 == 0:
print(
'Step: {step} with Current Losses mean: {loss}; with accuracy: {accur}'
.format(step=step,
loss=losses_mean.avg,
accur=accuracy_mean.avg))
else:
(data_idx, loss) = training_runners['sess'].run(
[model['data_idxs'], m.losses[0]])
losses_mean.update(loss)
if step % 100 == 0:
print('Step: {step} with Current Losses mean: {loss}'.format(
step=step, loss=losses_mean.avg))
if return_accuracy:
print('Final Losses mean: {loss}; with accuracy: {accur}'.format(
loss=losses_mean.avg, accur=accuracy_mean.avg))
else:
print('Final Losses mean: {loss}'.format(loss=losses_mean.avg))
end_train_time = time.time() - start_time
print('time to train %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
print('avg time per epoch: %.3f hrs' % ((end_train_time /
(60 * 60)) / cfg['num_epochs']))
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
import pdb
global synset
synset_1000 = [" ".join(i.split(" ")[1:]) for i in synset]
select = np.asarray([ 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1.,
1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,
0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 1.,
1., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0.,
0., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0.,
0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 0., 0., 1.,
0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 1., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,
1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 1., 1., 0., 0., 1., 0., 1.,
0., 1., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0.,
0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,
0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 1.,
0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0.,
0., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0.])
with open('/home/ubuntu/task-taxonomy-331b/lib/data/places_class_names.txt', 'r') as fp:
synset_places = [x.rstrip()[4:-1] for x,y in zip(fp.readlines(), select) if y == 1.]
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
if args.task is not 'NONE':
args.idx = list_of_tasks.index(args.task)
for idx, task in enumerate(list_of_tasks):
if idx != args.idx and args.idx != -1:
continue
if task == 'class_places':
synset = synset_places
elif task == 'class_1000':
synset = synset_1000
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() }
# task = '{f}__{t}__{hs}'.format(f=task_from, t=task_to, hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{TASK}'.format(TASK=task)
############## Load Configs ##############
cfg = utils.load_config( CONFIG_DIR, nopause=True )
RuntimeDeterminedEnviromentVars.register_dict( cfg )
split_file = os.path.join('/home/ubuntu/task-taxonomy-331b/assets/aws_data/', 'video2_info.pkl')
cfg['train_filenames'] = split_file
cfg['val_filenames'] = split_file
cfg['test_filenames'] = split_file
cfg['num_epochs'] = 2
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
print(cfg['log_root'])
cfg['model_path'] = os.path.join(
cfg['log_root'],
task,
'model.permanent-ckpt'
)
print( cfg['model_path'])
if cfg['model_path'] is None:
continue
cfg['dataset_dir'] = '/home/ubuntu'
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_representation_extraction
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input( cfg, is_training=ON_TEST_SET, use_filename_queue=False ) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model( inputs, cfg, is_training=IN_TRAIN_MODE )
m = model[ 'model' ]
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False )
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=( training_runners[ 'sess' ], training_runners[ 'coord' ] ))
prefetch_threads.start()
list_of_fname = np.load('/home/ubuntu/task-taxonomy-331b/assets/aws_data/video2_fname.npy')
import errno
try:
os.mkdir('/home/ubuntu/{}'.format(task))
os.mkdir('/home/ubuntu/{}/vid1'.format(task))
os.mkdir('/home/ubuntu/{}/vid2'.format(task))
os.mkdir('/home/ubuntu/{}/vid3'.format(task))
os.mkdir('/home/ubuntu/{}/vid4'.format(task))
except OSError as e:
if e.errno != errno.EEXIST:
raise
curr_comp = np.zeros((3,64))
curr_fit_img = np.zeros((256,256,3))
embeddings = []
############## Run First Batch ##############
for step_num in range(inputs['max_steps'] - 1):
#for step_num in range(1):
#if step_num > 0 and step_num % 20 == 0:
print(step_num)
if not hasattr(m, 'masks'):
(
input_batch, target_batch,
data_idx,
predicted, loss,
) = training_runners['sess'].run( [
m.input_images, m.targets,
model[ 'data_idxs' ],
m.decoder_output, m.total_loss] )
mask_batch = 1.
else:
(
input_batch, target_batch, mask_batch,
data_idx,
predicted, loss,
) = training_runners['sess'].run( [
m.input_images, m.targets, m.masks,
model[ 'data_idxs' ],
m.decoder_output, m.total_loss] )
if task == 'segment2d' or task == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32,256,256,3), dtype='float')
k_embed = 8
# for i in range(predicted.shape[0]):
# embedding_flattened = np.squeeze(predicted[i]).reshape((-1,64))
# pca = PCA(n_components=3)
# pca.fit(embedding_flattened)
# min_order = None
# min_dist = float('inf')
# for order in itertools.permutations([0,1,2]):
# reordered = pca.components_[list(order), :]
# dist = np.linalg.norm(curr_comp-reordered)
# if dist < min_dist:
# min_order = list(order)
# min_dist = dist
# print(min_order)
# pca.components_ = pca.components_[min_order, :]
# curr_comp = pca.components_
# lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
# lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
# x[i] = lower_dim
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape((-1,64))
embeddings.append(embedding_flattened)
if len(embeddings) > k_embed:
embeddings.pop(0)
pca = PCA(n_components=3)
pca.fit(np.vstack(embeddings))
min_order = None
min_dist = float('inf')
copy_of_comp = np.copy(pca.components_)
for order in itertools.permutations([0,1,2]):
#reordered = pca.components_[list(order), :]
#dist = np.linalg.norm(curr_comp-reordered)
pca.components_ = copy_of_comp[order, :]
lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
dist = np.linalg.norm(lower_dim - curr_fit_img)
if dist < min_dist:
min_order = order
min_dist = dist
pca.components_ = copy_of_comp[min_order, :]
lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min())
curr_fit_img = np.copy(lower_dim)
x[i] = lower_dim
predicted = x
if task == 'curvature':
std = [31.922, 21.658]
mean = [123.572, 120.1]
predicted = (predicted * std) + mean
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
predicted = np.squeeze(np.clip(predicted.astype(int) / 255., 0., 1. )[:,:,:,0])
just_rescale = ['autoencoder', 'denoise', 'edge2d',
'edge3d', 'keypoint2d', 'keypoint3d',
'reshade', 'rgb2sfnorm']
if task in just_rescale:
predicted = (predicted + 1.) / 2.
predicted = np.clip(predicted, 0., 1.)
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
just_clip = ['rgb2depth', 'rgb2mist']
if task in just_clip:
predicted[:,0,0,:] = 0.
predicted[:,1,0,:] = 1.
if task == 'segmentsemantic_rb':
label = np.argmax(predicted, axis=-1)
COLORS = ('white','red', 'blue', 'yellow', 'magenta',
'green', 'indigo', 'darkorange', 'cyan', 'pink',
'yellowgreen', 'black', 'darkgreen', 'brown', 'gray',
'purple', 'darkviolet')
rgb = (input_batch + 1.) / 2.
preds = [color.label2rgb(np.squeeze(x), np.squeeze(y), colors=COLORS, kind='overlay')[np.newaxis,:,:,:] for x,y in zip(label, rgb)]
predicted = np.vstack(preds)
if task in ['class_1000', 'class_places']:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode('utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
sorted_pred = np.argsort(predict_output)[::-1]
top_5_pred = [synset[sorted_pred[i]] for i in range(5)]
to_print_pred = "Top 5 prediction: \n {}\n {}\n {}\n {} \n {}".format(*top_5_pred)
img = Image.new('RGBA', (400, 200), (255, 255, 255))
d = ImageDraw.Draw(img)
fnt = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSerifCondensed.ttf', 25)
d.text((20, 5), to_print_pred, fill=(255, 0, 0), font=fnt)
img.save(to_store_name, 'PNG')
else:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode('utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
scipy.misc.toimage(np.squeeze(predict_output), cmin=0.0, cmax=1.0).save(to_store_name)
subprocess.call('tar -czvf /home/ubuntu/{t}.tar.gz /home/ubuntu/{t}'.format(t=task), shell=True)
subprocess.call('aws s3 cp /home/ubuntu/{t}.tar.gz s3://task-preprocessing-512-oregon/video2/'.format(t=task), shell=True)
subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid2/020%04d.png -vcodec libx264 -crf 15 -pix_fmt yuv420p {t}_2.mp4'.format(t=task), shell=True)
subprocess.call('aws s3 cp {t}_2.mp4 s3://task-preprocessing-512-oregon/video2/'.format(t=task), shell=True)
############## Clean Up ##############
training_runners[ 'coord' ].request_stop()
training_runners[ 'coord' ].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
print("Done: {}".format(task))
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def run_extract_representations(args, cfg, save_dir, given_task):
transfer = (cfg['model_type'] == architectures.TransferNet)
if transfer:
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer
setup_input_fn = utils.setup_input_transfer
else:
setup_input_fn = utils.setup_input
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn
# set up logging
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as g:
# create ops and placeholders
tf.logging.set_verbosity(tf.logging.INFO)
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics(inputs, model, cfg)
# execute training
start_time = time.time()
utils.print_start_info(cfg, inputs['max_steps'], is_training=False)
# start session and restore model
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
try:
if cfg['model_path'] is None:
print('Please specify a checkpoint directory')
return
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
utils.print_start_info(cfg, inputs['max_steps'], is_training=False)
data_prefetch_init_fn = get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=False)
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
# run one example so that we can calculate some statistics about the representations
filenames = []
representations, data_idx = training_runners['sess'].run(
[model['model'].encoder_output, inputs['data_idxs']])
# print("Step number: {}".format(gs))
filenames.extend(data_idx)
if type(representations) == list:
representations = representations[0]
print(representations.shape)
representations = representations.reshape(REPRESENTATIONS_SHAPE)
print('Got first batch representation with size: {0}'.format(
representations.shape))
# run the remaining examples
for step in range(inputs['max_steps'] - 1):
#for step in range( 10 ):
if step % 100 == 0:
print('Step {0} of {1}'.format(step,
inputs['max_steps'] - 1))
# This is just for GAN, for the LEO meeting
encoder_output, data_idx = training_runners['sess'].run(
[model['model'].encoder_output, inputs['data_idxs']])
if type(encoder_output) == list:
encoder_output = encoder_output[0]
encoder_output = encoder_output.reshape(REPRESENTATIONS_SHAPE)
representations = np.append(representations,
encoder_output,
axis=0)
filenames.extend(data_idx)
if training_runners['coord'].should_stop():
break
print(
'The size of representations is %s while we expect it to run for %d steps with batchsize %d'
% (representations.shape, inputs['max_steps'],
cfg['batch_size']))
end_train_time = time.time() - start_time
if args.imagenet:
save_path = os.path.join(
save_dir,
'{task}_{split}_imagenet_representations.pkl'.format(
task=given_task, split=args.data_split))
elif args.places:
save_path = os.path.join(
save_dir,
'{task}_{split}_places_representations.pkl'.format(
task=given_task, split=args.data_split))
elif args.vid:
save_path = os.path.join(
save_dir, '{task}_vid{vid_id}_representations.pkl'.format(
task=given_task, vid_id=args.vid_id))
else:
save_path = os.path.join(
save_dir, '{task}_{split}_representations.pkl'.format(
task=given_task, split=args.data_split))
with open(save_path, 'wb') as f:
pickle.dump(
{
'file_indexes': filenames,
'representations': representations
}, f)
copy_to = None
if args.out_dir:
os.makedirs(args.out_dir, exist_ok=True)
os.system("sudo cp {fp} {out}/".format(fp=save_path,
out=args.out_dir))
else:
if transfer:
copy_to = cfg['log_root']
else:
copy_to = os.path.join(cfg['log_root'], given_task)
os.system("sudo mv {fp} {dst}/".format(fp=save_path,
dst=copy_to))
print("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to))
print('saved representations to {0}'.format(save_path))
print('moved representations to {0}'.format(copy_to))
print('time to extract %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
finally:
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
def train(device,
model_dir,
train_loader,
val_loader,
nn_layers,
epochs,
lr,
weight_decay=0.0,
criterion=torch.nn.L1Loss(),
loss_weights=None):
"""
Model training and validation phase
:param device: torch device object
:type model_dir: str
:param train_loader: data loader with test data
:param val_loader: data loader with validation data
:type nn_layers: list[int]
:type epochs: int
:type lr: float
:type weight_decay: float
:param criterion: loss function
:type loss_weights: list[float]
"""
# get start time
start = time.time()
# set up model
model = setup_model(layers=nn_layers) # use default activation
# model = npz_to_torch_model(filename="../../params/models/autorally_nnet_09_12_2018.npz", model=model) # to load pretrained model
# load model onto device
model.to(device)
# set up optimizer
optimizer = optim.Adam(
model.parameters(), lr=lr,
weight_decay=weight_decay) # weight_decay is L2 penalty term to loss
# set up data loaders
data_loaders = {"train": train_loader, "val": val_loader}
dataset_sizes = {
"train": train_loader.dataset.__len__(),
"val": val_loader.dataset.__len__()
}
# dict to store train and val losses for plotting
losses = {"train": [], "val": []}
# label cols
label_cols = train_loader.dataset.label_cols
# dict to store losses of the different loss components
split_losses = {"train": {}, "val": {}}
for label_col in label_cols:
split_losses['train'][label_col] = []
split_losses['val'][label_col] = []
best_val_loss = np.inf
if loss_weights is None:
loss_weights = np.ones(
nn_layers[-1]
) # if no weights are specified for the loss just set the weights to 1
for epoch in range(epochs):
print('\nEpoch %i/%i' % (epoch, epochs - 1))
print('-' * 10)
for phase in ["train", "val"]:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
temp_split_losses = {}
for label_col in label_cols:
temp_split_losses[label_col] = 0.0
for inputs, labels in data_loaders[phase]:
inputs = inputs.to(device)
labels = labels.to(device)
# clear gradients
optimizer.zero_grad()
# forward
with torch.set_grad_enabled(phase == "train"):
# get output from model
outputs = model(inputs.double())
# apply the specified loss weights and compute the loss
loss = criterion(
torch.t(
torch.mul(
outputs,
torch.tensor(loss_weights,
dtype=torch.float64).to(device))),
torch.t(
torch.mul(
labels.double(),
torch.tensor(loss_weights,
dtype=torch.float64).to(device))))
# save loss splits
for label_col, split_loss in zip(label_cols, loss):
temp_split_losses[label_col] += torch.sum(
split_loss).item()
# apply reduction to loss
# loss = torch.sum(loss)
loss = torch.mean(loss)
if phase == "train":
# calculate the gradients
loss.backward()
# update all the parameters based on the gradients calculated
optimizer.step()
# updating stats
# running_loss += loss.item()
running_loss += loss.item() * inputs.size(
0
) # multiply by batch size since calculated loss was the mean
# calculate loss for epoch
epoch_loss = running_loss / dataset_sizes[phase]
losses[phase].append(epoch_loss)
print('%s Loss: %.4f' % (phase, epoch_loss))
# calculate split losses
for label_col in label_cols:
split_losses[phase][label_col].append(
temp_split_losses[label_col] / dataset_sizes[phase])
# update new best val loss and model
if phase == "val" and epoch_loss < best_val_loss:
best_val_loss = epoch_loss
torch.save(
{
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': best_val_loss
}, os.path.join(model_dir, "model.pt"))
gc.collect()
time_elapsed = time.time() - start
print("\nTraining complete in %.0fm %.0fs" %
(time_elapsed // 60, time_elapsed % 60))
print("Best val loss %0.5f" % best_val_loss)
# convert model to npz format for mppi usage
torch_model_to_npz(model, model_dir)
# plot loss monitoring
fig = plt.figure()
x = np.arange(epochs)
plt.plot(x, losses['train'], 'b-', x, losses['val'], 'r-')
plt.ylim(bottom=0.0)
plt.title("Loss")
plt.legend(['train', 'val'], loc='upper right')
plt.xlabel("epoch")
plt.ylabel("loss")
fig.savefig(os.path.join(model_dir, "loss.pdf"), format="pdf")
plt.close(fig)
# plot loss splits
fig = plt.figure()
for label_col in label_cols:
plt.plot(x, split_losses['val'][label_col], label=label_col)
plt.title("Val loss splits")
plt.xlabel("epoch")
plt.ylabel("loss")
plt.legend(label_cols, loc='best')
plt.ylim(bottom=0.0)
fig.savefig(os.path.join(model_dir, "loss_splits.pdf"), format="pdf")
plt.close(fig)
def run_to_task():
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
imgs = args.im_name.split(',')
if args.task == 'ego_motion' and len(imgs) != 3:
raise ValueError('Wrong number of images, expecting 3 but got {}'.format(len(imgs)))
if args.task != 'ego_motion' and len(imgs) != 2:
raise ValueError('Wrong number of images, expecting 2 but got {}'.format(len(imgs)))
task = args.task
if task not in list_of_tasks:
raise ValueError('Task not supported')
cfg = generate_cfg(task)
input_img = np.empty((len(imgs),256,256,3), dtype=np.float32)
for i,imname in enumerate(imgs):
img = load_raw_image_center_crop( imname )
img = skimage.img_as_float(img)
scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(imname)
img = cfg[ 'input_preprocessing_fn' ]( img, **cfg['input_preprocessing_fn_kwargs'] )
input_img[i,:,:,:] = img
input_img = input_img[np.newaxis, :]
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() }
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn( cfg, is_training=False, use_filename_queue=False )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model( inputs, cfg, is_training=False )
m = model[ 'model' ]
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
predicted, representation = training_runners['sess'].run(
[ m.decoder_output, m.encoder_output ], feed_dict={m.input_images: input_img} )
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(representation))
if args.store_pred:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}_pred.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(predicted))
if task == 'ego_motion':
ego_motion(predicted, args.store_name)
return
if task == 'fix_pose':
cam_pose(predicted, args.store_name, is_fixated=True)
return
if task == 'non_fixated_pose':
cam_pose(predicted, args.store_name, is_fixated=False)
return
if task == 'point_match':
prediction = np.argmax(predicted, axis=1)
print('the prediction (1 stands for match, 0 for unmatch)is: ', prediction)
return
############## Clean Up ##############
training_runners[ 'coord' ].request_stop()
training_runners[ 'coord' ].join()
print("Done: {}".format(config_name))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def run_to_task():
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
task = args.task
if task not in list_of_tasks:
raise ValueError('Task not supported')
cfg = utils.generate_cfg(task)
# Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
segment25d segment2d room_layout'.split()
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
############## Single Image ##############
if args.imgs_list:
with open(args.imgs_list) as imgs_list:
all_prediction = []
all_representation = []
for line in imgs_list:
filename = args.dir_name + line.strip().split(',')[0] # FIXME
img = prepare_image(task, filename, cfg)
predicted, representation = training_runners['sess'].run(
[m.decoder_output, m.encoder_output],
feed_dict={m.input_images: img})
utils.tasks(
task,
args,
predicted,
os.path.join(args.store_name +
line.split(os.path.sep)[-1].strip() + '.jpg'),
img=img)
all_prediction.append(np.squeeze(predicted))
all_representation.append(np.squeeze(representation))
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.array(all_representation))
if args.store_pred:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}_pred.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.array(all_prediction))
else:
img = prepare_image(task, args.im_name, cfg)
predicted, representation = training_runners['sess'].run(
[m.decoder_output, m.encoder_output],
feed_dict={m.input_images: img})
utils.tasks(task, args, predicted, representation, img)
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(representation))
if args.store_pred:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}_pred.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(predicted))
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# print("Done: {}".format(config_name))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def run_rand_baseline( args, cfg, given_task ):
# set up logging
tf.logging.set_verbosity( tf.logging.INFO )
with tf.Graph().as_default() as g:
# create ops and placeholders
tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input( cfg, is_training=False, use_filename_queue=False )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model( inputs, cfg, is_training=False )
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics( inputs, model, cfg )
# execute training
start_time = time.time()
utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
# start session and restore model
training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() }
try:
utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False )
#training_runners[ 'threads' ] = data_prefetch_init_fn( training_runners[ 'sess' ], training_runners[ 'coord' ] )
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=( training_runners[ 'sess' ], training_runners[ 'coord' ] ))
prefetch_threads.start()
# run one example so that we can calculate some statistics about the representations
targets = training_runners['sess'].run( inputs[ 'target_batch' ] )
# run the remaining examples
for step in range( inputs[ 'max_steps' ] - 1 ):
#for step in range( 10 ):
if step % 100 == 0:
print( 'Step {0} of {1}'.format( step, inputs[ 'max_steps' ] - 1 ))
target = training_runners['sess'].run( inputs[ 'target_batch' ] )
targets = np.append( targets, target, axis=0)
if training_runners['coord'].should_stop():
break
rand_idx = [random.randint(0, targets.shape[0] - 1) for i in range(targets.shape[0])]
rand_target = [targets[i] for i in rand_idx]
rand_target = np.vstack(rand_target)
counter = 0
sum = 0
for step in range( inputs[ 'max_steps' ] - 1 ):
#for step in range( 10 ):
if step % 100 == 0:
print( 'Step {0} of {1}'.format( step, inputs[ 'max_steps' ] - 1 ))
tar = targets[step*cfg['batch_size']:(step+1)*cfg['batch_size']]
rand = rand_target[step*cfg['batch_size']:(step+1)*cfg['batch_size']]
losses = training_runners['sess'].run( model['model'].losses, feed_dict={
inputs['target_batch']: tar, model['model'].final_output:rand})
sum += losses[0]
counter += 1
if training_runners['coord'].should_stop():
break
print(sum)
print(counter)
print('random_baseline has loss: {loss}'.format(loss=sum/counter))
end_train_time = time.time() - start_time
finally:
utils.request_data_loading_end( training_runners )
utils.end_data_loading_and_sess( training_runners )
parser.add_argument('--gpu',
'-g',
type=int,
default=0,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--model-setup',
required=True,
help='Model setup dictionary.')
parser.add_argument('--lsh',
action='store_true',
default=False,
help='If true, uses locally sensitive hashing \
(with k=10 NN) for NN search.')
args = parser.parse_args(['--model-setup', 'result/snli_bilstm/args.json'])
model, train, test, vocab, setup = setup_model(args)
if setup['dataset'] == 'snli':
converter = convert_snli_seq
else:
converter = convert_seq
# FIXME
args.batchsize = 64
max_beam_size = 5
with open(os.path.join(setup['save_path'], 'calib.json')) as f:
calibration_idx = json.load(f)
calibration = [train[i] for i in calibration_idx]
train = [x for i, x in enumerate(train) if i not in calibration_idx]
train = random.sample(train, 10000)
def generate_predictions(device,
model_dir,
data_path,
nn_layers,
state_cols,
state_der_cols,
ctrl_cols,
time_col='time',
time_horizon=2.5,
data_frac=1.0,
feature_scaler=None,
label_scaler=None,
skip_first_batch=True):
"""
Model test phase. Generates truth and nn predicted trajectory for each batch
NOTE: many parts of this test phase are currently hard coded to a specific problem
:param device: torch device object
:type model_dir: str
:type data_path: str
:type nn_layers: list[int]
:type state_cols: list[str]
:type state_der_cols: list[str]
:type ctrl_cols: list[str]
:type time_col: str
:param time_horizon: total time to propagate dynamics for
:param data_frac: fraction of test data to use
:param feature_scaler: sklearn standard scaler for features
:param label_scaler: sklearn standard scaler for labels
:param skip_first_batch: option to skip first batch
"""
print("\nGenerating predictions from trained model...")
# size of state space
state_dim = len(state_cols)
# folder to store all test phase files
test_phase_dir = os.path.join(model_dir, "test_phase/")
# get time step from data
time_step = pd.read_csv(data_path).head(2)[time_col].values[1]
print("time step: %.04f" % time_step)
# determine batch size
batch_size = int(np.ceil(time_horizon / time_step))
print("batch size: %.0f" % batch_size)
# setup data loader
indices = np.arange(int(data_frac * len(pd.read_csv(data_path))))
data_loader = make_test_data_loader(data_path,
batch_size,
state_cols,
state_der_cols,
ctrl_cols,
indices=indices,
time_col=time_col)
# number of batches to do
total_batches = data_loader.dataset.__len__() // batch_size
# load model architecture
model = setup_model(layers=nn_layers)
# load model onto device
model.to(device)
# load weights + biases from pretrained model
state_dict = torch.load(os.path.join(model_dir, "model.pt"))
# check if model was saved as part of a dict
if "model_state_dict" in state_dict.keys():
state_dict = state_dict["model_state_dict"]
model.load_state_dict(state_dict)
# var to keep track of errors from propagating dynamics
errors_list = []
# var to keep track of instantaneous errors
inst_errors = []
with torch.no_grad():
# set model to eval mode
model.eval()
# keep track of current batch number
batch_num = 0
# generate a trajectory for each batch
for truth_states, truth_state_ders, ctrls, time_data in data_loader:
print('\nBatch %i/%i' % (batch_num, total_batches))
print('-' * 10)
num_steps = truth_states.size(0)
# skip last batch if it is less than batch size
if num_steps < batch_size:
print("Skipping final batch...")
continue
# skip first batch if specified
if skip_first_batch and batch_num == 0:
print("Skipping first batch...")
batch_num += 1
continue
# make a new folder to store results from this batch
batch_folder = test_phase_dir + "batch_" + str(batch_num) + "/"
if not os.path.exists(batch_folder):
os.makedirs(batch_folder)
# update batch number
batch_num += 1
# init state variables
nn_states = np.full((num_steps, state_dim), 0, np.float)
# set initial conditions
nn_states[0] = truth_states[0].cpu().numpy()
# array to store all state derivatives
state_ders = np.full((num_steps, truth_state_ders.size(1)), 0,
np.float)
# TODO: remove hard coded stuff
# iterate through each step of trajectory
for idx in range(num_steps - 1):
# prep inputs to feed to neural network
# x1 is the input from continuously feeding the predicted state back into the model
x1 = torch.tensor([
nn_states[idx][3], nn_states[idx][4], nn_states[idx][5],
nn_states[idx][6], ctrls[idx][0], ctrls[idx][1]
])
# x2 is just the input from the truth state used to calculate instantaneous errors of the model
x2 = torch.tensor([
truth_states[idx][3], truth_states[idx][4],
truth_states[idx][5], truth_states[idx][6], ctrls[idx][0],
ctrls[idx][1]
])
# get the current state
curr_state = nn_states[idx]
# if data was standardized, apply transform on test data
if feature_scaler is not None:
x1 = torch.tensor(
feature_scaler.transform(x1.reshape(1, -1))[0])
x2 = torch.tensor(
feature_scaler.transform(x2.reshape(1, -1))[0])
# get outputs of neural network
output1 = model(x1.double().to(device)).cpu().numpy()
output2 = model(x2.double().to(device)).cpu().numpy()
# apply inverse transform on output
if label_scaler is not None:
output1 = label_scaler.inverse_transform(output1)
output2 = label_scaler.inverse_transform(output2)
# output1 = truth_state_ders[idx + 1].cpu().numpy() # use the truth derivatives
# compute the state derivatives
state_der = compute_state_ders(
curr_state, output1, negate_yaw_der=False
) # NOTE: set negate_yaw_der to True if using autorally's model
# update states
nn_states[idx + 1] = curr_state + state_der * time_step
# save state derivatives
state_ders[idx + 1] = state_der[3:]
# calculate the instantaneous signed error
inst_errors.append(truth_state_ders[idx].cpu().numpy() -
output2)
curr_errors = np.abs(nn_states - truth_states.numpy())
# compute yaw errors
curr_errors[:, 2] = [e % (2 * np.pi) for e in curr_errors[:, 2]]
curr_errors[:, 2] = [(2 * np.pi) - e if e > np.pi else e
for e in curr_errors[:, 2]]
# save errors
errors_list.append(curr_errors)
# print some errors on final time step
print("abs x error (m): %.02f" % curr_errors[-1][0])
print("abs y error (m): %.02f" % curr_errors[-1][1])
print("abs yaw error (rad): %.02f" % (curr_errors[-1][2]))
# convert time data to numpy
time_data = time_data.cpu().numpy()
# convert control data to numpy
ctrls = ctrls.cpu().numpy()
# create pandas DataFrames
df_nn = pd.DataFrame(data=np.concatenate((np.reshape(
time_data, (len(time_data), 1)), nn_states, state_ders, ctrls),
axis=1),
columns=np.concatenate(
([time_col], state_cols, state_der_cols,
ctrl_cols)))
df_nn.to_csv(os.path.join(batch_folder, "nn_state_variables.csv"),
index=False,
header=True)
# load ground truth data
df_truth = pd.DataFrame(data=np.concatenate((np.reshape(
time_data,
(len(time_data), 1)), truth_states, truth_state_ders, ctrls),
axis=1),
columns=np.concatenate(
([time_col], state_cols,
state_der_cols, ctrl_cols)))
df_truth.to_csv(os.path.join(batch_folder,
"truth_state_variables.csv"),
index=False,
header=True)
# plot trajectories and state vs. time
state_variable_plots(df1=df_truth,
df1_label="ground truth",
df2=df_nn,
df2_label="nn",
dir_path=batch_folder,
cols_to_include=np.concatenate(
(state_cols, ctrl_cols)))
# plot state der vs. time
state_der_plots(df1=df_truth,
df1_label="ground truth",
df2=df_nn,
df2_label="nn",
dir_path=batch_folder,
cols_to_include=np.concatenate(
(state_der_cols, ctrl_cols)))
# save raw multi-step errors to disk
errors_array = np.array(errors_list)
np.save(file=os.path.join(test_phase_dir, "multi_step_err.npy"),
arr=errors_array)
# hacky way to get first set of time data
_, _, _, time_data = iter(data_loader).next()
time_data = time_data.cpu().numpy()
# plot mean errors and their std
multi_step_error_plots(errors_array,
time_data,
x_idx=0,
y_idx=1,
yaw_idx=2,
dir_path=test_phase_dir,
num_box_plots=3,
plot_hists=True,
num_hist=6)
# save raw instantaneous errors to disk
inst_errors = np.array(inst_errors)
np.save(file=os.path.join(test_phase_dir, "inst_err.npy"),
arr=inst_errors)
# plot histogram of signed instantaneous errors
inst_error_plots(inst_errors, state_der_cols, test_phase_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu',
'-g',
type=int,
default=0,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--model-setup',
required=True,
help='Model setup dictionary.')
parser.add_argument('--lsh',
action='store_true',
default=False,
help='If true, uses locally sensitive hashing \
(with k=10 NN) for NN search.')
args = parser.parse_args()
model, train, test, vocab, setup = setup_model(args)
is_snli = setup['dataset'] == 'snli'
if is_snli:
converter = convert_snli_seq
else:
converter = convert_seq
# FIXME
args.batchsize = 64
max_beam_size = 5
test_iter = chainer.iterators.SerialIterator(test,
args.batchsize,
repeat=False,
shuffle=False)
checkpoint = []
n_batches = len(test) // args.batchsize
for batch_idx, batch in enumerate(tqdm(test_iter, total=n_batches)):
if batch_idx > 10:
break
batch = converter(batch, device=args.gpu)
xs = batch['xs']
reduced_xs, removed_indices = get_rawr(model,
xs,
max_beam_size=max_beam_size,
snli=is_snli)
n_finals = [len(r) for r in reduced_xs]
batch_size = len(xs[0]) if is_snli else len(xs)
assert len(reduced_xs) == batch_size
xp = cupy.get_array_module(xs[0][0])
if is_snli:
prem = xs[0]
_reduced_xs = []
for i in range(batch_size):
for x in reduced_xs[i]:
_reduced_xs.append([prem[i].copy(), xp.asarray(x)])
reduced_xs = list(map(list, zip(*_reduced_xs)))
else:
reduced_xs = list(itertools.chain(*reduced_xs))
reduced_xs = [xp.asarray(x) for x in reduced_xs]
# reduced_xs = converter(reduced_xs, device=args.gpu, with_label=False)
removed_indices = list(itertools.chain(*removed_indices))
with chainer.using_config('train', False):
ss_0 = xp.asnumpy(model.predict(xs, softmax=True))
ss_1 = xp.asnumpy(model.predict(reduced_xs, softmax=True))
ys_0 = np.argmax(ss_0, axis=1)
ys_1 = np.argmax(ss_1, axis=1)
if is_snli:
xs = list(map(list, zip(*xs)))
xs = [(a.tolist(), b.tolist()) for a, b in xs]
reduced_xs = list(map(list, zip(*reduced_xs)))
reduced_xs = [(a.tolist(), b.tolist()) for a, b in reduced_xs]
else:
xs = [x.tolist() for x in xs]
reduced_xs = [x.tolist() for x in reduced_xs]
start = 0
for example_idx in range(len(xs)):
oi = xs[example_idx] # original input
op = int(ys_0[example_idx]) # original predictoin
oos = ss_0[example_idx] # original output distribution
label = int(batch['ys'][example_idx])
checkpoint.append([])
for i in range(start, start + n_finals[example_idx]):
ri = reduced_xs[i]
rp = int(ys_1[i]) # reduced prediction
rs = ss_1[i] # reduced output distribution
rr = removed_indices[i]
entry = {
'original_input': oi,
'reduced_input': ri,
'original_prediction': op,
'reduced_prediction': rp,
'original_scores': oos,
'reduced_scores': rs,
'removed_indices': rr,
'label': label
}
checkpoint[-1].append(entry)
start += n_finals[example_idx]
with open(os.path.join('rawr_dev.pkl'), 'wb') as f:
pickle.dump(checkpoint, f)
def run_extract_representations( args, cfg ):
# set up logging
tf.logging.set_verbosity( tf.logging.INFO )
with tf.Graph().as_default() as g:
cfg['randomize'] = False
cfg['num_epochs'] = 1
# cfg['num_read_threads'] = 5
# cfg['batch_size']=2
#if cfg['model_path'] is None:
# cfg['model_path'] = tf.train.latest_checkpoint( os.path.join( args.cfg_dir, "logs/slim-train/" ) )
cfg['model_path'] = os.path.join( args.cfg_dir, "logs/slim-train/model.ckpt-59690")
# create ops and placeholders
tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input( cfg, is_training=False, use_filename_queue=True )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
# build model (and losses and train_op)
model = utils.setup_model( inputs, cfg, is_training=False )
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics( inputs, model, cfg )
# execute training
start_time = time.time()
utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
# start session and restore model
training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() }
try:
if cfg['model_path'] is None:
print('Please specify a checkpoint directory')
return
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=True )
training_runners[ 'threads' ] = data_prefetch_init_fn( training_runners[ 'sess' ], training_runners[ 'coord' ] )
# run one example so that we can calculate some statistics about the representations
filenames = []
representations, data_idx = training_runners['sess'].run( [
model['model'].encoder_output, inputs[ 'data_idxs' ] ] )
filenames += [ inputs[ 'filepaths_list'][ i ] for i in data_idx ]
print( 'Got first batch representation with size: {0}'.format( representations.shape ) )
# run the remaining examples
for step in xrange( inputs[ 'max_steps' ] - 1 ):
if step % 100 == 0:
print( 'Step {0} of {1}'.format( step, inputs[ 'max_steps' ] - 1 ))
encoder_output, data_idx = training_runners['sess'].run( [
model['model'].encoder_output, inputs[ 'data_idxs' ] ] )
representations = np.append(representations, encoder_output, axis=0)
filenames += [ inputs[ 'filepaths_list'][ i ] for i in data_idx ]
if training_runners['coord'].should_stop():
break
print('The size of representations is %s while we expect it to run for %d steps with batchsize %d' % (representations.shape, inputs['max_steps'], cfg['batch_size']))
end_train_time = time.time() - start_time
save_path = os.path.join( args.cfg_dir, '../representations.pkl' )
with open( save_path, 'wb' ) as f:
pickle.dump( { 'filenames': filenames, 'representations': representations }, f )
print( 'saved representations to {0}'.format( save_path ))
print('time to train %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time/(60*60)))
print('avg time per epoch: %.3f hrs' % ( (end_train_time/(60*60)) / cfg['num_epochs']) )
finally:
utils.request_data_loading_end( training_runners )
utils.end_data_loading_and_sess( training_runners )
def run_extract_losses(args, cfg, save_dir, given_task):
transfer = (cfg['model_type'] == architectures.TransferNet)
if transfer:
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer
setup_input_fn = utils.setup_input_transfer
else:
setup_input_fn = utils.setup_input
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn
# set up logging
tf.logging.set_verbosity(tf.logging.ERROR)
stats = Statistics()
print_every = int(args.print_every)
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
#RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
#RuntimeDeterminedEnviromentVars.populate_registered_variables()
max_steps = get_max_steps(inputs['max_steps'], args.data_split)
# pdb.set_trace()
# build model (and losses and train_op)
model = utils.setup_model(inputs, cfg, is_training=False)
loss_names, loss_ops = get_extractable_losses(cfg, model)
if 'l1_loss' in loss_names:
display_loss = 'l1_loss'
elif 'l2_loss' in loss_names:
display_loss = 'l2_loss'
elif 'xentropy' in loss_names:
display_loss = 'xentropy'
elif 'metric_loss' in loss_names:
display_loss = 'metric_loss'
elif 'cycle_loss' in loss_names:
display_loss = 'cycle_loss'
else:
display_loss = 'total_loss'
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics(inputs, model, cfg)
# execute training
start_time = time.time()
utils.print_start_info(cfg, max_steps, is_training=False)
# start session and restore model
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
try:
if cfg['model_path'] is None:
print('Please specify a checkpoint directory')
return
print('Attention, model_path is ', cfg['model_path'])
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
# var = [v for v in tf.global_variables() if 'decoder' in v.name][0]
# print(training_runners[ 'sess' ].run(var))
utils.print_start_info(cfg, max_steps, is_training=False)
data_prefetch_init_fn = get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=False)
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
# run one example so that we can calculate some statistics about the representations
filenames = []
loss_names_to_vals = {name: [] for name in loss_names}
results = training_runners['sess'].run([
inputs['data_idxs'], inputs['target_batch'],
inputs['mask_batch'], *loss_ops
])
#gs = results[1]
data_idx = results[0]
losses = results[3:]
target_input = results[1]
mask_input = results[2]
for i, name in enumerate(loss_names):
loss_names_to_vals[name].append(losses[i])
filenames.extend(data_idx)
print("Step number: {}".format(1), (data_idx))
# print(target_input, target_input.sum())
# return
# training_runners['sess'].run([v for v in tf.global_variables() if "transfer/rep_conv_1/weights" in v.name][0])
# run the remaining examples
start = time.perf_counter()
for step in range(max_steps - 1):
results = training_runners['sess'].run([
inputs['data_idxs'],
# [v for v in tf.global_variables() if "transfer/rep_conv_1/weights/(weights)" in v.name][0],
# model['model'].encoder_endpoints['net1_1_output'],
# model['model'].encoder_endpoints['net1_2_output'],
*loss_ops
])
data_idx = results[0]
# print(data_idx)
losses = results[1:]
# p, t, m = results[1], results[2], results[3]
# losses = results[4:]
# print(p.mean(), t)
for i, name in enumerate(loss_names):
loss_names_to_vals[name].append(losses[i])
filenames.extend(data_idx)
stats.push(loss_names_to_vals[display_loss][-1])
# baseline_loss = get_xentropy_loss(p, t, m)
# tf_loss = loss_names_to_vals[display_loss][-1]
# print('tf {} | ours {}'.format(tf_loss, baseline_loss))
# pdb.set_trace()
if step % print_every == 0 and step > 0:
print(
'Step {0} of {1}: ({5} loss: {2:.3f} || stddev: {3:.3f} :: ({4:.2f} secs/step)'
.format(
step,
max_steps - 1,
stats.mean(),
np.sqrt(stats.variance()),
# stats.variance(),
(time.perf_counter() - start) / print_every,
display_loss))
start = time.perf_counter()
if training_runners['coord'].should_stop():
break
print(
'The size of losses is %s while we expect it to run for %d steps with batchsize %d'
% (len(filenames), inputs['max_steps'], cfg['batch_size']))
end_train_time = time.time() - start_time
if args.out_name:
out_name = args.out_name
else:
out_name = '{task}_{split}_losses.pkl'.format(
task=given_task, split=args.data_split)
save_path = os.path.join(save_dir, out_name)
with open(save_path, 'wb') as f:
loss_names_to_vals['file_indexes'] = filenames
loss_names_to_vals['global_step'] = 0
pickle.dump(loss_names_to_vals, f)
if args.out_dir:
os.makedirs(args.out_dir, exist_ok=True)
os.system("sudo cp {fp} {out}/".format(fp=save_path,
out=args.out_dir))
else:
if transfer:
copy_to = cfg['log_root']
else:
copy_to = os.path.join(cfg['log_root'], given_task)
os.system("sudo mv {fp} {dst}/".format(fp=save_path,
dst=copy_to))
print("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to))
# if transfer:
# os.makedirs('/home/ubuntu/s3/model_log/losses_transfer/', exist_ok=True)
# os.system("sudo cp {fp} /home/ubuntu/s3/model_log/losses_transfer/".format(fp=save_path))
# else:
# os.makedirs('/home/ubuntu/s3/model_log/losses/', exist_ok=True)
# os.system("sudo cp {fp} /home/ubuntu/s3/model_log/losses/".format(fp=save_path))
print('saved losses to {0}'.format(save_path))
print('time to extract %d epochs: %.3f hrs' %
(cfg['num_epochs'], end_train_time / (60 * 60)))
finally:
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
'coord': tf.train.Coordinator()
}
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(
inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
'''
# encoder (extract features)
predicted, representation = training_runners['sess'].run(
[ m.decoder_output, m.encoder_output ], feed_dict={m.input_images: img} )
'''
representation = training_runners['sess'].run(
m.encoder_output, feed_dict={m.input_images: img})
'''
if task == 'segment2d' or task == 'segment25d':
segmentation_pca(predicted, storePath+task+im_name.split('.')[0]+'.png')
def run_extract_losses_5_steps(args, cfg, save_dir, given_task):
transfer = (cfg['model_type'] == architectures.TransferNet)
if transfer:
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer
setup_input_fn = utils.setup_input_transfer
else:
setup_input_fn = utils.setup_input
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn
# set up logging
tf.logging.set_verbosity(tf.logging.ERROR)
stats = Statistics()
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
#RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
#RuntimeDeterminedEnviromentVars.populate_registered_variables()
max_steps = get_max_steps(inputs['max_steps'], args.data_split)
# build model (and losses and train_op)
model = utils.setup_model(inputs, cfg, is_training=False)
loss_names, loss_ops = get_extractable_losses(cfg, model)
if 'l1_loss' in loss_names:
display_loss = 'l1_loss'
elif 'l2_loss' in loss_names:
display_loss = 'l2_loss'
elif 'xentropy' in loss_names:
display_loss = 'xentropy'
elif 'metric_loss' in loss_names:
display_loss = 'metric_loss'
elif 'cycle_loss' in loss_names:
display_loss = 'cycle_loss'
else:
display_loss = 'total_loss'
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics(inputs, model, cfg)
# execute training
start_time = time.time()
utils.print_start_info(cfg, max_steps, is_training=False)
# start session and restore model
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
if cfg['model_path'] is None:
print('Please specify a checkpoint directory')
return
print('Attention, model_path is ', cfg['model_path'])
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
utils.print_start_info(cfg, max_steps, is_training=False)
data_prefetch_init_fn = get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=False)
prefetch_threads = threading.Thread(target=data_prefetch_init_fn,
args=(training_runners['sess'],
training_runners['coord']))
prefetch_threads.start()
# run one example so that we can calculate some statistics about the representations
# results = training_runners['sess'].run( [ *loss_ops ] )
# losses = results[0]
x = 0
for step in range(3):
results = training_runners['sess'].run([*loss_ops])
x = x + results[0]
if training_runners['coord'].should_stop():
break
tf.reset_default_graph()
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
return x / 2.
def run_extract_losses( args, cfg, save_dir, given_task ):
transfer = (cfg['model_type'] == architectures.TransferNet)
if transfer:
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer_imagenet
setup_input_fn = utils.setup_input_transfer_imagenet
else:
setup_input_fn = utils.setup_input
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn
# set up logging
tf.logging.set_verbosity( tf.logging.ERROR )
stats = Statistics()
top5_stats = Statistics()
print_every = int(args.print_every)
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = setup_input_fn( cfg, is_training=False, use_filename_queue=False )
#RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
#RuntimeDeterminedEnviromentVars.populate_registered_variables()
max_steps = get_max_steps(inputs[ 'max_steps' ], args.data_split)
# build model (and losses and train_op)
model = utils.setup_model( inputs, cfg, is_training=False )
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics( inputs, model, cfg )
# execute training
start_time = time.time()
utils.print_start_info( cfg, max_steps, is_training=False )
# start session and restore model
training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() }
try:
if cfg['model_path'] is None:
print('Please specify a checkpoint directory')
return
print('Attention, model_path is ', cfg['model_path'])
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
# var = [v for v in tf.global_variables() if 'decoder' in v.name][0]
# print(training_runners[ 'sess' ].run(var))
utils.print_start_info( cfg, max_steps, is_training=False )
data_prefetch_init_fn = get_data_prefetch_threads_init_fn( inputs, cfg,
is_training=False, use_filename_queue=False )
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=( training_runners[ 'sess' ], training_runners[ 'coord' ] ))
prefetch_threads.start()
# run one example so that we can calculate some statistics about the representations
filenames = []
accuracies = []
if transfer:
accuracy_op = model['model'].decoder.accuracy
final_output = model['model'].decoder.final_output
else:
accuracy_op = model['model'].accuracy
final_output = model['model'].final_output
results = training_runners['sess'].run( [
inputs[ 'data_idxs' ], model['model'].global_step,
accuracy_op ] )
gs = results[1]
data_idx = results[0]
accuracy = results[2]
filenames.extend(data_idx)
accuracies.append(accuracy)
print("Step number: {}".format(gs))
# print(loss_names_to_vals, data_idx)
# return
# run the remaining examples
start = time.perf_counter()
for step in range( max_steps - 1 ):
results = training_runners['sess'].run( [
inputs[ 'data_idxs' ],
final_output,
inputs['target_batch'],
accuracy_op ] )
data_idx = results[0]
accuracy = results[-1]
logits = results[1]
gt = results[2]
sorted_top5 = np.argsort(logits[0])[::-1][:5]
sorted_gt = np.argsort(gt[0])[::-1][0]
top5 = 0.
if sorted_gt in sorted_top5:
top5 = 1.
filenames.extend(data_idx)
accuracies.append(accuracy)
stats.push(accuracy)
top5_stats.push(top5)
if step % print_every == 0 and step > 0:
print( 'Step {0} of {1}: ({5}: {2:.3f} || Top 5: {3:.3f} :: ({4:.2f} secs/step)'.format(
step, max_steps - 1,
stats.mean(),
top5_stats.mean(),
# stats.variance(),
(time.perf_counter() - start) / print_every,
'accuracy'
))
start = time.perf_counter()
if training_runners['coord'].should_stop():
break
os.system("sudo touch /home/ubuntu/s3/imagenet_accuracy/{}_{}_{}.txt".format(
given_task, int(stats.mean() * 1000) / 10., int(top5_stats.mean() * 1000) / 10.))
print('The size of losses is %s while we expect it to run for %d steps with batchsize %d' % (len(filenames), inputs['max_steps'], cfg['batch_size']))
end_train_time = time.time() - start_time
if args.out_name:
out_name = args.out_name
else:
out_name = '{task}_{split}_imagenet_accuracy.pkl'.format(task=given_task, split=args.data_split)
save_path = os.path.join( save_dir, out_name )
val_accuracy = {}
with open( save_path, 'wb' ) as f:
val_accuracy['file_indexes'] = filenames
val_accuracy['global_step'] = gs
val_accuracy['accuracy'] = accuracies
pickle.dump( val_accuracy, f )
if args.out_dir:
os.makedirs(args.out_dir, exist_ok=True)
os.system("sudo cp {fp} {out}/".format(fp=save_path, out=args.out_dir))
else:
if transfer:
copy_to = cfg['log_root']
else:
copy_to = os.path.join(cfg['log_root'], given_task)
os.system("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to))
print("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to))
# if transfer:
# os.makedirs('/home/ubuntu/s3/model_log/losses_transfer/', exist_ok=True)
# os.system("sudo cp {fp} /home/ubuntu/s3/model_log/losses_transfer/".format(fp=save_path))
# else:
# os.makedirs('/home/ubuntu/s3/model_log/losses/', exist_ok=True)
# os.system("sudo cp {fp} /home/ubuntu/s3/model_log/losses/".format(fp=save_path))
print( 'saved losses to {0}'.format( save_path ))
print('time to extract %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time/(60*60)))
finally:
utils.request_data_loading_end( training_runners )
utils.end_data_loading_and_sess( training_runners )
def run_extract_losses(args, cfg, save_dir, given_task):
transfer = (cfg['model_type'] == architectures.TransferNet)
if transfer:
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer
setup_input_fn = utils.setup_input_transfer
if given_task == 'pixels':
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer_imagenet
setup_input_fn = utils.setup_input_transfer_imagenet
else:
setup_input_fn = utils.setup_input
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn
# set up logging
tf.logging.set_verbosity(tf.logging.ERROR)
stats = Statistics()
print_every = int(args.print_every)
with tf.Graph().as_default() as g:
# create ops and placeholders
inputs = setup_input_fn(cfg,
is_training=True,
use_filename_queue=False)
#RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
#RuntimeDeterminedEnviromentVars.populate_registered_variables()
max_steps = get_max_steps(inputs['max_steps'], args.data_split)
# build model (and losses and train_op)
model = utils.setup_model(inputs, cfg, is_training=True)
# set up metrics to evaluate
names_to_values, names_to_updates = setup_metrics(inputs, model, cfg)
train_step_fn = model['train_step_fn']
# execute training
start_time = time.time()
utils.print_start_info(cfg, max_steps, is_training=True)
# start session and restore model
training_runners = {
'sess': tf.Session(),
'coord': tf.train.Coordinator()
}
try:
if cfg['model_path'] is None:
print('Please specify a checkpoint directory')
return
print('Attention, model_path is ', cfg['model_path'])
restore_ckpt = not args.from_scratch
if restore_ckpt:
non_encoder_var = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES)
adams = []
for v in tuple(non_encoder_var):
if 'Adam' in v.name:
non_encoder_var.remove(v)
adams.append(v)
continue
if 'finetune_encoder_imagenet' in cfg:
for x in model['model'].encoder_vars:
if v.name == x.name:
non_encoder_var.remove(v)
if not args.metric_only:
saver_for_transfer = tf.train.Saver(non_encoder_var)
#training_runners['sess'].run(saver_for_transfer)
saver_for_transfer.restore(training_runners['sess'],
cfg['model_path'])
else:
#saver_for_transfer = tf.train.Saver(non_encoder_var)
training_runners['sess'].run(
tf.variables_initializer(non_encoder_var))
training_runners['sess'].run(tf.variables_initializer(adams))
print('Loading Source Encoder:...')
if 'finetune_encoder_imagenet' in cfg:
model['init_fn'](training_runners['sess'])
print('Starting Training:..')
else:
init_op = tf.global_variables_initializer()
training_runners['sess'].run(init_op)
assign_op = model['global_step'].assign(0)
training_runners['sess'].run(assign_op)
# var = [v for v in tf.global_variables() if 'decoder' in v.name][0]
# print(training_runners[ 'sess' ].run(var))
utils.print_start_info(cfg, max_steps, is_training=True)
data_prefetch_init_fn = get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=True, use_filename_queue=False)
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
# run one example so that we can calculate some statistics about the representations
start = time.perf_counter()
saver = tf.train.Saver()
save_ckpt_name = 'places'
if args.from_scratch:
save_ckpt_name = 'places_scratch_{}_{}'.format(
args.layers, args.data_used)
if args.train_encoder:
save_ckpt_name = 'places_encoder'
for step in range(max_steps // 2 - 1):
#for step in range(10):
total_loss, should_stop = train_step_fn(
training_runners['sess'],
model['train_op'],
model['global_step'],
train_step_kwargs=model['train_step_kwargs'])
# print(data_idx)
# print(p.mean(), t)
stats.push(total_loss)
if step % print_every == 0 and step > 0:
print(
'Step {0} of {1}: ({5}: {2:.3f} || stddev: {3:.3f} :: ({4:.2f} secs/step)'
.format(
step,
max_steps - 1,
stats.mean(),
np.sqrt(stats.variance()),
# stats.variance(),
(time.perf_counter() - start) / print_every,
'Total_loss'))
start = time.perf_counter()
if step % 3000 == 2999:
saver.save(
training_runners['sess'],
os.path.join(cfg['log_root'], given_task,
'{}_{}'.format(save_ckpt_name, step)))
if training_runners['coord'].should_stop():
break
#print('Heressss')
saver.save(
training_runners['sess'],
os.path.join(cfg['log_root'], given_task, save_ckpt_name))
finally:
utils.request_data_loading_end(training_runners)
utils.end_data_loading_and_sess(training_runners)
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
from importlib import reload
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
# for arch in ['regular', 'shallow', 'dilated_shallow', 'dilated_regular']:
arch = args.arch
data_amount = args.data
if args.second_order:
global TRANSFER_TYPE
TRANSFER_TYPE = 'second_order'
if not args.no_regenerate_data:
#if False:
all_outputs = {}
pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format(
arch=arch, hs=args.hs, task_to=task_to)
subprocess.call(
"aws s3 cp s3://task-preprocessing-512-oregon/visualizations/transfer_viz/viz_{}.pkl {}"
.format(task_to, pickle_dir),
shell=True)
import os
if os.path.isfile(pickle_dir):
with open(pickle_dir, 'rb') as fp:
all_outputs = pickle.load(fp)
if args.second_order:
import itertools
with open(
'/home/ubuntu/task-taxonomy-331b/tools/ranked_first_order_transfers.pkl',
'rb') as fp:
data = pickle.load(fp)
list_of_src_tasks = list(
itertools.combinations(data[task_to][:5], 2))
list_of_src_tasks = [
'{}__{}'.format(x[0], x[1]) for x in list_of_src_tasks
]
with open(
'/home/ubuntu/task-taxonomy-331b/tools/second_order_should_flip.pkl',
'rb') as fp:
to_flip_dict = pickle.load(fp)
if args.find_all_src:
config_dir_root = '/home/ubuntu/task-taxonomy-331b/experiments/second_order/{arch}/{data}'.format(
arch=arch, data=data_amount)
all_configs = os.listdir(config_dir_root)
list_of_src_tasks = []
for i in all_configs:
if i.split('__')[-3] == task_to:
list_of_src_tasks.append(i)
list_of_src_tasks = [
'__'.join(x.split('__')[:2]) for x in list_of_src_tasks
]
rank_combo = {}
for task_from in list_of_src_tasks:
first, sec = task_from.split('__')
rank_combo[task_from] = (data[task_to].index(first),
data[task_to].index(sec))
global list_of_src_tasks
for i, task_from in enumerate(list_of_src_tasks):
if args.data == '16k':
if task_from in all_outputs:
print("{} already exists....\n\n\n".format(task_from))
continue
else:
if '{}_{}'.format(task_from, args.data) in all_outputs:
print("{} already exists....\n\n\n".format(task_from))
continue
print("Doing from {task_from} to {task_to}".format(
task_from=task_from, task_to=task_to))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
if task_from == "FULL" or task_from == "FULL_IMAGE":
task = '{f}__{t}__{hs}__unlocked'.format(f="FULL",
t=task_to,
hs=args.hs)
transfer_src = 'full_order' if task_from == "FULL" else 'full_order_image'
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=transfer_src,
arch=arch,
data=data_amount,
TASK=task)
elif task_from == "FULL_select" or task_from == "FULL_select_IMAGE":
task = '{f}__{t}__{hs}__unlocked'.format(f="FULL_select",
t=task_to,
hs=args.hs)
transfer_src = 'full_order_selected' if task_from == "FULL_select" else 'full_order_selected_image'
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=transfer_src,
arch=arch,
data=data_amount,
TASK=task)
else:
task = '{f}__{t}__{hs}__unlocked'.format(f=task_from,
t=task_to,
hs=args.hs)
CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format(
transfer_type=TRANSFER_TYPE,
arch=arch,
data=data_amount,
TASK=task)
print(CONFIG_DIR)
############## Load Configs ##############
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
if args.second_order and not args.find_all_src and not to_flip_dict[
task]:
cfg['val_representations_file'] = cfg[
'val_representations_file'][::-1]
cfg['num_epochs'] = 1
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train'
#'time'
))
# print(cfg['model_path'])
if cfg['model_path'] is None and task == 'random':
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train',
'time'))
if cfg['model_path'] is None:
continue
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input_transfer(
cfg, is_training=ON_TEST_SET, use_filename_queue=False
) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=True)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn_transfer(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False)
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
############## Run First Batch ##############
(
input_batch,
representation_batch,
target_batch,
data_idx,
encoder_output,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.input_representations, m.decoder.targets,
model['data_idxs'], m.encoder_output, m.decoder.decoder_output,
m.total_loss
])
if task_to == 'segment2d' or task_to == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
pca = PCA(n_components=3)
pca.fit(embedding_flattened)
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (
lower_dim.max() - lower_dim.min())
x[i] = lower_dim
predicted = x
if task_to == 'segmentsemantic_rb':
predicted = np.argmax(predicted, axis=-1)
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
to_store = {
'data_idx': data_idx,
'output': predicted,
'loss': loss
}
if args.second_order and args.find_all_src:
store_key = "{}_{}".format(task_from[:20],
rank_combo[task_from])
all_outputs[store_key] = to_store
elif args.data != '16k':
store_key = "{}_{}".format(task_from, args.data)
all_outputs[store_key] = to_store
else:
all_outputs[task_from] = to_store
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
#print(sys.modules.keys())
#del sys.modules[ 'config' ]
sys.path = remove_dups(sys.path)
print('Current Directory: ', os.getcwd())
pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format(
arch=arch, hs=args.hs, task_to=task_to)
with open(pickle_dir, 'wb') as fp:
pickle.dump(all_outputs, fp)
subprocess.call(
"aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/transfer_viz/viz_{}.pkl"
.format(pickle_dir, task_to),
shell=True)
# Run jupyter nb
print('Running Jupyter Notebooks...')
#os.makedirs("/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/transfer_{hs}_{arch}".format(hs=args.hs, arch=arch), exist_ok=True)
notebook_path = '/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/Visual_{task_to}'.format(
task_to=task_to)
if args.second_order and not args.find_all_src:
notebook_path = '{}-Copy1'.format(notebook_path)
subprocess.call("jupyter nbconvert \
--execute {notebook_path}.ipynb \
--to html \
--ExecutePreprocessor.kernel_name=python3 \
--ExecutePreprocessor.timeout=1200 ".format(
notebook_path=notebook_path, arch=arch, hs=args.hs, task_to=task_to),
shell=True)
subprocess.call(
"aws s3 cp {}.html s3://task-preprocessing-512-oregon/visualizations/{}/"
.format(notebook_path, TRANSFER_TYPE),
shell=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu',
'-g',
type=int,
default=0,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--model-setup',
required=True,
help='Model setup dictionary.')
parser.add_argument('--lsh',
action='store_true',
default=False,
help='If true, uses locally sensitive hashing \
(with k=10 NN) for NN search.')
args = parser.parse_args()
model, train, test, vocab, setup = setup_model(args)
if setup['dataset'] == 'snli':
converter = convert_snli_seq
use_snli = True
else:
converter = convert_seq
use_snli = False
with open(os.path.join(setup['save_path'], 'calib.json')) as f:
calibration_idx = json.load(f)
calibration = [train[i] for i in calibration_idx]
train = [x for i, x in enumerate(train) if i not in calibration_idx]
'''save dknn layers for training data'''
dknn = DkNN(model, lsh=args.lsh)
dknn.build(train,
batch_size=setup['batchsize'],
converter=converter,
device=args.gpu)
'''calibrate the dknn credibility values'''
dknn.calibrate(calibration,
batch_size=setup['batchsize'],
converter=converter,
device=args.gpu)
'''run dknn on evaluation data'''
test_iter = chainer.iterators.SerialIterator(test,
setup['batchsize'],
repeat=False)
test_iter.reset()
print('run dknn on evaluation data')
total = 0
n_reg_correct = 0
n_knn_correct = 0
n_batches = len(test) // setup['batchsize']
for test_batch in tqdm(test_iter, total=n_batches):
data = converter(test_batch, device=args.gpu, with_label=True)
text = data['xs']
knn_pred, knn_cred, knn_conf, reg_pred, reg_conf = dknn.predict(
text, snli=use_snli)
label = [int(x) for x in data['ys']]
total += len(label)
n_knn_correct += sum(x == y for x, y in zip(knn_pred, label))
n_reg_correct += sum(x == y for x, y in zip(reg_pred, label))
print('knn accuracy', n_knn_correct / total)
print('reg accuracy', n_reg_correct / total)
def run_to_task(args):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
tf.logging.set_verbosity(tf.logging.ERROR)
img = load_raw_image_center_crop(args.im_name)
img = skimage.img_as_float(img)
scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(args.im_name)
task = args.task
if task not in list_of_tasks:
raise ValueError('Task not supported')
cfg = generate_cfg(task)
# Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
segment25d segment2d room_layout'.split()
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
if task == 'jigsaw':
img = cfg['input_preprocessing_fn'](
img,
target=cfg['target_dict'][random.randint(0, 99)],
**cfg['input_preprocessing_fn_kwargs'])
else:
img = cfg['input_preprocessing_fn'](
img, **cfg['input_preprocessing_fn_kwargs'])
img = img[np.newaxis, :]
if task == 'class_places' or task == 'class_1000':
synset = get_synset(task)
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
predicted, representation = training_runners['sess'].run(
[m.decoder_output, m.encoder_endpoints[args.encoder_layer]],
feed_dict={m.input_images: img})
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
if args.compress_rep:
representation = tf.keras.layers.AveragePooling2D(
pool_size=(2, 2))(tf.constant(representation))
representation = training_runners['sess'].run(representation)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(representation))
if args.store_pred:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}_pred.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(predicted))
if task == 'segment2d' or task == 'segment25d':
segmentation_pca(predicted, args.store_name)
return
if task == 'colorization':
single_img_colorize(predicted, img, args.store_name)
return
if task == 'curvature':
curvature_single_image(predicted, args.store_name)
return
just_rescale = [
'autoencoder', 'denoise', 'edge2d', 'edge3d', 'keypoint2d',
'keypoint3d', 'reshade', 'rgb2sfnorm'
]
if task in just_rescale:
simple_rescale_img(predicted, args.store_name)
return
just_clip = ['rgb2depth', 'rgb2mist']
if task in just_clip:
depth_single_image(predicted, args.store_name)
return
if task == 'inpainting_whole':
inpainting_bbox(predicted, args.store_name)
return
if task == 'segmentsemantic':
semseg_single_image(predicted, img, args.store_name)
return
if task in ['class_1000', 'class_places']:
classification(predicted, synset, args.store_name)
return
if task == 'vanishing_point':
_ = plot_vanishing_point_smoothed(np.squeeze(predicted),
(np.squeeze(img) + 1.) / 2.,
args.store_name, [])
return
if task == 'room_layout':
mean = np.array([
0.006072743318127848, 0.010272365569691076, -3.135909774145468,
1.5603802322235532, 5.6228218371102496e-05, -1.5669352793761442,
5.622875878174759, 4.082800262277375, 2.7713941642895956
])
std = np.array([
0.8669452525283652, 0.687915294956501, 2.080513632043758,
0.19627420479282623, 0.014680602791251812, 0.4183827359302299,
3.991778013006544, 2.703495278378409, 1.2269185938626304
])
predicted = predicted * std + mean
plot_room_layout(np.squeeze(predicted), (np.squeeze(img) + 1.) / 2.,
args.store_name, [],
cube_only=True)
return
if task == 'jigsaw':
predicted = np.argmax(predicted, axis=1)
perm = cfg['target_dict'][predicted[0]]
show_jigsaw((np.squeeze(img) + 1.) / 2., perm, args.store_name)
return
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
print("Done: {}".format(config_name))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def deep_attribution():
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
tf.logging.set_verbosity(tf.logging.ERROR)
imlist_file_path = os.path.join(prj_dir, args.explain_result_root,
args.dataset, 'imlist.txt')
explain_result_root = os.path.join(prj_dir, args.explain_result_root,
args.dataset)
with open(imlist_file_path) as f:
lines = []
line = f.readline().strip()
while len(line) != 0:
lines += [os.path.join(prj_dir, 'dataset', args.dataset, line)]
line = f.readline().strip()
explain_methods = ['saliency', 'grad*input', 'elrp']
for task in list_of_tasks:
if not os.path.exists(os.path.join(explain_result_root, task)):
os.mkdir(os.path.join(explain_result_root, task))
cfg = generate_cfg(task)
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
sess = tf.Session()
training_runners = {'sess': sess, 'coord': tf.train.Coordinator()}
with DeepExplain(session=sess, graph=sess.graph) as de:
############## Set Up Inputs ##############
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
encoder_endpoints = model['model'].encoder_endpoints
endpoints = encoder_endpoints
print('There are {} images in {}'.format(len(lines), imlist_file_path))
img = load_raw_image_center_crop(lines[0])
img = skimage.img_as_float(img)
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
segment25d segment2d room_layout'.split()
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
img = cfg['input_preprocessing_fn'](
img, **cfg['input_preprocessing_fn_kwargs'])
imgs = np.zeros([args.imlist_size, 1] + list(img.shape), float)
for line_i in range(args.imlist_size):
img = load_raw_image_center_crop(lines[line_i])
img = skimage.img_as_float(img)
if task not in list_of_tasks:
raise ValueError('Task not supported')
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
if task == 'jigsaw':
img = cfg['input_preprocessing_fn'](
img,
target=cfg['target_dict'][random.randint(0, 99)],
**cfg['input_preprocessing_fn_kwargs'])
else:
img = cfg['input_preprocessing_fn'](
img, **cfg['input_preprocessing_fn_kwargs'])
imgs[line_i, :] = img[np.newaxis, :]
elrp = np.zeros([args.imlist_size] + list(img.shape), float)
saliency = np.zeros([args.imlist_size] + list(img.shape), float)
gradXinput = np.zeros([args.imlist_size] + list(img.shape), float)
for im_i in range(args.imlist_size):
with DeepExplain(session=sess) as de:
representation = training_runners['sess'].run(
[endpoints['encoder_output']],
feed_dict={m.input_images: imgs[im_i]})
attributions = {
explain_method:
de.explain(explain_method, endpoints['encoder_output'],
m.input_images, imgs[im_i])
for explain_method in explain_methods
}
elrp[im_i] = attributions['elrp']
saliency[im_i] = attributions['saliency']
gradXinput[im_i] = attributions['grad*input']
print('{} images done.'.format(im_i))
if ((im_i + 1) % 5) == 0:
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
sess = tf.Session()
training_runners = {
'sess': sess,
'coord': tf.train.Coordinator()
}
with DeepExplain(session=sess, graph=sess.graph) as de:
############## Set Up Inputs ##############
setup_input_fn = utils.setup_input
inputs = setup_input_fn(cfg,
is_training=False,
use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(
inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables(
)
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=False)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
encoder_endpoints = model['model'].encoder_endpoints
endpoints = encoder_endpoints
np.save(os.path.join(explain_result_root, task, 'elrp.npy'), elrp)
np.save(os.path.join(explain_result_root, task, 'saliency.npy'),
saliency)
np.save(os.path.join(explain_result_root, task, 'gradXinput.npy'),
gradXinput)
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
print('Task {} Done!'.format(task))
print('All Done.')
return
def run_to_task():
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
args = parser.parse_args()
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
tf.logging.set_verbosity(tf.logging.ERROR)
image_list = glob.glob(args.img_dir +"/*.png")
image_list.sort()
#image_list=image_list[:5]
print(len(image_list))
for task in tqdm(list_of_tasks):
print("Task is ", task)
if task not in list_of_tasks:
raise ValueError('Task not supported')
low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \
keypoint2d keypoint3d \
reshade rgb2depth rgb2mist rgb2sfnorm \
segment25d segment2d room_layout'.split()
cfg = generate_cfg(task)
if task in low_sat_tasks:
cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() }
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
setup_input_fn = utils.setup_input
inputs = setup_input_fn( cfg, is_training=False, use_filename_queue=False )
RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg )
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model( inputs, cfg, is_training=False )
m = model[ 'model' ]
model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] )
#Prints all the class variables and functions
print(dir(m))
for img_name in tqdm(image_list):
filename = img_name.split('/')[-1].split('.')[0]
img = load_raw_image_center_crop( img_name )
img = skimage.img_as_float(img)
scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(img_name)
# Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value
if task == 'jigsaw' :
img = cfg[ 'input_preprocessing_fn' ]( img, target=cfg['target_dict'][random.randint(0,99)],
**cfg['input_preprocessing_fn_kwargs'] )
else:
img = cfg[ 'input_preprocessing_fn' ]( img, **cfg['input_preprocessing_fn_kwargs'] )
img = img[np.newaxis,:]
if task in fc_task_list:
predicted, representation, decoder_features, encoder_features = training_runners['sess'].run(
[ m.decoder_output, m.encoder_output, m.metric_endpoints, m.encoder_endpoints ], feed_dict={m.input_images: img} )
else:
predicted, representation, decoder_features, encoder_features = training_runners['sess'].run(
[ m.decoder_output, m.encoder_output, m.decoder_endpoints, m.encoder_endpoints ], feed_dict={m.input_images: img} )
#np.save(save_path,value)
#for name,value in decoder_features.items():
# print (name)
## CKD : Uncomment below for loop
for name,value in encoder_features.items():
if name in encoder_save_list or name in feedforward_encoder_save_list:
#print (name)
name = name.replace('/', '_')
save_path = os.path.join(args.save_dir,filename+"_"+task + "_" + name + ".npy")
np.save(save_path,value)
if args.store_rep:
s_name, file_extension = os.path.splitext(args.store_name)
with open('{}.npy'.format(s_name), 'wb') as fp:
np.save(fp, np.squeeze(representation))
if args.store_pred:
save_path = os.path.join(args.save_dir,filename+"_"+task + "_" + "prediction" + ".npy")
with open(save_path, 'wb') as fp:
np.save(fp, np.squeeze(predicted))
#if task == 'segment2d' or task == 'segment25d':
# segmentation_pca(predicted, args.store_name)
#return
#if task == 'colorization':
# single_img_colorize(predicted, img , args.store_name)
#return
#if task == 'curvature':
# curvature_single_image(predicted, args.store_name)
#return
#just_rescale = ['autoencoder', 'denoise', 'edge2d',
# 'edge3d', 'keypoint2d', 'keypoint3d',
# 'reshade', 'rgb2sfnorm' ]
#if task in just_rescale:
# print(args.store_name)
#simple_rescale_img(predicted, args.store_name)
#return
#just_clip = ['rgb2depth', 'rgb2mist']
#if task in just_clip:
# depth_single_image(predicted, args.store_name)
# #return
#if task == 'inpainting_whole':
# inpainting_bbox(predicted, args.store_name)
#return
#if task == 'segmentsemantic':
# semseg_single_image( predicted, img, args.store_name)
#return
#if task in ['class_1000', 'class_places']:
# print("The shape of predicted is ---------------", predicted.shape)
#classification(predicted, synset, args.store_name)
#return
#if task == 'vanishing_point':
# _ = plot_vanishing_point_smoothed(np.squeeze(predicted), (np.squeeze(img) + 1. )/2., args.store_name, [])
#return
#if task == 'room_layout':
# mean = np.array([0.006072743318127848, 0.010272365569691076, -3.135909774145468,
# 1.5603802322235532, 5.6228218371102496e-05, -1.5669352793761442,
# 5.622875878174759, 4.082800262277375, 2.7713941642895956])
# std = np.array([0.8669452525283652, 0.687915294956501, 2.080513632043758,
# 0.19627420479282623, 0.014680602791251812, 0.4183827359302299,
# 3.991778013006544, 2.703495278378409, 1.2269185938626304])
# predicted = predicted * std + mean
# plot_room_layout(np.squeeze(predicted), (np.squeeze(img) + 1. )/2., args.store_name, [], cube_only=True)
#return
#if task == 'jigsaw':
# predicted = np.argmax(predicted, axis=1)
# perm = cfg[ 'target_dict' ][ predicted[0] ]
# show_jigsaw((np.squeeze(img) + 1. )/2., perm, args.store_name)
#return
############## Clean Up ##############
training_runners[ 'coord' ].request_stop()
training_runners[ 'coord' ].join()
print("Done: {}".format(task))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
from data.load_ops import rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
from data.task_data_loading import load_and_specify_preprocessors_for_input_depends_on_target
import lib.data.load_ops as load_ops
tf.logging.set_verbosity(tf.logging.ERROR)
args = parser.parse_args()
cfg, is_transfer, task, config_name = generate_cfg(args.config, args.vid,
args)
if task == 'class_places' or task == 'class_1000':
synset = get_synset(task)
if task == 'jigsaw':
cfg['preprocess_fn'] = load_and_specify_preprocessors_for_input_depends_on_target
print("Doing {task}".format(task=task))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
############## Start dataloading workers ##############
if is_transfer:
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer
setup_input_fn = utils.setup_input_transfer
else:
setup_input_fn = utils.setup_input
get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False)
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=IN_TRAIN_MODE)
m = model['model']
model['saver_op'].restore(training_runners['sess'], cfg['model_path'])
data_prefetch_init_fn = get_data_prefetch_threads_init_fn(
inputs, cfg, is_training=False, use_filename_queue=False)
prefetch_threads = threading.Thread(target=data_prefetch_init_fn,
args=(training_runners['sess'],
training_runners['coord']))
prefetch_threads.start()
list_of_fname = np.load(
'/home/ubuntu/task-taxonomy-331b/assets/aws_data/video{}_fname.npy'.
format(args.vid))
import errno
try:
os.mkdir('/home/ubuntu/{}'.format(task))
os.mkdir('/home/ubuntu/{}/vid1'.format(task))
os.mkdir('/home/ubuntu/{}/vid2'.format(task))
os.mkdir('/home/ubuntu/{}/vid3'.format(task))
os.mkdir('/home/ubuntu/{}/vid4'.format(task))
except OSError as e:
if e.errno != errno.EEXIST:
raise
curr_comp = np.zeros((3, 64))
curr_fit_img = np.zeros((256, 256, 3))
embeddings = []
curr_vp = []
curr_layout = []
############## Run First Batch ##############
def rescale_l_for_display(batch, rescale=True):
'''
Prepares network output for display by optionally rescaling from [-1,1],
and by setting some pixels to the min/max of 0/1. This prevents matplotlib
from rescaling the images.
'''
if rescale:
display_batch = [
rescale_image(im.copy(),
new_scale=[0, 100],
current_scale=[-1, 1]) for im in batch
]
else:
display_batch = batch.copy()
for im in display_batch:
im[0, 0,
0] = 1.0 # Adjust some values so that matplotlib doesn't rescale
im[0, 1, 0] = 0.0 # Now adjust the min
return display_batch
for step_num in range(inputs['max_steps'] - 1):
#for step_num in range(20):
#if step_num > 0 and step_num % 20 == 0:
print(step_num)
if is_transfer:
(input_batch, target_batch, data_idx,
predicted) = training_runners['sess'].run([
m.input_images, m.target_images, model['data_idxs'],
m.decoder.decoder_output
])
else:
(input_batch, target_batch, data_idx,
predicted) = training_runners['sess'].run([
m.input_images, m.targets, model['data_idxs'],
m.decoder_output
])
if task == 'segment2d' or task == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
k_embed = 8
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
embeddings.append(embedding_flattened)
if len(embeddings) > k_embed:
embeddings.pop(0)
pca = PCA(n_components=3)
pca.fit(np.vstack(embeddings))
min_order = None
min_dist = float('inf')
copy_of_comp = np.copy(pca.components_)
for order in itertools.permutations([0, 1, 2]):
#reordered = pca.components_[list(order), :]
#dist = np.linalg.norm(curr_comp-reordered)
pca.components_ = copy_of_comp[order, :]
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (
lower_dim.max() - lower_dim.min())
dist = np.linalg.norm(lower_dim - curr_fit_img)
if dist < min_dist:
min_order = order
min_dist = dist
pca.components_ = copy_of_comp[min_order, :]
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() -
lower_dim.min())
curr_fit_img = np.copy(lower_dim)
x[i] = lower_dim
predicted = x
if task == 'curvature':
std = [31.922, 21.658]
mean = [123.572, 120.1]
predicted = (predicted * std) + mean
predicted[:, 0, 0, :] = 0.
predicted[:, 1, 0, :] = 1.
predicted = np.squeeze(
np.clip(predicted.astype(int) / 255., 0., 1.)[:, :, :, 0])
if task == 'colorization':
maxs = np.amax(predicted, axis=-1)
softmax = np.exp(predicted - np.expand_dims(maxs, axis=-1))
sums = np.sum(softmax, axis=-1)
softmax = softmax / np.expand_dims(sums, -1)
kernel = np.load(
'/home/ubuntu/task-taxonomy-331b/lib/data/pts_in_hull.npy')
gen_target_no_temp = np.dot(softmax, kernel)
images_resized = np.zeros([0, 256, 256, 2], dtype=np.float32)
for image in range(gen_target_no_temp.shape[0]):
temp = scipy.ndimage.zoom(np.squeeze(
gen_target_no_temp[image]), (4, 4, 1),
mode='nearest')
images_resized = np.append(images_resized,
np.expand_dims(temp, axis=0),
axis=0)
inp_rescale = rescale_l_for_display(input_batch)
output_lab_no_temp = np.concatenate((inp_rescale, images_resized),
axis=3).astype(np.float64)
for i in range(input_batch.shape[0]):
output_lab_no_temp[i, :, :, :] = skimage.color.lab2rgb(
output_lab_no_temp[i, :, :, :])
predicted = output_lab_no_temp
just_rescale = [
'autoencoder', 'denoise', 'edge2d', 'edge3d', 'keypoint2d',
'keypoint3d', 'reshade', 'rgb2sfnorm', 'impainting_whole'
]
if task in just_rescale:
predicted = (predicted + 1.) / 2.
predicted = np.clip(predicted, 0., 1.)
predicted[:, 0, 0, :] = 0.
predicted[:, 1, 0, :] = 1.
just_clip = ['rgb2depth', 'rgb2mist']
if task in just_clip:
predicted = np.exp(predicted * np.log(2.0**16.0)) - 1.0
predicted = np.log(predicted) / 11.09
predicted = (predicted - 0.64) / 0.18
predicted = (predicted + 1.) / 2
predicted[:, 0, 0, :] = 0.
predicted[:, 1, 0, :] = 1.
if task == 'segmentsemantic_rb':
label = np.argmax(predicted, axis=-1)
COLORS = ('white', 'red', 'blue', 'yellow', 'magenta', 'green',
'indigo', 'darkorange', 'cyan', 'pink', 'yellowgreen',
'black', 'darkgreen', 'brown', 'gray', 'purple',
'darkviolet')
rgb = (input_batch + 1.) / 2.
preds = [
color.label2rgb(np.squeeze(x),
np.squeeze(y),
colors=COLORS,
kind='overlay')[np.newaxis, :, :, :]
for x, y in zip(label, rgb)
]
predicted = np.vstack(preds)
if task in ['class_1000', 'class_places']:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
sorted_pred = np.argsort(predict_output)[::-1]
top_5_pred = [synset[sorted_pred[i]] for i in range(5)]
to_print_pred = "Top 5 prediction: \n {}\n {}\n {}\n {} \n {}".format(
*top_5_pred)
img = Image.new('RGBA', (400, 200), (255, 255, 255))
d = ImageDraw.Draw(img)
fnt = ImageFont.truetype(
'/usr/share/fonts/truetype/dejavu/DejaVuSerifCondensed.ttf',
25)
d.text((20, 5), to_print_pred, fill=(255, 0, 0), font=fnt)
img.save(to_store_name, 'PNG')
elif task == 'vanishing_point_well_defined':
counter = 0
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
curr_vp.append(
plot_vanishing_point_smoothed(
predict_output, (input_batch[counter] + 1.) / 2.,
to_store_name, curr_vp))
if len(curr_vp) > 5:
curr_vp.pop(0)
counter += 1
#scipy.misc.toimage(result, cmin=0.0, cmax=1.0).save(to_store_name)
elif task == 'room_layout':
mean = np.array([
0.006072743318127848, 0.010272365569691076, -3.135909774145468,
1.5603802322235532, 5.6228218371102496e-05,
-1.5669352793761442, 5.622875878174759, 4.082800262277375,
2.7713941642895956
])
std = np.array([
0.8669452525283652, 0.687915294956501, 2.080513632043758,
0.19627420479282623, 0.014680602791251812, 0.4183827359302299,
3.991778013006544, 2.703495278378409, 1.2269185938626304
])
predicted = predicted * std + mean
counter = 0
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
plot_room_layout(predict_output,
(input_batch[counter] + 1.) / 2.,
to_store_name,
curr_layout,
cube_only=True)
curr_layout.append(predict_output)
if len(curr_layout) > 5:
curr_layout.pop(0)
#scipy.misc.toimage(result, cmin=0.0, cmax=1.0).save(to_store_name)
counter += 1
elif task == 'segmentsemantic_rb':
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
process_semseg_frame(predict_output, to_store_name)
elif task == 'jigsaw':
predicted = np.argmax(predicted, axis=1)
counter = 0
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
perm = cfg['target_dict'][predict_output]
show_jigsaw((input_batch[counter] + 1.) / 2., perm,
to_store_name)
counter += 1
else:
for file_idx, predict_output in zip(data_idx, predicted):
to_store_name = list_of_fname[file_idx].decode(
'utf-8').replace('video', task)
to_store_name = os.path.join('/home/ubuntu', to_store_name)
scipy.misc.toimage(np.squeeze(predict_output),
cmin=0.0,
cmax=1.0).save(to_store_name)
# subprocess.call('tar -czvf /home/ubuntu/{c}_{vid_id}.tar.gz /home/ubuntu/{t}/vid{vid_id}'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
# subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -vcodec libx264 -crf 15 {c}_{vid_id}.mp4'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
subprocess.call(
'ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -ss 00:01:54 -t 00:00:40 -c:v libvpx-vp9 -crf 10 -b:v 128k {c}_{vid_id}.webm'
.format(c=config_name, t=task, vid_id=args.vid),
shell=True)
# subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -vcodec libx264 -crf 15 -pix_fmt yuv420p {c}_{vid_id}.mp4'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
subprocess.call(
'sudo mkdir -p /home/ubuntu/s3/video_new/{t}'.format(t=task),
shell=True)
#subprocess.call('sudo mkdir -p /home/ubuntu/s3/video_new_all/{t}'.format(t=task), shell=True)
# subprocess.call('aws s3 cp /home/ubuntu/{c}_{vid_id}.tar.gz s3://task-preprocessing-512-oregon/video_new_all/{t}/'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
subprocess.call(
'aws s3 cp {c}_{vid_id}.webm s3://task-preprocessing-512-oregon/video_new/{t}/'
.format(c=config_name, t=task, vid_id=args.vid),
shell=True)
# subprocess.call('aws s3 cp /home/ubuntu/{c}_{vid_id}.tar.gz s3://taskonomy-unpacked-oregon/video_tar_all/{t}/'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
# subprocess.call('aws s3 cp {c}_{vid_id}.mp4 s3://taskonomy-unpacked-oregon/video_all/{t}/'.format(
# c=config_name, t=task, vid_id=args.vid), shell=True)
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
print("Done: {}".format(config_name))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
return
