def test_model_guest_creation(api_client, admin_user, model_name): model_data = setup_model(model_name, admin_user) response = api_client.post('/collab/{}/'.format(model_name), data=model_data, HTTP_ACCEPT="application/json") assert_response(response, status.HTTP_401_UNAUTHORIZED)
def load_model(f, nn_layers=None, from_npz=False): """ Loads neural network architecture specified in MPPI code, and loads weights and biases :param f: path of either .npz file or torch model :type f: str :param nn_layers: list consisting of number of nodes for each layer in network :param from_npz: if True, load .npz file which contains weights and biases of neural network :return: torch model """ # setup model architecture model = setup_model(layers=nn_layers, verbose=False) # load torch model if not from_npz: model.load_state_dict( torch.load(f) ) # add arg map_location=torch.device('cpu') to torch.load if using CPU else: # load npz file model = npz_to_torch_model(f, model) # load model onto GPU model.to(device) return model
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--model-setup', required=True, help='Model setup dictionary.') parser.add_argument('--lsh', action='store_true', default=False, help='If true, uses locally sensitive hashing \ (with k=10 NN) for NN search.') args = parser.parse_args() model, train, test, vocab, setup = setup_model(args) if setup['dataset'] == 'snli': converter = convert_snli_seq use_snli = True else: converter = convert_seq use_snli = False with open(os.path.join(setup['save_path'], 'calib.json')) as f: calibration_idx = json.load(f) calibration = [train[i] for i in calibration_idx] train = [x for i, x in enumerate(train) if i not in calibration_idx] sm = ScaledModel(model) optim = chainer.optimizers.Adam() optim.setup(sm.temperature) calib_iter = chainer.iterators.SerialIterator( calibration, setup['batchsize'], repeat=False) eceloss = ECELoss() num_epochs = 50 for i in range(num_epochs): calib_iter.reset() all_logits = [] all_labels = [] for i, batch in enumerate(calib_iter): batch = converter(batch, device=args.gpu, with_label=True) logits = sm.predict(batch['xs']) labels = F.concat(batch['ys'], axis=0) all_logits.append(logits.data) all_labels.append(labels.data) all_logits = F.concat(all_logits, axis=0) all_labels = F.concat(all_labels, axis=0) print(sm.temperature.temperature.data[0], eceloss(all_logits, all_labels)) logits = sm.temperature(all_logits) loss = F.softmax_cross_entropy(logits, all_labels) sm.temperature.zerograds() loss.backward() optim.update()
def test_model_creation(admin_api_client, admin_user, model_name): model_data = setup_model(model_name, admin_user) response = admin_api_client.post('/collab/{}/'.format(model_name), data=model_data, HTTP_ACCEPT='application/json') assert_response(response, status.HTTP_201_CREATED) projects_created = [response.data] response = admin_api_client.get('/collab/{}/'.format(model_name), HTTP_ACCEPT="application/json") assert_eq(response.data, projects_created)
def test_model_creation(admin_api_client, admin_user, model_name): model_data = setup_model(model_name, admin_user) response = admin_api_client.post('/collab/{}/'.format(model_name), data=model_data, HTTP_ACCEPT='application/json') # Manually handle matches, where API does not allow creation or # modification of objects, as they're read only if model_name == "matches": assert_response(response, status.HTTP_405_METHOD_NOT_ALLOWED) return assert_response(response, status.HTTP_201_CREATED) projects_created = [response.data] response = admin_api_client.get('/collab/{}/'.format(model_name), HTTP_ACCEPT="application/json") assert_response(response, status.HTTP_200_OK, projects_created)
# Command Line ardguments ap.add_argument('data_dir', nargs='*', action="store", default="./flowers/") ap.add_argument('--gpu', dest="gpu", action="store", default="gpu") ap.add_argument('--save_dir', dest="save_dir", action="store", default="./checkpoint.pth") ap.add_argument('--learning_rate', dest="learning_rate", action="store", default=0.003) ap.add_argument('--dropout', dest = "dropout", action = "store", default = 0.5) ap.add_argument('--epochs', dest="epochs", action="store", type=int, default=12) ap.add_argument('--arch', dest="arch", action="store", default="vgg16", type = str) pa = ap.parse_args() where = pa.data_dir path = pa.save_dir lr = pa.learning_rate structure = pa.arch dropout = pa.dropout power = pa.gpu epochs = pa.epochs trainloader, validationloader, testloader = utils.load_data(where) model, optimizer, criterion = utils.setup_model(structure,dropout,lr,power) utils.train_network(model, optimizer, criterion, epochs, 12, trainloader, power) utils.save_checkpoint(path,structure,dropout,lr)
def run_training(cfg, cfg_dir): # set up logging tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default() as g: # create ops and placeholders inputs = utils.setup_input_transfer(cfg, is_training=True) RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() # build model (and losses and train_op) model = utils.setup_model(inputs, cfg, is_training=True) # execute training start_time = time.time() utils.print_start_info(cfg, inputs['max_steps'], is_training=True) if cfg['model_type'] == 'empty': # Can't use tf slim because not trainable variables training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() } data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=True) training_runners['threads'] = data_prefetch_init_fn( training_runners['sess'], training_runners['coord']) try: # This just returns the imput as output. It is for testing data # input only. for step in xrange(inputs['max_steps']): input_batch, target_batch, data_idx = training_runners[ 'sess'].run([ model['input_batch'], model['target_batch'], model['data_idxs'] ]) if training_runners['coord'].should_stop(): break finally: utils.request_data_loading_end(training_runners) utils.end_data_loading_and_sess(training_runners) else: # Use tf.slim train_log_dir = os.path.join(cfg['log_dir'], 'slim-train') permanent_checkpoint_dir = os.path.join(cfg['log_dir'], 'checkpoints') session_config = tf.ConfigProto() session_config.gpu_options.allow_growth = True #max_to_keep = cfg['num_epochs'] * 2 max_to_keep = 10 if 'max_ckpts_to_keep' in cfg: max_to_keep = cfg['max_ckpts_to_keep'] # When ready to use a model, use the code below train( model['train_op'], train_log_dir, utils.get_data_prefetch_threads_init_fn_transfer( inputs, cfg, is_training=True), train_step_fn=model['train_step_fn'], train_step_kwargs=model['train_step_kwargs'], global_step=model['global_step'], number_of_steps=inputs['max_steps'], number_of_epochs=cfg['num_epochs'], init_fn=model['init_fn'], save_checkpoint_every=max(inputs['max_steps'] // (max_to_keep), 500), cfg_dir=cfg_dir, #RuntimeDeterminedEnviromentVars.steps_per_epoch, permanent_checkpoint_dir=permanent_checkpoint_dir, save_summaries_secs=cfg['summary_save_every_secs'], save_interval_secs=cfg['checkpoint_save_every_secs'], saver=model['saver_op'], return_accuracy='return_accuracy' in cfg and cfg['return_accuracy'], session_config=session_config) end_train_time = time.time() - start_time print('time to train %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time / (60 * 60))) print('avg time per epoch: %.3f hrs' % ((end_train_time / (60 * 60)) / cfg['num_epochs']))
def run_to_task(task_to): import general_utils from general_utils import RuntimeDeterminedEnviromentVars import models.architectures as architectures from data.load_ops import resize_rescale_image import utils from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction import lib.data.load_ops as load_ops tf.logging.set_verbosity(tf.logging.ERROR) all_outputs = {} pickle_dir = 'viz_output_single_task.pkl' import os if os.path.isfile(pickle_dir): with open(pickle_dir, 'rb') as fp: all_outputs = pickle.load(fp) for task in list_of_tasks: if task in all_outputs: print("{} already exists....\n\n\n".format(task)) continue print("Doing {task}".format(task=task)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } # task = '{f}__{t}__{hs}'.format(f=task_from, t=task_to, hs=args.hs) CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{TASK}'.format( TASK=task) ############## Load Configs ############## cfg = utils.load_config(CONFIG_DIR, nopause=True) RuntimeDeterminedEnviromentVars.register_dict(cfg) split_file = cfg['test_filenames'] if ON_TEST_SET else cfg[ 'val_filenames'] cfg['train_filenames'] = split_file cfg['val_filenames'] = split_file cfg['test_filenames'] = split_file cfg['num_epochs'] = 1 cfg['randomize'] = False root_dir = cfg['root_dir'] cfg['num_read_threads'] = 1 print(cfg['log_root']) if task == 'jigsaw': continue cfg['model_path'] = os.path.join(cfg['log_root'], task, 'model.permanent-ckpt') print(cfg['model_path']) if cfg['model_path'] is None: continue ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) inputs = utils.setup_input( cfg, is_training=ON_TEST_SET, use_filename_queue=False ) # is_training determines whether to use train/validaiton RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() # utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False ) ############## Set Up Model ############## model = utils.setup_model(inputs, cfg, is_training=IN_TRAIN_MODE) m = model['model'] model['saver_op'].restore(training_runners['sess'], cfg['model_path']) ############## Start dataloading workers ############## data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False) prefetch_threads = threading.Thread(target=data_prefetch_init_fn, args=(training_runners['sess'], training_runners['coord'])) prefetch_threads.start() ############## Run First Batch ############## if not hasattr(m, 'masks'): ( input_batch, target_batch, data_idx, predicted, loss, ) = training_runners['sess'].run([ m.input_images, m.targets, model['data_idxs'], m.decoder_output, m.total_loss ]) mask_batch = 1. else: ( input_batch, target_batch, mask_batch, data_idx, predicted, loss, ) = training_runners['sess'].run([ m.input_images, m.targets, m.masks, model['data_idxs'], m.decoder_output, m.total_loss ]) if task == 'segment2d' or task == 'segment25d': from sklearn.decomposition import PCA x = np.zeros((32, 256, 256, 3), dtype='float') for i in range(predicted.shape[0]): embedding_flattened = np.squeeze(predicted[i]).reshape( (-1, 64)) pca = PCA(n_components=3) pca.fit(embedding_flattened) lower_dim = pca.transform(embedding_flattened).reshape( (256, 256, -1)) lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min()) x[i] = lower_dim predicted = x ############## Clean Up ############## training_runners['coord'].request_stop() training_runners['coord'].join() # if os.path.isfile(pickle_dir): # with open(pickle_dir, 'rb') as fp: # all_outputs = pickle.load(fp) ############## Store to dict ############## to_store = { 'input': input_batch, 'target': target_batch, 'mask': mask_batch, 'data_idx': data_idx, 'output': predicted } all_outputs[task] = to_store print("Done: {}".format(task)) # os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir)) ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() try: del sys.modules['config'] except: pass sys.path = remove_dups(sys.path) print("FINISHED: {}\n\n\n\n\n\n".format(task)) pickle_dir = 'viz_output_single_task.pkl' with open(pickle_dir, 'wb') as fp: pickle.dump(all_outputs, fp) try: subprocess.call( "aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/" .format(pickle_dir), shell=True) except: subprocess.call( "sudo cp {} /home/ubuntu/s3/visualizations/".format( pickle_dir), shell=True) return
def run_val_test(cfg): # set up logging tf.logging.set_verbosity(tf.logging.INFO) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } # create ops and placeholders inputs = utils.setup_input(cfg, is_training=False) RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() # build model (and losses and train_op) model = utils.setup_model(inputs, cfg, is_training=False) # full_path = tf.train.latest_checkpoint(checkpoint_dir) # step = full_path.split('-')[-1] # model_path = os.path.join('/home/ubuntu/s3/model_log', cfg['task_name'], 'model.permanent-ckpt') model_path = os.path.join('/home/ubuntu/s3/model_log_final', cfg['task_name'], 'model.permanent-ckpt') model['saver_op'].restore(training_runners['sess'], model_path) m = model['model'] # execute training start_time = time.time() utils.print_start_info(cfg, inputs['max_steps'], is_training=False) data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False) prefetch_threads = threading.Thread(target=data_prefetch_init_fn, args=(training_runners['sess'], training_runners['coord'])) prefetch_threads.start() print("Dataloading workers dispatched....") return_accuracy = 'return_accuracy' in cfg and cfg['return_accuracy'], losses_mean = AverageMeter() accuracy_mean = AverageMeter() for step in range(inputs['max_steps']): #print(step) if return_accuracy: (data_idx, loss, accuracy) = training_runners['sess'].run( [model['data_idxs'], m.losses[0], m.accuracy]) losses_mean.update(loss) accuracy_mean.update(accuracy) if step % 100 == 0: print( 'Step: {step} with Current Losses mean: {loss}; with accuracy: {accur}' .format(step=step, loss=losses_mean.avg, accur=accuracy_mean.avg)) else: (data_idx, loss) = training_runners['sess'].run( [model['data_idxs'], m.losses[0]]) losses_mean.update(loss) if step % 100 == 0: print('Step: {step} with Current Losses mean: {loss}'.format( step=step, loss=losses_mean.avg)) if return_accuracy: print('Final Losses mean: {loss}; with accuracy: {accur}'.format( loss=losses_mean.avg, accur=accuracy_mean.avg)) else: print('Final Losses mean: {loss}'.format(loss=losses_mean.avg)) end_train_time = time.time() - start_time print('time to train %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time / (60 * 60))) print('avg time per epoch: %.3f hrs' % ((end_train_time / (60 * 60)) / cfg['num_epochs']))
def run_to_task(task_to): import general_utils from general_utils import RuntimeDeterminedEnviromentVars import models.architectures as architectures from data.load_ops import resize_rescale_image import utils from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction import lib.data.load_ops as load_ops import pdb global synset synset_1000 = [" ".join(i.split(" ")[1:]) for i in synset] select = np.asarray([ 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 0., 0., 1., 0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0.]) with open('/home/ubuntu/task-taxonomy-331b/lib/data/places_class_names.txt', 'r') as fp: synset_places = [x.rstrip()[4:-1] for x,y in zip(fp.readlines(), select) if y == 1.] tf.logging.set_verbosity(tf.logging.ERROR) args = parser.parse_args() if args.task is not 'NONE': args.idx = list_of_tasks.index(args.task) for idx, task in enumerate(list_of_tasks): if idx != args.idx and args.idx != -1: continue if task == 'class_places': synset = synset_places elif task == 'class_1000': synset = synset_1000 print("Doing {task}".format(task=task)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } # task = '{f}__{t}__{hs}'.format(f=task_from, t=task_to, hs=args.hs) CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/final/{TASK}'.format(TASK=task) ############## Load Configs ############## cfg = utils.load_config( CONFIG_DIR, nopause=True ) RuntimeDeterminedEnviromentVars.register_dict( cfg ) split_file = os.path.join('/home/ubuntu/task-taxonomy-331b/assets/aws_data/', 'video2_info.pkl') cfg['train_filenames'] = split_file cfg['val_filenames'] = split_file cfg['test_filenames'] = split_file cfg['num_epochs'] = 2 cfg['randomize'] = False root_dir = cfg['root_dir'] cfg['num_read_threads'] = 1 print(cfg['log_root']) cfg['model_path'] = os.path.join( cfg['log_root'], task, 'model.permanent-ckpt' ) print( cfg['model_path']) if cfg['model_path'] is None: continue cfg['dataset_dir'] = '/home/ubuntu' cfg['preprocess_fn'] = load_and_specify_preprocessors_for_representation_extraction ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) inputs = utils.setup_input( cfg, is_training=ON_TEST_SET, use_filename_queue=False ) # is_training determines whether to use train/validaiton RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() # utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False ) ############## Set Up Model ############## model = utils.setup_model( inputs, cfg, is_training=IN_TRAIN_MODE ) m = model[ 'model' ] model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] ) ############## Start dataloading workers ############## data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False ) prefetch_threads = threading.Thread( target=data_prefetch_init_fn, args=( training_runners[ 'sess' ], training_runners[ 'coord' ] )) prefetch_threads.start() list_of_fname = np.load('/home/ubuntu/task-taxonomy-331b/assets/aws_data/video2_fname.npy') import errno try: os.mkdir('/home/ubuntu/{}'.format(task)) os.mkdir('/home/ubuntu/{}/vid1'.format(task)) os.mkdir('/home/ubuntu/{}/vid2'.format(task)) os.mkdir('/home/ubuntu/{}/vid3'.format(task)) os.mkdir('/home/ubuntu/{}/vid4'.format(task)) except OSError as e: if e.errno != errno.EEXIST: raise curr_comp = np.zeros((3,64)) curr_fit_img = np.zeros((256,256,3)) embeddings = [] ############## Run First Batch ############## for step_num in range(inputs['max_steps'] - 1): #for step_num in range(1): #if step_num > 0 and step_num % 20 == 0: print(step_num) if not hasattr(m, 'masks'): ( input_batch, target_batch, data_idx, predicted, loss, ) = training_runners['sess'].run( [ m.input_images, m.targets, model[ 'data_idxs' ], m.decoder_output, m.total_loss] ) mask_batch = 1. else: ( input_batch, target_batch, mask_batch, data_idx, predicted, loss, ) = training_runners['sess'].run( [ m.input_images, m.targets, m.masks, model[ 'data_idxs' ], m.decoder_output, m.total_loss] ) if task == 'segment2d' or task == 'segment25d': from sklearn.decomposition import PCA x = np.zeros((32,256,256,3), dtype='float') k_embed = 8 # for i in range(predicted.shape[0]): # embedding_flattened = np.squeeze(predicted[i]).reshape((-1,64)) # pca = PCA(n_components=3) # pca.fit(embedding_flattened) # min_order = None # min_dist = float('inf') # for order in itertools.permutations([0,1,2]): # reordered = pca.components_[list(order), :] # dist = np.linalg.norm(curr_comp-reordered) # if dist < min_dist: # min_order = list(order) # min_dist = dist # print(min_order) # pca.components_ = pca.components_[min_order, :] # curr_comp = pca.components_ # lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1)) # lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min()) # x[i] = lower_dim for i in range(predicted.shape[0]): embedding_flattened = np.squeeze(predicted[i]).reshape((-1,64)) embeddings.append(embedding_flattened) if len(embeddings) > k_embed: embeddings.pop(0) pca = PCA(n_components=3) pca.fit(np.vstack(embeddings)) min_order = None min_dist = float('inf') copy_of_comp = np.copy(pca.components_) for order in itertools.permutations([0,1,2]): #reordered = pca.components_[list(order), :] #dist = np.linalg.norm(curr_comp-reordered) pca.components_ = copy_of_comp[order, :] lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1)) lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min()) dist = np.linalg.norm(lower_dim - curr_fit_img) if dist < min_dist: min_order = order min_dist = dist pca.components_ = copy_of_comp[min_order, :] lower_dim = pca.transform(embedding_flattened).reshape((256,256,-1)) lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min()) curr_fit_img = np.copy(lower_dim) x[i] = lower_dim predicted = x if task == 'curvature': std = [31.922, 21.658] mean = [123.572, 120.1] predicted = (predicted * std) + mean predicted[:,0,0,:] = 0. predicted[:,1,0,:] = 1. predicted = np.squeeze(np.clip(predicted.astype(int) / 255., 0., 1. )[:,:,:,0]) just_rescale = ['autoencoder', 'denoise', 'edge2d', 'edge3d', 'keypoint2d', 'keypoint3d', 'reshade', 'rgb2sfnorm'] if task in just_rescale: predicted = (predicted + 1.) / 2. predicted = np.clip(predicted, 0., 1.) predicted[:,0,0,:] = 0. predicted[:,1,0,:] = 1. just_clip = ['rgb2depth', 'rgb2mist'] if task in just_clip: predicted[:,0,0,:] = 0. predicted[:,1,0,:] = 1. if task == 'segmentsemantic_rb': label = np.argmax(predicted, axis=-1) COLORS = ('white','red', 'blue', 'yellow', 'magenta', 'green', 'indigo', 'darkorange', 'cyan', 'pink', 'yellowgreen', 'black', 'darkgreen', 'brown', 'gray', 'purple', 'darkviolet') rgb = (input_batch + 1.) / 2. preds = [color.label2rgb(np.squeeze(x), np.squeeze(y), colors=COLORS, kind='overlay')[np.newaxis,:,:,:] for x,y in zip(label, rgb)] predicted = np.vstack(preds) if task in ['class_1000', 'class_places']: for file_idx, predict_output in zip(data_idx, predicted): to_store_name = list_of_fname[file_idx].decode('utf-8').replace('video', task) to_store_name = os.path.join('/home/ubuntu', to_store_name) sorted_pred = np.argsort(predict_output)[::-1] top_5_pred = [synset[sorted_pred[i]] for i in range(5)] to_print_pred = "Top 5 prediction: \n {}\n {}\n {}\n {} \n {}".format(*top_5_pred) img = Image.new('RGBA', (400, 200), (255, 255, 255)) d = ImageDraw.Draw(img) fnt = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSerifCondensed.ttf', 25) d.text((20, 5), to_print_pred, fill=(255, 0, 0), font=fnt) img.save(to_store_name, 'PNG') else: for file_idx, predict_output in zip(data_idx, predicted): to_store_name = list_of_fname[file_idx].decode('utf-8').replace('video', task) to_store_name = os.path.join('/home/ubuntu', to_store_name) scipy.misc.toimage(np.squeeze(predict_output), cmin=0.0, cmax=1.0).save(to_store_name) subprocess.call('tar -czvf /home/ubuntu/{t}.tar.gz /home/ubuntu/{t}'.format(t=task), shell=True) subprocess.call('aws s3 cp /home/ubuntu/{t}.tar.gz s3://task-preprocessing-512-oregon/video2/'.format(t=task), shell=True) subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid2/020%04d.png -vcodec libx264 -crf 15 -pix_fmt yuv420p {t}_2.mp4'.format(t=task), shell=True) subprocess.call('aws s3 cp {t}_2.mp4 s3://task-preprocessing-512-oregon/video2/'.format(t=task), shell=True) ############## Clean Up ############## training_runners[ 'coord' ].request_stop() training_runners[ 'coord' ].join() # if os.path.isfile(pickle_dir): # with open(pickle_dir, 'rb') as fp: # all_outputs = pickle.load(fp) ############## Store to dict ############## print("Done: {}".format(task)) # os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir)) ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() return
def run_extract_representations(args, cfg, save_dir, given_task): transfer = (cfg['model_type'] == architectures.TransferNet) if transfer: get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer setup_input_fn = utils.setup_input_transfer else: setup_input_fn = utils.setup_input get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn # set up logging tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default() as g: # create ops and placeholders tf.logging.set_verbosity(tf.logging.INFO) inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() # build model (and losses and train_op) model = utils.setup_model(inputs, cfg, is_training=False) m = model['model'] # set up metrics to evaluate names_to_values, names_to_updates = setup_metrics(inputs, model, cfg) # execute training start_time = time.time() utils.print_start_info(cfg, inputs['max_steps'], is_training=False) # start session and restore model training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() } try: if cfg['model_path'] is None: print('Please specify a checkpoint directory') return model['saver_op'].restore(training_runners['sess'], cfg['model_path']) utils.print_start_info(cfg, inputs['max_steps'], is_training=False) data_prefetch_init_fn = get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False) prefetch_threads = threading.Thread( target=data_prefetch_init_fn, args=(training_runners['sess'], training_runners['coord'])) prefetch_threads.start() # run one example so that we can calculate some statistics about the representations filenames = [] representations, data_idx = training_runners['sess'].run( [model['model'].encoder_output, inputs['data_idxs']]) # print("Step number: {}".format(gs)) filenames.extend(data_idx) if type(representations) == list: representations = representations[0] print(representations.shape) representations = representations.reshape(REPRESENTATIONS_SHAPE) print('Got first batch representation with size: {0}'.format( representations.shape)) # run the remaining examples for step in range(inputs['max_steps'] - 1): #for step in range( 10 ): if step % 100 == 0: print('Step {0} of {1}'.format(step, inputs['max_steps'] - 1)) # This is just for GAN, for the LEO meeting encoder_output, data_idx = training_runners['sess'].run( [model['model'].encoder_output, inputs['data_idxs']]) if type(encoder_output) == list: encoder_output = encoder_output[0] encoder_output = encoder_output.reshape(REPRESENTATIONS_SHAPE) representations = np.append(representations, encoder_output, axis=0) filenames.extend(data_idx) if training_runners['coord'].should_stop(): break print( 'The size of representations is %s while we expect it to run for %d steps with batchsize %d' % (representations.shape, inputs['max_steps'], cfg['batch_size'])) end_train_time = time.time() - start_time if args.imagenet: save_path = os.path.join( save_dir, '{task}_{split}_imagenet_representations.pkl'.format( task=given_task, split=args.data_split)) elif args.places: save_path = os.path.join( save_dir, '{task}_{split}_places_representations.pkl'.format( task=given_task, split=args.data_split)) elif args.vid: save_path = os.path.join( save_dir, '{task}_vid{vid_id}_representations.pkl'.format( task=given_task, vid_id=args.vid_id)) else: save_path = os.path.join( save_dir, '{task}_{split}_representations.pkl'.format( task=given_task, split=args.data_split)) with open(save_path, 'wb') as f: pickle.dump( { 'file_indexes': filenames, 'representations': representations }, f) copy_to = None if args.out_dir: os.makedirs(args.out_dir, exist_ok=True) os.system("sudo cp {fp} {out}/".format(fp=save_path, out=args.out_dir)) else: if transfer: copy_to = cfg['log_root'] else: copy_to = os.path.join(cfg['log_root'], given_task) os.system("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to)) print("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to)) print('saved representations to {0}'.format(save_path)) print('moved representations to {0}'.format(copy_to)) print('time to extract %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time / (60 * 60))) finally: utils.request_data_loading_end(training_runners) utils.end_data_loading_and_sess(training_runners)
def train(device, model_dir, train_loader, val_loader, nn_layers, epochs, lr, weight_decay=0.0, criterion=torch.nn.L1Loss(), loss_weights=None): """ Model training and validation phase :param device: torch device object :type model_dir: str :param train_loader: data loader with test data :param val_loader: data loader with validation data :type nn_layers: list[int] :type epochs: int :type lr: float :type weight_decay: float :param criterion: loss function :type loss_weights: list[float] """ # get start time start = time.time() # set up model model = setup_model(layers=nn_layers) # use default activation # model = npz_to_torch_model(filename="../../params/models/autorally_nnet_09_12_2018.npz", model=model) # to load pretrained model # load model onto device model.to(device) # set up optimizer optimizer = optim.Adam( model.parameters(), lr=lr, weight_decay=weight_decay) # weight_decay is L2 penalty term to loss # set up data loaders data_loaders = {"train": train_loader, "val": val_loader} dataset_sizes = { "train": train_loader.dataset.__len__(), "val": val_loader.dataset.__len__() } # dict to store train and val losses for plotting losses = {"train": [], "val": []} # label cols label_cols = train_loader.dataset.label_cols # dict to store losses of the different loss components split_losses = {"train": {}, "val": {}} for label_col in label_cols: split_losses['train'][label_col] = [] split_losses['val'][label_col] = [] best_val_loss = np.inf if loss_weights is None: loss_weights = np.ones( nn_layers[-1] ) # if no weights are specified for the loss just set the weights to 1 for epoch in range(epochs): print('\nEpoch %i/%i' % (epoch, epochs - 1)) print('-' * 10) for phase in ["train", "val"]: if phase == 'train': model.train() else: model.eval() running_loss = 0.0 temp_split_losses = {} for label_col in label_cols: temp_split_losses[label_col] = 0.0 for inputs, labels in data_loaders[phase]: inputs = inputs.to(device) labels = labels.to(device) # clear gradients optimizer.zero_grad() # forward with torch.set_grad_enabled(phase == "train"): # get output from model outputs = model(inputs.double()) # apply the specified loss weights and compute the loss loss = criterion( torch.t( torch.mul( outputs, torch.tensor(loss_weights, dtype=torch.float64).to(device))), torch.t( torch.mul( labels.double(), torch.tensor(loss_weights, dtype=torch.float64).to(device)))) # save loss splits for label_col, split_loss in zip(label_cols, loss): temp_split_losses[label_col] += torch.sum( split_loss).item() # apply reduction to loss # loss = torch.sum(loss) loss = torch.mean(loss) if phase == "train": # calculate the gradients loss.backward() # update all the parameters based on the gradients calculated optimizer.step() # updating stats # running_loss += loss.item() running_loss += loss.item() * inputs.size( 0 ) # multiply by batch size since calculated loss was the mean # calculate loss for epoch epoch_loss = running_loss / dataset_sizes[phase] losses[phase].append(epoch_loss) print('%s Loss: %.4f' % (phase, epoch_loss)) # calculate split losses for label_col in label_cols: split_losses[phase][label_col].append( temp_split_losses[label_col] / dataset_sizes[phase]) # update new best val loss and model if phase == "val" and epoch_loss < best_val_loss: best_val_loss = epoch_loss torch.save( { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': best_val_loss }, os.path.join(model_dir, "model.pt")) gc.collect() time_elapsed = time.time() - start print("\nTraining complete in %.0fm %.0fs" % (time_elapsed // 60, time_elapsed % 60)) print("Best val loss %0.5f" % best_val_loss) # convert model to npz format for mppi usage torch_model_to_npz(model, model_dir) # plot loss monitoring fig = plt.figure() x = np.arange(epochs) plt.plot(x, losses['train'], 'b-', x, losses['val'], 'r-') plt.ylim(bottom=0.0) plt.title("Loss") plt.legend(['train', 'val'], loc='upper right') plt.xlabel("epoch") plt.ylabel("loss") fig.savefig(os.path.join(model_dir, "loss.pdf"), format="pdf") plt.close(fig) # plot loss splits fig = plt.figure() for label_col in label_cols: plt.plot(x, split_losses['val'][label_col], label=label_col) plt.title("Val loss splits") plt.xlabel("epoch") plt.ylabel("loss") plt.legend(label_cols, loc='best') plt.ylim(bottom=0.0) fig.savefig(os.path.join(model_dir, "loss_splits.pdf"), format="pdf") plt.close(fig)
def run_to_task(): import general_utils from general_utils import RuntimeDeterminedEnviromentVars tf.logging.set_verbosity(tf.logging.ERROR) args = parser.parse_args() imgs = args.im_name.split(',') if args.task == 'ego_motion' and len(imgs) != 3: raise ValueError('Wrong number of images, expecting 3 but got {}'.format(len(imgs))) if args.task != 'ego_motion' and len(imgs) != 2: raise ValueError('Wrong number of images, expecting 2 but got {}'.format(len(imgs))) task = args.task if task not in list_of_tasks: raise ValueError('Task not supported') cfg = generate_cfg(task) input_img = np.empty((len(imgs),256,256,3), dtype=np.float32) for i,imname in enumerate(imgs): img = load_raw_image_center_crop( imname ) img = skimage.img_as_float(img) scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(imname) img = cfg[ 'input_preprocessing_fn' ]( img, **cfg['input_preprocessing_fn_kwargs'] ) input_img[i,:,:,:] = img input_img = input_img[np.newaxis, :] print("Doing {task}".format(task=task)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) setup_input_fn = utils.setup_input inputs = setup_input_fn( cfg, is_training=False, use_filename_queue=False ) RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() ############## Set Up Model ############## model = utils.setup_model( inputs, cfg, is_training=False ) m = model[ 'model' ] model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] ) predicted, representation = training_runners['sess'].run( [ m.decoder_output, m.encoder_output ], feed_dict={m.input_images: input_img} ) if args.store_rep: s_name, file_extension = os.path.splitext(args.store_name) with open('{}.npy'.format(s_name), 'wb') as fp: np.save(fp, np.squeeze(representation)) if args.store_pred: s_name, file_extension = os.path.splitext(args.store_name) with open('{}_pred.npy'.format(s_name), 'wb') as fp: np.save(fp, np.squeeze(predicted)) if task == 'ego_motion': ego_motion(predicted, args.store_name) return if task == 'fix_pose': cam_pose(predicted, args.store_name, is_fixated=True) return if task == 'non_fixated_pose': cam_pose(predicted, args.store_name, is_fixated=False) return if task == 'point_match': prediction = np.argmax(predicted, axis=1) print('the prediction (1 stands for match, 0 for unmatch)is: ', prediction) return ############## Clean Up ############## training_runners[ 'coord' ].request_stop() training_runners[ 'coord' ].join() print("Done: {}".format(config_name)) ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() return
def run_to_task(): import general_utils from general_utils import RuntimeDeterminedEnviromentVars tf.logging.set_verbosity(tf.logging.ERROR) args = parser.parse_args() task = args.task if task not in list_of_tasks: raise ValueError('Task not supported') cfg = utils.generate_cfg(task) # Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \ keypoint2d keypoint3d \ reshade rgb2depth rgb2mist rgb2sfnorm \ segment25d segment2d room_layout'.split() if task in low_sat_tasks: cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat print("Doing {task}".format(task=task)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) setup_input_fn = utils.setup_input inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() ############## Set Up Model ############## model = utils.setup_model(inputs, cfg, is_training=False) m = model['model'] model['saver_op'].restore(training_runners['sess'], cfg['model_path']) ############## Single Image ############## if args.imgs_list: with open(args.imgs_list) as imgs_list: all_prediction = [] all_representation = [] for line in imgs_list: filename = args.dir_name + line.strip().split(',')[0] # FIXME img = prepare_image(task, filename, cfg) predicted, representation = training_runners['sess'].run( [m.decoder_output, m.encoder_output], feed_dict={m.input_images: img}) utils.tasks( task, args, predicted, os.path.join(args.store_name + line.split(os.path.sep)[-1].strip() + '.jpg'), img=img) all_prediction.append(np.squeeze(predicted)) all_representation.append(np.squeeze(representation)) if args.store_rep: s_name, file_extension = os.path.splitext(args.store_name) with open('{}.npy'.format(s_name), 'wb') as fp: np.save(fp, np.array(all_representation)) if args.store_pred: s_name, file_extension = os.path.splitext(args.store_name) with open('{}_pred.npy'.format(s_name), 'wb') as fp: np.save(fp, np.array(all_prediction)) else: img = prepare_image(task, args.im_name, cfg) predicted, representation = training_runners['sess'].run( [m.decoder_output, m.encoder_output], feed_dict={m.input_images: img}) utils.tasks(task, args, predicted, representation, img) if args.store_rep: s_name, file_extension = os.path.splitext(args.store_name) with open('{}.npy'.format(s_name), 'wb') as fp: np.save(fp, np.squeeze(representation)) if args.store_pred: s_name, file_extension = os.path.splitext(args.store_name) with open('{}_pred.npy'.format(s_name), 'wb') as fp: np.save(fp, np.squeeze(predicted)) ############## Clean Up ############## training_runners['coord'].request_stop() training_runners['coord'].join() # print("Done: {}".format(config_name)) ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() return
def run_rand_baseline( args, cfg, given_task ): # set up logging tf.logging.set_verbosity( tf.logging.INFO ) with tf.Graph().as_default() as g: # create ops and placeholders tf.logging.set_verbosity( tf.logging.INFO ) inputs = utils.setup_input( cfg, is_training=False, use_filename_queue=False ) RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) RuntimeDeterminedEnviromentVars.populate_registered_variables() # build model (and losses and train_op) model = utils.setup_model( inputs, cfg, is_training=False ) # set up metrics to evaluate names_to_values, names_to_updates = setup_metrics( inputs, model, cfg ) # execute training start_time = time.time() utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False ) # start session and restore model training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() } try: utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False ) data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False ) #training_runners[ 'threads' ] = data_prefetch_init_fn( training_runners[ 'sess' ], training_runners[ 'coord' ] ) prefetch_threads = threading.Thread( target=data_prefetch_init_fn, args=( training_runners[ 'sess' ], training_runners[ 'coord' ] )) prefetch_threads.start() # run one example so that we can calculate some statistics about the representations targets = training_runners['sess'].run( inputs[ 'target_batch' ] ) # run the remaining examples for step in range( inputs[ 'max_steps' ] - 1 ): #for step in range( 10 ): if step % 100 == 0: print( 'Step {0} of {1}'.format( step, inputs[ 'max_steps' ] - 1 )) target = training_runners['sess'].run( inputs[ 'target_batch' ] ) targets = np.append( targets, target, axis=0) if training_runners['coord'].should_stop(): break rand_idx = [random.randint(0, targets.shape[0] - 1) for i in range(targets.shape[0])] rand_target = [targets[i] for i in rand_idx] rand_target = np.vstack(rand_target) counter = 0 sum = 0 for step in range( inputs[ 'max_steps' ] - 1 ): #for step in range( 10 ): if step % 100 == 0: print( 'Step {0} of {1}'.format( step, inputs[ 'max_steps' ] - 1 )) tar = targets[step*cfg['batch_size']:(step+1)*cfg['batch_size']] rand = rand_target[step*cfg['batch_size']:(step+1)*cfg['batch_size']] losses = training_runners['sess'].run( model['model'].losses, feed_dict={ inputs['target_batch']: tar, model['model'].final_output:rand}) sum += losses[0] counter += 1 if training_runners['coord'].should_stop(): break print(sum) print(counter) print('random_baseline has loss: {loss}'.format(loss=sum/counter)) end_train_time = time.time() - start_time finally: utils.request_data_loading_end( training_runners ) utils.end_data_loading_and_sess( training_runners )
parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--model-setup', required=True, help='Model setup dictionary.') parser.add_argument('--lsh', action='store_true', default=False, help='If true, uses locally sensitive hashing \ (with k=10 NN) for NN search.') args = parser.parse_args(['--model-setup', 'result/snli_bilstm/args.json']) model, train, test, vocab, setup = setup_model(args) if setup['dataset'] == 'snli': converter = convert_snli_seq else: converter = convert_seq # FIXME args.batchsize = 64 max_beam_size = 5 with open(os.path.join(setup['save_path'], 'calib.json')) as f: calibration_idx = json.load(f) calibration = [train[i] for i in calibration_idx] train = [x for i, x in enumerate(train) if i not in calibration_idx] train = random.sample(train, 10000)
def generate_predictions(device, model_dir, data_path, nn_layers, state_cols, state_der_cols, ctrl_cols, time_col='time', time_horizon=2.5, data_frac=1.0, feature_scaler=None, label_scaler=None, skip_first_batch=True): """ Model test phase. Generates truth and nn predicted trajectory for each batch NOTE: many parts of this test phase are currently hard coded to a specific problem :param device: torch device object :type model_dir: str :type data_path: str :type nn_layers: list[int] :type state_cols: list[str] :type state_der_cols: list[str] :type ctrl_cols: list[str] :type time_col: str :param time_horizon: total time to propagate dynamics for :param data_frac: fraction of test data to use :param feature_scaler: sklearn standard scaler for features :param label_scaler: sklearn standard scaler for labels :param skip_first_batch: option to skip first batch """ print("\nGenerating predictions from trained model...") # size of state space state_dim = len(state_cols) # folder to store all test phase files test_phase_dir = os.path.join(model_dir, "test_phase/") # get time step from data time_step = pd.read_csv(data_path).head(2)[time_col].values[1] print("time step: %.04f" % time_step) # determine batch size batch_size = int(np.ceil(time_horizon / time_step)) print("batch size: %.0f" % batch_size) # setup data loader indices = np.arange(int(data_frac * len(pd.read_csv(data_path)))) data_loader = make_test_data_loader(data_path, batch_size, state_cols, state_der_cols, ctrl_cols, indices=indices, time_col=time_col) # number of batches to do total_batches = data_loader.dataset.__len__() // batch_size # load model architecture model = setup_model(layers=nn_layers) # load model onto device model.to(device) # load weights + biases from pretrained model state_dict = torch.load(os.path.join(model_dir, "model.pt")) # check if model was saved as part of a dict if "model_state_dict" in state_dict.keys(): state_dict = state_dict["model_state_dict"] model.load_state_dict(state_dict) # var to keep track of errors from propagating dynamics errors_list = [] # var to keep track of instantaneous errors inst_errors = [] with torch.no_grad(): # set model to eval mode model.eval() # keep track of current batch number batch_num = 0 # generate a trajectory for each batch for truth_states, truth_state_ders, ctrls, time_data in data_loader: print('\nBatch %i/%i' % (batch_num, total_batches)) print('-' * 10) num_steps = truth_states.size(0) # skip last batch if it is less than batch size if num_steps < batch_size: print("Skipping final batch...") continue # skip first batch if specified if skip_first_batch and batch_num == 0: print("Skipping first batch...") batch_num += 1 continue # make a new folder to store results from this batch batch_folder = test_phase_dir + "batch_" + str(batch_num) + "/" if not os.path.exists(batch_folder): os.makedirs(batch_folder) # update batch number batch_num += 1 # init state variables nn_states = np.full((num_steps, state_dim), 0, np.float) # set initial conditions nn_states[0] = truth_states[0].cpu().numpy() # array to store all state derivatives state_ders = np.full((num_steps, truth_state_ders.size(1)), 0, np.float) # TODO: remove hard coded stuff # iterate through each step of trajectory for idx in range(num_steps - 1): # prep inputs to feed to neural network # x1 is the input from continuously feeding the predicted state back into the model x1 = torch.tensor([ nn_states[idx][3], nn_states[idx][4], nn_states[idx][5], nn_states[idx][6], ctrls[idx][0], ctrls[idx][1] ]) # x2 is just the input from the truth state used to calculate instantaneous errors of the model x2 = torch.tensor([ truth_states[idx][3], truth_states[idx][4], truth_states[idx][5], truth_states[idx][6], ctrls[idx][0], ctrls[idx][1] ]) # get the current state curr_state = nn_states[idx] # if data was standardized, apply transform on test data if feature_scaler is not None: x1 = torch.tensor( feature_scaler.transform(x1.reshape(1, -1))[0]) x2 = torch.tensor( feature_scaler.transform(x2.reshape(1, -1))[0]) # get outputs of neural network output1 = model(x1.double().to(device)).cpu().numpy() output2 = model(x2.double().to(device)).cpu().numpy() # apply inverse transform on output if label_scaler is not None: output1 = label_scaler.inverse_transform(output1) output2 = label_scaler.inverse_transform(output2) # output1 = truth_state_ders[idx + 1].cpu().numpy() # use the truth derivatives # compute the state derivatives state_der = compute_state_ders( curr_state, output1, negate_yaw_der=False ) # NOTE: set negate_yaw_der to True if using autorally's model # update states nn_states[idx + 1] = curr_state + state_der * time_step # save state derivatives state_ders[idx + 1] = state_der[3:] # calculate the instantaneous signed error inst_errors.append(truth_state_ders[idx].cpu().numpy() - output2) curr_errors = np.abs(nn_states - truth_states.numpy()) # compute yaw errors curr_errors[:, 2] = [e % (2 * np.pi) for e in curr_errors[:, 2]] curr_errors[:, 2] = [(2 * np.pi) - e if e > np.pi else e for e in curr_errors[:, 2]] # save errors errors_list.append(curr_errors) # print some errors on final time step print("abs x error (m): %.02f" % curr_errors[-1][0]) print("abs y error (m): %.02f" % curr_errors[-1][1]) print("abs yaw error (rad): %.02f" % (curr_errors[-1][2])) # convert time data to numpy time_data = time_data.cpu().numpy() # convert control data to numpy ctrls = ctrls.cpu().numpy() # create pandas DataFrames df_nn = pd.DataFrame(data=np.concatenate((np.reshape( time_data, (len(time_data), 1)), nn_states, state_ders, ctrls), axis=1), columns=np.concatenate( ([time_col], state_cols, state_der_cols, ctrl_cols))) df_nn.to_csv(os.path.join(batch_folder, "nn_state_variables.csv"), index=False, header=True) # load ground truth data df_truth = pd.DataFrame(data=np.concatenate((np.reshape( time_data, (len(time_data), 1)), truth_states, truth_state_ders, ctrls), axis=1), columns=np.concatenate( ([time_col], state_cols, state_der_cols, ctrl_cols))) df_truth.to_csv(os.path.join(batch_folder, "truth_state_variables.csv"), index=False, header=True) # plot trajectories and state vs. time state_variable_plots(df1=df_truth, df1_label="ground truth", df2=df_nn, df2_label="nn", dir_path=batch_folder, cols_to_include=np.concatenate( (state_cols, ctrl_cols))) # plot state der vs. time state_der_plots(df1=df_truth, df1_label="ground truth", df2=df_nn, df2_label="nn", dir_path=batch_folder, cols_to_include=np.concatenate( (state_der_cols, ctrl_cols))) # save raw multi-step errors to disk errors_array = np.array(errors_list) np.save(file=os.path.join(test_phase_dir, "multi_step_err.npy"), arr=errors_array) # hacky way to get first set of time data _, _, _, time_data = iter(data_loader).next() time_data = time_data.cpu().numpy() # plot mean errors and their std multi_step_error_plots(errors_array, time_data, x_idx=0, y_idx=1, yaw_idx=2, dir_path=test_phase_dir, num_box_plots=3, plot_hists=True, num_hist=6) # save raw instantaneous errors to disk inst_errors = np.array(inst_errors) np.save(file=os.path.join(test_phase_dir, "inst_err.npy"), arr=inst_errors) # plot histogram of signed instantaneous errors inst_error_plots(inst_errors, state_der_cols, test_phase_dir)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--model-setup', required=True, help='Model setup dictionary.') parser.add_argument('--lsh', action='store_true', default=False, help='If true, uses locally sensitive hashing \ (with k=10 NN) for NN search.') args = parser.parse_args() model, train, test, vocab, setup = setup_model(args) is_snli = setup['dataset'] == 'snli' if is_snli: converter = convert_snli_seq else: converter = convert_seq # FIXME args.batchsize = 64 max_beam_size = 5 test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) checkpoint = [] n_batches = len(test) // args.batchsize for batch_idx, batch in enumerate(tqdm(test_iter, total=n_batches)): if batch_idx > 10: break batch = converter(batch, device=args.gpu) xs = batch['xs'] reduced_xs, removed_indices = get_rawr(model, xs, max_beam_size=max_beam_size, snli=is_snli) n_finals = [len(r) for r in reduced_xs] batch_size = len(xs[0]) if is_snli else len(xs) assert len(reduced_xs) == batch_size xp = cupy.get_array_module(xs[0][0]) if is_snli: prem = xs[0] _reduced_xs = [] for i in range(batch_size): for x in reduced_xs[i]: _reduced_xs.append([prem[i].copy(), xp.asarray(x)]) reduced_xs = list(map(list, zip(*_reduced_xs))) else: reduced_xs = list(itertools.chain(*reduced_xs)) reduced_xs = [xp.asarray(x) for x in reduced_xs] # reduced_xs = converter(reduced_xs, device=args.gpu, with_label=False) removed_indices = list(itertools.chain(*removed_indices)) with chainer.using_config('train', False): ss_0 = xp.asnumpy(model.predict(xs, softmax=True)) ss_1 = xp.asnumpy(model.predict(reduced_xs, softmax=True)) ys_0 = np.argmax(ss_0, axis=1) ys_1 = np.argmax(ss_1, axis=1) if is_snli: xs = list(map(list, zip(*xs))) xs = [(a.tolist(), b.tolist()) for a, b in xs] reduced_xs = list(map(list, zip(*reduced_xs))) reduced_xs = [(a.tolist(), b.tolist()) for a, b in reduced_xs] else: xs = [x.tolist() for x in xs] reduced_xs = [x.tolist() for x in reduced_xs] start = 0 for example_idx in range(len(xs)): oi = xs[example_idx] # original input op = int(ys_0[example_idx]) # original predictoin oos = ss_0[example_idx] # original output distribution label = int(batch['ys'][example_idx]) checkpoint.append([]) for i in range(start, start + n_finals[example_idx]): ri = reduced_xs[i] rp = int(ys_1[i]) # reduced prediction rs = ss_1[i] # reduced output distribution rr = removed_indices[i] entry = { 'original_input': oi, 'reduced_input': ri, 'original_prediction': op, 'reduced_prediction': rp, 'original_scores': oos, 'reduced_scores': rs, 'removed_indices': rr, 'label': label } checkpoint[-1].append(entry) start += n_finals[example_idx] with open(os.path.join('rawr_dev.pkl'), 'wb') as f: pickle.dump(checkpoint, f)
def run_extract_representations( args, cfg ): # set up logging tf.logging.set_verbosity( tf.logging.INFO ) with tf.Graph().as_default() as g: cfg['randomize'] = False cfg['num_epochs'] = 1 # cfg['num_read_threads'] = 5 # cfg['batch_size']=2 #if cfg['model_path'] is None: # cfg['model_path'] = tf.train.latest_checkpoint( os.path.join( args.cfg_dir, "logs/slim-train/" ) ) cfg['model_path'] = os.path.join( args.cfg_dir, "logs/slim-train/model.ckpt-59690") # create ops and placeholders tf.logging.set_verbosity( tf.logging.INFO ) inputs = utils.setup_input( cfg, is_training=False, use_filename_queue=True ) RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) RuntimeDeterminedEnviromentVars.populate_registered_variables() # build model (and losses and train_op) model = utils.setup_model( inputs, cfg, is_training=False ) # set up metrics to evaluate names_to_values, names_to_updates = setup_metrics( inputs, model, cfg ) # execute training start_time = time.time() utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False ) # start session and restore model training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() } try: if cfg['model_path'] is None: print('Please specify a checkpoint directory') return model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] ) utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False ) data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=True ) training_runners[ 'threads' ] = data_prefetch_init_fn( training_runners[ 'sess' ], training_runners[ 'coord' ] ) # run one example so that we can calculate some statistics about the representations filenames = [] representations, data_idx = training_runners['sess'].run( [ model['model'].encoder_output, inputs[ 'data_idxs' ] ] ) filenames += [ inputs[ 'filepaths_list'][ i ] for i in data_idx ] print( 'Got first batch representation with size: {0}'.format( representations.shape ) ) # run the remaining examples for step in xrange( inputs[ 'max_steps' ] - 1 ): if step % 100 == 0: print( 'Step {0} of {1}'.format( step, inputs[ 'max_steps' ] - 1 )) encoder_output, data_idx = training_runners['sess'].run( [ model['model'].encoder_output, inputs[ 'data_idxs' ] ] ) representations = np.append(representations, encoder_output, axis=0) filenames += [ inputs[ 'filepaths_list'][ i ] for i in data_idx ] if training_runners['coord'].should_stop(): break print('The size of representations is %s while we expect it to run for %d steps with batchsize %d' % (representations.shape, inputs['max_steps'], cfg['batch_size'])) end_train_time = time.time() - start_time save_path = os.path.join( args.cfg_dir, '../representations.pkl' ) with open( save_path, 'wb' ) as f: pickle.dump( { 'filenames': filenames, 'representations': representations }, f ) print( 'saved representations to {0}'.format( save_path )) print('time to train %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time/(60*60))) print('avg time per epoch: %.3f hrs' % ( (end_train_time/(60*60)) / cfg['num_epochs']) ) finally: utils.request_data_loading_end( training_runners ) utils.end_data_loading_and_sess( training_runners )
def run_extract_losses(args, cfg, save_dir, given_task): transfer = (cfg['model_type'] == architectures.TransferNet) if transfer: get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer setup_input_fn = utils.setup_input_transfer else: setup_input_fn = utils.setup_input get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn # set up logging tf.logging.set_verbosity(tf.logging.ERROR) stats = Statistics() print_every = int(args.print_every) with tf.Graph().as_default() as g: # create ops and placeholders inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) #RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) #RuntimeDeterminedEnviromentVars.populate_registered_variables() max_steps = get_max_steps(inputs['max_steps'], args.data_split) # pdb.set_trace() # build model (and losses and train_op) model = utils.setup_model(inputs, cfg, is_training=False) loss_names, loss_ops = get_extractable_losses(cfg, model) if 'l1_loss' in loss_names: display_loss = 'l1_loss' elif 'l2_loss' in loss_names: display_loss = 'l2_loss' elif 'xentropy' in loss_names: display_loss = 'xentropy' elif 'metric_loss' in loss_names: display_loss = 'metric_loss' elif 'cycle_loss' in loss_names: display_loss = 'cycle_loss' else: display_loss = 'total_loss' # set up metrics to evaluate names_to_values, names_to_updates = setup_metrics(inputs, model, cfg) # execute training start_time = time.time() utils.print_start_info(cfg, max_steps, is_training=False) # start session and restore model training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() } try: if cfg['model_path'] is None: print('Please specify a checkpoint directory') return print('Attention, model_path is ', cfg['model_path']) model['saver_op'].restore(training_runners['sess'], cfg['model_path']) # var = [v for v in tf.global_variables() if 'decoder' in v.name][0] # print(training_runners[ 'sess' ].run(var)) utils.print_start_info(cfg, max_steps, is_training=False) data_prefetch_init_fn = get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False) prefetch_threads = threading.Thread( target=data_prefetch_init_fn, args=(training_runners['sess'], training_runners['coord'])) prefetch_threads.start() # run one example so that we can calculate some statistics about the representations filenames = [] loss_names_to_vals = {name: [] for name in loss_names} results = training_runners['sess'].run([ inputs['data_idxs'], inputs['target_batch'], inputs['mask_batch'], *loss_ops ]) #gs = results[1] data_idx = results[0] losses = results[3:] target_input = results[1] mask_input = results[2] for i, name in enumerate(loss_names): loss_names_to_vals[name].append(losses[i]) filenames.extend(data_idx) print("Step number: {}".format(1), (data_idx)) # print(target_input, target_input.sum()) # return # training_runners['sess'].run([v for v in tf.global_variables() if "transfer/rep_conv_1/weights" in v.name][0]) # run the remaining examples start = time.perf_counter() for step in range(max_steps - 1): results = training_runners['sess'].run([ inputs['data_idxs'], # [v for v in tf.global_variables() if "transfer/rep_conv_1/weights/(weights)" in v.name][0], # model['model'].encoder_endpoints['net1_1_output'], # model['model'].encoder_endpoints['net1_2_output'], *loss_ops ]) data_idx = results[0] # print(data_idx) losses = results[1:] # p, t, m = results[1], results[2], results[3] # losses = results[4:] # print(p.mean(), t) for i, name in enumerate(loss_names): loss_names_to_vals[name].append(losses[i]) filenames.extend(data_idx) stats.push(loss_names_to_vals[display_loss][-1]) # baseline_loss = get_xentropy_loss(p, t, m) # tf_loss = loss_names_to_vals[display_loss][-1] # print('tf {} | ours {}'.format(tf_loss, baseline_loss)) # pdb.set_trace() if step % print_every == 0 and step > 0: print( 'Step {0} of {1}: ({5} loss: {2:.3f} || stddev: {3:.3f} :: ({4:.2f} secs/step)' .format( step, max_steps - 1, stats.mean(), np.sqrt(stats.variance()), # stats.variance(), (time.perf_counter() - start) / print_every, display_loss)) start = time.perf_counter() if training_runners['coord'].should_stop(): break print( 'The size of losses is %s while we expect it to run for %d steps with batchsize %d' % (len(filenames), inputs['max_steps'], cfg['batch_size'])) end_train_time = time.time() - start_time if args.out_name: out_name = args.out_name else: out_name = '{task}_{split}_losses.pkl'.format( task=given_task, split=args.data_split) save_path = os.path.join(save_dir, out_name) with open(save_path, 'wb') as f: loss_names_to_vals['file_indexes'] = filenames loss_names_to_vals['global_step'] = 0 pickle.dump(loss_names_to_vals, f) if args.out_dir: os.makedirs(args.out_dir, exist_ok=True) os.system("sudo cp {fp} {out}/".format(fp=save_path, out=args.out_dir)) else: if transfer: copy_to = cfg['log_root'] else: copy_to = os.path.join(cfg['log_root'], given_task) os.system("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to)) print("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to)) # if transfer: # os.makedirs('/home/ubuntu/s3/model_log/losses_transfer/', exist_ok=True) # os.system("sudo cp {fp} /home/ubuntu/s3/model_log/losses_transfer/".format(fp=save_path)) # else: # os.makedirs('/home/ubuntu/s3/model_log/losses/', exist_ok=True) # os.system("sudo cp {fp} /home/ubuntu/s3/model_log/losses/".format(fp=save_path)) print('saved losses to {0}'.format(save_path)) print('time to extract %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time / (60 * 60))) finally: utils.request_data_loading_end(training_runners) utils.end_data_loading_and_sess(training_runners)
'coord': tf.train.Coordinator() } ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) setup_input_fn = utils.setup_input inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() ############## Set Up Model ############## model = utils.setup_model(inputs, cfg, is_training=False) m = model['model'] model['saver_op'].restore(training_runners['sess'], cfg['model_path']) ''' # encoder (extract features) predicted, representation = training_runners['sess'].run( [ m.decoder_output, m.encoder_output ], feed_dict={m.input_images: img} ) ''' representation = training_runners['sess'].run( m.encoder_output, feed_dict={m.input_images: img}) ''' if task == 'segment2d' or task == 'segment25d': segmentation_pca(predicted, storePath+task+im_name.split('.')[0]+'.png')
def run_extract_losses_5_steps(args, cfg, save_dir, given_task): transfer = (cfg['model_type'] == architectures.TransferNet) if transfer: get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer setup_input_fn = utils.setup_input_transfer else: setup_input_fn = utils.setup_input get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn # set up logging tf.logging.set_verbosity(tf.logging.ERROR) stats = Statistics() with tf.Graph().as_default() as g: # create ops and placeholders inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) #RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) #RuntimeDeterminedEnviromentVars.populate_registered_variables() max_steps = get_max_steps(inputs['max_steps'], args.data_split) # build model (and losses and train_op) model = utils.setup_model(inputs, cfg, is_training=False) loss_names, loss_ops = get_extractable_losses(cfg, model) if 'l1_loss' in loss_names: display_loss = 'l1_loss' elif 'l2_loss' in loss_names: display_loss = 'l2_loss' elif 'xentropy' in loss_names: display_loss = 'xentropy' elif 'metric_loss' in loss_names: display_loss = 'metric_loss' elif 'cycle_loss' in loss_names: display_loss = 'cycle_loss' else: display_loss = 'total_loss' # set up metrics to evaluate names_to_values, names_to_updates = setup_metrics(inputs, model, cfg) # execute training start_time = time.time() utils.print_start_info(cfg, max_steps, is_training=False) # start session and restore model training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() } if cfg['model_path'] is None: print('Please specify a checkpoint directory') return print('Attention, model_path is ', cfg['model_path']) model['saver_op'].restore(training_runners['sess'], cfg['model_path']) utils.print_start_info(cfg, max_steps, is_training=False) data_prefetch_init_fn = get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False) prefetch_threads = threading.Thread(target=data_prefetch_init_fn, args=(training_runners['sess'], training_runners['coord'])) prefetch_threads.start() # run one example so that we can calculate some statistics about the representations # results = training_runners['sess'].run( [ *loss_ops ] ) # losses = results[0] x = 0 for step in range(3): results = training_runners['sess'].run([*loss_ops]) x = x + results[0] if training_runners['coord'].should_stop(): break tf.reset_default_graph() utils.request_data_loading_end(training_runners) utils.end_data_loading_and_sess(training_runners) return x / 2.
def run_extract_losses( args, cfg, save_dir, given_task ): transfer = (cfg['model_type'] == architectures.TransferNet) if transfer: get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer_imagenet setup_input_fn = utils.setup_input_transfer_imagenet else: setup_input_fn = utils.setup_input get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn # set up logging tf.logging.set_verbosity( tf.logging.ERROR ) stats = Statistics() top5_stats = Statistics() print_every = int(args.print_every) with tf.Graph().as_default() as g: # create ops and placeholders inputs = setup_input_fn( cfg, is_training=False, use_filename_queue=False ) #RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) #RuntimeDeterminedEnviromentVars.populate_registered_variables() max_steps = get_max_steps(inputs[ 'max_steps' ], args.data_split) # build model (and losses and train_op) model = utils.setup_model( inputs, cfg, is_training=False ) # set up metrics to evaluate names_to_values, names_to_updates = setup_metrics( inputs, model, cfg ) # execute training start_time = time.time() utils.print_start_info( cfg, max_steps, is_training=False ) # start session and restore model training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() } try: if cfg['model_path'] is None: print('Please specify a checkpoint directory') return print('Attention, model_path is ', cfg['model_path']) model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] ) # var = [v for v in tf.global_variables() if 'decoder' in v.name][0] # print(training_runners[ 'sess' ].run(var)) utils.print_start_info( cfg, max_steps, is_training=False ) data_prefetch_init_fn = get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False ) prefetch_threads = threading.Thread( target=data_prefetch_init_fn, args=( training_runners[ 'sess' ], training_runners[ 'coord' ] )) prefetch_threads.start() # run one example so that we can calculate some statistics about the representations filenames = [] accuracies = [] if transfer: accuracy_op = model['model'].decoder.accuracy final_output = model['model'].decoder.final_output else: accuracy_op = model['model'].accuracy final_output = model['model'].final_output results = training_runners['sess'].run( [ inputs[ 'data_idxs' ], model['model'].global_step, accuracy_op ] ) gs = results[1] data_idx = results[0] accuracy = results[2] filenames.extend(data_idx) accuracies.append(accuracy) print("Step number: {}".format(gs)) # print(loss_names_to_vals, data_idx) # return # run the remaining examples start = time.perf_counter() for step in range( max_steps - 1 ): results = training_runners['sess'].run( [ inputs[ 'data_idxs' ], final_output, inputs['target_batch'], accuracy_op ] ) data_idx = results[0] accuracy = results[-1] logits = results[1] gt = results[2] sorted_top5 = np.argsort(logits[0])[::-1][:5] sorted_gt = np.argsort(gt[0])[::-1][0] top5 = 0. if sorted_gt in sorted_top5: top5 = 1. filenames.extend(data_idx) accuracies.append(accuracy) stats.push(accuracy) top5_stats.push(top5) if step % print_every == 0 and step > 0: print( 'Step {0} of {1}: ({5}: {2:.3f} || Top 5: {3:.3f} :: ({4:.2f} secs/step)'.format( step, max_steps - 1, stats.mean(), top5_stats.mean(), # stats.variance(), (time.perf_counter() - start) / print_every, 'accuracy' )) start = time.perf_counter() if training_runners['coord'].should_stop(): break os.system("sudo touch /home/ubuntu/s3/imagenet_accuracy/{}_{}_{}.txt".format( given_task, int(stats.mean() * 1000) / 10., int(top5_stats.mean() * 1000) / 10.)) print('The size of losses is %s while we expect it to run for %d steps with batchsize %d' % (len(filenames), inputs['max_steps'], cfg['batch_size'])) end_train_time = time.time() - start_time if args.out_name: out_name = args.out_name else: out_name = '{task}_{split}_imagenet_accuracy.pkl'.format(task=given_task, split=args.data_split) save_path = os.path.join( save_dir, out_name ) val_accuracy = {} with open( save_path, 'wb' ) as f: val_accuracy['file_indexes'] = filenames val_accuracy['global_step'] = gs val_accuracy['accuracy'] = accuracies pickle.dump( val_accuracy, f ) if args.out_dir: os.makedirs(args.out_dir, exist_ok=True) os.system("sudo cp {fp} {out}/".format(fp=save_path, out=args.out_dir)) else: if transfer: copy_to = cfg['log_root'] else: copy_to = os.path.join(cfg['log_root'], given_task) os.system("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to)) print("sudo mv {fp} {dst}/".format(fp=save_path, dst=copy_to)) # if transfer: # os.makedirs('/home/ubuntu/s3/model_log/losses_transfer/', exist_ok=True) # os.system("sudo cp {fp} /home/ubuntu/s3/model_log/losses_transfer/".format(fp=save_path)) # else: # os.makedirs('/home/ubuntu/s3/model_log/losses/', exist_ok=True) # os.system("sudo cp {fp} /home/ubuntu/s3/model_log/losses/".format(fp=save_path)) print( 'saved losses to {0}'.format( save_path )) print('time to extract %d epochs: %.3f hrs' % (cfg['num_epochs'], end_train_time/(60*60))) finally: utils.request_data_loading_end( training_runners ) utils.end_data_loading_and_sess( training_runners )
def run_extract_losses(args, cfg, save_dir, given_task): transfer = (cfg['model_type'] == architectures.TransferNet) if transfer: get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer setup_input_fn = utils.setup_input_transfer if given_task == 'pixels': get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer_imagenet setup_input_fn = utils.setup_input_transfer_imagenet else: setup_input_fn = utils.setup_input get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn # set up logging tf.logging.set_verbosity(tf.logging.ERROR) stats = Statistics() print_every = int(args.print_every) with tf.Graph().as_default() as g: # create ops and placeholders inputs = setup_input_fn(cfg, is_training=True, use_filename_queue=False) #RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) #RuntimeDeterminedEnviromentVars.populate_registered_variables() max_steps = get_max_steps(inputs['max_steps'], args.data_split) # build model (and losses and train_op) model = utils.setup_model(inputs, cfg, is_training=True) # set up metrics to evaluate names_to_values, names_to_updates = setup_metrics(inputs, model, cfg) train_step_fn = model['train_step_fn'] # execute training start_time = time.time() utils.print_start_info(cfg, max_steps, is_training=True) # start session and restore model training_runners = { 'sess': tf.Session(), 'coord': tf.train.Coordinator() } try: if cfg['model_path'] is None: print('Please specify a checkpoint directory') return print('Attention, model_path is ', cfg['model_path']) restore_ckpt = not args.from_scratch if restore_ckpt: non_encoder_var = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES) adams = [] for v in tuple(non_encoder_var): if 'Adam' in v.name: non_encoder_var.remove(v) adams.append(v) continue if 'finetune_encoder_imagenet' in cfg: for x in model['model'].encoder_vars: if v.name == x.name: non_encoder_var.remove(v) if not args.metric_only: saver_for_transfer = tf.train.Saver(non_encoder_var) #training_runners['sess'].run(saver_for_transfer) saver_for_transfer.restore(training_runners['sess'], cfg['model_path']) else: #saver_for_transfer = tf.train.Saver(non_encoder_var) training_runners['sess'].run( tf.variables_initializer(non_encoder_var)) training_runners['sess'].run(tf.variables_initializer(adams)) print('Loading Source Encoder:...') if 'finetune_encoder_imagenet' in cfg: model['init_fn'](training_runners['sess']) print('Starting Training:..') else: init_op = tf.global_variables_initializer() training_runners['sess'].run(init_op) assign_op = model['global_step'].assign(0) training_runners['sess'].run(assign_op) # var = [v for v in tf.global_variables() if 'decoder' in v.name][0] # print(training_runners[ 'sess' ].run(var)) utils.print_start_info(cfg, max_steps, is_training=True) data_prefetch_init_fn = get_data_prefetch_threads_init_fn( inputs, cfg, is_training=True, use_filename_queue=False) prefetch_threads = threading.Thread( target=data_prefetch_init_fn, args=(training_runners['sess'], training_runners['coord'])) prefetch_threads.start() # run one example so that we can calculate some statistics about the representations start = time.perf_counter() saver = tf.train.Saver() save_ckpt_name = 'places' if args.from_scratch: save_ckpt_name = 'places_scratch_{}_{}'.format( args.layers, args.data_used) if args.train_encoder: save_ckpt_name = 'places_encoder' for step in range(max_steps // 2 - 1): #for step in range(10): total_loss, should_stop = train_step_fn( training_runners['sess'], model['train_op'], model['global_step'], train_step_kwargs=model['train_step_kwargs']) # print(data_idx) # print(p.mean(), t) stats.push(total_loss) if step % print_every == 0 and step > 0: print( 'Step {0} of {1}: ({5}: {2:.3f} || stddev: {3:.3f} :: ({4:.2f} secs/step)' .format( step, max_steps - 1, stats.mean(), np.sqrt(stats.variance()), # stats.variance(), (time.perf_counter() - start) / print_every, 'Total_loss')) start = time.perf_counter() if step % 3000 == 2999: saver.save( training_runners['sess'], os.path.join(cfg['log_root'], given_task, '{}_{}'.format(save_ckpt_name, step))) if training_runners['coord'].should_stop(): break #print('Heressss') saver.save( training_runners['sess'], os.path.join(cfg['log_root'], given_task, save_ckpt_name)) finally: utils.request_data_loading_end(training_runners) utils.end_data_loading_and_sess(training_runners)
def run_to_task(task_to): import general_utils from general_utils import RuntimeDeterminedEnviromentVars import models.architectures as architectures from data.load_ops import resize_rescale_image import utils from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction import lib.data.load_ops as load_ops from importlib import reload import tensorflow as tf tf.logging.set_verbosity(tf.logging.ERROR) # for arch in ['regular', 'shallow', 'dilated_shallow', 'dilated_regular']: arch = args.arch data_amount = args.data if args.second_order: global TRANSFER_TYPE TRANSFER_TYPE = 'second_order' if not args.no_regenerate_data: #if False: all_outputs = {} pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format( arch=arch, hs=args.hs, task_to=task_to) subprocess.call( "aws s3 cp s3://task-preprocessing-512-oregon/visualizations/transfer_viz/viz_{}.pkl {}" .format(task_to, pickle_dir), shell=True) import os if os.path.isfile(pickle_dir): with open(pickle_dir, 'rb') as fp: all_outputs = pickle.load(fp) if args.second_order: import itertools with open( '/home/ubuntu/task-taxonomy-331b/tools/ranked_first_order_transfers.pkl', 'rb') as fp: data = pickle.load(fp) list_of_src_tasks = list( itertools.combinations(data[task_to][:5], 2)) list_of_src_tasks = [ '{}__{}'.format(x[0], x[1]) for x in list_of_src_tasks ] with open( '/home/ubuntu/task-taxonomy-331b/tools/second_order_should_flip.pkl', 'rb') as fp: to_flip_dict = pickle.load(fp) if args.find_all_src: config_dir_root = '/home/ubuntu/task-taxonomy-331b/experiments/second_order/{arch}/{data}'.format( arch=arch, data=data_amount) all_configs = os.listdir(config_dir_root) list_of_src_tasks = [] for i in all_configs: if i.split('__')[-3] == task_to: list_of_src_tasks.append(i) list_of_src_tasks = [ '__'.join(x.split('__')[:2]) for x in list_of_src_tasks ] rank_combo = {} for task_from in list_of_src_tasks: first, sec = task_from.split('__') rank_combo[task_from] = (data[task_to].index(first), data[task_to].index(sec)) global list_of_src_tasks for i, task_from in enumerate(list_of_src_tasks): if args.data == '16k': if task_from in all_outputs: print("{} already exists....\n\n\n".format(task_from)) continue else: if '{}_{}'.format(task_from, args.data) in all_outputs: print("{} already exists....\n\n\n".format(task_from)) continue print("Doing from {task_from} to {task_to}".format( task_from=task_from, task_to=task_to)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } if task_from == "FULL" or task_from == "FULL_IMAGE": task = '{f}__{t}__{hs}__unlocked'.format(f="FULL", t=task_to, hs=args.hs) transfer_src = 'full_order' if task_from == "FULL" else 'full_order_image' CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format( transfer_type=transfer_src, arch=arch, data=data_amount, TASK=task) elif task_from == "FULL_select" or task_from == "FULL_select_IMAGE": task = '{f}__{t}__{hs}__unlocked'.format(f="FULL_select", t=task_to, hs=args.hs) transfer_src = 'full_order_selected' if task_from == "FULL_select" else 'full_order_selected_image' CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format( transfer_type=transfer_src, arch=arch, data=data_amount, TASK=task) else: task = '{f}__{t}__{hs}__unlocked'.format(f=task_from, t=task_to, hs=args.hs) CONFIG_DIR = '/home/ubuntu/task-taxonomy-331b/experiments/{transfer_type}/{arch}/{data}/{TASK}'.format( transfer_type=TRANSFER_TYPE, arch=arch, data=data_amount, TASK=task) print(CONFIG_DIR) ############## Load Configs ############## cfg = utils.load_config(CONFIG_DIR, nopause=True) RuntimeDeterminedEnviromentVars.register_dict(cfg) if args.second_order and not args.find_all_src and not to_flip_dict[ task]: cfg['val_representations_file'] = cfg[ 'val_representations_file'][::-1] cfg['num_epochs'] = 1 cfg['randomize'] = False root_dir = cfg['root_dir'] cfg['num_read_threads'] = 1 cfg['model_path'] = tf.train.latest_checkpoint( os.path.join(cfg['log_root'], 'logs', 'slim-train' #'time' )) # print(cfg['model_path']) if cfg['model_path'] is None and task == 'random': cfg['model_path'] = tf.train.latest_checkpoint( os.path.join(cfg['log_root'], 'logs', 'slim-train', 'time')) if cfg['model_path'] is None: continue ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) inputs = utils.setup_input_transfer( cfg, is_training=ON_TEST_SET, use_filename_queue=False ) # is_training determines whether to use train/validaiton RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() # utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False ) ############## Set Up Model ############## model = utils.setup_model(inputs, cfg, is_training=True) m = model['model'] model['saver_op'].restore(training_runners['sess'], cfg['model_path']) ############## Start dataloading workers ############## data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn_transfer( inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False) prefetch_threads = threading.Thread( target=data_prefetch_init_fn, args=(training_runners['sess'], training_runners['coord'])) prefetch_threads.start() ############## Run First Batch ############## ( input_batch, representation_batch, target_batch, data_idx, encoder_output, predicted, loss, ) = training_runners['sess'].run([ m.input_images, m.input_representations, m.decoder.targets, model['data_idxs'], m.encoder_output, m.decoder.decoder_output, m.total_loss ]) if task_to == 'segment2d' or task_to == 'segment25d': from sklearn.decomposition import PCA x = np.zeros((32, 256, 256, 3), dtype='float') for i in range(predicted.shape[0]): embedding_flattened = np.squeeze(predicted[i]).reshape( (-1, 64)) pca = PCA(n_components=3) pca.fit(embedding_flattened) lower_dim = pca.transform(embedding_flattened).reshape( (256, 256, -1)) lower_dim = (lower_dim - lower_dim.min()) / ( lower_dim.max() - lower_dim.min()) x[i] = lower_dim predicted = x if task_to == 'segmentsemantic_rb': predicted = np.argmax(predicted, axis=-1) ############## Clean Up ############## training_runners['coord'].request_stop() training_runners['coord'].join() # if os.path.isfile(pickle_dir): # with open(pickle_dir, 'rb') as fp: # all_outputs = pickle.load(fp) ############## Store to dict ############## to_store = { 'data_idx': data_idx, 'output': predicted, 'loss': loss } if args.second_order and args.find_all_src: store_key = "{}_{}".format(task_from[:20], rank_combo[task_from]) all_outputs[store_key] = to_store elif args.data != '16k': store_key = "{}_{}".format(task_from, args.data) all_outputs[store_key] = to_store else: all_outputs[task_from] = to_store # os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir)) ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() #print(sys.modules.keys()) #del sys.modules[ 'config' ] sys.path = remove_dups(sys.path) print('Current Directory: ', os.getcwd()) pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format( arch=arch, hs=args.hs, task_to=task_to) with open(pickle_dir, 'wb') as fp: pickle.dump(all_outputs, fp) subprocess.call( "aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/transfer_viz/viz_{}.pkl" .format(pickle_dir, task_to), shell=True) # Run jupyter nb print('Running Jupyter Notebooks...') #os.makedirs("/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/transfer_{hs}_{arch}".format(hs=args.hs, arch=arch), exist_ok=True) notebook_path = '/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/Visual_{task_to}'.format( task_to=task_to) if args.second_order and not args.find_all_src: notebook_path = '{}-Copy1'.format(notebook_path) subprocess.call("jupyter nbconvert \ --execute {notebook_path}.ipynb \ --to html \ --ExecutePreprocessor.kernel_name=python3 \ --ExecutePreprocessor.timeout=1200 ".format( notebook_path=notebook_path, arch=arch, hs=args.hs, task_to=task_to), shell=True) subprocess.call( "aws s3 cp {}.html s3://task-preprocessing-512-oregon/visualizations/{}/" .format(notebook_path, TRANSFER_TYPE), shell=True)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--model-setup', required=True, help='Model setup dictionary.') parser.add_argument('--lsh', action='store_true', default=False, help='If true, uses locally sensitive hashing \ (with k=10 NN) for NN search.') args = parser.parse_args() model, train, test, vocab, setup = setup_model(args) if setup['dataset'] == 'snli': converter = convert_snli_seq use_snli = True else: converter = convert_seq use_snli = False with open(os.path.join(setup['save_path'], 'calib.json')) as f: calibration_idx = json.load(f) calibration = [train[i] for i in calibration_idx] train = [x for i, x in enumerate(train) if i not in calibration_idx] '''save dknn layers for training data''' dknn = DkNN(model, lsh=args.lsh) dknn.build(train, batch_size=setup['batchsize'], converter=converter, device=args.gpu) '''calibrate the dknn credibility values''' dknn.calibrate(calibration, batch_size=setup['batchsize'], converter=converter, device=args.gpu) '''run dknn on evaluation data''' test_iter = chainer.iterators.SerialIterator(test, setup['batchsize'], repeat=False) test_iter.reset() print('run dknn on evaluation data') total = 0 n_reg_correct = 0 n_knn_correct = 0 n_batches = len(test) // setup['batchsize'] for test_batch in tqdm(test_iter, total=n_batches): data = converter(test_batch, device=args.gpu, with_label=True) text = data['xs'] knn_pred, knn_cred, knn_conf, reg_pred, reg_conf = dknn.predict( text, snli=use_snli) label = [int(x) for x in data['ys']] total += len(label) n_knn_correct += sum(x == y for x, y in zip(knn_pred, label)) n_reg_correct += sum(x == y for x, y in zip(reg_pred, label)) print('knn accuracy', n_knn_correct / total) print('reg accuracy', n_reg_correct / total)
def run_to_task(args): import general_utils from general_utils import RuntimeDeterminedEnviromentVars tf.logging.set_verbosity(tf.logging.ERROR) img = load_raw_image_center_crop(args.im_name) img = skimage.img_as_float(img) scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(args.im_name) task = args.task if task not in list_of_tasks: raise ValueError('Task not supported') cfg = generate_cfg(task) # Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \ keypoint2d keypoint3d \ reshade rgb2depth rgb2mist rgb2sfnorm \ segment25d segment2d room_layout'.split() if task in low_sat_tasks: cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat if task == 'jigsaw': img = cfg['input_preprocessing_fn']( img, target=cfg['target_dict'][random.randint(0, 99)], **cfg['input_preprocessing_fn_kwargs']) else: img = cfg['input_preprocessing_fn']( img, **cfg['input_preprocessing_fn_kwargs']) img = img[np.newaxis, :] if task == 'class_places' or task == 'class_1000': synset = get_synset(task) print("Doing {task}".format(task=task)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) setup_input_fn = utils.setup_input inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() ############## Set Up Model ############## model = utils.setup_model(inputs, cfg, is_training=False) m = model['model'] model['saver_op'].restore(training_runners['sess'], cfg['model_path']) predicted, representation = training_runners['sess'].run( [m.decoder_output, m.encoder_endpoints[args.encoder_layer]], feed_dict={m.input_images: img}) if args.store_rep: s_name, file_extension = os.path.splitext(args.store_name) if args.compress_rep: representation = tf.keras.layers.AveragePooling2D( pool_size=(2, 2))(tf.constant(representation)) representation = training_runners['sess'].run(representation) with open('{}.npy'.format(s_name), 'wb') as fp: np.save(fp, np.squeeze(representation)) if args.store_pred: s_name, file_extension = os.path.splitext(args.store_name) with open('{}_pred.npy'.format(s_name), 'wb') as fp: np.save(fp, np.squeeze(predicted)) if task == 'segment2d' or task == 'segment25d': segmentation_pca(predicted, args.store_name) return if task == 'colorization': single_img_colorize(predicted, img, args.store_name) return if task == 'curvature': curvature_single_image(predicted, args.store_name) return just_rescale = [ 'autoencoder', 'denoise', 'edge2d', 'edge3d', 'keypoint2d', 'keypoint3d', 'reshade', 'rgb2sfnorm' ] if task in just_rescale: simple_rescale_img(predicted, args.store_name) return just_clip = ['rgb2depth', 'rgb2mist'] if task in just_clip: depth_single_image(predicted, args.store_name) return if task == 'inpainting_whole': inpainting_bbox(predicted, args.store_name) return if task == 'segmentsemantic': semseg_single_image(predicted, img, args.store_name) return if task in ['class_1000', 'class_places']: classification(predicted, synset, args.store_name) return if task == 'vanishing_point': _ = plot_vanishing_point_smoothed(np.squeeze(predicted), (np.squeeze(img) + 1.) / 2., args.store_name, []) return if task == 'room_layout': mean = np.array([ 0.006072743318127848, 0.010272365569691076, -3.135909774145468, 1.5603802322235532, 5.6228218371102496e-05, -1.5669352793761442, 5.622875878174759, 4.082800262277375, 2.7713941642895956 ]) std = np.array([ 0.8669452525283652, 0.687915294956501, 2.080513632043758, 0.19627420479282623, 0.014680602791251812, 0.4183827359302299, 3.991778013006544, 2.703495278378409, 1.2269185938626304 ]) predicted = predicted * std + mean plot_room_layout(np.squeeze(predicted), (np.squeeze(img) + 1.) / 2., args.store_name, [], cube_only=True) return if task == 'jigsaw': predicted = np.argmax(predicted, axis=1) perm = cfg['target_dict'][predicted[0]] show_jigsaw((np.squeeze(img) + 1.) / 2., perm, args.store_name) return ############## Clean Up ############## training_runners['coord'].request_stop() training_runners['coord'].join() print("Done: {}".format(config_name)) ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() return
def deep_attribution(): import general_utils from general_utils import RuntimeDeterminedEnviromentVars tf.logging.set_verbosity(tf.logging.ERROR) imlist_file_path = os.path.join(prj_dir, args.explain_result_root, args.dataset, 'imlist.txt') explain_result_root = os.path.join(prj_dir, args.explain_result_root, args.dataset) with open(imlist_file_path) as f: lines = [] line = f.readline().strip() while len(line) != 0: lines += [os.path.join(prj_dir, 'dataset', args.dataset, line)] line = f.readline().strip() explain_methods = ['saliency', 'grad*input', 'elrp'] for task in list_of_tasks: if not os.path.exists(os.path.join(explain_result_root, task)): os.mkdir(os.path.join(explain_result_root, task)) cfg = generate_cfg(task) print("Doing {task}".format(task=task)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() sess = tf.Session() training_runners = {'sess': sess, 'coord': tf.train.Coordinator()} with DeepExplain(session=sess, graph=sess.graph) as de: ############## Set Up Inputs ############## setup_input_fn = utils.setup_input inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() ############## Set Up Model ############## model = utils.setup_model(inputs, cfg, is_training=False) m = model['model'] model['saver_op'].restore(training_runners['sess'], cfg['model_path']) encoder_endpoints = model['model'].encoder_endpoints endpoints = encoder_endpoints print('There are {} images in {}'.format(len(lines), imlist_file_path)) img = load_raw_image_center_crop(lines[0]) img = skimage.img_as_float(img) low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \ keypoint2d keypoint3d \ reshade rgb2depth rgb2mist rgb2sfnorm \ segment25d segment2d room_layout'.split() if task in low_sat_tasks: cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat img = cfg['input_preprocessing_fn']( img, **cfg['input_preprocessing_fn_kwargs']) imgs = np.zeros([args.imlist_size, 1] + list(img.shape), float) for line_i in range(args.imlist_size): img = load_raw_image_center_crop(lines[line_i]) img = skimage.img_as_float(img) if task not in list_of_tasks: raise ValueError('Task not supported') if task in low_sat_tasks: cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat if task == 'jigsaw': img = cfg['input_preprocessing_fn']( img, target=cfg['target_dict'][random.randint(0, 99)], **cfg['input_preprocessing_fn_kwargs']) else: img = cfg['input_preprocessing_fn']( img, **cfg['input_preprocessing_fn_kwargs']) imgs[line_i, :] = img[np.newaxis, :] elrp = np.zeros([args.imlist_size] + list(img.shape), float) saliency = np.zeros([args.imlist_size] + list(img.shape), float) gradXinput = np.zeros([args.imlist_size] + list(img.shape), float) for im_i in range(args.imlist_size): with DeepExplain(session=sess) as de: representation = training_runners['sess'].run( [endpoints['encoder_output']], feed_dict={m.input_images: imgs[im_i]}) attributions = { explain_method: de.explain(explain_method, endpoints['encoder_output'], m.input_images, imgs[im_i]) for explain_method in explain_methods } elrp[im_i] = attributions['elrp'] saliency[im_i] = attributions['saliency'] gradXinput[im_i] = attributions['grad*input'] print('{} images done.'.format(im_i)) if ((im_i + 1) % 5) == 0: ############## Clean Up ############## training_runners['coord'].request_stop() training_runners['coord'].join() ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() sess = tf.Session() training_runners = { 'sess': sess, 'coord': tf.train.Coordinator() } with DeepExplain(session=sess, graph=sess.graph) as de: ############## Set Up Inputs ############## setup_input_fn = utils.setup_input inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables( ) ############## Set Up Model ############## model = utils.setup_model(inputs, cfg, is_training=False) m = model['model'] model['saver_op'].restore(training_runners['sess'], cfg['model_path']) encoder_endpoints = model['model'].encoder_endpoints endpoints = encoder_endpoints np.save(os.path.join(explain_result_root, task, 'elrp.npy'), elrp) np.save(os.path.join(explain_result_root, task, 'saliency.npy'), saliency) np.save(os.path.join(explain_result_root, task, 'gradXinput.npy'), gradXinput) ############## Clean Up ############## training_runners['coord'].request_stop() training_runners['coord'].join() ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() print('Task {} Done!'.format(task)) print('All Done.') return
def run_to_task(): import general_utils from general_utils import RuntimeDeterminedEnviromentVars args = parser.parse_args() if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) tf.logging.set_verbosity(tf.logging.ERROR) image_list = glob.glob(args.img_dir +"/*.png") image_list.sort() #image_list=image_list[:5] print(len(image_list)) for task in tqdm(list_of_tasks): print("Task is ", task) if task not in list_of_tasks: raise ValueError('Task not supported') low_sat_tasks = 'autoencoder curvature denoise edge2d edge3d \ keypoint2d keypoint3d \ reshade rgb2depth rgb2mist rgb2sfnorm \ segment25d segment2d room_layout'.split() cfg = generate_cfg(task) if task in low_sat_tasks: cfg['input_preprocessing_fn'] = load_ops.resize_rescale_image_low_sat print("Doing {task}".format(task=task)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) setup_input_fn = utils.setup_input inputs = setup_input_fn( cfg, is_training=False, use_filename_queue=False ) RuntimeDeterminedEnviromentVars.load_dynamic_variables( inputs, cfg ) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() ############## Set Up Model ############## model = utils.setup_model( inputs, cfg, is_training=False ) m = model[ 'model' ] model[ 'saver_op' ].restore( training_runners[ 'sess' ], cfg[ 'model_path' ] ) #Prints all the class variables and functions print(dir(m)) for img_name in tqdm(image_list): filename = img_name.split('/')[-1].split('.')[0] img = load_raw_image_center_crop( img_name ) img = skimage.img_as_float(img) scipy.misc.toimage(np.squeeze(img), cmin=0.0, cmax=1.0).save(img_name) # Since we observe that areas with pixel values closes to either 0 or 1 sometimes overflows, we clip pixels value if task == 'jigsaw' : img = cfg[ 'input_preprocessing_fn' ]( img, target=cfg['target_dict'][random.randint(0,99)], **cfg['input_preprocessing_fn_kwargs'] ) else: img = cfg[ 'input_preprocessing_fn' ]( img, **cfg['input_preprocessing_fn_kwargs'] ) img = img[np.newaxis,:] if task in fc_task_list: predicted, representation, decoder_features, encoder_features = training_runners['sess'].run( [ m.decoder_output, m.encoder_output, m.metric_endpoints, m.encoder_endpoints ], feed_dict={m.input_images: img} ) else: predicted, representation, decoder_features, encoder_features = training_runners['sess'].run( [ m.decoder_output, m.encoder_output, m.decoder_endpoints, m.encoder_endpoints ], feed_dict={m.input_images: img} ) #np.save(save_path,value) #for name,value in decoder_features.items(): # print (name) ## CKD : Uncomment below for loop for name,value in encoder_features.items(): if name in encoder_save_list or name in feedforward_encoder_save_list: #print (name) name = name.replace('/', '_') save_path = os.path.join(args.save_dir,filename+"_"+task + "_" + name + ".npy") np.save(save_path,value) if args.store_rep: s_name, file_extension = os.path.splitext(args.store_name) with open('{}.npy'.format(s_name), 'wb') as fp: np.save(fp, np.squeeze(representation)) if args.store_pred: save_path = os.path.join(args.save_dir,filename+"_"+task + "_" + "prediction" + ".npy") with open(save_path, 'wb') as fp: np.save(fp, np.squeeze(predicted)) #if task == 'segment2d' or task == 'segment25d': # segmentation_pca(predicted, args.store_name) #return #if task == 'colorization': # single_img_colorize(predicted, img , args.store_name) #return #if task == 'curvature': # curvature_single_image(predicted, args.store_name) #return #just_rescale = ['autoencoder', 'denoise', 'edge2d', # 'edge3d', 'keypoint2d', 'keypoint3d', # 'reshade', 'rgb2sfnorm' ] #if task in just_rescale: # print(args.store_name) #simple_rescale_img(predicted, args.store_name) #return #just_clip = ['rgb2depth', 'rgb2mist'] #if task in just_clip: # depth_single_image(predicted, args.store_name) # #return #if task == 'inpainting_whole': # inpainting_bbox(predicted, args.store_name) #return #if task == 'segmentsemantic': # semseg_single_image( predicted, img, args.store_name) #return #if task in ['class_1000', 'class_places']: # print("The shape of predicted is ---------------", predicted.shape) #classification(predicted, synset, args.store_name) #return #if task == 'vanishing_point': # _ = plot_vanishing_point_smoothed(np.squeeze(predicted), (np.squeeze(img) + 1. )/2., args.store_name, []) #return #if task == 'room_layout': # mean = np.array([0.006072743318127848, 0.010272365569691076, -3.135909774145468, # 1.5603802322235532, 5.6228218371102496e-05, -1.5669352793761442, # 5.622875878174759, 4.082800262277375, 2.7713941642895956]) # std = np.array([0.8669452525283652, 0.687915294956501, 2.080513632043758, # 0.19627420479282623, 0.014680602791251812, 0.4183827359302299, # 3.991778013006544, 2.703495278378409, 1.2269185938626304]) # predicted = predicted * std + mean # plot_room_layout(np.squeeze(predicted), (np.squeeze(img) + 1. )/2., args.store_name, [], cube_only=True) #return #if task == 'jigsaw': # predicted = np.argmax(predicted, axis=1) # perm = cfg[ 'target_dict' ][ predicted[0] ] # show_jigsaw((np.squeeze(img) + 1. )/2., perm, args.store_name) #return ############## Clean Up ############## training_runners[ 'coord' ].request_stop() training_runners[ 'coord' ].join() print("Done: {}".format(task)) ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() return
def run_to_task(task_to): import general_utils from general_utils import RuntimeDeterminedEnviromentVars import models.architectures as architectures from data.load_ops import resize_rescale_image from data.load_ops import rescale_image import utils from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction from data.task_data_loading import load_and_specify_preprocessors_for_input_depends_on_target import lib.data.load_ops as load_ops tf.logging.set_verbosity(tf.logging.ERROR) args = parser.parse_args() cfg, is_transfer, task, config_name = generate_cfg(args.config, args.vid, args) if task == 'class_places' or task == 'class_1000': synset = get_synset(task) if task == 'jigsaw': cfg['preprocess_fn'] = load_and_specify_preprocessors_for_input_depends_on_target print("Doing {task}".format(task=task)) general_utils = importlib.reload(general_utils) tf.reset_default_graph() training_runners = { 'sess': tf.InteractiveSession(), 'coord': tf.train.Coordinator() } ############## Start dataloading workers ############## if is_transfer: get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn_transfer setup_input_fn = utils.setup_input_transfer else: setup_input_fn = utils.setup_input get_data_prefetch_threads_init_fn = utils.get_data_prefetch_threads_init_fn ############## Set Up Inputs ############## # tf.logging.set_verbosity( tf.logging.INFO ) inputs = setup_input_fn(cfg, is_training=False, use_filename_queue=False) RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg) RuntimeDeterminedEnviromentVars.populate_registered_variables() start_time = time.time() ############## Set Up Model ############## model = utils.setup_model(inputs, cfg, is_training=IN_TRAIN_MODE) m = model['model'] model['saver_op'].restore(training_runners['sess'], cfg['model_path']) data_prefetch_init_fn = get_data_prefetch_threads_init_fn( inputs, cfg, is_training=False, use_filename_queue=False) prefetch_threads = threading.Thread(target=data_prefetch_init_fn, args=(training_runners['sess'], training_runners['coord'])) prefetch_threads.start() list_of_fname = np.load( '/home/ubuntu/task-taxonomy-331b/assets/aws_data/video{}_fname.npy'. format(args.vid)) import errno try: os.mkdir('/home/ubuntu/{}'.format(task)) os.mkdir('/home/ubuntu/{}/vid1'.format(task)) os.mkdir('/home/ubuntu/{}/vid2'.format(task)) os.mkdir('/home/ubuntu/{}/vid3'.format(task)) os.mkdir('/home/ubuntu/{}/vid4'.format(task)) except OSError as e: if e.errno != errno.EEXIST: raise curr_comp = np.zeros((3, 64)) curr_fit_img = np.zeros((256, 256, 3)) embeddings = [] curr_vp = [] curr_layout = [] ############## Run First Batch ############## def rescale_l_for_display(batch, rescale=True): ''' Prepares network output for display by optionally rescaling from [-1,1], and by setting some pixels to the min/max of 0/1. This prevents matplotlib from rescaling the images. ''' if rescale: display_batch = [ rescale_image(im.copy(), new_scale=[0, 100], current_scale=[-1, 1]) for im in batch ] else: display_batch = batch.copy() for im in display_batch: im[0, 0, 0] = 1.0 # Adjust some values so that matplotlib doesn't rescale im[0, 1, 0] = 0.0 # Now adjust the min return display_batch for step_num in range(inputs['max_steps'] - 1): #for step_num in range(20): #if step_num > 0 and step_num % 20 == 0: print(step_num) if is_transfer: (input_batch, target_batch, data_idx, predicted) = training_runners['sess'].run([ m.input_images, m.target_images, model['data_idxs'], m.decoder.decoder_output ]) else: (input_batch, target_batch, data_idx, predicted) = training_runners['sess'].run([ m.input_images, m.targets, model['data_idxs'], m.decoder_output ]) if task == 'segment2d' or task == 'segment25d': from sklearn.decomposition import PCA x = np.zeros((32, 256, 256, 3), dtype='float') k_embed = 8 for i in range(predicted.shape[0]): embedding_flattened = np.squeeze(predicted[i]).reshape( (-1, 64)) embeddings.append(embedding_flattened) if len(embeddings) > k_embed: embeddings.pop(0) pca = PCA(n_components=3) pca.fit(np.vstack(embeddings)) min_order = None min_dist = float('inf') copy_of_comp = np.copy(pca.components_) for order in itertools.permutations([0, 1, 2]): #reordered = pca.components_[list(order), :] #dist = np.linalg.norm(curr_comp-reordered) pca.components_ = copy_of_comp[order, :] lower_dim = pca.transform(embedding_flattened).reshape( (256, 256, -1)) lower_dim = (lower_dim - lower_dim.min()) / ( lower_dim.max() - lower_dim.min()) dist = np.linalg.norm(lower_dim - curr_fit_img) if dist < min_dist: min_order = order min_dist = dist pca.components_ = copy_of_comp[min_order, :] lower_dim = pca.transform(embedding_flattened).reshape( (256, 256, -1)) lower_dim = (lower_dim - lower_dim.min()) / (lower_dim.max() - lower_dim.min()) curr_fit_img = np.copy(lower_dim) x[i] = lower_dim predicted = x if task == 'curvature': std = [31.922, 21.658] mean = [123.572, 120.1] predicted = (predicted * std) + mean predicted[:, 0, 0, :] = 0. predicted[:, 1, 0, :] = 1. predicted = np.squeeze( np.clip(predicted.astype(int) / 255., 0., 1.)[:, :, :, 0]) if task == 'colorization': maxs = np.amax(predicted, axis=-1) softmax = np.exp(predicted - np.expand_dims(maxs, axis=-1)) sums = np.sum(softmax, axis=-1) softmax = softmax / np.expand_dims(sums, -1) kernel = np.load( '/home/ubuntu/task-taxonomy-331b/lib/data/pts_in_hull.npy') gen_target_no_temp = np.dot(softmax, kernel) images_resized = np.zeros([0, 256, 256, 2], dtype=np.float32) for image in range(gen_target_no_temp.shape[0]): temp = scipy.ndimage.zoom(np.squeeze( gen_target_no_temp[image]), (4, 4, 1), mode='nearest') images_resized = np.append(images_resized, np.expand_dims(temp, axis=0), axis=0) inp_rescale = rescale_l_for_display(input_batch) output_lab_no_temp = np.concatenate((inp_rescale, images_resized), axis=3).astype(np.float64) for i in range(input_batch.shape[0]): output_lab_no_temp[i, :, :, :] = skimage.color.lab2rgb( output_lab_no_temp[i, :, :, :]) predicted = output_lab_no_temp just_rescale = [ 'autoencoder', 'denoise', 'edge2d', 'edge3d', 'keypoint2d', 'keypoint3d', 'reshade', 'rgb2sfnorm', 'impainting_whole' ] if task in just_rescale: predicted = (predicted + 1.) / 2. predicted = np.clip(predicted, 0., 1.) predicted[:, 0, 0, :] = 0. predicted[:, 1, 0, :] = 1. just_clip = ['rgb2depth', 'rgb2mist'] if task in just_clip: predicted = np.exp(predicted * np.log(2.0**16.0)) - 1.0 predicted = np.log(predicted) / 11.09 predicted = (predicted - 0.64) / 0.18 predicted = (predicted + 1.) / 2 predicted[:, 0, 0, :] = 0. predicted[:, 1, 0, :] = 1. if task == 'segmentsemantic_rb': label = np.argmax(predicted, axis=-1) COLORS = ('white', 'red', 'blue', 'yellow', 'magenta', 'green', 'indigo', 'darkorange', 'cyan', 'pink', 'yellowgreen', 'black', 'darkgreen', 'brown', 'gray', 'purple', 'darkviolet') rgb = (input_batch + 1.) / 2. preds = [ color.label2rgb(np.squeeze(x), np.squeeze(y), colors=COLORS, kind='overlay')[np.newaxis, :, :, :] for x, y in zip(label, rgb) ] predicted = np.vstack(preds) if task in ['class_1000', 'class_places']: for file_idx, predict_output in zip(data_idx, predicted): to_store_name = list_of_fname[file_idx].decode( 'utf-8').replace('video', task) to_store_name = os.path.join('/home/ubuntu', to_store_name) sorted_pred = np.argsort(predict_output)[::-1] top_5_pred = [synset[sorted_pred[i]] for i in range(5)] to_print_pred = "Top 5 prediction: \n {}\n {}\n {}\n {} \n {}".format( *top_5_pred) img = Image.new('RGBA', (400, 200), (255, 255, 255)) d = ImageDraw.Draw(img) fnt = ImageFont.truetype( '/usr/share/fonts/truetype/dejavu/DejaVuSerifCondensed.ttf', 25) d.text((20, 5), to_print_pred, fill=(255, 0, 0), font=fnt) img.save(to_store_name, 'PNG') elif task == 'vanishing_point_well_defined': counter = 0 for file_idx, predict_output in zip(data_idx, predicted): to_store_name = list_of_fname[file_idx].decode( 'utf-8').replace('video', task) to_store_name = os.path.join('/home/ubuntu', to_store_name) curr_vp.append( plot_vanishing_point_smoothed( predict_output, (input_batch[counter] + 1.) / 2., to_store_name, curr_vp)) if len(curr_vp) > 5: curr_vp.pop(0) counter += 1 #scipy.misc.toimage(result, cmin=0.0, cmax=1.0).save(to_store_name) elif task == 'room_layout': mean = np.array([ 0.006072743318127848, 0.010272365569691076, -3.135909774145468, 1.5603802322235532, 5.6228218371102496e-05, -1.5669352793761442, 5.622875878174759, 4.082800262277375, 2.7713941642895956 ]) std = np.array([ 0.8669452525283652, 0.687915294956501, 2.080513632043758, 0.19627420479282623, 0.014680602791251812, 0.4183827359302299, 3.991778013006544, 2.703495278378409, 1.2269185938626304 ]) predicted = predicted * std + mean counter = 0 for file_idx, predict_output in zip(data_idx, predicted): to_store_name = list_of_fname[file_idx].decode( 'utf-8').replace('video', task) to_store_name = os.path.join('/home/ubuntu', to_store_name) plot_room_layout(predict_output, (input_batch[counter] + 1.) / 2., to_store_name, curr_layout, cube_only=True) curr_layout.append(predict_output) if len(curr_layout) > 5: curr_layout.pop(0) #scipy.misc.toimage(result, cmin=0.0, cmax=1.0).save(to_store_name) counter += 1 elif task == 'segmentsemantic_rb': for file_idx, predict_output in zip(data_idx, predicted): to_store_name = list_of_fname[file_idx].decode( 'utf-8').replace('video', task) to_store_name = os.path.join('/home/ubuntu', to_store_name) process_semseg_frame(predict_output, to_store_name) elif task == 'jigsaw': predicted = np.argmax(predicted, axis=1) counter = 0 for file_idx, predict_output in zip(data_idx, predicted): to_store_name = list_of_fname[file_idx].decode( 'utf-8').replace('video', task) to_store_name = os.path.join('/home/ubuntu', to_store_name) perm = cfg['target_dict'][predict_output] show_jigsaw((input_batch[counter] + 1.) / 2., perm, to_store_name) counter += 1 else: for file_idx, predict_output in zip(data_idx, predicted): to_store_name = list_of_fname[file_idx].decode( 'utf-8').replace('video', task) to_store_name = os.path.join('/home/ubuntu', to_store_name) scipy.misc.toimage(np.squeeze(predict_output), cmin=0.0, cmax=1.0).save(to_store_name) # subprocess.call('tar -czvf /home/ubuntu/{c}_{vid_id}.tar.gz /home/ubuntu/{t}/vid{vid_id}'.format( # c=config_name, t=task, vid_id=args.vid), shell=True) # subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -vcodec libx264 -crf 15 {c}_{vid_id}.mp4'.format( # c=config_name, t=task, vid_id=args.vid), shell=True) subprocess.call( 'ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -ss 00:01:54 -t 00:00:40 -c:v libvpx-vp9 -crf 10 -b:v 128k {c}_{vid_id}.webm' .format(c=config_name, t=task, vid_id=args.vid), shell=True) # subprocess.call('ffmpeg -r 29.97 -f image2 -s 256x256 -i /home/ubuntu/{t}/vid{vid_id}/0{vid_id}0%04d.png -vcodec libx264 -crf 15 -pix_fmt yuv420p {c}_{vid_id}.mp4'.format( # c=config_name, t=task, vid_id=args.vid), shell=True) subprocess.call( 'sudo mkdir -p /home/ubuntu/s3/video_new/{t}'.format(t=task), shell=True) #subprocess.call('sudo mkdir -p /home/ubuntu/s3/video_new_all/{t}'.format(t=task), shell=True) # subprocess.call('aws s3 cp /home/ubuntu/{c}_{vid_id}.tar.gz s3://task-preprocessing-512-oregon/video_new_all/{t}/'.format( # c=config_name, t=task, vid_id=args.vid), shell=True) subprocess.call( 'aws s3 cp {c}_{vid_id}.webm s3://task-preprocessing-512-oregon/video_new/{t}/' .format(c=config_name, t=task, vid_id=args.vid), shell=True) # subprocess.call('aws s3 cp /home/ubuntu/{c}_{vid_id}.tar.gz s3://taskonomy-unpacked-oregon/video_tar_all/{t}/'.format( # c=config_name, t=task, vid_id=args.vid), shell=True) # subprocess.call('aws s3 cp {c}_{vid_id}.mp4 s3://taskonomy-unpacked-oregon/video_all/{t}/'.format( # c=config_name, t=task, vid_id=args.vid), shell=True) ############## Clean Up ############## training_runners['coord'].request_stop() training_runners['coord'].join() print("Done: {}".format(config_name)) ############## Reset graph and paths ############## tf.reset_default_graph() training_runners['sess'].close() return