def build_model (): assert FLAGS.fix_width > 0 assert FLAGS.fix_height > 0 model = make_model(FLAGS.net, [FLAGS.fix_height, FLAGS.fix_width, FLAGS.channels]) model.compile(optimizer=Adam(lr=0.0001), loss='sparse_categorical_crossentropy', metrics=[acc]) return model
def calculate_metrics_for_td(self, td, batches): if td.metric_eval_timestamp != 'None': return td else: model = make_model(td.model_name, (None, None, 3)) model.load_weights(opjoin(self.nn_models_dir, td.model_file_name)) td.add_metrics(self.calculate_metrics_for_model(model, batches)) return td
def visualize_for_train_data(self, td, batch, number_to_show=4): model = make_model(td.model_name, (None, None, 3)) model.load_weights(opjoin(self.nn_models_dir, td.model_file_name)) x, y = batch pred = model.predict(x, batch_size=16) for i in range(number_to_show): image, image_parts = x[i] mask, mask_parts = y[i] pred_parts = model.predict(image_parts, batch_size=len(image_parts)) pred_parts = pred_parts.reshape(16, 16, 256, 256).swapaxes(1, 2).reshape(16 * 256, 16 * 256) pred_parts = cv2.resize(pred_parts, (256, 256)) show(image, mask, pred_parts, self.predict_threshold)
def generate_predictions(file_name, model_name, target_size, test_iter): """ Using model - model_name and saved weights in file_name, generates predictions for files in test_path """ weight = os.path.join('{}.h5'.format(file_name)) model = make_model(model_name, (target_size, target_size, 3), 1, "sigmoid") model.load_weights(weight, by_name=True) #model = load_model(weight) predictions = model.predict_generator(test_iter, steps=len(test_iter.filenames), verbose=1) return predictions
def run_parallel_map(): """ Run a test with synthetic data and MCMC inference """ # Parse command line args (options, args) = parse_cmd_line_args() # Load the data data = load_data(options) # Get a model for the data model_type = 'standard_glm' model = make_model(model_type, N=data['N']) # Get parallel clients rc = Client(profile="sge") dview = rc[:] # dview = get_engines(n_workers=8) # Load imports on the client load_imports_on_client(dview) # Initialize population objects on the clients dview.apply_sync(initialize_client, (model_type,N,data))
def run_parallel_map(): """ Run a test with synthetic data and MCMC inference """ options, popn, data, client, popn_true, x_true = initialize_parallel_test_harness() # Get the list of models for cross validation base_model = make_model(options.model, N=data['N']) models = get_xv_models(base_model) # Segment data into training and cross validation sets train_frac = 0.75 T_split = data['T'] * train_frac train_data = segment_data(data, (0,T_split)) xv_data = segment_data(data, (T_split,data['T'])) # Sample random initial state x0 = popn.sample() # Track the best model and parameters best_ind = -1 best_xv_ll = -np.Inf best_x = x0 best_model = None use_existing = True # Fit each model using the optimum of the previous models train_lls = np.zeros(len(models)) xv_lls = np.zeros(len(models)) total_lls = np.zeros(len(models)) for (i,model) in enumerate(models): print "Evaluating model %d" % i set_hyperparameters_on_engines(client[:], model) set_data_on_engines(client[:], train_data) if use_existing and \ os.path.exists(os.path.join(options.resultsDir, 'results.partial.%d.pkl' % i)): print "Found existing results for model %d" % i with open(os.path.join(options.resultsDir, 'results.partial.%d.pkl' % i)) as f: (x_inf, ll_train, ll_xv, ll_total) = cPickle.load(f) train_lls[i] = ll_train xv_lls[i] = ll_xv total_lls[i] = ll_total else: x0 = copy.deepcopy(best_x) # set_data_on_engines(client[:], train_data) ll0 = parallel_compute_ll(client[:], x0, data['N']) print "Training LL0: %f" % ll0 # Perform inference x_inf = parallel_coord_descent(client, data['N'], x0=x0, maxiter=1, use_hessian=False, use_rop=False) ll_train = parallel_compute_ll(client[:], x_inf, data['N']) print "Training LL_inf: %f" % ll_train train_lls[i] = ll_train # Compute log lkhd on xv data set_data_on_engines(client[:], xv_data) ll_xv = parallel_compute_ll(client[:], x_inf, data['N']) print "Cross Validation LL: %f" % ll_xv xv_lls[i] = ll_xv # Compute log lkhd on total dataset set_data_on_engines(client[:], data) ll_total = parallel_compute_ll(client[:], x_inf, data['N']) print "Total LL: %f" % ll_total total_lls[i] = ll_total print "Saving partial results" with open(os.path.join(options.resultsDir, 'results.partial.%d.pkl' % i),'w') as f: cPickle.dump((x_inf, ll_train, ll_xv, ll_total) ,f, protocol=-1) # Update best model if ll_xv > best_xv_ll: best_ind = i best_xv_ll = ll_xv best_x = copy.deepcopy(x_inf) best_model = copy.deepcopy(model) print "Training the best model (%d) with the full dataset" % best_ind # Set the best hyperparameters set_hyperparameters_on_engines(client[:], best_model) set_data_on_engines(client[:], data) # Fit the best model on the full training data best_x = parallel_coord_descent(client, data['N'], x0=best_x, maxiter=1, use_hessian=False, use_rop=False) # Print results summary for i in np.arange(len(models)): print "Model %d:\tTrain LL: %.1f\tXV LL: %.1f\tTotal LL: %.1f" % (i, train_lls[i], xv_lls[i], total_lls[i]) print "Best model: %d" % best_ind print "Best Total LL: %f" % parallel_compute_ll(client[:], best_x, data['N']) print "True LL: %f" % popn_true.compute_ll(x_true) # Save results with open(os.path.join(options.resultsDir, 'results.pkl'),'w') as f: cPickle.dump(best_x, f, protocol=-1) # Plot results plot_results(popn, best_x, popn_true, x_true, do_plot_imp_responses=(model['N']<64), resdir=options.resultsDir)
def one_step_forecast(test_data, opt_manager, model_name="rnf"): """ N.b. Given our main use case is for real-time data streams where speed is key, we test the RNF by running it continuously (i.e. states are not reset during operation) """ print("*** Running one-step-ahead predictions ***") # Get hyperparam stuff params = opt_manager.get_best_params() checkpoint_path = opt_manager.checkpoint_path # Param setup params['total_time_steps'] = int(1e4) params['minibatch_size'] = 1 output_size = int(params['output_size']) hidden_layer_size = int(params['hidden_layer_size']) # Setup data batcher = configs.get_batcher(model_name) col_defn = data_formatter.get_column_definition() test_batches = batcher.batch(test_data, col_defn, lookback=params['total_time_steps']) test_inputs, test_outputs, test_flags = test_batches # To make both LSTM & RNF outputs consistent if not isinstance(test_inputs, list): test_inputs = [test_inputs] batches, timesteps, _ = test_outputs.shape # Set model K.clear_session() states = [np.zeros((1, hidden_layer_size)) for _ in range(2)] model = model_factory.make_model(model_name, params, set_initial_states=True) _ = model.predict([ip[:1, ...] for ip in test_inputs] + states) model.load_weights(checkpoint_path) preds = [] for i in range(batches): print("Predicting {}/{} trajs".format(i + 1, batches)) outputs, state_h, state_c = model.predict( [ip[i:i + 1, ...] for ip in test_inputs] + states) states = [state_h, state_c] means, stds = outputs[..., :output_size], outputs[..., output_size:] preds.append(means) preds = np.concatenate(preds, axis=0) mse = np.sum( (preds - test_outputs)**2 * test_flags[..., np.newaxis]) / np.sum( test_flags[..., np.newaxis]) return mse
def run_synth_test(): """ Run a test with synthetic data and MCMC inference """ options, popn, data, client, popn_true, x_true = initialize_parallel_test_harness() raise Exception("Make sur ethe sparsity is set properly!") # If x0 specified, load x0 from file x0 = None if options.x0_file is not None: with open(options.x0_file, 'r') as f: print "Initializing with state from: %s" % options.x0_file prev_x0 = cPickle.load(f) if isinstance(prev_x0, list): x0 = prev_x0[-1] else: mle_x0 = prev_x0 # HACK: We're assuming x0 came from a standard GLM mle_model = make_model('standard_glm', N=data['N']) mle_popn = Population(mle_model) mle_popn.set_data(data) x0 = popn.sample(None) x0 = convert_model(mle_popn, mle_model, mle_x0, popn, popn.model, x0) # !!!!DEBUG!!!!! # Initialize with true variables # import copy # x0 = copy.deepcopy(x_true) use_existing = False if use_existing and \ os.path.exists(os.path.join(options.resultsDir, 'results.pkl')): print "Found existing results" with open(os.path.join(options.resultsDir, 'results.pkl')) as f: x_smpls = cPickle.load(f) N_samples = len(x_smpls) else: N_samples = 1000 # Define a callback to evaluate log likelihoods and predictive log likelihoods print "Creating synthetic test data" T_test = 15 popn_test = Population(popn.model) test_data = gen_synth_data(data['N'], T_test, popn_true, x_true) popn_test.set_data(test_data) # Compute pred ll under true model popn_true.set_data(test_data) x_true['predll'] = popn_true.compute_ll(x_true) popn_true.set_data(data) # Compute the predictive log likelihood under a homogeneous PP model wiht MLE # homog_pred_lls[j] = compute_homog_pp(train_data, test_data) pred_lls = np.zeros(N_samples) smpl = [0] def pred_ll_cbk(x): pred_ll = popn_test.compute_ll(x) pred_lls[smpl[0]] = pred_ll x['predll'] = pred_ll smpl[0] += 1 print "Pred LL: %.2f" % pred_ll pred_ll_cbk = None # Perform inference print "Performing parallel inference" start_time = time.time() x_smpls = parallel_gibbs_sample(client, data, x0=x0, N_samples=N_samples, save_interval=50, results_dir=options.resultsDir, callback=pred_ll_cbk) stop_time = time.time() # Save results print "Saving results to %s" % os.path.join(options.resultsDir, 'results.pkl') with open(os.path.join(options.resultsDir, 'results.pkl'),'w') as f: cPickle.dump(x_smpls, f, protocol=-1) # Save runtime with open(os.path.join(options.resultsDir, 'runtime.pkl'),'w') as f: cPickle.dump(stop_time-start_time, f, protocol=-1) # Plot average of last 20% of samples print "Plotting results" smpl_frac = 1.0 # Only plot the impulse response matrix for small N do_plot = data['N'] < 20 do_plot_imp_responses = data['N'] < 30 if do_plot: plot_results(popn, x_smpls[-1*int(smpl_frac*len(x_smpls)):], popn_true, x_true, do_plot_imp_responses=do_plot_imp_responses, resdir=options.resultsDir)
def load_set_of_results(N, T, graph_model='er', sample_frac=0.1): data_dir = os.path.join('/group', 'hips', 'scott', 'pyglm', 'data', 'synth', graph_model, 'N%dT%d' % (N, T)) # Evaluate the state for each of the parameter settings s_infs_mcmc = [] s_infs_map = [] s_trues = [] # Enumerate the subdirectories containing the data subdirs = os.listdir(data_dir) subdirs = reduce(lambda sd, d: sd + [d] \ if os.path.isdir(os.path.join(data_dir, d)) \ else sd, subdirs, []) # For each data subdirectory, load the true data, the MAP estimate, and the MCMC results print "WARNING: Make sure we sample all subdirs" # import pdb; pdb.set_trace() for d in subdirs: print "Loading data and results from %s" % d print "Loading true data" with open(os.path.join(data_dir, d, 'data.pkl'), 'r') as f: data = cPickle.load(f) print "Loading model" with open(os.path.join(data_dir, d, 'model.pkl'), 'r') as f: model_data = cPickle.load(f) #HACK if 'N_dims' not in model_data['network']['graph']: model_data['network']['graph']['N_dims'] = 1 if 'location_prior' not in model_data['network']['graph']: model_data['network']['graph']['location_prior'] = \ { 'type' : 'gaussian', 'mu' : 0.0, 'sigma' : 1.0 } if 'L' in data['vars']['net']['graph']: data['vars']['net']['graph']['L'] = data['vars']['net']['graph']['L'].ravel() popn_data = Population(model_data) popn_data.set_data(data) s_trues.append(popn_data.eval_state(data['vars'])) try: print "Loading map estimate" with open(os.path.join(data_dir, d, 'map', 'results.pkl'), 'r') as f: x_map = cPickle.load(f) model_map = make_model('standard_glm', N=data['N']) popn_map = Population(model_map) popn_map.set_data(data) print "Evaluating MAP state" s_infs_map.append(popn_map.eval_state(x_map)) except Exception as e: print "ERROR: Failed to load MAP estimate" try: print "Loading mcmc estimate" with open(os.path.join(data_dir, d, 'mcmc', 'results.pkl'), 'r') as f: x_mcmc = cPickle.load(f) model_mcmc = make_model('sparse_weighted_model', N=data['N']) popn_mcmc = Population(model_mcmc) popn_mcmc.set_data(data) # Now compute the true and false positive rates for MCMC # For MCMC results, only consider the tail of the samples print "Evaluating MCMC states" N_samples = len(x_mcmc) start_smpl = int(np.floor(N_samples - sample_frac*N_samples)) # Evaluate the state this_s_mcmc = [] for i in range(start_smpl, N_samples): this_s_mcmc.append(popn_mcmc.eval_state(x_mcmc[i])) s_infs_mcmc.append(this_s_mcmc) except Exception as e: print "ERROR: Failed to load MCMC estimate" return s_trues, s_infs_map, s_infs_mcmc
def main(args): #initialize Horovod. hvd.init() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.visible_device_list = str(hvd.local_rank()) K.set_session(tf.Session(config=config)) fold = args.data_path.split("fold_")[1] if hvd.rank()==0: print("================================") if args.use_lovasz: print("Fine tuning with ") print("Fold {}".format(fold)) #Find best saved model best_model_file = 'weights/{}/fold_{}_{epoch}_best.h5'.format(args.model, fold, epoch='{epoch}') resume_from_epoch = 0 for try_epoch in range(args.epochs, 0, -1): if os.path.exists(best_model_file.format(epoch=try_epoch)): resume_from_epoch = try_epoch break if hvd.rank()==0: print("Last model saved: {}".format(best_model_file.format(epoch=resume_from_epoch))) resume_from_epoch = hvd.broadcast(resume_from_epoch, 0, name='resume_from_epoch') #verbose mode for one node if hvd.rank()==0: verbose = 1 else: verbose = 0 #Create dataset dataset = TGSDataset(data_path=args.data_path, batch_size=args.batch_size) input_shape = (args.target_size, args.target_size) mask_shape = (101, 101) train_data_generator = dataset.get_train_data_generator(input_size=input_shape, mask_size=mask_shape, seed=np.random.rand()) val_data_generator = dataset.get_val_data_generator(input_size=input_shape, mask_size=mask_shape, seed=np.random.rand()) train_step_size = dataset.train_step_size // hvd.size() val_step_size = dataset.val_step_size // hvd.size() #Create model model = make_model(args.model, (args.target_size, args.target_size, 3), 2) #load weights if resume_from_epoch > 0: model.load_weights(best_model_file.format(epoch=resume_from_epoch)) size = hvd.size() opt = hvd.DistributedOptimizer(SGD(lr=args.learning_rate * size, momentum=0.9, nesterov=True)) #Loss loss = losses.c_lovasz_loss if args.use_lovasz else losses.c_binary_crossentropy model.compile(loss=loss, optimizer=opt, metrics=[metrics.c_binary_accuracy, metrics.c_iou]) #h5 model best_model = ModelCheckpointMGPU(model, filepath=best_model_file, monitor='val_loss', verbose=1, mode='min', period=1, save_best_only=True, save_weights_only=True) callbacks = [ # Horovod: broadcast initial variable states from rank 0 to all other processes. # This is necessary to ensure consistent initialization of all workers when # training is started with random weights or restored from a checkpoint. hvd.callbacks.BroadcastGlobalVariablesCallback(0), # Horovod: average metrics among workers at the end of every epoch. # # Note: This callback must be in the list before the ReduceLROnPlateau, # TensorBoard, or other metrics-based callbacks. hvd.callbacks.MetricAverageCallback(), # Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final # accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during # the first five epochs. See https://arxiv.org/abs/1706.02677 for details. hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=args.warmup_epochs, verbose=True) ] # Horovod: save checkpoints only on the first worker to prevent other workers from corrupting them. if hvd.rank() == 0: callbacks.append(keras.callbacks.TensorBoard(args.log_dir)) callbacks.append(best_model) #Fit model history = model.fit_generator(train_data_generator, steps_per_epoch=train_step_size, callbacks=callbacks, epochs=args.epochs, verbose=verbose, workers=4, initial_epoch=resume_from_epoch, validation_data=val_data_generator, validation_steps=val_step_size) score = hvd.allreduce(model.evaluate_generator(val_data_generator, val_step_size, workers=4)) print('Test loss:', score[0]) print('Test accuracy:', score[1])
valid_inputs, valid_outputs, valid_flags = valid_batches test_inputs, test_outputs, test_flags = test_batches while len(opt_manager.results.columns) < hyperparam_iterations: K.clear_session() print() print("# -----------------------------------------------------------") print("# Running hyperparam optimisation {} of {} for {}".format( len(opt_manager.results.columns) + 1, hyperparam_iterations, expt_name)) print("# -----------------------------------------------------------") print() params = opt_manager.get_next_parameters() model = model_factory.make_model(model_name, params) tmp_folder = os.path.join(model_folder, 'tmp') tmp_model = os.path.join(tmp_folder, "model") callbacks = [ tf.keras.callbacks.EarlyStopping( monitor='val_loss', patience=early_stopping, min_delta=1e-4), tf.keras.callbacks.ModelCheckpoint( filepath=tmp_model, monitor='val_loss', save_best_only=True, save_weights_only=True), tf.keras.callbacks.TerminateOnNaN()
def postprocess(popn, x_inf, popn_true, x_true, options): """ Compute an ROC curve from a set of inferred samples and the true state """ true_state = popn_true.eval_state(x_true) # Make sure we have a list of x's if not isinstance(x_inf, list): x_inf = [x_inf] inf_state = [popn.eval_state(x) for x in x_inf] # Check if the inference model is a standard GLM or a network GLM if # Now compute the true and false positive rates for MAP (map_tpr, map_fpr) = compute_roc_from_std_glm(true_state, map_state) map_tprs.append(map_tpr) map_fprs.append(map_fpr) print "Loading mcmc estimate" x_mcmc = None with open(os.path.join(data_dir, d, 'mcmc', 'results.pkl'), 'r') as f: x_mcmc = cPickle.load(f) model_mcmc = make_model('sparse_weighted_model', N=data['N']) popn_mcmc = Population(model_mcmc) popn_mcmc.set_data(data) # Evaluate the state mcmc_state = [] for x in x_mcmc: mcmc_state.append(popn_mcmc.eval_state(x)) # Now compute the true and false positive rates for MCMC # For MCMC results, only consider the tail of the samples N_samples = len(mcmc_state) sample_frac = 0.2 start_smpl = int(np.floor(N_samples - sample_frac*N_samples)) (mcmc_tpr, mcmc_fpr) = compute_roc_from_sparse_glm_smpls(true_state, mcmc_state[start_smpl:]) mcmc_tprs.append(mcmc_tpr) mcmc_fprs.append(mcmc_fpr) # Pickle the roc results with open(PKL_FNAME, 'w') as f: # TODO Dump the MCMC results too cPickle.dump({'map_tprs' : map_tprs, 'map_fprs' : map_fprs}, f, protocol=-1) # Plot the actual ROC curve # Subsample to get about 10 errorbars subsample = N*N//10 f = plt.figure() ax = f.add_subplot(111) plot_roc_curve(map_tprs, map_fprs, ax=ax, color='b', subsample=subsample) # plot_roc_curve(mcmc_tprs, mcmc_fprs, ax=ax, color='r', subsample=subsample) fname = os.path.join(PLOTDIR,'roc_N%dT%d.pdf' % (N,T)) print "Saving ROC to %s" % fname f.savefig(fname) plt.close(f)
def run_synth_test(): """ Run a test with synthetic data and MCMC inference """ options, popn, data, client, popn_true, x_true = initialize_parallel_test_harness() # If x0 specified, load x0 from file x0 = None if options.x0_file is not None: with open(options.x0_file, 'r') as f: print "Initializing with state from: %s" % options.x0_file prev_x0 = cPickle.load(f) if isinstance(prev_x0, list): x0 = prev_x0[-1] else: mle_x0 = prev_x0 # HACK: We're assuming x0 came from a standard GLM mle_model = make_model('standard_glm', N=data['N']) mle_popn = Population(mle_model) mle_popn.set_data(data) x0 = popn.sample() x0 = convert_model(mle_popn, mle_model, mle_x0, popn, popn.model, x0) use_existing = False if use_existing and \ os.path.exists(os.path.join(options.resultsDir, 'results.pkl')): print "Found existing results" with open(os.path.join(options.resultsDir, 'results.pkl')) as f: x_smpls = cPickle.load(f) N_samples = len(x_smpls) else: # Perform inference print "Performing parallel inference" N_samples = 1000 x_smpls = parallel_gibbs_sample(client, data, x0=x0, N_samples=N_samples, save_interval=50, results_dir=options.resultsDir) # Save results print "Saving results to %s" % os.path.join(options.resultsDir, 'results.pkl') with open(os.path.join(options.resultsDir, 'results.pkl'),'w') as f: cPickle.dump(x_smpls, f, protocol=-1) # Plot average of last 20% of samples print "Plotting results" smpl_frac = 0.5 # Only plot the impulse response matrix for small N do_plot = data['N'] < 20 do_plot_imp_responses = data['N'] < 30 if do_plot: plot_results(popn, x_smpls[-1*int(smpl_frac*N_samples):], popn_true, x_true, do_plot_imp_responses=do_plot_imp_responses, resdir=options.resultsDir)
def run_synth_test(): """ Run a test with synthetic data and MAP inference with cross validation """ options, popn, data, popn_true, x_true = initialize_test_harness() # Get the list of models for cross validation base_model = make_model(options.model, N=data['N']) models = get_xv_models(base_model) # TODO Segment data into training and cross validation sets train_frac = 0.75 T_split = data['T'] * train_frac train_data = segment_data(data, (0,T_split)) xv_data = segment_data(data, (T_split,data['T'])) # Sample random initial state x0 = popn.sample() # Track the best model and parameters best_ind = -1 best_xv_ll = -np.Inf best_x = x0 best_model = None # Fit each model using the optimum of the previous models train_lls = np.zeros(len(models)) xv_lls = np.zeros(len(models)) total_lls = np.zeros(len(models)) for (i,model) in enumerate(models): print "Training model %d" % i x0 = copy.deepcopy(best_x) popn.set_hyperparameters(model) popn.set_data(train_data) ll0 = popn.compute_log_p(x0) print "Training LL0: %f" % ll0 # Perform inference x_inf = coord_descent(popn, data, x0=x0, maxiter=1, use_hessian=False, use_rop=False) ll_train = popn.compute_log_p(x_inf) print "Training LL_inf: %f" % ll_train train_lls[i] = ll_train # Compute log lkhd on xv data popn.set_data(xv_data) ll_xv = popn.compute_ll(x_inf) print "Cross Validation LL: %f" % ll_xv xv_lls[i] = ll_xv # Compute log lkhd on total dataset popn.set_data(data) ll_total = popn.compute_ll(x_inf) print "Tota LL: %f" % ll_total total_lls[i] = ll_total # Update best model if ll_xv > best_xv_ll: best_ind = i best_xv_ll = ll_xv best_x = copy.deepcopy(x_inf) best_model = copy.deepcopy(model) # Create a population with the best model popn.set_hyperparameters(best_model) popn.set_data(data) # Fit the best model on the full training data best_x = coord_descent(popn, data, x0=x0, maxiter=1, use_hessian=False, use_rop=False) # Print results summary for i in np.arange(len(models)): print "Model %d:\tTrain LL: %.1f\tXV LL: %.1f\tTotal LL: %.1f" % (i, train_lls[i], xv_lls[i], total_lls[i]) print "Best model: %d" % best_ind print "Best Total LL: %f" % popn.compute_ll(best_x) print "True LL: %f" % popn_true.compute_ll(x_true) # Save results results_file = os.path.join(options.resultsDir, 'results.pkl') print "Saving results to %s" % results_file with open(results_file, 'w') as f: cPickle.dump(best_x, f) # Plot results plot_results(popn, best_x, popn_true, x_true, resdir=options.resultsDir)
if __name__ == '__main__': logging.basicConfig( format='%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s', datefmt='%d-%b-%y %H:%M:%S') logger = logging.getLogger("main") logger.setLevel(logging.INFO) t0 = timeit.default_timer() logger.info("Loading model weights") weights = [path.join(args.models_dir, m) for m in args.models] models = [] num_tiles = int(args.num_tiles) for w in weights: model = make_model(args.network, (None, None, 3)) logger.info("Building model {} from weights {} ".format( args.network, w)) model.load_weights(w) models.append(model) os.makedirs(test_pred, exist_ok=True) #print('Predicting test') for d in tqdm(listdir(test_folder)): logger.info("Predicting Image: {}".format(d)) fid = d full_img = cv2.imread(path.join(test_folder, fid), cv2.IMREAD_COLOR)[..., ::-1] if num_tiles == None: num_tiles = 1
def gibbs_sample(population, data, N_samples=1000, x0=None, init_from_mle=True): """ Sample the posterior distribution over parameters using MCMC. """ N = population.model['N'] # Draw initial state from prior if not given if x0 is None: x0 = population.sample() if init_from_mle: print "Initializing with coordinate descent" from models.model_factory import make_model, convert_model from population import Population mle_model = make_model('standard_glm', N=N) mle_popn = Population(mle_model) mle_popn.set_data(data) mle_x0 = mle_popn.sample() # Initialize with MLE under standard GLM mle_x0 = coord_descent(mle_popn, data, x0=mle_x0, maxiter=1) # Convert between inferred parameters of the standard GLM # and the parameters of this model. Eg. Convert unweighted # networks to weighted networks with normalized impulse responses. x0 = convert_model(mle_popn, mle_model, mle_x0, population, population.model, x0) # Create updates for this population serial_updates, parallel_updates = initialize_updates(population) # DEBUG Profile the Gibbs sampling loop import cProfile, pstats, StringIO pr = cProfile.Profile() pr.enable() # Alternate fitting the network and fitting the GLMs x_smpls = [x0] x = x0 import time start_time = time.clock() for smpl in np.arange(N_samples): # Print the current log likelihood lp = population.compute_log_p(x) # Compute iters per second stop_time = time.clock() if stop_time - start_time == 0: print "Gibbs iteration %d. Iter/s exceeds time resolution. Log prob: %.3f" % (smpl, lp) else: print "Gibbs iteration %d. Iter/s = %f. Log prob: %.3f" % (smpl, 1.0/(stop_time-start_time), lp) start_time = stop_time # Go through each parallel MH update for parallel_update in parallel_updates: for n in np.arange(N): parallel_update.update(x, n) # Sample the serial updates for serial_update in serial_updates: serial_update.update(x) x_smpls.append(copy.deepcopy(x)) pr.disable() s = StringIO.StringIO() sortby = 'cumulative' ps = pstats.Stats(pr, stream=s).sort_stats(sortby) ps.print_stats() with open('mcmc.prof.txt', 'w') as f: f.write(s.getvalue()) f.close() return x_smpls
def main(data_path='/data/SN6_buildings/train/AOI_11_Rotterdam/', config_path='/project/configs/senet154_gcc_fold1.py', gpu='0'): os.environ["CUDA_VISIBLE_DEVICES"] = gpu config = get_config(config_path) model_name = config['model_name'] fold_number = config['fold_number'] alias = config['alias'] log_path = osp.join(config['logs_path'], alias + str(fold_number) + '_' + model_name) device = torch.device(config['device']) weights = config['weights'] loss_name = config['loss'] optimizer_name = config['optimizer'] lr = config['lr'] decay = config['decay'] momentum = config['momentum'] epochs = config['epochs'] fp16 = config['fp16'] n_classes = config['n_classes'] input_channels = config['input_channels'] main_metric = config['main_metric'] best_models_count = config['best_models_count'] minimize_metric = config['minimize_metric'] min_delta = config['min_delta'] train_images = data_path data_type = config['data_type'] masks_data_path = config['masks_data_path'] folds_file = config['folds_file'] train_augs = config['train_augs'] preprocessing_fn = config['preprocessing_fn'] limit_files = config['limit_files'] batch_size = config['batch_size'] shuffle = config['shuffle'] num_workers = config['num_workers'] valid_augs = config['valid_augs'] val_batch_size = config['val_batch_size'] multiplier = config['multiplier'] train_dataset = SemSegDataset(images_dir=train_images, data_type=data_type, masks_dir=masks_data_path, mode='train', n_classes=n_classes, folds_file=folds_file, fold_number=fold_number, augmentation=train_augs, preprocessing=preprocessing_fn, limit_files=limit_files, multiplier=multiplier) train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) valid_dataset = SemSegDataset(images_dir=train_images, data_type=data_type, mode='valid', folds_file=folds_file, n_classes=n_classes, fold_number=fold_number, augmentation=valid_augs, preprocessing=preprocessing_fn, limit_files=limit_files) valid_loader = DataLoader(dataset=valid_dataset, batch_size=val_batch_size, shuffle=False, num_workers=num_workers) model = make_model(model_name=model_name, weights=weights, n_classes=n_classes, input_channels=input_channels).to(device) loss = get_loss(loss_name=loss_name) optimizer = get_optimizer(optimizer_name=optimizer_name, model=model, lr=lr, momentum=momentum, decay=decay) if config['scheduler'] == 'reduce_on_plateau': print('reduce lr') alpha = config['alpha'] patience = config['patience'] threshold = config['thershold'] min_lr = config['min_lr'] mode = config['scheduler_mode'] scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer=optimizer, factor=alpha, verbose=True, patience=patience, mode=mode, threshold=threshold, min_lr=min_lr) elif config['scheduler'] == 'steps': print('steps lr') steps = config['steps'] step_gamma = config['step_gamma'] scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer=optimizer, milestones=steps, gamma=step_gamma) else: scheduler = None callbacks = [] dice_callback = DiceCallback() callbacks.append(dice_callback) callbacks.append(CheckpointCallback(save_n_best=best_models_count)) callbacks.append( EarlyStoppingCallback(patience=config['early_stopping'], metric=main_metric, minimize=minimize_metric, min_delta=min_delta)) runner = SupervisedRunner(device=device) loaders = {'train': train_loader, 'valid': valid_loader} runner.train(model=model, criterion=loss, optimizer=optimizer, loaders=loaders, scheduler=scheduler, callbacks=callbacks, logdir=log_path, num_epochs=epochs, verbose=True, main_metric=main_metric, minimize_metric=minimize_metric, fp16=fp16)
def main(config_path='/project/configs/senet154_gcc_fold1.py', test_images='/data/SN6_buildings/test_public/AOI_11_Rotterdam/', test_predict_result='/wdata/folds_predicts/', batch_size=1, workers=1, gpu='1'): with torch.no_grad(): config = get_config(config_path) model_name = config['model_name'] weights_path = config['load_from'] device = config['device'] val_batch_size = batch_size input_channels = config['input_channels'] original_size = config['original_size'] cropper = albu.Compose( [albu.CenterCrop(original_size[0], original_size[1], p=1.0)]) n_classes = config['n_classes'] preprocessing_fn = config['preprocessing_fn'] valid_augs = config['valid_augs'] limit_files = config['limit_files'] num_workers = workers os.environ["CUDA_VISIBLE_DEVICES"] = gpu if not os.path.exists(test_predict_result): os.mkdir(test_predict_result) fold_name = weights_path.split('/')[-3] folder_to_save = os.path.join(test_predict_result, fold_name) if os.path.exists(folder_to_save): shutil.rmtree(folder_to_save) os.mkdir(folder_to_save) test_dataset = TestSemSegDataset(images_dir=os.path.join( test_images, 'SAR-Intensity'), preprocessing=preprocessing_fn, augmentation=valid_augs, limit_files=limit_files) test_loader = DataLoader(dataset=test_dataset, batch_size=val_batch_size, shuffle=False, num_workers=num_workers) print('Loading {}'.format(weights_path)) model = make_model(model_name=model_name, weights=None, n_classes=n_classes, input_channels=input_channels).to(device) model.load_state_dict(torch.load(weights_path)['model_state_dict']) model.eval() model = tta.TTAWrapper(model, flip_image2mask) model = torch.nn.DataParallel(model).cuda() file_names = sorted(test_dataset.ids) for batch_i, test_batch in enumerate(tqdm(test_loader)): runner_out = model(test_batch.cuda()) image_pred = runner_out image_pred = image_pred.cpu().detach().numpy() names = file_names[batch_i * val_batch_size:(batch_i + 1) * val_batch_size] for i in range(len(names)): file_name = os.path.join(folder_to_save, names[i].split('.')[0] + '.png') data = image_pred[i, ...] data = np.moveaxis(data, 0, -1) sample = cropper(image=data) data = sample['image'] data = (data * 255).astype(np.uint8) cv2.imwrite(file_name, data)
def main(args): num_of_folds = len(os.listdir(args.data_path)) print("Found {} number of folds".format(num_of_folds)) for fold_index in range(num_of_folds): print("================================") print("Starting fold {}".format(fold_index)) #Create dataset dataset = TGSDataset(data_path="{}/fold_{}".format( args.data_path, fold_index), batch_size=args.batch_size) input_shape = (args.target_size, args.target_size) mask_shape = (101, 101) train_data_generator = dataset.get_train_data_generator( input_size=input_shape, mask_size=mask_shape, seed=args.seed) val_data_generator = dataset.get_val_data_generator( input_size=input_shape, mask_size=mask_shape, seed=args.seed) #Find best saved model best_model_file = 'weights/{}/fold_{}_{epoch}_best.h5'.format( args.model, fold_index, epoch='{epoch}') resume_from_epoch = 0 for try_epoch in range(args.epochs, 0, -1): if os.path.exists(best_model_file.format(epoch=try_epoch)): resume_from_epoch = try_epoch break if resume_from_epoch > 0: print("Resuming from epoch {}".format(resume_from_epoch)) model = load_model(best_model_file.format(epoch=resume_from_epoch), custom_objects={'c_iou': metrics.c_iou}) else: model = make_model(args.model, (args.target_size, args.target_size, 3), 1) #Optimizer opt = adam(lr=args.learning_rate) #Compile model model.compile(loss=binary_crossentropy, optimizer=opt, metrics=[binary_accuracy, metrics.c_iou]) #Keras callbacks callbacks = [ keras.callbacks.TensorBoard(args.log_dir), keras.callbacks.ModelCheckpoint(best_model_file, save_best_only=True, save_weights_only=False), keras.callbacks.EarlyStopping(monitor='c_iou', patience=20, verbose=0, mode='max') ] train_step_size = dataset.train_step_size val_step_size = dataset.val_step_size history = model.fit_generator(train_data_generator, steps_per_epoch=train_step_size, callbacks=callbacks, epochs=args.epochs, verbose=args.v, workers=4, initial_epoch=resume_from_epoch, validation_data=val_data_generator, validation_steps=val_step_size) #Load weights resume_from_epoch = 0 for try_epoch in range(args.epochs, 0, -1): if os.path.exists(best_model_file.format(epoch=try_epoch)): resume_from_epoch = try_epoch break if resume_from_epoch > 0: print("Resuming from epoch {}".format(resume_from_epoch)) model_with_lovasz = load_model( best_model_file.format(epoch=resume_from_epoch), custom_objects={"c_iou": metrics.c_iou}) else: model_with_lovasz = make_model( args.model, (args.target_size, args.target_size, 3), 1) #Lovasz Loss #Optimizer #Keras callbacks callbacks = [ keras.callbacks.TensorBoard(args.log_dir), keras.callbacks.ModelCheckpoint(best_model_file, save_best_only=True, save_weights_only=False), keras.callbacks.EarlyStopping(monitor='c_iou_zero', mode='max', patience=20, verbose=0) ] train_data_generator = dataset.get_train_data_generator( input_size=input_shape, mask_size=mask_shape, seed=args.seed) val_data_generator = dataset.get_val_data_generator( input_size=input_shape, mask_size=mask_shape, seed=args.seed) model_with_lovasz = Model(model_with_lovasz.layers[0].input, model_with_lovasz.layers[-1].input) opt = adam(lr=args.learning_rate) model_with_lovasz.compile( loss=losses.c_lovasz_loss, optimizer=opt, metrics=[binary_accuracy, metrics.c_iou_zero]) print("Fine tuning with lovasz loss") model_with_lovasz.fit_generator(train_data_generator, steps_per_epoch=train_step_size, callbacks=callbacks, epochs=args.epochs, verbose=args.v, workers=4, initial_epoch=resume_from_epoch, validation_data=val_data_generator, validation_steps=val_step_size) # Evaluate the model on the validation data set. score = model_with_lovasz.evaluate_generator(val_data_generator, val_step_size) print('Test loss:', score[0]) print('Test accuracy:', score[1])
def fit_latent_network_to_mle(): """ Run a test with synthetic data and MCMC inference """ options, popn, data, popn_true, x_true = initialize_test_harness() import pdb; pdb.set_trace() # Load MLE parameters from command line mle_x = None if options.x0_file is not None: with open(options.x0_file, 'r') as f: print "Initializing with state from: %s" % options.x0_file mle_x = cPickle.load(f) mle_model = make_model('standard_glm', N=data['N']) mle_popn = Population(mle_model) mle_popn.set_data(data) # Create a location sampler print "Initializing latent location sampler" loc_sampler = LatentDistanceNetworkUpdate() loc_sampler.preprocess(popn) # Convert the mle results into a weighted adjacency matrix x_aw = popn.sample() x_aw = convert_model(mle_popn, mle_model, mle_x, popn, popn.model, x_aw) # Get rid of unnecessary keys del x_aw['glms'] # Fit the latent distance network to a thresholded adjacency matrix ws = np.sort(np.abs(x_aw['net']['weights']['W'])) wperm = np.argsort(np.abs(x_aw['net']['weights']['W'])) nthrsh = 20 threshs = np.arange(ws.size, step=ws.size/nthrsh) res = [] N = popn.N for th in threshs: print "Fitting network for threshold: %.3f" % th A = np.zeros_like(ws, dtype=np.int8) A[wperm[th:]] = 1 A = A.reshape((N,N)) # A = (np.abs(x_aw['net']['weights']['W']) >= th).astype(np.int8).reshape((N,N)) # Make sure the diag is still all 1s A[np.diag_indices(N)] = 1 x = copy.deepcopy(x_aw) x['net']['graph']['A'] = A smpls = fit_latent_network_given_A(x, loc_sampler) # Index the results by the overall sparsity of A key = (np.sum(A)-N) / (np.float(np.size(A))-N) res.append((key, smpls)) # Save results results_file = os.path.join(options.resultsDir, 'fit_latent_network_results.pkl') print "Saving results to %s" % results_file with open(results_file, 'w') as f: cPickle.dump(res, f)
def main(): if args.crop_size: print('Using crops of shape ({}, {})'.format(args.crop_size, args.crop_size)) else: print('Using full size images') folds = [int(f) for f in args.fold.split(",")] for fold in folds: channels = 3 if args.multi_gpu: with K.tf.device("/cpu:0"): model = make_model(args.network, (None, None, 3)) else: model = make_model(args.network, (None, None, channels)) if args.weights is None: print('No weights passed, training from scratch') else: weights_path = args.weights.format(fold) print('Loading weights from {}'.format(weights_path)) model.load_weights(weights_path, by_name=True) freeze_model(model, args.freeze_till_layer) optimizer = RMSprop(lr=args.learning_rate) if args.optimizer: if args.optimizer == 'rmsprop': optimizer = RMSprop(lr=args.learning_rate, decay=float(args.decay)) elif args.optimizer == 'adam': optimizer = Adam(lr=args.learning_rate, decay=float(args.decay)) elif args.optimizer == 'amsgrad': optimizer = Adam(lr=args.learning_rate, decay=float(args.decay), amsgrad=True) elif args.optimizer == 'sgd': optimizer = SGD(lr=args.learning_rate, momentum=0.9, nesterov=True, decay=float(args.decay)) dataset = DSB2018BinaryDataset(args.images_dir, args.masks_dir, args.labels_dir, fold, args.n_folds, seed=args.seed) random_transform = aug_mega_hardcore() train_generator = dataset.train_generator( (args.crop_size, args.crop_size), args.preprocessing_function, random_transform, batch_size=args.batch_size) val_generator = dataset.val_generator(args.preprocessing_function, batch_size=1) best_model_file = '{}/best_{}{}_fold{}.h5'.format( args.models_dir, args.alias, args.network, fold) best_model = ModelCheckpointMGPU(model, filepath=best_model_file, monitor='val_loss', verbose=1, mode='min', period=args.save_period, save_best_only=True, save_weights_only=True) last_model_file = '{}/last_{}{}_fold{}.h5'.format( args.models_dir, args.alias, args.network, fold) last_model = ModelCheckpointMGPU(model, filepath=last_model_file, monitor='val_loss', verbose=1, mode='min', period=args.save_period, save_best_only=False, save_weights_only=True) if args.multi_gpu: model = multi_gpu_model(model, len(gpus)) model.compile( loss=make_loss(args.loss_function), optimizer=optimizer, metrics=[binary_crossentropy, hard_dice_coef_ch1, hard_dice_coef]) def schedule_steps(epoch, steps): for step in steps: if step[1] > epoch: print("Setting learning rate to {}".format(step[0])) return step[0] print("Setting learning rate to {}".format(steps[-1][0])) return steps[-1][0] callbacks = [best_model, last_model] if args.schedule is not None: steps = [(float(step.split(":")[0]), int(step.split(":")[1])) for step in args.schedule.split(",")] lrSchedule = LearningRateScheduler( lambda epoch: schedule_steps(epoch, steps)) callbacks.insert(0, lrSchedule) tb = TensorBoard("logs/{}_{}".format(args.network, fold)) callbacks.append(tb) steps_per_epoch = len(dataset.train_ids) / args.batch_size + 1 if args.steps_per_epoch > 0: steps_per_epoch = args.steps_per_epoch validation_data = val_generator validation_steps = len(dataset.val_ids) model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=args.epochs, validation_data=validation_data, validation_steps=validation_steps, callbacks=callbacks, max_queue_size=5, verbose=1, workers=args.num_workers) del model K.clear_session() gc.collect()
def run_gen_synth_data(): """ Run a test with synthetic data and MCMC inference """ options, args = parse_cmd_line_args() # Create the model model = make_model(options.model, N=options.N) # Set the sparsity level to minimize the risk of unstable networks stabilize_sparsity(model) print "Creating master population object" popn = Population(model) # Sample random parameters from the model x_true = popn.sample() # Check stability of matrix assert check_stability(model, x_true, options.N), "ERROR: Sampled network is unstable!" # Save the model so it can be loaded alongside the data fname_model = os.path.join(options.resultsDir, 'model.pkl') print "Saving data to %s" % fname_model with open(fname_model,'w') as f: cPickle.dump(model, f, protocol=-1) print "Generating synthetic data with %d neurons and %.2f seconds." % \ (options.N, options.T_stop) # Set simulation parametrs dt = 0.001 dt_stim = 0.1 D_stim = 1 stim = np.random.randn(options.T_stop/dt_stim, D_stim) data = gen_synth_data(options.N, options.T_stop, popn, x_true, dt, dt_stim, D_stim, stim) # Set the data so that the population state can be evaluated popn.set_data(data) # DEBUG Evaluate the firing rate and the simulated firing rate state = popn.eval_state(x_true) for n in np.arange(options.N): lam_true = state['glms'][n]['lam'] lam_sim = popn.glm.nlin_model.f_nlin(data['X'][:,n]) assert np.allclose(lam_true, lam_sim) # Save the data for reuse #fname_mat = os.path.join(options.resultsDir, 'data.mat') #print "Saving data to %s" % fname_mat #scipy.io.savemat(fname_mat, data, oned_as='row') # Pickle the data so we can open it more easily fname_pkl = os.path.join(options.resultsDir, 'data.pkl') print "Saving data to %s" % fname_pkl with open(fname_pkl,'w') as f: cPickle.dump(data, f, protocol=-1) # Plot firing rates, stimulus responses, etc plot_results(popn, data['vars'], resdir=options.resultsDir, do_plot_stim_resp=False, do_plot_imp_responses=False)