def submission(): """ Generate submission file for the trained models. """ print('Loading and compiling models...') model_systole = get_model() model_diastole = get_model() print('Loading models weights...') model_systole.load_weights('weights_systole23.hdf5') model_diastole.load_weights('weights_diastole50.hdf5') # load val losses to use as sigmas for CDF with open('val_loss.txt', mode='r') as f: val_loss_systole = float(f.readline()) val_loss_diastole = float(f.readline()) print('Loading validation data...') X, ids = load_validation_data() #print('Pre-processing images...') #X = preprocess(X) batch_size = 32 print('Predicting on validation data...') pred_systole = model_systole.predict(X, batch_size=batch_size, verbose=1) pred_diastole = model_diastole.predict(X, batch_size=batch_size, verbose=1) # real predictions to CDF cdf_pred_systole = real_to_cdf(pred_systole, val_loss_systole) cdf_pred_diastole = real_to_cdf(pred_diastole, val_loss_diastole) print('Accumulating results...') sub_systole = accumulate_study_results(ids, cdf_pred_systole) sub_diastole = accumulate_study_results(ids, cdf_pred_diastole) # write to submission file print('Writing submission to file...') fi = csv.reader(open('data/sample_submission_validate.csv')) f = open('submission.csv', 'w') fo = csv.writer(f, lineterminator='\n') fo.writerow(fi.next()) for line in fi: idx = line[0] key, target = idx.split('_') key = int(key) out = [idx] if key in sub_systole: if target == 'Diastole': out.extend(list(sub_diastole[key][0])) else: out.extend(list(sub_systole[key][0])) else: print('Miss {0}'.format(idx)) fo.writerow(out) f.close() print('Done.')
def submission(): """ Generate submission file for the trained models. """ logging.info('Loading and compiling models...') model_systole = get_model() model_diastole = get_model() logging.info('Loading models weights...') model_systole.load_weights('../models/weights/weights_systole_best.hdf5') model_diastole.load_weights('../models/weights/weights_diastole_best.hdf5') logging.info('Loading validation data...') X, ids = load_validation_data() logging.info('Pre-processing images...') X = preprocess(X) batch_size = 32 logging.info('Predicting on validation data...') pred_systole = model_systole.predict(X, batch_size=batch_size, verbose=1) pred_diastole = model_diastole.predict(X, batch_size=batch_size, verbose=1) # real predictions to CDF cdf_pred_systole = correct_cdf(pred_systole) cdf_pred_diastole = correct_cdf(pred_diastole) logging.info('Accumulating results...') sub_systole = accumulate_study_results(ids, cdf_pred_systole) sub_diastole = accumulate_study_results(ids, cdf_pred_diastole) # write to submission file logging.info('Writing submission to file...') fi = csv.reader(open('../input/sample_submission_validate.csv')) f = open('../submissions/submission_13.csv', 'w') fo = csv.writer(f, lineterminator='\n') fo.writerow(next(fi)) for line in fi: idx = line[0] key, target = idx.split('_') key = int(key) out = [idx] if key in sub_systole: if target == 'Diastole': out.extend(list(sub_diastole[key][0])) else: out.extend(list(sub_systole[key][0])) else: logging.info('Miss {0}'.format(idx)) fo.writerow(out) f.close() logging.info('Done.')
def train(args): device = args.device load_path = args.load_path # load data train_data = load_data('train') val_data = load_data('validation') # load model with tf.device('/gpu:%d' % device): model = get_model('policy') # trainer init optimizer = Config.optimizer train_step = optimizer.minimize(model.loss) # init session and server sess = tf.InteractiveSession() saver = tf.train.Saver() if load_path==None: sess.run(tf.initialize_all_variables()) else: saver.restore(sess, load_path) print("Model restored from %s" % load_path) # accuracy pred = tf.reshape(model.pred, [-1, 9*10*16]) label = tf.reshape(model.label, [-1, 9*10*16]) correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(label,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) logging.basicConfig(filename='log.txt', level=logging.DEBUG) # train steps for i in range(Config.n_epoch): # training step batch_data, batch_label = train_data.next_batch(Config.minibatch_size) input_dict = {model.label:batch_label} for var, data in zip(model.inputs, batch_data): input_dict[var]=data #from IPython import embed;embed() sess.run(train_step, feed_dict=input_dict) # evalue step if (i+1)%Config.evalue_point == 0: batch_data, batch_label = val_data.next_batch(Config.minibatch_size) val_dict = {model.label:batch_label} for var, data in zip(model.inputs, batch_data): val_dict[var]=data score = accuracy.eval(feed_dict=val_dict) print("epoch %d, accuracy is %.2f" % (i,score)) logging.info("epoch %d, accuracy is %.2f" % (i,score)) # save step if (i+1)%Config.check_point == 0: save_path = saver.save(sess, "%s/epoch-%d" %(Config.save_path, i)) print("Model saved in file: %s" % save_path) logging.info("Model saved in file: %s" % save_path)
def export_input_graph(model_folder): sys.path.append(model_folder) from model import get_model with tf.Session() as sess: model = get_model('policy') saver = tf.train.Saver() tf.train.write_graph(sess.graph_def, model_folder, 'input_graph.pb', as_text=True)
def hard_train(data_prefix, prefix, seed, col): what = ['systole', 'diastole'][col % 2] print('We are going to train hard {} {}'.format(what, col)) print('Loading training data...') X, y = load_train_data(data_prefix, seed) X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2) model = get_model() nb_iter = 200 epochs_per_iter = 1 batch_size = 32 min_val = sys.float_info.max datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) height_shift_range=0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=True) # randomly flip images print('-'*50) print('Training...') print('-'*50) datagen.fit(X_train) checkpointer_best = ModelCheckpoint(filepath=prefix + "weights_{}_best.hdf5".format(what), verbose=1, save_best_only=True) checkpointer = ModelCheckpoint(filepath=prefix + "weights_{}.hdf5".format(what), verbose=1, save_best_only=False) hist = model.fit_generator(datagen.flow(X_train, y_train[:, col], batch_size=batch_size), samples_per_epoch=X_train.shape[0], nb_epoch=nb_iter, show_accuracy=False, validation_data=(X_test, y_test[:, col]), callbacks=[checkpointer, checkpointer_best], nb_worker=4) loss = hist.history['loss'][-1] val_loss = hist.history['val_loss'][-1] with open(prefix + 'val_loss.txt', mode='w+') as f: f.write(str(min(hist.history['val_loss']))) f.write('\n')
def _main(config, config_idx, train): base_filename = config.name + '_cfg' + str(config_idx) logger = set_up_logger('logs/' + base_filename + '.log') title = '{}: {} ({}) config index {}'.format(__file__, config.name, config.desc, config_idx) logger.info('START ' + title + '\n\n{}\n'.format(config)) data = get_data(config, train) if config.device != 'cpu': assert 'theano' not in sys.modules import theano.sandbox.cuda theano.sandbox.cuda.use(config.device) from model import get_model model = get_model(config, data) if not train: assert config.tst_load_model_path if not model.load(config.tst_load_model_path): raise AssertionError('Failed loading model weights from {}'.format(config.tst_load_model_path)) ans_hats = _tst_epoch(config, model, data) write_test_predictions(ans_hats, config.pred_json_path) logger.info('END ' + title) return # Training loop epoch_results = [] max_em = -np.inf max_f1 = -np.inf np_rng = np.random.RandomState(config.seed // 2) for epoch in range(1, config.max_num_epochs+1): trn_loss, trn_acc, trn_samples_per_sec = _trn_epoch(config, model, data, epoch, np_rng) dev_loss, dev_acc, dev_em, dev_f1 = _dev_epoch(config, model, data) if dev_em > max_em: model.save('models/' + base_filename + '_best_em.pkl') max_em = dev_em if dev_f1 > max_f1: model.save('models/' + base_filename + '_best_f1.pkl') max_f1 = dev_f1 if config.save_freq and epoch % config.save_freq == 0: model.save('models/' + base_filename + '_e{:03d}.pkl'.format(epoch)) epoch_results.append( EpochResult(trn_loss, trn_acc, dev_loss, dev_acc, dev_em, dev_f1)) if config.plot: plot_epoch_results(epoch_results, 'logs/' + base_filename + '.png') logger.info('\n\nFinished epoch {} for: (config index {}) (samples/sec: {:<.1f})\n{}\n\nResults:\n{}\n\n'.format( epoch, config_idx, trn_samples_per_sec, config.format_compared(), format_epoch_results(epoch_results))) logger.info('END ' + title)
def train(args): normalLogger.debug('loading data...') data = pd.read_csv(args.data_dir) normalLogger.debug('split data into train and test...') target, features = data[args.y_col], data.drop([args.y_col], axis=1) X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.25, random_state=33) ##### output test_set.csv ##### # X_test[args.y_col] = y_test # X_test.to_csv('test_set.csv',index=False) # X_test.drop([args.y_col], axis=1, inplace=True) ##### drop_useful_col(X_train) drop_useful_col(X_test) ##### normalLogger.debug('X_train size:' + str(X_train.shape)) normalLogger.debug('X_test size:' + str(X_test.shape)) normalLogger.debug('y_train size:' + str(y_train.shape)) normalLogger.debug('y_test size:' + str(y_test.shape)) normalLogger.debug('create preprocess from training data...') preprocessor = preprocess(encoder='label', normalize=(args.algorithm == 'nn')) if args.algorithm == 'nn': # note: target is for target encoder and nn to get output class count. # if you don't use target encoder and algorithm is not nn, then target is not matter X_train_encoder = preprocessor.fit_transform(X_train, auto_fill=True, target=y_train) else: X_train_encoder = preprocessor.fit_transform(X_train, auto_fill=False, target=y_train) # save preprocessor to pickle with open('./model_data/preprocessor.pkl', 'wb') as output: pickle.dump(preprocessor, output, pickle.HIGHEST_PROTOCOL) scaler = sum(y_train != 1) / sum(y_train == 1) #restrict the max scale time if sum(y_train != 1) / sum(y_train == 1) > 100: scaler = 100 #np.floor(np.sqrt(sum(y_train!=1)/sum(y_train==1))) normalLogger.debug('initialize %s model...' % args.algorithm) model, param_grid = get_model(args.algorithm, scaler=scaler, in_features=len(X_train.columns), num_classes=len(set(y_train)), mid_features=256) normalLogger.debug('getting model: ') normalLogger.debug(model) if args.algorithm == 'nn': from nn_factory import nn_factory import torch device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") grid = nn_factory(model, device, X_train_encoder, y_train, batch_size=32) grid.fit(epoch=30) else: normalLogger.debug('Hyperparameter tuning...') grid_start = time.time() scroer = make_scorer(fbeta_score, beta=3) if len(param_grid) > 0: grid = GridSearchCV(estimator=model, cv=5, n_jobs=-1, param_grid=param_grid, scoring=scroer) #grid = RandomizedSearchCV(estimator=model,cv=5, n_jobs=-1 , param_distributions=param_grid, scoring='f1_micro', n_iter=100) else: grid = model grid.fit(X_train_encoder, y_train) grid_end = time.time() - grid_start normalLogger.debug('finish grid search, it took %.5f min' % (grid_end / 60)) #print(grid.cv_results_) # save model for future inference normalLogger.debug('saving model to ./model_data') normalLogger.debug(grid.best_estimator_) joblib.dump(grid.best_estimator_, os.path.join(cwd, 'model_data', 'grid.pkl')) normalLogger.debug('saving feature importance') feature_importance(X_train_encoder, grid.best_estimator_) #see training performance normalLogger.debug('prediction on training set...') train_preds = grid.predict(X_train_encoder) train_auc = roc_auc_score(y_train, train_preds) train_recall = recall_score(y_train, train_preds, average=None) normalLogger.debug('compute and save the confusion matrix...') train_conf = confusion_matrix(y_train, train_preds) # graph confusion table and save colormap = sns.diverging_palette(220, 10, as_cmap=True) sns.set(font_scale=1.4) plt.figure() train_plot = sns.heatmap(train_conf, cmap=colormap, annot=True, cbar=False, fmt='d') train_fig = train_plot.get_figure() plt.title('train auc: %.3f, recall:%s' % (train_auc, str(train_recall))) train_fig.savefig("train_confusion.png") ##### see testset performance ##### normalLogger.debug('prediction on testing set...') normalLogger.debug('preprocess for testing set...') del preprocessor with open('./model_data/preprocessor.pkl', 'rb') as input: preprocessor_test = pickle.load(input) X_test_encoder = preprocessor_test.transform(X_test) test_preds = grid.predict(X_test_encoder) test_auc = roc_auc_score(y_test, test_preds) test_recall = recall_score(y_test, test_preds, average=None) test_conf = confusion_matrix(y_test, test_preds) # graph confusion table and save plt.figure() test_plot = sns.heatmap(test_conf, cmap=colormap, annot=True, cbar=False, fmt='d') test_fig = test_plot.get_figure() plt.title('test auc: %.3f, recall:%s' % (test_auc, str(test_recall))) test_fig.savefig("test_confusion.png")
def main(_): start_time = time.time() MAX_STEPS = 10000 # Maximum steps to train logging.info("TensorFlow version: %s", tf.__version__) logging.info("TensorFlow git version: %s", tf.__git_version__) if KUBERNETES: tf_config_json = os.environ.get("TF_CONFIG", "{}") tf_config = json.loads(tf_config_json) logging.info("tf_config: {}".format(tf_config)) task = tf_config.get("task", {}) task_index = task["index"] job_name = task["type"] logging.info("task: {}".format(task)) cluster_spec = tf_config.get("cluster", {}) logging.info("cluster_spec: {}".format(cluster_spec)) else: # Local testing task_index = FLAGS.task_index job_name = FLAGS.job_name cluster_spec = { "ps": ["localhost:2222"], "worker": ["localhost:2223", "localhost:2224"] } worker_list = cluster_spec.get("worker", "{}") ps_list = cluster_spec.get("ps", "{}") logging.info("job_name: {}".format(job_name)) logging.info("task_index: {}".format(task_index)) config = tf.ConfigProto(inter_op_parallelism_threads=num_inter_op_threads, intra_op_parallelism_threads=num_intra_op_threads) cluster = tf.train.ClusterSpec(cluster_spec) server = tf.train.Server(cluster, job_name=job_name, task_index=task_index) is_sync = (FLAGS.is_sync == 1) # Synchronous or asynchronous updates is_chief = (task_index == 0) # Am I the chief node (always task 0) if job_name == "ps": logging.info( "I am parameter server #{}. " "I will join the server and will " "need to be explicitly terminated when all jobs end. " "Kubernetes should do this automatically." "Otherwise, use CTRL-\\ for manual termination".format(task_index)) server.join() elif job_name == "worker": if is_chief: logging.info("I am the chief worker {} with task #{}".format( worker_list[task_index], task_index)) else: logging.info("I am worker {} with task #{}".format( worker_list[task_index], task_index)) # Graph worker_device = "/job:{}/task:{}".format(job_name, task_index) setter = tf.train. \ replica_device_setter(ps_tasks=len(ps_list), worker_device=worker_device) with tf.device(setter): """ BEGIN: MODEL DEFINE """ input_tensor = tf.placeholder(tf.float32) label_tensor = tf.placeholder(tf.float32) model = get_model(input_tensor, label_tensor, FLAGS, is_chief, MAX_STEPS, len(worker_list)) """ END: MODEL DEFINE """ # Monitored Training Session checkpoint_dir = None # if is_chief: # checkpoint_dir = FLAGS.CHECKPOINTS_DIRECTORY # else: # checkpoint_dir = None params = dict(master=server.target, is_chief=is_chief, config=config, hooks=model["hooks"], checkpoint_dir=checkpoint_dir, stop_grace_period_secs=10) sess = tf.train.MonitoredTrainingSession(**params) logging.info("Starting training on worker {}".format(task_index)) if is_chief: time.sleep(5) """ Just predict a simple line. """ slope = 8.16 intercept = -19.71 while not sess.should_stop(): train_x = np.random.randn(1) * 10 train_y = slope * train_x + \ intercept + np.random.randn(1) * 0.33 _, result, loss, m, b, step = sess.run([ model["optimizer"], model["prediction"], model["loss"], model["m"], model["b"], model["global_step"] ], feed_dict={ input_tensor: train_x, label_tensor: train_y }) logging.info("worker {}, step {}: loss = {:.4}, " "target: [{}, {}], prediction: [{:.4}, {:.4}]".format( task_index, step, loss, slope, intercept, m, b)) logging.info("Finished on task {}".format(task_index)) logging.info( "Session from worker {} closed cleanly".format(task_index))
# training config settings = {} settings['model_selection'] = 'cnn_version_1' #settings['model_selection'] = 'resnet_version_1' settings['time_len'] = 1024 # tf computing graph noised_inp_ = tf.placeholder(dtype=tf.float32, shape=[None, settings['time_len']], name='model_input') clear_ground_truth_ = tf.placeholder(dtype=tf.float32, shape=[None, settings['time_len']], name='ground_truth') denoised_out_ = model.get_model(noised_inp_, settings['model_selection']) # check_point saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables(), max_to_keep=100) ################################################# # load check point (load weights) if 1: saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) data_str = 'd2020-08-11_t1624' checkpoint_str = 'E:\\dnn_denoising\\traing_log\\' + settings[ 'model_selection'] + '\\' + data_str + '\\chech_point-19' #checkpoint_str = r'E:\dnn_denoising\traing_log'+'/'+settings['model_selection']+'/'+ data_str+'r\chech_point-19' print('### checkpoint_str: {0}'.format(checkpoint_str)) saver.restore(sess, checkpoint_str)
def train(epoch=None, show=None): """ 训练模型 """ model = get_model() model.train(epoch=epoch, show=show)
def run_main(ARGS): print("#######################################") print("Current execution paramenters:") for arg, value in sorted(vars(ARGS).items()): print("{}: {}".format(arg, value)) print("#######################################") bandwidth = ARGS.conv_depth / (16 * 3) print("\nBandwidth: ", bandwidth) dirname = "mode{mode}/alpha{alpha:.2f}B{b}E{e}band{band:.2f}".format( mode=ARGS.loss_type, alpha=ARGS.alpha, b=int(ARGS.snr_legit_train), e=int(ARGS.snr_adv_train), band=bandwidth) train_dir = ARGS.train_dir + "/" + dirname + "/train" save_dir = ARGS.train_dir + "/" + dirname + "/saved" test_dir = ARGS.test_dir + "/" + dirname + "/test" for dir in [train_dir, save_dir, test_dir]: if not os.path.exists(dir): os.makedirs(dir) img_height = DATASETS[ARGS.dataset]._HEIGHT img_width = DATASETS[ARGS.dataset]._WIDTH num_channels = DATASETS[ARGS.dataset]._NUM_CHANNELS num_classes = DATASETS[ARGS.dataset]._NUM_CLASSES u = tf.compat.v1.placeholder( tf.float32, shape=[None, img_height, img_width, num_channels], name='u') p = tf.compat.v1.placeholder(tf.float32, shape=[None, num_classes], name='p') snr_legit = ARGS.snr_legit_train snr_adv = ARGS.snr_adv_train model_vars, model_metrics, model_losses, model_collections = get_model( u, p, params={ 'ARGS': ARGS, 'snr_legit': snr_legit, 'snr_adv': snr_adv }) DATASETS[ARGS.dataset].data_path = ARGS.data_dir_test DATASETS[ARGS.dataset].maybe_download_and_extract() u_test, s_test, p_test = DATASETS[ARGS.dataset].load_test_data() def test(session): feed_dict_test = {u: u_test, p: p_test} results = session.run(model_metrics, feed_dict=feed_dict_test) return results session_config = tf.ConfigProto(inter_op_parallelism_threads=0, intra_op_parallelism_threads=0, allow_soft_placement=True) session_config.gpu_options.allow_growth = True if ARGS.mode in ("train", "Train"): DATASETS[ARGS.dataset].data_path = ARGS.data_dir_train DATASETS[ARGS.dataset].maybe_download_and_extract() u_train, s_train, p_train = DATASETS[ARGS.dataset].load_training_data() batch_size = ARGS.batch_size train_iters_legit_1 = ARGS.train_iters_legit_1 train_iters_legit_3 = ARGS.train_iters_legit_3 train_iters_adv_2 = ARGS.train_iters_adv_2 train_iters_adv_3 = ARGS.train_iters_adv_3 global_step = tf.Variable(initial_value=0, name='global_step', trainable=False) legit_vars = model_collections['legitimate_vars'] adv_vars = model_collections['adversary_vars'] optimizers = [ tf.train.AdamOptimizer(learning_rate=ARGS.learn_rate).minimize( model_losses['loss_legit_prelim'], var_list=legit_vars), tf.train.AdamOptimizer(learning_rate=ARGS.learn_rate).minimize( model_losses['loss_adv'], var_list=adv_vars), tf.train.AdamOptimizer(learning_rate=ARGS.learn_rate).minimize( model_losses['loss_legit'], var_list=legit_vars) ] def get_random_batch(batch_size): idx = np.random.choice(len(u_train), size=batch_size) u_batch = u_train[idx, :, :, :] p_batch = p_train[idx, :] return u_batch, p_batch def train(session, optimizer, iters): for i in range(iters): u_batch, p_batch = get_random_batch(batch_size) feed_dict_train = {u: u_batch, p: p_batch} i_global, _ = session.run([global_step, optimizer], feed_dict=feed_dict_train) for n in range(ARGS.num_simulations): print("Beginning simulation number: ", n) sim_slug = "/sim{}".format(n) for dir in [train_dir, save_dir, test_dir]: if not os.path.exists(dir + sim_slug): os.makedirs(dir + sim_slug) session = tf.Session(config=session_config) init = tf.compat.v1.global_variables_initializer() session.run(init) for phase in range(3): measures = { 'legit_iters': [], 'adv_iters': [], 'mse': [], 'psnr': [], 'cross_entropy': [], 'accuracy': [], 'avg_power_y': [], 'avg_power_z': [] } def write_results(): filename = train_dir + sim_slug + "/results_" + str( phase + 1) + ".csv" df = pd.DataFrame(measures) df.to_csv(filename) print("\tResults saved in ", filename) t_epoch = 0 tot_iters_legit = 0 tot_iters_adv = 0 test_results = test(session) def append_results(results): print("\t -Total main network iterations: ", tot_iters_legit) print("\t -Total adversary iterations: ", tot_iters_adv) measures['legit_iters'].append(tot_iters_legit) measures['adv_iters'].append(tot_iters_adv) for key in results: if key in measures: if t_epoch % 1 == 0: print("\t -", key, ": {0}".format(results[key])) measures[key].append(results[key]) if phase == 0: # Phase 1: preliminary training of main network print("\tPhase 1: Preliminary training of main network.") saver = tf.compat.v1.train.Saver(var_list=legit_vars) save_path = save_dir + sim_slug + "/model_phase_1.ckpt" if os.path.exists(save_path + ".index" ) and not ARGS.delete_prev_model_1: saver.restore(session, save_path) print("\tModel restored from: ", save_path) if ARGS.skip_phase_1: print("\tSkipping phase 1") continue start_time = time.time() test_results = test(session) append_results(test_results) mse_new = test_results['mse'] while True: t_epoch = t_epoch + 1 print("\tEpoch number: ", t_epoch) mse_old = mse_new train(session, optimizers[0], train_iters_legit_1) test_results = test(session) tot_iters_legit = tot_iters_legit + train_iters_legit_1 append_results(test_results) write_results() mse_new = test_results['mse'] # save model every 10 epochs if t_epoch % 10 == 0: saver.save(session, save_path) print("\tModel saved in path: ", save_path) # if a specific condition is satisfied, exit from the loop condition = t_epoch >= ARGS.train_epochs_1 if ARGS.reach_convergence_1: condition = abs(mse_old - mse_new) < ARGS.mse_epsilon_1 if condition: break elif phase == 1: # Phase 2: preliminary training of the adversary print("\tPhase 2: Preliminary training of the adversary.") saver = tf.compat.v1.train.Saver(var_list=adv_vars) save_path = save_dir + sim_slug + "/model_phase_2.ckpt" if os.path.exists(save_path + ".index" ) and not ARGS.delete_prev_model_2: saver.restore(session, save_path) print("\tModel restored from: ", save_path) if ARGS.skip_phase_2: print("\tSkipping phase 2") continue start_time = time.time() test_results = test(session) append_results(test_results) mse_new = test_results['accuracy'] while True: t_epoch = t_epoch + 1 print("\tEpoch number: ", t_epoch) mse_old = mse_new train(session, optimizers[1], train_iters_adv_2) test_results = test(session) tot_iters_adv = tot_iters_adv + train_iters_adv_2 append_results(test_results) write_results() mse_new = test_results['accuracy'] # save model every 10 epochs if t_epoch % 10 == 0: saver.save(session, save_path) print("\tModel saved in path: ", save_path) # if a specific condition is satisfied, exit from the loop condition = t_epoch >= ARGS.train_epochs_2 if ARGS.reach_convergence_2: condition = abs(acc_old - acc_new) < ARGS.acc_epsilon_2 if condition: break else: # Phase 3: adversarial training of the network (minimax) print( "\tPhase 3: Adversarial training of the network (minimax)." ) saver = tf.compat.v1.train.Saver() save_path = save_dir + sim_slug + "/model.ckpt" start_time = time.time() test_results = test(session) append_results(test_results) mse_new = test_results['mse'] acc_new = test_results['accuracy'] while True: t_epoch = t_epoch + 1 print("\tEpoch number: ", t_epoch) mse_old = mse_new acc_old = acc_new train(session, optimizers[2], train_iters_legit_3) test_results = test(session) tot_iters_legit = tot_iters_legit + train_iters_legit_3 append_results(test_results) write_results() train(session, optimizers[1], train_iters_adv_3) test_results = test(session) tot_iters_adv = tot_iters_adv + train_iters_adv_3 append_results(test_results) write_results() acc_new = test_results['accuracy'] # save model every 10 epochs if t_epoch % 10 == 0: saver.save(session, save_path) print("\tModel saved in path: ", save_path) # if a specific condition is satisfied, exit from the loop condition = t_epoch >= ARGS.train_epochs_3 if ARGS.reach_convergence_3: condition = abs(mse_old - mse_new) < ARGS.mse_epsilon_3 condition = condition and abs( acc_old - acc_new) < ARGS.acc_epsilon_3 if condition: break elif ARGS.mode in ("test", "Test"): DATASETS[ARGS.dataset].data_path = ARGS.data_dir_test DATASETS[ARGS.dataset].maybe_download_and_extract() # count the number of simulation folders num_simulations = 0 folders = os.walk(save_dir)[1] for name in folders: if "sim" in name: num_simulations = num_simulations + 1 for n in num_simulations: print("Beginning simulation number: ", n) sim_slug = "/sim{}".format(n) measures = { 'snr_legit': [], 'snr_adv': [], 'mse': [], 'psnr': [], 'cross_entropy': [], 'accuracy': [], 'avg_power_y': [], 'avg_power_z': [] } def write_results(): filename = test_dir + sim_slug + "/results.csv" df = pd.DataFrame(measures) df.to_csv(filename) print("\tResults saved in ", filename) def append_results(results): measures['snr_legit'].append(snr_legit) measures['snr_adv'].append(snr_adv) for key in results: if key in measures: if (t_epoch % 1 == 0): print("\t -", key, ": {0}".format(results[key])) measures[key].append(results[key]) snr_range = [5 * i for i in range(-5, 6)] # [-25, -20, ..., 20, 25] snr_legit = ARGS.snr_legit_train for snr_adv in snr_range: model_vars, model_metrics, model_losses, model_collections = get_model( u, p, params={ 'ARGS': ARGS, 'snr_legit': snr_legit, 'snr_adv': snr_adv }) session = tf.Session(config=session_config) saver = tf.compat.v1.train.Saver() save_path = save_dir + sim_slug + "/model.ckpt" if os.path.exists(save_path + ".index"): saver.restore(session, save_path) print("Model restored from: ", save_path) test_results = test(session) append_results(test_results) write_results() else: print("ERROR: Model not found.") else: print("Error: the only available options are 'train' or 'test'.")
default="dataset/predict/input", help="input video folder") parser.add_argument('--output', type=str, default="dataset/predict/output", help="output video folder") parser.add_argument( "--scale", type=int, default=4, help='Increase the frames by Nx. Example scale=2 ==> 2x frames') args = parser.parse_args() # Flow compute flow_compute = get_model("FC") model_load(flow_compute, "FC", args.checkpoint) device = model_device() flow_compute.to(device) flow_compute.eval() # Flow interpolate flow_interpolate = get_model("AT") model_load(flow_interpolate, "AT", args.checkpoint) flow_interpolate.to(device) flow_interpolate.eval() # if os.environ["ENABLE_APEX"] == "YES": # from apex import amp # model = amp.initialize(model, opt_level="O1")
def train(): """ Training systole and diastole models. """ print('Loading and compiling models...') model_systole = get_model() model_diastole = get_model() #import best model if it exists if os.path.isfile('/data/run2/weights_systole_best.hdf5'): print('loading weights') model_systole.load_weights('/data/run2/weights_systole_best.hdf5') if os.path.isfile('/data/run2/weights_diastole_best.hdf5'): model_diastole.load_weights('/data/run2/weights_diastole_best.hdf5') print('Loading training data...') X, y, metadata = load_train_data() #print('Pre-processing images...') #X = preprocess(X) #np.save('/data/pre/pre/X_train.npy', X) # split to training and test X_train, y_train, X_test, y_test, metadata_train, metadata_test = split_data(X, y, metadata, split_ratio=0.2) nb_iter = 200 epochs_per_iter = 1 batch_size = 8 calc_crps = 5 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed) # remember min val. losses (best iterations), used as sigmas for submission min_val_loss_systole = sys.float_info.max min_val_loss_diastole = sys.float_info.max print('-'*50) print('Training...') print('-'*50) for i in range(0,nb_iter): print('-'*50) print('Iteration {0}/{1}'.format(i + 1, nb_iter)) print('-'*50) # print('Augmenting images - rotations') # X_train_aug = rotation_augmentation(X_train, 15) # print('Augmenting images - shifts') # X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1) # print('Augmenting images - shifts') # X_train_aug = shift_augmentation(X_train, 0.1, 0.1) X_train_aug = X_train print('Fitting systole model...') hist_systole = model_systole.fit({'input1':X_train_aug, 'input2':metadata_train, 'output':y_train[:, 0]}, shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data={'input1':X_test,'input2':metadata_test, 'output':y_test[:, 0]}) print('Fitting diastole model...') hist_diastole = model_diastole.fit({'input1':X_train_aug, 'input2':metadata_train, 'output':y_train[:, 1]}, shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data={'input1':X_test, 'input2':metadata_test, 'output':y_test[:, 1]}) # sigmas for predicted data, actually loss function values (RMSE) loss_systole = hist_systole.history['loss'][-1] loss_diastole = hist_diastole.history['loss'][-1] val_loss_systole = hist_systole.history['val_loss'][-1] val_loss_diastole = hist_diastole.history['val_loss'][-1] if calc_crps > 0 and i % calc_crps == 0: print('Evaluating CRPS...') pred_systole = model_systole.predict({'input1':X_train, 'input2':metadata_train, 'output':y_train[:, 0]}, batch_size=batch_size, verbose=1)['output'] pred_diastole = model_diastole.predict({'input1':X_train, 'input2':metadata_train, 'output':y_train[:, 1]}, batch_size=batch_size, verbose=1)['output'] val_pred_systole = model_systole.predict({'input1':X_test, 'input2':metadata_test, 'output':y_test[:, 0]}, batch_size=batch_size, verbose=1)['output'] val_pred_diastole = model_diastole.predict({'input1':X_test, 'input2':metadata_test, 'output':y_test[:, 1]}, batch_size=batch_size, verbose=1)['output'] # Get sigmas # sigma_systole = as_tensor_variable(root_mean_squared_error(y_train[:, 0], pred_systole)) # sigma_diastole = as_tensor_variable(root_mean_squared_error(y_train[:, 1], pred_systole)) # val_sigma_systole = as_tensor_variable(root_mean_squared_error(y_test[:, 0], val_pred_systole)) # val_sigma_diastole = as_tensor_variable(root_mean_squared_error(y_test[:, 1], val_pred_diastole)) # CDF for train and test data (actually a step function) cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1]))) cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1]))) # CDF for predicted data cdf_pred_systole = real_to_cdf(pred_systole, loss_systole) cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole) cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole) cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole) # evaluate CRPS on training data crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole))) print('CRPS(train) = {0}'.format(crps_train)) # evaluate CRPS on test data crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole))) print('CRPS(test) = {0}'.format(crps_test)) # for best (lowest) val losses, save weights if val_loss_systole < min_val_loss_systole: min_val_loss_systole = val_loss_systole model_systole.save_weights('/data/run2/weights_systole_best.hdf5', overwrite=True) if val_loss_diastole < min_val_loss_diastole: min_val_loss_diastole = val_loss_diastole model_diastole.save_weights('/data/run2/weights_diastole_best.hdf5', overwrite=True) # save best (lowest) val losses in file (to be later used for generating submission) with open('/data/run2/val_loss.txt', mode='w+') as f: f.write(str(min_val_loss_systole)) f.write('\n') f.write(str(min_val_loss_diastole)) with open("/data/run2/loss.txt", "a+") as myfile: myfile.write('\t'.join((str(i+1), str(loss_systole),str(loss_diastole),str(val_loss_systole),str(val_loss_diastole), str(crps_train), str(crps_test)))) myfile.write('\n')
def test(): """ 模型测试 """ model = get_model() return model.test()
def cut(sentence): """ 分词 """ model = get_model() return model.cut(sentence)
params["n_sents"], params["n_targets"], params["batch_size"], encoding) # add post-processing experiment params embedding_input = batches.char_indexer.vocab_len() n_classes = batches.word_indexer.vocab_len() params.update({"embedding_input": embedding_input, "n_classes": n_classes}) X_test, y_test = batches.train_test_split() y_test = np_utils.to_categorical(y_test, n_classes) # model print("compiling model...") m = get_model(**params) print("learning...") if params["batch_training"]: for e in range(params["n_epoch"]): print("epoch number: %d" % e) for X_batch, y_batch in batches: y_batch = np_utils.to_categorical(y_batch, n_classes) m.fit(X_batch, y_batch, batch_size=params["mini_batch_size"], nb_epoch=1, verbose=1, validation_data=(X_test, y_test)) # loss = m.train_on_batch(X_batch, y_batch) # print("loss: [%d]" % loss[0]) else: # canister callback
def refresh(self): fname, fkey = self.reference.split(".") fmod = get_model(fname) self.count = fmod.query(getattr(fmod, fkey) == self.target).count()
from theano.tensor.nnet import categorical_crossentropy from transformers import OneHotEncode, RandomHorizontalFlip batch_size = 32 test_dataset = CIFAR10(('test',), subset=slice(0, 9984)) test_stream = DataStream.default_stream( test_dataset, iteration_scheme=SequentialScheme(test_dataset.num_examples, batch_size) ) test_stream = OneHotEncode(test_stream, which_sources=('targets',)) X = tensor.ftensor4('features') targets = tensor.fmatrix('targets') output, output_test, all_parameters, acc_parameters = get_model(X, batch_size, (32, 32)) saved_parameters = numpy.load('./best_weights.npz')['arr_0'] for param_w, param in zip(saved_parameters, all_parameters): param.set_value(param_w) saved_parameters_bn = numpy.load('./best_weights_bn.npz')['arr_0'] for param_w, param in zip(saved_parameters_bn, acc_parameters): param[1].set_value(param_w) error_test = tensor.neq(tensor.argmax(output_test[:,:,0,0], axis=1), tensor.argmax(targets, axis=1)).mean() error_test.name = 'error_test' f_valid = theano.function( inputs=[X, targets], outputs=[error_test],
def train(continue_training=False): ''' Run training ''' print('Loading training data...') train = np.load(train_file) train,val = split_data(train, split_ratio = 0.2) test = np.load(test_file) print "done." print('Prepare data...') input_train = model_data(train) input_val = model_data(val) input_test = model_data(test) nFeatures = input_train["X_input"].shape[1] nTin = input_train["RNN_input"].shape[1] nRNNFeatures = input_train["RNN_input"].shape[2] nTin2 = input_train["RNN2_input"].shape[1] nRNNFeatures2 = input_train["RNN2_input"].shape[2] print "using %d/%d/%d samples with %d features RNN(t,features,hidden)=(%d,%d,%d) and (%d,%d,%d)" % \ (input_train["X_input"].shape[0],input_val["X_input"].shape[0],input_test["X_input"].shape[0],\ nFeatures,nTin,nRNNFeatures,nRNNHidden,nTin2,nRNNFeatures2,nRNNHidden2) print('Loading and compiling models...') model = get_model(nTin=nTin, nRNNFeatures=nRNNFeatures,nRNNHidden=nRNNHidden, \ nTin2=nTin2, nRNNFeatures2=nRNNFeatures2,nRNNHidden2=nRNNHidden, \ nFeatures=nFeatures, nOutput=N_ZONES) if continue_training: print('Loading models weights...') model.load_weights(output_weights_best_file) print "done." print('-'*50) print('Training model...') print('-'*50) nIterations = 10000 # 300 epochs_per_iter = 1 batch_size = 64 # 1024 # 64 loss_val_min = sys.float_info.max losses_train = [] losses_val = [] losses_test = [] errors_train = [] errors_val = [] errors_test = [] for iIteration in range(nIterations): print('-'*50) print('Iteration {0}/{1}'.format(iIteration + 1,nIterations)) print('-'*50) print('Fitting model...') hist = model.fit(input_train, validation_data=(input_val), \ shuffle=True, nb_epoch=epochs_per_iter, verbose=1,batch_size=batch_size) loss_train = hist.history['loss'][-1] loss_val = hist.history['val_loss'][-1] losses_train.append(loss_train) losses_val.append(loss_val) print("Loss train/test = %f / %f" % (loss_train, loss_val)) print('Calculate predictions...') Ypred_train = model.predict(input_train, batch_size=batch_size, verbose=1)["out"] Ypred_val = model.predict(input_val, batch_size=batch_size, verbose=1)["out"] Y_train = input_train["out"] Y_val = input_val["out"] error_train = np.sqrt(np.mean((Y_train - Ypred_train) * (Y_train - Ypred_train))) / np.mean(Y_train) error_val = np.sqrt(np.mean((Y_val - Ypred_val) * (Y_val - Ypred_val))) / np.mean(Y_val) errors_train.append(error_train) errors_val.append(error_val) print("Error train/test = %f / %f" % (error_train, error_val)) print "done." print('Save Losses...') csv_file = open(output_losses_file, "w") csv_file.write("iter,train_loss,test_loss\n") for i in range(len(losses_train)): csv_file.write("%d,%f,%f,%f,%f\n" % (i, losses_train[i], losses_val[i],errors_train[i],errors_val[i])) csv_file.close() print "done." print('Saving weights...') model.save_weights(output_weights_file, overwrite=True) if loss_val < loss_val_min: # csv_file = open("output/kerasRNN/accuracy", "w") # csv_file.write("id,actual,predicted\n") # for id in id_to_pred_test.keys(): # csv_file.write("%s,%f,%f\n" % (id, id_to_actual_test[id], id_to_pred_test[id])) # csv_file.close() loss_val_min = loss_val model.save_weights(output_weights_best_file, overwrite=True) print "done." # force deletion del hist model.training_data = None model.validation_data = None gc.collect() print "Plot example..." plot(Y_val,Ypred_val,i=0,iZone=0) print "done." print "END."
def tagger(sentence): """ 词性标注 """ model = get_model() return model.predict(sentence)
def main(): """ main function """ ### header parser = argparse.ArgumentParser() # path parser.add_argument('--root-path', default=CFG.root_path, help="root path") parser.add_argument('--log-path', default=CFG.log_path, help="log path") parser.add_argument('--model-path', default=CFG.model_path, help="model path") parser.add_argument('--pretrained-path', help='pretrained path') # image parser.add_argument('--transform-version', default=0, type=int, help="image transform version ex) 0, 1, 2 ...") parser.add_argument('--image-size', default=256, type=int, help="image size(256)") # model parser.add_argument('--model-name', default=CFG.model_name, help=f"model name({CFG.model_name})") parser.add_argument('--backbone-name', default=CFG.backbone_name, help=f"backbone name({CFG.backbone_name})") # learning parser.add_argument('--batch-size', default=CFG.batch_size, type=int, help=f"batch size({CFG.batch_size})") parser.add_argument('--learning-rate', default=CFG.learning_rate, type=float, help=f"learning rate({CFG.learning_rate})") parser.add_argument('--num-epochs', default=CFG.num_epochs, type=int, help=f"number of epochs({CFG.num_epochs})") # etc parser.add_argument("--seed", default=CFG.seed, type=int, help=f"seed({CFG.seed})") parser.add_argument("--workers", default=CFG.workers, type=int, help=f"number of workers({CFG.workers})") parser.add_argument("--debug", action="store_true", help="debug mode") args = parser.parse_args() # path CFG.root_path = args.root_path CFG.model_path = args.model_path CFG.log_path = args.log_path CFG.pretrained_path = args.pretrained_path # image CFG.transform_version = args.transform_version CFG.image_size = args.image_size # model CFG.model_name = args.model_name CFG.backbone_name = args.backbone_name # learning CFG.batch_size = args.batch_size CFG.learning_rate = args.learning_rate CFG.num_epochs = args.num_epochs # etc CFG.seed = args.seed CFG.workers = args.workers CFG.debug = args.debug # get device CFG.device = get_device() # get version _, version, _ = sys.argv[0].split('/') CFG.version = version # update log path if not CFG.debug: CFG.log_path = os.path.join(CFG.log_path, CFG.version) os.makedirs(CFG.log_path, exist_ok=True) CFG.log_path = os.path.join( CFG.log_path, f'exp_{get_exp_id(CFG.log_path, prefix="exp_")}') os.makedirs(CFG.log_path, exist_ok=True) else: CFG.log_path = os.path.join(CFG.log_path, "debug") os.makedirs(CFG.log_path, exist_ok=True) CFG.log_path = os.path.join(CFG.log_path, "debug") os.makedirs(CFG.log_path, exist_ok=True) # update model path if not CFG.debug: CFG.model_path = os.path.join(CFG.model_path, version) os.makedirs(CFG.model_path, exist_ok=True) CFG.model_path = os.path.join( CFG.model_path, f'exp_{get_exp_id(CFG.model_path, prefix="exp_")}') os.makedirs(CFG.model_path, exist_ok=True) else: CFG.model_path = os.path.join(CFG.model_path, "debug") os.makedirs(CFG.model_path, exist_ok=True) CFG.model_path = os.path.join(CFG.model_path, "debug") os.makedirs(CFG.model_path, exist_ok=True) pprint({k: v for k, v in dict(CFG.__dict__).items() if '__' not in k}) json.dump({k: v for k, v in dict(CFG.__dict__).items() if '__' not in k}, open(os.path.join(CFG.log_path, 'CFG.json'), "w")) print() ### seed all seed_everything(CFG.seed) ### data related # load data print("Load Raw Data") data_df, test_df = load_data(CFG) # preprocess data print("Preprocess Data") data_df = preprocess_data(CFG, data_df) # split data into train with valid print("Split Data") data_df = split_data(CFG, data_df) # get transform print("Get Transform") train_transforms, test_transforms = get_transform(CFG) # train test split for fold in range(CFG.n_folds): print(f"\nValidation Fold: {fold}") train_df = data_df[data_df['fold'] != fold].reset_index(drop=True) valid_df = data_df[data_df['fold'] == fold].reset_index(drop=True) print( f"... Train Shape: {train_df.shape}, Valid Shape: {valid_df.shape}" ) # dataset trn_data = MelanomaDataset(CFG, train_df, train_transforms) val_data = MelanomaDataset(CFG, valid_df, test_transforms) ### model related # get learner learner = Learner(CFG) learner.name = f"model.fold_{fold}" if CFG.pretrained_path: print("Load Pretrained Model") print(f"... Pretrained Info - {CFG.pretrained_path}") learner.load(CFG.pretrained_path, f"model_state_dict") # get model if CFG.pretrained_path: print(f"Get Model") model = learner.best_model.to(CFG.deivce) else: print(f"Get Model") model = get_model(CFG) model = model.to(CFG.device) if torch.cuda.device_count() > 1: model = nn.DataParallel(model) # get optimizer optimizer = optim.Adam(model.parameters(), lr=CFG.learning_rate) optimizer = torchcontrib.optim.SWA(optimizer) # get scheduler # scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( # optimizer, mode='min', patience=1, verbose=False, factor=0.2) # scheduler = CosineAnnealingLRWarmup(optimizer, T_min=int(CFG.num_epochs / 5), T_max=CFG.num_epochs) scheduler = CosineAnnealingLRWarmup(optimizer, T_min=0, T_max=CFG.num_epochs // 4) ### train related # train model learner.train(trn_data, val_data, model, optimizer, scheduler) print()
def train(): """ 训练模型 """ model = get_model() model.train()
from tqdm import tqdm from util import load_data, get_tokens, get_keep_tokens from config import BaseConfig from model import get_model from dataset import data_generator # 加载数据集 data = load_data(BaseConfig.train_path) train_data = [d for i, d in enumerate(data) if i % 10 != 0] valid_data = [d for i, d in enumerate(data) if i % 10 == 0] test_data = load_data(BaseConfig.test_path) test_generator = data_generator(test_data, BaseConfig.batch_size) # 数据集词频 tokens = get_tokens(data + test_data) # BERT词频 keep_tokens = get_keep_tokens() model = get_model(tokens, keep_tokens) model.load_weights('best_model.h5') F = open("result.csv", mode="w") for x_true, _ in tqdm(test_generator): y_pred = model.predict(x_true)[:, 0, 5:7] y_pred = y_pred[:, 1] / (y_pred.sum(axis=1) + 1e-8) for p in y_pred: F.write('%f\n' % p) F.close()
def recognize(sentence): """ 命名实体识别 """ model = get_model() return model.predict(sentence)
def train(): """ Training model. """ # Compile training and testing functions [model, train_fn, val_fn, predict_fn] = get_model() # Load training data print('Loading training data...') X, y = load_train_data() #print('Pre-processing images...') #X = preprocess(X) # split to training and test X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2) nb_epoch = 200 batch_size = 32 calc_crps = 0 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed) NOT IMPLEMENTED YET print('-'*50) print('Training...') print('-'*50) min_val_err = sys.float_info.max patience = 0 for i in range(nb_epoch): print('-'*50) print('Iteration {0}/{1}'.format(i + 1, nb_epoch)) print('-'*50) print('Augmenting images - rotations') X_train_aug = rotation_augmentation(X_train, 15) print('Augmenting images - shifts') X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1) # In each epoch, we do a full pass over the training data: print('Fitting model...') train_err = 0 train_batches = 0 for batch in iterate_minibatches(X_train_aug, y_train, batch_size, shuffle=True): inputs, targets = batch train_err += train_fn(inputs, targets) train_batches += 1 # And a full pass over the validation data: val_err = 0 val_batches = 0 for batch in iterate_minibatches(X_test, y_test, batch_size, shuffle=False): inputs, targets = batch val_err += val_fn(inputs, targets) val_batches += 1 print('Saving weights...') # save weights so they can be loaded later # np.savez('weights.npz', *get_all_param_values(model)) # for best (lowest) val losses, save weights if val_err < min_val_err: patience = 0 min_val_err = val_err np.savez('weights_best.npz', *get_all_param_values(model)) else: patience += 1 print('error on validation set: ' + str(val_err)) print('patience variable is: ' + str(patience)) print('\n') # save best (lowest) val losses in file (to be later used for generating submission) with open('val_loss.txt', mode='a') as f: f.write(str(val_err)) f.write('\n') if (patience == 8): break
def main(): opt = parse_args() logging.info(json.dumps(vars(opt), indent=2)) rootpath = opt.rootpath testCollection = opt.testCollection assert collectionStrt == "multiple" resume = os.path.join(opt.logger_name, opt.checkpoint_name) if not os.path.exists(resume): logging.info(resume + ' not exists.') sys.exit(0) checkpoint = torch.load(resume) start_epoch = checkpoint['epoch'] best_rsum = checkpoint['best_rsum'] logging.info("=> loaded checkpoint '{}' (epoch {}, best_rsum {})".format( resume, start_epoch, best_rsum)) options = checkpoint['opt'] trainCollection = options.trainCollection valCollection = options.valCollection visual_feat_file = BigFile( os.path.join(rootpath, testCollection, 'FeatureData', options.visual_feature)) assert options.visual_feat_dim == visual_feat_file.ndims video2frame = read_dict( os.path.join(rootpath, testCollection, 'FeatureData', options.visual_feature, 'video2frames.txt')) vid_data_loader = data.get_vis_data_loader(visual_feat_file, opt.batch_size, opt.workers, video2frame) vis_embs = None # set bow vocabulary and encoding bow_vocab_file = os.path.join(rootpath, options.trainCollection, 'TextData', 'vocabulary', 'bow', options.vocab + '.pkl') bow_vocab = pickle.load(open(bow_vocab_file, 'rb')) bow2vec = get_text_encoder('bow')(bow_vocab) options.bow_vocab_size = len(bow_vocab) # set rnn vocabulary rnn_vocab_file = os.path.join(rootpath, options.trainCollection, 'TextData', 'vocabulary', 'rnn', options.vocab + '.pkl') rnn_vocab = pickle.load(open(rnn_vocab_file, 'rb')) options.vocab_size = len(rnn_vocab) model = get_model(options.model)(options) model.load_state_dict(checkpoint['model']) model.val_start() output_dir = resume.replace(trainCollection, testCollection) for query_set in opt.query_sets.strip().split(','): output_dir_tmp = output_dir.replace( valCollection, '%s/%s/%s' % (query_set, trainCollection, valCollection)) output_dir_tmp = output_dir_tmp.replace('/%s/' % options.cv_name, '/results/') pred_result_file = os.path.join(output_dir_tmp, 'id.sent.score.txt') logging.info(pred_result_file) if checkToSkip(pred_result_file, opt.overwrite): sys.exit(0) makedirsforfile(pred_result_file) # query data loader query_file = os.path.join(rootpath, testCollection, 'TextData', query_set) query_loader = data.get_txt_data_loader(query_file, rnn_vocab, bow2vec, opt.batch_size, opt.workers) # encode videos if vis_embs is None: start = time.time() if options.space == 'hybrid': video_embs, video_tag_probs, video_ids = evaluation.encode_text_or_vid_tag_hist_prob( model.embed_vis, vid_data_loader) else: video_embs, video_ids = evaluation.encode_text_or_vid( model.embed_vis, vid_data_loader) logging.info("encode video time: %.3f s" % (time.time() - start)) # encode text start = time.time() if options.space == 'hybrid': query_embs, query_tag_probs, query_ids = evaluation.encode_text_or_vid_tag_hist_prob( model.embed_txt, query_loader) else: query_embs, query_ids = evaluation.encode_text_or_vid( model.embed_txt, query_loader) logging.info("encode text time: %.3f s" % (time.time() - start)) if options.space == 'hybrid': t2v_matrix_1 = evaluation.cal_simi(query_embs, video_embs) # eval_avs(t2v_matrix_1, query_ids, video_ids, pred_result_file, rootpath, testCollection, query_set) t2v_matrix_2 = evaluation.cal_simi(query_tag_probs, video_tag_probs) # pred_result_file = os.path.join(output_dir_tmp, 'id.sent.score_2.txt') # eval_avs(t2v_matrix_2, query_ids, video_ids, pred_result_file, rootpath, testCollection, query_set) t2v_matrix_1 = norm_score(t2v_matrix_1) t2v_matrix_2 = norm_score(t2v_matrix_2) for w in [0.8]: print("\n") t2v_matrix = w * t2v_matrix_1 + (1 - w) * t2v_matrix_2 pred_result_file = os.path.join(output_dir_tmp, 'id.sent.score_%.1f.txt' % w) eval_avs(t2v_matrix, query_ids, video_ids, pred_result_file, rootpath, testCollection, query_set) else: t2v_matrix_1 = evaluation.cal_simi(query_embs, video_embs) eval_avs(t2v_matrix_1, query_ids, video_ids, pred_result_file, rootpath, testCollection, query_set)
def train(data_prefix, prefix, seed, run): """ Training systole and diastole models. """ print('Loading training data...') X, y = load_train_data(data_prefix, seed) print('Loading and compiling models...') model_systole = get_model() model_diastole = get_model() # split to training and test X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2) nb_iter = 200 epochs_per_iter = 1 batch_size = 32 calc_crps = 1 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed) # remember min val. losses (best iterations), used as sigmas for submission min_val_loss_systole = sys.float_info.max min_val_loss_diastole = sys.float_info.max print('-'*50) print('Training...') print('-'*50) datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) height_shift_range=0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=True) # randomly flip images # compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied) datagen.fit(X_train) systole_checkpointer_best = ModelCheckpoint(filepath=prefix + "weights_systole_best.hdf5", verbose=1, save_best_only=True) diastole_checkpointer_best = ModelCheckpoint(filepath=prefix + "weights_diastole_best.hdf5", verbose=1, save_best_only=True) systole_checkpointer = ModelCheckpoint(filepath=prefix + "weights_systole.hdf5", verbose=1, save_best_only=False) diastole_checkpointer = ModelCheckpoint(filepath=prefix + "weights_diastole.hdf5", verbose=1, save_best_only=False) if run == 0 or run == 1: print('Fitting Systole Shapes') hist_systole = model_systole.fit_generator(datagen.flow(X_train, y_train[:, 2], batch_size=batch_size), samples_per_epoch=X_train.shape[0], nb_epoch=nb_iter, show_accuracy=False, validation_data=(X_test, y_test[:, 2]), callbacks=[systole_checkpointer, systole_checkpointer_best], nb_worker=4) if run == 0 or run == 2: print('Fitting Diastole Shapes') hist_diastole = model_diastole.fit_generator(datagen.flow(X_train, y_train[:, 2], batch_size=batch_size), samples_per_epoch=X_train.shape[0], nb_epoch=nb_iter, show_accuracy=False, validation_data=(X_test, y_test[:, 2]), callbacks=[diastole_checkpointer, diastole_checkpointer_best], nb_worker=4) if run == 0 or run == 1: loss_systole = hist_systole.history['loss'][-1] val_loss_systole = hist_systole.history['val_loss'][-1] if run == 0 or run == 2: loss_diastole = hist_diastole.history['loss'][-1] val_loss_diastole = hist_diastole.history['val_loss'][-1] if calc_crps > 0 and run == 0: print('Evaluating CRPS...') pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1) val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1) pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1) val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1) # CDF for train and test data (actually a step function) cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1]))) cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1]))) # CDF for predicted data cdf_pred_systole = real_to_cdf(pred_systole, loss_systole) cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole) cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole) cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole) # evaluate CRPS on training data crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole))) print('CRPS(train) = {0}'.format(crps_train)) # evaluate CRPS on test data crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole))) print('CRPS(test) = {0}'.format(crps_test)) # save best (lowest) val losses in file (to be later used for generating submission) with open(prefix + 'val_loss.txt', mode='w+') as f: if run == 0 or run == 1: f.write(str(min(hist_systole.history['val_loss']))) f.write('\n') if run == 0 or run == 2: f.write(str(min(hist_diastole.history['val_loss'])))
def submission(): """ Generate submission file for the trained models. """ print('Loading and compiling models...') [model, train_fn, val_fn, predict_fn] = get_model() print('Loading models weights...') with np.load('weights_best.hdf5.npz') as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] set_all_param_values(model, param_values) # load val losses to use as sigmas for CDF # with open('val_loss.txt', mode='r') as f: # val_loss_systole = float(f.readline()) # val_loss_diastole = float(f.readline()) print('Loading validation data...') X, ids = load_validation_data() print('Pre-processing images...') #X = preprocess(X) #batch_size = 32 pred_systole = np.zeros([X.shape[0], 600]) pred_diastole = np.zeros([X.shape[0], 600]) print('Predicting on validation data...') nb_of_batches = 10 list_indexes = np.linspace(0,X.shape[0],nb_of_batches + 1,dtype=np.int) for i in range(nb_of_batches): pred = predict_fn(X[list_indexes[i]:list_indexes[i+1]]) pred_systole[list_indexes[i]:list_indexes[i+1]] = pred[:,:600] pred_diastole[list_indexes[i]:list_indexes[i+1]] = pred[:,600:] # CDF for train and test prediction cdf_pred_systole = np.cumsum(pred_systole, axis=1) cdf_pred_diastole = np.cumsum(pred_diastole, axis=1) print('Accumulating results...') sub_systole = accumulate_study_results(ids, cdf_pred_systole) sub_diastole = accumulate_study_results(ids, cdf_pred_diastole) # write to submission file print('Writing submission to file...') fi = csv.reader(open('data/sample_submission_validate.csv')) f = open('submission.csv', 'w') fo = csv.writer(f, lineterminator='\n') fo.writerow(fi.next()) for line in fi: idx = line[0] key, target = idx.split('_') key = int(key) out = [idx] if key in sub_systole: if target == 'Diastole': out.extend(list(sub_diastole[key][0])) else: out.extend(list(sub_systole[key][0])) else: print('Miss {0}'.format(idx)) fo.writerow(out) f.close()
def load_model(input_shape, n_outputs=100): '''Loads and compiles pre-trained model to be used for real-time predictions''' model = m.get_model(input_size=input_shape, n_outputs=n_outputs) model.load_weights("pre_trained_weights/latest_model_weights.hdf5") model.compile(loss='categorical_crossentropy', optimizer='adam') return model
def build_submission(config): model_systole = get_model() model_diastole = get_model() print('Loading models weights...') model_systole.load_weights(config.systole_weights) model_diastole.load_weights(config.diastole_weights) # load val losses to use as sigmas for CDF with open(config.val_loss_systole, 'r') as f: val_loss_systole = float(f.readline()) with open(config.val_loss_diastole, 'r') as f: val_loss_diastole = float(f.readline()) print('Loading validation data...') X, ids, mult = load_validation_data() batch_size = 32 print('Predicting on validation data...') pred_normed_systole = model_systole.predict(X, batch_size=batch_size, verbose=1) pred_normed_diastole = model_diastole.predict(X, batch_size=batch_size, verbose=1) print('Normed_systole:', pred_normed_systole.shape) print('Normed_diastole:', pred_normed_diastole.shape) print('mult:', mult.shape) pred_systole = pred_normed_systole[:,0] * mult pred_diastole = pred_normed_diastole[:,0] * mult print('systole:', pred_systole.shape) print('diastole:', pred_diastole.shape) # real predictions to CDF cdf_pred_systole = real_to_cdf(pred_systole, val_loss_systole) cdf_pred_diastole = real_to_cdf(pred_diastole, val_loss_diastole) print('Accumulating results...') sub_systole = accumulate_study_results(ids, cdf_pred_systole) sub_diastole = accumulate_study_results(ids, cdf_pred_diastole) # write to submission file print('Writing submission to file...') fi = csv.reader(open('/data/sample_submission_validate.csv')) f = open(config.submission, 'w') fo = csv.writer(f, lineterminator='\n') fo.writerow(next(fi)) for line in fi: idx = line[0] key, target = idx.split('_') key = int(key) out = [idx] if key in sub_systole: if target == 'Diastole': out.extend(list(sub_diastole[key][0])) else: out.extend(list(sub_systole[key][0])) else: print('Miss {0}'.format(idx)) fo.writerow(out) f.close() print('Done.')
from flask import Flask, render_template import preprocessor from preprocessor import preprocess_text import model from model import get_model import tokenizer from tokenizer import get_tokenizer import pandas as pd import numpy as np from keras.preprocessing.sequence import pad_sequences from mappings import map_emoji tokenizer = get_tokenizer() model = get_model() app = Flask(__name__) @app.route('/<string:text>') def index(text): text_processed = preprocess_text(text) df_test = pd.DataFrame({'Text': [text_processed]}) X_predict = tokenizer.texts_to_sequences(df_test['Text'].values) print("Here") X_predict = pad_sequences(X_predict) pred = model.predict(X_predict) pred_labels = np.argmax(pred, axis=1) pred_emojis = map_emoji(pred_labels) return pred_emojis[0]
def train(config): self_summary_strs = [] # summary string to print out for later # first, read both data and filter stuff, to get the word2vec idx, train_data = read_data(config, 'train', config.load) val_data = read_data( config, 'val', True ) # dev should always load model shared data(word2idx etc.) from train # now that the dataset is loaded , we get the max_word_size from the dataset # then adjust the max based on the threshold as well # also get the vocab size config_vars = vars(config) str_ = "threshold setting--\n" + "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.thresmeta]) print str_ self_summary_strs.append(str_) # cap the numbers # max sentence word count etc. update_config(config, [train_data, val_data], showMeta=True) # all word num is <= max_thres str_ = "renewed ----\n" + "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.maxmeta]) print str_ self_summary_strs.append(str_) # now we initialize the matrix for word embedding for word not in glove word2vec_dict = train_data.shared['word2vec'] word2idx_dict = train_data.shared[ 'word2idx'] # this is the word not in word2vec # we are not fine tuning , so this should be empty idx2vec_dict = { word2idx_dict[word]: vec for word, vec in word2vec_dict.items() if word in word2idx_dict } #print len(idx2vec_dict) # 0 # config.word_vocab_size = len(train_data.shared['word2idx']) # the word not in word2vec # so the emb_mat should all be a random vector # np.random.multivariate_normal gets mean of zero and co of 1 for each dim, like #>>> np.random.multivariate_normal(np.zeros(5),np.eye(5)) #array([-0.73663652, -1.16417783, -0.74083293, -0.80016731, 0.060182 ]) # random initial embedding matrix for new words config.emb_mat = np.array([ idx2vec_dict[idx] if idx2vec_dict.has_key(idx) else np.random.multivariate_normal( np.zeros(config.word_emb_size), np.eye(config.word_emb_size)) for idx in xrange(config.word_vocab_size) ], dtype="float32") model = get_model(config) # construct model under gpu0 trainer = Trainer(model, config) tester = Tester(model, config) saver = tf.train.Saver(max_to_keep=5) # how many model to keep bestsaver = tf.train.Saver(max_to_keep=5) # just for saving the best model save_period = config.save_period # also the eval period # start training! # allow_soft_placement : tf will auto select other device if the tf.device(*) not available tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all # or you can set hard limit #tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4 with tf.Session(config=tfconfig) as sess: # calculate total parameters totalParam = cal_total_param() str_ = "total parameters: %s" % (totalParam) print str_ self_summary_strs.append(str_) initialize(load=config.load, load_best=config.load_best, model=model, config=config, sess=sess) # the total step (iteration) the model will run last_time = time.time() # total / batchSize * epoch num_steps = int( math.ceil(train_data.num_examples / float(config.batch_size))) * config.num_epochs # get_batches is a generator, run on the fly # there will be num_steps batch str_ = " batch_size:%s, epoch:%s,total step:%s,eval/save every %s steps" % ( config.batch_size, config.num_epochs, num_steps, config.save_period) print str_ self_summary_strs.append(str_) best = { "acc": 0.0, "step": -1 } # remember the best eval acc during training finalAcc = None isStart = True for batch in tqdm(train_data.get_batches(config.batch_size, num_steps=num_steps), total=num_steps): # each batch has (batch_idxs,Dataset(batch_data, full_shared)) # batch_data has {"q":,"y":..."pidx2feat",.."photo_idxs"..} global_step = sess.run(model.global_step) + 1 # start from 0 # if load from existing model, save if first if config.load and isStart: tqdm.write("saving original model...") tqdm.write("\tsaving model...") saver.save(sess, config.save_dir_model, global_step=global_step) tqdm.write("\tdone") isStart = False id2predanswers = {} id2realanswers = {} for evalbatch in val_data.get_batches( config.batch_size, num_steps=config.val_num_batches, shuffle=False, cap=True): yp = tester.step( sess, evalbatch ) # [N,4] # id2realanswersprob for each answer pred, gt = getAnswers( yp, evalbatch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) evalAcc = getEvalScore(id2predanswers, id2realanswers) tqdm.write( "\teval on validation %s batches Acc:%s, (best:%s at step %s) " % (config.val_num_batches, evalAcc, best['acc'], best['step'])) # remember the best acc if (evalAcc > best['acc']): best['acc'] = evalAcc best['step'] = global_step # save the best model tqdm.write("\t saving best model...") bestsaver.save(sess, config.save_dir_best_model, global_step=global_step) tqdm.write("\t done.") finalAcc = evalAcc loss, train_op = trainer.step(sess, batch) if global_step % save_period == 0: # time to save model duration = time.time() - last_time # in seconds sec_per_step = duration / float(save_period) last_time = time.time() #use tqdm to print tqdm.write( "step:%s/%s (epoch:%.3f), took %s, loss:%s, estimate remaining:%s" % (global_step, num_steps, (config.num_epochs * global_step / float(num_steps)), sec2time(duration), loss, sec2time((num_steps - global_step) * sec_per_step))) tqdm.write("\tsaving model...") saver.save(sess, config.save_dir_model, global_step=global_step) tqdm.write("\tdone") id2predanswers = {} id2realanswers = {} for evalbatch in val_data.get_batches( config.batch_size, num_steps=config.val_num_batches, shuffle=False, cap=True): yp = tester.step( sess, evalbatch ) # [N,4] # id2realanswersprob for each answer pred, gt = getAnswers( yp, evalbatch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) evalAcc = getEvalScore(id2predanswers, id2realanswers) tqdm.write( "\teval on validation %s batches Acc:%s, (best:%s at step %s) " % (config.val_num_batches, evalAcc, best['acc'], best['step'])) # remember the best acc if (evalAcc > best['acc']): best['acc'] = evalAcc best['step'] = global_step # save the best model tqdm.write("\t saving best model...") bestsaver.save(sess, config.save_dir_best_model, global_step=global_step) tqdm.write("\t done.") finalAcc = evalAcc if global_step % save_period != 0: # time to save model saver.save(sess, config.save_dir_model, global_step=global_step) str_ = "best eval on val Accurucy: %s at %s step, final step %s Acc is %s" % ( best['acc'], best['step'], global_step, finalAcc) print str_ self_summary_strs.append(str_) if config.write_self_sum: f = open(config.self_summary_path, "w") f.writelines("%s" % ("\n".join(self_summary_strs))) f.close()
def main(_): start_time = time.time() logging.info("TensorFlow version: %s", tf.__version__) logging.info("TensorFlow git version: %s", tf.__git_version__) if KUBERNETES: tf_config_json = os.environ.get("TF_CONFIG", "{}") tf_config = json.loads(tf_config_json) logging.info("tf_config: {}".format(tf_config)) task = tf_config.get("task", {}) task_index = task["index"] job_name = task["type"] logging.info("task: {}".format(task)) cluster_spec = tf_config.get("cluster", {}) logging.info("cluster_spec: {}".format(cluster_spec)) else: # Local testing task_index = FLAGS.task_index job_name = FLAGS.job_name cluster_spec = {"ps": ["localhost:2222"], "worker": ["localhost:2223", "localhost:2224"]} worker_list = cluster_spec.get("worker", "{}") ps_list = cluster_spec.get("ps", "{}") logging.info("job_name: {}".format(job_name)) logging.info("task_index: {}".format(task_index)) config = tf.ConfigProto( inter_op_parallelism_threads=num_inter_op_threads, intra_op_parallelism_threads=num_intra_op_threads) cluster = tf.train.ClusterSpec(cluster_spec) server = tf.train.Server(cluster, job_name=job_name, task_index=task_index) is_chief = (task_index == 0) # Am I the chief node (always task 0) if job_name == "ps": logging.info("I am parameter server #{}. " "I will join the server and will " "need to be explicitly terminated when all jobs end. " "Kubernetes should do this automatically." "Otherwise, use CTRL-\\ for manual termination". format(task_index)) server.join() elif job_name == "worker": if is_chief: logging.info("I am the chief worker {} with task #{}".format( worker_list[task_index], task_index)) else: logging.info("I am worker {} with task #{}".format( worker_list[task_index], task_index)) # Graph worker_device = "/job:{}/task:{}".format(job_name, task_index) setter = tf.train. \ replica_device_setter(ps_tasks=len(ps_list), worker_device=worker_device) with tf.device(setter): """ BEGIN: Data loader """ data = load_data() """ END: Data loader """ """ BEGIN: MODEL DEFINE """ model = get_model(data, FLAGS, is_chief, len(worker_list)) """ END: MODEL DEFINE """ """ Monitored Training Session """ checkpoint_dir = None # if is_chief: # checkpoint_dir = FLAGS.CHECKPOINTS_DIRECTORY # else: # checkpoint_dir = None params = dict(master=server.target, is_chief=is_chief, config=config, hooks=model["hooks"], checkpoint_dir=checkpoint_dir, stop_grace_period_secs=10) with tf.train.MonitoredTrainingSession(**params) as sess: logging.info("Starting training on worker {}".format( task_index)) batch_start = 0 batch_stop = batch_start + FLAGS.batch_size epoch = 1 while not sess.should_stop(): x_train = data["x_train"][batch_start:batch_stop] y_train = data["y_train"][batch_start:batch_stop] _, loss, step = sess.run([model["optimizer"], model["loss"], model["global_step"]], feed_dict={ model["input_tensor"]: x_train, model["label_tensor"]: y_train}) batch_start += FLAGS.batch_size if batch_start >= data["num_train"]: batch_start = 0 epoch += 1 logging.info("Epoch {}, Batch Size = {}".format( epoch, FLAGS.batch_size)) batch_stop = batch_start + FLAGS.batch_size logging.info("worker {}, step {} of {}: loss = {:.4}". format(task_index, step, FLAGS.steps_to_train, loss)) if ((step % FLAGS.steps_to_validate) == 0) and is_chief: loss, accuracy = sess.run([model["loss"], model["accuracy"]], feed_dict={ model["input_tensor"]: data["x_test"], model["label_tensor"]: data["y_test"]}) logging.info("{} test validation: loss = {:.4f}" ", accuracy of the model is {:.3f}".format( data["name"], loss, accuracy)) logging.info("Finished on task {} in {} seconds".format( task_index, time.time() - start_time)) logging.info( "Session from worker {} closed cleanly".format(task_index))
def test(config): if config.is_test_on_val: test_data = read_data(config, 'val', True) print "total val samples:%s" % test_data.num_examples else: test_data = read_data( config, 'test', True ) # here will load shared.p from config.outpath (outbase/modelname/runId/) print "total test samples:%s" % test_data.num_examples # get the max_sent_size and other stuff print "threshold setting--" config_vars = vars(config) print "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.thresmeta]) # cap the numbers update_config(config, [test_data], showMeta=True) print "renewed ----" print "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.maxmeta]) model = get_model(config) # update each batch forward into this dict id2predanswers = {} id2realanswers = {} id2yp = {} tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all # or you can set hard limit #tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4 with tf.Session(config=tfconfig) as sess: initialize(load=True, load_best=config.load_best, model=model, config=config, sess=sess) if config.is_save_weights: weight_dict = {} weight_sum = open(os.path.join(config.weights_path, "all.txt"), "w") for var in tf.trainable_variables(): shape = var.get_shape() weight_sum.writelines("%s %s\n" % (var.name, shape)) var_val = sess.run(var) weight_dict[var.name] = var_val np.savez(os.path.join(config.weights_path, "weights.npz"), **weight_dict) weight_sum.close() last_time = time.time() # num_epoch should be 1 num_steps = int( math.ceil(test_data.num_examples / float(config.batch_size))) * config.num_epochs # load the graph and variables tester = Tester(model, config, sess) count = 0 print "total step:%s" % num_steps for batch in tqdm(test_data.get_batches(config.batch_size, num_steps=num_steps, shuffle=False), total=num_steps): count += 1 yp = tester.step( sess, batch) # [N,4] # id2realanswersprob for each answer if config.get_yp: pred, gt, yp = getAnswers_yp(yp, batch) id2yp.update(yp) else: pred, gt = getAnswers(yp, batch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) acc = getEvalScore(id2predanswers, id2realanswers) print "done, got %s answers, accuracy:%s" % (len(id2predanswers), acc) json.dump(id2predanswers, open("%s/answers.json" % config.val_path, "w")) if config.get_yp: json.dump({id_: "%s" % (id2yp[id_]) for id_ in id2yp}, open("%s/yps.json" % config.val_path, "w"))
NUM_LAYERS = args['num_layers'] MODEL_WEIGHTS = args['model_weights'] if TAR_PATH is not None: untar(TAR_PATH) if TEST_TAR_PATH is not None: untar(TEST_TAR_PATH) X, Y, input_vocab, output_vocab, input_dict, output_dict = prep_data( DATASET_PATH, INPUT_LENGTH, OUTPUT_LENGTH) X_test, Y_test, test_input_vocab, test_output_vocab, _, _ = prep_data( TEST_PATH, INPUT_LENGTH, OUTPUT_LENGTH) auto_encoder = get_model(len(input_vocab), INPUT_LENGTH, len(output_vocab), OUTPUT_LENGTH, HIDDEN_SIZE, NUM_LAYERS) auto_encoder.load_weights(MODEL_WEIGHTS) auto_encoder.summary() expected_codes = vec_to_words(Y_test, test_output_vocab) tasks = input_vec_to_words(X_test, test_input_vocab) codes = predict_codes(auto_encoder, X_test, input_dict, INPUT_LENGTH, output_vocab) print('*************************************************') print('MODEL EVALUATION') print(average_bleu(expected_codes, codes)) print(average_code_compilance(codes)) pp_results(tasks, codes) print('*************************************************')
def train(): """ model training """ model = get_model() model.train()
X_train.append(training_set_scaled[i - 30:i, 0]) y_train.append(training_set_scaled[i, 0]) X_train, y_train = np.array(X_train), np.array(y_train) X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1)) X_test = [] y_test = [] for i in range(30, test_set.shape[0]): X_test.append(test_set_scaled[i - 30:i, 0]) y_test.append(test_set_scaled[i, 0]) X_test, y_test = np.array(X_test), np.array(y_test) X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1)) model = get_model(X_train) model.compile(loss='mse', optimizer='Adam') model.fit(X_train[:X_train.shape[0] * 80 // 100], y_train[:X_train.shape[0] * 80 // 100], epochs=100, batch_size=512, validation_data=(X_train[X_train.shape[0] * 80 // 100:], y_train[X_train.shape[0] * 80 // 100:]), verbose=2, shuffle=False) model.save(os.path.join(experiment_name, "model.h5"))
def denoise(self): # DEBUG, INFO, WARN, ERROR, or FATAL tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.FATAL) # silence deprecate warning message config = tf.ConfigProto() # for ubuntu nvidia driver config.gpu_options.allow_growth = True # for ubuntu nvidia driver tf.keras.backend.set_session(tf.Session(config=config)) # for ubuntu nvidia driver image_dir = self.input_dir weight_file = self.weight_path is_sharpening = False val_noise_model = get_noise_model('gaussian,20,20') model = get_model(self.model_category) model.load_weights(weight_file) # if args.output_dir: # output_dir = Path(args.output_dir) # output_dir.mkdir(parents=True, exist_ok=True) # # save_arg_filename = Path(output_dir) / 'args.txt' # with open(str(save_arg_filename), 'w') as f: # json.dump(args.__dict__, f, indent=2) # --kw: glob -> rglob-- image_paths = list(Path(image_dir).expanduser().rglob("*.png")) image_paths += list(Path(image_dir).expanduser().rglob("*.tif")) image_paths += list(Path(image_dir).expanduser().rglob("*.jpg")) image_paths += list(Path(image_dir).expanduser().rglob("*.bmp")) for image_path in tqdm(image_paths): image_gray = cv2.imread(str(image_path), cv2.IMREAD_GRAYSCALE) image = cv2.merge([image_gray, image_gray, image_gray]) h, w, _ = image.shape image = image[:(h // 16) * 16, :(w // 16) * 16] # for stride (maximum 16) h, w, _ = image.shape out_image = np.zeros((h, w * 3, 3), dtype=np.uint8) noise_image = val_noise_model(image) prediction = model.predict(np.expand_dims(noise_image, 0)) denoised_image = self.clip_image(prediction[0]) # if is_sharpening == 'True': # print('sharpening') # dst = cv2.Laplacian(denoised_image, cv2.CV_64F, ksize=1) # denoised_image_float = denoised_image.astype(np.float64) # denoised_image = denoised_image_float - dst # denoised_image = np.clip(denoised_image, 0, 255).astype(np.uint8) out_image[:, :w] = image out_image[:, w:w * 2] = noise_image out_image[:, w * 2:] = denoised_image out_image = cv2.cvtColor(out_image, cv2.COLOR_BGR2GRAY) # diff = cv2.absdiff(out_image[:, w * 2:], out_image[:, :w]) # fig_diff, ax = plt.subplots(figsize=(10, 5)) # fig_diff.subplots_adjust(hspace=0.2, wspace=0.2) # 設定子圖的間隔 # plt.subplot(1, 2, 1) # plt.imshow(diff, vmin=0, vmax=255) # plt.subplot(1, 2, 2) # plt.hist(diff.ravel(), bins=256, range=[0, 256]) # plt.show() if not self.debugMode: output_dir = Path(self.output_dir).expanduser().resolve() # print('output_dir', output_dir) rawImgNameWithoutExtension = str(Path(image_path).stem) save_file_name = output_dir / str('deN-'+rawImgNameWithoutExtension+'.png') # save_histogram_name = str(save_img_name)[:-4] + '_histogram.png' # print('save_img_name', save_img_name) # print('save_histogram_name', save_histogram_name) if not Path(output_dir).exists(): os.makedirs(output_dir) cv2.imwrite(str(save_file_name), out_image) # plt.savefig(save_histogram_name, dpi=300) # plt.close() else: cv2.imshow("result", out_image) key = cv2.waitKey(-1) # "q": quit if key == 113: return 0
model = joblib.load(f) elif os.path.basename(args.model).endswith('.pt'): # pytorch model align_data = preprocessor.align_data train_target = preprocessor.target import torch import torch.nn.functional as F device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") normalLogger.debug('load model .pt file...') model, _ = get_model( args.algorithm, #scaler=scaler, in_features=len(align_data.columns), num_classes=len(set(train_target)), mid_features=256) state = torch.load(args.model) #state = torch.load('./checkpoint/epoch-5-val_loss0.037-val_acc0.972.pt') model.load_state_dict(state['state_dict']) print(model) model.eval() normalLogger.debug('successfuly load model') except: normalLogger.debug( 'fail to load model...check model path or do training model') assert False, 'fail to load model'
type=str, default="config.yaml", help='path of config file') args = parser.parse_args() with open(args.config, 'r') as file: config = yaml.load(file) paths = config["paths"] data = config["data"] dataloader = DataLoader(config) dataloader.load() input_shape = (data["imsize"], data["imsize"], data["imchannel"]) model = get_model(input_shape, config, top=False) model.load_weights(paths["load"], by_name=True) X_batch, y_batch = dataloader.get_random_batch(k=-1) #embeddings = X_batch.reshape(-1, 784) embeddings = model.predict(X_batch, batch_size=config["train"]["batch-size"], verbose=1) tsne = TSNE(n_components=2, perplexity=config["tsne"]["perplexity"], verbose=1, n_iter=config["tsne"]["n_iter"]) tsne_embeds = tsne.fit_transform(embeddings)
def main(): opt = parse_args() print(json.dumps(vars(opt), indent=2)) rootpath = opt.rootpath collectionStrt = opt.collectionStrt resume = os.path.join(opt.logger_name, opt.checkpoint_name) if not os.path.exists(resume): logging.info(resume + ' not exists.') sys.exit(0) checkpoint = torch.load(resume) start_epoch = checkpoint['epoch'] best_rsum = checkpoint['best_rsum'] print("=> loaded checkpoint '{}' (epoch {}, best_rsum {})".format( resume, start_epoch, best_rsum)) options = checkpoint['opt'] # collection setting testCollection = opt.testCollection collections_pathname = options.collections_pathname collections_pathname['test'] = testCollection trainCollection = options.trainCollection output_dir = resume.replace(trainCollection, testCollection) if 'checkpoints' in output_dir: output_dir = output_dir.replace('/checkpoints/', '/results/') else: output_dir = output_dir.replace( '/%s/' % options.cv_name, '/results/%s/%s/' % (options.cv_name, trainCollection)) result_pred_sents = os.path.join(output_dir, 'id.sent.score.txt') pred_error_matrix_file = os.path.join(output_dir, 'pred_errors_matrix.pth.tar') if checkToSkip(pred_error_matrix_file, opt.overwrite): sys.exit(0) makedirsforfile(pred_error_matrix_file) log_config(output_dir) logging.info(json.dumps(vars(opt), indent=2)) # data loader prepare test_cap = os.path.join(rootpath, collections_pathname['test'], 'TextData', '%s.caption.txt' % testCollection) if collectionStrt == 'single': test_cap = os.path.join( rootpath, collections_pathname['test'], 'TextData', '%s%s.caption.txt' % (testCollection, opt.split)) elif collectionStrt == 'multiple': test_cap = os.path.join(rootpath, collections_pathname['test'], 'TextData', '%s.caption.txt' % testCollection) else: raise NotImplementedError('collection structure %s not implemented' % collectionStrt) caption_files = {'test': test_cap} img_feat_path = os.path.join(rootpath, collections_pathname['test'], 'FeatureData', options.visual_feature) visual_feats = {'test': BigFile(img_feat_path)} assert options.visual_feat_dim == visual_feats['test'].ndims video2frames = { 'test': read_dict( os.path.join(rootpath, collections_pathname['test'], 'FeatureData', options.visual_feature, 'video2frames.txt')) } # set bow vocabulary and encoding bow_vocab_file = os.path.join(rootpath, collections_pathname['train'], 'TextData', 'vocabulary', 'bow', options.vocab + '.pkl') bow_vocab = pickle.load(open(bow_vocab_file, 'rb')) bow2vec = get_text_encoder('bow')(bow_vocab) options.bow_vocab_size = len(bow_vocab) # set rnn vocabulary rnn_vocab_file = os.path.join(rootpath, collections_pathname['train'], 'TextData', 'vocabulary', 'rnn', options.vocab + '.pkl') rnn_vocab = pickle.load(open(rnn_vocab_file, 'rb')) options.vocab_size = len(rnn_vocab) # Construct the model model = get_model(options.model)(options) model.load_state_dict(checkpoint['model']) model.Eiters = checkpoint['Eiters'] model.val_start() # set data loader video_ids_list = data.read_video_ids(caption_files['test']) vid_data_loader = data.get_vis_data_loader(visual_feats['test'], opt.batch_size, opt.workers, video2frames['test'], video_ids=video_ids_list) text_data_loader = data.get_txt_data_loader(caption_files['test'], rnn_vocab, bow2vec, opt.batch_size, opt.workers) # mapping if options.space == 'hybrid': video_embs, video_tag_probs, video_ids = evaluation.encode_text_or_vid_tag_hist_prob( model.embed_vis, vid_data_loader) cap_embs, cap_tag_probs, caption_ids = evaluation.encode_text_or_vid_tag_hist_prob( model.embed_txt, text_data_loader) else: video_embs, video_ids = evaluation.encode_text_or_vid( model.embed_vis, vid_data_loader) cap_embs, caption_ids = evaluation.encode_text_or_vid( model.embed_txt, text_data_loader) v2t_gt, t2v_gt = metrics.get_gt(video_ids, caption_ids) logging.info("write into: %s" % output_dir) if options.space != 'latent': tag_vocab_path = os.path.join( rootpath, collections_pathname['train'], 'TextData', 'tags', 'video_label_th_1', 'tag_vocab_%d.json' % options.tag_vocab_size) evaluation.pred_tag(video_tag_probs, video_ids, tag_vocab_path, os.path.join(output_dir, 'video')) evaluation.pred_tag(cap_tag_probs, caption_ids, tag_vocab_path, os.path.join(output_dir, 'text')) if options.space in ['latent', 'hybrid']: # logging.info("=======Latent Space=======") t2v_all_errors_1 = evaluation.cal_error(video_embs, cap_embs, options.measure) if options.space in ['concept', 'hybrid']: # logging.info("=======Concept Space=======") t2v_all_errors_2 = evaluation.cal_error_batch(video_tag_probs, cap_tag_probs, options.measure_2) if options.space in ['hybrid']: w = 0.6 t2v_all_errors_1 = norm_score(t2v_all_errors_1) t2v_all_errors_2 = norm_score(t2v_all_errors_2) t2v_tag_all_errors = w * t2v_all_errors_1 + (1 - w) * t2v_all_errors_2 cal_perf(t2v_tag_all_errors, v2t_gt, t2v_gt) torch.save( { 'errors': t2v_tag_all_errors, 'videos': video_ids, 'captions': caption_ids }, pred_error_matrix_file) logging.info("write into: %s" % pred_error_matrix_file) elif options.space in ['latent']: cal_perf(t2v_all_errors_1, v2t_gt, t2v_gt) torch.save( { 'errors': t2v_all_errors_1, 'videos': video_ids, 'captions': caption_ids }, pred_error_matrix_file) logging.info("write into: %s" % pred_error_matrix_file)
def predict(*args, **kwargs): def remove_sents(sentences_t, labels_t, max_length): remove_idxs = [] for i, sentence in enumerate(sentences_t): if len(sentence) > max_length: remove_idxs.append(i) for i in remove_idxs: sentences_t.pop(i) labels_t.pop(i) word_index = {} label_index = {} max_length = 0 with open(configs.DICT_FILE, 'r', encoding='utf-8') as file: dicts = file.read() dicts = dicts.split("\n") word_index = eval(dicts[0]) label_index = eval(dicts[1]) max_length = eval(dicts[2]) with open(configs.EMBEDDINGS_FILE, 'rb') as file: word_embeddings = np.load(file) #Loading test sequences sentences_t, labels_t, max_length_t = prd.read_data(configs.TEST_FILE) remove_sents(sentences_t, labels_t, max_length) print('Total no of test sequences: ', len(sentences_t)) char_index = prd.get_vocabulory(word_index) char_idxs = prd.get_chars(sentences_t, max_length, char_index) label_idxs = prd.get_sequence_indices(labels_t, label_index, max_length) seq_idxs = prd.get_sequence_indices(sentences_t, word_index, max_length) assert (len(seq_idxs) == len(label_idxs) ), "length of I/O sequences doesn't match" index_labels = {} for item, i in label_index.items(): index_labels[i] = item model_t = mdl.get_model(word_embeddings, max_length, len(char_index), len(index_labels), True) # Predict labels for test data pred_label = np.asarray(model_t.predict([seq_idxs, char_idxs])) pred_label = np.argmax(pred_label, axis=-1) #Skipping padded sequences pred_label = prd.get_orig_labels(pred_label, index_labels, labels_t) print("Predicted Labels--->\n", pred_label[0]) outputfile = configs.OUTPUT_FILE with open(outputfile, 'w', encoding='utf-8') as f: f.write('WORD' + ' ' + 'TRUE_LABEL' + ' ' + 'PRED_LABEL') f.write('\n') f.write('\n') for i in range(len(pred_label)): cur_sentences = sentences_t[i] cur_labels = labels_t[i] cur_pred = pred_label[i] for j in range(len(cur_sentences)): f.write(cur_sentences[j] + ' ' + cur_labels[j] + ' ' + cur_pred[j]) f.write('\n') f.write('\n') f.write('\n') with open(outputfile, 'r', encoding='utf-8') as f: for line in f.readlines(): print(line) return ([labels_t, pred_label])
if not args_dict.model_file: # run all training model, model_val = init_models(args_dict) model, model_name = trainloop(args_dict,model,model_val = model_val) epoch_start = args_dict.nepochs del model K.clear_session() else: epoch_start = 0 ### Fine Tune ConvNet ### # get fine tuning params in place args_dict.lr = args_dict.ftlr args_dict.nepochs = args_dict.ftnepochs model = get_model(args_dict) opt = get_opt(args_dict) if args_dict.model_file: print "Loading snapshot: ", args_dict.model_file model.load_weights(args_dict.model_file) else: model.load_weights(model_name) for i,layer in enumerate(model.layers[1].layers): if i > args_dict.finetune_start_layer: layer.trainable = True model.compile(optimizer=opt,loss='categorical_crossentropy', sample_weight_mode="temporal") args_dict.cnn_train = True
return if __name__ == "__main__": print("Loading Config") config_file = "./default-config,yaml" config = load_config_file(config_file) # dataloaders print("Getting Dataloader") test_loader, total_classes = load_dataset(config) # creating model print("Loading Model") model = get_model(num_classes=total_classes) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(device) # loading model if not os.path.isfile(config["MODEL_TEST_PATH"]): print("Model checkpoint weights does no exist, exiting....") import sys sys.exit() else: ckpt = torch.load(config["MODEL_TEST_PATH"]) if "model_state_dict" in ckpt: model.load_state_dict(ckpt["model_state_dict"]) if "accuracy" in ckpt: print( "The loaded model has Validation accuracy of: {:.2f} %\n".
import dataset import engine from model import get_model if __name__ == '__main__': data_path = 'C:/Users/Arpan/Downloads/Education/ML/Approaching-almost-any-Machine-Learning-Problem/Computer Vision/Data' device = 'cuda' df = pd.read_csv(os.path.join(data_path, 'train.csv')) epochs = 10 images = df['ImageId'].values.tolist() images = [os.path.join(data_path, 'train_png', i + '.png') for i in images] targets = df['target'].values model = get_model(pretrained=True, model_name='resnet34') model.to(device) # imagenet stats mean = (0.485, 0.456, 0.406) std = (0.229, 0.224, 0.225) aug = albumentations.Compose([ albumentations.Normalize(mean, std, max_pixel_value=255, always_apply=True) ]) train_images, val_images, train_targets, val_targets = train_test_split(
def run(epochs=500, training_percentage=0.4, validation_percentage=0.1, extract=True, cont=True, size=256, top_k=5): '''Does the routine required to get the data, put them in needed format and start training the model saves weights whenever the model produces a better test result and keeps track of the best loss''' if extract: print("Extracting data..") X, y = data.extract_data(size=size) print("Preprocessing data..") X, y, nb_samples, num_categories = data.preprocess_data(X, y, save=True, subtract_mean=True) else: print("Loading data..") h5f = h5py.File('data.hdf5', 'r') nb_samples = h5f['nb_samples'].value num_categories = h5f['n_categories'].value h5f.close() print("Number of categories: {}".format(num_categories)) print("Number of samples {}".format(nb_samples)) data_ids = np.arange(start=0, stop=nb_samples) val_ids = data.produce_validation_indices(data_ids, nb_samples * validation_percentage) train_ids = data.produce_train_indices(dataset_indx=data_ids, number_of_samples=nb_samples * training_percentage, val_indx=val_ids) # X_train, y_train, X_test, y_test = data.split_data(X, y, split_ratio=split) X_train, y_train, X_val, y_val = data.load_dataset_bit_from_hdf5(train_ids, val_ids, only_train=False) X_val = X_val / 255 print("Building and Compiling model..") model = m.get_model(n_outputs=num_categories, input_size=size) if cont: # model.load_weights_until_layer("pre_trained_weights/latest_model_weights.hdf5", 26) model.load_weights("pre_trained_weights/latest_model_weights.hdf5") model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=["accuracy"]) print("Training..") best_performance = np.inf for i in range(epochs): train_ids = data.produce_train_indices(dataset_indx=data_ids, number_of_samples=15000, val_indx=val_ids) X_train, y_train = data.load_dataset_bit_from_hdf5(train_ids, val_ids, only_train=True) X_train = X_train / 255 X_train = data.augment_data(X_train) # fit the model on the batches generated by datagen.flow() metadata = model.fit(X_train, y_train, validation_data=[X_val, y_val], batch_size=64, nb_epoch=1, verbose=1, shuffle=True, class_weight=None, sample_weight=None) current_loss = metadata.history['loss'][-1] current_val_loss = metadata.history['val_loss'][-1] preds = model.predict_proba(X_val, batch_size=64) print("Loss: {}".format(current_loss)) print("Val_loss: {}".format(current_val_loss)) top_3_error = get_top_n_error(preds, y_val, top_k) print("Top 3 error: {}".format(top_3_error)) if current_val_loss < best_performance: model.save_weights("pre_trained_weights/model_weights.hdf5", overwrite=True) best_performance = current_val_loss print("Saving weights..") model.save_weights("pre_trained_weights/latest_model_weights.hdf5", overwrite=True)
import os import glob import sys sys.path.append('/home/ubuntu/vdp/clevr_inference/scene_parse/attr_net') import json from options import get_options from datasets import get_dataloader from model import get_model import utils COMP_CAT_DICT_PATH = '/home/ubuntu/vdp/clevr_inference/scene_parse/attr_net/tools/clevr_comp_cat_dict.json' opt = get_options('test') test_loader = get_dataloader(opt, 'test') model = get_model(opt) if opt.use_cat_label: with open(COMP_CAT_DICT_PATH) as f: cat_dict = utils.invert_dict(json.load(f)) if opt.dataset == 'clevr': num_images = len(glob.glob(opt.clevr_val_img_dir + '/*.png')) scenes = [{ 'image_index': i, 'image_filename': 'CLEVR_val_%06d.png' % i, 'objects': [] } for i in range(num_images)] # print("run_test.py", scenes) count = 0
def training(model_name): feature, label, _ = data.get_matrix('train', True) classifier = model.get_model(model_name) classifier.fit(feature, label) return classifier
vocab = pickle.load(open(os.path.join(data_dir, 'instr_vocab.pkl'),'rb')) ingr_vocab_size = len(ingrs_vocab) instrs_vocab_size = len(vocab) output_dim = instrs_vocab_size print(ingr_vocab_size) print(instrs_vocab_size) kk = 'cuda' device = torch.device(kk) t= time.time() import sys sys.argv = [''] del sys args = get_parser() args.maxseqlen = 15 args.ingr_only = False model = get_model(args,ingr_vocab_size,instrs_vocab_size) model.load_state_dict(torch.load('../checkpoints/inversecooking/im2ingr/checkpoints/modelbest.ckpt', map_location = None)) model.to(device) model.eval() model.ingrs_only = False model.recipe_only = False transf_list_batch = [] transf_list_batch.append(transforms.ToTensor()) transf_list_batch.append(transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225))) to_input_transf = transforms.Compose(transf_list_batch) greedy = [True, False, False, False] beam = [-1, -1, -1, -1] temperature = 1.0
def train(): print('Loading and compiling models...') model_systole = get_model() model_diastole = get_model() print('Loading training data...') X, y = load_train_data() print('Pre-processing images...') X = preprocess(X) X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2) nb_iter = 200 epochs_per_iter = 1 batch_size = 32 calc_crps = 1 min_val_loss_systole = sys.float_info.max min_val_loss_diastole = sys.float_info.max print('-'*50) print('Training...') print('-'*50) for i in range(nb_iter): print('-'*50) print('Iteration {0}/{1}'.format(i + 1, nb_iter)) print('-'*50) print('Augmenting images - rotations') X_train_aug = rotation_augmentation(X_train, 15) print('Augmenting images - shifts') X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1) print('Fitting systole model...') hist_systole = model_systole.fit(X_train_aug, y_train[:, 0], shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data=(X_test, y_test[:, 0])) print('Fitting diastole model...') hist_diastole = model_diastole.fit(X_train_aug, y_train[:, 1], shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data=(X_test, y_test[:, 1])) loss_systole = hist_systole.history['loss'][-1] loss_diastole = hist_diastole.history['loss'][-1] val_loss_systole = hist_systole.history['val_loss'][-1] val_loss_diastole = hist_diastole.history['val_loss'][-1] if calc_crps > 0 and i % calc_crps == 0: print('Evaluating CRPS...') pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1) pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1) val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1) val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1) cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1]))) cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1]))) cdf_pred_systole = real_to_cdf(pred_systole, loss_systole) cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole) cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole) cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole) crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole))) print('CRPS(train) = {0}'.format(crps_train)) crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole))) print('CRPS(test) = {0}'.format(crps_test)) print('Saving weights...') model_systole.save_weights('weights_systole.hdf5', overwrite=True) model_diastole.save_weights('weights_diastole.hdf5', overwrite=True) if val_loss_systole < min_val_loss_systole: min_val_loss_systole = val_loss_systole model_systole.save_weights('weights_systole_best.hdf5', overwrite=True) if val_loss_diastole < min_val_loss_diastole: min_val_loss_diastole = val_loss_diastole model_diastole.save_weights('weights_diastole_best.hdf5', overwrite=True) with open('val_loss.txt', mode='w+') as f: f.write(str(min_val_loss_systole)) f.write('\n') f.write(str(min_val_loss_diastole))
# plt.ylabel('RMSE') # plt.title('RMSE Loss Function') # plt.legend('') # plt.show() # #crps # plt.plot(x,y) # plt.xlabel('Iteration') # plt.ylabel('CRPS') # plt.title('') # plt.legend('') # plt.show() print('Loading and compiling models...') model_systole = get_model() model_diastole = get_model() #import best model if it exists if os.path.isfile('/data/run2/weights_systole_best.hdf5'): print('loading weights') model_systole.load_weights('/data/run2/weights_systole_best.hdf5') if os.path.isfile('/data/run2/weights_diastole_best.hdf5'): model_diastole.load_weights('/data/run2/weights_diastole_best.hdf5') print('Loading training data...') X, y, metadata = load_train_data() print(metadata[0, :].shape, metadata[0].shape, metadata[0:1, :].shape) pred_systole = model_systole.predict({'input1':X[40:50, :, : , :], 'input2':metadata[40:50, :], 'output':y[40:50, 0]})['output']
def main(): height = 512 width = 512 noise = 'gauss' mode = 'clean' args = get_args() image_dir = args.image_dir weight_file = 'weights_{}_{}.hdf5'.format(noise, mode) #args.weight_file if mode != 'clean': val_noise_model = get_noise_model(args.test_noise_model) else: model = get_model(height, width, args.model) model.load_weights(weight_file) model.summary() # saved_model tf.saved_model.save( model, 'saved_model_{}_{}_{}_{}x{}'.format(args.model, noise, mode, height, width)) # pb full_model = tf.function(lambda inputs: model(inputs)) full_model = full_model.get_concrete_function( inputs=[tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype)]) frozen_func = convert_variables_to_constants_v2(full_model, lower_control_flow=False) frozen_func.graph.as_graph_def() tf.io.write_graph(graph_or_graph_def=frozen_func.graph, logdir=".", name="noise2noise_{}_{}_{}_{}x{}_float32.pb".format( args.model, noise, mode, height, width), as_text=False) # No Quantization - Input/Output=float32 converter = tf.lite.TFLiteConverter.from_keras_model(model) tflite_model = converter.convert() with open( 'noise2noise_{}_{}_{}_{}x{}_float32.tflite'.format( args.model, noise, mode, height, width), 'wb') as w: w.write(tflite_model) print( "tflite convert complete! - noise2noise_{}_{}_{}_{}x{}_float32.tflite". format(args.model, noise, mode, height, width)) # Weight Quantization - Input/Output=float32 converter = tf.lite.TFLiteConverter.from_keras_model(model) converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE] tflite_model = converter.convert() with open( 'noise2noise_{}_{}_{}_{}x{}_weight_quant.tflite'.format( args.model, noise, mode, height, width), 'wb') as w: w.write(tflite_model) print( 'Weight Quantization complete! - noise2noise_{}_{}_{}_{}x{}_weight_quant.tflite' .format(args.model, noise, mode, height, width)) # Float16 Quantization - Input/Output=float32 converter = tf.lite.TFLiteConverter.from_keras_model(model) converter.optimizations = [tf.lite.Optimize.DEFAULT] converter.target_spec.supported_types = [tf.float16] tflite_quant_model = converter.convert() with open( 'noise2noise_{}_{}_{}_{}x{}_float16_quant.tflite'.format( args.model, noise, mode, height, width), 'wb') as w: w.write(tflite_quant_model) print( 'Float16 Quantization complete! - noise2noise_{}_{}_{}_{}x{}_float16_quant.tflite' .format(args.model, noise, mode, height, width)) def representative_dataset_gen(): for data in raw_test_data.take(10): image = data['image'].numpy() image = tf.image.resize(image, (height, width)) image = image[np.newaxis, :, :, :] # image = image / 127.5 - 1.0 yield [image] raw_test_data, info = tfds.load(name="coco/2017", with_info=True, split="test", data_dir="~/TFDS", download=False) # Integer Quantization - Input/Output=float32 converter = tf.lite.TFLiteConverter.from_keras_model(model) converter.optimizations = [tf.lite.Optimize.DEFAULT] converter.representative_dataset = representative_dataset_gen tflite_quant_model = converter.convert() with open( 'noise2noise_{}_{}_{}_{}x{}_integer_quant.tflite'.format( args.model, noise, mode, height, width), 'wb') as w: w.write(tflite_quant_model) print( 'Integer Quantization complete! - noise2noise_{}_{}_{}_{}x{}_integer_quant.tflite' .format(args.model, noise, mode, height, width)) # Full Integer Quantization - Input/Output=int8 converter = tf.lite.TFLiteConverter.from_keras_model(model) converter.optimizations = [tf.lite.Optimize.DEFAULT] converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] converter.inference_input_type = tf.uint8 converter.inference_output_type = tf.uint8 converter.representative_dataset = representative_dataset_gen tflite_quant_model = converter.convert() with open( 'noise2noise_{}_{}_{}_{}x{}_full_integer_quant.tflite'.format( args.model, noise, mode, height, width), 'wb') as w: w.write(tflite_quant_model) print( 'Integer Quantization complete! - noise2noise_{}_{}_{}_{}x{}_full_integer_quant.tflite' .format(args.model, noise, mode, height, width)) # # EdgeTPU # import subprocess # result = subprocess.check_output(["edgetpu_compiler", "-s", "noise2noise_{}_{}_{}_{}x{}_full_integer_quant.tflite".format(args.model, noise, mode, height, width)]) # print(result) sys.exit(0) if args.output_dir: output_dir = Path(args.output_dir) output_dir.mkdir(parents=True, exist_ok=True) image_paths = list(Path(image_dir).glob("*.*")) for image_path in image_paths: image = cv2.imread(str(image_path)) h, w, _ = image.shape image = image[:(h // 16) * 16, :(w // 16) * 16] # for stride (maximum 16) h, w, _ = image.shape out_image = np.zeros((h, w * 3, 3), dtype=np.uint8) noise_image = val_noise_model(image) pred = model.predict(np.expand_dims(noise_image, 0)) denoised_image = get_image(pred[0]) out_image[:, :w] = image out_image[:, w:w * 2] = noise_image out_image[:, w * 2:] = denoised_image if args.output_dir: cv2.imwrite( str(output_dir.joinpath(image_path.name))[:-4] + ".png", out_image) else: cv2.imshow("result", out_image) key = cv2.waitKey(-1) # "q": quit if key == 113: return 0
from __future__ import print_function from keras.callbacks import LambdaCallback from model import get_model from data import data_load from config import Config from sample import print_sample (chars, char_indices, indices_char, maxlen, X, y, text) = data_load(Config) model = get_model(Config, maxlen, chars) model_name = Config['model_name'] print_sample(Config, text, maxlen, chars, char_indices, model, indices_char)
def main(): nb_epochs = config.MAXIMUM_EPOCHS batch_size = config.batch_size lr = 0.1 momentum = 0.9 model_name = 'ResNet50' image_size = config.IMAGE_SIZE output_dir = 'checkpoints' experiment_name = 'data_augmentation' early_stop_patience = config.EARLY_STOP_EPOCHS train_path = os.path.join(image_directory, 'train') validation_path = os.path.join(image_directory, 'validation') test_path = os.path.join(image_directory, 'test') PARAMS = { 'epoch_nr': nb_epochs, 'batch_size': batch_size, 'learning_rate': lr, 'momentum': momentum, # 'input_shape': (512, 32, 3), 'early_stop': early_stop_patience, 'image_size': image_size, 'network': model_name } if log_experiment: neptune.init(project_qualified_name='4ND4/sandbox') neptune_tb.integrate_with_keras() result = neptune.create_experiment(name=experiment_name, params=PARAMS) name = result.id print(name) else: name = 'debug' train_gen, val_gen, test_gen = getdata(train_path, validation_path, test_path) model = get_model(model_name=model_name, image_size=image_size, number_classes=nb_classes) sgd = SGD(lr=lr, momentum=momentum, nesterov=True) model.compile(optimizer=sgd, loss="categorical_crossentropy", metrics=[age_mae]) model.summary() output_dir = Path(__file__).resolve().parent.joinpath(output_dir) if not output_dir.exists(): output_dir.mkdir(parents=True) if not os.path.exists('checkpoints/{}'.format(name)): os.mkdir('checkpoints/{}'.format(name)) callbacks = [ EarlyStopping(monitor='val_age_mae', mode='min', verbose=1, patience=early_stop_patience), LearningRateScheduler(schedule=Schedule(nb_epochs, initial_lr=lr)), ModelCheckpoint(os.path.join(output_dir, name) + "/weights.{epoch:03d}-{val_loss:.3f}-{" "val_age_mae:.3f}.hdf5", monitor="val_age_mae", verbose=1, save_best_only=True, mode="min") ] hist = model.fit_generator(generator=train_gen, steps_per_epoch=train_gen.samples // batch_size, validation_data=val_gen, validation_steps=val_gen.samples // batch_size, epochs=nb_epochs, verbose=1, callbacks=callbacks) np.savez(str(output_dir.joinpath("history_{}.npz".format(name))), history=hist.history)
def train(): """ Training systole and diastole models. """ print('Loading and compiling models...') model_systole = get_model() model_diastole = get_model() print('Loading training data...') X, y = load_train_data() print('Pre-processing images...') X = preprocess(X) # split to training and test X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2) nb_iter = 200 epochs_per_iter = 1 batch_size = 32 calc_crps = 1 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed) # remember min val. losses (best iterations), used as sigmas for submission min_val_loss_systole = sys.float_info.max min_val_loss_diastole = sys.float_info.max print('-'*50) print('Training...') print('-'*50) for i in range(nb_iter): print('-'*50) print('Iteration {0}/{1}'.format(i + 1, nb_iter)) print('-'*50) print('Augmenting images - rotations') X_train_aug = rotation_augmentation(X_train, 15) print('Augmenting images - shifts') X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1) print('Fitting systole model...') hist_systole = model_systole.fit(X_train_aug, y_train[:, 0], shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data=(X_test, y_test[:, 0])) print('Fitting diastole model...') hist_diastole = model_diastole.fit(X_train_aug, y_train[:, 1], shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data=(X_test, y_test[:, 1])) # sigmas for predicted data, actually loss function values (RMSE) loss_systole = hist_systole.history['loss'][-1] loss_diastole = hist_diastole.history['loss'][-1] val_loss_systole = hist_systole.history['val_loss'][-1] val_loss_diastole = hist_diastole.history['val_loss'][-1] if calc_crps > 0 and i % calc_crps == 0: print('Evaluating CRPS...') pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1) pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1) val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1) val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1) # CDF for train and test data (actually a step function) cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1]))) cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1]))) # CDF for predicted data cdf_pred_systole = real_to_cdf(pred_systole, loss_systole) cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole) cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole) cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole) # evaluate CRPS on training data crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole))) print('CRPS(train) = {0}'.format(crps_train)) # evaluate CRPS on test data crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole))) print('CRPS(test) = {0}'.format(crps_test)) print('Saving weights...') # save weights so they can be loaded later model_systole.save_weights('weights_systole.hdf5', overwrite=True) model_diastole.save_weights('weights_diastole.hdf5', overwrite=True) # for best (lowest) val losses, save weights if val_loss_systole < min_val_loss_systole: min_val_loss_systole = val_loss_systole model_systole.save_weights('weights_systole_best.hdf5', overwrite=True) if val_loss_diastole < min_val_loss_diastole: min_val_loss_diastole = val_loss_diastole model_diastole.save_weights('weights_diastole_best.hdf5', overwrite=True) # save best (lowest) val losses in file (to be later used for generating submission) with open('val_loss.txt', mode='w+') as f: f.write(str(min_val_loss_systole)) f.write('\n') f.write(str(min_val_loss_diastole))
def parser(sentence): """ 依存句法分析 """ model = get_model() return model.predict(sentence)
def train(): """ Training systole and diastole models. """ print('Loading and compiling models...') model_systole = get_model(img_size) model_diastole = get_model(img_size) print('Loading training data...') X, y = load_train_data() print('Pre-processing images...') X = preprocess(X) # split to training and test X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2) # define image generator for random rotations datagen = ImageDataGenerator(featurewise_center=False, featurewise_std_normalization=False, rotation_range=15) nb_iter = 300 epochs_per_iter = 1 batch_size = 64 calc_crps = 1 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed) # remember min val. losses (best iterations), used as sigmas for submission min_val_loss_systole = sys.float_info.max min_val_loss_diastole = sys.float_info.max if not os.path.exists(STATS): os.makedirs(STATS) with open(STATS + 'RMSE_CRPS.txt', 'w') as f: names = ['train_RMSE_d', 'train_RMSE_s', 'test_RMSE_d', 'test_RMSE_s', 'train_crps', 'test_crps'] f.write('\t'.join([str(name) for name in names]) + '\n') print('-'*50) print('Training...') print('-'*50) for i in range(nb_iter): print('-'*50) print('Iteration {0}/{1}'.format(i + 1, nb_iter)) print('-'*50) print('Augmenting images - rotations') X_train_aug = rotation_augmentation(X_train, 15) print('Augmenting images - shifts') X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1) print('Fitting systole model...') hist_systole = model_systole.fit(X_train_aug, y_train[:, 0], shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data=(X_test, y_test[:, 0])) print('Fitting diastole model...') hist_diastole = model_diastole.fit(X_train_aug, y_train[:, 1], shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data=(X_test, y_test[:, 1])) # sigmas for predicted data, actually loss function values (RMSE) loss_systole = hist_systole.history['loss'][-1] loss_diastole = hist_diastole.history['loss'][-1] val_loss_systole = hist_systole.history['val_loss'][-1] val_loss_diastole = hist_diastole.history['val_loss'][-1] if calc_crps > 0 and i % calc_crps == 0: print('Evaluating CRPS...') pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1) pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1) val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1) val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1) # CDF for train and test data (actually a step function) cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1]))) cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1]))) # CDF for predicted data cdf_pred_systole = real_to_cdf(pred_systole, loss_systole) cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole) cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole) cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole) # evaluate CRPS on training data crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole))) print('CRPS(train) = {0}'.format(crps_train)) # evaluate CRPS on test data crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole))) print('CRPS(test) = {0}'.format(crps_test)) print('Saving weights...') # save weights so they can be loaded later model_systole.save_weights(MODELS + 'weights_systole.hdf5', overwrite=True) model_diastole.save_weights(MODELS + 'weights_diastole.hdf5', overwrite=True) # for best (lowest) val losses, save weights if val_loss_systole < min_val_loss_systole: min_val_loss_systole = val_loss_systole model_systole.save_weights(MODELS + 'weights_systole_best.hdf5', overwrite=True) if val_loss_diastole < min_val_loss_diastole: min_val_loss_diastole = val_loss_diastole model_diastole.save_weights(MODELS + 'weights_diastole_best.hdf5', overwrite=True) # save best (lowest) val losses in file (to be later used for generating submission) with open(MODELS + 'val_loss.txt', mode='w+') as f: f.write(str(min_val_loss_systole)) f.write('\n') f.write(str(min_val_loss_diastole)) with open(STATS + 'RMSE_CRPS.txt', 'a') as f: # train_RMSE_d train_RMSE_s test_RMSE_d test_RMSE_s train_crps test_crps rmse_values = [loss_diastole, loss_systole, val_loss_diastole, val_loss_systole] crps_values = [crps_train, crps_test] f.write('\t'.join([str(val) for val in rmse_values + crps_values]) + '\n') print('Saving stats images...') write_images(STATS) if (i != 0) & ((i + 1) % 100 == 0): print('Submitting learned model....') SUBMISSION_FOLDER = SUBMISSION + preproc_type + "/" + model_name + "/" + get_name() + "_ITERS" + str(i + 1) + "/" if not os.path.exists(SUBMISSION_FOLDER): os.makedirs(SUBMISSION_FOLDER) copyfile(MODELS + 'weights_systole_best.hdf5', SUBMISSION_FOLDER + 'weights_systole_best.hdf5') copyfile(MODELS + 'weights_diastole_best.hdf5', SUBMISSION_FOLDER + 'weights_diastole_best.hdf5') copyfile(MODELS + 'val_loss.txt', SUBMISSION_FOLDER + 'val_loss.txt') os.system('python submission.py %s %s %s' % (preproc_type, model_name, SUBMISSION_FOLDER))