def run_exploding_grads_test(net, data, labels, max_seq_len, n_pretrain): """ Visualize the increase in the norm of the gradient when gradient clipping or similar mitigation schemes are not applied. This will only be the case when gradient clipping is not yet implemented. This function will use gradient clipping as soon as its implemented and may be used for verification. :param net: list of layers in the network :param data: dims = (maxSeqLen, nVocab) input data :param labels: dims = (maxSeqLen, nVocab) labels corresponding to inputs :param max_seq_len: length of the longest sequence for which the gradient should be computed :param n_pretrain: number of pretraining iterations on network, so that the exploding gradient becomes more realistic :return: show the plot of the gradient's magnitude against the sequence length """ # Run pretraining n_seq_pretrain = 3 print(f"Pre-training for {n_pretrain} iterations...") for i in range(n_pretrain): pre_data, pre_labels = dg.generate_random_data(n_seq_pretrain, data.shape[1]) tr.run_training(net, pre_data, pre_labels, clip_gradients=False) # Empty array that is used for storing the magnitudes of the computed analytical derivatives grads = [] # Testing exploding gradients print(f"Computing plot...") for i in range(1, max_seq_len): _ = tr.run_training(net, data[0:i + 1], labels[0:i + 1], clip_gradients=True, update_weights=False) grads_tmp = [] for layer in net: grads_tmp.append( np.concatenate( [grad.flatten() for _, grad, _ in layer.get_params()])) layer.clear_stored_states() layer.clear_stored_derivs() grads_tmp = np.abs(np.concatenate(grads_tmp)) grads.append(np.mean(grads_tmp)) grads = np.array(grads) plt.figure() plt.title("Visualization of exploding gradients") plt.loglog(range(1, max_seq_len), grads) plt.xlabel("Number of Iterations") plt.ylabel("Mean Absolute Derivative") plt.show()
def main(): df = get_data_set('idao_dataset/train', save_to_csv=False) create_folds(df, 5, config) run_training(1, config, mode='clf') run_training(1, config, mode='reg') clf_preds, reg_preds = predict(config) sub_df = pd.read_csv(sub_df_path) sub_df['classification_predictions'] = clf_preds sub_df['regression_predictions'] = reg_preds sub_df['regression_predictions'] = sub_df['regression_predictions'].apply( transform) sub_df.to_csv('Final_Submission.csv', index=False)
def main(args): print(args) if (args.do_training): tra.run_training(args.dataset_dir, args.logging_dir, model_name=args.model_name, img_format=args.image_format, max_step=args.max_step) else: images = prep.readImagesInDirectory(args.predict_dir, format=args.image_format) model_dir = args.logging_dir + '/' + args.model_name + '/train' result, labels = pred.classify_images( args.model_name, model_dir, images, labels=['Hsia Yu-chiao', 'Sung Yun-hua']) print(labels) print(result)
def cross_validation(): config = {'label_csv_mame': '../2CvTrain/label.csv', 'test_label_csv_mame': '../1TestData/label.csv', 'test_img_txt_path': "../1TestData/*.txt", 'valid_label_csv_mame': '../2CvValid/label.csv', 'valid_img_txt_path': "../2CvValid/*.txt", # label csv column names 'name_list': ["img_name", "c", "gx", "gy", "gw", "gh"], 'cfg_path': "../4Others/yolo.cfg", 'params_dir': '../4Others/params.txt'} # entire trainin label prep_label_config = {'label_csv_mame': '../1TrainData/label.csv', 'img_txt_path': "../1TrainData/*.txt", 'name_list': config['name_list']} # cv train label cv_train_label_config = {'label_csv_mame': config['label_csv_mame'], 'img_txt_path': "../2CvTrain/*.txt", 'name_list': config['name_list']} torch.backends.cudnn.enabled = False torch.backends.cudnn.benchmark = False # you can specify any tuning hyper-parammeters here , but do not use # np.arrange() as it will result an error for json.dumps() tune_params = {'seed': list(range(1, 5)), 'batch_size': list(range(2, 7, 2)), 'height': [416, 448] } index, values = zip(*tune_params.items()) experiments_params = [dict(zip(index, v)) for v in it.product(*values)] # prepare label for the entire training set prep_labels(**prep_label_config) cv_results_list = [] for experiment_params in experiments_params: cv_split_config = {'n_splits': 5, 'cv': True, 'train_size': 0.7, 'name_list': config['name_list'], 'random_state': experiment_params['seed'], 'train_cv_path': "../2CvTrain/", 'valid_cv_path': "../2CvValid/", 'label_name': prep_label_config['label_csv_mame']} for _ in split_train(**cv_split_config): prep_labels(**cv_train_label_config) params = prep_params(config['params_dir'], cv_train_label_config['label_csv_mame'], experiment_params) best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\ map_frame = run_training(params=params, **config) cv_results_list.append(params) shutil.rmtree(cv_split_config['train_cv_path']) shutil.rmtree(cv_split_config['valid_cv_path']) cv_results_df = pd.DataFrame(cv_results_list) cv_results_df.to_csv(f"{params['working_dir']}cv_results.csv")
def main(): hparams = hyperparameters.hparams print("************************************") print("*********** Begin Train ************") print("************************************") print("Log directory: %s" % hparams.log_dir) #sign_changer.change_sign("./free_with_back.txt") #don't actually use loss for now loss, optimizer, accuracy, data_placeholder, label_placeholder, summaries = graph.build_graph( hparams) input_pairs = [] input_pairs.append(preprocessing.preprocess(hparams, 'butterfly')) input_pairs.append(preprocessing.preprocess(hparams, 'backstroke')) #input_pairs.append(preprocessing.preprocess(hparams, 'breastroke')) input_pairs.append(preprocessing.preprocess(hparams, 'freestyle')) #preprocessing.augment_data(hparams) train.run_training(hparams, data_placeholder, label_placeholder, optimizer, accuracy, input_pairs, summaries)
Dense(config.hidden_layers[0], input_dim=input_dim, kernel_regularizer=l2(config.l2_reg))) model.add(Activation(config.activation)) if config.batch_norm: model.add(BatchNormalization()) for n in config.hidden_layers[1:]: model.add(Dense(n)) model.add(Activation(config.activation)) if config.batch_norm: model.add(BatchNormalization()) model.add(Dense(1)) model.add(Activation(config.output_activation)) model.compile(optimizer=config.optimizer, loss=config.loss, metrics=config.metrics) K.set_value(model.optimizer.lr, config.learning_rate) return model num_trainings = 20 #used for model hyperparameter searching for __ in range(num_trainings): config = get_config( config_name) #randomly initialized config for hyperparameter search model = build_model(config) run_training(model, config) if not config.hyper_search: break
""" This script is the entrypoint to the model docker container for SageMaker. It expects a parameter of either "train" or "serve" and then triggers the appropriate script. """ from argparse import ArgumentParser from train import run_training from serve import start_server if __name__ == '__main__': parser = ArgumentParser( description='select model training or serving entrypoint') parser.add_argument('entrypoint', choices=['train', 'serve']) args = parser.parse_args() if args.entrypoint == 'train': run_training() else: start_server()
'label_csv_mame': config['label_csv_mame'], 'img_txt_path': "../1TrainData/*.txt", 'name_list': config['name_list'] } # torch.backends.cudnn.enabled = False # torch.backends.cudnn.benchmark = False # you can specify any tuning hyper-parammeters here , but do not use # np.arrange() as it will result an error for json.dumps() tune_params = { 'seed': list(range(1, 3)), 'height': [448, 480], "decay": [0.0005, 0.001], "steps": [6, 10], "optimizer": ['sgd', 'adam'], "epochs": [40, 80] } index, values = zip(*tune_params.items()) experiments_params = [dict(zip(index, v)) for v in it.product(*values)] # prepare training lables prep_labels(**prep_label_config) for experiment_params in experiments_params: params = prep_params(config['params_dir'], config['label_csv_mame'], experiment_params) try: best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\ map_frame = run_training(params=params, **config) except: continue
def train(lr=None, image_dir=None): run_training(lr, image_dir)
import optuna from functools import partial from train import objective, run_training DEVICE = "cuda" EPOCHS = 100 if __name__ == "__main__": partial_obj = partial(objective) study = optuna.create_study(direction="minimize") study.optimize(partial_obj, n_trials=150) print("Best trial:") trial_ = study.best_trial print(f"Value: {trial_.value}") print("Params: ") best_params = trial_.params print(best_params) scores = 0 for j in range(5): score = run_training(fold=j, params=best_params, save_model=True) scores += score print(f"OOF Score {scores/5}")
config.dataset['folder'] = DATADIR else: config.dataset['folder'] = os.path.join(DATADIR, dataset) config.validationset['folder'] = config.dataset['folder'] config.model['channels'][-1] = channel config.LOGDIR += f'{size}/{channel}/ae' config.superfluous_strides = int(log2(size / min(config.sizes))) return config DATADIR = '' # directory containing the LOCAL = os.getcwd() + '/' DATASETS = ['pokemon', 'stl10', 'celeba'] DATASET = DATASETS[2] SIZES = [(6, [48]) ] # used to run experiments on ly for a certain shape instead of all SIZES = SIZES or Config.from_file( f'configs/{DATASET}_config_ae.py').sizes.items() if __name__ == '__main__': for size, channels in SIZES: for channel in channels: config = prep_config(DATASET, size, channel, LOCAL) print('running', size, channel) config.save('temp_config.py', False) config = Config.from_file('temp_config.py') run_training(config)
data_train = data_generator.generate_sin_data(sz_dataset, FLAGS.sz_batch) iterator = tf.data.Iterator.from_structure(data_train.output_types, data_train.output_shapes) train_next_element = iterator.get_next() train_series, train_label = train_next_element it_train_init = iterator.make_initializer(data_train) train_outputs = model.build_graph_train(cell, rnn_out_wrapper, train_series) train_loss = train.get_loss_sin(train_outputs, train_label) train_op = train.get_train_op(train_loss, FLAGS.lr) with tf.Session() as sess: train.run_training(sess, FLAGS.nbr_epochs, train_op, train_loss, it_train_init) test.run_test_sin(sess, cell, rnn_out_wrapper, nbr_test_samples, FLAGS.RNN) elif FLAGS.task == 2: sz_dataset = 10000 db_size_valid = 1000 db_size_test = 1000 shape_letter = 10 to_remember_len = 8 blank_separation_len = 5 if FLAGS.RNN == 'BasicRNN': cell = BasicRNNCell.BasicRNNCell(shape_letter, FLAGS.sz_rnn) elif FLAGS.RNN == 'LSTM':
def do_train(): """ callback function """ train.run_training() training_msg.set("Training Complete!")
params["reg_ratio"] = np.random.rand() * 0.0015 params["batch_size"] = np.random.randint(26, 256) params["bidirectional"] = bool(np.random.randint(0, 2)) cfg = AcousticLLDConfig(**params) model = RNN(cfg) elif args.model_type == "acoustic-spectrogram": test_features, test_labels, val_features, val_labels, train_features, train_labels = load_spectrogram_dataset( ) params["fc_size"] = np.random.randint(10, 200) params["dropout"] = 0.3 + np.random.rand() * 0.6 cfg = AcousticSpectrogramConfig(**params) model = CNN(cfg) else: raise Exception( "model_type parameter has to be one of [linguistic|acoustic-lld|acoustic-spectrogram]" ) print( "Subsets sizes: test_features:{}, test_labels:{}, val_features:{}, val_labels:{}, train_features:{}, train_labels:{}" .format(test_features.shape[0], test_labels.shape[0], val_features.shape[0], val_labels.shape[0], train_features.shape[0], train_labels.shape[0])) """Converting model to specified hardware and format""" model.float() model = model.to(get_device()) run_training(model, cfg, test_features, test_labels, train_features, train_labels, val_features, val_labels)
import train import evaluate if __name__ == '__main__': # 训练模型 train.run_training() # 评估模型 # evaluate.evaluate_one_image()
def run_check_grads(net, data, labels, num_checks): """ Check the gradient computed by the backpropagation algorithm using finite differences :param net: list of layers in the network :param data: dims = (nSamples, nVocab) input data :param labels: dims = (nSamples, nVocab) labels corresponding to inputs :param num_checks: number of checks to perform per each parameter tensor :return: prints out the parameter name and the error associated with randomly selected values """ # Delta for numerical difference quotient delta = 1e-7 # One complete pass across the data to obtain the analytical derivatives without updating the weights, so that # the network is not changed tr.run_training(net, data, labels, clip_gradients=False, update_weights=False) grad_err_found = False # Compute num_checks numerical derivatives for each layer in the RNN for layer in net: for param, grad, name in layer.get_params(): print(f"Working on {layer} and parameter: {name}") size_param = param.size n = 0 while n < num_checks: # Select a random parameter for which to calculate the derivative select = int(np.random.uniform(0, size_param)) # Select the analytical derivative d_analytical = grad.flat[select].copy() # Compute the numerical derivative orig_val = param.flat[select] param.flat[select] = orig_val + delta L1, _ = run_forward_pass( net, data, labels, clear_after=True) # loss with positive disturbance param.flat[select] = orig_val - delta L2, _ = run_forward_pass( net, data, labels, clear_after=True) # loss with negative disturbance param.flat[ select] = orig_val # reset the parameter to the original value d_numerical = (L1 - L2) / ( 2 * delta) # compute the numerical derivative if d_numerical == 0: continue # Compute and print absolute and relative errors abs_error = abs(d_analytical - d_numerical) rel_error = abs_error / abs(d_numerical) # print(f"Numerical Grad {d_numerical}, Analytic Grad {d_analytical}") print( f"Absolute error {abs_error}, Relative Error {rel_error}") if abs_error > 1e-7 or rel_error > 0.005: grad_err_found = True print("WARNING: Gradient seems to be wrong") n += 1 print("------------------") print("------------------") if grad_err_found: print("WARNING: Gradient seems to be wrong!") else: print("Gradient seems to be right! Enjoy!") print("------------------") print("------------------")
# -*- coding: utf-8 -*- ''' # Created on 2019/01/26 10:45:55 # main.py # @author: ZhangYachen #@Version : 1.0 #@Contact : [email protected] ''' # here put the import lib import train import test from tool.config import FLAGS if __name__ == '__main__': if FLAGS.run_flag == "0": train.run_training(FLAGS) else: begin_char = input('## 请输入第一个字:') poem = test.gen_poem(FLAGS, begin_char) test.pretty_print_poem(poem_=poem)