def run_exploding_grads_test(net, data, labels, max_seq_len, n_pretrain):
"""
Visualize the increase in the norm of the gradient when gradient clipping or similar mitigation schemes are not
applied. This will only be the case when gradient clipping is not yet implemented. This function will use gradient
clipping as soon as its implemented and may be used for verification.
:param net: list of layers in the network
:param data: dims = (maxSeqLen, nVocab) input data
:param labels: dims = (maxSeqLen, nVocab) labels corresponding to inputs
:param max_seq_len: length of the longest sequence for which the gradient should be computed
:param n_pretrain: number of pretraining iterations on network, so that the exploding gradient becomes more
realistic
:return: show the plot of the gradient's magnitude against the sequence length
"""
# Run pretraining
n_seq_pretrain = 3
print(f"Pre-training for {n_pretrain} iterations...")
for i in range(n_pretrain):
pre_data, pre_labels = dg.generate_random_data(n_seq_pretrain,
data.shape[1])
tr.run_training(net, pre_data, pre_labels, clip_gradients=False)
# Empty array that is used for storing the magnitudes of the computed analytical derivatives
grads = []
# Testing exploding gradients
print(f"Computing plot...")
for i in range(1, max_seq_len):
_ = tr.run_training(net,
data[0:i + 1],
labels[0:i + 1],
clip_gradients=True,
update_weights=False)
grads_tmp = []
for layer in net:
grads_tmp.append(
np.concatenate(
[grad.flatten() for _, grad, _ in layer.get_params()]))
layer.clear_stored_states()
layer.clear_stored_derivs()
grads_tmp = np.abs(np.concatenate(grads_tmp))
grads.append(np.mean(grads_tmp))
grads = np.array(grads)
plt.figure()
plt.title("Visualization of exploding gradients")
plt.loglog(range(1, max_seq_len), grads)
plt.xlabel("Number of Iterations")
plt.ylabel("Mean Absolute Derivative")
plt.show()
def main():
df = get_data_set('idao_dataset/train', save_to_csv=False)
create_folds(df, 5, config)
run_training(1, config, mode='clf')
run_training(1, config, mode='reg')
clf_preds, reg_preds = predict(config)
sub_df = pd.read_csv(sub_df_path)
sub_df['classification_predictions'] = clf_preds
sub_df['regression_predictions'] = reg_preds
sub_df['regression_predictions'] = sub_df['regression_predictions'].apply(
transform)
sub_df.to_csv('Final_Submission.csv', index=False)
def main(args):
print(args)
if (args.do_training):
tra.run_training(args.dataset_dir,
args.logging_dir,
model_name=args.model_name,
img_format=args.image_format,
max_step=args.max_step)
else:
images = prep.readImagesInDirectory(args.predict_dir,
format=args.image_format)
model_dir = args.logging_dir + '/' + args.model_name + '/train'
result, labels = pred.classify_images(
args.model_name,
model_dir,
images,
labels=['Hsia Yu-chiao', 'Sung Yun-hua'])
print(labels)
print(result)
def cross_validation():
config = {'label_csv_mame': '../2CvTrain/label.csv',
'test_label_csv_mame': '../1TestData/label.csv',
'test_img_txt_path': "../1TestData/*.txt",
'valid_label_csv_mame': '../2CvValid/label.csv',
'valid_img_txt_path': "../2CvValid/*.txt",
# label csv column names
'name_list': ["img_name", "c", "gx", "gy", "gw", "gh"],
'cfg_path': "../4Others/yolo.cfg",
'params_dir': '../4Others/params.txt'}
# entire trainin label
prep_label_config = {'label_csv_mame': '../1TrainData/label.csv',
'img_txt_path': "../1TrainData/*.txt",
'name_list': config['name_list']}
# cv train label
cv_train_label_config = {'label_csv_mame': config['label_csv_mame'],
'img_txt_path': "../2CvTrain/*.txt",
'name_list': config['name_list']}
torch.backends.cudnn.enabled = False
torch.backends.cudnn.benchmark = False
# you can specify any tuning hyper-parammeters here , but do not use
# np.arrange() as it will result an error for json.dumps()
tune_params = {'seed': list(range(1, 5)),
'batch_size': list(range(2, 7, 2)),
'height': [416, 448]
}
index, values = zip(*tune_params.items())
experiments_params = [dict(zip(index, v)) for v in it.product(*values)]
# prepare label for the entire training set
prep_labels(**prep_label_config)
cv_results_list = []
for experiment_params in experiments_params:
cv_split_config = {'n_splits': 5,
'cv': True,
'train_size': 0.7,
'name_list': config['name_list'],
'random_state': experiment_params['seed'],
'train_cv_path': "../2CvTrain/",
'valid_cv_path': "../2CvValid/",
'label_name': prep_label_config['label_csv_mame']}
for _ in split_train(**cv_split_config):
prep_labels(**cv_train_label_config)
params = prep_params(config['params_dir'],
cv_train_label_config['label_csv_mame'],
experiment_params)
best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\
map_frame = run_training(params=params, **config)
cv_results_list.append(params)
shutil.rmtree(cv_split_config['train_cv_path'])
shutil.rmtree(cv_split_config['valid_cv_path'])
cv_results_df = pd.DataFrame(cv_results_list)
cv_results_df.to_csv(f"{params['working_dir']}cv_results.csv")
def main():
hparams = hyperparameters.hparams
print("************************************")
print("*********** Begin Train ************")
print("************************************")
print("Log directory: %s" % hparams.log_dir)
#sign_changer.change_sign("./free_with_back.txt")
#don't actually use loss for now
loss, optimizer, accuracy, data_placeholder, label_placeholder, summaries = graph.build_graph(
hparams)
input_pairs = []
input_pairs.append(preprocessing.preprocess(hparams, 'butterfly'))
input_pairs.append(preprocessing.preprocess(hparams, 'backstroke'))
#input_pairs.append(preprocessing.preprocess(hparams, 'breastroke'))
input_pairs.append(preprocessing.preprocess(hparams, 'freestyle'))
#preprocessing.augment_data(hparams)
train.run_training(hparams, data_placeholder, label_placeholder, optimizer,
accuracy, input_pairs, summaries)
Dense(config.hidden_layers[0],
input_dim=input_dim,
kernel_regularizer=l2(config.l2_reg)))
model.add(Activation(config.activation))
if config.batch_norm:
model.add(BatchNormalization())
for n in config.hidden_layers[1:]:
model.add(Dense(n))
model.add(Activation(config.activation))
if config.batch_norm:
model.add(BatchNormalization())
model.add(Dense(1))
model.add(Activation(config.output_activation))
model.compile(optimizer=config.optimizer,
loss=config.loss,
metrics=config.metrics)
K.set_value(model.optimizer.lr, config.learning_rate)
return model
num_trainings = 20 #used for model hyperparameter searching
for __ in range(num_trainings):
config = get_config(
config_name) #randomly initialized config for hyperparameter search
model = build_model(config)
run_training(model, config)
if not config.hyper_search:
break
"""
This script is the entrypoint to the model docker container for SageMaker. It expects a parameter of either "train" or
"serve" and then triggers the appropriate script.
"""
from argparse import ArgumentParser
from train import run_training
from serve import start_server
if __name__ == '__main__':
parser = ArgumentParser(
description='select model training or serving entrypoint')
parser.add_argument('entrypoint', choices=['train', 'serve'])
args = parser.parse_args()
if args.entrypoint == 'train':
run_training()
else:
start_server()
'label_csv_mame': config['label_csv_mame'],
'img_txt_path': "../1TrainData/*.txt",
'name_list': config['name_list']
}
# torch.backends.cudnn.enabled = False
# torch.backends.cudnn.benchmark = False
# you can specify any tuning hyper-parammeters here , but do not use
# np.arrange() as it will result an error for json.dumps()
tune_params = {
'seed': list(range(1, 3)),
'height': [448, 480],
"decay": [0.0005, 0.001],
"steps": [6, 10],
"optimizer": ['sgd', 'adam'],
"epochs": [40, 80]
}
index, values = zip(*tune_params.items())
experiments_params = [dict(zip(index, v)) for v in it.product(*values)]
# prepare training lables
prep_labels(**prep_label_config)
for experiment_params in experiments_params:
params = prep_params(config['params_dir'], config['label_csv_mame'],
experiment_params)
try:
best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\
map_frame = run_training(params=params, **config)
except:
continue
def train(lr=None, image_dir=None):
run_training(lr, image_dir)
import optuna
from functools import partial
from train import objective, run_training
DEVICE = "cuda"
EPOCHS = 100
if __name__ == "__main__":
partial_obj = partial(objective)
study = optuna.create_study(direction="minimize")
study.optimize(partial_obj, n_trials=150)
print("Best trial:")
trial_ = study.best_trial
print(f"Value: {trial_.value}")
print("Params: ")
best_params = trial_.params
print(best_params)
scores = 0
for j in range(5):
score = run_training(fold=j, params=best_params, save_model=True)
scores += score
print(f"OOF Score {scores/5}")
config.dataset['folder'] = DATADIR
else:
config.dataset['folder'] = os.path.join(DATADIR, dataset)
config.validationset['folder'] = config.dataset['folder']
config.model['channels'][-1] = channel
config.LOGDIR += f'{size}/{channel}/ae'
config.superfluous_strides = int(log2(size / min(config.sizes)))
return config
DATADIR = '' # directory containing the
LOCAL = os.getcwd() + '/'
DATASETS = ['pokemon', 'stl10', 'celeba']
DATASET = DATASETS[2]
SIZES = [(6, [48])
] # used to run experiments on ly for a certain shape instead of all
SIZES = SIZES or Config.from_file(
f'configs/{DATASET}_config_ae.py').sizes.items()
if __name__ == '__main__':
for size, channels in SIZES:
for channel in channels:
config = prep_config(DATASET, size, channel, LOCAL)
print('running', size, channel)
config.save('temp_config.py', False)
config = Config.from_file('temp_config.py')
run_training(config)
data_train = data_generator.generate_sin_data(sz_dataset,
FLAGS.sz_batch)
iterator = tf.data.Iterator.from_structure(data_train.output_types,
data_train.output_shapes)
train_next_element = iterator.get_next()
train_series, train_label = train_next_element
it_train_init = iterator.make_initializer(data_train)
train_outputs = model.build_graph_train(cell, rnn_out_wrapper,
train_series)
train_loss = train.get_loss_sin(train_outputs, train_label)
train_op = train.get_train_op(train_loss, FLAGS.lr)
with tf.Session() as sess:
train.run_training(sess, FLAGS.nbr_epochs, train_op, train_loss,
it_train_init)
test.run_test_sin(sess, cell, rnn_out_wrapper, nbr_test_samples,
FLAGS.RNN)
elif FLAGS.task == 2:
sz_dataset = 10000
db_size_valid = 1000
db_size_test = 1000
shape_letter = 10
to_remember_len = 8
blank_separation_len = 5
if FLAGS.RNN == 'BasicRNN':
cell = BasicRNNCell.BasicRNNCell(shape_letter, FLAGS.sz_rnn)
elif FLAGS.RNN == 'LSTM':
def do_train():
""" callback function """
train.run_training()
training_msg.set("Training Complete!")
params["reg_ratio"] = np.random.rand() * 0.0015
params["batch_size"] = np.random.randint(26, 256)
params["bidirectional"] = bool(np.random.randint(0, 2))
cfg = AcousticLLDConfig(**params)
model = RNN(cfg)
elif args.model_type == "acoustic-spectrogram":
test_features, test_labels, val_features, val_labels, train_features, train_labels = load_spectrogram_dataset(
)
params["fc_size"] = np.random.randint(10, 200)
params["dropout"] = 0.3 + np.random.rand() * 0.6
cfg = AcousticSpectrogramConfig(**params)
model = CNN(cfg)
else:
raise Exception(
"model_type parameter has to be one of [linguistic|acoustic-lld|acoustic-spectrogram]"
)
print(
"Subsets sizes: test_features:{}, test_labels:{}, val_features:{}, val_labels:{}, train_features:{}, train_labels:{}"
.format(test_features.shape[0], test_labels.shape[0],
val_features.shape[0], val_labels.shape[0],
train_features.shape[0], train_labels.shape[0]))
"""Converting model to specified hardware and format"""
model.float()
model = model.to(get_device())
run_training(model, cfg, test_features, test_labels, train_features,
train_labels, val_features, val_labels)
import train
import evaluate
if __name__ == '__main__':
# 训练模型
train.run_training()
# 评估模型
# evaluate.evaluate_one_image()
def run_check_grads(net, data, labels, num_checks):
"""
Check the gradient computed by the backpropagation algorithm using finite differences
:param net: list of layers in the network
:param data: dims = (nSamples, nVocab) input data
:param labels: dims = (nSamples, nVocab) labels corresponding to inputs
:param num_checks: number of checks to perform per each parameter tensor
:return: prints out the parameter name and the error associated with randomly selected values
"""
# Delta for numerical difference quotient
delta = 1e-7
# One complete pass across the data to obtain the analytical derivatives without updating the weights, so that
# the network is not changed
tr.run_training(net,
data,
labels,
clip_gradients=False,
update_weights=False)
grad_err_found = False
# Compute num_checks numerical derivatives for each layer in the RNN
for layer in net:
for param, grad, name in layer.get_params():
print(f"Working on {layer} and parameter: {name}")
size_param = param.size
n = 0
while n < num_checks:
# Select a random parameter for which to calculate the derivative
select = int(np.random.uniform(0, size_param))
# Select the analytical derivative
d_analytical = grad.flat[select].copy()
# Compute the numerical derivative
orig_val = param.flat[select]
param.flat[select] = orig_val + delta
L1, _ = run_forward_pass(
net, data, labels,
clear_after=True) # loss with positive disturbance
param.flat[select] = orig_val - delta
L2, _ = run_forward_pass(
net, data, labels,
clear_after=True) # loss with negative disturbance
param.flat[
select] = orig_val # reset the parameter to the original value
d_numerical = (L1 - L2) / (
2 * delta) # compute the numerical derivative
if d_numerical == 0:
continue
# Compute and print absolute and relative errors
abs_error = abs(d_analytical - d_numerical)
rel_error = abs_error / abs(d_numerical)
# print(f"Numerical Grad {d_numerical}, Analytic Grad {d_analytical}")
print(
f"Absolute error {abs_error}, Relative Error {rel_error}")
if abs_error > 1e-7 or rel_error > 0.005:
grad_err_found = True
print("WARNING: Gradient seems to be wrong")
n += 1
print("------------------")
print("------------------")
if grad_err_found:
print("WARNING: Gradient seems to be wrong!")
else:
print("Gradient seems to be right! Enjoy!")
print("------------------")
print("------------------")
# -*- coding: utf-8 -*- ''' # Created on 2019/01/26 10:45:55 # main.py # @author: ZhangYachen #@Version : 1.0 #@Contact : [email protected] ''' # here put the import lib import train import test from tool.config import FLAGS if __name__ == '__main__': if FLAGS.run_flag == "0": train.run_training(FLAGS) else: begin_char = input('## 请输入第一个字:') poem = test.gen_poem(FLAGS, begin_char) test.pretty_print_poem(poem_=poem)