def train_and_test(wv):
# merged_summary, writer, saver = configure_saver()
x_train, y_train = utils.get_processed_data(wv, train_file)
# x_val, y_val = utils.get_processed_data(wv, val_file)
x_test, y_test = utils.get_processed_data(wv, test_file)
session = tf.Session()
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
# writer.add_graph(session.graph)
cur_batch = 0
for x_batch, y_batch in utils.generate_batch(x_train, y_train):
feed = {
cnn_model.input_x: x_batch,
cnn_model.input_y: y_batch,
cnn_model.mode: tf.estimator.ModeKeys.TRAIN
}
session.run(cnn_model.train_op, feed_dict=feed)
cur_batch += 1
# val_acc, val_pre, val_recall, val_f1 = validate(x_val, y_val, session)
test_acc, test_pre, test_recall, teste_f1 = validate(
x_test, y_test, session)
# saver.save(sess=session, save_path=save_path)
return test_acc, test_pre, test_recall, teste_f1
def run(args):
""" Load data, generate training batch, initiate model and train VAE. """
print('\nLoad dataset')
if args.dataset == 'kddcup99':
keep_cols = args.keep_cols[1:-1].replace("'",
"").replace(" ",
"").split(",")
data = get_kdd_data(keep_cols=keep_cols)
else:
raise ValueError('Only "kddcup99" dataset supported.')
print('\nGenerate training batch')
X, _ = generate_batch(data, args.samples, 0.)
print('\nInitiate outlier detector model')
n_features = data.shape[1] - 1 # nb of features
vae = model(n_features,
hidden_layers=args.hidden_layers,
latent_dim=args.latent_dim,
hidden_dim=args.hidden_dim)
if args.print_progress:
vae.summary()
print('\nTrain outlier detector')
train(vae, X, args)
def run(args):
""" Load data, generate training batch, initiate model and train VAE. """
print('\nLoad dataset')
if args.dataset == 'kddcup99':
keep_cols = args.keep_cols[1:-1].replace("'",
"").replace(" ",
"").split(",")
data = get_kdd_data(keep_cols=keep_cols)
else:
raise ValueError('Only "kddcup99" dataset supported.')
print('\nGenerate training batch')
X, _ = generate_batch(data, args.samples, 0.)
print('\nInitiate outlier detector model')
n_features = data.shape[1] - 1 # nb of features
vae = model(n_features,
hidden_layers=args.hidden_layers,
latent_dim=args.latent_dim,
hidden_dim=args.hidden_dim,
output_activation=args.output_activation,
learning_rate=args.learning_rate)
if args.continue_training:
print('\nLoad pre-trained model')
vae.load_weights(args.load_path + args.model_name +
'_weights.h5') # load pretrained model weights
if args.print_progress:
vae.summary()
print('\nTrain outlier detector')
train(vae, X, args)
def run(args):
""" Load data, generate training batch and train Isolation Forest. """
print('\nLoad dataset')
if args.dataset=='kddcup99':
keep_cols = args.keep_cols[1:-1].replace("'","").replace(" ","").split(",")
data = get_kdd_data(keep_cols=keep_cols)
else:
raise ValueError('Only "kddcup99" dataset supported.')
print('\nGenerate training batch')
X, _ = generate_batch(data,args.samples,args.contamination)
print('\nTrain outlier detector')
train(X,args)
print('\nTraining done!')
def do_epoch(model, dataset, batch_size, train_mode=True, is_shuffle = True):
model.train(mode=train_mode)
if is_shuffle:
random.shuffle(dataset)
acc_loss = 0
for iter, (x, y) in enumerate(generate_batch(dataset, batch_size=batch_size)):
# forward pass and compute loss
loss = (- model.loss(x, y)) / x.size(1)
acc_loss += loss.view(-1).data.tolist()[0]
if train_mode:
model.zero_grad()
loss.backward() # compute gradients
optim.step() # update parameters
return acc_loss / (iter + 1)
def create_dataloader(dataset,
trans_fn=None,
mode='train',
batch_size=1,
use_gpu=False,
pad_token_id=0):
"""
Creats dataloader.
Args:
dataset(obj:`paddle.io.Dataset`): Dataset instance.
mode(obj:`str`, optional, defaults to obj:`train`): If mode is 'train', it will shuffle the dataset randomly.
batch_size(obj:`int`, optional, defaults to 1): The sample number of a mini-batch.
use_gpu(obj:`bool`, optional, defaults to obj:`False`): Whether to use gpu to run.
pad_token_id(obj:`int`, optional, defaults to 0): The pad token index.
Returns:
dataloader(obj:`paddle.io.DataLoader`): The dataloader which generates batches.
"""
if trans_fn:
dataset = dataset.apply(trans_fn, lazy=True)
if mode == 'train' and use_gpu:
sampler = paddle.io.DistributedBatchSampler(dataset=dataset,
batch_size=batch_size,
shuffle=True)
else:
shuffle = True if mode == 'train' else False
sampler = paddle.io.BatchSampler(dataset=dataset,
batch_size=batch_size,
shuffle=shuffle)
dataloader = paddle.io.DataLoader(dataset,
batch_sampler=sampler,
return_list=True,
collate_fn=lambda batch: generate_batch(
batch, pad_token_id=pad_token_id))
return dataloader
def main():
# make dirs
target_dirs = [
args.checkpoint_dir, args.logdir, args.train_result_dir,
args.train_LR_result_dir, args.valid_result_dir,
args.valid_LR_result_dir
]
create_dirs(target_dirs)
# set logger
logflag = set_logger(args)
log(logflag, 'Training script start', 'info')
NLR_data = tf.placeholder(tf.float32,
shape=[None, None, None, args.channel],
name='NLR_input')
CLR_data = tf.placeholder(tf.float32,
shape=[None, None, None, args.channel],
name='CLR_input')
NHR_data = tf.placeholder(tf.float32,
shape=[None, None, None, args.channel],
name='NHR_input')
CHR_data = tf.placeholder(tf.float32,
shape=[None, None, None, args.channel],
name='CHR_input')
# build Generator and Discriminator
network = Network(args,
NLR_data=NLR_data,
CLR_data=CLR_data,
NHR_data=NHR_data,
CHR_data=CHR_data)
CLR_C1, NLR_C1, CLR_C2, CHR_C3, NLR_C3, CHR_C4, CLR_I1, CHR_I1, CHR_I2 = network.train_generator(
)
D_out, Y_out, C_out = network.train_discriminator(CLR_C1, CHR_C3)
# build loss function
loss = Loss()
gen_loss, g_gen_loss, dis_loss, Y_dis_loss, C_dis_loss = loss.gan_loss(
args, NLR_data, CLR_data, NHR_data, CHR_data, CLR_C1, NLR_C1, CLR_C2,
CHR_C3, NLR_C3, CHR_C4, CLR_I1, CHR_I1, CHR_I2, D_out, Y_out, C_out)
# define optimizers
global_iter = tf.Variable(0, trainable=False)
dis_var, dis_optimizer, gen_var, gen_optimizer, Y_dis_optimizer, C_dis_optimizer = Optimizer(
).gan_optimizer(args, global_iter, dis_loss, gen_loss, Y_dis_loss,
C_dis_loss)
# build summary writer
tr_summary = tf.summary.merge(loss.add_summary_writer())
fetches = {
'dis_optimizer': dis_optimizer,
'Y_dis_optimizer': Y_dis_optimizer,
'C_dis_optimizer': C_dis_optimizer,
'gen_optimizer': gen_optimizer,
'dis_loss': dis_loss,
'Y_dis_loss': Y_dis_loss,
'C_dis_loss': C_dis_loss,
'gen_loss': gen_loss,
'g_gen_loss': g_gen_loss,
'CHR_out': CHR_C3,
'CLR_out': CLR_C1,
'summary': tr_summary
}
"""
fetches = {'dis_optimizer': dis_optimizer,'dis_optimizerLR': dis_optimizerLR, 'gen_optimizer': gen_optimizer,
'dis_loss': dis_loss, 'gen_loss': gen_loss,
'CHR_out': CHR_C3, 'CLR_out' : CLR_C1, 'summary' : tr_summary}
"""
gc.collect()
config = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True, visible_device_list=args.gpu_dev_num))
# Start Session
with tf.Session(config=config) as sess:
log(logflag, 'Training ESRGAN starts', 'info')
sess.run(tf.global_variables_initializer())
sess.run(global_iter.initializer)
writer = tf.summary.FileWriter(args.logdir, graph=sess.graph)
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,scope='generator/Generator1')\
+ tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,scope='generator/Generator2')\
+ tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,scope='discriminator')
pre_saver = tf.train.Saver(var_list=var_list)
pre_saver.restore(
sess, tf.train.latest_checkpoint(args.pre_train_checkpoint_dir1))
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='generator/SR')
pre_saver = tf.train.Saver(var_list=var_list)
pre_saver.restore(
sess, tf.train.latest_checkpoint(args.pre_train_checkpoint_dir2))
if args.perceptual_loss == 'VGG19':
sess.run(load_vgg19_weight(args))
saver = tf.train.Saver(max_to_keep=10)
saver.restore(sess, tf.train.latest_checkpoint(args.checkpoint_dir))
_datapathNLR = np.sort(
np.asarray(
glob(
os.path.join(args.data_dir + '/source/train_HR_aug/x4/',
'*.png'))))
_datapathNHR = np.sort(
np.asarray(
glob(
os.path.join(args.data_dir + '/source/train_HR_aug/x4/',
'*.png'))))
_datapathCLR = np.sort(
np.asarray(
glob(
os.path.join(args.data_dir + '/target/train_LR_aug/x4/',
'*.png'))))
_datapathCHR = np.sort(
np.asarray(
glob(
os.path.join(args.data_dir + '/target/train_HR_aug/x4/',
'*.png'))))
idxLR = np.random.permutation(len(_datapathNLR))
datapathNLR = _datapathNLR[idxLR]
datapathNHR = _datapathNHR[idxLR]
idxHR = np.random.permutation(len(_datapathCHR))
datapathCLR = _datapathCLR[idxHR]
datapathCHR = _datapathCHR[idxHR]
epoch = 0
counterNLR = 0
counterCLR = 0
g_loss = 0.0
d_loss = 0.0
steps = 0
psnr_max = 20
while True:
if counterNLR >= len(_datapathNLR):
log(
logflag,
'Train Epoch: {0} g.loss : {1} d.loss : {2}'.format(
epoch, g_loss / steps, d_loss / steps), 'info')
idx = np.random.permutation(len(_datapathNLR))
datapathNLR = _datapathNLR[idx]
datapathNHR = _datapathNHR[idx]
counterNLR = 0
g_loss = 0.0
d_loss = 0.0
steps = 0
epoch += 1
if epoch == 200:
break
if counterCLR >= len(_datapathCHR):
idx = np.random.permutation(len(_datapathCHR))
datapathCHR = _datapathCHR[idx]
datapathCLR = _datapathCLR[idx]
counterCLR = 0
dataNLR, dataNHR, dataCLR, dataCHR = generate_batch(
datapathNLR[counterNLR:counterNLR + args.image_batch_size],
datapathNHR[counterNLR:counterNLR + args.image_batch_size],
datapathCLR[counterCLR:counterCLR + args.image_batch_size],
datapathCHR[counterCLR:counterCLR + args.image_batch_size],
args)
counterNLR += args.image_batch_size
counterCLR += args.image_batch_size
for iteration in range(0, dataCLR.shape[0], args.batch_size):
_CHR_data = dataCHR[iteration:iteration + args.batch_size]
_CLR_data = dataCLR[iteration:iteration + args.batch_size]
_NLR_data = dataNLR[iteration:iteration + args.batch_size]
_NHR_data = dataNHR[iteration:iteration + args.batch_size]
feed_dict = {
CHR_data: _CHR_data,
CLR_data: _CLR_data,
NLR_data: _NLR_data,
NHR_data: _NHR_data,
}
# update weights
result = sess.run(fetches=fetches, feed_dict=feed_dict)
current_iter = tf.train.global_step(sess, global_iter)
g_loss += result['gen_loss']
d_loss += result['dis_loss']
steps += 1
# save summary every n iter
if current_iter % args.train_summary_save_freq == 0:
writer.add_summary(result['summary'],
global_step=current_iter)
# save samples every n iter
if current_iter % 100 == 0:
log(
logflag,
'Mymodel iteration : {0}, gen_loss : {1}, dis_loss : {2},'
' Y_dis_loss {3} C_dis_loss {4} g_gen_loss {5}'.format(
current_iter, result['gen_loss'],
result['dis_loss'], result['Y_dis_loss'],
result['C_dis_loss'], result['g_gen_loss']),
'info')
if current_iter % args.train_sample_save_freq == 0:
validpathLR = np.sort(
np.asarray(
glob(
os.path.join(args.data_dir + '/validation/x/',
'*.png'))))
validpathHR = np.sort(
np.asarray(
glob(
os.path.join(args.data_dir + '/validation/y/',
'*.png'))))
psnr_avg = 0.0
for valid_ii in range(100):
#valid_i = np.random.randint(100)
validLR, validHR = generate_testset(
validpathLR[valid_ii], validpathHR[valid_ii], args)
validLR = np.transpose(validLR[:, :, :, np.newaxis],
(3, 0, 1, 2))
validHR = np.transpose(validHR[:, :, :, np.newaxis],
(3, 0, 1, 2))
valid_out, valid_out_LR = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out += np.rot90(valid_out90, 3, axes=(1, 2))
valid_out_LR += np.rot90(valid_out_LR90,
3,
axes=(1, 2))
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out += np.rot90(valid_out90, 2, axes=(1, 2))
valid_out_LR += np.rot90(valid_out_LR90,
2,
axes=(1, 2))
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out += np.rot90(valid_out90, 1, axes=(1, 2))
valid_out_LR += np.rot90(valid_out_LR90,
1,
axes=(1, 2))
valid_out /= 4.
valid_out_LR /= 4.
from utils import de_normalize_image
validHR = np.rot90(validHR, 1, axes=(1, 2))
F = de_normalize_image(validHR) / 255.
G = de_normalize_image(valid_out) / 255.
E = F - G
N = np.size(E)
PSNR = 10 * np.log10(N / np.sum(E**2))
print(PSNR)
psnr_avg += PSNR / 100
if valid_ii < 5:
save_image(args,
valid_out,
'valid',
current_iter + valid_ii,
save_max_num=5)
save_image(args,
valid_out_LR,
'valid_LR',
current_iter + valid_ii,
save_max_num=5)
if psnr_avg > psnr_max:
print("max psnr : %f" % psnr_avg)
psnr_max = psnr_avg
saver.save(sess,
os.path.join('./best_checkpoint', 'gen'),
global_step=current_iter)
#save_image(args, result['gen_HR'], 'My_train', current_iter, save_max_num=5)
#save_image(args, result['gen_out_LR'], 'My_train_LR', current_iter, save_max_num=5)
# save checkpoints
if current_iter % args.train_ckpt_save_freq == 0:
saver.save(sess,
os.path.join(args.checkpoint_dir, 'gen'),
global_step=current_iter)
writer.close()
log(logflag, 'Training ESRGAN end', 'info')
log(logflag, 'Training script end', 'info')
hidden_sizes=[64, 64],
act_fn=tf.nn.tanh)
#pi = GaussianPolicy(env.observation_space, env.action_space, scope='pi', train_type = 'reinforce', hidden_sizes = [16, 16])
pi.load_policy(FLAGS.pi_weightfile)
# setting behavior policy
if FLAGS.pi_b == 4:
pi_b = pi
num_mc_trajs = FLAGS.num_mc_trajs
num_mc_sampled_states = FLAGS.num_mc_sampled_states
num_mc_rollouts = FLAGS.num_mc_rollouts
# evaluating "true" value of certain states based on evaluation policy
# generate trajectories to sample states
mc_trajs = generate_batch(env, env_name, pi, num_mc_trajs)
# sample states to compute MC estimates for
mc_sampled_states = sample_states(mc_trajs, num_mc_sampled_states)
assert len(mc_sampled_states) == num_mc_sampled_states
#assert len(mc_sampled_states[0]) == env.observation_space.shape[0]
assert len(mc_sampled_states[0][0]) == env.observation_space.shape[0]
# get MC estimates for sampled states (as "true" value)
vf = mc_rollouts(env, env_name, pi, num_mc_rollouts, mc_sampled_states)
# removing the full state
mc_sampled_states = [s[0] for s in mc_sampled_states]
# generate batch of data for actual (RIS)-TD(0) with behavior policy
num_batch_trajs = FLAGS.num_trajs
batch_raw = generate_batch(env, env_name, pi_b, num_batch_trajs)
def train():
# 创建结果保存的路径
if not os.path.exists(FLAGS.result_dir):
os.makedirs(FLAGS.result_dir)
if FLAGS.model_prefix == 'poems':
poems_vector, word_to_int, vocabularies = build_dataset(
FLAGS.file_path)
elif FLAGS.model_prefix == 'names':
poems_vector, word_to_int, vocabularies = build_name_dataset(
FLAGS.file_path)
batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size,
poems_vector, word_to_int)
input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
end_points = char_rnn(model='lstm',
input_data=input_data,
output_data=output_targets,
vocab_size=len(vocabularies),
rnn_size=128,
num_layers=2,
batch_size=FLAGS.batch_size,
learning_rate=FLAGS.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
start_epoch = 0
checkpoint = tf.train.latest_checkpoint(FLAGS.result_dir)
if checkpoint: # 从上次结束的地方继续训练
saver.restore(sess, checkpoint)
print("## restore from the checkpoint {0}".format(checkpoint))
start_epoch += int(checkpoint.split('-')[-1])
print('## start training...')
try:
for epoch in range(start_epoch, FLAGS.epochs):
n = 0
n_chunk = len(poems_vector) // FLAGS.batch_size
for batch in range(n_chunk):
loss, _, _ = sess.run(
[
end_points['total_loss'], end_points['last_state'],
end_points['train_op']
],
feed_dict={
input_data: batches_inputs[n],
output_targets: batches_outputs[n]
})
n += 1
print('Epoch: %d, batch: %d, training loss: %.6f' %
(epoch, batch, loss))
if epoch % 10 == 0:
saver.save(sess,
os.path.join(FLAGS.result_dir,
FLAGS.model_prefix),
global_step=epoch)
except KeyboardInterrupt:
print('## Interrupt manually, try saving checkpoint for now...')
saver.save(sess,
os.path.join(FLAGS.result_dir, FLAGS.model_prefix),
global_step=epoch)
print(
'## Last epoch were saved, next time will start from epoch {}.'
.format(epoch))
def train():
FLAG = FLAGS()
poems_vector, word_to_int, vocabularies = build_dataset(
FLAG.poems_path, FLAG.name_path)
batches_inputs, batches_outputs = generate_batch(FLAG.batch_size,
poems_vector, word_to_int)
input_data = tf.placeholder(tf.int32, [FLAG.batch_size, None],
name="Input")
output_targets = tf.placeholder(tf.int32, [FLAG.batch_size, None])
#z = tf.log(output_targets, name="namemodel")
end_points = char_rnn(model='lstm',
input_data=input_data,
output_data=output_targets,
vocab_size=len(vocabularies),
rnn_size=FLAG.rnn_size,
num_layers=FLAG.num_layers,
batch_size=FLAG.batch_size,
learning_rate=FLAG.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
start_epoch = 0
checkpoint = tf.train.latest_checkpoint(FLAG.result_dir)
if checkpoint: # 从上次结束的地方继续训练
saver.restore(sess, checkpoint)
print("## restore from the checkpoint {0}".format(checkpoint))
start_epoch += int(checkpoint.split('-')[-1])
print('## start training...')
try:
for epoch in range(start_epoch, FLAG.epochs):
n = 0
n_chunk = len(poems_vector) // FLAG.batch_size
for batch in range(n_chunk):
loss, _, _ = sess.run(
[
end_points['total_loss'], end_points['last_state'],
end_points['train_op']
],
feed_dict={
input_data: batches_inputs[n],
output_targets: batches_outputs[n]
})
n += 1
print('Epoch: %d, batch: %d, training loss: %.6f' %
(epoch, batch, loss))
if epoch % 10 == 0:
saver.save(sess,
os.path.join(FLAG.result_dir,
FLAG.model_prefix),
global_step=epoch)
except KeyboardInterrupt:
print('## Interrupt manually, try saving checkpoint for now...')
saver.save(sess,
os.path.join(FLAG.result_dir, FLAG.model_prefix),
global_step=epoch)
print(
'## Last epoch were saved, next time will start from epoch {}.'
.format(epoch))
#saver.save(sess, FLAG.result_dir+'/model/'+"model.ckpt")
#tf.train.write_graph(sess.graph_def, FLAG.result_dir+'/model/', 'graph.pb')
builder = tf.saved_model.builder.SavedModelBuilder(FLAG.result_dir +
"/model_complex")
SignatureDef = tf.saved_model.signature_def_utils.build_signature_def(
inputs={
'input_data':
tf.saved_model.utils.build_tensor_info(input_data),
'output_targets':
tf.saved_model.utils.build_tensor_info(output_targets)
},
outputs={
'prediction':
tf.saved_model.utils.build_tensor_info(
end_points['prediction'])
})
builder.add_meta_graph_and_variables(
sess, [tag_constants.TRAINING],
signature_def_map={
tf.saved_model.signature_constants.CLASSIFY_INPUTS:
SignatureDef
})
builder.save()
default='ptb',
help='dataset to use (default: ptb)')
args = parser.parse_args()
# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
if torch.cuda.is_available():
if not args.cuda:
print(
"WARNING: You have a CUDA device, so you should probably run with --cuda"
)
file_path = 'data/poems.txt'
poems_vector, word_to_int, vocabularies = process_poems(file_path)
batches_inputs, batches_outputs = generate_batch(args.batch_size, poems_vector,
word_to_int, args.seq_len)
batches_inputs = Variable(torch.from_numpy(np.array(batches_inputs)).long())
batches_outputs = Variable(torch.from_numpy(np.array(batches_outputs)).long())
if args.cuda:
batches_inputs = batches_inputs.cuda()
batches_outputs = batches_outputs.cuda()
n_characters = len(vocabularies)
num_chans = [args.nhid] * (args.levels - 1) + [args.emsize]
k_size = args.ksize
dropout = args.dropout
emb_dropout = args.emb_dropout
model = TCN(args.emsize,
n_characters,
