def validate(message):
K.clear_session()
global status
global data
global task
global progress
data = None
status = WorkerStatus.WORKING
task = TaskType.EVALUATE
progress = 0
try:
model_path = "{}/{}".format(work_dir, message.taskName)
try:
shutil.rmtree(model_path)
except FileNotFoundError:
pass
pathlib.Path(model_path).mkdir(parents=True, exist_ok=True)
io_input = BytesIO(message.data)
with tarfile.open(fileobj=io_input, mode="r:gz") as tar:
tar.extractall(work_dir)
model = Seq2Seq(load=True, working_dir=model_path)
model.save_full_report()
progress = 1
model.evaluate_checkpoints(progress=lambda p: set_progress(p + 1),
data_type=['validate'])
compress_and_set_result(model_path)
except Exception:
status = WorkerStatus.ERROR
task = TaskType.NONE
progress = 0
traceback.print_exc()
def main():
# Load data
print('Loading data...')
train_iter, val_iter, test_iter, DE, EN = load_dataset(batch_size=32)
PAD_IDX = EN.vocab.stoi[EN.pad_token]
# Build model and optimizer
print('Building model and optimizer...')
s2s = Seq2Seq(enc_vocab_size=len(DE.vocab),
dec_vocab_size=len(EN.vocab),
enc_embed_dim=1500,
dec_embed_dim=1500,
hidden_size=1500,
enc_num_layers=2,
dec_num_layers=2,
padding_idx=PAD_IDX,
dropout_rate=0.2)
if USE_CUDA:
s2s = s2s.cuda()
optimizer = optim.SGD(params=s2s.parameters(), lr=1)
# optimizer = optim.Adam(params=s2s.parameters(), lr=10**-2)
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.3, threshold=10**-3, patience=0)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[7, 14],
gamma=0.5,
last_epoch=-1)
loss_func = nn.NLLLoss(size_average=False, ignore_index=PAD_IDX)
# Train model
print('Training...')
print('-' * 102)
N_EPOCHS = 20
for i in range(N_EPOCHS):
# Train a single epoch
lr = optimizer.param_groups[0]['lr']
start = time.time()
train_loss = train(s2s,
train_iter,
loss_func,
optimizer,
pad_idx=PAD_IDX,
max_grad_norm=5)
end = time.time()
train_loss = train_loss[0]
train_time = (end - start)
# Calculate loss and ppl on validation set, adjust learning rate if needed
val_loss = eval(s2s, val_iter, loss_func, PAD_IDX)
val_loss = val_loss[0]
ppl = np.exp(val_loss)
scheduler.step()
# Save model
torch.save(s2s, 's2s_model.pt')
# Print epoch update
print(
'| Epoch #{:2d} | train loss {:5.4f} | train time {:5.2f} | val loss {:5.4f} | val ppl {:7.4f} | lr {:4f}'
.format(i, train_loss, train_time, val_loss, ppl, lr))
print('-' * 102)
def train(model_def_path, epochs=300, batch_size=64):
model_def = load_model(model_def_path)
records = load_records()
model = Seq2Seq(model_def,
working_dir=time.strftime(
"./models/{}-%Y_%m_%d-%H%M%S".format(
model_def['name'])))
model.save_no_train()
model.train(data=records, epochs=epochs, batch_size=batch_size)
model.save()
return model
def validate_model(path):
with open("{}/result_train.csv".format(path), "r") as ftrain:
train = process_csv(ftrain.readlines())
with open("{}/result_validate.csv".format(path), "r") as fvalidate:
validate = process_csv(fvalidate.readlines())
full = [train[0] + validate[0], train[1] + validate[1]]
model = Seq2Seq(load=True, working_dir=path)
# model.save_for_inference_tf()
train_acc = model.validate(train[0], train[1])
model._acc_report(train_acc, len(train[0]))
validate_acc = model.validate(validate[0], validate[1])
model._acc_report(validate_acc, len(validate[0]))
full_acc = model.validate(full[0], full[1])
model._acc_report(full_acc, len(full[0]))
return model
def train(message):
K.clear_session()
global status
global data
global task
global progress
data = None
status = WorkerStatus.WORKING
task = TaskType.TRAIN
progress = 0
try:
model_def = json.loads(message.modelDefinition)
records = load_records()
model_path = "{}/{}".format(work_dir, message.taskName)
try:
shutil.rmtree(model_path)
except FileNotFoundError:
pass
model = Seq2Seq(model_def, working_dir=model_path)
model.train(
records,
epochs=model_def["epochs"],
batch_size=model_def["batch_size"],
callbacks=[
LambdaCallback(
on_epoch_end=lambda epoch, logs: set_progress(epoch + 1))
])
model.save_model()
model.save_model_def()
model.save_no_train()
model.save_history()
model.save_for_inference_tf()
compress_and_set_result(model_path)
except Exception:
status = WorkerStatus.ERROR
task = TaskType.NONE
progress = 0
traceback.print_exc()
for _ in range(m_l):
next_tok = torch.argmax(log_prob)
translation = translation + [next_tok.item()]
if next_tok == eosindx:
break
log_prob, hidden = model.decode(hidden, next_tok.item())
translations[i] = translation
return translations
if __name__ == "__main__":
from Seq2Seq import Seq2Seq
sentences = [
"I am an apple", "There is a dog", "look a tree",
'here is a realy long sentence just for verifying'
]
words = []
for s in sentences:
for w in s.split():
words.append(w)
words = set(words)
word2idx = {w: i for i, w in enumerate(list(words))}
x_words = word2idx
seq2seq = Seq2Seq(word2idx.copy(), word2idx.copy())
translation = SimpleGreedyDecodingTranslation(seq2seq, sentences)
#translation = GreedyDecodingTranslation(seq2seq, sentences)
print(translation)
print(len(translation))
#translation = BeamDecodingTranslation(seq2seq, sentences, 2)
print(len(translation))
'./model_def_phase1_topology/S2S-T4-GRU-Nadam.json',
'./model_def_phase1_topology/S2S-T4-LSTM-Nadam.json',
'./model_def_phase1_topology/S2S-T5-GRU-Nadam-ReLU.json',
'./model_def_phase1_topology/S2S-T5-LSTM-Nadam-ReLU.json',
'./model_def_phase1_topology/S2S-T6-GRU-Nadam-ReLU.json',
'./model_def_phase1_topology/S2S-T6-LSTM-Nadam-ReLU.json',
'./model_def_phase2_activation/S2S-T5-GRU-Nadam-LReLU.json',
'./model_def_phase2_activation/S2S-T5-GRU-Nadam-Sigmoid.json',
'./model_def_phase2_activation/S2S-T5-GRU-Nadam-Tanh.json',
'./model_def_phase2_activation/S2S-T5-GRU-Nadam-Elu.json',
'./model_def_phase2_activation/S2S-T5-LSTM-Nadam-LReLU.json',
'./model_def_phase2_activation/S2S-T5-LSTM-Nadam-Sigmoid.json',
'./model_def_phase2_activation/S2S-T5-LSTM-Nadam-Tanh.json',
'./model_def_phase2_activation/S2S-T5-LSTM-Nadam-Elu.json',
'./model_def_phase2_activation/S2S-T6-GRU-Nadam-LReLU.json',
'./model_def_phase2_activation/S2S-T6-GRU-Nadam-Sigmoid.json',
'./model_def_phase2_activation/S2S-T6-GRU-Nadam-Tanh.json',
'./model_def_phase2_activation/S2S-T6-GRU-Nadam-Elu.json',
'./model_def_phase2_activation/S2S-T6-LSTM-Nadam-LReLU.json',
'./model_def_phase2_activation/S2S-T6-LSTM-Nadam-Sigmoid.json',
'./model_def_phase2_activation/S2S-T6-LSTM-Nadam-Tanh.json',
'./model_def_phase2_activation/S2S-T6-LSTM-Nadam-Elu.json'
]
for model in models:
for i in range(0, 2):
mdl = train(model, 300, 64)
K.clear_session()
Seq2Seq(load=True, working_dir=mdl.working_dir)
K.clear_session()
def __init__(self, config):
self.restore_ckpt = None
if 'restore_ckpt' in config:
self.restore_ckpt = config['restore_ckpt']
self.eval_interval = 1000
self.save_interval = 1000
self.show_interval = 100
self.train_epoch = 100
self.max_iter = 1000000
#train dataset parse
self.train_dataset_type = config['train_dataset_type']
self.train_dataset = None
if self.train_dataset_type == 'tfrecord':
self.train_dataset = RecordDataset
elif self.train_dataset_type == 'filelist':
self.train_dataset = FileDataset
train_ds_config = {}
train_ds_config['norm_h'] = int(config['norm_h'])
train_ds_config['expand_rate'] = float(config['expand_rate'])
train_ds_config['file_list'] = config['train_file_list']
train_ds_config['num_parallel'] = config['num_parallel']
train_ds_config['batch_size'] = config['batch_size']
train_ds_config['char_dict'] = config['char_dict']
train_ds_config['model_type'] = config['model_type']
train_ds_config['mode'] = 'train'
self.train_dataset = self.train_dataset(train_ds_config).data_reader(
self.train_epoch)
#eval dataset parse
self.eval_dataset_type = config['eval_dataset_type']
self.eval_dataset = None
if self.eval_dataset_type == 'tfrecord':
self.eval_dataset = RecordDataset
elif self.eval_dataset_type == 'filelist':
self.eval_dataset = FileDataset
eval_ds_config = {}
eval_ds_config['norm_h'] = int(config['norm_h'])
eval_ds_config['expand_rate'] = float(config['expand_rate'])
eval_ds_config['file_list'] = config['eval_file_list']
eval_ds_config['num_parallel'] = config['num_parallel']
eval_ds_config['batch_size'] = config['batch_size']
eval_ds_config['char_dict'] = config['char_dict']
eval_ds_config['model_type'] = config['model_type']
eval_ds_config['mode'] = 'test'
self.eval_dataset = self.eval_dataset(eval_ds_config).data_reader()
#charset parse
self.char_dict = config['char_dict']
self.charset = Charset(self.char_dict)
self.step_counter = tf.Variable(tf.constant(0),
trainable=False,
name='step_counter')
self.decoder_dict = {}
self.loss_value = 0.0
self.learning_rate = 0.01
if 'learning_rate' in config:
self.learning_rate = float(config['learning_rate'])
self.learning_rate_decay = None
if 'learning_rate_decay' in config:
self.learning_rate_decay = config['learning_rate_decay']
'''
if self.learning_rate_decay == 'piecewise_constant':
boundaries = [30000, 60000]
values = [self.learning_rate, self.learning_rate*0.5, self.learning_rate*0.2]
self.learning_rate = tf.train.piecewise_constant(self.step_counter, boundaries, values)
elif self.learning_rate_decay == 'exponential_decay':
decay_rate = 0.8
decay_steps = 40000
self.learning_rate = tf.train.exponential_decay(self.learning_rate,
self.step_counter,
decay_steps,
decay_rate)
elif self.learning_rate_decay == 'linear_cosine_decay':
decay_steps = 30000
self.learning_rate = tf.train.linear_cosine_decay(self.learning_rate,
self.step_counter,
decay_steps)
'''
if 'eval_interval' in config:
self.eval_interval = int(config['eval_interval'])
if 'save_interval' in config:
self.save_interval = int(config['save_interval'])
if 'train_epoch' in config:
self.train_epoch = int(config['train_epoch'])
if 'max_iter' in config:
self.max_iter = int(config['max_iter'])
self.max_padding = 20
self.eos_id = self.charset.get_eosid()
self.sos_id = self.charset.get_sosid()
self.vocab_size = self.charset.get_size()
self.embedding_dim = 96
self.enc_units = 128
self.dec_units = 128
self.seq2seq = Seq2Seq(vocab_size=self.vocab_size,
embedding_dim=self.embedding_dim,
SOS_ID=self.sos_id,
EOS_ID=self.eos_id,
dec_units=self.dec_units,
enc_units=self.enc_units,
attention_name='luong',
attention_type=0,
rnn_type='gru',
max_length=self.max_padding)
self.optimizer_type = 'sgd'
if 'optimizer_type' in config:
self.optimizer_type = config['optimizer_type']
if self.optimizer_type not in ('sgd', 'momentum', 'adam'):
print(
"Solover Error: optimizer_type {} not in [sgd, momentum, adam]"
.format(self.optimizer_type))
self.optimizer = tf.keras.optimizers.Adam(self.learning_rate)
if self.optimizer_type == 'sgd':
self.optimizer = tf.keras.optimizers.SGD(self.learning_rate)
elif self.optimizer_type == 'momentum':
self.optimizer = tf.keras.optimizers.SGD(self.learning_rate, 0.95)
elif self.optimizer_type == 'adam':
self.optimizer = tf.keras.optimizers.Adam(self.learning_rate)
self.checkpoint_dir = 'training_checkpoints'
if 'checkpoint_dir' in config:
self.checkpoint_dir = config['checkpoint_dir']
if not os.path.isdir(self.checkpoint_dir):
os.makedirs(self.checkpoint_dir)
self.checkpoint_prefix = os.path.join(self.checkpoint_dir, "ckpt")
self.checkpoint = tf.train.Checkpoint(optimizer=self.optimizer,
model=self.seq2seq)
if self.restore_ckpt is not None and os.path.isdir(self.restore_ckpt):
if tf.train.latest_checkpoint(self.restore_ckpt) is not None:
self.checkpoint.restore(
tf.train.latest_checkpoint(self.restore_ckpt))
else:
if tf.train.latest_checkpoint(self.checkpoint_dir) is not None:
self.checkpoint.restore(
tf.train.latest_checkpoint(self.checkpoint_dir))
from Seq2Seq import Seq2Seq,data_generator
import numpy as np
train_houses = [2,3,4,5,6]
test_house = 1
#appliance = 'washer_dryer'
appliance = 'refrigerator'
windows_length = 600
epochs = 15
#Type = 'CNN-RNN'
#Type = 'ConvLSTM'
#Type = 'Dense'
#Type = 'CNN-2d' # 适用 mode：'od'
#Type = 'CNN-1d-2'
Type = 'CNN(Chaoyun)'
seq2seq = Seq2Seq('o')
seq2seq.get_houses_data(train_houses,test_house,
appliance,windows_length,2000,200)
seq2seq.build_network(Type)
#seq2seq.load_model_and_history(Type)
seq2seq.train(epochs=epochs)
seq2seq.save_model_and_history()
seq2seq.plot_training_history()
seq2seq._demo_show()
def __init__(self, config):
self.restore_ckpt = None
if 'restore_ckpt' in config:
self.restore_ckpt = config['restore_ckpt']
self.eval_interval = 1000
self.save_interval = 10
self.show_interval = 100
self.train_epoch = 100
self.max_iter = 1000000
#train dataset parse
self.train_dataset_type = config['train_dataset_type']
self.train_dataset = None
if self.train_dataset_type == 'tfrecord':
self.train_dataset = RecordDataset
elif self.train_dataset_type == 'filelist':
self.train_dataset = FileDataset
train_ds_config = {}
train_ds_config['norm_h'] = int(config['norm_h'])
train_ds_config['expand_rate'] = float(config['expand_rate'])
train_ds_config['file_list'] = config['train_file_list']
train_ds_config['num_parallel'] = config['num_parallel']
train_ds_config['batch_size'] = config['batch_size']
train_ds_config['char_dict'] = config['char_dict']
train_ds_config['model_type'] = config['model_type']
train_ds_config['mode'] = 'train'
self.train_dataset = self.train_dataset(train_ds_config).data_reader_ctc_attention(self.train_epoch)
#eval dataset parse
self.eval_dataset_type = config['eval_dataset_type']
self.eval_dataset = None
if self.eval_dataset_type == 'tfrecord':
self.eval_dataset = RecordDataset
elif self.eval_dataset_type == 'filelist':
self.eval_dataset = FileDataset
eval_ds_config = {}
eval_ds_config['norm_h'] = int(config['norm_h'])
eval_ds_config['expand_rate'] = float(config['expand_rate'])
eval_ds_config['file_list'] = config['eval_file_list']
eval_ds_config['num_parallel'] = config['num_parallel']
eval_ds_config['batch_size'] = config['batch_size']
eval_ds_config['char_dict'] = config['char_dict']
eval_ds_config['model_type'] = config['model_type']
eval_ds_config['mode'] = 'test'
self.eval_dataset = self.eval_dataset(eval_ds_config).data_reader_ctc_attention()
#charset parse
self.char_dict = config['char_dict']
self.model_type = config['model_type']
self.charset = Charset(self.char_dict, self.model_type)
self.step_counter = tf.train.get_or_create_global_step()
self.decoder_dict = {}
self.loss_value = 0.0
self.learning_rate = 0.01
if 'learning_rate' in config:
self.learning_rate = float(config['learning_rate'])
self.learning_rate_decay = None
if 'learning_rate_decay' in config:
self.learning_rate_decay = config['learning_rate_decay']
if self.learning_rate_decay == 'piecewise_constant':
boundaries = [30000, 60000]
values = [self.learning_rate, self.learning_rate*0.5, self.learning_rate*0.2]
self.learning_rate = tf.train.piecewise_constant(self.step_counter, boundaries, values)
elif self.learning_rate_decay == 'exponential_decay':
decay_rate = 0.8
decay_steps = 40000
self.learning_rate = tf.train.exponential_decay(self.learning_rate,
self.step_counter,
decay_steps,
decay_rate)
elif self.learning_rate_decay == 'linear_cosine_decay':
decay_steps = 30000
self.learning_rate = tf.train.linear_cosine_decay(self.learning_rate,
self.step_counter,
decay_steps)
if 'eval_interval' in config:
self.eval_interval = int(config['eval_interval'])
if 'save_interval' in config:
self.save_interval = int(config['save_interval'])
if 'train_epoch' in config:
self.train_epoch = int(config['train_epoch'])
if 'max_iter' in config:
self.max_iter = int(config['max_iter'])
self.max_dec_length = 20
self.max_enc_length = 1200
self.eos_id = self.charset.get_eosid()
self.sos_id = self.charset.get_sosid()
self.vocab_size = self.charset.get_size()
self.enc_used_rnn = False
self.enc_num_layers = 0
self.dec_num_layers = 1
self.d_model = 512
self.enc_num_heads = 4
self.dec_num_heads = 4
self.enc_dff = 1024
self.dec_dff = 1024
self.enc_rate = 0.0
self.dec_rate = 0.0
self.seq2seq = Seq2Seq(enc_num_layers=self.enc_num_layers,
dec_num_layers=self.dec_num_layers,
d_model=self.d_model,
vocab_size=self.vocab_size,
enc_num_heads=self.enc_num_heads,
dec_num_heads=self.dec_num_heads,
enc_dff=self.enc_dff,
dec_dff=self.dec_dff,
enc_used_rnn=self.enc_used_rnn,
sos_id=self.sos_id,
eos_id=self.eos_id,
max_enc_length=self.max_enc_length,
max_dec_length=self.max_dec_length,
enc_rate=self.enc_rate,
dec_rate=self.dec_rate)
self.optimizer_type = 'adam'
if 'optimizer_type' in config:
self.optimizer_type = config['optimizer_type']
if self.optimizer_type not in ('sgd', 'momentum', 'adam'):
print("Solover Error: optimizer_type {} not in [sgd, momentum, adam]".format(self.optimizer_type))
self.optimizer = tf.train.AdamOptimizer(self.learning_rate)
if self.optimizer_type == 'sgd':
self.optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
elif self.optimizer_type == 'momentum':
self.optimizer = tf.train.MomentumOptimizer(self.learning_rate, 0.95)
elif self.optimizer_type == 'adam':
self.optimizer = tf.train.AdamOptimizer(self.learning_rate)
self.checkpoint_dir = 'training_checkpoints'
if 'checkpoint_dir' in config:
self.checkpoint_dir = config['checkpoint_dir']
if not os.path.isdir(self.checkpoint_dir):
os.makedirs(self.checkpoint_dir)
self.checkpoint_prefix = os.path.join(self.checkpoint_dir, "ckpt")
self.checkpoint = tf.train.Checkpoint(optimizer=self.optimizer,
seq2seq=self.seq2seq,
global_step=self.step_counter)
if self.restore_ckpt is not None and os.path.isdir(self.restore_ckpt):
if tf.train.latest_checkpoint(self.restore_ckpt) is not None:
self.checkpoint.restore(tf.train.latest_checkpoint(self.restore_ckpt))
else:
if tf.train.latest_checkpoint(self.checkpoint_dir) is not None:
self.checkpoint.restore(tf.train.latest_checkpoint(self.checkpoint_dir))
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
torch.backends.cudnn.enabled = False
print("Input dim: ", n_vocab)
################################# DEFINE MODEL ################################
model = Seq2Seq(n_vocab = n_vocab,
n_speaker = n_speaker,
n_tags = n_tags,
n_embed_text = 128,
n_embed_speaker = 64,
n_embed_tags = 128,
n_embed_dec = 2,
n_hidden_enc = 128,
n_hidden_dec = 128,
n_layers = 1,
n_output = 2,
dropout = 0.5)
model = model.cuda()
print(f'The model has {count_parameters(model):,} trainable parameters')
'''
# set single/multi gpu usage
if not args.gpu is None:
torch.cuda.manual_seed(0)
if len(args.gpu) == 1:
torch.cuda.set_device(int(args.gpu))
def main():
# TODO: Add a function to parse arguments.
# Global setting here.
mode_list = ["train", "visualization", "analysis"]
mode = mode_list[0]
task = "control_length"
# Each model has its own file. User can add information (epochs, randon seed, task) in the file name.
model_path = "../saved_model/control_length_units=4"
if mode == "train" or mode == "analysis":
units = 4
if mode == 'train':
# Train a Seq2Seq model.
# Define other variables only used in train mode.
epochs = 10
log_file = "training_control_length_units=4.txt"
seq2seq = Seq2Seq(mode, task, model_path, log_file, units, epochs)
seq2seq.seq2seq_model, seq2seq.encoder_model, seq2seq.decoder_model = build_model.seq2seq(
seq2seq.src_max_len,
seq2seq.tgt_max_len,
seq2seq.src_token_size,
seq2seq.tgt_token_size,
latent_dim=seq2seq.units)
seq2seq.train()
seq2seq.check_accuracy(check_list=["word", "length"
]) # Check accuracy after training.
print("data =", seq2seq.data)
print("units =", seq2seq.units)
print("=" * 50)
if mode == "analysis":
# Evaluate a trained Seq2Seq and get trained weights and hidden values from the model.
seq2seq = Seq2Seq(mode, task, model_path, units=units)
seq2seq.seq2seq_model, seq2seq.encoder_model, seq2seq.decoder_model = build_model.seq2seq(
seq2seq.src_max_len,
seq2seq.tgt_max_len,
seq2seq.src_token_size,
seq2seq.tgt_token_size,
latent_dim=seq2seq.units)
seq2seq.load_seq2seq(model_path)
# seq2seq.check_accuracy(check_list=["word", "length"])
# Get the trained weights and save them.
weights = analysis.get_gru_weights(seq2seq)
with open(os.path.join(model_path, 'gate_values'),
'rb') as filehandler:
gate_values = pickle.load(filehandler)
analysis_weight.main(weights, gate_values)
return
# Get hidden state by their labels and save them.
# TODO: Parse N (sample number of each label and each time step) into this function.
sample = analysis.get_sample(task)
# Hidden values in the last fully connected layer.
dense_values = analysis.get_dense_values(seq2seq, sample)
with open(os.path.join(model_path, 'dense_values'),
'wb') as filehandler:
print('(Main) dense_values =',
dense_values.shape) # Shape = (10, 12, 100, 8)
pickle.dump(dense_values, filehandler)
# Hidden values in the decoder GRU layer.
hidden_state = analysis.get_hidden_state(
seq2seq, sample) # Shape = (10, 100, 12, 16)
hidden_state = hidden_state.transpose([0, 2, 1, 3])
with open(os.path.join(model_path, 'hidden_state'),
'wb') as filehandler:
print('(Main) hidden_state =',
hidden_state.shape) # Shape = (10, 12, 100, 16)
pickle.dump(hidden_state, filehandler)
# Gate values in the decoder GRU layer.
gate_values = analysis.get_gate_values(seq2seq, sample)
with open(os.path.join(model_path, 'gate_values'),
'wb') as filehandler:
print('(Main) gate_values[z] =',
gate_values['z'].shape) # Shape = (10, 12, 100, 16)
pickle.dump(gate_values, filehandler)
# pdb.set_trace()
# analysis.dim_reduction_plot(hidden_state, sample)
if mode == 'visualization':
# Load the hidden state and visualize it.
# TODO: Move this part to visualization.main().
# Load values.
with open(os.path.join(model_path, 'gate_values'),
'rb') as filehandler:
gate_values = pickle.load(filehandler)
if task == "control_length":
title = ['Length=' + str(i) for i in range(1, 11)]
for gate in ['h', 'z', 'r', 'hh']:
visualization.plot_xy(gate_values[gate],
title,
name='%s' % gate)
# return
for gate in ['h', 'z', 'r', 'hh']:
visualization.scatter_gate_values(gate_values[gate], name=gate)
break
options = None
run_metadata = None
# options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
# run_metadata = tf.RunMetadata()
train_timeline_fname = 'timeline_01.json'
valid_timeline_fname = "timeline_infer_1s"
if params_common["model_name"][0] == "Transformer":
model_params = params_models[params_common["model_name"][0]][
params_common["model_name"][1]]
model_params.update(vocab_size=max(params_common["source_vocab_size"],
params_common["target_vocab_size"]))
model = Transformer(params_common, model_params)
else:
model_params = params_models[params_common["model_name"][0]]
model = Seq2Seq(params_common, model_params)
print(model_params)
if params_common["mode"] == "train":
train_iter = BatchIterator(path_train_x, path_train_y, source_vocab2id,
target_vocab2id, params_common["start_id"],
params_common["end_id"],
params_common["unk_id"],
params_common["pad_id"],
params_common["reverse_target"])
valid_iter = BatchIterator(path_valid_x, path_valid_y, source_vocab2id,
target_vocab2id, params_common["start_id"],
params_common["end_id"],
params_common["unk_id"],
DEVICE = 'cpu'
# initialize encoder, decoder and seq2seq model classes
enc = Encoder(INPUT_DIM,
ENC_HID_DIM,
N_LAYERS,
ENC_DROPOUT,
is_bidirectional=False)
attn = CosAtt(enc, DEC_HID_DIM, N_LAYERS)
dec = Decoder(OUTPUT_DIM,
DEC_HID_DIM,
N_LAYERS,
DEC_DROPOUT,
enc,
attention=attn)
model = Seq2Seq(enc, dec, DEVICE, attention=attn)
# initialize values of learnable parameters
def init_weights(m):
for name, param in m.named_parameters():
nn.init.uniform_(param.data, -0.08, 0.08)
model.apply(init_weights)
# count parameters
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
if __name__ == '__main__':
source_path = '../data/test/source.txt'
target_path = '../data/test/target.txt'
vocab_path = '../data/vocab.txt'
model_path = '../model/model.pth'
tokenizer = Tokenizer(vocab_path)
config = Config()
fr = open('../result/test.txt','w',encoding='utf-8-sig') # 存储预测结果
loader = DataLoader(dataset=MyDataSet(source_path, target_path, tokenizer), batch_size=config.batch_size, shuffle=True,
num_workers=2,collate_fn=pad,drop_last=False) # 最后一个batch数据集不丢弃
if not torch.cuda.is_available():
print('No cuda is available!')
exit()
device = torch.device('cuda:0')
model = Seq2Seq(config)
model.to(device)
# 加载模型
checkpoint = torch.load(model_path,map_location=device)
model.load_state_dict(checkpoint['model'])
for iter, (batch_x, batch_y, batch_source_lens,batch_target_lens) in enumerate(loader):
batch_x = batch_x.cuda()
batch_source_lens = torch.as_tensor(batch_source_lens)
# 预测结果和相应时刻的注意力权重
results = model.BatchSample(batch_x,batch_source_lens)
for i in range(len(results)):
words = tokenizer.convert_ids_to_tokens(results[i])
if i % 100 == 0:
print(''.join(words))
fr.write(''.join(words))
def __init__(self, config):
self.restore_ckpt = None
if 'restore_ckpt' in config:
self.restore_ckpt = config['restore_ckpt']
self.clip_max_gradient = 3.0
if 'clip_max_gradient' in config:
self.clip_max_gradient = float(config['clip_max_gradient'])
self.clip_min_gradient = -3.0
if 'clip_min_gradient' in config:
self.clip_min_gradient = float(config['clip_min_gradient'])
self.eval_interval = 2000
self.save_interval = 1000
self.show_interval = 10
self.train_epoch = 100
self.max_iter = 1000000
self.mirrored_strategy = None
self.num_replicas_in_sync = 1
self.batch_per_replica = int(config['batch_size'])
self.devices = configs['devices'].split(',')
assert(len(self.devices) > 1)
self.mirrored_strategy = tf.distribute.MirroredStrategy(devices=self.devices)
self.num_replicas_in_sync = self.mirrored_strategy.num_replicas_in_sync
self.global_batch_size = self.batch_per_replica * self.num_replicas_in_sync
#train dataset parse
self.train_dataset_type = config['train_dataset_type']
self.train_dataset = None
if self.train_dataset_type == 'tfrecord':
self.train_dataset = RecordDataset
elif self.train_dataset_type == 'filelist':
self.train_dataset = FileDataset
train_ds_config = dict()
train_ds_config['norm_h'] = int(config['norm_h'])
train_ds_config['expand_rate'] = float(config['expand_rate'])
train_ds_config['file_list'] = config['train_file_list']
train_ds_config['num_parallel'] = config['num_parallel']
train_ds_config['batch_size'] = self.global_batch_size
train_ds_config['model_type'] = config['model_type']
train_ds_config['char_dict'] = config['char_dict']
train_ds_config['mode'] = 'train'
self.train_dataset = self.train_dataset(train_ds_config).data_reader(self.train_epoch)
#eval dataset parse
self.eval_dataset_type = config['eval_dataset_type']
self.eval_dataset = None
if self.eval_dataset_type == 'tfrecord':
self.eval_dataset = RecordDataset
elif self.eval_dataset_type == 'filelist':
self.eval_dataset = FileDataset
eval_ds_config = dict()
eval_ds_config['norm_h'] = int(config['norm_h'])
eval_ds_config['expand_rate'] = float(config['expand_rate'])
eval_ds_config['file_list'] = config['eval_file_list']
eval_ds_config['num_parallel'] = config['num_parallel']
eval_ds_config['batch_size'] = self.global_batch_size
eval_ds_config['char_dict'] = config['char_dict']
eval_ds_config['model_type'] = config['model_type']
eval_ds_config['mode'] = 'test'
self.eval_dataset = self.eval_dataset(eval_ds_config).data_reader()
self.train_dataset = self.mirrored_strategy.experimental_distribute_dataset(self.train_dataset)
self.eval_dataset = self.mirrored_strategy.experimental_distribute_dataset(self.eval_dataset)
#charset parse
self.char_dict = config['char_dict']
self.model_type = config['model_type']
self.charset = Charset(self.char_dict, self.model_type)
self.step_counter = 0
#self.step_counter = tf.Variable(tf.constant(0), trainable=False, name='step_counter')
self.learning_rate = 0.01
if 'learning_rate' in config:
self.learning_rate = float(config['learning_rate'])
lr_decay_steps = 200000
lr_decay_rate = 0.96
if 'lr_decay_steps' in configs:
lr_decay_steps = int(configs['lr_decay_steps'])
if 'lr_decay_rate' in configs:
lr_decay_rate = float(configs['lr_decay_rate'])
self.schedules_type = 'ExponentialDecay'
if self.schedules_type == 'ExponentialDecay':
self.learning_rate = tf.keras.optimizers.schedules.ExponentialDecay(self.learning_rate,
lr_decay_steps,
lr_decay_rate)
elif self.schedules_type == 'InverseTimeDecay':
self.learning_rate = tf.keras.optimizers.schedules.InverseTimeDecay(self.learning_rate,
lr_decay_steps,
lr_decay_rate)
if 'eval_interval' in config:
self.eval_interval = int(config['eval_interval'])
if 'save_interval' in config:
self.save_interval = int(config['save_interval'])
if 'train_epoch' in config:
self.train_epoch = int(config['train_epoch'])
if 'max_iter' in config:
self.max_iter = int(config['max_iter'])
self.optimizer_type = 'adam'
if 'optimizer_type' in config:
self.optimizer_type = config['optimizer_type']
if self.optimizer_type not in ('sgd', 'momentum', 'adam', 'rmsprop', 'adadelte'):
print("Solover Error: optimizer_type {} not in [sgd, momentum, adam, rmsprop, adadelte]".format(self.optimizer_type))
self.eos_id = self.charset.get_eosid()
self.vocab_size = self.charset.get_size()
with self.mirrored_strategy.scope():
self.seq2seq = Seq2Seq(vocab_size=self.vocab_size, eos_id=self.eos_id)
self.train_loss = tf.keras.metrics.Mean('train_loss')
self.seq_err_cnt = tf.keras.metrics.Sum('seqerr_count')
self.seq_all_cnt = tf.keras.metrics.Sum('seqerr_count')
self.char_err_cnt = tf.keras.metrics.Sum('charerr_count')
self.char_all_cnt = tf.keras.metrics.Sum('charall_count')
self.optimizer = tf.keras.optimizers.Adam(self.learning_rate)
if self.optimizer_type == 'sgd':
self.optimizer = tf.keras.optimizers.SGD(self.learning_rate)
elif self.optimizer_type == 'momentum':
self.optimizer = tf.keras.optimizers.SGD(self.learning_rate, 0.95)
elif self.optimizer_type == 'adam':
self.optimizer = tf.keras.optimizers.Adam(self.learning_rate)
elif self.optimizer_type == 'rmsprop':
self.optimizer = tf.keras.optimizers.RMSprop(self.learning_rate)
elif self.optimizer_type == 'adadelte':
self.optimizer = tf.keras.optimizers.Adadelta(self.learning_rate)
self.checkpoint_dir = 'training_checkpoints'
if 'checkpoint_dir' in config:
self.checkpoint_dir = config['checkpoint_dir']
if not os.path.isdir(self.checkpoint_dir):
os.makedirs(self.checkpoint_dir)
self.checkpoint_prefix = os.path.join(self.checkpoint_dir, "ckpt")
self.checkpoint = tf.train.Checkpoint(optimizer=self.optimizer, model=self.seq2seq)
if self.restore_ckpt is not None and os.path.isdir(self.restore_ckpt):
if tf.train.latest_checkpoint(self.restore_ckpt) is not None:
self.checkpoint.restore(tf.train.latest_checkpoint(self.restore_ckpt))
else:
if tf.train.latest_checkpoint(self.checkpoint_dir) is not None:
self.checkpoint.restore(tf.train.latest_checkpoint(self.checkpoint_dir))
batch_size=config.batch_size,
shuffle=True,
num_workers=0,
collate_fn=pad,
drop_last=False) # 最后一个batch数据集不丢弃
# 评估集
eval_loader = DataLoader(dataset=MyDataSet(eval_source_path,
eval_target_path,
eval_keyword_path, tokenizer),
batch_size=config.batch_size,
shuffle=True,
num_workers=0,
collate_fn=pad,
drop_last=False) # 最后一个batch数据集不丢弃
model = Seq2Seq(config, bert_model)
model.to(device)
checkpoint = torch.load(pre_trainModel)
model.load_state_dict(checkpoint['model'])
#optimizer = optim.SGD(model.parameters(),lr=config.learning_rate)
optimizer = optim.Adam(model.parameters(), lr=config.learning_rate)
optimizer.load_state_dict(checkpoint['optimizer'])
criterion = nn.CrossEntropyLoss()
total_loss_criterion = 0
PAD_ID = 0
print_every = 20000
num_training_steps = 600000 # 至少训练多少个step
print_every_loss = [] # 记录每print_every个step的batch大小数据的平均损失，用于输出查看损失曲线
num_eval = 8 # 评估集最少评估次数
Rouge_threshold = 3 # 评估集上前后num_eval次评估时，ROUGE差值若低于该值，则满足早停条件，ROUGE采用百分制
#TransformerParameters
emsize = args.transformer_embedding_dim # 200 embedding dimension
nhid = args.transformer_hidden_dim #200 the dimension of the feedforward network model in nn.TransformerEncoder
nlayers = args.transformer_n_layers #2 the number of nn.TransformerEncoderLayer in nn.TransformerEncoder
nhead = args.transformer_n_head #2 the number of heads in the multiheadattention models
dropout = args.transformer_dropout # 0.2 the dropout value
# encoder
if args.model == 'seq2seq':
enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS,
ENC_DROPOUT).to(device)
dec = Decoder(DEC_EMB_DIM, OUTPUT_DIM, HID_DIM, N_LAYERS,
DEC_DROPOUT).to(device)
model = Seq2Seq(enc, dec, attn=False).to(device)
optimizer = optim.Adam(model.parameters())
elif args.model == 'hred':
enc = RecurrentEncoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS,
ENC_DROPOUT).to(device)
dec = AttnDecoder(DEC_EMB_DIM, OUTPUT_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT,
MAX_LENGTH).to(device)
model = Seq2Seq(enc, dec, attn=True).to(device)
optimizer = optim.Adam(model.parameters())
elif args.model == 'seq2seq_attn':
enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, HRED_N_LAYERS,
ENC_DROPOUT).to(device)
dec = AttnDecoder(DEC_EMB_DIM, OUTPUT_DIM, HID_DIM, HRED_N_LAYERS,
DEC_DROPOUT, MAX_LENGTH).to(device)
model = Seq2Seq(enc, dec, attn=True).to(device)
optimizer = optim.Adam(model.parameters())
return elapsed_mins, elapsed_secs
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
torch.backends.cudnn.enabled = False
print("Input dim: ", n_vocab)
################################# DEFINE MODEL ################################
model = Seq2Seq(n_vocab=n_vocab,
n_embed_text=128,
n_embed_dec=2,
n_hidden_enc=128,
n_hidden_dec=128,
n_layers=1,
n_output=2,
dropout=0.5)
model = model.cuda()
print(f'The model has {count_parameters(model):,} trainable parameters')
if args.load_from > 0:
model.load_state_dict(torch.load("_epoch_" + str(args.load_from) + ".pt"))
########################### DEFINE LOSS & OPTIMIZER ###########################
positive_label_weights = torch.tensor([args.pos_weight]).cuda()
#criterion = nn.BCEWithLogitsLoss(pos_weight=positive_label_weights)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(),
lr=0.001,
# intialization hyper-par
INPUT_DIM = len(SRC.vocab)
OUTPUT_DIM = len(TRG.vocab)
ENC_EMB_DIM = 100
DEC_EMB_DIM = 100
ENC_HID_DIM = 2
DEC_HID_DIM = 2
N_LAYERS = 2
ENC_DROPOUT = 0.5
DEC_DROPOUT = 0.5
device = 'cpu'#torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# initialize encoder, decoder and model onject
enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, N_LAYERS, ENC_DROPOUT)
dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, DEC_HID_DIM, N_LAYERS, DEC_DROPOUT)
model = Seq2Seq(enc, dec, device).to(device)
# initialize encoder, decoder, model objects with attention
attn = Attention(ENC_HID_DIM, DEC_HID_DIM)
enc_a = EncoderWAttention(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, N_LAYERS, ENC_DROPOUT)
dec_a = DecoderWAttention(OUTPUT_DIM, DEC_EMB_DIM, DEC_HID_DIM, ENC_HID_DIM, DEC_DROPOUT, attn)
model_a = Seq2SeqWAttention(enc_a, dec_a, device).to(device)
# initialize weights
def init_weights(m):
for name, param in m.named_parameters():
nn.init.uniform_(param.data, -0.08, 0.08)
model.apply(init_weights)
model_a.apply(init_weights)
# count parameters