def train_seq2seq():
# seq2seq models require a dim 3 input matrix (rows, timesteps, num_features )
in_train_x, in_test_x, out_test_x = np.expand_dims(
in_train, axis=2), np.expand_dims(in_test, axis=2), np.expand_dims(out_test, axis=2)
seq2seq_kwargs = {}
seq2seq_kwargs["encoder_dim"] = [10]
seq2seq_kwargs["decoder_dim"] = [20]
seq2seq_kwargs["epochs"] = 40
seq2seq_kwargs["batch_size"] = 256
seq2seq_kwargs["learning_rate"] = 0.01
n_features = 1 # single value per feature
seq2seq = Seq2SeqModel(n_features, **seq2seq_kwargs)
seq2seq.train(in_train_x, in_test_x)
seq2seq.save_model()
# seq2seq.load_model()
inlier_scores = seq2seq.compute_anomaly_score(
in_test_x[np.random.randint(100, size=400), :])
outlier_scores = seq2seq.compute_anomaly_score(
out_test_x[np.random.randint(100, size=80), :])
print(inlier_scores[:5])
print(outlier_scores[:5])
metrics = eval_utils.evaluate_model(
inlier_scores, outlier_scores, model_name="seq2seq", show_plot=False)
print(metrics)
return metrics
def __init__(self, args=None, vocab=None, emb_matrix=None, model_file=None, use_cuda=False, teacher_model_file=None):
self.use_cuda = use_cuda
self.vocab = vocab
if model_file is not None:
# load everything from file
self.load(model_file, use_cuda, args=args, vocab=vocab)
else:
# build model from scratch
self.args = args
self.model = Seq2SeqModel(args, emb_matrix=emb_matrix, use_cuda=use_cuda)
# dict-based components
self.word_dict = dict()
self.composite_dict = dict()
self.crit = torch.nn.NLLLoss(ignore_index=constant.PAD_ID)
self.nce_parameters = []
if self.args['lambda2'] != 0:
self.init_teacher(teacher_model_file, use_cuda, emb_matrix=emb_matrix)
self.teacher_update_count = 0
self.parameters = [p for p in self.model.parameters() if p.requires_grad]
if use_cuda:
self.model.cuda()
self.crit.cuda()
else:
self.model.cpu()
self.crit.cpu()
self.optimizer = utils.get_optimizer(self.args['optim'], self.parameters, self.args['lr'])
def run_training():
news = SougouNews()
batches = news.batch_sequences(length_from=3, length_to=8, vocab_lower=2, vocab_upper=10,
batch_size=100)
print(batches)
print(np.array(batches).shape)
loss_track = []
model = Seq2SeqModel()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
try:
for batch in range(5000):
batch_data = next(batches)
feed_data = model.make_train_inputs(batch_data, batch_data)
_, loss = sess.run([model.train_op, model.loss], feed_dict=feed_data)
loss_track.append(loss)
if batch == 0 or batch % 500 == 0:
logging.info('batch {1}, mini batch loss: {2}'.format(batch, loss))
for i, (e_in, dt_pred) in enumerate(zip(
feed_data[model.encoder_inputs].T,
sess.run(model.decoder_prediction_train, feed_data).T
)):
print(' sample {}:'.format(i + 1))
print(' enc input > {}'.format(e_in))
print(' dec train predicted > {}'.format(dt_pred))
except KeyboardInterrupt:
logging.info('training interrupted.')
def load(self, filename, use_cuda=False, args=None, vocab=None):
try:
print(f"Loading model from '{filename}'...")
checkpoint = torch.load(filename, lambda storage, loc: storage)
except BaseException:
print("Cannot load model from {}".format(filename))
sys.exit(1)
self.args = checkpoint['config']
if args is not None:
self.args['lambda1'] = args['lambda1']
self.args['lambda2'] = args['lambda2']
self.args['lambda_reinforce'] = args['lambda_reinforce']
self.args['teacher_elmo'] = args['teacher_elmo']
self.args['teacher_spec_only'] = args['teacher_spec_only']
self.args['teacher_info_only'] = args['teacher_info_only']
if vocab is not None:
self.args['vocab'] = vocab
self.model = Seq2SeqModel(self.args, use_cuda=use_cuda)
self.model.load_state_dict(checkpoint['model'])
def _model():
with open(FLAGS.write_vocab) as f:
vocab = json.load(f)
with open(FLAGS.write_utts) as f:
utts = json.load(f)
with open(FLAGS.write_lfs) as f:
lfs = json.load(f)
utt_reps = torch.tensor(np.load(FLAGS.write_utt_reps)).to(_device())
word_reps = torch.tensor(np.load(FLAGS.write_word_reps).astype(np.float32)).to(_device())
representer = _sent_representer(vocab)
embedder = _word_representer(vocab)
if FLAGS.model == "sim":
model = SimModel(utt_reps, word_reps, utts, lfs, representer, embedder, _device())
elif FLAGS.model == "seq2seq":
model = Seq2SeqModel(word_reps.shape[2], vocab, utt_reps, utts, lfs, embedder, _device())
else:
assert False
return model