def parameter_count(spec, experiment_directory):
# spec, experiment_directory = args
# Unpack some of the specification information
try:
spec = set_spec_default_values(spec)
algorithm = spec["algorithm"]
batch_size = spec['batch_size']
bptt_len = spec['bptt_len']
spec['device'] = 'cpu'
device = 'cpu'
hmm_hidden = spec['hmm_hidden']
max_step = spec['max_step']
name = spec['name']
sequence_dependence = spec['sequence_dependence']
vocab = spec['vocab']
# Unpack additional arguments <here>
except KeyError:
print("Invalid experiment specification: {}".format(spec))
raise
logging.basicConfig(level=logging.DEBUG)
# filename=J(experiment_directory, 'out.log'),
# filemode='w')
logger = logging.getLogger('exp_runner')
logger.info("Starting the parameter counter!")
logger.info(str(spec))
# Create the directory
if not os.path.exists(experiment_directory):
os.makedirs(experiment_directory)
else:
assert c.EXPERIMENT_RUNNER_SHOULD_OVERWRITE, "Experiment directory {} already exists".format(
experiment_directory)
# Choose sequence model type
if algorithm == 'transformer':
sequence_model = TransformerXL(**spec)
elif algorithm == 'lstm':
sequence_model = LSTMModel(**spec)
elif algorithm == 'cnn':
sequence_model = GatedCNN(**spec)
else:
print(spec)
# Model
model = sequence_model.get_model()
pp = 0
for p in list(model.parameters()):
nn = 1
for s in list(p.size()):
nn = nn * s
pp += nn
print(pp)
np.save(J(experiment_directory, 'parameters.npy'), [pp])
def run(is_on):
print("Start")
wordemb, vecmat = load_vector('new_embdding.txt')
embedding = merge_embdding(wordemb, vecmat)
train_corpus = Corpus('train.csv')
manual_features = syntax_features(train_corpus)
CONF = dict(veclen=len(embedding[0]),
maxlen=64,
word_index=wordemb,
embedding_matrix=embedding,
syntax_features=manual_features)
model = LSTMModel()
model.build_model(CONF, is_on)
model.model.summary()
model.train(train_corpus, CONF, 100, 64)
val_corpus = Corpus('test.csv')
CONF['syntax_features'] = syntax_features(val_corpus)
result = model.predict(val_corpus, CONF, 'train')
print(result)
val_corpus.set_predict(result)
title = ('Anger', 'Anticipation', 'Disgust', 'Fear', 'Joy', 'Sadness',
'Surprise', 'Trust')
val_corpus.set_label_title(title)
print(val_corpus.eval())
print("OK")
class KerasClient(object):
def __init__(self, iden, X, y):
self.iden = iden
self.X = X
self.y = y
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "lstm":
self.model = LSTMModel()
else:
raise ValueError("Model {0} not supported.".format(model_type))
def train(self, weights, config):
logging.info('Training just started.')
assert weights != None, 'weights must not be None.'
self.model.set_weights(weights)
self.model.train(X)
logging.info('Training complete.')
new_weights = self.model.get_weights()
return new_weights, len(X)
def validate(self, t, weights, config):
# TODO: Need to implement Keras validation.
pass
def get_initial_weights(self):
return self.model.get_initial_weights()
def __init__(self, cfg, weights_matrix=None):
super(SentenceEncoder, self).__init__()
vocab_size = cfg.get('vocab_size', {})
pretrained_embeddings = cfg.get('pretrained_embeddings', False)
if pretrained_embeddings:
vocab = pickle.load(open('vocab/words.pkl', 'rb'))
weights_matrix = pretrained_weights_matrix(
vocab, cfg, vocab_size.get('sentences', None))
self.embedding = create_emb_layer(cfg,
vocab_size.get('sentences',
None), weights_matrix)
self.encoder_type = cfg.get('encoder_type', 'lstm')
if self.encoder_type not in ['lstm', 'transformer']:
raise ValueError('Encoder needs to be valid type.')
if self.encoder_type == 'lstm':
self.encoder = LSTMModel(cfg)
elif self.encoder_type == 'transformer':
self.encoder = TransformerModel(cfg)
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
if not cfg.get('use_cuda', True):
self.device = torch.device('cpu')
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "lstm":
self.model = LSTMModel()
else:
raise ValueError("Model {0} not supported.".format(model_type))
def main(model_name="LSTM"):
# choose an initial state
print('Initialization')
x = np.array([3 * np.pi / 7, 3 * np.pi / 4, 0, 0], dtype=np.float32) #x1
t = np.linspace(0, 20, num=301) # num=305??? # t2
x1_analytical = jax.device_get(solve_analytical(x, t))
#cart_coords_ana = get_dynamics_coords(x1_analytical, model_name="Analytical solution")
if model_name == 'LSTM':
model = LSTMModel()
model.load_state_dict(torch.load("./logs/lstm_18_04_2022.14_22.pth"))
model.eval()
# next lines ??????
# test_list=[]
# test_list.append(x_analytical)
# test_set=[]
# for seq in test_list:
# x = seq[[0,1,2,3],:]
# y = seq[4:]
# test_set.append((x,y))
# output = predict_lstm(model, test_set[0][0], 301)
x1_model = predict_lstm(model, x1_analytical[[0, 1, 2, 3], :], 301)
seg = x1_analytical[:4]
x1_model = np.insert(x1_model, 0, seg, axis=0)
elif model_name == 'FC':
model = BaselineNN()
model.load_state_dict(torch.load("./logs/fc_18_04_2022.14_21.pth"))
model.eval()
x1_model = predict_fc(model, x, 301)
elif model_name == "LNN":
print('Making prediction')
#model_name = 'lnn_model_15_05_2022.13_06.pickle' #best
#model_filename = 'lnn_model_17_05_2022.19_53.pickle'
#model_filename = 'lnn_model_18_05_2022.14_10.pickle'
model_filename = 'lnn_model_18_05_2022.14_18.pickle'
with open(f'./logs/{model_filename}', 'rb') as f:
model = pickle.load(f)
x1_model = predict_lnn(model, x, t=t)
elif model_name == "new_LNN":
print('Making prediction')
#model_filename = 'new_lnn_model_17_05_2022.16_45.pickle'
model_filename = 'new_lnn_model_18_05_2022.14_52.pickle'
with open(f'./logs/{model_filename}', 'rb') as f:
model = pickle.load(f)
x1_model = predict_lnn_new(model, x, t=t)
else:
print(f'No realisation for {model_name}')
cart_coords_mod = get_dynamics_coords(x1_model, model_name=model_name)
def run_experiment(spec, experiment_directory):
"""Runs an experiment based on the desired experiment specification.
This process will record the desired response variables and write them to the experiment directory.
Args:
spec (dict): The JSON object specifying the experiment to run.
experiment_directory (str): The directory path to which to write the response variables.
"""
# spec, experiment_directory = args
# Unpack some of the specification information
try:
spec = set_spec_default_values(spec)
algorithm = spec["algorithm"]
batch_size = spec['batch_size']
bptt_len = spec['bptt_len']
device = spec['device']
hmm_hidden = spec['hmm_hidden']
max_step = spec['max_step']
name = spec['name']
sequence_dependence = spec['sequence_dependence']
vocab = spec['vocab']
# Unpack additional arguments <here>
except KeyError:
print("Invalid experiment specification: {}".format(spec))
raise
logging.basicConfig(level=logging.DEBUG)
# filename=J(experiment_directory, 'out.log'),
# filemode='w')
logger = logging.getLogger('exp_runner')
logger.info("Starting the experiment!")
logger.info(str(spec))
# Create the directory
if not os.path.exists(experiment_directory):
os.makedirs(experiment_directory)
else:
assert c.EXPERIMENT_RUNNER_SHOULD_OVERWRITE, "Experiment directory {} already exists".format(
experiment_directory)
# Output a copy of the experiment specification
with open(J(experiment_directory, 'params.json'), 'w') as f:
json.dump(spec, f)
# Choose sequence model type
if algorithm == 'transformer':
sequence_model = TransformerXL(**spec)
elif algorithm == 'lstm':
sequence_model = LSTMModel(**spec)
elif algorithm == 'cnn':
sequence_model = GatedCNN(**spec)
else:
print(spec)
# TODO: loop over trainig files/algorithm specification
ROOT_PATH = 'generated_data'
DATA_FILE = 'V{}_hmm_hidden_{}_lag_{}_vocab_{}.txt'.format(
c.DATA_GENERATION_VERSION, hmm_hidden, sequence_dependence, vocab)
train_file = 'train_' + DATA_FILE
test_file = 'test_' + DATA_FILE
device = torch.device(device)
# Create dataset iterators
train_iter, test_iter = torchtext_batch_iterators(ROOT_PATH,
train_file,
test_file,
batch_size=batch_size,
bptt_len=bptt_len,
device=device,
batch_first=True,
repeat=False)
train_perplex_iter, test_perplex_iter = torchtext_batch_iterators(
ROOT_PATH,
train_file,
test_file,
batch_size=batch_size,
bptt_len=bptt_len,
device=device,
batch_first=True,
repeat=False)
# Model
model = sequence_model.get_model()
optimizer = sequence_model.get_optimizer()
scheduler = sequence_model.get_scheduler()
max_step = spec['max_step']
eval_steps = spec["eval_steps"]
train_step = 0
train_loss = 0
best_val_loss = None
losses = []
test_performance = []
train_performance = []
step_to_performance = []
num_steps = 0
# Training Loop
tqdm_out = TqdmLogger(logger, level=logging.INFO)
progress = tqdm.tqdm(total=max_step, )
try:
for epoch in itertools.count(start=1):
model.train()
mems = tuple()
print()
for train_step, batch in enumerate(train_iter):
num_steps += 1
progress.update()
loss = sequence_model.train_step(batch.text,
batch.target,
mems=mems)
losses.append(loss)
progress.set_description("Loss {:.4f}".format(loss))
# Update scheduler
sequence_model.update_scheduler(num_steps)
if num_steps % 500 == 0:
progress.write("Saving loss performance!")
np.save(J(experiment_directory, 'losses.npy'), losses)
np.save(J(experiment_directory, 'test_performance.npy'),
test_performance)
np.save(J(experiment_directory, 'train_performance.npy'),
train_performance)
np.save(J(experiment_directory, 'step_to_performance.npy'),
step_to_performance)
if num_steps % 1000 == 0:
# Calculate perplexity
progress.write("-" * 100)
progress.write("Model Performance:")
test_performance.append(
evaluate_model(sequence_model, test_perplex_iter, 2000,
vocab))
train_performance.append(
evaluate_model(sequence_model, train_perplex_iter,
1000, vocab))
step_to_performance.append(num_steps)
progress.write(
"Test (Perplex, Accuracy): {:.6f}, {:.6f}".format(
*test_performance[-1]))
progress.write(
"Train (Perplex, Accuracy): {:.6f}, {:.6f}".format(
*train_performance[-1]))
progress.write("Average loss (past 1000): {}".format(
np.mean(losses[-1000:])))
if num_steps >= max_step:
break
if num_steps >= max_step:
progress.write('-' * 100)
progress.write('End of training')
break
# if val_loss is None or val_loss < best_val_loss:
# best_val_loss = val_loss
# # TODO: save the best performing model so far(and its stats)
except KeyboardInterrupt:
logger.info('-' * 100)
logger.info('Exiting from training early')
raise
def run(cls, run_type, dataset, model_type, epochs):
x_train, y_train, x_test, y_test = Processor.get_data(
run_type,
dataset,
)
# reshape input to be [samples, timesteps, features]
x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1])
x_test = x_test.reshape(x_test.shape[0], 1, x_test.shape[1])
if run_type == 1:
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
start = time.time()
if model_type == 1:
model = Conv1DModel.model(
run_type,
dataset,
(x_train.shape[1], x_train.shape[2]),
)
elif model_type == 2:
model = DNNModel.model(
run_type,
dataset,
(x_train.shape[1], x_train.shape[2]),
)
elif model_type == 3:
model = GRUModel.model(
run_type,
dataset,
(x_train.shape[1], x_train.shape[2]),
)
elif model_type == 4:
model = LSTMModel.model(
run_type,
dataset,
(x_train.shape[1], x_train.shape[2]),
)
elif model_type == 5:
model = RNNModel.model(
run_type,
dataset,
(x_train.shape[1], x_train.shape[2]),
)
model.summary()
# optimizer
adam = Adam(lr=0.0005)
if run_type == 0:
model.compile(
optimizer=adam,
loss='binary_crossentropy',
metrics=['accuracy'],
)
else:
model.compile(
optimizer=adam,
loss='categorical_crossentropy',
metrics=['accuracy'],
)
model.fit(
x_train,
y_train,
validation_data=(x_test, y_test),
epochs=epochs,
batch_size=32,
)
# save the model
# model.save(f"model{run_type}.hdf5")
print("--- %s seconds ---" % (time.time() - start))
loss, accuracy = model.evaluate(x_test, y_test, batch_size=32)
print("\nLoss: %.2f, Accuracy: %.2f%%" % (loss, accuracy * 100))
y_pred = model.predict_classes(x_test)
print("\nAnomaly in Test: ", np.count_nonzero(y_test, axis=0))
print("\nAnomaly in Prediction: ", np.count_nonzero(y_pred, axis=0))
if run_type == 0:
print('Confusion Matrix')
print(confusion_matrix(y_test, y_pred))
print('Classification Report')
print(classification_report(y_test, y_pred))
def train(dataset,
test_set,
num_epochs=10,
lr=0.01,
batch_size=256,
log_dir='./logs'):
"""
"""
run_name = datetime.now().strftime("%d_%m_%Y.%H_%M")
model = LSTMModel()
trainDataLoader = DataLoader(dataset, batch_size=batch_size)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
train_losses, valid_losses, train_losses_final = [], [], []
print('Training Start')
for epoch in tqdm(range(num_epochs)):
model.train()
running_loss = 0.0
train_loss = 0.0
num_batches = len(trainDataLoader)
for i, data in enumerate(trainDataLoader, 0):
inputs, labels = data
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# print statistics
running_loss += loss.item()
train_loss += loss.item()
if i % 100 == 99: # print every 100 mini-batches
#print('[%d, %5d] loss: %.6f' %
#(epoch + 1, i + 1, running_loss / 100))
train_losses.append(running_loss)
running_loss = 0.0
train_losses_final.append(train_loss / num_batches)
train_loss = 0.0
#validate
index = 1 #random.randint(1,9)
val_output = torch.from_numpy(
predict_lstm(model, test_set[index][0], 301))
target = torch.from_numpy(test_set[index][1])
valid_losses.append(criterion(val_output, target).item())
print('Finished Training')
if log_dir is not None:
plot_loss(train_losses_final,
valid_losses,
model_name='lstm',
log_dir=log_dir)
torch.save(model.state_dict(),
os.path.join(log_dir, f"lstm_{run_name}.pth"))