def create_and_train_VAE(
self,
model_id,
num_epochs=30,
batch_size=64,
hidden_dim=10,
latent_dim=50,
is_synthetic=False,
is_take_existing_VAE=False,
teacher_VAE=None,
is_new_categories_to_addded_to_existing_task=False,
is_completely_new_task=False,
epoch_improvement_limit=30,
learning_rate=0.00035,
betas=(0.5, .999),
weight_decay=0.0001,
sample_limit=float('Inf'),
is_save=False,
):
"""Creates and trains a VAE either from scratch, or by applying a previously trained model (and if required, amending the dimensions of fully connected layers). """
self.num_VAE_epochs = num_epochs
# if not using a pre-trained VAE, create new VAE
if not is_take_existing_VAE:
self.VAE_most_recent = CVAE.CVAE(
self.dataset_interface.original_input_dimensions, hidden_dim,
latent_dim, self.num_categories_in_task,
self.dataset_interface.original_channel_number, self.device)
print("create new VAE to train on ", self.num_categories_in_task,
self.categories_list)
# if using a pre-trained VAE, copy 'teacher' VAE and use its weights as a template
else:
self.VAE_most_recent = copy.deepcopy(teacher_VAE)
# if adding a new task category to learn, then need to changed fully connected layer dimensions
if (is_completely_new_task
or is_new_categories_to_addded_to_existing_task):
fc3 = nn.Linear(self.num_categories_in_task, 1000)
fc2 = nn.Linear(self.num_categories_in_task, 1000)
# update CVAE class parameters
self.VAE_most_recent.update_y_layers(fc3, fc2)
self.VAE_most_recent.n_categories = self.num_categories_in_task
self.VAE_most_recent.to(self.device)
# intialise variables before training
patience_counter = 0
best_train_loss = 100000000000
self.VAE_optimizer = torch.optim.Adam(
self.VAE_most_recent.parameters(),
lr=learning_rate,
betas=betas,
weight_decay=weight_decay)
# obtain pytorch dataloaders for training
dataloaders = self.dataset_interface.return_data_loaders(
'VAE', BATCH_SIZE=batch_size)
start_timer = time.time()
# cycle through each epoch
for epoch in range(num_epochs):
# fix epochs
train_loss = self.run_a_VAE_epoch_calculate_loss(
dataloaders['train'], is_train=True, sample_limit=sample_limit)
val_loss = self.run_a_VAE_epoch_calculate_loss(dataloaders['val'],
is_train=False)
# this is required if we reached our training set size threshold. It ensures that avergae calculations are correct
if sample_limit < float('inf'):
train_size = sample_limit
else:
train_size = self.dataset_interface.training_set_size
train_loss /= train_size
val_loss /= self.dataset_interface.val_set_size
time_elapsed = time.time() - start_timer
# if training set metrics has improved, update
if best_train_loss > train_loss:
best_train_loss = train_loss
patience_counter = 1
best_model_wts = copy.deepcopy(
self.VAE_most_recent.state_dict())
print(
f'Epoch {epoch}, Train Loss: {train_loss:.2f}, Val Loss: {val_loss:.2f} ',
time_elapsed, "**** new best ****")
else:
print(
f'Epoch {epoch}, Train Loss: {train_loss:.2f}, Val Loss: {val_loss:.2f} ',
time_elapsed)
patience_counter += 1
# if still haven't seen performance improvement on training set after threshold, cancel training
if patience_counter > epoch_improvement_limit:
break
print('\nTraining complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best train loss: {:4f}'.format(best_train_loss))
# load best trained model parameters
self.VAE_most_recent.load_state_dict(best_model_wts)
# post training we save model and store benchmarking variables
model_string = "VAE " + self.task_name + " epochs" + str(
num_epochs) + ",batch" + str(batch_size) + ",z_d" + str(
latent_dim) + ",synth" + str(is_synthetic) + ",rebuilt" + str(
is_take_existing_VAE
) + ",lr" + str(learning_rate) + ",betas" + str(
betas) + "lowest_error " + str(best_train_loss) + " "
model_string += model_id
self.VAE_most_recent_path = self.initial_directory_path + model_string
self.overall_average_reconstruction_error = best_train_loss
self.VAE_most_recent.cpu()
# reset related task queue variables
self.reset_queue_variables()
if is_save:
torch.save(self.VAE_most_recent, self.VAE_most_recent_path)
self.obtain_VAE_recon_error_mean_and_std_per_class(
self.initial_directory_path + model_string)
# send model back to GPU and delete cache
self.VAE_most_recent.send_all_to_CPU()
torch.cuda.empty_cache()
data_pipeline = data_pipeline("MNIST")
train_xs, train_ys, valid_xs, valid_ys, test_xs, test_ys = data_pipeline.load_preprocess_data(
)
_, height, width, channel = np.shape(train_xs)
n_cls = np.shape(train_ys)[1]
X = tf.placeholder(dtype=tf.float32,
shape=[None, height, width, channel],
name="Input")
X_noised = tf.placeholder(dtype=tf.float32,
shape=[None, height, width, channel],
name="Input_noised")
Y = tf.placeholder(dtype=tf.float32, shape=[None, n_cls], name="labels")
keep_prob = tf.placeholder(dtype=tf.float32, name="drop_rate")
CVAE = CVAE([_, height, width, channel], n_cls, [500, 500, 500, 500],
FLAGS.n_z, keep_prob)
z, output, loss = CVAE.Conditional_Variational_AutoEncoder(
X, X_noised, Y, keep_prob)
latent = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.n_z],
name="latent_input")
global_step = tf.Variable(0, trainable=False)
if FLAGS.PMLR is True: # code for plot manifold learning Results
assert FLAGS.n_z == 2, "n_z should be 2!"
images_manifold = CVAE.conditional_bernoulli_decoder(
latent, Y, keep_prob)
if FLAGS.PARR is True: # code for plot analogical reasoning result
images_PARR = CVAE.conditional_bernoulli_decoder(latent, Y, keep_prob)
if __name__ == '__main__':
# get verbose flag
parser = argparse.ArgumentParser(description='Get verbose flag.')
parser.add_argument('--verbose', action='store_true') # default is False
verbose = parser.parse_args().verbose
# get dataset
ts_start = time.time()
BATCH_SIZE = 100
data_dict = load_MNIST_dataset.load(batch_size=BATCH_SIZE)
ts_end = time.time()
print('Prepare dataset took {:.3f} sec.'.format(ts_end - ts_start))
# setup model and optimizer
ts_start = time.time()
cvae_model = CVAE.CVAE(
latent_dim=50,
input_shape=data_dict['data_shape'],
optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),
)
ts_end = time.time()
print('Prepare model took {:.3f} sec.'.format(ts_end - ts_start))
# test run
total_epoch = 10
print('Start epoch loop ({} epochs in total).'.format(total_epoch))
for epoch in range(1, total_epoch + 1):
# train loop
ts_start = time.time()
train_loss = []
for index, train_x in tqdm.tqdm(
iterable=enumerate(data_dict['train_dataset']),
desc='train',
total=data_dict['train_batch_count'],
results["cossim_w1what1"] = np.zeros(results_shape)
results["cossim_w1what2"] = np.zeros(results_shape)
results["cossim_w2what1"] = np.zeros(results_shape)
results["cossim_w2what2"] = np.zeros(results_shape)
results["cossim_what1what2"] = np.zeros(results_shape)
for i_obj, obj in enumerate(obj_sweep):
for i_lam, lam in enumerate(lambda_sweep):
for i_randseed, randseed in enumerate(randseed_sweep):
print("Objective %d/%d, lambda %d/%d, randseed %d/%d..." % \
(i_obj+1, len(obj_sweep),
i_lam+1, len(lambda_sweep),
i_randseed+1, len(randseed_sweep)))
trial_results = CVAE(steps=steps,
lam_ML=lam,
objective=obj,
decoder_net='nonLinGauss',
classifier_net='hyperplane',
randseed=randseed,
save_output=False,
debug_level=0)
for key in results.keys():
results[key][:, i_obj, i_lam, i_randseed] = trial_results[key]
print("Trial complete.")
print("All trials complete! Saving...")
timestamp = ''.join(re.findall(r'\d+', str(datetime.datetime.now())[:19]))
matfilename = 'results_' + timestamp + '.mat'
sio.savemat(matfilename, {'params': sweep_params, 'data': results})
print('Complete! Saved to ' + matfilename + '.')
TOTAL_EPOCH = args.epoch
LEARNING_RATE = 10**(args.log10_lr)
output_name = args.out
# get dataset
ts_start = time.time()
data_dict = load_MNIST_dataset.load(batch_size=BATCH_SIZE)
ts_end = time.time()
print('Prepare dataset took {:.3f} sec.'.format(ts_end - ts_start),
flush=True)
# setup model and optimizer
ts_start = time.time()
cvae_model = CVAE.CVAE(
latent_dim=LATENT_DIM,
input_shape=data_dict['data_shape'],
optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),
)
ts_end = time.time()
print('Prepare model took {:.3f} sec.'.format(ts_end - ts_start),
flush=True)
# utility functions
def train():
ts_start = time.time()
train_loss = []
for index, train_x in tqdm.tqdm(
iterable=enumerate(data_dict['train_dataset']),
desc='train',
total=data_dict['train_batch_count'],
disable=not VERBOSE,
import numpy as np
import pandas as pd
import yfinance as yf
import pickle
import plotly.graph_objects as go
import CVAE as cvae
encoder, _, _ = cvae.build_cvae()
encoder.load_weights('encoder.h5')
ngb = pickle.load(open('ngboost.pkl', 'rb'))
symbols = pd.read_csv('forex.csv')
ticks = symbols.Symbol.tolist()
def LagguerreRSI(data, gamma):
# Create base data structures
out = []
l = [np.zeros(4), [data.iloc[0]] * 4]
cu = np.zeros(3)
cd = np.zeros(3)
for i in data.index:
# Different calculation for if it is the first price value
l[0][0] = ((1 - gamma) * data[i]) + (gamma * l[1][0])
l[0][1] = ((-gamma) * l[0][0]) + l[1][0] + (gamma * l[1][1])
l[0][2] = ((-gamma) * l[0][1]) + l[1][1] + (gamma * l[1][2])
l[0][3] = ((-gamma) * l[0][2]) + l[1][2] + (gamma * l[1][3])
# Calculate the Count Up and Count Down
cu[0] = l[0][0] - l[0][1] if l[0][0] >= l[0][1] else 0
cu[1] = cu[0] + (l[0][1] - l[0][2]) if l[0][1] >= l[0][2] else cu[0]
cu[2] = cu[1] + (l[0][2] - l[0][3]) if l[0][2] >= l[0][3] else cu[1]