def __init__(self, args):
self.log_path = args.log_path
self.device = torch.device("cuda:0" if args.cuda else "cpu")
self.img_size = args.img_size
self.sample_num = args.sample_num
self.transform = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize([64, 64], 1),
transforms.ToTensor(),
transforms.Normalize([.5], [.5])
])
self.pil_transform = transforms.ToPILImage(mode="RGB")
self.norm_scale = np.loadtxt(os.path.join(args.config_path,
"norm_scale.txt"),
dtype=np.float32,
delimiter=",")[None]
self.norm_min = np.loadtxt(os.path.join(args.config_path,
"norm_min.txt"),
dtype=np.float32,
delimiter=",")[None]
self.pb_list = torch.from_numpy(
np.loadtxt(os.path.join(args.config_path, "pb_list.txt"),
dtype=np.float32,
delimiter=","))
self.kmeans = KMeans(n_clusters=2)
self.kmeans.fit(self.pb_list)
print("=" * 5, "Init LSTMPB", "=" * 5)
self.rnn = LSTMPB(args, pb_unit=self.pb_list[5][None])
pt_file = load_model(args.model_load_path, "*/*LSTMPB*.pt")
self.rnn.load_state_dict(torch.load(pt_file))
print("=" * 5, "Init VAE", "=" * 5)
self.vae = VAE(img_size=args.img_size, z_dim=args.vae_z_dims)
pt_file = load_model(args.model_load_path, "*/VAE*.pt")
self.vae.load_state_dict(torch.load(pt_file))
self.vae.eval()
print("=" * 5, "Init CVAE", "=" * 5)
self.cvae = CVAE(img_size=args.img_size, z_dim=args.cvae_z_dims)
pt_file = load_model(args.model_load_path, "*/*CVAE*.pt")
self.cvae.load_state_dict(torch.load(pt_file))
self.cvae.eval()
self.norm_mode = {
"joint": [0, 1, 2, 3, 4],
"visual": [5, 6, 7, 8, 9, 10, 11]
}
self.norm_mode[
"all"] = self.norm_mode["joint"] + self.norm_mode["visual"]
self.global_step = 0
self.his_log = HistoryWindow(maxlen=args.window_size)
#visualize current goal
_, goal = self.vae.decoder(self.denorm(self.goal, "visual"))
goal = ((goal[0] * .5 + .5) * 255).to(torch.int8)
self.goal_img = self.pil_transform(goal)
def train_conditional(data, optimizer):
(X_train, y_train), (X_test, y_test) = data
lb = LabelBinarizer()
y_train = lb.fit_transform(y_train)
y_test = lb.transform(y_test)
fname = os.path.join(data_dir, 'cvae_train.csv')
logger = tf.keras.callbacks.CSVLogger(filename=fname)
encoder, decoder, cvae = CVAE(input_dim=X_train.shape[-1],
latent_dim=latent_dim,
aux_dim=y_train.shape[-1],
beta=beta,
output_activation='sigmoid').build()
print(cvae.summary())
cvae.compile(optimizer, loss=None)
cvae.fit(x=[X_train, y_train],
y=None,
validation_data=([X_test, y_test], None),
epochs=epochs,
batch_size=bs,
shuffle=True,
verbose=2,
callbacks=[logger])
print('\nfinished training, saving models...')
cvae.save(os.path.join(model_dir, 'cvae.h5'))
encoder.save(os.path.join(model_dir, 'conditional_encoder.h5'))
decoder.save(os.path.join(model_dir, 'conditional_decoder.h5'))
def __init__(self, env, silent=False):
super().__init__(env)
from vae import CVAE
self.vae = CVAE()
self.vae.set_weights(np.load("vae_weights.npy", allow_pickle=True))
self.observation_space = Box(low=float("-inf"),
high=float("inf"),
shape=(40, ))
self.silent = silent
def main(args):
transform = transforms.Compose([
transforms.Resize([64, 64], 1),
transforms.ToTensor(),
transforms.Normalize([.5], [.5])
])
dataset = datasets.ImageFolder(root=args.data_path, transform=transform)
data = []
for i, j in dataset:
data.append(i)
data = torch.stack(data)
cvae = CVAE(img_size=data.shape[1:], z_dim=args.z_dim)
cvae.eval()
pt_files = glob.glob(os.path.join(args.model_load_path, "*.pt"))
pt_files.sort()
#data_1 = data[0][None, :, :, :].repeat([20, 1, 1, 1])
#data_2 = data[1][None, :, :, :].repeat([20, 1, 1, 1])
for i in range(len(pt_files)):
print(pt_files[i])
cvae.load_state_dict(torch.load(pt_files[i]))
#z_data = [torch.randn(data.shape[0], args.z_dim), data]
z_data = [
torch.randn(32, args.z_dim), data[None, 0].repeat([32, 1, 1, 1])
]
_, rec_img = cvae.decoder(*z_data)
grid_img = make_result_img(
[data[None, 0].repeat([32, 1, 1, 1]), rec_img],
normalize=True,
range=(-1., 1.))
utils.save_image(
grid_img, "{}CVAE_gen_result_{:0>2d}.png".format(args.log_path, i))
def model(tree_def, cond_tree_def, activation):
return CVAE(
FLAGS.embedding_size,
det_encoder=Encoder(
tree_def=tree_def,
embedding_size=FLAGS.embedding_size,
cut_arity=FLAGS.cut_arity,
max_arity=FLAGS.max_arity,
variable_arity_strategy=FLAGS.enc_variable_arity_strategy,
cellsbuilder=EncoderCellsBuilder(
EncoderCellsBuilder.simple_cell_builder(
hidden_coef=FLAGS.hidden_cell_coef,
activation=activation,
gate=FLAGS.encoder_gate),
EncoderCellsBuilder.simple_dense_embedder_builder(
activation=activation),
EncoderCellsBuilder.simple_categorical_merger_builder(
hidden_coef=FLAGS.hidden_cell_coef, activation=activation),
),
name='encoder'),
det_decoder=Decoder(
tree_def=tree_def,
embedding_size=FLAGS.embedding_size,
max_node_count=FLAGS.max_node_count,
max_depth=FLAGS.max_depth,
max_arity=FLAGS.max_arity,
cut_arity=FLAGS.cut_arity,
cellbuilder=DecoderCellsBuilder(
DecoderCellsBuilder.simple_distrib_cell_builder(
FLAGS.hidden_cell_coef, activation=activation),
DecoderCellsBuilder.simple_1ofk_value_inflater_builder(
0.5, activation=tf.tanh),
DecoderCellsBuilder.simple_node_inflater_builder(
FLAGS.hidden_cell_coef,
activation=activation,
gate=FLAGS.decoder_gate)),
variable_arity_strategy=FLAGS.dec_variable_arity_strategy,
attention=False),
cond_encoder=Encoder(
tree_def=cond_tree_def,
embedding_size=FLAGS.embedding_size,
cut_arity=FLAGS.cut_arity,
cellsbuilder=EncoderCellsBuilder(
EncoderCellsBuilder.simple_cell_builder(
hidden_coef=FLAGS.hidden_cell_coef,
activation=activation,
gate=FLAGS.encoder_gate),
EncoderCellsBuilder.simple_categorical_embedder_builder(
FLAGS.hidden_cell_coef, activation=activation),
EncoderCellsBuilder.simple_categorical_merger_builder(
hidden_coef=FLAGS.hidden_cell_coef,
activation=activation)),
max_arity=FLAGS.max_arity,
name="condition_encoder",
variable_arity_strategy=FLAGS.enc_variable_arity_strategy))
def main(args):
transform = transforms.Compose([
transforms.Resize([64, 64], 1),
transforms.ToTensor(),
transforms.Normalize([.5], [.5])
])
dataset = datasets.ImageFolder(root=args.data_path, transform=transform)
data = []
for i, j in dataset:
data.append(i)
data = torch.stack(data)
cvae = CVAE(img_size=data.shape[1:], z_dim=args.z_dim)
pt_file = load_model(args.model_load_path, "*10.pt")
cvae.load_state_dict(torch.load(pt_file))
cvae.eval()
if args.decode:
#z_data = [torch.randn(data.shape[0], args.z_dim), data]
#_, rec_img = cvae.decoder(*z_data)
#grid_img = make_result_img([data, rec_img], normalize=True, range=(-1., 1.))
#utils.save_image(grid_img, "{}CVAE_gen_result.png".format(args.log_path))
cond_img = data[None, 0].repeat([32, 1, 1, 1])
z_data = [torch.randn(cond_img.shape[0], args.z_dim), cond_img]
_, rec_img = cvae.decoder(*z_data)
#grid_img = make_result_img([rec_img], normalize=True, range=(-1., 1.))
#utils.save_image(grid_img, "{}CVAE_gen_result_same_cond.png".format(args.log_path))
for i in range(rec_img.shape[0]):
utils.save_image(rec_img[i],
"{}goal{:0>6d}.png".format(args.log_path, i),
normalize=True,
range=(-1., 1.))
if args.gen_seq:
for i, d in enumerate(data):
cond_img = data[None, i]
z_data = [torch.randn(1, args.z_dim), cond_img]
_, rec_img = cvae.decoder(*z_data)
grid_img = make_result_img([cond_img, rec_img],
normalize=True,
range=(-1., 1.))
utils.save_image(
grid_img, "{}res_state-goal/CVAE_gen_{:0>6d}.png".format(
args.log_path, i))
utils.save_image(rec_img,
"{}res_goal/CVAE_gen_{:0>6d}.png".format(
args.log_path, i),
normalize=True,
range=(-1., 1.))
test_size=validation_set_percent,
stratify=X_autoencoder_tumor_type,
random_state=42)
# Order the features correctly before training
X_train = X_train.reindex(sorted(X_train.columns), axis="columns")
X_val = X_val.reindex(sorted(X_val.columns), axis="columns")
#Train the Model
cvae = CVAE(original_dim=X_autoencoder_train.shape[1],
intermediate_dim=hidden_dim,
latent_dim=latent_dim,
cond_dim=32,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
dropout_rate_input=dropout_input,
dropout_rate_hidden=dropout_hidden,
dropout_decoder=dropout_decoder,
freeze_weights=freeze_weights,
classifier_use_z=classifier_use_z,
rec_loss=reconstruction_loss)
cvae.initialize_model()
cvae.train_vae(
train_df=X_autoencoder_train,
train_cond_df=pd.get_dummies(X_autoencoder_tumor_type_train),
val_df=X_autoencoder_val,
val_cond_df=pd.get_dummies(X_autoencoder_tumor_type_val))
# Build and train stacked classifier
class NNModel(object):
def __init__(self, args):
self.log_path = args.log_path
self.device = torch.device("cuda:0" if args.cuda else "cpu")
self.img_size = args.img_size
self.sample_num = args.sample_num
self.transform = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize([64, 64], 1),
transforms.ToTensor(),
transforms.Normalize([.5], [.5])
])
self.pil_transform = transforms.ToPILImage(mode="RGB")
self.norm_scale = np.loadtxt(os.path.join(args.config_path,
"norm_scale.txt"),
dtype=np.float32,
delimiter=",")[None]
self.norm_min = np.loadtxt(os.path.join(args.config_path,
"norm_min.txt"),
dtype=np.float32,
delimiter=",")[None]
self.pb_list = torch.from_numpy(
np.loadtxt(os.path.join(args.config_path, "pb_list.txt"),
dtype=np.float32,
delimiter=","))
self.kmeans = KMeans(n_clusters=2)
self.kmeans.fit(self.pb_list)
print("=" * 5, "Init LSTMPB", "=" * 5)
self.rnn = LSTMPB(args, pb_unit=self.pb_list[5][None])
pt_file = load_model(args.model_load_path, "*/*LSTMPB*.pt")
self.rnn.load_state_dict(torch.load(pt_file))
print("=" * 5, "Init VAE", "=" * 5)
self.vae = VAE(img_size=args.img_size, z_dim=args.vae_z_dims)
pt_file = load_model(args.model_load_path, "*/VAE*.pt")
self.vae.load_state_dict(torch.load(pt_file))
self.vae.eval()
print("=" * 5, "Init CVAE", "=" * 5)
self.cvae = CVAE(img_size=args.img_size, z_dim=args.cvae_z_dims)
pt_file = load_model(args.model_load_path, "*/*CVAE*.pt")
self.cvae.load_state_dict(torch.load(pt_file))
self.cvae.eval()
self.norm_mode = {
"joint": [0, 1, 2, 3, 4],
"visual": [5, 6, 7, 8, 9, 10, 11]
}
self.norm_mode[
"all"] = self.norm_mode["joint"] + self.norm_mode["visual"]
self.global_step = 0
self.his_log = HistoryWindow(maxlen=args.window_size)
#visualize current goal
_, goal = self.vae.decoder(self.denorm(self.goal, "visual"))
goal = ((goal[0] * .5 + .5) * 255).to(torch.int8)
self.goal_img = self.pil_transform(goal)
def on_predict(self, cur_joint, cur_img, state=None):
cur_joint = torch.Tensor(cur_joint)[None]
cur_img = self.transform(cur_img[:, :, ::-1])[None]
utils.save_image(cur_img[0],
"./result/visual_{:0>6d}.png".format(
self.global_step),
normalize=True,
range=(-1, 1))
img_feature = self.vae.reparam(*self.vae.encoder(cur_img))
inputs = torch.cat([cur_joint, img_feature], axis=-1).detach()
inputs = self.norm(inputs).to(torch.float32)
outputs, state = self.rnn.step(inputs, state)
outputs, state = outputs.detach().cpu(), \
(state[0].detach().cpu(), state[1].detach().cpu())
self.global_step += 1
return outputs, state, self.denorm(outputs).to(torch.float32)
def off_predict(self, cur_joint, img_feature, state=None):
assert isinstance(cur_joint, (list, np.ndarray))
assert isinstance(img_feature, (list, np.ndarray))
cur_joint = torch.Tensor(cur_joint).to(torch.float32)[None]
img_feature = torch.Tensor(img_feature).to(torch.float32)[None]
inputs = torch.cat([cur_joint, img_feature], axis=-1)
outputs, state = self.rnn.step(inputs, state)
outputs, state = outputs.detach().cpu(), \
(state[0].detach().cpu(), state[1].detach().cpu())
self.his_log.put([outputs, inputs, state])
return outputs, state, self.denorm(outputs).to(torch.float32)
def gen_goal(self, visual_img):
visual_img = self.transform(visual_img)[None].repeat(
self.sample_num, 1, 1, 1)
sampled_z = torch.randn(self.sample_num, self.cvae.z_dim)
_, gen_goals = self.cvae.decoder(z=sampled_z, cond=visual_img)
pb_list = self.vae.reparam(*self.vae.encoder(gen_goals)).detach().cpu()
#for i in range(gen_goals.shape[0]):
# utils.save_image(gen_goals[i], "{}gen_goal{:0>6d}.png".format("./", i),normalize=True, range=(-1., 1.))
pb_label = self.kmeans.predict(pb_list.numpy())
print(pb_label)
pb_list = torch.stack(
[pb_list[pb_label == 0].mean(0), pb_list[pb_label == 1].mean(0)])
_, goal_list = self.vae.decoder(pb_list)
pb_list = self.norm(pb_list, "visual")
goal_list = ((goal_list * .5 + .5) * 255).to(torch.int8)
goal_list = [self.pil_transform(goal) for goal in goal_list]
return goal_list, pb_list
def pem(self):
assert len(self.his_log), "the history window is empty!"
for param in self.rnn.parameters():
param.requires_grad = False
self.rnn.pb_unit = nn.Parameter(self.rnn.pb_unit, requires_grad=True)
optim_param = [
param for param in self.rnn.parameters()
if param.requires_grad == True
]
optim = torch.optim.Adam(optim_param, lr=0.01)
mse_loss = nn.MSELoss()
pred_his, actual_his, state_his = self.his_log.get()
pb_log = []
for i in range(80):
log = []
cur_input = torch.cat([pred_his[:, 0, :5], actual_his[:, 0, 5:]],
dim=-1)
state = state_his[0]
for step in range(1, len(state_his)):
cur_input, state = self.rnn.step(cur_input, state)
log.append(cur_input)
log = torch.stack(log, dim=1)
loss = mse_loss(log[0, :, 5:], actual_his[0, 1:, 5:]) + \
(self.rnn.pb_unit - self.pb_list).pow(2).mean()
pb_log.append(self.rnn.pb_unit.data.clone())
loss.backward()
optim.step()
print("PEM loss, step {}, loss: {}".format(i, loss.item()))
@property
def goal(self):
return self.rnn.pb_unit
@goal.setter
def goal(self, pb):
if pb.ndim == 1:
pb = torch.unsqueeze(pb, 0)
self.rnn.pb_unit = pb
def norm(self, inputs, mode="all"):
assert mode in ["joint", "visual", "all"]
i_slice = self.norm_mode[mode]
return inputs * self.norm_scale[:, i_slice] + self.norm_min[:, i_slice]
def denorm(self, outputs, mode="all"):
assert mode in ["joint", "visual", "all"]
i_slice = self.norm_mode[mode]
return (outputs - self.norm_min[:, i_slice]) / self.norm_scale[:,
i_slice]
train_unlabeled = pickle.load(open("train_unlabeled.p", "rb"))
train_unlabeled.train_labels = torch.ones([47000])
train_unlabeled.k = 47000
val_data = pickle.load(open("validation.p", "rb"))
#test_data = pickle.load(open("test.p","rb"))
train_loader_unlabeled = torch.utils.data.DataLoader(train_unlabeled,
batch_size=64,
shuffle=True)
train_loader_labeled = torch.utils.data.DataLoader(train_labeled,
batch_size=64,
shuffle=True)
val_loader = torch.utils.data.DataLoader(val_data, batch_size=64, shuffle=True)
#test_loader = torch.utils.data.DataLoader(test_data, batch_size=64, shuffle=True)
vae = CVAE()
model = CNN()
#----------------------------------------
#Unsupevised VAE training
#----------------------------------------
opt = optim.Adam(vae.parameters(), lr=0.001)
mse = nn.MSELoss()
bce = nn.BCELoss()
def test_vae():
vae.eval()
test_loss = 0
class VaeCarWrapper(gym.ObservationWrapper):
def __init__(self, env, silent=False):
super().__init__(env)
from vae import CVAE
self.vae = CVAE()
self.vae.set_weights(np.load("vae_weights.npy", allow_pickle=True))
self.observation_space = Box(low=float("-inf"),
high=float("inf"),
shape=(40, ))
self.silent = silent
def _process_frame(self, frame):
obs = (frame[0:84, :, :] * 255).astype(np.uint8)
obs = Image.fromarray(obs, mode="RGB").resize((64, 64))
obs = np.array(obs)
return np.array(self.vae.encode(obs.reshape(1, 64, 64, 3) / 255)[0])
def observation(self, frame):
# far-front spike
car_body = np.sum((frame[56:59, 47, 1] > 0.5).flatten())
# main headlights
car_body = np.sum((frame[59:74, 46:49, 1] > 0.5).flatten())
# rear wheels
car_body += np.sum((frame[72:76, 44, 1] > 0.5).flatten())
car_body += np.sum((frame[72:76, 50, 1] > 0.5).flatten())
#sides
car_body += np.sum((frame[67:77, 45, 1] > 0.5).flatten())
car_body += np.sum((frame[67:77, 49, 1] > 0.5).flatten())
self.green = car_body / 55.0
self.speed = sum(frame[85:, 2, 0]) / 5
self.abs1 = sum(frame[85:, 9, 2])
self.abs2 = sum(frame[85:, 14, 2])
self.abs3 = sum(frame[85:, 19, 2])
self.abs4 = sum(frame[85:, 24, 2])
steering_input_left = sum(frame[90, 37:48, 1])
steering_input_right = sum(frame[90, 47:58, 1])
self.steering = steering_input_right - steering_input_left
rotation_left = sum(frame[90, 59:72, 0])
rotation_right = sum(frame[90, 72:85, 0])
self.rotation = rotation_right - rotation_left
if not self.silent:
print(
f"green:{self.green}\tspeed:{self.speed}\tabs:\t{self.abs1}\t{self.abs2}\t{self.abs3}\t{self.abs4}\tsteering:{self.steering}\trotation:{self.rotation}"
)
features = self._process_frame(frame)
return np.concatenate([
features,
[
self.speed, self.green, self.abs1, self.abs2, self.abs3,
self.abs4, self.steering, self.rotation
]
])
print('[!] Error: Unable to continue - missing file ' + WEIGHTS)
print(
'[*] To generate the missing file, please train the model by running vae.py'
)
exit()
# Encoding the entire dataset, if not already encoded
if not isfile(ENCODED_DATASET):
print(
'[!] Warning: Missing file {}. Generating...'.format(ENCODED_DATASET))
dataset = glob(PATH)
if not dataset:
print('[!] Error: Unable to encode dataset - missing files at ' + PATH)
exit()
else:
net = CVAE()
net.vae.load_weights(WEIGHTS)
encode(ENCODED_DATASET, dataset, net)
# Generating the Eigen values and vectors, if they do not already exist.
if not isfile(EIGENVALUES) or not isfile(EIGENVECTORS):
print('[!] Warning: Missing files {},{}. Generating...'.format(
EIGENVALUES, EIGENVECTORS))
pca(ENCODED_DATASET)
# Searching for the optional cascade files
cascades = [EYE_CASCADE, MOUTH_CASCADE, FACE_CASCADE]
DEMO_FLAG = True
if not all([isfile(x) for x in cascades]):
print('[!] Warning: Missing the following files:')
print(*cascades, sep=', ')
import tensorflow as tf
import numpy as np
from vae import CVAE
from data.circle_generator import gen_circles
model = CVAE(2)
model.load_weights('../saved_models/vae/vae_weights')
numTests = 10
size = 28
dataset = tf.data.Dataset.from_generator(
gen_circles,
args=[3, size],
output_types=(tf.float32),
output_shapes=((size, size, 1))
)
for img in dataset:
@tf.function
def compute_apply_gradients(model, x, optimizer):
with tf.GradientTape() as tape:
loss = compute_loss(model, x)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
epochs = 75
if challenge == 'position':
latent_dim = 2
elif challenge == 'with_size':
latent_dim = 3
model = CVAE(size, latent_dim)
for epoch in range(1, epochs + 1):
start_time = time.time()
for train_x in train_dataset:
compute_apply_gradients(model, train_x, optimizer)
end_time = time.time()
if epoch % 1 == 0:
loss = tf.keras.metrics.Mean()
for test_x in test_dataset:
loss(compute_loss(model, test_x))
elbo = -loss.result()
display.clear_output(wait=False)
print('Epoch: {}, Test set ELBO: {}, '
'time elapse for current epoch {}'.format(