def main():
parser = argparse.ArgumentParser(description="latent space enjoy")
parser.add_argument('--log-dir',
default='',
type=str,
help='directory to load model')
parser.add_argument('-vae',
'--vae-path',
help='Path to saved VAE',
type=str,
default='')
args = parser.parse_args()
vae = VAEController()
vae.load(args.vae_path)
fig_name = "Decoder for the VAE"
# TODO: load data to infer bounds
bound_min = -10
bound_max = 10
create_figure_and_sliders(fig_name, vae.z_size)
should_exit = False
while not should_exit:
# stop if escape is pressed
k = cv2.waitKey(1) & 0xFF
if k == 27:
break
state = []
for i in range(vae.z_size):
state.append(cv2.getTrackbarPos(str(i), 'slider for ' + fig_name))
# Rescale the values to fit the bounds of the representation
state = (np.array(state) / 100) * (bound_max - bound_min) + bound_min
reconstructed_image = vae.decode(state[None])[0]
reconstructed_image = cv2.cvtColor(reconstructed_image,
cv2.COLOR_RGB2BGR)
# stop if user closed a window
if (cv2.getWindowProperty(fig_name, 0) < 0) or (cv2.getWindowProperty(
'slider for ' + fig_name, 0) < 0):
should_exit = True
break
cv2.imshow(fig_name, reconstructed_image)
# gracefully close
cv2.destroyAllWindows()
def load_vae(path=None, z_size=None):
"""
:param path: (str)
:param z_size: (int)
:return: (VAEController)
"""
# z_size will be recovered from saved model
if z_size is None:
assert path is not None
vae = VAEController(z_size=z_size)
if path is not None:
vae.load(path)
print("Dim VAE = {}".format(vae.z_size))
return vae
def load_vae(path=None, z_size=None):
# z_size will be recovered from saved model
if z_size is None:
assert path is not None
vae = VAEController(z_size=z_size)
if path is not None:
if path.endswith('.json'):
vae.load_json(path)
else:
vae.load(path)
print("Dim VAE = {}".format(vae.z_size))
return vae
# split indices into minibatches. minibatchlist is a list of lists; each
# list is the id of the observation preserved through the training
minibatchlist = [
np.array(sorted(indices[start_idx:start_idx + args.batch_size]))
for start_idx in range(0,
len(indices) - args.batch_size + 1, args.batch_size)
]
data_loader = DataLoader(minibatchlist,
images,
n_workers=4,
is_training=True,
infinite_loop=True) # wyb
vae_controller = VAEController(z_size=SIZE_Z)
vae_controller.vae = vae
best_loss = np.inf
save_path = PATH_MODEL
best_model_path = PATH_MODEL_BEST
os.makedirs(os.path.dirname(save_path), exist_ok=True)
for epoch in range(args.n_epochs):
pbar = tqdm(total=len(minibatchlist))
for _, obs, target_obs in data_loader: # wyb
feed = {vae.input_tensor: obs, vae.target_tensor: target_obs}
(train_loss, r_loss, kl_loss, train_step, _) = vae.sess.run(
[vae.loss, vae.r_loss, vae.kl_loss, vae.global_step, vae.train_op],
feed)
pbar.update(1)
pbar.close()
np.random.shuffle(indices)
# split indices into minibatches. minibatchlist is a list of lists; each
# list is the id of the observation preserved through the training
minibatchlist = [
np.array(sorted(indices[start_idx:start_idx + args.batch_size]))
for start_idx in range(0,
len(indices) - args.batch_size + 1, args.batch_size)
]
data_loader = DataLoader(minibatchlist,
images,
n_workers=2,
folder=args.folder)
vae_controller = VAEController(z_size=args.z_size)
vae_controller.vae = vae
for epoch in range(args.n_epochs):
pbar = tqdm(total=len(minibatchlist))
for obs in data_loader:
feed = {vae.input_tensor: obs}
(train_loss, r_loss, kl_loss, train_step, _) = vae.sess.run(
[vae.loss, vae.r_loss, vae.kl_loss, vae.global_step, vae.train_op],
feed)
pbar.update(1)
pbar.close()
print("Epoch {:3}/{}".format(epoch + 1, args.n_epochs))
print("VAE: optimization step", (train_step + 1), train_loss, r_loss,
kl_loss)
import os
from jetracer.nvidia_racecar import NvidiaRacecar
from jetcam.csi_camera import CSICamera
import time
import vae
from vae.controller import VAEController
from stable_baselines import SAC
car = NvidiaRacecar()
throttle = 0.8
#path = "jetcar_weights.pkl"
path = "logs/sac/JetVae-v0_46/JetVae-v0.pkl"
input_array = np.zeros((1, 256))
try:
i = 0
v = VAEController()
v.load("logs/vae-256.pkl")
#model = keras.models.load_model(path)
model = SAC.load(path)
camera = CSICamera(width=112, height=112)
camera.running = True
print("Imported Camera! Ready to Start!")
while True:
print("Starting to read image")
image = camera.value.copy()
print("Image Read!")
#image = cv2.resize(image, (224//2, 224//2))
print(type(image))
tmp = v.encode_from_raw_image(image)
print("Got image")
sys.path.append('/home/frcvision1/Final/My_Environments/')
import Carla_new
env = gym.make('CarlaEnv-0_9_corl-v0')
print('Made env')
PATH_MODEL_VAE = "vae.json"
# Final filename will be PATH_MODEL_DDPG + ".pkl"
PATH_MODEL_DDPG = "ddpg_carla_segmented_corl"
logger.configure(folder='/tmp/ddpg_carla/')
# Initialize VAE model and add it to gym environment.
# VAE does image post processing to latent vector and
# buffers raw image for future optimization.
vae = VAEController()
env.unwrapped.set_vae(vae)
# Run in test mode of trained models exist.
if os.path.exists(PATH_MODEL_DDPG + ".pkl") and \
os.path.exists(PATH_MODEL_VAE):
print("Task: test")
ddpg = DDPG.load(PATH_MODEL_DDPG, env)
vae.load(PATH_MODEL_VAE)
obs = env.reset()
while True:
action, _states = ddpg.predict(obs)
# print(action)
obs, reward, done, info = env.step(action)
import os
from custom_sac import SACWithVAE
from stable_baselines.common.vec_env import VecFrameStack, DummyVecEnv
from gym_airsim.airsim_car_env import AirSimCarEnv
from vae.controller import VAEController
import numpy as np
import cv2
PATH_MODEL_SAC = "sac.zip"
PATH_MODEL_VAE = "vae.json"
vae = VAEController()
airsim_env = lambda: AirSimCarEnv(vae)
env = DummyVecEnv([airsim_env])
env = VecFrameStack(env,4)
# Run in test mode if trained models exist.
if os.path.exists(PATH_MODEL_SAC) and os.path.exists(PATH_MODEL_VAE):
print("Task: test")
sac = SACWithVAE.load(PATH_MODEL_SAC, env)
vae.load(PATH_MODEL_VAE)
obs = env.reset()
while True:
arr = vae.decode(obs[:,:, :512].reshape(1, 512))
arr = np.round(arr).astype(np.uint8)
arr = arr.reshape(80, 160, 3)
# to visualize what car sees
#cv2.imwrite("decoded_img.png", arr)
from vae.controller import VAEController
parser = argparse.ArgumentParser()
parser.add_argument('-f', '--folder', help='Log folder', type=str, default='logs/recorded_data/')
parser.add_argument('-vae', '--vae-path', help='Path to saved VAE', type=str, default='')
parser.add_argument('--n-samples', help='Max number of samples', type=int, default=20)
parser.add_argument('--seed', help='Random generator seed', type=int, default=0)
args = parser.parse_args()
set_global_seeds(args.seed)
if not args.folder.endswith('/'):
args.folder += '/'
vae = VAEController()
vae.load(args.vae_path)
images = [im for im in os.listdir(args.folder) if im.endswith('.png')]
images = np.array(images)
n_samples = len(images)
for i in range(args.n_samples):
# Load test image
image_idx = np.random.randint(n_samples)
image_path = args.folder + images[image_idx]
image = cv2.imread(image_path)
image = cv2.resize(image, (112, 112))
encoded = vae.encode_from_raw_image(image)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-f',
'--folder',
help='Path to a folder containing images for training',
type=str,
default='logs/recorded_data/')
parser.add_argument('--z-size', help='Latent space', type=int, default=512)
parser.add_argument('--seed',
help='Random generator seed',
type=int,
default=0)
parser.add_argument('--n-samples',
help='Max number of samples',
type=int,
default=-1)
parser.add_argument('--batch-size',
help='Batch size',
type=int,
default=64)
parser.add_argument('--learning-rate',
help='Learning rate',
type=float,
default=1e-4)
parser.add_argument('--kl-tolerance',
help='KL tolerance (to cap KL loss)',
type=float,
default=0.5)
parser.add_argument(''
'--beta',
help='Weight for kl loss',
type=float,
default=1.0)
parser.add_argument('--n-epochs',
help='Number of epochs',
type=int,
default=10)
parser.add_argument('--verbose', help='Verbosity', type=int, default=1)
args = parser.parse_args()
set_global_seeds(args.seed)
if not args.folder.endswith('/'):
args.folder += '/'
vae = ConvVAE(z_size=args.z_size,
batch_size=args.batch_size,
learning_rate=args.learning_rate,
kl_tolerance=args.kl_tolerance,
beta=args.beta,
is_training=True,
reuse=False)
images = [im for im in os.listdir(args.folder) if im.endswith('.jpg')]
images = np.array(images)
n_samples = len(images)
if args.n_samples > 0:
n_samples = min(n_samples, args.n_samples)
print("{} images".format(n_samples))
# indices for all time steps where the episode continues
indices = np.arange(n_samples, dtype='int64')
np.random.shuffle(indices)
# split indices into minibatches. minibatchlist is a list of lists; each
# list is the id of the observation preserved through the training
minibatchlist = [
np.array(sorted(indices[start_idx:start_idx + args.batch_size]))
for start_idx in range(0,
len(indices) - args.batch_size +
1, args.batch_size)
]
data_loader = DataLoader(minibatchlist,
images,
n_workers=2,
folder=args.folder)
vae_controller = VAEController(z_size=args.z_size)
vae_controller.vae = vae
for epoch in range(args.n_epochs):
pbar = tqdm(total=len(minibatchlist))
for obs in data_loader:
feed = {vae.input_tensor: obs}
(train_loss, r_loss, kl_loss, train_step, _) = vae.sess.run([
vae.loss, vae.r_loss, vae.kl_loss, vae.global_step,
vae.train_op
], feed)
pbar.update(1)
pbar.close()
print("Epoch {:3}/{}".format(epoch + 1, args.n_epochs))
print("VAE: optimization step", (train_step + 1), train_loss, r_loss,
kl_loss)
# Update params
vae_controller.set_target_params()
# Load test image
if args.verbose >= 1:
image_idx = np.random.randint(n_samples)
image_path = args.folder + images[image_idx]
image = cv2.imread(image_path)
r = ROI
im = image[int(r[1]):int(r[1] + r[3]), int(r[0]):int(r[0] + r[2])]
encoded = vae_controller.encode(im)
reconstructed_image = vae_controller.decode(encoded)[0]
# Plot reconstruction
cv2.imshow("Original", image)
cv2.imshow("Reconstruction", reconstructed_image)
cv2.waitKey(1)
save_path = "logs/vae-{}".format(args.z_size)
os.makedirs(save_path, exist_ok=True)
print("Saving to {}".format(save_path))
vae_controller.set_target_params()
vae_controller.save(save_path)
import sys
sys.path.append(
'/home/frcvision1/Final/My_Environments/Carla2/alta/environment')
sys.path.append('/home/frcvision1/Final/learning-to-drive-in-a-day')
from carla_synchronous.env2 import CarlaEnv
from vae.controller import VAEController
vae = VAEController()
env = CarlaEnv()
env.set_vae(vae)
env.reset()
for i in range(0, 5000):
env.step()