def __init__(self, num_iters, compile_flag, interactive, dev_str, f):
# ivy
f = choose_random_framework() if f is None else f
ivy.set_framework(f)
ivy.seed(0)
# device
if dev_str is None:
dev_str = 'gpu:0' if ivy.gpu_is_available() else 'cpu'
self._dev_str = dev_str
# Load input images and poses
this_dir = os.path.dirname(os.path.realpath(__file__))
data = np.load(os.path.join(this_dir, 'nerf_data/tiny_nerf_data.npz'))
images = ivy.array(data['images'], 'float32', dev_str)
inv_ext_mats = ivy.array(data['poses'], 'float32', dev_str)
# intrinsics
focal_lengths = ivy.array(np.tile(np.reshape(data['focal'], (1, 1)), [100, 2]), 'float32', dev_str)
self._img_dims = images.shape[1:3]
pp_offsets = ivy.tile(ivy.array([[dim/2 - 0.5 for dim in self._img_dims]], dev_str=dev_str), [100, 1])
# train data
self._images = images[:100, ..., :3]
self._intrinsics = ivy_vision.focal_lengths_and_pp_offsets_to_intrinsics_object(
focal_lengths, pp_offsets, self._img_dims)
self._cam_geoms = ivy_vision.inv_ext_mat_and_intrinsics_to_cam_geometry_object(
inv_ext_mats[:100, 0:3], self._intrinsics)
# test data
self._test_img = images[101]
self._test_cam_geom = ivy_vision.inv_ext_mat_and_intrinsics_to_cam_geometry_object(
inv_ext_mats[101, 0:3], self._intrinsics.slice(0))
# train config
self._embed_length = 6
self._lr = 5e-4
self._num_samples = 64
self._num_iters = num_iters
# log config
self._interactive = interactive
self._log_freq = 1
self._vis_freq = 25 if self._interactive else -1
self._vis_log_dir = 'nerf_renderings'
if os.path.exists(self._vis_log_dir):
shutil.rmtree(self._vis_log_dir)
os.makedirs(self._vis_log_dir)
# model
self._model = Model(4, 256, self._embed_length, dev_str)
# compile
if compile_flag:
rays_o, rays_d = self._get_rays(self._cam_geoms.slice(0))
target = self._images[0]
self._loss_fn = ivy.compile_fn(self._loss_fn, False,
example_inputs=[self._model, rays_o, rays_d, target, self._model.v])
def test_jit(self, dev_str, dtype_str, call):
op = top_hat
if call in [helpers.jnp_call, helpers.torch_call]:
# compiled jax tensors do not have device_buffer attribute, preventing device info retrieval,
# pytorch scripting does not support .type() casting, nor Union or Numbers for type hinting
pytest.skip()
op_compiled = ivy.compile_fn(op)
input_ = ivy.cast(ivy.random_uniform(shape=(1, 2, 7, 7), dev_str=dev_str), dtype_str)
kernel = ivy.ones((3, 3), dev_str=dev_str, dtype_str=dtype_str)
actual = call(op_compiled, input_, kernel)
expected = call(op, input_, kernel)
assert np.allclose(actual, expected)
def __init__(self, lr, dev_str, compile_step=False):
"""
Construct an general Optimizer. This is an abstract class, and must be derived.
:param lr: Learning rate.
:type lr: float
:param compile_step: Whether to compile the optimizer step, default is False.
:type compile_step: bool, option
"""
self._lr = lr
self._compile_step = compile_step
if compile_step:
self._step_fn = ivy.compile_fn(self._step)
else:
self._step_fn = self._step
def main(env_str, steps=100, iters=10000, lr=0.001, seed=0, log_freq=100, vis_freq=1000, visualize=True, f=None):
# config
f = choose_random_framework(excluded=['numpy']) if f is None else f
ivy.seed(seed)
env = getattr(ivy_gym, env_str)()
starting_obs = env.reset()
# policy
in_size = starting_obs.shape[0]
ac_dim = env.action_space.shape[0]
policy = Policy(in_size, ac_dim)
# compile loss function
compiled_loss_fn = ivy.compile_fn(lambda initial_state, pol_vs:
loss_fn(env, initial_state, policy, pol_vs, steps),
False, example_inputs=[env.get_state(), policy.v])
# optimizer
optimizer = ivy.Adam(lr=lr)
# train
scores = []
for iteration in range(iters):
if iteration % vis_freq == 0 and visualize:
obs = env.reset()
env.render()
for _ in range(steps):
ac = policy(obs)
obs, _, _, _ = env.step(ac)
env.render()
env.reset()
if iteration == 0:
print('\nCompiling loss function for {} environment steps... This may take a while...\n'.format(steps))
score = train_step(compiled_loss_fn, optimizer, env.get_state(), policy, f)
if iteration == 0:
print('\nLoss function compiled!\n')
print('iteration {} score {}'.format(iteration, ivy.to_numpy(score).item()))
scores.append(ivy.to_numpy(score)[0])
if len(scores) == log_freq:
print('\nIterations: {} Mean Score: {}\n'.format(iteration + 1, np.mean(scores)))
scores.clear()
def main(batch_size=32,
num_train_steps=31250,
compile_flag=True,
num_bits=8,
seq_len=28,
ctrl_output_size=100,
memory_size=128,
memory_vector_dim=28,
overfit_flag=False,
interactive=True,
f=None):
f = choose_random_framework() if f is None else f
set_framework(f)
# train config
lr = 1e-3 if not overfit_flag else 1e-2
batch_size = batch_size if not overfit_flag else 1
num_train_steps = num_train_steps if not overfit_flag else 150
max_grad_norm = 50
# logging config
vis_freq = 250 if not overfit_flag else 1
# optimizer
optimizer = ivy.Adam(lr=lr)
# ntm
ntm = NTM(input_dim=num_bits + 1,
output_dim=num_bits,
ctrl_output_size=ctrl_output_size,
ctrl_layers=1,
memory_size=memory_size,
memory_vector_dim=memory_vector_dim,
read_head_num=1,
write_head_num=1)
# compile loss fn
total_seq_example = ivy.random_uniform(shape=(batch_size, 2 * seq_len + 1,
num_bits + 1))
target_seq_example = total_seq_example[:, 0:seq_len, :-1]
if compile_flag:
loss_fn_maybe_compiled = ivy.compile_fn(
lambda v, ttl_sq, trgt_sq, sq_ln: loss_fn(ntm, v, ttl_sq, trgt_sq,
sq_ln),
dynamic=False,
example_inputs=[
ntm.v, total_seq_example, target_seq_example, seq_len
])
else:
loss_fn_maybe_compiled = lambda v, ttl_sq, trgt_sq, sq_ln: loss_fn(
ntm, v, ttl_sq, trgt_sq, sq_ln)
# init
input_seq_m1 = ivy.cast(
ivy.random_uniform(0., 1., (batch_size, seq_len, num_bits)) > 0.5,
'float32')
mw = None
vw = None
for i in range(num_train_steps):
# sequence to copy
if not overfit_flag:
input_seq_m1 = ivy.cast(
ivy.random_uniform(0., 1.,
(batch_size, seq_len, num_bits)) > 0.5,
'float32')
target_seq = input_seq_m1
input_seq = ivy.concatenate(
(input_seq_m1, ivy.zeros((batch_size, seq_len, 1))), -1)
eos = ivy.ones((batch_size, 1, num_bits + 1))
output_seq = ivy.zeros_like(input_seq)
total_seq = ivy.concatenate((input_seq, eos, output_seq), -2)
# train step
loss, pred_vals = train_step(loss_fn_maybe_compiled, optimizer, ntm,
total_seq, target_seq, seq_len, mw, vw,
ivy.array(i + 1,
'float32'), max_grad_norm)
# log
print('step: {}, loss: {}'.format(i, ivy.to_numpy(loss).item()))
# visualize
if i % vis_freq == 0:
target_to_vis = (ivy.to_numpy(target_seq[0] * 255)).astype(
np.uint8)
target_to_vis = np.transpose(
cv2.resize(target_to_vis, (560, 160),
interpolation=cv2.INTER_NEAREST), (1, 0))
pred_to_vis = (ivy.to_numpy(pred_vals[0] * 255)).astype(np.uint8)
pred_to_vis = np.transpose(
cv2.resize(pred_to_vis, (560, 160),
interpolation=cv2.INTER_NEAREST), (1, 0))
img_to_vis = np.concatenate((pred_to_vis, target_to_vis), 0)
img_to_vis = cv2.resize(img_to_vis, (1120, 640),
interpolation=cv2.INTER_NEAREST)
img_to_vis[0:60, -200:] = 0
img_to_vis[5:55, -195:-5] = 255
cv2.putText(img_to_vis, 'step {}'.format(i), (935, 42),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
img_to_vis[0:60, 0:200] = 0
img_to_vis[5:55, 5:195] = 255
cv2.putText(img_to_vis, 'prediction', (7, 42),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
img_to_vis[320:380, 0:130] = 0
img_to_vis[325:375, 5:125] = 255
cv2.putText(img_to_vis, 'target', (7, 362),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
if interactive:
cv2.imshow('prediction_and_target', img_to_vis)
if overfit_flag:
cv2.waitKey(1)
else:
cv2.waitKey(100)
cv2.destroyAllWindows()
def assert_compilable(fn):
try:
ivy.compile_fn(fn)
except Exception as e:
raise e