def test_model(self, episode=None): #no explore here
if episode is None:
episode = self.tested_episodes
# Set models to eval mode
self.transition_model.eval()
self.observation_model.eval()
self.reward_model.eval()
self.encoder.eval()
# Initialise parallelised test environments
test_envs = EnvBatcher(ControlSuiteEnv, (self.parms.env_name, self.parms.seed, self.parms.max_episode_length, self.parms.bit_depth), {}, self.parms.test_episodes)
total_steps = self.parms.max_episode_length // test_envs.action_repeat
rewards = np.zeros(self.parms.test_episodes)
real_rew = torch.zeros([total_steps,self.parms.test_episodes])
predicted_rew = torch.zeros([total_steps,self.parms.test_episodes])
with torch.no_grad():
observation, total_rewards, video_frames = test_envs.reset(), np.zeros((self.parms.test_episodes, )), []
belief, posterior_state, action = torch.zeros(self.parms.test_episodes, self.parms.belief_size, device=self.parms.device), torch.zeros(self.parms.test_episodes, self.parms.state_size, device=self.parms.device), torch.zeros(self.parms.test_episodes, self.env.action_size, device=self.parms.device)
tqdm.write("Testing model.")
for t in range(total_steps):
belief, posterior_state, action, next_observation, rewards, done, pred_next_rew = self.update_belief_and_act(test_envs, belief, posterior_state, action, observation.to(device=self.parms.device), list(rewards), self.env.action_range[0], self.env.action_range[1])
total_rewards += rewards.numpy()
real_rew[t] = rewards
predicted_rew[t] = pred_next_rew
observation = self.env.get_original_frame().unsqueeze(dim=0)
video_frames.append(make_grid(torch.cat([observation, self.observation_model(belief, posterior_state).cpu()], dim=3) + 0.5, nrow=5).numpy()) # Decentre
observation = next_observation
if done.sum().item() == self.parms.test_episodes:
break
real_rew = torch.transpose(real_rew, 0, 1)
predicted_rew = torch.transpose(predicted_rew, 0, 1)
#save and plot metrics
self.tested_episodes += 1
self.metrics['test_episodes'].append(episode)
self.metrics['test_rewards'].append(total_rewards.tolist())
lineplot(self.metrics['test_episodes'], self.metrics['test_rewards'], 'test_rewards', self.statistics_path)
write_video(video_frames, 'test_episode_%s' % str(episode), self.video_path) # Lossy compression
# Set models to train mode
self.transition_model.train()
self.observation_model.train()
self.reward_model.train()
self.encoder.train()
# Close test environments
test_envs.close()
return self.metrics
def create_and_dump_plan(self, env, belief, posterior_state, action, observation, reward, min_action=-inf, max_action=inf):
tqdm.write("Dumping plan")
video_frames = []
encoded_obs = self.encoder(observation).unsqueeze(dim=0)
belief, _, _, _, posterior_state, _, _ = self.transition_model(posterior_state, action.unsqueeze(dim=0), belief, encoded_obs)
belief, posterior_state = belief.squeeze(dim=0), posterior_state.squeeze(dim=0) # Remove time dimension from belief/state
next_action,_, beliefs, states, plan = self.planner(belief, posterior_state,False) # Get action from planner(q(s_t|o≤t,a<t), p)
predicted_frames = self.observation_model(beliefs, states).to(device=self.parms.device)
for i in range(self.parms.planning_horizon):
plan[i].clamp_(min=env.action_range[0], max=self.env.action_range[1]) # Clip action range
next_observation, reward, done = env.step(plan[i].cpu())
next_observation = next_observation.squeeze(dim=0)
video_frames.append(make_grid(torch.cat([next_observation, predicted_frames[i]], dim=1) + 0.5, nrow=2).numpy()) # Decentre
write_video(video_frames, 'dump_plan', self.dump_plan_path, dump_frame=True)
transition_model.rnn.memory = None
# Update and plot reward metrics (and write video if applicable) and save metrics
metrics['test_episodes'].append(episode)
metrics['test_rewards'].append(total_rewards.tolist())
lineplot(metrics['test_episodes'], metrics['test_rewards'],
'test_rewards', results_dir)
lineplot(np.asarray(
metrics['steps'])[np.asarray(metrics['test_episodes']) - 1],
metrics['test_rewards'],
'test_rewards_steps',
results_dir,
xaxis='step')
if not args.symbolic_env:
episode_str = str(episode).zfill(len(str(args.episodes)))
write_video(video_frames, 'test_episode_%s' % episode_str,
results_dir) # Lossy compression
save_image(
torch.as_tensor(video_frames[-1]),
os.path.join(results_dir, 'test_episode_%s.png' % episode_str))
torch.save(metrics, os.path.join(results_dir, 'metrics.pth'))
# Set models to train mode
transition_model.train()
observation_model.train()
reward_model.train()
encoder.train()
# Close test environments
test_envs.close()
# Checkpoint models
print("Completed episode {}".format(episode))
and we must update camera attributes."""
current_status, illumination = _check_twilight(illumination)
if current_status == prior_status:
"""Implement camera.wait_recording(1) to wait on the
video encoder for 1 timeout second. Modify timeout
seconds as necessary for your specific application."""
if result:
"""If result, motion detected. Capture start time,
set file name, record magnnitude of motion, and
write the video."""
#print "Motion Detected!"
start = datetime.now()
file_names, motion_magnitude = _get_file_names(
start.isoformat()), magnitude
camera.capture(file_names[1])
write_video(stream, file_names[0])
while result:
"""While motion is detected, wait on the video
encoder for 6 timeout seconds. Modify minimum timeout
seconds as necessary for your specific application.
The more timeout seconds, the more video captured
after motion is detected. If motion continues after
timeout, recording continues."""
camera.wait_recording(6)
"""Motion ended. Capture end time, upload video to s3,
remove video file from system, and save motion data
to db for analysis."""
end = datetime.now()
#print 'Motion Ended'
_upload_and_remove_files(file_names)
save_motion(start, end, motion_magnitude, file_names,
def test(self, episode):
print("Test model")
# Set models to eval mode
self.transition_model.eval()
self.observation_model.eval()
self.reward_model.eval()
self.encoder.eval()
self.algorithms.train_to_eval()
# self.actor_model_g.eval()
# self.value_model_g.eval()
# Initialise parallelised test environments
test_envs = EnvBatcher(
Env, (args.env, args.symbolic_env, args.seed,
args.max_episode_length, args.action_repeat, args.bit_depth),
{}, args.test_episodes)
with torch.no_grad():
observation, total_rewards, video_frames = test_envs.reset(
), np.zeros((args.test_episodes, )), []
belief, posterior_state, action = torch.zeros(
args.test_episodes, args.belief_size,
device=args.device), torch.zeros(
args.test_episodes, args.state_size,
device=args.device), torch.zeros(args.test_episodes,
self.env.action_size,
device=args.device)
pbar = tqdm(range(args.max_episode_length // args.action_repeat))
for t in pbar:
belief, posterior_state, action, next_observation, reward, done = self.update_belief_and_act(
args, test_envs, belief, posterior_state, action,
observation.to(device=args.device))
total_rewards += reward.numpy()
if not args.symbolic_env: # Collect real vs. predicted frames for video
video_frames.append(
make_grid(torch.cat([
observation,
self.observation_model(belief,
posterior_state).cpu()
],
dim=3) + 0.5,
nrow=5).numpy()) # Decentre
observation = next_observation
if done.sum().item() == args.test_episodes:
pbar.close()
break
# Update and plot reward metrics (and write video if applicable) and save metrics
self.metrics['test_episodes'].append(episode)
self.metrics['test_rewards'].append(total_rewards.tolist())
Save_Txt(self.metrics['test_episodes'][-1],
self.metrics['test_rewards'][-1], 'test_rewards',
args.results_dir)
# Save_Txt(np.asarray(metrics['steps'])[np.asarray(metrics['test_episodes']) - 1], metrics['test_rewards'],'test_rewards_steps', results_dir, xaxis='step')
# lineplot(metrics['test_episodes'], metrics['test_rewards'], 'test_rewards', results_dir)
# lineplot(np.asarray(metrics['steps'])[np.asarray(metrics['test_episodes']) - 1], metrics['test_rewards'], 'test_rewards_steps', results_dir, xaxis='step')
if not args.symbolic_env:
episode_str = str(episode).zfill(len(str(args.episodes)))
write_video(video_frames, 'test_episode_%s' % episode_str,
args.results_dir) # Lossy compression
save_image(
torch.as_tensor(video_frames[-1]),
os.path.join(args.results_dir,
'test_episode_%s.png' % episode_str))
torch.save(self.metrics, os.path.join(args.results_dir, 'metrics.pth'))
# Set models to train mode
self.transition_model.train()
self.observation_model.train()
self.reward_model.train()
self.encoder.train()
# self.actor_model_g.train()
# self.value_model_g.train()
self.algorithms.eval_to_train()
# Close test environments
test_envs.close()
def run_trial(test_env, encoder, transition_model, decoder, trpath, args):
observation = test_env.reset().cuda()
total_reward = 0
fr = []
## Replanning Iterations
for ptw in range(2):
## Goal state
goal = observation.clone()
goal[:, :3, :, :] = goal[:, 3:, :, :]
## Encode current state and goal state
encoding = encoder(observation.unsqueeze(1))
encoding = encoding[:, :, :args.hidden_size]
goal_encoding = encoder(goal.unsqueeze(1))
goal_encoding = goal_encoding[:, :, :args.hidden_size]
ph = 15
numsamples = 1000
## CEM Iterations
for itr in range(3):
itrpath = os.path.join(trpath, "iter_" + str(ptw) + "_" + str(itr))
os.makedirs(itrpath, exist_ok=True)
if itr == 0:
## Generate action samples
action_samples = []
for n in range(numsamples):
action_trajs = []
r = np.random.randint(test_env.action_size)
for j in range(ph):
action_trajs.append(test_env.sample_random_action())
action_trajs = torch.stack(action_trajs)
action_samples.append(action_trajs)
action_samples = torch.stack(action_samples).cuda()
else:
sortid = costs.argsort()
actions_sorted = action_samples[sortid]
actions_ranked = actions_sorted[:10]
## Refitting to Best Trajs
mean, std = actions_ranked.mean(0), actions_ranked.std(0)
smp = torch.empty(action_samples.shape).normal_(mean=0,
std=1).cuda()
mean = mean.unsqueeze(0).repeat(numsamples, 1, 1)
std = std.unsqueeze(0).repeat(numsamples, 1, 1)
action_samples = smp * std + mean
curr_states = encoding.repeat(numsamples, 1, 1)
all_states = []
all_states.append(curr_states)
## Forward Predictions and Cost Evaluation
for j in range(ph):
next_states = transition_model(
curr_states, action_samples[:, j].unsqueeze(1))
curr_states = next_states
all_states.append(curr_states)
all_states = torch.stack(all_states)
# all_res = decoder(all_states.squeeze())
costs = ((goal_encoding.repeat(numsamples, ph + 1, 1) -
all_states.squeeze().permute(1, 0, 2))**2).mean(dim=(1,
2))
_, best_traj = costs.min(0)
acts = action_samples[best_traj]
## Step in env
for j in range(ph):
next_observation, reward, done = test_env.step(acts[j].cpu())
reward /= args.action_repeat
total_reward += reward
fr.append(
next_observation[:, :3].squeeze().cpu().detach().numpy() /
255.0)
observation = next_observation.cuda()
write_video(fr, str(total_reward) + 'rollout', trpath)
## Check Success
success = test_env._env.is_goal()
return success
## Logging Models/Images
if (s % args.checkpoint_interval == 0):
newpath = os.path.join(results_dir, str(s))
os.makedirs(newpath, exist_ok=True)
metrics['teststeps'].append(s)
video_frames = []
for p in range(predlen + 1):
video_frames.append(
make_grid(torch.cat([
observations[p, :5, :3, :, :].cpu().detach(),
observations[0, :5, 3:, :, :].cpu().detach(),
target[p, :5, :, :, :].cpu().detach(),
target_preds_logging[p, :5, :, :, :].cpu().detach(),
],
dim=3),
nrow=1).numpy() / 255.0)
write_video(video_frames, 'train_step%s' % s,
newpath) # Lossy compression
lineplot(metrics['trainsteps'][-len(metrics['observation_loss']):],
metrics['observation_loss'], 'observation_loss', results_dir)
torch.save(
{
'transition_model': transition_model.state_dict(),
'decoder': decoder.state_dict(),
'encoder': encoder.state_dict(),
'optimiser': optimiser.state_dict()
}, os.path.join(results_dir, 'models_%d.pth' % s))
def main(data_dir):
"""
Generates videos with X2Face using the same driving video and source
video combinations used with Face2Face.
Args:
data_dir: Base directory of the FaceForensics++ dataset.
"""
face2face_dir = os.path.join(data_dir,
'manipulated_sequences/Face2Face/c0/videos')
orig_dir = os.path.join(data_dir, 'original_sequences/c0/videos')
image_dir = os.path.join(data_dir, 'original_sequences_images',
COMPRESSION_LEVEL, 'images')
output_dir = os.path.join(data_dir, 'manipulated_sequences/X2Face',
COMPRESSION_LEVEL, 'videos')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
model = load_x2face_model()
pairs = get_seq_combos(face2face_dir)
reenact_count = 0
for source_id, driver_id in pairs:
output_path = os.path.join(output_dir,
'{}_{}.mp4'.format(source_id, driver_id))
if os.path.exists(output_path):
# Do not recreate a video if it already exists.
# If the user wants to recreated a video
# the existing video must be deleted first.
continue
print('Computing reenactment for {} onto {}...'.format(
driver_id, source_id))
# Validate that input files exist.
video_path = os.path.join(orig_dir, '{}.mp4'.format(driver_id))
image_path = os.path.join(image_dir, '{}.png'.format(source_id))
if not os.path.isfile(video_path):
print('Failed to find video sequence {}'.format(source_id),
file=stderr)
continue
if not os.path.isfile(image_path):
print('Failed to find image for sequence {}'.format(source_id),
file=stderr)
continue
# Load and crop each frame of the video.
video = cv2.VideoCapture(video_path)
fps = video.get(cv2.CAP_PROP_FPS)
# Load image and crop around a face.
image = cv2.imread(image_path)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
face_locations = face_recognition.face_locations(gray, model='hog')
if len(face_locations) == 0:
print('No face found in image for sequence {}'.format(source_id),
file=stderr)
continue
cropped_src = crop_face(image, face_locations[0], ZOOMOUT_FACTOR)
# Compute reenactment.
frames = reenact(model, video, cropped_src)
video.release()
# Write reenactment to disk.
output_path = os.path.abspath(output_path)
print('Writing video to "{}"'.format(output_path))
try:
write_video(frames, fps, (256, 256), output_path)
except KeyboardInterrupt as e:
# Safely handle premature termination.
# Remove unfinished file.
if os.exists(output_path):
os.remove(output_path)
raise e
reenact_count += 1
if reenact_count == 0:
print('No reenactments were created')
else:
print('{} reenactments created'.format(reenact_count))
break
# Update and plot reward metrics (and write video if applicable) and save metrics
metrics['test_episodes'].append(episode)
metrics['test_rewards'].append(total_rewards.tolist())
lineplot(metrics['test_episodes'], metrics['test_rewards'],
'test_rewards', 'results')
lineplot(np.asarray(
metrics['steps'])[np.asarray(metrics['test_episodes']) - 1],
metrics['test_rewards'],
'test_rewards_steps',
'results',
xaxis='step')
if not args.symbolic_env:
write_video(
video_frames, 'test_episode_%s' %
str(episode).zfill(len(str(args.episodes))), 'results')
torch.save(metrics, os.path.join('results', 'metrics.pth'))
# Set models to train mode
transition_model.train()
observation_model.train()
reward_model.train()
encoder.train()
# Close test environments
test_envs.close()
# Checkpoint models
if episode % args.checkpoint_interval == 0:
torch.save(
{
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
device = torch.device('cuda')
torch.cuda.manual_seed(args.seed)
else:
device = torch.device('cpu')
# Initialise training environment and experience replay memory
env = Env(args.env, args.seed, args.max_episode_length, args.action_repeat)
D = ExperienceReplay(args.experience_size, env.observation_size, env.action_size, device)
# Initialise dataset D with S random seed episodes
for s in range(1, args.seed_episodes + 1):
observation, done, t = env.reset(), False, 0
epdata = []
while not done:
action = env.sample_random_action()
next_observation, reward, done = env.step(action)
D.append(observation, action, reward, done)
epdata.append(next_observation)
observation = next_observation
t += 1
epdata = np.concatenate(epdata)
frames = torch.FloatTensor(epdata[:,:3,:,:]) / 255.
write_video(frames, "Episode"+str(s), logdir)
print(epdata.shape)
goal = np.swapaxes(np.swapaxes(epdata[0,3:], 0, 1), 1, 2)
cv2.imwrite(logdir+"/goal"+str(s)+".jpg", cv2.cvtColor(goal, cv2.COLOR_BGR2RGB))
torch.save(D, os.path.join(results_dir, 'experience.pth'))
env.close()
def main(data_dir):
"""
Generates videos with GANnotation using the same driving video and source
video combinations used with Face2Face.
Args:
data_dir: Base directory of the FaceForensics++ dataset.
"""
face2face_dir = '{}/manipulated_sequences/Face2Face/c0/videos'.format(
data_dir)
orig_dir = '{}/original_sequences/c0/videos'.format(data_dir)
base_dir = '{}/manipulated_sequences/GANnotation'.format(data_dir)
output_enc_dir = '{}/encodings'.format(base_dir)
output_vid_dir = '{}/{}/videos'.format(base_dir, COMPRESSION_LEVEL)
pairs = get_seq_combos(face2face_dir)
# Compute all video encodings and save them to disk.
# We precompute these because they take roughly 10 times as long to compute
# as the reenactments, and we may want to recompute the reenactments with
# different images later.
print('Computing video encodings...')
if not os.path.exists(output_enc_dir):
os.makedirs(output_enc_dir)
enc_count = 0
for source_id, _ in pairs:
encoding_path = get_encoding_path(output_enc_dir, source_id)
if os.path.exists(encoding_path):
continue # Encoding already calculated for this video sequence.
print('Computing encoding for sequence {}...'.format(source_id))
video_path = '{}/{}.mp4'.format(orig_dir, source_id)
cap = cv2.VideoCapture(video_path)
points = compute_video_encoding(cap)
cap.release()
try:
np.savetxt(encoding_path, points.reshape((132, -1)).transpose())
except KeyboardInterrupt as e:
# Safely handle premature termination.
# Remove unfinished file.
if os.exists(encoding_path):
os.remove(encoding_path)
raise e
enc_count += 1
if enc_count == 0:
print('No encodings were calculated')
else:
print('{} video sequences encoded'.format(enc_count))
print()
print('Computing reenactments...')
# Load pre-trained model.
gann_path = os.path.join(dirname, 'models/myGEN.pth')
my_gann = GANnotation.GANnotation(path_to_model=gann_path)
image_dir = '{}/original_sequences_images/{}/images'.format(
data_dir, COMPRESSION_LEVEL)
if not os.path.exists(output_vid_dir):
os.makedirs(output_vid_dir)
reenact_count = 0
for source_id, driver_id in pairs:
output_path = '{}/{}_{}.mp4'.format(output_vid_dir, source_id,
driver_id)
if os.path.exists(output_path):
# Do not recreate a video if it already exists.
# If the user wants to recreated a video
# the existing video must be deleted first.
continue
print('Computing reenactment for {} onto {}...'.format(
driver_id, source_id))
# Validate that input files exist.
encoding_path = get_encoding_path(output_enc_dir, driver_id)
if not os.path.isfile(encoding_path):
print('Failed to find encoding for video sequence {}'.format(
driver_id),
file=stderr)
continue
image_path = '{}/{}.png'.format(image_dir, source_id)
if not os.path.isfile(image_path):
print('Failed to find image for sequence {}'.format(source_id),
file=stderr)
continue
points = np.loadtxt(encoding_path).transpose().reshape(66, 2, -1)
# Load and transform image for inputting.
image = cv2.imread(image_path)
cropped = get_gann_cropped_face(image)
# Compute reenactment.
frames, _ = my_gann.reenactment(cropped, points)
output_path = os.path.abspath(output_path)
print('Writing video to "{}"'.format(output_path))
try:
write_video(frames, FPS, (128, 128), output_path)
except KeyboardInterrupt as e:
# Safely handle premature termination.
# Remove unfinished file.
if os.exists(output_path):
os.remove(output_path)
raise e
reenact_count += 1
if reenact_count == 0:
print('No reenactments were created')
else:
print('{} reenactments created'.format(reenact_count))
