def evaluate(experiment_dir, args):
"""
Evaluate the model stored in the given directory. It loads the latest available checkpoint and iterates over
the test set.
Args:
experiment_dir: The model directory.
args: Commandline arguments.
"""
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9,
allow_growth=True)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
test_model, test_data, config, means, vars, stand = create_and_restore_test_model(
sess, experiment_dir, args)
print("Evaluating test set ...")
eval_result = evaluate_model(sess, test_model, test_data, means, vars,
stand)
if args.export:
# Export the results into a csv file that can be submitted.
fname = os.path.join(
experiment_dir,
"predictions_in{}_out{}.csv".format(config['source_seq_len'],
config['target_seq_len']))
export_results(eval_result, fname)
# Export a zip file containing the code that generated the results.
code_files = glob.glob('./*.py', recursive=False)
export_code(code_files, os.path.join(experiment_dir, 'code.zip'))
if args.visualize:
# Visualize the seed sequence and the prediction for some random samples in the test set.
fk_engine = SMPLForwardKinematics()
visualizer = Visualizer(fk_engine)
n_samples_viz = 10
rng = np.random.RandomState(42)
idxs = rng.randint(0, len(eval_result), size=n_samples_viz)
sample_keys = [list(sorted(eval_result.keys()))[i] for i in idxs]
for k in sample_keys:
visualizer.visualize(eval_result[k][1],
eval_result[k][0],
title=k)
output_file_name = rest[0]
else:
output_file_name = 'output.txt'
algorithms = {'d': dfs, 'b': bfs, 'a': partial(a_star, h=heuristic)}
problem = formulate_maze_problem(input_file_name)
visualizer = Visualizer(problem)
if problem:
os.system('clear')
visualizer.visualize_state(problem.initial_state)
print_possible_actions()
option = input('Enter an option: ').lower()
algorithm = algorithms.get(option)
if not algorithm:
print('Press only the first letter of the algorithm')
sys.exit(1)
os.system('clear')
solution = algorithm(problem)
if solution:
visualizer.visualize(solution)
output = problem.prettify(solution)
with open(output_file_name, 'w') as f:
print(output, file=f)
print(output)
print(f'Output is written to file {output_file_name}!')
else:
print('There is a "problem" with the problem :)')
sys.exit(1)
for model, name in zip(models, model_names):
clf = model
clf.fit(stat_training_attrs, stat_training_values)
stat_pred = clf.predict(stat_training_attrs)
print stat_pred
diff =[]
for trueValue, predValue in zip(stat_testing_values, stat_pred):
diff.append(abs(trueValue - predValue))
print 'Errors for model', name, ':', sum(diff), 'on value set', value_set
# Visualize data
if args.v:
visualizer = Visualizer()
visualizer.visualize(data, EDGE_FUNCS[args.edge], split=args.split,
save=args.save, show=args.show)
if args.p:
# Select prediction method
if args.p == 'kmeans':
print 'Using k-means clustering metric.'
attribute_clusters, attribute_and_friendship_clusters, weighted_attribute_and_friendship_clusters = k_means_clustering(data, featureWeightMap, args.show)
attribute_clusters = _convert_kmeans_format(attribute_clusters)
attribute_and_friendship_clusters = _convert_kmeans_format(attribute_and_friendship_clusters)
weighted_attribute_and_friendship_clusters = _convert_kmeans_format(weighted_attribute_and_friendship_clusters)
real_training_data = 'real_training_data.csv'
kmeans_attrs = 'kmeans_attrs.csv'
kmeans_attrs_friends = 'kmeans_attrs_friends.csv'
kmeans_weighted_attrs_friends = 'kmeans_weighted_attrs_friends.csv'
clf.fit(stat_training_attrs, stat_training_values)
stat_pred = clf.predict(stat_training_attrs)
print stat_pred
diff = []
for trueValue, predValue in zip(stat_testing_values,
stat_pred):
diff.append(abs(trueValue - predValue))
print 'Errors for model', name, ':', sum(
diff), 'on value set', value_set
# Visualize data
if args.v:
visualizer = Visualizer()
visualizer.visualize(data,
EDGE_FUNCS[args.edge],
split=args.split,
save=args.save,
show=args.show)
if args.p:
# Select prediction method
if args.p == 'kmeans':
print 'Using k-means clustering metric.'
attribute_clusters, attribute_and_friendship_clusters, weighted_attribute_and_friendship_clusters = k_means_clustering(
data, featureWeightMap, args.show)
attribute_clusters = _convert_kmeans_format(attribute_clusters)
attribute_and_friendship_clusters = _convert_kmeans_format(
attribute_and_friendship_clusters)
weighted_attribute_and_friendship_clusters = _convert_kmeans_format(
weighted_attribute_and_friendship_clusters)
# Statespace can take CircleRobot or RectangleRobot objects
robot = RectangleRobot(0.04, 0.02) #Rectangle
# Takes discrete values, divide continuous values by resolution
# Parameters: environment length, width, 2D array with obstacle parameters
# e.g. [[l1, w1, x1, x2], [l2, w2, x2, y2],..., [ln, wn, xn, yn]]
env = Environment(30, 30, [[6, 2, 19, 17], [2, 2, 14, 26]])
# Parameters: resolution (m), number of theta values, robot object,
# and environment object
state_space = StateSpace(resolution_m, 8, robot, env)
planner = AStar(state_space)
path = []
pts = [0.1, 0.1, 0.2, 0.25]
# Input x (m), y (m)
if planner.set_start(pts[0], pts[1], pi/4) and planner.set_goal(pts[2], pts[3], pi/4):
# Planner return whether or not it was successful,
# the number of expansions, and time taken (s)
success, num_expansions, planning_time = planner.plan()
if success:
_, path = planner.extract_path()
# Remove this when running optimization
vis = Visualizer(env, state_space, robot)
vis.visualize(path, start_end=pts)
def pipeline(args):
feed = InputFeeder(args.i)
feed.load_data()
FaceDetectionPipe = FaceDetection(args.m_fd, args.pt, args.d, args.cpu_ext)
load_time = time.time()
FaceDetectionPipe.load_model()
load_time_fd = time.time() - load_time
FacialLandmarksPipe = FacialLandmarks(args.m_ld, args.d, args.cpu_ext)
load_time = time.time()
FacialLandmarksPipe.load_model()
load_time_ld = time.time() - load_time
HeadPoseEstimationPipe = HeadPoseEstimation(args.m_hpe, args.d,
args.cpu_ext)
load_time = time.time()
HeadPoseEstimationPipe.load_model()
load_time_hpe = time.time() - load_time
GazeEstimationPipe = GazeEstimation(args.m_ge, args.d, args.cpu_ext)
load_time = time.time()
GazeEstimationPipe.load_model()
load_time_ge = time.time() - load_time
log.info('Load time for face detection model: ' + str(load_time_fd))
log.info('Load time for landmark detection model: ' + str(load_time_ld))
log.info('Load time for head pose estimation model: ' + str(load_time_hpe))
log.info('Load time for gaze estimation model: ' + str(load_time_ge))
inf_time_fd = inf_time_ld = inf_time_hpe = inf_time_ge = frame_count = 0
for frame in feed.next_batch():
if frame is None:
break
if cv2.waitKey(1) & 0xFF == ord('q'):
break
frame_count += 1
inf_time = time.time()
fd_img_output, fd_coords = FaceDetectionPipe.predict(frame)
inf_time_fd = time.time() - inf_time
if (fd_coords == []):
log.info('No face detected')
else:
inf_time = time.time()
eye_l_image, eye_r_image, ld_coords = FacialLandmarksPipe.predict(
fd_img_output)
inf_time_ld = time.time() - inf_time
inf_time = time.time()
hpe_output = HeadPoseEstimationPipe.predict(fd_img_output)
inf_time_hpe = time.time() - inf_time
yaw, pitch, roll = hpe_output
inf_time = time.time()
ge_output = GazeEstimationPipe.predict(eye_l_image, eye_r_image,
[yaw, pitch, roll])
inf_time_ge = time.time() - inf_time
if frame_count % 5 == 0:
pointer = MouseController('medium', 'fast')
pointer.move(ge_output[0], ge_output[1])
fps_fd = 1 / inf_time_fd
fps_ld = 1 / inf_time_ld
fps_hpe = 1 / inf_time_hpe
fps_ge = 1 / inf_time_ge
if (args.v):
v = Visualizer(frame, fd_img_output, fd_coords, ld_coords,
hpe_output)
v.visualize()
log.info('Average inference time for face detection model: ' +
str(inf_time_fd))
log.info('Average inference time for landmark detection model: ' +
str(inf_time_ld))
log.info(
'Average inference time for head pose estimation model: ' +
str(inf_time_hpe))
log.info('Average inference time for gaze estimation model: ' +
str(inf_time_ge))
log.info('FPS for face detection model: ' + str(fps_fd))
log.info('FPS for landmark detection model: ' + str(fps_ld))
log.info('FPS for head pose estimation model: ' + str(fps_hpe))
log.info('FPS for gaze estimation model: ' + str(fps_ge))
log.info('Frames Count:' + str(frame_count))
mm = ModelMetrics()
log.info('Writing stats to file...')
mm.save_to_file('stats_fd.txt', 'FD/' + model_precision,
inf_time_fd, fps_fd, load_time_fd)
mm.save_to_file('stats_ld.txt', model_precision, inf_time_ld,
fps_ld, load_time_ld)
mm.save_to_file('stats_hpe.txt', model_precision, inf_time_hpe,
fps_hpe, load_time_hpe)
mm.save_to_file('stats_ge.txt', model_precision, inf_time_ge,
fps_ge, load_time_ge)
feed.close()
def run_planner(env_parameters, render=None):
global_vars[N_RUNS] += 1
resolution_m = 0.01
with open("temp-data-params.txt", "w") as f:
f.write(",".join(env_parameters.astype(str)))
obs_params = clamp_obs(env_parameters[:M])
# Statespace can take a PointRobot, SquareRobot, RectangleRobot objects
# robot = PointRobot(0, 0)
# robot = CircleRobot(3)
# robot = RectangleRobot(4,4)
# robot = RectangleRobot(3,1)
robot = RectangleRobot(0.04, 0.02)
# Takes discrete values, divide continuous values by resolution
# Parameters: environment length, width, 2D array with obstacle parameters
# e.g. [[l1, w1, x1, y1], [l2, w2, x2, y2],..., [ln, wn, xn, yn]]
if ORACLE_L[1]:
env = Environment(30, 30, [])
else:
env = Environment(30, 30, [obs_params[:4], obs_params[4:8]])
# Parameters: resolution (m), number of theta values, robot object,
# and environment object
state_space = StateSpace(resolution_m, 8, robot, env)
planner = AStar(state_space)
error = False
path = ([], [])
success, num_expansions, planning_time = True, 0, 0.0
# Input x (m), y (m)
if len(env_parameters) > 8:
sx, sy, gx, gy = env_parameters[
8:12] / 116 + 0.02 # ensure b/w 0.02 and 0.28
else:
sx, sy, gx, gy = [0.05, 0.05, 0.25, 0.25]
# Optimizing orientation
if len(env_parameters) > 12:
so, go = env_parameters[12], env_parameters[13]
else:
so, go = pi / 4, pi / 4
if not (planner.set_start(sx, sy, so)):
success = False # no expansions, since initial config was invalid
if not (planner.set_goal(gx, gy, go)):
success = False # ditto
# Planner return whether or not it was successful,
# the number of expansions, and time taken (s)
if success:
try:
success, num_expansions, planning_time = planner.plan()
if success:
print("Extracting path")
path = planner.extract_path()
# BUG 3 -- Oracle
if path is None and ORACLE_L[2]:
print("ERROR: Incorrect goal")
error = True
# BUG 4 -- Oracle
elif (not planner.check_consistency(path[0])) and ORACLE_L[3]:
print("ERROR: Path not consistence")
error = True
else:
# BUG 2 -- Oracle
if ORACLE_L[1]:
print('ERROR: Constrained goal')
error = True
# BUG 1 -- Oracle
if len(planner.visited) >= state_space.get_max_expansions(
) and ORACLE_L[0]:
print("ERROR: Expanded every node in the environment")
error = True
# BUG 6 -- Oracle
if planning_time >= TIMEOUT:
print("Planning timed out")
if ORACLE_L[5]:
error = True
# BUG 5 -- Oracle
except Exception:
print("Unexpected error:", sys.exc_info())
error = True
if error or render:
filename = None
if render == True or render is None:
filename = save_vars[IMG_PATH]
else:
filename = render
vis = Visualizer(env, state_space, robot)
if path is None:
path = ([], [])
vis.visualize(path[1], filename=filename, start_end=[sx, sy, gx, gy])
else:
print(" ", end='')
print("Success:", success, "\tExpansions:", num_expansions, "\tTime:",
planning_time)
save_vars[CSV_WRITER].writerow([
env_parameters,
error,
not not render,
success,
num_expansions,
planning_time,
])
save_vars[DATA_FILE].flush()
return error, success, num_expansions, planning_time
def main(cfg):
if not torch.cuda.is_available():
print('CPU mode is not supported')
exit(1)
device = torch.device('cuda:0')
torch.manual_seed(cfg.seed)
torch.cuda.manual_seed_all(cfg.seed)
np.random.seed(cfg.seed)
random.seed(cfg.seed)
if cfg.ckpt:
if not os.path.exists(cfg.ckpt):
print('Invalid ckpt path -->', cfg.ckpt)
exit(1)
ckpt = torch.load(cfg.ckpt, map_location=lambda storage, loc: storage)
print(cfg.ckpt, 'loaded')
loaded_cfg = ckpt['cfg'].__dict__
pprint(loaded_cfg)
del loaded_cfg['train']
del loaded_cfg['test']
del loaded_cfg['visualize']
del loaded_cfg['batch_size']
del loaded_cfg['ckpt']
cfg.__dict__.update(loaded_cfg)
print()
print('Merged Config')
pprint(cfg.__dict__)
print()
step = ckpt['step']
else:
os.makedirs(os.path.join(cfg.log_dir, 'ckpt'))
step = 0
dataloader = MNIST_Dataset(cfg=cfg, )
model = ICVAE(
cfg=cfg,
device=device,
)
print()
print(model)
print()
if torch.cuda.device_count() > 1 and cfg.multi_gpu:
model = nn.DataParallel(model)
model.to(device)
optimizer = Adam(
params=model.parameters(),
lr=cfg.lr,
# weight_decay=cfg.weight_decay,
)
if cfg.ckpt is not None:
model.load_state_dict(ckpt['model'])
optimizer.load_state_dict(ckpt['optimizer'])
if cfg.train:
trainer = Trainer(
cfg=cfg,
dataloader=dataloader,
model=model,
optimizer=optimizer,
device=device,
step=step,
)
trainer.train()
elif cfg.test:
tester = Tester(
cfg=cfg,
dataloader=dataloader,
model=model,
device=device,
)
tester.test()
elif cfg.visualize:
cfg.batch_size = 1
visualizer = Visualizer(
cfg=cfg,
dataloader=dataloader,
model=model,
device=device,
)
visualizer.visualize()
else:
print('Select mode')
exit(1)
