def prepare_result_images(self, np_images, boxes, scores, classes):
result_images = list()
# Filter all Boxes with a Score >= 0.5, for bounding boxes.
# In visualize_boxes_and_labels_on_image_array function this threshold is build in as param
if self.image_visualization == 1:
filtered_boxes_per_image = []
for image_idx in range(np.size(np_images, 0)):
filtered_boxes = []
for score_idx, score in enumerate(scores[image_idx]):
if score >= self.threshold:
filtered_boxes.append(boxes[image_idx][score_idx])
filtered_boxes_per_image.append(np.array(filtered_boxes))
for index in range(np.size(np_images, 0)):
image_np = np_images[index]
# Draw bounding boxes in image
if self.image_visualization == 1:
vis_util.draw_bounding_boxes_on_image_array(
image_np,
filtered_boxes_per_image[index],
thickness=4,
)
# Draw bounding boxes and scores in image
if self.image_visualization == 0:
vis_util.visualize_boxes_and_labels_on_image_array(
image_np,
np.squeeze(boxes[index]),
np.squeeze(classes[index]).astype(np.int32),
np.squeeze(scores[index]),
self.category_index,
use_normalized_coordinates=True,
line_thickness=4,
agnostic_mode=False)
result_images.append(image_np)
return result_images
def recreate_images_with_bounding_boxes(inp_files, input_tensor, res):
boxes, classes, scores, num_det = res
for i, fname in enumerate(inp_files):
n_obj = int(num_det[i])
image_array = input_tensor[i]
print("{} - found objects:".format(fname))
target_boxes = []
for j in range(n_obj):
# check score
cl_id = int(classes[i][j])
label = coco.IMAGE_CLASSES[cl_id]
score = scores[i][j]
if score < 0.5:
continue
# print each data
box = boxes[i][j]
print(" ", cl_id, label, score, box)
target_boxes.append(box)
# recreate image with bounding boxes
visualization_utils.draw_bounding_boxes_on_image_array(
image_array, np.array(target_boxes))
out_file_name = os.path.splitext(fname)[0] + "_with_boxes.png"
Image.fromarray(image_array).save(out_file_name)
def test_draw_bounding_boxes_on_image_array(self):
test_image = self.create_colorful_test_image()
width_original = test_image.shape[0]
height_original = test_image.shape[1]
boxes = np.array([[0.25, 0.75, 0.4, 0.6],
[0.1, 0.1, 0.9, 0.9]])
visualization_utils.draw_bounding_boxes_on_image_array(test_image, boxes)
width_final = test_image.shape[0]
height_final = test_image.shape[1]
self.assertEqual(width_original, width_final)
self.assertEqual(height_original, height_final)
def test_draw_bounding_boxes_on_image_array(self):
test_image = self.create_colorful_test_image()
width_original = test_image.shape[0]
height_original = test_image.shape[1]
boxes = np.array([[0.25, 0.75, 0.4, 0.6], [0.1, 0.1, 0.9, 0.9]])
visualization_utils.draw_bounding_boxes_on_image_array(
test_image, boxes)
width_final = test_image.shape[0]
height_final = test_image.shape[1]
self.assertEqual(width_original, width_final)
self.assertEqual(height_original, height_final)
detected_scores_stack=detected_scores
else:
x_stack=np.hstack((x_stack,x_))
y_stack=np.hstack((y_stack,y_))
detected_boxes_stack=np.vstack((detected_boxes_stack,detected_boxes))
detected_scores_stack=np.vstack((detected_scores_stack,detected_scores))
# Visulalise some high confidence images
if FLAGS.vis_path and detected_scores.any():
if np.max(detected_scores)>FLAGS.vis_threshold:
for j in range(FLAGS.batch_size):
if np.max(detected_scores[j])>FLAGS.vis_threshold:
# Draw boxes
boxes_ii=np.where(detected_scores[j]>FLAGS.vis_threshold)
boxes=detected_boxes[j][boxes_ii]
vis_utils.draw_bounding_boxes_on_image_array(image[j],boxes)
# Save
vis_utils.save_image_array_as_png(image[j],os.path.join(FLAGS.vis_path,str(x_[j])+"-"+str(y_[j])+".png"))
except tf.errors.OutOfRangeError:
# Catch exceptions
tf.logging.info('Finished processing records')
finally:
if len(sat_gen.map):
sat_gen.write(x_stack,y_stack,(1024,1024),detected_boxes_stack,detected_scores_stack,batch_size=FLAGS.batch_size,stack_size=int(len(x_stack)/FLAGS.batch_size),count=True)
tf.logging.info('Finished writting residual stacks')
end=time.time()
tf.logging.info("Elapsed time {}".format(end-start))
if __name__ == '__main__':
dtype=np.uint8)
img_rgb = img.copy()
tree = ET.parse(file.split('.')[0] + '.xml')
root = tree.getroot()
obj = root.findall('object')
bndbox = obj[0].find('bndbox')
xmin = int(bndbox.find('xmin').text)
xmax = int(bndbox.find('xmax').text)
ymin = int(bndbox.find('ymin').text)
ymax = int(bndbox.find('ymax').text)
vis_util.draw_bounding_boxes_on_image_array(
img_rgb,
np.array([[(float(ymin) / 720.0), (float(xmin) / 1280.0),
(float(ymax) / 720.0), (float(xmax) / 1280.0)]]),
color='black',
thickness=4)
bb2 = {
'x1': xmin - 1280 / 2,
'x2': xmax - 1280 / 2,
'y1': 720 / 2 - ymin,
'y2': 720 / 2 - ymax
}
with open(file, 'rb') as f:
xmin, xmax, ymin, ymax = [
int(float(x)) for x in f.readline().split()
]
vis_util.draw_bounding_boxes_on_image_array(
img_rgb,
def step(self, action):
_state = State()
file = 'data/' + str(self._sequences[self.current_sequence]) + '/' + \
str(self.frames_sequences[self.current_sequence][self.current_frame])
frame = np.asarray(Image.open(file.split('.')[0] +
'.png').convert('RGB'),
dtype=np.uint8).copy()
# GROUNDTRUTH
tree = ET.parse(file.split('.')[0] + '.xml')
root = tree.getroot()
obj = root.findall('object')
bndbox = obj[0].find('bndbox')
xmin = int(bndbox.find('xmin').text)
xmax = int(bndbox.find('xmax').text)
ymin = int(bndbox.find('ymin').text)
ymax = int(bndbox.find('ymax').text)
gt_box = {
'x1': xmin - self.im_width / 2,
'x2': xmax - self.im_width / 2,
'y1': self.im_height / 2 - ymin,
'y2': self.im_height / 2 - ymax
}
# TRACKER
det_box = None
with open(file, 'rb') as f:
xmint, xmaxt, ymint, ymaxt = [
int(float(x)) for x in f.readline().split()
]
det_box = {
'x1': xmint - self.im_width / 2,
'x2': xmaxt - self.im_width / 2,
'y1': self.im_height / 2 - ymint,
'y2': self.im_height / 2 - ymaxt
}
POS_X = int(float(det_box['x1'] + det_box['x2']) / 2.0)
POS_Y = int(float(det_box['y1'] + det_box['y2']) / 2.0)
WIDTH = int(float(det_box['x2'] - det_box['x1']))
HEIGHT = int(float(det_box['y1'] - det_box['y2']))
# GREEDY (BASELINE) AGENT
step_x = self.nearest_to_step(
int(float(gt_box['x1'] + gt_box['x2']) / 2.0) - self.old_gx)
step_y = self.nearest_to_step(
int(float(gt_box['y1'] + gt_box['y2']) / 2.0) - self.old_gy)
new_gx = self.old_gx + step_x
new_gy = self.old_gy + step_y
baseline_box = {
'x1': new_gx - WIDTH / 2,
'x2': new_gx + WIDTH / 2,
'y1': new_gy + HEIGHT / 2,
'y2': new_gy - HEIGHT / 2
}
# CUSTOM AGENT
print "\n-----Parameters-----"
print "Delta X:", self.current_state.DELTA_X
print "Delta Y:", self.current_state.DELTA_Y
new_x = self.old_x + action[0]
new_y = self.old_y + action[1]
agent_box = {
'x1': new_x - WIDTH / 2,
'x2': new_x + WIDTH / 2,
'y1': new_y + HEIGHT / 2,
'y2': new_y - HEIGHT / 2
}
if self.current_episode > self.max_guided_eps:
self.old_x = new_x # POS_X
self.old_y = new_y # POS_Y
else:
self.old_x = POS_X
self.old_y = POS_Y
self.old_gx = new_gx
self.old_gy = new_gy
iou_unconstrained = self.get_iou(gt_box, det_box)
iou_constrained = self.get_iou(gt_box, agent_box)
iou_baseline = self.get_iou(gt_box, baseline_box)
print "-----IoU Stats-----"
print "Uncons IoU:", iou_unconstrained
print "Cons IoU:", iou_constrained
print "Greedy IoU:", iou_baseline
dist_x = float(agent_box['x1'] + agent_box['x2']) / 2.0 - float(
gt_box['x1'] + gt_box['x2']) / 2.0
dist_y = float(agent_box['y1'] + agent_box['y2']) / 2.0 - float(
gt_box['y1'] + gt_box['y2']) / 2.0
dist = np.linalg.norm([dist_x, dist_y]) / self.max_dist
reward = ((1 - dist) * self.SCALE_DIST + iou_constrained *
self.SCALE_IOU) / (self.SCALE_DIST + self.SCALE_IOU)
print "Distance :", dist, \
"\nIoU :", iou_constrained, \
"\nDist Reward:", (1-dist)*self.SCALE_DIST, \
"\nIoU Reward :", iou_constrained*self.SCALE_IOU
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(ymin) / float(self.im_height)),
(float(xmin) / float(self.im_width)),
(float(ymax) / float(self.im_height)),
(float(xmax) / float(self.im_width))]]),
color='black',
thickness=7)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - det_box['y1']) /
float(self.im_height)),
(float(det_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - det_box['y2']) /
float(self.im_height)),
(float(det_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='blue',
thickness=3)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - agent_box['y1']) /
float(self.im_height)),
(float(agent_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - agent_box['y2']) /
float(self.im_height)),
(float(agent_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='yellow',
thickness=5)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - baseline_box['y1']) /
float(self.im_height)),
(float(baseline_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - baseline_box['y2']) /
float(self.im_height)),
(float(baseline_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='red',
thickness=5)
result = Image.fromarray(frame)
path_prefix = 'data/output_train/' + self._sequences[
self.current_sequence] + '/'
if not os.path.exists(path_prefix):
os.makedirs(path_prefix)
img_path = path_prefix + self.frames_sequences[self.current_sequence][
self.current_frame].split('.')[0] + '.jpg'
result.save(img_path)
done = 0
self.current_frame += 1
if self.current_frame >= self.length_sequences[self.current_sequence]:
done = 1
self.current_sequence = (self.current_sequence + 1) % len(
self._sequences)
print "Action RL :", action, "pixels"
print "Action BASE:", (step_x, step_y), "pixels"
print "Reward :", reward
print "Done :", done
# cv2.imshow('Simulation', frame)
# cv2.waitKey(10)
_state = State()
if not done:
file = 'data/' + str(self._sequences[self.current_sequence]) + '/' + \
str(self.frames_sequences[self.current_sequence][self.current_frame])
det_box = None
with open(file, 'rb') as f:
xmint, xmaxt, ymint, ymaxt = [
int(float(x)) for x in f.readline().split()
]
det_box = {
'x1': xmint - self.im_width / 2,
'x2': xmaxt - self.im_width / 2,
'y1': self.im_height / 2 - ymint,
'y2': self.im_height / 2 - ymaxt
}
POS_X = int(float(det_box['x1'] + det_box['x2']) / 2.0)
POS_Y = int(float(det_box['y1'] + det_box['y2']) / 2.0)
WIDTH = int(float(det_box['x2'] - det_box['x1']))
HEIGHT = int(float(det_box['y1'] - det_box['y2']))
_state.DELTA_X = POS_X - self.old_x
_state.DELTA_Y = POS_Y - self.old_y
self.current_state = _state
return self.state_to_array(_state), reward, done
def test(_):
start=time.time()
tf.logging.set_verbosity(tf.logging.DEBUG)
#Check FLAGS
SatellitleGeneratorTest.check_flags(required_flags = ['input_path','inference_graph'])
# Intialise and load shard
if FLAGS.test_size==-1:
# Use default test size of 10
sat_gen=SatellitleGeneratorTest(FLAGS.input_path,FLAGS.batch_size)
else:
sat_gen=SatellitleGeneratorTest(FLAGS.input_path,FLAGS.batch_size,FLAGS.test_size)
sat_gen.load_shard(FLAGS.shard_path,FLAGS.shard)
# Test a sample patch read
coord=(sat_gen.map[0][0],sat_gen.map[1][0])
sat_gen.read_patch_test(coord,FLAGS.input_path)
# Setup dataset object
sat_gen.setup_data()
# Turn on log_device_placement for verbosity in ops
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,log_device_placement=False)) as sess:
sess.run(sat_gen.iter.initializer)
if FLAGS.test_images:
while True:
x,y,image_tensors=sat_gen.iter.get_next()
sys.exit(1)
# Read and Fill Graph
tf.logging.info('Reading graph and building model...')
with tf.gfile.Open(FLAGS.inference_graph, 'rb') as graph_def_file:
graph_content = graph_def_file.read()
try:
for counter in itertools.count():
tf.logging.info('Reading Image No; \t {} SHARD{}'.format(counter,FLAGS.shard))
x,y,image_tensors=sat_gen.iter.get_next()
if not counter:
# Build Graph tensors only once
(num_detections_tensor,detected_boxes_tensor, detected_scores_tensor,
detected_labels_tensor) = sat_gen.build_inference_graph(
image_tensors, graph_content)
tf.logging.info('Running inference')
(detected_boxes, detected_scores,detected_classes, num_detections) = sess.run([detected_boxes_tensor,\
detected_scores_tensor,detected_labels_tensor,num_detections_tensor])
tf.logging.log_every_n(tf.logging.INFO, 'Processed %d images...', 10,counter)
# Threshold SCORES
ii=np.where(detected_scores>FLAGS.threshold)
detected_scores=detected_scores[ii]
detected_boxes=detected_boxes[ii]
detected_classes=detected_classes[ii]
tf.logging.debug("DETETED SCORES .{}".format((detected_scores)))
tf.logging.debug("DETECTED BOXES {}".format(len(detected_boxes)))
x_,y_=sess.run([x,y])
sat_gen.write(x_,y_,(1024,1024),detected_boxes,detected_scores,count=counter)
# Visulalise some high confidence images
if FLAGS.vis_path and detected_scores.any():
if np.max(detected_scores)>FLAGS.vis_threshold:
image=np.squeeze(sess.run(image_tensors))
# Draw boxes
vis_utils.draw_bounding_boxes_on_image_array(image, detected_boxes)
# Save
x_=x_[0]
y_=y_[0]
vis_utils.save_image_array_as_png(image,os.path.join(FLAGS.vis_path,str(x_)+"-"+str(y_)+".png"))
except tf.errors.OutOfRangeError:
# Catch exceptions
tf.logging.info('Finished processing records')
finally:
end=time.time()
tf.logging.info("Elapsed time {}".format(end-start))
def step(self, action):
# TRACKER
done = 0
reward = 0.0
frame = self.current_frame.copy()
output = self.current_output
POS_X = output[0]
POS_Y = output[1]
WIDTH = output[2]
HEIGHT = output[3]
det_box = {
'x1': POS_X - WIDTH / 2,
'x2': POS_X + WIDTH / 2,
'y1': POS_Y + HEIGHT / 2,
'y2': POS_Y - HEIGHT / 2
}
print "\n-----Parameters-----"
print "Delta X:", self.current_state.DELTA_X
print "Delta Y:", self.current_state.DELTA_Y
if action is not None:
# GREEDY (BASELINE) AGENT
step_x = self.nearest_to_step(POS_X - self.old_gx)
step_y = self.nearest_to_step(POS_Y - self.old_gy)
new_gx = self.old_gx + step_x
new_gy = self.old_gy + step_y
baseline_box = {
'x1': new_gx - WIDTH / 2,
'x2': new_gx + WIDTH / 2,
'y1': new_gy + HEIGHT / 2,
'y2': new_gy - HEIGHT / 2
}
# CUSTOM AGENT
new_x = self.old_x + action[0]
new_y = self.old_y + action[1]
agent_box = {
'x1': new_x - WIDTH / 2,
'x2': new_x + WIDTH / 2,
'y1': new_y + HEIGHT / 2,
'y2': new_y - HEIGHT / 2
}
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - det_box['y1']) /
float(self.im_height)),
(float(det_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - det_box['y2']) /
float(self.im_height)),
(float(det_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='blue',
thickness=3)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - agent_box['y1']) /
float(self.im_height)),
(float(agent_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - agent_box['y2']) /
float(self.im_height)),
(float(agent_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='yellow',
thickness=5)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - baseline_box['y1']) /
float(self.im_height)),
(float(baseline_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - baseline_box['y2']) /
float(self.im_height)),
(float(baseline_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='red',
thickness=5)
iou_constrained = self.get_iou(det_box, agent_box)
iou_baseline = self.get_iou(det_box, baseline_box)
print "-----IoU Stats-----"
print "~Cons IoU :", iou_constrained
print "~Grdy IoU :", iou_baseline
if not self.TEST:
dist_x = float(agent_box['x1'] +
agent_box['x2']) / 2.0 - float(
det_box['x1'] + det_box['x2']) / 2.0
dist_y = float(agent_box['y1'] +
agent_box['y2']) / 2.0 - float(
det_box['y1'] + det_box['y2']) / 2.0
dist = np.linalg.norm([dist_x, dist_y]) / self.max_dist
reward = ((1 - dist) * self.SCALE_DIST + iou_constrained *
self.SCALE_IOU) / (self.SCALE_DIST + self.SCALE_IOU)
print "Distance :", dist, \
"\nIoU :", iou_constrained, \
"\nDist Reward:", (1-dist)*self.SCALE_DIST, \
"\nIoU Reward :", iou_constrained*self.SCALE_IOU
print "Action RL :", action, "pixels"
print "Action BASE:", (step_x, step_y), "pixels"
# cv2.imshow('Simulation', frame)
# cv2.waitKey(2)
self.old_x = new_x
self.old_y = new_y
if (not self.TEST) and self.current_episode > self.max_guided_eps:
self.old_x = POS_X
self.old_y = POS_Y
self.old_gx = new_gx
self.old_gy = new_gy
else:
self.old_x = POS_X
self.old_y = POS_Y
self.old_gx = POS_X
self.old_gy = POS_Y
# NEXT frame
_state = State()
frame = self._connector.get_frame()
output = self._detector.detect(frame)
if (not output) or reward <= self.min_reward:
done = 1
reward = -20.0
else:
POS_X = output[0]
POS_Y = output[1]
WIDTH = output[2]
HEIGHT = output[3]
_state.DELTA_X = POS_X - self.old_x
_state.DELTA_Y = POS_Y - self.old_y
self.current_state = _state
self.current_output = output
self.current_frame = frame
print "Reward :", reward
print "Done :", done
return self.state_to_array(_state), reward, done
def step(self, action):
file = 'data/' + str(self._sequences[self.current_sequence]) + '/' + \
str(self.frames_sequences[self.current_sequence][self.current_frame])
frame = np.asarray(Image.open(file.split('.')[0] +
'.png').convert('RGB'),
dtype=np.uint8).copy()
# GROUNDTRUTH
tree = ET.parse(file.split('.')[0] + '.xml')
root = tree.getroot()
obj = root.findall('object')
bndbox = obj[0].find('bndbox')
xmin = int(bndbox.find('xmin').text)
xmax = int(bndbox.find('xmax').text)
ymin = int(bndbox.find('ymin').text)
ymax = int(bndbox.find('ymax').text)
gt_box = {
'x1': xmin - 1280 / 2,
'x2': xmax - 1280 / 2,
'y1': 720 / 2 - ymin,
'y2': 720 / 2 - ymax
}
# TRACKER
det_box = None
with open(file, 'rb') as f:
xmint, xmaxt, ymint, ymaxt = [
int(float(x)) for x in f.readline().split()
]
det_box = {
'x1': xmint - 1280 / 2,
'x2': xmaxt - 1280 / 2,
'y1': 720 / 2 - ymint,
'y2': 720 / 2 - ymaxt
}
# output = self._detector.detect(frame)
# POS_X = output[0]
# POS_Y = output[1]
# WIDTH = output[2]
# HEIGHT = output[3]
POS_X = int(float(det_box['x1'] + det_box['x2']) / 2.0)
POS_Y = int(float(det_box['y1'] + det_box['y2']) / 2.0)
WIDTH = int(float(det_box['x2'] - det_box['x1']))
HEIGHT = int(float(det_box['y1'] - det_box['y2']))
# det_box = {
# 'x1': POS_X - WIDTH/2,
# 'x2': POS_X + WIDTH/2,
# 'y1': POS_Y - HEIGHT/2,
# 'y2': POS_Y + HEIGHT/2
# }
# CUSTOM AGENT
print "\n-----Parameters-----"
print "Delta X:", self.current_state.DELTA_X
print "Delta Y:", self.current_state.DELTA_Y
new_x = self.old_x + action[0]
new_y = self.old_y + action[1]
self.old_x = POS_X
self.old_y = POS_Y
agent_box = {
'x1': new_x - WIDTH / 2,
'x2': new_x + WIDTH / 2,
'y1': new_y + HEIGHT / 2,
'y2': new_y - HEIGHT / 2
}
iou_unconstrained = self.get_iou(gt_box, det_box)
iou_constrained = self.get_iou(gt_box, agent_box)
print "-----IoU Stats-----"
print "Uncons IoU:", iou_unconstrained
print "Cons IoU:", iou_constrained
dist_x = float(agent_box['x1'] + agent_box['x2']) / 2.0 - float(
gt_box['x1'] + gt_box['x2']) / 2.0
dist_y = float(agent_box['y1'] + agent_box['y2']) / 2.0 - float(
gt_box['y1'] + gt_box['y2']) / 2.0
dist = np.linalg.norm([dist_x, dist_y]) / self.max_dist
reward = (1 -
dist) * self.SCALE_DIST + iou_constrained * self.SCALE_IOU
print "Distance :", dist, \
"\nIoU :", iou_constrained, \
"\nDist Reward:", (1-dist)*self.SCALE_DIST, \
"\nIoU Reward :", iou_constrained*self.SCALE_IOU
vis_util.draw_bounding_boxes_on_image_array(frame,
np.array([[
(float(ymin) / 720.0),
(float(xmin) / 1280.0),
(float(ymax) / 720.0),
(float(xmax) / 1280.0)
]]),
color='black',
thickness=7)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(720 / 2 - det_box['y1']) / 720.0),
(float(det_box['x1'] + 1280 / 2) / 1280.0),
(float(720 / 2 - det_box['y2']) / 720.0),
(float(det_box['x2'] + 1280 / 2) / 1280.0)]]),
color='blue',
thickness=3)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(720 / 2 - agent_box['y1']) / 720.0),
(float(agent_box['x1'] + 1280 / 2) / 1280.0),
(float(720 / 2 - agent_box['y2']) / 720.0),
(float(agent_box['x2'] + 1280 / 2) / 1280.0)]]),
color='yellow',
thickness=5)
done = 0
self.current_frame += 1
if self.current_frame >= self.length_sequences[self.current_sequence]:
done = 1
self.current_sequence = (self.current_sequence + 1) % len(
self._sequences)
print "Action :", action, "pixels"
print "Reward :", reward
print "Done :", done
# imgplot = plt.imshow(frame)
# plt.show()
if not self.fig:
plt.ion()
self.fig = plt.figure()
self.plot = plt.subplot(1, 1, 1)
plt.imshow(frame)
ax = self.fig.gca()
ax.set_xticks(np.arange(0., 1280., 85.33))
ax.set_yticks(np.arange(0., 720., 48.))
plt.grid()
self.fig.show()
else:
plt.imshow(frame)
self.plot.relim()
self.fig.canvas.flush_events()
_state = State()
if not done:
file = 'data/' + str(self._sequences[self.current_sequence]) + '/' + \
str(self.frames_sequences[self.current_sequence][self.current_frame])
det_box = None
with open(file, 'rb') as f:
xmint, xmaxt, ymint, ymaxt = [
int(float(x)) for x in f.readline().split()
]
det_box = {
'x1': xmint - 1280 / 2,
'x2': xmaxt - 1280 / 2,
'y1': 720 / 2 - ymint,
'y2': 720 / 2 - ymaxt
}
# output = self._detector.detect(frame)
# POS_X = output[0]
# POS_Y = output[1]
# WIDTH = output[2]
# HEIGHT = output[3]
POS_X = int(float(det_box['x1'] + det_box['x2']) / 2.0)
POS_Y = int(float(det_box['y1'] + det_box['y2']) / 2.0)
WIDTH = int(float(det_box['x2'] - det_box['x1']))
HEIGHT = int(float(det_box['y1'] - det_box['y2']))
_state.DELTA_X = POS_X - self.old_x
_state.DELTA_Y = POS_Y - self.old_y
self.current_state = _state
return self.state_to_array(_state), reward, done
def step(self, action):
# TRACKER
frame = self.current_frame.copy()
output = self.current_output
POS_X = output[0]
POS_Y = output[1]
WIDTH = output[2]
HEIGHT = output[3]
det_box = {
'x1': POS_X - WIDTH / 2,
'x2': POS_X + WIDTH / 2,
'y1': POS_Y + HEIGHT / 2,
'y2': POS_Y - HEIGHT / 2
}
print "\n-----Parameters-----"
print "Delta X:", self.current_state.DELTA_X
print "Delta Y:", self.current_state.DELTA_Y
if action is not None:
# GREEDY (BASELINE) AGENT
step_x = self.nearest_to_step(POS_X - self.old_gx)
step_y = self.nearest_to_step(POS_Y - self.old_gy)
new_gx = self.old_gx + step_x
new_gy = self.old_gy + step_y
baseline_box = {
'x1': new_gx - WIDTH / 2,
'x2': new_gx + WIDTH / 2,
'y1': new_gy + HEIGHT / 2,
'y2': new_gy - HEIGHT / 2
}
# CUSTOM AGENT
new_x = self.old_x + action[0]
new_y = self.old_y + action[1]
agent_box = {
'x1': new_x - WIDTH / 2,
'x2': new_x + WIDTH / 2,
'y1': new_y + HEIGHT / 2,
'y2': new_y - HEIGHT / 2
}
print "[DEBUG]:", det_box
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - det_box['y1']) /
float(self.im_height)),
(float(det_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - det_box['y2']) /
float(self.im_height)),
(float(det_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='blue',
thickness=3)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - agent_box['y1']) /
float(self.im_height)),
(float(agent_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - agent_box['y2']) /
float(self.im_height)),
(float(agent_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='yellow',
thickness=5)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - baseline_box['y1']) /
float(self.im_height)),
(float(baseline_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - baseline_box['y2']) /
float(self.im_height)),
(float(baseline_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='red',
thickness=5)
print "Action :", action, "pixels"
# cv2.imshow('Simulation', frame)
# cv2.waitKey(10)
self.old_x = new_x
self.old_y = new_y
self.old_gx = new_gx
self.old_gy = new_gy
else:
self.old_x = POS_X
self.old_y = POS_Y
self.old_gx = POS_X
self.old_gy = POS_Y
# NEXT frame
_state = State()
frame = self._connector.get_frame()
output = self._detector.detect(frame)
if not output:
raise Exception('Unable to Detect')
POS_X = output[0]
POS_Y = output[1]
WIDTH = output[2]
HEIGHT = output[3]
_state.DELTA_X = POS_X - self.old_x
_state.DELTA_Y = POS_Y - self.old_y
self.current_state = _state
self.current_output = output
self.current_frame = frame
return self.state_to_array(_state)
def main(_):
start=time.time()
tf.logging.set_verbosity(tf.logging.INFO)
#Check FLAGS
SatellitleGenerator.check_flags(required_flags = ['input_path','inference_graph'])
# Intialise and load shard
sat_gen=SatellitleGenerator(FLAGS.input_path,FLAGS.batch_size,FLAGS.test_size)
if FLAGS.residue_run:
print("residue run")
sat_gen.load_residues(FLAGS.output_path)
sat_gen.load_shard(FLAGS.shard_path,FLAGS.shard)
if FLAGS.restore_from_json:
message=sat_gen.restore_from_json(os.path.join(FLAGS.output_path,'result-'+str(FLAGS.shard)+'.json'))
if message:
tf.logging.info("Finished all evaluations here. Exitting")
didnotrun=True
sys.exit(0)
else:
didnotrun=False
else:
didnotrun=False
# Test a sample patch read
'''
coord=(sat_gen.map[0][0],sat_gen.map[1][0])
sat_gen.read_patch_test(coord,FLAGS.input_path)
'''
# Setup dataset object
sat_gen.setup_data()
# Session configs
config=tf.ConfigProto(allow_soft_placement=True,log_device_placement=False)
config.gpu_options.allow_growth = True
# Turn on log_device_placement for verbosity in ops
# Covered png's
total_covered=[]
with tf.Session(config=config) as sess:
sess.run(sat_gen.iter.initializer)
# Read and Fill Graph
tf.logging.info('Reading graph and building model...')
with tf.gfile.Open(FLAGS.inference_graph, 'rb') as graph_def_file:
graph_content = graph_def_file.read()
x,y,image_tensors,covered=sat_gen.iter.get_next()
(num_detections_tensor,detected_boxes_tensor, detected_scores_tensor,
detected_labels_tensor) = sat_gen.build_inference_graph(
image_tensors, graph_content)
# Chrome tracing
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
try:
for counter in itertools.count():
tf.logging.info('Reading Image No; \t {} SHARD{}'.format(counter,FLAGS.shard))
tf.logging.info('Running inference')
if counter%10==0:
(detected_boxes, detected_scores, num_detections,x_,y_,image,covered_) = sess.run([detected_boxes_tensor,\
detected_scores_tensor,num_detections_tensor,x,y,image_tensors,covered],
options=options, run_metadata=run_metadata)
else:
(detected_boxes, detected_scores, num_detections,x_,y_,image,covered_) = sess.run([detected_boxes_tensor,\
detected_scores_tensor,num_detections_tensor,x,y,image_tensors,covered])
tf.logging.log_every_n(tf.logging.INFO, 'Processed %d images...', 10,counter)
total_covered.append(covered_)
if counter==0:
# Initialise stacks
tf.logging.info("Initialsied stacks")
x_stack=x_
y_stack=y_
detected_boxes_stack=detected_boxes
detected_scores_stack=detected_scores
# Calibrate to write at counter = FLAGS.write_every-1, 2*FLAGS.write_every-1, ...
if (counter)%FLAGS.write_every==FLAGS.write_every-1 :
# Write to JSON
tf.logging.info("Writting to JSON SHARD {}".format(FLAGS.shard) )
sat_gen.write(x_stack,y_stack,(1024,1024),detected_boxes_stack,detected_scores_stack,count=counter,batch_size=FLAGS.batch_size,stack_size=FLAGS.write_every)
x_stack=x_
y_stack=y_
detected_boxes_stack=detected_boxes
detected_scores_stack=detected_scores
np.save(FLAGS.total_covered,np.array(total_covered).flatten().astype(str))
else:
x_stack=np.hstack((x_stack,x_))
y_stack=np.hstack((y_stack,y_))
detected_boxes_stack=np.vstack((detected_boxes_stack,detected_boxes))
detected_scores_stack=np.vstack((detected_scores_stack,detected_scores))
# Visulalise some high confidence images
if FLAGS.vis_path and detected_scores.any():
if np.max(detected_scores)>FLAGS.vis_threshold:
for j in range(FLAGS.batch_size):
if np.max(detected_scores[j])>FLAGS.vis_threshold:
# Draw boxes
boxes_ii=np.where(detected_scores[j]>FLAGS.vis_threshold)
boxes=detected_boxes[j][boxes_ii]
vis_utils.draw_bounding_boxes_on_image_array(image[j],boxes)
# Save
vis_utils.save_image_array_as_png(image[j],os.path.join(FLAGS.vis_path,str(x_[j])+"-"+str(y_[j])+".png"))
'''
# Chrome Profiling trace
if counter%100==0:
fetched_timeline = timeline.Timeline(run_metadata.step_stats)
chrome_trace = fetched_timeline.generate_chrome_trace_format()
with open(os.path.join(FLAGS.trace_path,'timeline_'+str(counter)+'.json'), 'w') as f:
f.write(chrome_trace)
'''
except tf.errors.OutOfRangeError:
# Catch exceptions
tf.logging.info('Finished processing records')
finally:
if didnotrun:
pass
else:
# print(x_stack.shape)
sat_gen.write(x_stack,y_stack,(1024,1024),detected_boxes_stack,detected_scores_stack,batch_size=FLAGS.batch_size,stack_size=int(len(x_stack)/FLAGS.batch_size),count=True)
tf.logging.info('Finished writting residual stacks of size {}'.format(len(x_stack)))
end=time.time()
tf.logging.info("Elapsed time {}".format(end-start))
def main(_):
start = time.time()
tf.logging.set_verbosity(tf.logging.INFO)
#Check FLAGS
CellImagesRun.check_flags(required_flags=['input_path', 'inference_graph'])
# Intialise and load shard
cell_run = CellImagesRun(FLAGS.input_path, FLAGS.batch_size)
if FLAGS.test:
cell_run.map = cell_run.map[:10]
# Setup dataset object
cell_run.setup_data()
# Turn on log_device_placement for verbosity in ops
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
sess.run(cell_run.iter.initializer)
# Read and Fill Graph
tf.logging.info('Reading graph and building model...')
with tf.gfile.Open(FLAGS.inference_graph, 'rb') as graph_def_file:
graph_content = graph_def_file.read()
id, image_tensors = cell_run.iter.get_next()
(num_detections_tensor, detected_boxes_tensor, detected_scores_tensor,
detected_labels_tensor) = cell_run.build_inference_graph(
image_tensors, graph_content)
try:
for counter in itertools.count():
tf.logging.info('Reading Image No; \t {} '.format(counter))
tf.logging.info('Running inference')
try:
(detected_boxes, detected_scores,detected_classes, num_detections,id_,image) = sess.run([detected_boxes_tensor,\
detected_scores_tensor,detected_labels_tensor,num_detections_tensor,id,image_tensors])
except:
sess.run([id])
continue
tf.logging.log_every_n(tf.logging.INFO,
'Processed %d images...', 10, counter)
# Threshold SCORES
ii = np.where(detected_scores > FLAGS.threshold)
detected_scores = detected_scores[ii]
detected_boxes = detected_boxes[ii]
detected_classes = detected_classes[ii]
tf.logging.debug("DETETED SCORES .{}".format(
(detected_scores)))
tf.logging.debug("DETECTED BOXES {}".format(
len(detected_boxes)))
# Visulalise some high confidence images
if FLAGS.vis_path and detected_scores.any():
if np.max(detected_scores) > FLAGS.vis_threshold:
image = np.squeeze(image)
# Draw boxes
id_ = id_[0].decode("utf-8")
vis_utils.draw_bounding_boxes_on_image_array(
image, detected_boxes)
vis_utils.save_image_array_as_png(
image, os.path.join(FLAGS.vis_path, id_))
except tf.errors.OutOfRangeError:
# Catch exceptions
tf.logging.info('Finished processing records')
finally:
end = time.time()
tf.logging.info("Elapsed time {}".format(end - start))
def detect(self, image_np, gt_box=None):
image = image_np.copy()
# image_np_expanded = np.expand_dims(image_np, axis=0)
output_dict = self.run_inference_for_single_image(image_np)
if gt_box is not None:
vis_util.draw_bounding_boxes_on_image_array(image,
np.array([gt_box]),
color='black',
thickness=4)
# Visualization of the results of a detection.
vis_util.visualize_boxes_and_labels_on_image_array(
image,
output_dict['detection_boxes'],
output_dict['detection_classes'],
output_dict['detection_scores'],
self.category_index,
min_score_thresh=self.min_score_thresh,
instance_masks=output_dict.get('detection_masks'),
use_normalized_coordinates=True,
skip_scores=False,
skip_labels=True,
line_thickness=4)
# cv2.imshow('Simulation', image)
# cv2.waitKey(10)
bboxes = output_dict['detection_boxes']
classes = output_dict['detection_classes']
scores = output_dict['detection_scores']
bboxes = [
bbox for bbox, _ in sorted(
zip(bboxes, scores), key=lambda pair: pair[1], reverse=True)
]
classes = [
clss for clss, _ in sorted(
zip(classes, scores), key=lambda pair: pair[1], reverse=True)
]
scores = sorted(scores, key=lambda x: x, reverse=True)
im_height, im_width = image_np.shape[0:2]
for i in range(len(bboxes)):
if scores is None or scores[i] > self.min_score_thresh:
if classes[i] in self.category_index.keys():
class_name = self.category_index[classes[i]]['name']
if class_name == 'car':
box = tuple(bboxes[i].tolist())
ymin, xmin, ymax, xmax = box
left = xmin * im_width
right = xmax * im_width
top = ymin * im_height
bottom = ymax * im_height
POS_X = (left + right - im_width) / 2.0
POS_Y = (im_height - top - bottom) / 2.0
WIDTH = right - left
HEIGHT = bottom - top
return (POS_X, POS_Y, WIDTH, HEIGHT)
return None
