def val_to_data(source):
text_lines=[]
frames_list=[]
frame = None
progress = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ',progressbar.Percentage(), ' ',progressbar.ETA()])
with open(source, 'r') as s:
for line in s:
id_frame, id_class, conf, xmin, ymin, xmax, ymax = line.strip().split(' ')
text_lines.append((id_frame, id_class, conf, xmin, ymin, xmax, ymax))
for i in range(0, len(text_lines)):
if frame is None:
frame = fm.Frame_Info()
frame.frame= text_lines[i][0]
rect= multiclass_rectangle.Rectangle_Multiclass()
# Not all the inserted values are really used
rect.load_labeled_rect(0, text_lines[i][2], text_lines[i][2], text_lines[i][3], text_lines[i][4], text_lines[i][5], text_lines[i][6], text_lines[i][1], text_lines[i][1], text_lines[i][1])
frame.append_labeled_rect(rect)
else :
if frame.frame == text_lines[i][0]:
rect= multiclass_rectangle.Rectangle_Multiclass()
# Not all the inserted values are really used
rect.load_labeled_rect(0, text_lines[i][2], text_lines[i][2], text_lines[i][3], text_lines[i][4], text_lines[i][5], text_lines[i][6], text_lines[i][1], text_lines[i][1], text_lines[i][1])
frame.append_labeled_rect(rect)
else :
frames_list.append(frame)
frame = fm.Frame_Info()
frame.frame= text_lines[i][0]
rect= multiclass_rectangle.Rectangle_Multiclass()
# Not all the inserted values are really used
rect.load_labeled_rect(0, text_lines[i][2], text_lines[i][2], text_lines[i][3], text_lines[i][4], text_lines[i][5], text_lines[i][6], text_lines[i][1], text_lines[i][1], text_lines[i][1])
frame.append_labeled_rect(rect)
frames_list.append(frame)
return frames_list
def track_min_objects(video_info):
previous_frame = None
previous_num_obj = -1
tracked_video = []
frame_id = 0
min_rects = []
min_frame_id = None
min_num_obj = None
for frame_info in video_info:
if (min_num_obj is None) & (len(frame_info.rects) > 0):
min_num_obj = len(frame_info.rects)
min_frame_id = frame_id
if (len(frame_info.rects) < min_num_obj) & (len(frame_info.rects) > 0):
min_num_obj = len(frame_info.rects)
min_frame_id = frame_id
frame_id = frame_id + 1
min_rects = multiclass_rectangle.duplicate_rects(
video_info[min_frame_id].rects)
("Min num object video:%d" % min_num_obj)
for frame_info in video_info:
print("Tracking Frame Nr: %d" % frame_info.frame)
print("Len Rects Frame: %d" % len(frame_info.rects))
current_frame = frame.Frame_Info()
current_frame = frame_info.duplicate()
current_frame.rects = []
if previous_frame is not None:
print("Min num object video:%d" % min_num_obj)
print("Len Previous Rects Frame: %d" % len(previous_frame.rects))
for rect in previous_frame.rects:
print(len(current_frame.rects))
if len(current_frame.rects) <= min_num_obj:
current_rect = multiclass_rectangle.pop_max_iou(
frame_info.rects, rect)
current_rect.load_trackID(rect.trackID)
current_frame.append_labeled_rect(current_rect)
else:
trackID = 1
for rect in min_rects:
if len(current_frame.rects) < min_num_obj:
current_rect = multiclass_rectangle.pop_max_iou(
frame_info.rects, rect)
current_rect.load_trackID(trackID)
current_frame.append_labeled_rect(current_rect)
trackID = trackID + 1
previous_frame = current_frame.duplicate()
previous_frame.rects = multiclass_rectangle.duplicate_rects(
current_frame.rects)
print("Current Frame obj:%d" % len(current_frame.rects))
tracked_video.insert(len(tracked_video), current_frame)
return tracked_video
def parse_XML_to_data(xml_list_video):
frames_list=[]
video_list=[]
# image_multi_class= None
# rectangle_multi = None
progress = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ',progressbar.Percentage(), ' ',progressbar.ETA()])
for i in progress(list(range(0, len(xml_list_video)))):
# print "Iterating on Video:"+ str(xml_list_video[i][0][0])
for j in range(0, len(xml_list_video[i])):
# print "Iterating on Frame:"+ str(xml_list_video[i][j][0])
with open(xml_list_video[i][j][0], 'rt') as f:
tree = ElementTree.parse(f)
for obj in tree.findall('object'):
name = obj.find('name').text
class_code= name
name = vid_classes.code_to_class_string(name)
if name in ["nothing"]:
continue
else:
#The files with the original data path are made in both: multiclass e single class
jump=0
image_multi_class= fm.Frame_Info()
image_multi_class.frame= xml_list_video[i][j][1]
# print image_multi_class.frame
rectangle_multi= multiclass_rectangle.Rectangle_Multiclass()
for node in tree.iter():
tag=str(node.tag)
if tag in ['name']:
if str(vid_classes.code_to_class_string(str(node.text))) in ["nothing"]:
jump = 1
else :
jump=0
rectangle_multi.label_chall=int(vid_classes.class_string_to_comp_code(str(vid_classes.code_to_class_string(str(node.text)))))
# print rectangle_multi.label_chall
rectangle_multi.label_code=str(node.text)
rectangle_multi.label=vid_classes.code_to_class_string(str(node.text))
if tag in ["xmax"]:
if jump == 0:
rectangle_multi.x2=float(node.text)
if tag in ["xmin"]:
if jump == 0:
rectangle_multi.x1=float(node.text)
if tag in ["ymax"]:
if jump == 0:
rectangle_multi.y2=float(node.text)
if tag in ["ymin"]:
if jump == 0:
rectangle_multi.y1=float(node.text)
image_multi_class.append_rect(rectangle_multi)
if jump == 0:
image_multi_class.append_labeled_rect(rectangle_multi)
break
frames_list.append(image_multi_class)
video_list.append(frames_list)
# frames_list=None
# frames_list=[]
return video_list
def track_objects(video_info):
previous_frame = None
previous_num_obj = -1
tracked_video = []
for frame_info in video_info:
print("Tracking Frame Nr: %d" % frame_info.frame)
print("Len Rects Frame: %d" % len(frame_info.rects))
current_frame = frame.Frame_Info()
current_frame = frame_info.duplicate()
current_frame.rects = []
if previous_frame is not None:
print("Len Previous Rects Frame: %d" % len(previous_frame.rects))
for rect in previous_frame.rects:
print(len(current_frame.rects))
current_rect = multiclass_rectangle.pop_max_iou(
frame_info.rects, rect)
current_rect.load_trackID(rect.trackID)
current_frame.append_labeled_rect(current_rect)
else:
trackID = 1
picked_rect = Utils_Tensorbox.NMS(frame_info.rects)
for rect in picked_rect:
current_rect = rect.duplicate()
current_rect.load_trackID(trackID)
current_frame.append_labeled_rect(current_rect)
trackID = trackID + 1
previous_frame = current_frame.duplicate()
previous_frame.rects = multiclass_rectangle.duplicate_rects(
current_frame.rects)
print("Current Frame obj:%d" % len(current_frame.rects))
tracked_video.insert(len(tracked_video), current_frame)
return tracked_video
def bbox_det_TENSORBOX_multiclass(frames_list, path_video_folder, hypes_file,
weights_file, pred_idl):
from train import build_forward
print("Starting DET Phase")
#### START TENSORBOX CODE ###
lenght = int(len(frames_list))
video_info = []
### Opening Hypes file for parameters
with open(hypes_file, 'r') as f:
H = json.load(f)
### Building Network
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32,
name='x_in',
shape=[H['image_height'], H['image_width'], 3])
if H['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
grid_area = H['grid_height'] * H['grid_width']
pred_confidences = tf.reshape(
tf.nn.softmax(
tf.reshape(pred_confs_deltas,
[grid_area * H['rnn_len'], H['num_classes']])),
[grid_area, H['rnn_len'], H['num_classes']])
pred_logits = tf.reshape(
tf.nn.softmax(
tf.reshape(pred_logits,
[grid_area * H['rnn_len'], H['num_classes']])),
[grid_area, H['rnn_len'], H['num_classes']])
if H['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(
sess, weights_file
) ##### Restore a Session of the Model to get weights and everything working
#### Starting Evaluating the images
print(("%d Frames to DET" % len(frames_list)))
progress = progressbar.ProgressBar(widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage(), ' ',
progressbar.ETA()
])
frameNr = 0
skipped = 0
for i in progress(list(range(0, len(frames_list)))):
current_frame = frame.Frame_Info()
current_frame.frame = frameNr
current_frame.filename = frames_list[i]
if utils_image.isnotBlack(
frames_list[i]) & utils_image.check_image_with_pil(
frames_list[i]):
img = imread(frames_list[i])
# test(frames_list[i])
feed = {x_in: img}
(np_pred_boxes, np_pred_logits,
np_pred_confidences) = sess.run(
[pred_boxes, pred_logits, pred_confidences],
feed_dict=feed)
_, rects = get_multiclass_rectangles(H,
np_pred_confidences,
np_pred_boxes,
rnn_len=H['rnn_len'])
if len(rects) > 0:
# pick = NMS(rects)
pick = rects
print((len(rects), len(pick)))
current_frame.rects = pick
frameNr = frameNr + 1
video_info.insert(len(video_info), current_frame)
print((len(current_frame.rects)))
else:
skipped = skipped + 1
else:
skipped = skipped + 1
print(("Skipped %d Black Frames" % skipped))
#### END TENSORBOX CODE ###
return video_info
def track_and_label_objects(video_info):
previous_frame = None
previous_num_obj = -1
cropped_img_array = []
tracked_video = []
for frame_info in video_info:
print("Tracking Frame Nr: %d" % frame_info.frame)
print(len(frame_info.rects))
current_frame = frame.Frame_Info()
current_frame = frame_info.duplicate()
current_frame.rects = []
print(len(frame_info.rects))
if previous_frame is not None:
print("Previous Frame obj:%d" % previous_num_obj)
for rect in frame_info.rects:
print("Entered into the rect check")
max_rect = None
max_iou = 0
current_rect = Rectangle_Multiclass()
trackID = -1
if previous_num_obj > 0: ### If i come here means that there's the same number of object between the previous and the current frame
print("Entered into the rect check with :%d objects" %
previous_num_obj)
id_rect = 0
max_id = 0
for prev_rect in previous_frame.rects:
print("Entered")
if rect.iou(prev_rect) > max_iou:
max_iou = rect.iou(prev_rect)
max_id = id_rect
id_rect = id_rect + 1
print("Lenght previous rects array: %d" %
len(previous_frame.rects))
print("max_rect track ID: %d" %
previous_frame.rects[max_id].trackID)
print("max_rect label: %s" %
previous_frame.rects[max_id].label)
current_rect.load_labeled_rect(
previous_frame.rects[max_id].trackID,
previous_frame.rects[max_id].true_confidence,
previous_frame.rects[max_id].label_confidence,
previous_frame.rects[max_id].x1,
previous_frame.rects[max_id].y1,
previous_frame.rects[max_id].x2,
previous_frame.rects[max_id].y2,
previous_frame.rects[max_id].label,
previous_frame.rects[max_id].label_chall,
previous_frame.rects[max_id].label_code)
current_frame.append_labeled_rect(current_rect)
rect.load_label(
previous_frame.rects[max_id].trackID,
previous_frame.rects[max_id].label_confidence,
previous_frame.rects[max_id].label,
previous_frame.rects[max_id].label_chall,
previous_frame.rects[max_id].label_code)
previous_frame.rects.pop(max_id)
previous_num_obj = previous_num_obj - 1
else:
### If i come here means that there's more objects in the current frame respect to che previous
if previous_num_obj == 0:
trackID = len(frame_info.rects)
previous_num_obj = -1
current_rect = Rectangle_Multiclass()
img = Image.open(frame_info.filename)
cor = (rect.x1, rect.y1, rect.x2, rect.y2)
cropped_img = img.crop(cor)
cropped_img_name = "cropped_frame_%d.JPEG" % (
frame_info.frame)
cropped_img.save(cropped_img_name)
cropped_img_array.append(cropped_img_name)
label, confidence = Utils_Imagenet.run_inception_once(
cropped_img_name)
rect.load_label(
trackID, confidence,
vid_classes.code_to_class_string(label),
vid_classes.code_to_code_chall(vid_classes), label)
current_rect.load_labeled_rect(
trackID, rect.true_confidence, confidence, rect.x1,
rect.y1, rect.x2, rect.y2,
vid_classes.code_to_class_string(label),
vid_classes.code_to_code_chall(vid_classes), label)
print("current_rect track ID: %d" % current_rect.trackID)
print("current_rect label: %s" % current_rect.label)
current_frame.append_labeled_rect(current_rect)
else:
trackID = 1
for rect in frame_info.rects:
current_rect = Rectangle_Multiclass()
img = Image.open(frame_info.filename)
cor = (rect.x1, rect.y1, rect.x2, rect.y2)
cropped_img = img.crop(cor)
cropped_img_name = "cropped_frame_%d.JPEG" % (frame_info.frame)
cropped_img.save(cropped_img_name)
cropped_img_array.append(cropped_img_name)
label, confidence = Utils_Imagenet.run_inception_once(
cropped_img_name)
rect.load_label(trackID, confidence,
vid_classes.code_to_class_string(label),
vid_classes.code_to_code_chall(vid_classes),
label)
current_rect.load_labeled_rect(
trackID, rect.true_confidence, confidence, rect.x1,
rect.y1, rect.x2, rect.y2,
vid_classes.code_to_class_string(label),
vid_classes.code_to_code_chall(vid_classes), label)
current_frame.append_labeled_rect(current_rect)
trackID = trackID + 1
previous_num_obj = len(frame_info.rects)
previous_frame = frame_info.duplicate()
previous_frame.duplicate_rects(frame_info.rects)
print(previous_frame)
print("Previous Frame obj:%d" % previous_num_obj)
print("prev_rect 0 track ID: %d" % previous_frame.rects[0].trackID)
print("prev_rect 0 label: %s" % previous_frame.rects[0].label)
tracked_video.insert(len(tracked_video), current_frame)
return tracked_video
def recurrent_track_objects(video_info):
previous_frame = None
previous_num_obj = -1
tracked_video = []
deltas_video = []
deltas_frame = []
dx1, dx2, dy1, dy2 = 0, 0, 0, 0
for frame_info in video_info:
print("Tracking Frame Nr: %d" % frame_info.frame)
print("Len Rects Frame: %d" % len(frame_info.rects))
current_frame = frame.Frame_Info()
current_frame = frame_info.duplicate()
current_frame.rects = []
if previous_frame is not None:
deltas_frame = []
if frame_info.frame > 1:
print("Len Previous Rects Frame: %d" %
len(previous_frame.rects))
rect_idx = 0
for rect in previous_frame.rects:
print(len(current_frame.rects))
rect.add_delta(
deltas_video[frame_info.frame - 2][rect_idx][0],
deltas_video[frame_info.frame - 2][rect_idx][1],
deltas_video[frame_info.frame - 2][rect_idx][2],
deltas_video[frame_info.frame - 2][rect_idx][3])
current_rect = multiclass_rectangle.pop_max_iou(
frame_info.rects, rect)
if current_rect is not None:
current_rect.load_trackID(rect.trackID)
current_rect.check_rects_motion(
frame_info.filename, rect,
deltas_video[frame_info.frame - 2][rect_idx][0],
deltas_video[frame_info.frame - 2][rect_idx][1],
deltas_video[frame_info.frame - 2][rect_idx][2],
deltas_video[frame_info.frame - 2][rect_idx][3])
current_frame.append_labeled_rect(current_rect)
dx1 = current_rect.x1 - rect.x1
dx2 = current_rect.x2 - rect.x2
dy1 = current_rect.y1 - rect.y1
dy2 = current_rect.y2 - rect.y2
deltas_frame.append((dx1, dx2, dy1, dy2))
else:
break
else:
print("Len Previous Rects Frame: %d" %
len(previous_frame.rects))
for rect in previous_frame.rects:
print(len(current_frame.rects))
current_rect = multiclass_rectangle.pop_max_iou(
frame_info.rects, rect)
if current_rect is not None:
dx1 = current_rect.x1 - rect.x1
dx2 = current_rect.x2 - rect.x2
dy1 = current_rect.y1 - rect.y1
dy2 = current_rect.y2 - rect.y2
deltas_frame.append((dx1, dx2, dy1, dy2))
current_rect.load_trackID(rect.trackID)
current_frame.append_labeled_rect(current_rect)
else:
break
deltas_video.append(deltas_frame)
else:
trackID = 1
picked_rect = Utils_Tensorbox.NMS(frame_info.rects)
for rect in picked_rect:
current_rect = rect.duplicate()
current_rect.load_trackID(trackID)
current_frame.append_labeled_rect(current_rect)
trackID = trackID + 1
previous_frame = current_frame.duplicate()
previous_frame.rects = multiclass_rectangle.duplicate_rects(
current_frame.rects)
print("Current Frame obj:%d" % len(current_frame.rects))
tracked_video.insert(len(tracked_video), current_frame)
return tracked_video
