def extract_real_labelled_plates():
output_folder = real_labelled_plates_folder
name_to_sequence = {}
for folder in db.get_folders():
for frame in folder.frames:
if not frame.has_run_rfcn(): continue
lazy_img = LazyImage(frame.absolute_path())
for car_bbox in frame.parts:
if car_bbox.name != 'car': continue
for plate_bbox in car_bbox.parts:
if plate_bbox.name != 'plate': continue
if not plate_bbox.auto_sequence_confirmed(): continue
auto_sequence = plate_bbox.auto_sequence()
""" compute crop boundary """
img = lazy_img.get()
x1 = clip(min([x for x,y,char in auto_sequence]) - 100, 0, img.shape[1])
y1 = clip(min([y for x,y,char in auto_sequence]) - 100, 0, img.shape[0])
x2 = clip(max([x for x,y,char in auto_sequence]) + 100, 0, img.shape[1])
y2 = clip(max([y for x,y,char in auto_sequence]) + 100, 0, img.shape[0])
new_sequence = [(x-x1,y-y1, char) for x,y,char in auto_sequence]
img_crop = img[y1:y2, x1:x2]
imname = plate_bbox.unique_name(with_jpg=True)
cv2.imwrite(os.path.join(output_folder, imname), img_crop)
name_to_sequence[imname] = new_sequence
cPickle.dump(name_to_sequence, open(os.path.join(output_folder, 'name_to_sequence.pkl'), 'wb'))
def get_good_crop(car_bbox, plate_bbox, img):
"""
Takes a reasonably sized crop of the plate
- 1-line plate: horozontally expand 0.3 randomly, vertically 1/3 of that
- 2-line plate: vertically expand 0.3 randomly, horizontally 3x of that
"""
assert isinstance(car_bbox, db2.BBox)
assert isinstance(plate_bbox, db2.BBox)
# cx,cy,cw,ch = car_bbox.xywh()
px, py, pw, ph = plate_bbox.xywh()
centerx = px + pw / 2
centery = py + ph / 2
# determine crop size
H_target = make_multiple(ph * 1.5, 24)
W_target = make_multiple(pw * 1.5, 24)
"""
is_2line = (pw / float(ph)) < 2.4
if is_2line: # 2 line
H_target = make_multiple(int(ph * 1.3), 24)
W_target = make_multiple(3 * H_target, 24)
assert H_target >= ph
assert W_target >= pw
else: # one line
W_target = int(pw * 1.3)
H_target = int(W_target / 3)
assert H_target >= ph
assert W_target >= pw
"""
# choose a center crop
x1 = centerx - W_target / 2
x2 = centerx + W_target / 2
y1 = centery - H_target / 2
y2 = centery + H_target / 2
x1 = clip(x1, 0, img.shape[1])
x2 = clip(x2, 0, img.shape[1])
y1 = clip(y1, 0, img.shape[0])
y2 = clip(y2, 0, img.shape[0])
crop = img[y1:y2, x1:x2]
return crop
def __getitem__(self, i):
"""
1. Load image
2. Take a random crop
3. Perform other augmentation
4. Generate label
"""
global black_white
fname, bboxes = self.crops_info[i]
# load the image in BGR order, and keep it this way
if black_white:
img_np = cv2.imread(fname, 0)
H, W = img_np.shape[:2]
img_np = img_np.reshape(H, W, 1) # (H,W) -> (H,W,1)
else:
img_np = cv2.imread(fname)
H, W = img_np.shape[:2]
""" taking a random crop from the image """
if True: # crop it
paste_y = 0
""" the case where all we have is a narrow, frontal view of the car """
if W / float(H) > 1.6:
crop_len_W = int(W * 0.75)
crop_len_H = H
crop_x = random.randint(0, W - crop_len_W)
crop_y = 0
paste_y = random.randint(0, crop_len_W - crop_len_H)
""" create a square image, and paste the region we want into it """
img_crop = np.zeros((crop_len_W, crop_len_W, num_input_cn),
np.uint8)
img_crop[paste_y:paste_y+crop_len_H, :crop_len_W, :] = \
img_np[crop_y:crop_y+crop_len_H, crop_x:crop_x+crop_len_W, :]
else:
min_side = min(H, W)
max_side = max(H, W)
if max_side / float(min_side) > 1.5:
crop_len = min_side
else:
crop_len = int(min_side * random.uniform(0.6, 0.8))
crop_x = random.randint(0, W - crop_len)
crop_y = random.randint(0, H - crop_len)
img_crop = img_np[crop_y:crop_y + crop_len,
crop_x:crop_x + crop_len, :]
""" other augmentation """
if not black_white and random.randint(0, 1): # swap channels
img_crop = swap_channels(img_crop)
"""
if black_white and random.randint(0,1): # invert image
img_crop = 255 - img_crop
"""
if random.randint(0, 1): # normalize
img_crop = normalize_image(img_crop)
flip = random.randint(0, 1)
if flip: # horizontal flip
img_crop = cv2.flip(img_crop, 1)
if black_white: img_crop = np.expand_dims(img_crop, 2)
if random.uniform(0, 1) < 0: # mess up contrast
img_crop = mess_contrast(img_crop, (0.8, 1.4), (-40, 40))
band = random.randint(0, 0) # FIXME DISABLED FOR NOW
if band: # add black band to bottom-right
band_width, band_height = add_black_band(img_crop)
# resize to input_H by input_W
factor_Y = input_H / float(img_crop.shape[0])
factor_X = input_W / float(img_crop.shape[1])
img_resized = cv2.resize(img_crop, (input_W, input_H))
if black_white:
img_resized = img_resized.reshape(input_W, input_H, 1)
img = torch.from_numpy(img_resized.transpose([2, 0, 1])).float()
label = torch.zeros(1, label_H, label_W).float()
""" Generate heat label """
for bbox in bboxes:
x1, y1, x2, y2 = bbox
x1 = clip(int(round((x1 - crop_x) * factor_X / downsample_times)),
0, label_W)
y1 = clip(
int(
round((y1 - crop_y + paste_y) * factor_Y /
downsample_times)), 0, label_H)
x2 = clip(int(round((x2 - crop_x) * factor_X / downsample_times)),
0, label_W)
y2 = clip(
int(
round((y2 - crop_y + paste_y) * factor_Y /
downsample_times)), 0, label_H)
if flip:
x1, x2 = label_W - x2, label_W - x1
if x1 == x2 or y1 == y2: continue # out-of-crop boxes
label[0, y1:y2, x1:x2] = 1
if band:
label[0,
-int(round(band_height / float(downsample_times))):, :] = 0
label[0, :, -int(round(band_width / float(downsample_times))):] = 0
return img, label
def output_bigcrops_for_training(output_folder):
"""
Generate the training set for training localizer models.
"""
assert os.path.exists(output_folder)
crops_info = []
for folder in db.get_folders():
for frame in folder.frames:
""" watch for both manual and auto plates when we add heat"""
all_plates = [
p for c in frame.parts for p in c.parts if
p.typename == 'plate' and p.label_type() in ['auto', 'manual']
]
plate_bboxes = [p.bbox for p in all_plates]
lazy_img = LazyImage(frame.absolute_path())
for car_bbox in frame.parts:
if len(car_bbox.parts) == 0: continue
assert len(car_bbox.parts) == 1
plate_bbox = car_bbox.parts[0]
assert isinstance(plate_bbox, db2.BBox)
assert plate_bbox.typename == 'plate'
if plate_bbox.label_type() == 'auto' and random.uniform(
0, 1) < 0.7:
continue
if plate_bbox.label_type() == 'none' and random.uniform(
0, 1) < 0.5:
continue
"""
this sample can be used for training if it is manually labelled. Both 'manual' and 'none' are manually
labelled. 'none' just means this crop doesn't have a plate
"""
assert plate_bbox.label_type() in ['manual', 'none', 'auto']
img = lazy_img.get()
img_H = img.shape[0]
img_W = img.shape[1]
""" expand the bounding box and crop """
x1, y1, x2, y2 = car_bbox.bbox
width = x2 - x1
height = y2 - y1
width_inc = width / 2
height_inc = height / 2
x1_new = huva.clip(x1 - width_inc, 0, img_W)
y1_new = huva.clip(y1 - height_inc, 0, img_H)
x2_new = huva.clip(x2 + width_inc, 0, img_W)
y2_new = huva.clip(y2 + height_inc, 0, img_H)
width_new = x2_new - x1_new
height_new = y2_new - y1_new
img_crop = img[y1_new:y2_new, x1_new:x2_new, :].copy()
""" find all plate bbox that lay within the new crop """
overlaps = rfcn.get_overlapping_bboxes(
[x1_new, y1_new, x2_new, y2_new], plate_bboxes)
heatboxes = []
for overlap in overlaps:
x1_o, y1_o, x2_o, y2_o = overlap
x1_o = clip(x1_o, 0, width_new)
y1_o = clip(y1_o, 0, height_new)
x2_o = clip(x2_o, 0, width_new)
y2_o = clip(y2_o, 0, height_new)
heatboxes.append((x1_o, y1_o, x2_o, y2_o))
""" wrap up """
save_path = os.path.join(output_folder,
car_bbox.unique_name(with_jpg=True))
cv2.imwrite(save_path, img_crop)
crops_info.append((save_path, heatboxes))
path_crops_info = os.path.join(output_folder, 'crops_info.pkl')
cPickle.dump(crops_info, open(path_crops_info, 'wb'))
def auto_label_frame(frame,
model_name,
threshold=0.7,
mode='display',
skip_existing=True):
"""
1. load the image
2. for each cbi, take a corresponding crop and feed through model
- run the output through cv2.findContours then cv2.boundingRect
- show the boundingRect is the contour area is greater than some value
On cropping:
instead of cropping only the tight bounding box, we can crop an extended bounding box, then remove any heat
generated from outside the tight bounding box during contour generation. This allows us to input data in a way
similar to training data, without getting labels from non-bb'ed cars
mode is either 'display' or 'label'
- 'display': plt.imshow the auto-detected labels
- 'label'; directly add the detected label to corresponding cbi
"""
model.eval()
img = cv2.imread(frame.absolute_path())
H, W = img.shape[0], img.shape[1]
num_cars = 0
num_detected = 0
# for every car box, crop
for i, car_bbox in enumerate(frame.parts):
assert isinstance(car_bbox, db2.BBox)
assert car_bbox.typename == 'car'
x1, y1, x2, y2 = car_bbox.bbox
""" skip all car_bbox that already have a plate label """
plate_bboxes = car_bbox.get_typed_parts('plate')
if skip_existing and len(plate_bboxes) > 0:
print('{} already has {} plate_bbox'.format(
car_bbox.unique_name(), len(plate_bboxes)))
continue
num_cars += 1
""" Expand the bounding bbox """
if True:
x_inc = (x2 - x1) * 0.3
y_inc = (y2 - y1) * 0.3
x1_new = huva.clip(int(x1 - x_inc), 0, W)
y1_new = huva.clip(int(y1 - y_inc), 0, H)
x2_new = huva.clip(int(x2 + x_inc), 0, W)
y2_new = huva.clip(int(y2 + y_inc), 0, H)
img_crop = img[y1_new:y2_new, x1_new:x2_new]
width_new = x2_new - x1_new
height_new = y2_new - y1_new
crop_H, crop_W = img_crop.shape[0], img_crop.shape[1]
max_side = max(crop_H, crop_W)
""" Create heat mask """
x1_lab = int(label_W * (float(x1 - x1_new) / max_side))
y1_lab = int(label_H * (float(y1 - y1_new) / max_side))
x2_lab = int(label_W * (float(x2 - x1_new) / max_side))
y2_lab = int(label_H * (float(y2 - y1_new) / max_side))
label_mask = np.zeros((label_H, label_W), np.float32)
label_mask[y1_lab:y2_lab, x1_lab:x2_lab] = 1
else:
img_crop = img[y1:y2, x1:x2]
crop_H, crop_W = img_crop.shape[0], img_crop.shape[1]
max_side = max(crop_H, crop_W)
factor = input_W / float(max_side)
resized_H = int(crop_H * factor)
resized_W = int(crop_W * factor)
img_resized = cv2.resize(img_crop, (resized_W, resized_H))
img_input = np.zeros((input_H, input_W, 3), np.float32)
img_input[:resized_H, :resized_W] = img_resized
# produce input
if black_white:
img_input = np.expand_dims(
cv2.cvtColor(img_input, cv2.COLOR_BGR2GRAY), 2)
imgs = torch.from_numpy(img_input.transpose(
[2, 0, 1])).contiguous().view(1, num_input_cn, input_H, input_W)
v_imgs = Variable(imgs - (
mean_bw if black_white else mean_bgr).expand_as(imgs)).cuda()
v_outs = model(v_imgs)
output = v_outs.data.cpu()
out = output[0].numpy().reshape(label_H, label_W)
# mask the non-bb region of the heat output
out = out * label_mask
threshed = out > threshold
"""
jet = get_jet(img_input.astype(np.uint8), cv2.resize(label_mask, (img_input.shape[0], img_input.shape[1])))
plt.imshow(jet)
plt.show()
"""
contours, hierarchy = cv2.findContours(threshed.astype(np.uint8), 1, 2)
for j, cnt in enumerate(contours):
area = cv2.contourArea(cnt)
if mode == 'display':
print("area: {}".format(area))
if area < 5: continue
x, y, w, h = cv2.boundingRect(cnt)
x1_det, y1_det, x2_det, y2_det = x, y, x + w, y + h
label = torch.zeros(1, label_H, label_W)
label[0, y1_det:y2_det, x1_det:x2_det] = 1
if mode == 'display':
jet = th_get_jet(imgs[0], label)
plt.imshow(jet)
print('area: {}'.format(area))
plt.show()
elif mode == 'count': #count stuffs
num_detected += 1
elif mode == 'label': # label mode
"""
recover x1_det and so on to the full-frame coordinate system
1. multiply by downsample_times/factor to get back to a coordinate with origin at (x1_new, y1_new)
2. add (x1_new, y1_new) to recover the origin at (0,0)
"""
x1_rec = int(
round((x1_det * downsample_times / factor) + x1_new))
y1_rec = int(
round((y1_det * downsample_times / factor) + y1_new))
x2_rec = int(
round((x2_det * downsample_times / factor) + x1_new))
y2_rec = int(
round((y2_det * downsample_times / factor) + y1_new))
""" Now, put the bloody label on the shit """
print(x1_det, y1_det, x2_det, y2_det)
print(x1_rec, y1_rec, x2_rec, y2_rec)
plate_bbox = db2.BBox(car_bbox, 'plate',
(x1_rec, y1_rec, x2_rec, y2_rec))
plate_bbox.label_type('auto')
plate_bbox.auto_model(model_name)
car_bbox.add_part(plate_bbox)
else:
assert False, 'unknown mode {}'.format(mode)
# for every cbi we only detect on bbox, so break directly
break
if mode == 'display':
jet = th_get_jet(imgs[0], output[0])
""" draw the bbox of the car """
x1_b = int(round((x1 - x1_new) * factor))
y1_b = int(round((y1 - y1_new) * factor))
x2_b = int(round((x2 - x1_new) * factor))
y2_b = int(round((y2 - y1_new) * factor))
jet = jet.copy()
cv2.rectangle(jet, (x1_b, y1_b), (x2_b, y2_b), (0, 0, 255))
plt.imshow(jet)
print('max: {}'.format(out.max()))
plt.show()
if mode == 'count':
return num_cars, num_detected
def sequence_demo(show=False, show_small=False, write_out=False, draw_mispredict=False, mode='detect'):
"""
1. For every image in output_root/with_plates/*.jpg
2. Get all the plate proposal regions from name_to_boxes.pkl
3. For each proposed region, try to find a license sequence
4. If found, draw it onto the image
5. Output the annotated image to output_root/with_numbers_2/*.jpg
6. Also count number of proposals and numbers of plates detected
-- model6: 17466 / 10302 (margin=0.15, base=60, target_range=135)
- (0.35, 0.15), (110)
-- model8: 17466 / 12179 (margin=0.15, base=60, target_range=135)
- (0.35, 0.15), (110)
"""
output_root = demo_root
raw_filenames = sorted(glob(output_root+'/raw/*.jpg'))
name_to_boxes = cPickle.load(open(os.path.join(output_root, 'with_plates', 'name_to_boxes.pkl')))
short_names = sorted(name_to_boxes.keys())
""" Count (number of plate proposals, number of plates detected) """
if mode=='detect':
num_plates_proposed = 0
num_plates_detected = 0
elif mode=='integrate':
fidx_to_car_insts = []
integrator = LPRFrameIntegrator()
for fidx, short_name in enumerate(short_names):
img_color = cv2.imread(os.path.join(output_root, 'raw', short_name))
img = cv2.imread(os.path.join(output_root, 'raw', short_name), 0)
boxes = name_to_boxes[short_name]
if mode=='integrate':
car_insts = [] # [(x1,y1,x2,y2), car_info] for LPRFrameIntegrator
for box in boxes:
x1,y1,x2,y2 = box
w = x2 - x1
h = y2 - y1
""" expand the crop box """
w_ratio = 0.15
h_ratio = w_ratio
if w/h < 2:
w_ratio = 0.35
h_ratio = 0.15
x1 = clip(int(x1 - w*w_ratio), 0, img.shape[1])
x2 = clip(int(x2 + w*w_ratio), 0, img.shape[1])
y1 = clip(int(y1 - h*h_ratio), 0, img.shape[0])
y2 = clip(int(y2 + h*h_ratio), 0, img.shape[0])
w = x2 - x1
h = y2 - y1
if w<10 or h < 10: continue
""" crop """
plate_crop = img[y1:y2, x1:x2]
if show_small:
print('plate_crop')
plt.imshow(plate_crop, cmap='gray');
plt.show()
""" resize crop """
plate_crop_scaled = get_scaled_crop(plate_crop)
if show_small:
print('plate_crop_scaled')
plt.imshow(plate_crop_scaled[:,:,0], cmap='gray');
plt.show()
""" enhance crop """
plate_crop_scaled = plate_crop_scaled.astype(np.float32)
plate_crop_scaled = change_contrast(plate_crop_scaled, 60, 135)
""" pass to model and get sequence """
heatmaps = pass_imgs(np.expand_dims(plate_crop_scaled, 0)).numpy()
sequences, filtered = infer_sequences(heatmaps[0])
""" mode-specific stuffs """
if mode=='detect':
num_plates_proposed += 1
if write_out:
cv2.rectangle(img_color, (x1,y1), (x2,y2), (0,0,255), 3)
if len(filtered) != 0:
num_plates_detected += 1
str_seq = filtered[0].get_str_seq()
if write_out:
cv2.putText(img_color, str_seq, (x1,y1-5), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 3)
elif write_out and draw_mispredict and len(sequences)>0:
str_seq = sequences[0].get_str_seq()
cv2.putText(img_color, str_seq, (x1,y1-5), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 3)
elif mode=='integrate':
x = (x1+x2)/2
y = (y1+y2)/2
# find car_info of previous car
if fidx==0: # just create new car
car_info = CarInfo()
else:
prev_car_instance = find_nearest_car(box, fidx_to_car_insts[fidx-1])
if prev_car_instance is None:
car_info = CarInfo()
else:
car_info = prev_car_instance[1]
if len(filtered):
car_info.add_plate(box, filtered[0].get_str_seq(), filtered[0].prob)
car_instance = (box, car_info)
car_insts.append(car_instance)
print(short_name)
if mode=='detect' and write_out:
out_path = os.path.join(output_root, 'with_numbers_2', short_name)
cv2.imwrite(out_path, img_color)
elif mode=='integrate':
fidx_to_car_insts.append(car_insts)
integrator.add_frame(short_name, img_color, car_insts)
else:
assert False, 'unknown mode={}'.format(mode)
if mode=='detect':
print(num_plates_proposed, num_plates_detected)
elif mode=='integrate':
integrator.flush()
def get_real_labelled_sample(show=False):
length = len(name_to_sequence)
idx = random.randint(0, length-1)
imname = name_to_sequence.keys()[idx]
sequence = name_to_sequence[imname]
img = cv2.imread(os.path.join(real_labelled_plates_folder, imname), 0)
""" (xmin,ymin) (xmax,ymax) is the minimum region that must be included in the crop """
xmin = clip(min([x for x,y,char in sequence]) - 10, 0, img.shape[1])
ymin = clip(min([y for x,y,char in sequence]) - 10, 0, img.shape[0])
xmax = clip(max([x for x,y,char in sequence]) + 10, 0, img.shape[1])
ymax = clip(max([y for x,y,char in sequence]) + 10, 0, img.shape[0])
"""
compute crop region
left-extend random amount, right-extend random amount, clip
divide width by 3 to get height
"""
x1 = clip(xmin - random.randint(5,int((xmax-xmin)*0.5)), 0, img.shape[1])
x2 = clip(xmax + random.randint(5,int((xmax-xmin)*0.5)), 0, img.shape[1])
width = x2 - x1
height = int(round(width / 3.0))
""" if this is a 2-line plate, it's possible that height < ymax-ymin """
if height < ymax - ymin:
y1 = clip(ymin - random.randint(5, int((ymax-ymin)*0.5)), 0, img.shape[0])
y2 = clip(ymax + random.randint(5, int((ymax-ymin)*0.5)), 0, img.shape[0])
height = y2-y1
width = int(round(height * 3.0))
while width < xmax - xmin:
print(width, xmax, xmin, img.shape)
width = xmax - xmin
x1 = xmin - random.randint(0, width - (xmax - xmin))
x2 = x1 + width
else:
y1 = ymin - random.randint(0, height - (ymax - ymin))
y2 = y1 + height
x1 = clip(x1, 0, img.shape[1])
y1 = clip(y1, 0, img.shape[0])
x2 = clip(x2, 0, img.shape[1])
y2 = clip(y2, 0, img.shape[0])
img_crop = img[y1:y2, x1:x2]
height, width = img_crop.shape[:2]
""" resize image """
factor_x = input_W / float(width)
factor_y = input_H / float(height)
new_sequence = [(int((x-x1)*factor_x),int((y-y1)*factor_y),char) for x,y,char in sequence]
img_resized = cv2.resize(img_crop, (input_W, input_H))
""" heat it """
label_np = np.zeros((num_classes, label_H, label_W), np.float32)
ss = heat_radius
for x,y,char in new_sequence:
charid = char_to_charid(char)
label_np[charid, y-ss:y+ss+1, x-ss:x+ss+1] = 1.0
label_np[num_classes-1] = 1 - label_np.max(0)
""" visualize if asked """
if show:
for x,y,char in new_sequence:
img_resized[y-ss:y+ss+1, x-ss:x+ss+1] = 255
plt.imshow(img_resized)
plt.show()
label = torch.from_numpy(label_np)
""" some augmentation """
if random.randint(0,1):
img_resized = apply_clahe(img_resized)
img = torch.from_numpy(np.expand_dims(img_resized, 0)).float()
if random.randint(0,1):
img = mess_contrast(img)
return img, label