def show(self, item=None):
ultility.show(self.org)
if self.success:
plt.gca().add_patch(
patches.Arrow(*self.back_point,
*(self.front_point - self.back_point),
width=25,
color='w'))
def test():
src = ['board-images-new/image{}.jpg'.format(i) for i in range(180, 200)]
for img in src:
plt.cla()
image = cv.imread(img)
ultility.show(image)
positions = predict(image)
plt.scatter(positions[:, 0], positions[:, 1])
plt.pause(1)
def predict(image, debug=False):
w, h = image.shape[1], image.shape[0]
edges = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
edges = cv.medianBlur(edges, 3) # median blur to avoid dust near a corner
image = Image.fromarray(np.uint8(image[:, :, ::-1]))
x = torch.unsqueeze(transform(image), 0)
model.eval() # set model to evaluation mode
with torch.no_grad():
x = x.to(device=device, dtype=dtype) # move to device, e.g. GPU
scores = model(x)
positions = np.reshape(scores.cpu().numpy(), (4, 2))
positions = inv_parameterize(positions, w, h)
positions = positions.astype(np.int16)
if debug:
print('corners detected:')
print(positions)
print('corners refined offsets:')
corner_images = []
corner_pos = []
for i in range(4):
s = 20
corner = cv.goodFeaturesToTrack(
edges[max(positions[i, 1] - s, 0):min(positions[i, 1] + s, h),
max(positions[i, 0] - s, 0):min(positions[i, 0] + s, w)],
1, 5e-2, 0)
if debug:
corner_images.append(
edges[max(positions[i, 1] -
s, 0):min(positions[i, 1] + s, h),
max(positions[i, 0] -
s, 0):min(positions[i, 0] + s, w)])
corner_pos.append(corner[0].reshape(-1))
if corner is not None:
print(corner - s)
positions[i] += np.int16(corner[0]).reshape(-1) - s
if debug:
fig = plt.Figure()
ultility.show(np.concatenate(corner_images[:2], axis=0),
np.concatenate(corner_images[:1:-1], axis=0))
corner_pos = np.array(corner_pos)
corner_pos[2:, 0] += 2 * s
corner_pos[1:3, 1] += 2 * s
plt.scatter(corner_pos[:, 0], corner_pos[:, 1])
plt.show()
return positions
def __call__(self, img, debug=False):
W, H = img.shape[1], img.shape[0]
cx = max(min(self.x, W - self.r - 1), self.r)
cy = max(min(self.y, H - self.r - 1), self.r)
if debug:
ultility.show(img)
roi = img[cy - self.r:cy + self.r + 1, cx - self.r:cx + self.r + 1]
corner = cv.goodFeaturesToTrack(roi, 1, 0.5, 0)
if corner is not None:
corner = np.int16(corner[0]).reshape(-1) - self.r
self.x = cx + corner[0]
self.y = cy + corner[1]
return self.x, self.y
def read(debug=False):
global cap, positions
success = False
tic = time.time()
for t in range(frames_per_read):
success, frame = cap.read(0)
if not success: return -1
toc = time.time()
print('Read a frame used {:.2f} s'.format(toc-tic))
if not fix_camera:
positions = match.match(frame)
img = warp_perspective(frame, positions, perspective+shift, size+2*shift)
plt.cla()
coords = track.track(img, debug=debug)
if not debug: # show the origin image if not in debug mode
ultility.show(img, frame)
ultility.plot(vertices)
plt.pause(interval)
return coords
def main():
cap = cv.VideoCapture('demo_large.mp4')
ret, frame = cap.read(0)
positions = np.array([[135.88205, 123.48814], [116.24813, 371.36624],
[535.10486, 368.09393], [501.5636, 114.489265]],
dtype=np.float32)
#positions = np.array([[52.576714, 69.34139], [23.18219, 432.99365],
# [634.5883, 426.2749], [585.03754, 56.74374]],
# dtype=np.float32)
width = 297
height = 210
scale = 2
width = int(width * scale)
height = int(height * scale)
perspective = np.array(((0, 0), (0, height), (width, height), (width, 0)),
dtype=np.float32)
def warp_perspective(image):
image = cv.warpPerspective(
image, cv.getPerspectiveTransform(positions, perspective),
(width, height))
return image
img = warp_perspective(frame)
img = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
init(frame, positions)
while True:
success, frame = cap.read(0)
if not success:
break
plt.cla()
positions = match(frame, False)
ultility.show(frame)
positions = np.concatenate([positions, positions[0:1, :]])
plt.plot(positions[:, 0], positions[:, 1])
plt.pause(0.03)
def extract(frame, debug=False):
"""
Extract the path in an image.
Inputs:
- frame: The input image.
Returns:
- vertices: The nodes of the path in order,
though we don't care whether the first node is the start or the end.
Each node is reporesented as a list [x, y]
"""
# put the image from BGR to gray
edges = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
# ignore a boundary
color = 0 if use_adaptive_threshold else 255
edges[:margin, :] = color
edges[-margin:, :] = color
edges[:, :margin] = color
edges[:, -margin:] = color
# transform into binary image
if use_adaptive_threshold:
edges = cv.medianBlur(edges, 11)
edges = cv.adaptiveThreshold(edges, 255, cv.ADAPTIVE_THRESH_MEAN_C,
cv.THRESH_BINARY, 41, 4)
else:
threshold = k_means_clustering(edges)['threshold']
_, edges = cv.threshold(edges, threshold, 255, cv.THRESH_BINARY)
# make the image thinner a little bit
edges = cv.dilate(
edges,
cv.getStructuringElement(cv.MORPH_CROSS, (s * 2 + 1, ) * 2, (s, ) * 2))
# make the image as thin as possible (until refine(edges) returns 0)
#while refine(edges) > end_refine: pass
# detect lines roughly with Hough transformation
lines = cv.HoughLinesP(255 - edges, 1, np.pi / 360, 20, None, 30, 20)
lines = [l[0] for l in lines]
# show the binarization and Hough transform result for a while
ultility.show(edges)
for line in lines:
plt.plot(line[::2], line[1::2])
plt.pause(1)
if debug:
print('image refined | lines detected | lines rearranged')
ultility.show(edges)
for line in lines:
plt.plot(frame.shape[1] + line[::2], line[1::2])
# rearrange the lines and detect the nodes of the path.
arrange(lines)
if debug:
for line in lines:
plt.plot(frame.shape[1] * 2 + line[::2], line[1::2])
plt.show()
vertices = nodes(lines)
# add the middle point of each segments
new = []
for i in range(len(vertices) - 1):
new.append(vertices[i])
new.append(((vertices[i][0] + vertices[i + 1][0]) / 2,
(vertices[i][1] + vertices[i + 1][1]) / 2))
new.append(vertices[-1])
vertices = new
# ensure that the start point is in the left
if vertices[0][0] > vertices[-1][0]:
vertices.reverse()
return vertices
def init(debug=False):
global lan, address, positions, cap, vertices
track.init()
if not manual_transform:
localization.init()
if load_from_video:
cap = cv.VideoCapture('demo_large.mp4')
#image = cv.imread('./board-images-new/image199.jpg')
else:
# string = input('Please check if the LAN address ({}) is correct \
# (press ENTER to confirm or type a new address):'\
# .format(lan))
# if string != '':
# lan = string
address = 'http://{}:{}@{}/video'.format(username, password, lan)
cap = cv.VideoCapture(address)
if cap.isOpened():
print('Connected to the IP camera sccessfully.')
else:
print('IP camera not available. Please check the settings.')
os._exit(0)
_, image = cap.read(0)
if manual_transform or debug:
plt.ion()
fig = plt.figure()
print('Original image:')
ultility.show(image)
plt.show()
positions = []
if manual_transform:
def on_click(e):
if len(positions) == 4: return
if e.button == 1:
positions.append((e.xdata, e.ydata))
plt.scatter(e.xdata, e.ydata)
plt.gca().figure.canvas.draw()
fig.canvas.mpl_connect('button_press_event', on_click)
while len(positions) < 4:
plt.pause(interval)
else:
tic = time.time()
positions = localization.predict(image, debug=debug)
toc = time.time()
print('Use time: {}'.format(toc-tic))
positions = np.float32(np.array(positions))
print(positions)
img = warp_perspective(image, positions, perspective, size)
plt.cla()
ultility.show(img)
vertices = extract.extract(img, debug=debug)
vertices = np.array(vertices)+shift
print(vertices)
if debug:
print('path:', vertices)
ultility.show(img)
ultility.plot(vertices, width, height)
plt.show()
match.init(image, positions)
if not load_from_video:
cap = cv.VideoCapture(address)
return vertices