# cv2.waitKey(0)
# print('Prev: ',img_raw.shape)
try:
img_raw = cv2.resize(img_raw, (resize, resize),
interpolation=cv2.INTER_AREA)
except:
print('Couldn\'t resize')
dropped_from_rotate_store.append(image)
rotate_by = random.choice(keys)
if (rotate_by != -1):
img_raw = cv2.rotate(img_raw, possible_rotations[rotate_by])
# print('After: ',img_raw.shape)
# cv2.imshow('Window',img_raw)
# cv2.waitKey(0)
write_to = os.path.join(resized_folder, str(possible_rotations[rotate_by]))
if img_raw.shape[0] == 224:
print(img_raw.shape)
if not os.path.exists(os.path.join(write_to, image)):
cv2.imwrite(os.path.join(write_to, image), img_raw)
rotation_list.append(possible_rotations[rotate_by])
i = i + 1
def seg(img, t=8, A=200, L=50):
# t: Threshold => the threshold used to segment the image (value of 8-10 works best. Otsu and Isodata values do not led to best result)
# A: Threshold area => All the segments less than A in area are to be removed and considered as noise
# L: Threshold length => All the centrelines less than L in length are to be removed
# Resize image to ~(1000px, 1000px) for best results
# Green Channel
g = img[:, :, 1]
#Creating mask for restricting FOV
_, mask = cv2.threshold(g, 10, 255, cv2.THRESH_BINARY)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15, 15))
mask = cv2.erode(mask, kernel, iterations=3)
# CLAHE and background estimation
clahe = cv2.createCLAHE(clipLimit=3, tileGridSize=(9, 9))
g_cl = clahe.apply(g)
g_cl1 = cv2.medianBlur(g_cl, 5)
bg = cv2.GaussianBlur(g_cl1, (55, 55), 0)
# Background subtraction
norm = np.float32(bg) - np.float32(g_cl1)
norm = norm * (norm > 0)
# Thresholding for segmentation
_, t = cv2.threshold(norm, t, 255, cv2.THRESH_BINARY)
# Removing noise points by coloring the contours
t = np.uint8(t)
th = t.copy()
contours, hierarchy = cv2.findContours(t, cv2.RETR_TREE,
cv2.CHAIN_APPROX_NONE)
for c in contours:
if (cv2.contourArea(c) < A):
cv2.drawContours(th, [c], 0, 0, -1)
th = th * (mask / 255)
th = np.uint8(th)
#plt.imshow(th, cmap='gryay') # THE SEGMENTED IMAGE
# Distance transform for maximum diameter
vessels = th.copy()
_, ves = cv2.threshold(vessels, 30, 255, cv2.THRESH_BINARY)
dist = cv2.distanceTransform(vessels, cv2.DIST_L2, 3)
_, mv, _, mp = cv2.minMaxLoc(dist)
print("Maximum diameter:", mv * 2, "at the point:", mp)
print("Select the vessel and press Q after selection.")
# Centerline extraction using Zeun-Shang's thinning algorithm
# Using opencv-contrib-python which provides very fast and efficient thinning algorithm
# The package can be installed using pip
thinned = cv2.ximgproc.thinning(th)
# Filling broken lines via morphological closing using a linear kernel
kernel = np.ones((1, 10), np.uint8)
d_im = cv2.dilate(thinned, kernel)
e_im = cv2.erode(d_im, kernel)
num_rows, num_cols = thinned.shape
for i in range(1, 360 // 15):
rotation_matrix = cv2.getRotationMatrix2D((num_cols / 2, num_rows / 2),
15 * i, 1)
img_rotation = cv2.warpAffine(thinned, rotation_matrix,
(num_cols, num_rows))
temp_d_im = cv2.dilate(img_rotation, kernel)
temp_e_im = cv2.erode(temp_d_im, kernel)
rotation_matrix = cv2.getRotationMatrix2D((num_cols / 2, num_rows / 2),
-15 * i, 1)
im = cv2.warpAffine(temp_e_im, rotation_matrix, (num_cols, num_rows))
e_im = np.maximum(im, e_im)
# Skeletonizing again to remove unwanted noise
thinned1 = cv2.ximgproc.thinning(e_im)
thinned1 = thinned1 * (mask / 255)
# Removing bifurcation points by using specially designed kernels
# Can be optimized further! (not the best implementation)
thinned1 = np.uint8(thinned1)
thh = thinned1.copy()
hi = thinned1.copy()
thi = thinned1.copy()
hi = cv2.cvtColor(hi, cv2.COLOR_GRAY2BGR)
thi = cv2.cvtColor(thi, cv2.COLOR_GRAY2BGR)
thh = thh / 255
kernel1 = np.array([[1, 0, 1], [0, 1, 0], [0, 1, 0]])
kernel2 = np.array([[0, 1, 0], [1, 1, 1], [0, 0, 0]])
kernel3 = np.array([[0, 1, 0], [0, 1, 1], [1, 0, 0]])
kernel4 = np.array([[1, 0, 1], [0, 1, 0], [0, 0, 1]])
kernel5 = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]])
kernels = [kernel1, kernel2, kernel3, kernel4, kernel5]
for k in kernels:
k1 = k
k2 = cv2.rotate(k1, cv2.ROTATE_90_CLOCKWISE)
k3 = cv2.rotate(k2, cv2.ROTATE_90_CLOCKWISE)
k4 = cv2.rotate(k3, cv2.ROTATE_90_CLOCKWISE)
ks = [k1, k2, k3, k4]
for kernel in ks:
th = cv2.filter2D(thh, -1, kernel)
for i in range(th.shape[0]):
for j in range(th.shape[1]):
if (th[i, j] == 4.0):
cv2.circle(hi, (j, i), 2, (0, 255, 0), 2)
cv2.circle(thi, (j, i), 2, (0, 0, 0), 2)
#plt.figure(figsize=(20, 14))
thi = cv2.cvtColor(thi, cv2.COLOR_BGR2GRAY)
#plt.imshow(hi, cmap='gray') # This image shows all the bifurcation points
# Removing centerlines which are smaller than L=50 px in length
cl = thi.copy()
contours, hierarchy = cv2.findContours(thi, cv2.RETR_TREE,
cv2.CHAIN_APPROX_NONE)
for c in contours:
if (c.size < L):
cv2.drawContours(cl, [c], 0, 0, -1)
# Centerline superimposed on green channel
colors = [(100, 0, 150), (102, 0, 255), (0, 128, 255), (255, 255, 0),
(10, 200, 10)]
colbgr = [(193, 182, 255), (255, 0, 102), (255, 128, 0), (0, 255, 255),
(10, 200, 10)]
im = g.copy()
im = cv2.cvtColor(im, cv2.COLOR_GRAY2BGR)
thc = cl
thh = thc.copy()
thh = cv2.cvtColor(thh, cv2.COLOR_GRAY2BGR)
contours, heirarchy = cv2.findContours(thc, cv2.RETR_LIST,
cv2.CHAIN_APPROX_NONE)
for c in contours:
color = np.random.randint(len(colors))
cv2.drawContours(im, c, -1, colbgr[color], 2, cv2.LINE_AA)
#cv2.namedWindow('image', cv2.WINDOW_NORMAL)
#cv2.resizeWindow('image', (int(im.shape[1]/2), int(im.shape[0]/2)))
#cv2.moveWindow('image', 40,30) # Move it to (40,30)
#cv2.imshow('image', cv2.cvtColor(im, cv2.COLOR_BGR2RGB))
#cv2.waitKey()
#cv2.destroyAllWindows()
# Maximum diameter estimate
d = mv * 1.5
return im, cl, d
(x, y, w, h) = cv2.boundingRect(cnt)
#cv2.drawContours(roi, [cnt], -1, (0, 0, 255), 3)
cv2.rectangle(roi, (x, y), (x + w, y + h), (255, 0, 0), 2)
cv2.line(roi, (x + int(w / 2), 0), (x + int(w / 2), rows), (0, 255, 0),
2)
cv2.line(roi, (0, y + int(h / 2)), (cols, y + int(h / 2)), (0, 255, 0),
2)
# coordinate of center of pupils
print(x + int(w / 2), "\t", y + int(h / 2))
break
# Rotate stuff
# frame = cv2.rotate(frame, cv2.ROTATE_180)
roi = cv2.rotate(roi, cv2.ROTATE_180)
gray_roi = cv2.rotate(gray_roi, cv2.ROTATE_180)
threshold = cv2.rotate(threshold, cv2.ROTATE_180)
cv2.imshow('Contours', roi)
cv2.imshow("Threshold", threshold)
# cv2.imshow('Input', frame)
cv2.imshow('Eyes', gray_roi)
c = cv2.waitKey(1)
if c == 64:
a += 1
print("Yes")
if c == 27:
#?green
#l_orange = np.array([0,0,100])
#u_orange = np.array([255,100,255])
#dark blue
#l_orange = np.array([200,0,50])
#u_orange = np.array([255,255,75])
#red
l_orange = np.array([0, 50, 0])
u_orange = np.array([10, 255, 255])
b, img = cam.read()
if b:
rotated = cv2.rotate(img, cv2.ROTATE_180)
final = cv2.convertScaleAbs(rotated, alpha=3, beta=0)
hsv = cv2.cvtColor(rotated, cv2.COLOR_BGR2HSV)
masked = cv2.inRange(hsv, l_orange, u_orange)
scary = cv2.bitwise_and(rotated, rotated, mask=masked)
cnts = cv2.findContours(masked.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
center = None
if len(cnts) > 0:
c = max(cnts, key=cv2.contourArea)
((x, y), radius) = cv2.minEnclosingCircle(c)
def calibrate_Burgess_session(calibration_data_name, vid_pair, num_frames_for_intrinsics=50, min_frames_for_intrinsics=10, num_frames_for_stereo=20, min_frames_for_stereo=5):
'''
:param calibration_data_name:
:param num_frames_for_intrinsics:
:param min_frames_for_intrinsics: minimum number of frames to use for intrinsics calibration (if number of valid frames
is less than num_frames_for_intrinsics, it will use all available frames. If min_frames_for_intrinsics is greater
than the number of available frames, calibration will be skipped).
:return:
'''
# create video objects for each calibration_video
vid_obj = []
for i_vid, vid_name in enumerate(vid_pair):
vid_obj.append(cv2.VideoCapture(vid_name))
CALIBRATION_FLAGS = cv2.CALIB_FIX_PRINCIPAL_POINT + cv2.CALIB_ZERO_TANGENT_DIST + cv2.CALIB_FIX_ASPECT_RATIO
STEREO_FLAGS = cv2.CALIB_FIX_INTRINSIC + cv2.CALIB_FIX_PRINCIPAL_POINT
# initialize camera intrinsics to have an aspect ratio of 1 and assume the center of the 1280 x 1024 field is [639.5, 511.5]
init_mtx = np.array([[2000, 0, 639.5],[0, 2000, 511.5],[0, 0, 1]])
cal_data = skilled_reaching_io.read_pickle(calibration_data_name)
# get intrinsics and distortion for each camera
num_cams = len(cal_data['cam_objpoints'])
cal_data['mtx'] = []
cal_data['dist'] = []
cal_data['frame_nums_for_intrinsics'] = []
for i_cam in range(num_cams):
current_cam = cal_data['calvid_metadata'][i_cam]['cam_num']
session_date_string = navigation_utilities.datetime_to_string_for_fname(
cal_data['calvid_metadata'][i_cam]['session_datetime'])
# if 'mtx' in cal_data.keys():
# # if intrinsics have already been calculated for this camera, skip
# if i_cam >= len(cal_data['mtx']):
# # this camera number is larger than the number of cameras for which intrinsics have been stored
# print('intrinsics already calculated for {}, camera {:02d}'.format(session_date_string, current_cam))
# continue
print('working on {}, camera {:02d} intrinsics calibration'.format(session_date_string, current_cam))
# select num_frames_for_intrinsics evenly spaced frames
cam_objpoints = cal_data['cam_objpoints'][i_cam]
cam_imgpoints = cal_data['cam_imgpoints'][i_cam]
total_valid_frames = np.shape(cam_objpoints)[0]
num_frames_to_use = min(num_frames_for_intrinsics, total_valid_frames)
if num_frames_to_use < min_frames_for_intrinsics:
mtx = np.zeros((3, 3))
dist = np.zeros((1, 5))
frame_numbers = []
else:
objpoints_for_intrinsics, imgpoints_for_intrinsics, frame_numbers = select_cboards_for_calibration(cam_objpoints, cam_imgpoints, num_frames_to_use)
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints_for_intrinsics,
imgpoints_for_intrinsics,
cal_data['im_size'][i_cam],
init_mtx,
None,
flags=CALIBRATION_FLAGS)
intrinsics_frames = np.shape(objpoints_for_intrinsics)[0]
valid_frames = cal_data['valid_frames'][i_cam]
for i_frame in range(intrinsics_frames):
pp = test_reprojection(objpoints_for_intrinsics[i_frame], imgpoints_for_intrinsics[i_frame], mtx, rvecs[i_frame], tvecs[i_frame], dist)
cur_frame = [ii for ii, vf in enumerate(valid_frames) if vf == True][frame_numbers[i_frame]]
vid_obj[i_cam].set(cv2.CAP_PROP_POS_FRAMES, cur_frame)
ret, cur_img = vid_obj[i_cam].read()
if current_cam == 1:
# rotate the image 180 degrees
cur_img = cv2.rotate(cur_img, cv2.ROTATE_180)
# corners_img = cv2.drawChessboardCorners(cur_img, cal_data['cb_size'], imgpoints_for_intrinsics[i_frame], True)
# reproj_img = cv2.drawChessboardCorners(corners_img, cal_data['cb_size'], pp, False)
#
# img_name = '/home/levlab/Public/mouse_SR_videos_to_analyze/mouse_SR_calibration_data/test_frame_{}_cam{:02d}_frame{:03d}.jpg'.format(session_date_string, current_cam, cur_frame)
# cv2.imwrite(img_name, reproj_img)
cal_data['mtx'].append(np.copy(mtx))
cal_data['dist'].append(np.copy(dist))
cal_data['frame_nums_for_intrinsics'].append(frame_numbers)
skilled_reaching_io.write_pickle(calibration_data_name, cal_data)
# now perform stereo calibration
# num_frames_for_stereo = 20, min_frames_for_stereo = 5
stereo_objpoints = cal_data['stereo_objpoints']
stereo_imgpoints = cal_data['stereo_imgpoints']
num_stereo_pairs = np.shape(stereo_objpoints)[0]
num_frames_to_use = min(num_frames_for_stereo, num_stereo_pairs)
objpoints, imgpoints, stereo_frame_idx = select_cboards_for_stereo_calibration(stereo_objpoints, stereo_imgpoints, num_frames_to_use)
frames_for_stereo_calibration = [sf_idx for sf_idx in cal_data['stereo_frames']]
mtx = cal_data['mtx']
dist = cal_data['dist']
im_size = cal_data['im_size']
# im_size must be the same for both cameras
if all([ims == im_size[0] for ims in im_size]) and num_frames_to_use >= min_frames_for_stereo:
# all images have the same size
print('working on stereo calibration for {}'.format(session_date_string))
im_size = im_size[0]
ret, mtx1, dist1, mtx2, dist2, R, T, E, F = cv2.stereoCalibrate(objpoints, imgpoints[0], imgpoints[1], mtx[0], dist[0], mtx[1], dist[1], im_size, flags=STEREO_FLAGS)
else:
ret = False
mtx1 = np.zeros((3, 3))
mtx2 = np.zeros((3, 3))
dist1 = np.zeros((1, 5))
dist2 = np.zeros((1, 5))
R = np.zeros((3, 3))
T = np.zeros((3, 1))
E = np.zeros((3, 3))
F = np.zeros((3, 3))
cal_data['R'] = R
cal_data['T'] = T
cal_data['E'] = E
cal_data['F'] = F
cal_data['frames_for_stereo_calibration'] = frames_for_stereo_calibration # frame numbers in original calibration video used for the stereo calibration
# if valid_frames[0][i_frame] and valid_frames[1][i_frame]:
# # checkerboards were identified in matching frames
# stereo_objpoints.append(objp)
# for i_vid, corner_pts in enumerate(corners2):
# stereo_imgpoints[i_vid].append(corner_pts)
skilled_reaching_io.write_pickle(calibration_data_name, cal_data)
def main(camera_id, filename, hrnet_m, hrnet_c, hrnet_j, hrnet_weights, hrnet_joints_set, image_resolution,
single_person, use_tiny_yolo, disable_tracking, max_batch_size, disable_vidgear, save_video, video_format,
video_framerate, device):
# filename = 'D:/test.mp4'
if device is not None:
device = torch.device(device)
else:
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
device = torch.device('cuda')
else:
device = torch.device('cpu')
# print(device)
image_resolution = ast.literal_eval(image_resolution)
has_display = 'DISPLAY' in os.environ.keys() or sys.platform == 'win32'
video_writer = None
if filename is not None:
rotation_code = check_video_rotation(filename)
video = cv2.VideoCapture(filename)
assert video.isOpened()
else:
rotation_code = None
if disable_vidgear:
video = cv2.VideoCapture(camera_id)
assert video.isOpened()
else:
video = CamGear(camera_id).start()
if use_tiny_yolo:
yolo_model_def="../models/detectors/yolo/config/yolov3-tiny.cfg"
yolo_class_path="../models/detectors/yolo/data/coco.names"
yolo_weights_path="../models/detectors/yolo/weights/yolov3-tiny.weights"
else:
yolo_model_def="../models/detectors/yolo/config/yolov3.cfg"
yolo_class_path="../models/detectors/yolo/data/coco.names"
yolo_weights_path="../models/detectors/yolo/weights/yolov3.weights"
model = SimpleHRNet(
hrnet_c,
hrnet_j,
hrnet_weights,
model_name=hrnet_m,
resolution=image_resolution,
multiperson=not single_person,
return_bounding_boxes=not disable_tracking,
max_batch_size=max_batch_size,
yolo_model_def=yolo_model_def,
yolo_class_path=yolo_class_path,
yolo_weights_path=yolo_weights_path,
device=device
)
if not disable_tracking:
prev_boxes = None
prev_pts = None
prev_person_ids = None
next_person_id = 0
while True:
t = time.time()
if filename is not None or disable_vidgear:
ret, frame = video.read()
if not ret:
break
if rotation_code is not None:
frame = cv2.rotate(frame, rotation_code)
else:
frame = video.read()
if frame is None:
break
pts = model.predict(frame)
if not disable_tracking:
boxes, pts = pts
if not disable_tracking:
if len(pts) > 0:
if prev_pts is None and prev_person_ids is None:
person_ids = np.arange(next_person_id, len(pts) + next_person_id, dtype=np.int32)
next_person_id = len(pts) + 1
else:
boxes, pts, person_ids = find_person_id_associations(
boxes=boxes, pts=pts, prev_boxes=prev_boxes, prev_pts=prev_pts, prev_person_ids=prev_person_ids,
next_person_id=next_person_id, pose_alpha=0.2, similarity_threshold=0.4, smoothing_alpha=0.1,
)
next_person_id = max(next_person_id, np.max(person_ids) + 1)
else:
person_ids = np.array((), dtype=np.int32)
prev_boxes = boxes.copy()
prev_pts = pts.copy()
prev_person_ids = person_ids
else:
person_ids = np.arange(len(pts), dtype=np.int32)
for i, (pt, pid) in enumerate(zip(pts, person_ids)):
frame = draw_points_and_skeleton(frame, pt, joints_dict()[hrnet_joints_set]['skeleton'], person_index=pid,
points_color_palette='gist_rainbow', skeleton_color_palette='jet',
points_palette_samples=10)
fps = 1. / (time.time() - t)
print('\rframerate: %f fps' % fps, end='')
if has_display:
cv2.imshow('frame.png', frame)
k = cv2.waitKey(1)
if k == 27: # Esc button
if disable_vidgear:
video.release()
else:
video.stop()
break
else:
cv2.imwrite('frame.png', frame)
if save_video:
if video_writer is None:
fourcc = cv2.VideoWriter_fourcc(*video_format) # video format
video_writer = cv2.VideoWriter('output.avi', fourcc, video_framerate, (frame.shape[1], frame.shape[0]))
video_writer.write(frame)
if save_video:
video_writer.release()
def main():
if not os.path.exists(args.data_path):
raise ValueError('data path does not exist.')
positive_data_path = os.path.join(args.output_path, 'positive', 'Images')
negative_data_path = os.path.join(args.output_path, 'negative', 'Images')
if not os.path.exists(args.output_path):
os.makedirs(positive_data_path)
os.makedirs(negative_data_path)
f_info_positive = open(
os.path.join(args.output_path, 'positive', 'dataInfo.txt'), 'a')
f_info_negative = open(
os.path.join(args.output_path, 'negative', 'dataInfo.txt'), 'a')
num_pos = len(os.listdir(positive_data_path))
num_neg = len(os.listdir(negative_data_path))
num_data = num_pos + num_neg
with open(args.label_path, 'r') as f:
annotations = f.readlines()
for i in xrange(len(annotations)):
file_path, grasp_index, label = annotations[i].split(' ')
if file_path.endswith('.jpg'):
file_name = file_path.split('/')[-1]
else:
file_name = file_path + '.jpg'
angle_index = int(grasp_index)
label = int(label)
img = cv2.imread(os.path.join(args.data_path, file_name))
if args.argumented:
for j in range(4):
grasp_angle = ((
(angle_index + j * 9) % 18) * 10 - 90) * 1.0 / 180 * pi
rotated_img = img if j == 0 else cv2.rotate(img, j - 1)
if label == 0:
cv2.imwrite(
positive_data_path +
'/{:06d}.jpg'.format(i * 4 + j + num_data),
rotated_img)
f_info_positive.write(','.join([
'{:06d}.jpg'.format(i * 4 + j + num_data),
'{}\n'.format(grasp_angle)
]))
else:
cv2.imwrite(
negative_data_path +
'/{:06d}.jpg'.format(i * 4 + j + num_data),
rotated_img)
f_info_negative.write(','.join([
'{:06d}.jpg'.format(i * 4 + j + num_data),
'{}\n'.format(grasp_angle)
]))
else:
grasp_angle = (angle_index * 10 - 90) * 1.0 / 180 * pi
if label == 0:
cv2.imwrite(
positive_data_path + '/{:06d}.jpg'.format(i + num_data),
img)
f_info_positive.write(','.join([
'{:06d}.jpg'.format(i + num_data),
'{}\n'.format(grasp_angle)
]))
else:
cv2.imwrite(
negative_data_path + '/{:06d}.jpg'.format(i + num_data),
img)
f_info_negative.write(','.join([
'{:06d}.jpg'.format(i + num_data),
'{}\n'.format(grasp_angle)
]))
f_info_positive.close()
f_info_negative.close()
ret, frame = cap.read()
if not ret:
cap = cv2.VideoCapture('green.mp4')
ret, frame = cap.read()
continue
hmin = cv2.getTrackbarPos("hmin", "frame1")
hmax = cv2.getTrackbarPos("hmax", "frame1")
smin = cv2.getTrackbarPos("smin", "frame1")
smax = cv2.getTrackbarPos("smax", "frame1")
vmin = cv2.getTrackbarPos("vmin", "frame1")
vmax = cv2.getTrackbarPos("vmax", "frame1")
lower_green = np.array([hmin, smin, vmin])
upper_green = np.array([hmax, smax, vmax])
frame = cv2.rotate(frame, 0)
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(hsv, lower_green, upper_green)
kernel = np.ones((15, 15), np.int)
dilated = cv2.dilate(mask, kernel)
res = cv2.bitwise_and(frame, frame, mask=mask)
ret, thrshed = cv2.threshold(cv2.cvtColor(res, cv2.COLOR_BGR2GRAY), 3, 255,
cv2.THRESH_BINARY)
_, cnts, _ = cv2.findContours(thrshed, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
areas = [cv2.contourArea(cnt) for cnt in cnts]
max_index = np.argmax(areas)
cnt = cnts[max_index]
M = cv2.moments(cnt)
if M["m00"] != 0:
rotate_degrees = 90 # Clockwise rotation (90, 180, 270)
flip_ver = False
flip_hor = False
display_video = True
# capture frames from the camera
for frame in camera.capture_continuous(rawCapture,
format="bgr",
use_video_port=True):
# grab the raw NumPy array representing the image, then initialize the timestamp
# and occupied/unoccupied text
image = frame.array
if rotate: # If True rotate by rotate_degrees
if rotate_degrees == 90:
image = cv.rotate(image, cv.ROTATE_90_CLOCKWISE)
if rotate_degrees == 180:
image = cv.rotate(image, cv.ROTATE_180_CLOCKWISE)
if rotate_degrees == 270:
image = cv.rotate(image, cv.ROTATE_270_CLOCKWISE)
# If True flip
if flip_ver: image = cv.flip(image, 0)
if flip_hor: image = cv.flip(image, 1)
if scale: # If True scale the image
# calculate the new dimensions
width = int(image.shape[1] * scale_percent / 100)
height = int(image.shape[0] * scale_percent / 100)
dsize = (width, height)
# resize image
returnCode, resolution, image = vrep.simxGetVisionSensorImage(clientID, v1, 0, vrep.simx_opmode_streaming)
returnCode, detectionState, detectedPoint_lidar, detectedObjectHandle, detectedSurfaceNormalVector = vrep.simxReadProximitySensor(clientID,Sonar,vrep.simx_opmode_streaming);
returnCode, detectionState, detectedPoint_lidar, detectedObjectHandle, detectedSurfaceNormalVector = vrep.simxReadProximitySensor(clientID,Lidar,vrep.simx_opmode_streaming);
while (vrep.simxGetConnectionId(clientID) != -1):
returnCode, resolution, image = vrep.simxGetVisionSensorImage(clientID, v1, 0, vrep.simx_opmode_buffer)
returnCode, detectionState, detectedPoint_lidar, detectedObjectHandle, detectedSurfaceNormalVector = vrep.simxReadProximitySensor(clientID,Sonar,vrep.simx_opmode_buffer);
returnCode, detectionState, detectedPoint_lidar, detectedObjectHandle, detectedSurfaceNormalVector = vrep.simxReadProximitySensor(clientID,Lidar,vrep.simx_opmode_buffer);
norm_distance = np.linalg.norm(detectedPoint_lidar)
filtered_data = alphatrimmer(7,2,str(norm_distance))
if returnCode == vrep.simx_return_ok:
img = np.array(image,dtype=np.uint8)
img.resize([resolution[1],resolution[0],3])
output = cv2.rotate(img, cv2.ROTATE_180)
out = cv2.VideoWriter('output.avi', -1, 20.0, (1024,512))
cv2.imshow('img',output)
output_filter = next(filtered_data)
# cv2.putText(image, str(filtered_data), org, font, fontScale, color, thickness, cv2.LINE_AA)
print(output_filter*100)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
elif returnCode == vrep.simx_return_novalue_flag:
pass
else:
print (returnCode)
else:
vrep.simxFinish(clientID)
cv2.destroyAllWindows()
def track_bees(args):
# Handle file not found
if not os.path.exists(args.video_file_in):
sys.exit("Error. File not found. (-2)")
model, device = setup_model_dataloader(args, batch_size=1)
vid = cv2.VideoCapture(args.video_file_in)
ic("Processing ", args.video_file_in)
# Get some metadata
num_frames = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = vid.get(cv2.CAP_PROP_FPS)
ic("Num frames: ", num_frames)
ic("Width x Height: ", (width, height))
# I don't think fps is accurate!
ic("Source FPS: ", fps)
if args.fps != 0:
frame_duration = 1000 // int(args.fps) # ms
else:
frame_duration = 1000 // int(fps) # ms
# Don't actually need the dataset. Just need the input dimensions
bee_ds = BeePointDataset(root_dir="/dev/null")
input_size = bee_ds.input_size
ic(bee_ds.input_size)
# Vars for tracking
prev_pts = np.array([])
running_pairs = []
# Vars for profiling
total_exec_time_accumulator = 0
exec_time_accumulator = 0
num_exec_frames = 0
for frame_num in range(num_frames):
# Read frame from video
ret, frame = vid.read()
if frame is None:
break
if args.enable_timing:
start_time = time.time()
# Rotate if rotation is set
if args.rotate == -90:
frame = cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
elif args.rotate == 90:
frame = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
# Resize to network input size
frame = cv2.resize(frame, (input_size[1], input_size[0]))
if 0:
cv2.imshow("Video", frame)
key = cv2.waitKey(30)
if key == 'q' or key == 27:
break
# Convert to tensor and forward pass
tensor_frame = TF.to_tensor(frame)
tensor_frame.to(device)
tensor_frame = torch.unsqueeze(tensor_frame, 0)
pred = helper.model_forward(model, tensor_frame, device, ENABLE_TIMING)
# Get prediction centroids (predicted points)
pred_pts = helper.get_centroids(pred)
num_bees = len(pred_pts)
# Since what we really care about is # bees, stop time profiling here
# The rest is for visualization
if args.enable_timing and frame_num > 0:
iter_time = time.time() - start_time
total_exec_time_accumulator += iter_time
exec_time_accumulator += iter_time
num_exec_frames += 1
if frame_num % int(fps) == 0:
avg_exec_time = exec_time_accumulator / num_exec_frames
ic("Avg exec time: ", avg_exec_time)
exec_time_accumulator = 0
num_exec_frames = 0
# Use bipartite graph minimum weight matching to associate detections
if len(pred_pts) > 0 and len(
prev_pts) > 0 and not args.disable_tracking:
pairs = helper.calculate_pairs(prev_pts, pred_pts, args.threshold)
# Extract actual points based on indices in original arrays
point_pairs = []
for pair in pairs:
point_pairs.append((prev_pts[pair[1]], pred_pts[pair[0]]))
running_pairs.append(point_pairs)
if len(running_pairs) > args.tracker_frame_len:
running_pairs.pop(0)
# Draw the tracking lines
frame = draw_tracking_lines(frame, running_pairs)
elif len(running_pairs) > 0:
running_pairs.pop(0)
if len(pred_pts) > 0:
for pred_pt in pred_pts:
cv2.circle(frame, tuple((pred_pt[1], pred_pt[0])), 5,
VIS_COLOR, cv2.FILLED)
# Draw # of bees in text on bottom left of image
num_bees_text = "# Bees: %i" % num_bees
bottom_left = (0, frame.shape[0] - 20)
frame = cv2.putText(frame,
num_bees_text,
org=bottom_left,
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=1,
color=(255, 0, 0),
thickness=2,
lineType=cv2.LINE_AA)
# Show results
helper.show_image("Predictions", frame, delay=frame_duration)
prev_pts = deepcopy(pred_pts)
# Calculate the average processing fps. num_frames-1 b/c we didnt' count the first frame initialization delay
avg_exec_time = total_exec_time_accumulator / (num_frames - 1)
measured_fps = 1 / avg_exec_time
ic("Overall avg exec time: ", avg_exec_time)
ic("Overall FPS: ", measured_fps)
def main():
if not os.path.isfile(args.face):
raise ValueError('--face argument must be a valid path to video/image file')
elif args.face.split('.')[1] in ['jpg', 'png', 'jpeg']:
full_frames = [cv2.imread(args.face)]
fps = args.fps
else:
video_stream = cv2.VideoCapture(args.face)
fps = video_stream.get(cv2.CAP_PROP_FPS)
print('Reading video frames...')
full_frames = []
while 1:
still_reading, frame = video_stream.read()
if not still_reading:
video_stream.release()
break
if args.resize_factor > 1:
frame = cv2.resize(frame, (frame.shape[1]//args.resize_factor, frame.shape[0]//args.resize_factor))
if args.rotate:
frame = cv2.rotate(frame, cv2.cv2.ROTATE_90_CLOCKWISE)
y1, y2, x1, x2 = args.crop
if x2 == -1: x2 = frame.shape[1]
if y2 == -1: y2 = frame.shape[0]
frame = frame[y1:y2, x1:x2]
full_frames.append(frame)
print ("Number of frames available for inference: "+str(len(full_frames)))
if not args.audio.endswith('.wav'):
print('Extracting raw audio...')
command = 'ffmpeg -y -i {} -strict -2 {}'.format(args.audio, 'temp/temp.wav')
subprocess.call(command, shell=True)
args.audio = 'temp/temp.wav'
wav = audio.load_wav(args.audio, 16000)
mel = audio.melspectrogram(wav)
print(mel.shape)
if np.isnan(mel.reshape(-1)).sum() > 0:
raise ValueError('Mel contains nan! Using a TTS voice? Add a small epsilon noise to the wav file and try again')
mel_chunks = []
mel_idx_multiplier = 80./fps
i = 0
while 1:
start_idx = int(i * mel_idx_multiplier)
if start_idx + mel_step_size > len(mel[0]):
mel_chunks.append(mel[:, len(mel[0]) - mel_step_size:])
break
mel_chunks.append(mel[:, start_idx : start_idx + mel_step_size])
i += 1
print("Length of mel chunks: {}".format(len(mel_chunks)))
full_frames = full_frames[:len(mel_chunks)]
batch_size = args.wav2lip_batch_size
gen = datagen(full_frames.copy(), mel_chunks)
for i, (img_batch, mel_batch, frames, coords) in enumerate(tqdm(gen,
total=int(np.ceil(float(len(mel_chunks))/batch_size)))):
if i == 0:
model = load_model(args.checkpoint_path)
print ("Model loaded")
frame_h, frame_w = full_frames[0].shape[:-1]
out = cv2.VideoWriter('temp/result.avi',
cv2.VideoWriter_fourcc(*'DIVX'), fps, (frame_w, frame_h))
img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device)
mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device)
with torch.no_grad():
pred = model(mel_batch, img_batch)
pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255.
for p, f, c in zip(pred, frames, coords):
y1, y2, x1, x2 = c
p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1))
f[y1:y2, x1:x2] = p
out.write(f)
out.release()
command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio, 'temp/result.avi', args.outfile)
subprocess.call(command, shell=True)
frame_counter = 0
c = 0
while cap.isOpened():
ret, frame = cap.read()
# frame_counter += 1
# #If the last frame is reached, reset the capture and the frame_counter
# if frame_counter == cap.get(cv.CAP_PROP_FRAME_COUNT):
# frame_counter = 0 #Or whatever as long as it is the same as next line
# cap.set(cv.CAP_PROP_POS_FRAMES, 0)
# if frame is read correctly ret is True
if not ret:
print("Can't receive frame (stream end?). Exiting ...")
break
if c % 20 == 0:
# Region Of Interest(ROI)
# crop = img[y:y+h, x:x+w]
crop = cv.rotate(frame[140:327, 54:238], cv.ROTATE_180)
cv.imshow('frame', frame)
file = 'live.png'
cv.imwrite(file, crop)
# # print OCR text
print(detect_text(file))
# data = detect_text(file).replace(" ", "")
# print(float(data.splitlines()[1]))
c += 1
if cv.waitKey(1) == ord('q'):
break
cap.release()
cv.destroyAllWindows()
if not os.path.exists(args.movie):
print("Movie file %s missing" % args.movie)
sys.exit(1)
cam = cv2.VideoCapture(args.movie)
width = int(cam.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cam.get(cv2.CAP_PROP_FRAME_HEIGHT))
objdet = ObjectDetector()
counter = 0
ret, img = cam.read()
while ret == True:
img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
h, w, _ = img.shape
expand = np.expand_dims(img, axis=0)
result = objdet.detect(expand)
boxes = []
for i in range(result['num_detections']):
if result['detection_scores'][i] > 0.75:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
score = result['detection_scores'][i]
y1, x1 = int(box[0] * h), int(box[1] * w)
y2, x2 = int(box[2] * h), int(box[3] * w)
boxes.append((class_, score, x1, y1, x2, y2))
for box in boxes:
def play_as_human():
pygame.init()
surface = pygame.display.set_mode((int(WIDTH), int(HEIGHT)))
pygame.display.set_caption('Geometry Friends')
clock = pygame.time.Clock()
font = pygame.font.SysFont("Lato", 22)
font.set_bold(True)
gameQuit = False
setup = game_setup.setups[SETUP](surface)
obstacles, ball, diamonds = (setup.obstacles, setup.ball, setup.diamonds)
if GIF:
t = 0
start_ticks = pygame.time.get_ticks() ## Start tick
while not gameQuit:
for event in pygame.event.get():
# print(event)
if event.type == pygame.QUIT:
gameQuit = True
## Press the arrows
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_RIGHT:
ball.pushedRight = True
if event.key == pygame.K_LEFT:
ball.pushedLeft = True
if event.key == pygame.K_UP and ball.isPushable:
ball.pushedY = True
## 'q' to exit
if event.key == pygame.K_q:
gameQuit = True
## 'r' to reset
if event.key == pygame.K_r:
play_as_human()
## Release the arrows
if event.type == pygame.KEYUP:
if (event.key == pygame.K_RIGHT):
ball.pushedRight = False
if (event.key == pygame.K_LEFT):
ball.pushedLeft = False
surface.fill(BACKGROUND)
ball.act()
ball.pushedY = False
for diamond in diamonds:
if diamond.exists and diamond.isCollectedBy(ball):
diamond.exists = False
setup.scoreCounter += 1
diamond.draw()
if all([not diamond.exists for diamond in diamonds]):
setup.refresh()
for obstacle in obstacles:
if obstacle is not None:
obstacle.draw()
setup.printScoreAndTime(surface=surface,
start_ticks=start_ticks,
font=font)
if GIF:
image = pygame.surfarray.array3d(pygame.display.get_surface())
image = np.fliplr(cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE))
image = cv2.resize(
image, (int(0.75 * params.WIDTH), int(0.75 * params.HEIGHT)),
interpolation=cv2.INTER_NEAREST)
plt.imsave("images/frame_{:0>3}.jpg".format(t), image)
t += 1
pygame.display.update()
clock.tick(FPS)
if (setup.scoreCounter / len(diamonds)) == GAME_OVER:
break
name, score = setup.saveScore(start_ticks=start_ticks,
totalCollect=setup.scoreCounter,
player="human")
print(("PLAYER {}, TIME PER SCORE {}, FOR SETUP {}".format(
name, score, SETUP)))
STYLE = GOALIER
t = time()
connectStateTime = -1
blockedConnect = False
dd = 0
kickBlocked = False
kickState = -1
ballOuterRadius = 25
ballOuterArea = PointSet(
curveToPoints(Point(0, 0), ballOuterRadius, fr=0, to=2 * PI))
while 1:
### READ FRAME
_, frame = cap.read()
#frame = cv.resize(frame, None, fx=0.5, fy=0.5)
frame = frame[0:-10, 150:-130, :]
frame = cv.rotate(frame, cv.ROTATE_90_COUNTERCLOCKWISE)
### RECALCULATE FRAME SIZE
H, W, _ = frame.shape
### DENOISE
frame = cv.GaussianBlur(frame, (3, 3), 0)
### READ SERIAL
readSTM()
#print(f'{world.robot.pos} eq {world.fieldEngine.goalieTarget})')
### ALGO LOGIC
plotter = []
if STYLE == GOALIER:
delta = world.robot.pos - world.fieldEngine.goalieTarget
# Rescale target
targets[:, 2:6] *= configs.img_size
# Get yaw angle
targets[:, 6] = torch.atan2(targets[:, 6], targets[:, 7])
img_bev = imgs.squeeze() * 255
img_bev = img_bev.permute(1, 2, 0).numpy().astype(np.uint8)
img_bev = cv2.resize(img_bev, (configs.img_size, configs.img_size))
for c, x, y, w, l, yaw in targets[:, 1:7].numpy():
# Draw rotated box
bev_utils.drawRotatedBox(img_bev, x, y, w, l, yaw,
cnf.colors[int(c)])
img_bev = cv2.rotate(img_bev, cv2.ROTATE_180)
if configs.mosaic and configs.show_train_data:
if configs.save_img:
fn = os.path.basename(img_file)
cv2.imwrite(os.path.join(configs.saved_dir, fn), img_bev)
else:
cv2.imshow('mosaic_sample', img_bev)
else:
out_img = merge_rgb_to_bev(img_rgb,
img_bev,
output_width=configs.output_width)
if configs.save_img:
fn = os.path.basename(img_file)
cv2.imwrite(os.path.join(configs.saved_dir, fn), out_img)
else:
def collect_cbpoints_Burgess(vid_pair, cal_data_parent, cb_size=(7, 10), checkerboard_square_size=7):
'''
:param vid_pair:
:param cal_data_parent: parent directory for folders containing pickle files with calibration results. Has structure:
cal_data_parent-->calibration_data_YYYY-->calibration_data_YYYYmm
:param cb_size:
:param checkerboard_square_size:
:return:
'''
# extract metadata from file names. Note that cam 01 is upside down
calvid_metadata = [navigation_utilities.parse_Burgess_calibration_vid_name(vid) for vid in vid_pair]
cal_data_name = navigation_utilities.create_optitrack_calibration_data_name(cal_data_parent,
calvid_metadata[0]['session_datetime'])
if os.path.isfile(cal_data_name):
# if file already exists, assume cb points have already been collected
return
# camera calibrations have been performed, now need to do stereo calibration
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# CBOARD_FLAGS = cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE
# create video objects for each calibration_video
vid_obj = []
num_frames = []
im_size = []
vid_root_names = []
for i_vid, vid_name in enumerate(vid_pair):
_, cur_root_name = os.path.split(vid_name)
vid_root_names.append(cur_root_name)
vid_obj.append(cv2.VideoCapture(vid_name))
num_frames.append(int(vid_obj[i_vid].get(cv2.CAP_PROP_FRAME_COUNT)))
im_size.append((int(vid_obj[i_vid].get(cv2.CAP_PROP_FRAME_WIDTH)), int(vid_obj[i_vid].get(cv2.CAP_PROP_FRAME_HEIGHT))))
if all(nf == num_frames[0] for nf in num_frames):
# check that there are the same number of frames in each calibration video
# Prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
cbrow = cb_size[0]
cbcol = cb_size[1]
objp = np.zeros((cbrow * cbcol, 3), np.float32)
objp[:, :2] = np.mgrid[0:cbrow, 0:cbcol].T.reshape(-1, 2)
cam_objpoints = [[] for ii in vid_pair]
stereo_objpoints = []
cam_imgpoints = [[] for ii in vid_pair]
stereo_imgpoints = [[] for ii in vid_pair]
# create boolean lists to track which frames have valid checkerboard images. Writing so that can expand to more
# views eventually
# valid_frames is a list of length num_cameras of lists that each are of length num_frames
valid_frames = [[False for frame_num in range(num_frames[0])] for ii in vid_pair]
stereo_frames = []
for i_frame in range(num_frames[0]):
print('frame number: {:04d} for {} and {}'.format(i_frame, vid_root_names[0], vid_root_names[1]))
corners2 = [[] for ii in vid_pair]
cur_img = [[] for ii in vid_pair]
for i_vid, vo in enumerate(vid_obj):
vo.set(cv2.CAP_PROP_POS_FRAMES, i_frame)
ret, cur_img[i_vid] = vo.read()
# test_img_name = os.path.join(cal_data_parent, 'test_cam{:02d}.jpg'.format(calibration_metadata[i_vid]['cam_num']))
# cv2.imwrite(test_img_name, cur_img[i_vid])
if ret:
if calvid_metadata[i_vid]['cam_num'] == 1:
# rotate the image 180 degrees
cur_img[i_vid] = cv2.rotate(cur_img[i_vid], cv2.ROTATE_180)
cur_img_gray = cv2.cvtColor(cur_img[i_vid], cv2.COLOR_BGR2GRAY)
found_valid_chessboard, corners = cv2.findChessboardCorners(cur_img_gray, cb_size)
valid_frames[i_vid][i_frame] = found_valid_chessboard
if found_valid_chessboard:
corners2[i_vid] = cv2.cornerSubPix(cur_img_gray, corners, (11, 11), (-1, -1), criteria)
cam_objpoints[i_vid].append(objp)
cam_imgpoints[i_vid].append(corners2[i_vid])
corners_img = cv2.drawChessboardCorners(cur_img[i_vid], cb_size, corners2[i_vid],
found_valid_chessboard)
else:
corners_img = cv2.drawChessboardCorners(cur_img[i_vid], cb_size, corners,
found_valid_chessboard)
# vid_path, vid_name = os.path.split(calibration_vids[i_vid])
# vid_name, _ = os.path.splitext(vid_name)
# frame_path = os.path.join(vid_path, vid_name)
# if not os.path.isdir(frame_path):
# os.makedirs(frame_path)
session_date_string = navigation_utilities.datetime_to_string_for_fname(calvid_metadata[i_vid]['session_datetime'])
test_save_dir = os.path.join(cal_data_parent, 'corner_images', session_date_string, 'cam{:02d}'.format(calvid_metadata[i_vid]['cam_num']))
if not os.path.isdir(test_save_dir):
os.makedirs(test_save_dir)
test_img_name = os.path.join(test_save_dir,
'test_cboard_{}_cam{:02d}_frame{:04d}.jpg'.format(session_date_string, calvid_metadata[i_vid]['cam_num'], i_frame))
# frame_name = vid_name + '_frame{:03d}'.format(i_frame) + '.png'
cv2.imwrite(test_img_name, corners_img)
# collect all checkerboard points visible in pairs of images
if valid_frames[0][i_frame] and valid_frames[1][i_frame]:
# checkerboards were identified in matching frames
stereo_objpoints.append(objp)
stereo_frames.append(i_frame)
for i_vid, corner_pts in enumerate(corners2):
stereo_imgpoints[i_vid].append(corner_pts)
# todo: test that the checkerboard points in each imgpoint array are correct
# below is for checking if corners were correctly identified
# for i_vid in range(3):
# if valid_frames[i_vid][i_frame]:
# corners_img = cv2.drawChessboardCorners(cur_img[i_vid], cb_size, corners2[i_vid], found_valid_chessboard)
# cv2.imwrite(corners_img_name, corners_img)
# plt.imshow(corners_img)
# plt.show()
calibration_data = {
'cam_objpoints': cam_objpoints,
'cam_imgpoints': cam_imgpoints,
'stereo_objpoints': stereo_objpoints,
'stereo_imgpoints': stereo_imgpoints,
'stereo_frames': stereo_frames, # frame numbers for which valid checkerboards were found for both cameras
'valid_frames': valid_frames,
'im_size': im_size,
'cb_size': cb_size,
'checkerboard_square_size': checkerboard_square_size,
'calvid_metadata': calvid_metadata
}
skilled_reaching_io.write_pickle(cal_data_name, calibration_data)
for vo in vid_obj:
vo.release()
def cif_logpolar_auto_90(img):
img = cv2.rotate(img, cv2.ROTATE_90_COUNTERCLOCKWISE)
img_polar = cv2.logPolar(img, (3000, 3000), 758, cv2.WARP_FILL_OUTLIERS)
img_rotated = cv2.rotate(img_polar, cv2.ROTATE_90_COUNTERCLOCKWISE)
img_rotated = img_rotated[:300, :]
return img_rotated
import numpy as np
import cv2
import os
import matplotlib.pyplot as plt
import time
imgs = []
folder = "AE4317_2019_datasets/sim_poles/20190121-160844"
for filename in os.listdir(folder):
img = cv2.imread(os.path.join(folder, filename))
if img is not None:
img = cv2.rotate(img, cv2.ROTATE_90_COUNTERCLOCKWISE)
scale = 100 # in percent
width = int(img.shape[1] * scale / 100)
height = int(img.shape[0] * scale / 100)
img = cv2.resize(img, (width, height))
imgs.append(img)
frame1 = imgs[0]
prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
mags = []
angs = []
flows = []
diffs = []
durations = []
i = 1
h_crop_1 = 80
h_crop_2 = 160
v_mid = int(imgs[0].shape[1] / 2)
while (i < 476):
start = time.time()
if i < 476:
def texture(final_path, texture_path, floor_type, wall_type, tiling):
# Find array of x, y coordinates for given special tile type
def match(template, image):
w, h = template.shape[::-1]
res = cv2.matchTemplate(image, template, cv2.TM_CCOEFF_NORMED)
threshold = 0.98
loc = np.where(res >= threshold)
return loc
# Apply special tile textures to pre-determined locations
def paste(x, y):
anchor_x = x
anchor_y = y
background_width = background.shape[1]
background_height = background.shape[0]
foreground_width = foreground.shape[1]
foreground_height = foreground.shape[0]
alpha = 1
start_x = anchor_x
start_y = anchor_y
end_x = anchor_x + foreground_width
end_y = anchor_y + foreground_height
blended_portion = cv2.addWeighted(
foreground, alpha, background[start_y:end_y, start_x:end_x, :],
1 - alpha, 0, background)
background[start_y:end_y, start_x:end_x, :] = blended_portion
# Load required images into variables
# ------------------------------------------------------------------------------
original = cv2.imread(texture_path, -1)
original_bw = cv2.cvtColor(original, cv2.COLOR_BGR2GRAY)
if tiling == False:
if floor_type == 'Grass':
floor_texture = cv2.imread('../Textures/Untiled/grass.png', -1)
elif floor_type == 'Sand':
floor_texture = cv2.imread('../Textures/Untiled/sand.png', -1)
elif floor_type == 'Rocks':
floor_texture = cv2.imread('../Textures/Untiled/rocks.png', -1)
elif floor_type == 'Snow':
floor_texture = cv2.imread('../Textures/Untiled/snow.png', -1)
else:
floor_texture = cv2.imread('../Textures/Untiled/tiles.png', -1)
if wall_type == 'Stone':
wall_texture = cv2.imread('../Textures/Untiled/tiles.png', -1)
elif wall_type == 'Gold':
wall_texture = cv2.imread('../Textures/Untiled/gold.png', -1)
elif wall_type == 'Black':
wall_texture = cv2.imread('../Textures/Untiled/black.png', -1)
elif wall_type == 'Ice':
wall_texture = cv2.imread('../Textures/Untiled/ice.png', -1)
else:
wall_texture = cv2.imread('../Textures/Untiled/grass.png', -1)
# Dilate and draw contours
# ------------------------------------------------------------------------------
print('Drawing Walls...')
kernel = np.ones((5, 5), np.uint8)
dilated = cv2.dilate(original, kernel, iterations=2)
dilated = cv2.cvtColor(dilated, cv2.COLOR_BGR2GRAY)
(thresh, dilated) = cv2.threshold(dilated, 128, 255, 0)
contours, hierarchy = cv2.findContours(dilated, cv2.RETR_TREE,
cv2.CHAIN_APPROX_NONE)
dilated = cv2.cvtColor(dilated, cv2.COLOR_GRAY2BGR)
cv2.drawContours(dilated, contours, -1, (0, 255, 0), 10)
# Texture walls
# ------------------------------------------------------------------------------
print('Texturing Walls...')
textured_walls = Image.fromarray(dilated)
textured_walls = textured_walls.convert('RGBA')
data = textured_walls.load()
# Convert all pixels within contour lines to transparent
width, height = textured_walls.size
for y in range(height):
for x in range(width):
if data[x, y] != (0, 0, 0, 255) and data[x, y] != (255, 255, 255,
255):
data[x, y] = (0, 255, 0, 0)
wall_texture = cv2.resize(wall_texture,
(int(original.shape[1]), int(original.shape[0])))
textured_walls = cv2.cvtColor(np.array(textured_walls),
cv2.COLOR_RGBA2BGRA)
# Separate RGB and Alpha channels, apply texture to Alpha portion, recombine
textured_walls_img = textured_walls[:, :, :3]
textured_walls_mask = textured_walls[:, :, 3:]
background_mask = 255 - textured_walls_mask
textured_walls_mask = cv2.cvtColor(textured_walls_mask, cv2.COLOR_GRAY2BGR)
background_mask = cv2.cvtColor(background_mask, cv2.COLOR_GRAY2BGR)
wall_texture_part = (wall_texture * (1 / 255.0)) * (background_mask *
(1 / 255.0))
textured_walls_part = (textured_walls_img *
(1 / 255.0)) * (textured_walls_mask * (1 / 255.0))
textured_walls = np.uint8(
cv2.addWeighted(wall_texture_part, 255.0, textured_walls_part, 255.0,
0.0))
# Texture floor
# ------------------------------------------------------------------------------
print('Texturing Floor...')
if tiling == False:
textured_walls = cv2.cvtColor(textured_walls, cv2.COLOR_BGRA2RGBA)
textured_combination = Image.fromarray(textured_walls)
textured_combination = textured_combination.convert('RGBA')
data = textured_combination.load()
# Convert all non-transparent white pixels to transparent
width, height = textured_combination.size
for y in range(height):
for x in range(width):
if data[x, y] == (255, 255, 255, 255):
data[x, y] = (255, 255, 255, 0)
floor_texture = cv2.resize(
floor_texture, (int(original.shape[1]), int(original.shape[0])))
textured_combination = cv2.cvtColor(np.array(textured_combination),
cv2.COLOR_RGBA2BGRA)
# Separate RGB and Alpha channels, apply texture to Alpha portion, recombine
textured_combination_img = textured_combination[:, :, :3]
textured_combination_mask = textured_combination[:, :, 3:]
background_mask = 255 - textured_combination_mask
textured_combination_mask = cv2.cvtColor(textured_combination_mask,
cv2.COLOR_GRAY2BGR)
background_mask = cv2.cvtColor(background_mask, cv2.COLOR_GRAY2BGR)
floor_texture_part = (floor_texture * (1 / 255.0)) * (background_mask *
(1 / 255.0))
textured_combination_part = (
textured_combination_img *
(1 / 255.0)) * (textured_combination_mask * (1 / 255.0))
textured_combination = np.uint8(
cv2.addWeighted(floor_texture_part, 255.0,
textured_combination_part, 255.0, 0.0))
else:
path = '../Templates/Tiles/'
floor_tile_templates = [
file for file in listdir(path) if isfile(join(path, file))
]
rotation = [
cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_180,
cv2.ROTATE_90_COUNTERCLOCKWISE
]
textured_combination = textured_walls
background = textured_walls
if floor_type == 'Brown':
tile_path = '../Textures/Brown_Tiles/'
elif floor_type == 'Cave':
tile_path = '../Textures/Cave_Tiles/'
elif floor_type == 'Runes':
tile_path = '../Textures/Runes_Tiles/'
elif floor_type == 'White':
tile_path = '../Textures/White_Tiles/'
tile_count = len(next(os.walk(path)))
for template in floor_tile_templates:
tile_template = cv2.imread(path + template, 0)
loc = match(tile_template, original_bw)
for pt in zip(*loc[::-1]):
foreground = cv2.imread(
tile_path + 'tile' + str(random.randint(1, tile_count)) +
'.png', -1)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
degrees = random.randint(0, 3)
if (degrees != 3):
foreground = cv2.rotate(foreground, rotation[degrees])
foreground = cv2.resize(foreground, (int(51), int(51)))
paste(pt[0], pt[1])
#loc = match(tile_template, original_bw)
# Texture Special Tiles
# Doors
# ------------------------------------------------------------------------------
print('Texturing Special Tiles...')
background = textured_combination
#foreground = cv2.resize(foreground, (int(51), int(51)))
path = '../Templates/Doors/Horizontal/'
horizontal_door_templates = [
file for file in listdir(path) if isfile(join(path, file))
]
for template in horizontal_door_templates:
door_template = cv2.imread(path + template, 0)
loc = match(door_template, original_bw)
foreground = cv2.imread('../Textures/Doors/doorh_small.png', -1)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1] + 18)
path = '../Templates/Doors/Vertical/'
vertical_door_templates = [
file for file in listdir(path) if isfile(join(path, file))
]
for template in vertical_door_templates:
door_template = cv2.imread(path + template, 0)
loc = match(door_template, original_bw)
foreground = cv2.imread('../Textures/Doors/doorv_small.png', -1)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0] + 18, pt[1])
# Stairs
# ------------------------------------------------------------------------------
temp = cv2.imread('../Templates/Stairs/stairs_pointdown.png', 0)
loc = match(temp, original_bw)
foreground = cv2.imread('../Textures/Stairs/stairsv.png', -1)
foreground = cv2.rotate(foreground, cv2.ROTATE_180)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1])
temp = cv2.imread('../Templates/Stairs/stairs_pointleft.png', 0)
loc = match(temp, original_bw)
foreground = cv2.imread('../Textures/Stairs/stairsh.png', -1)
foreground = cv2.rotate(foreground, cv2.ROTATE_180)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1])
temp = cv2.imread('../Templates/Stairs/stairs_pointright.png', 0)
loc = match(temp, original_bw)
foreground = cv2.imread('../Textures/Stairs/stairsh.png', -1)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1])
temp = cv2.imread('../Templates/Stairs/stairs_pointup.png', 0)
loc = match(temp, original_bw)
foreground = cv2.imread('../Textures/Stairs/stairsv.png', -1)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1])
temp = cv2.imread('../Templates/Stairs/stairs_opendown.png', 0)
loc = match(temp, original_bw)
foreground = cv2.imread('../Textures/Stairs/stairsv.png', -1)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1])
temp = cv2.imread('../Templates/Stairs/stairs_openleft.png', 0)
loc = match(temp, original_bw)
foreground = cv2.imread('../Textures/Stairs/stairsh.png', -1)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1])
temp = cv2.imread('../Templates/Stairs/stairs_openright.png', 0)
loc = match(temp, original_bw)
foreground = cv2.imread('../Textures/Stairs/stairsh.png', -1)
foreground = cv2.rotate(foreground, cv2.ROTATE_180)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1])
temp = cv2.imread('../Templates/Stairs/stairs_openup.png', 0)
loc = match(temp, original_bw)
foreground = cv2.imread('../Textures/Stairs/stairsv.png', -1)
foreground = cv2.rotate(foreground, cv2.ROTATE_180)
foreground = cv2.cvtColor(foreground, cv2.COLOR_BGRA2BGR)
for pt in zip(*loc[::-1]):
paste(pt[0], pt[1])
# Final image
cv2.imwrite(final_path, background)
def main():
read_sequence = os.listdir(operatedir_video) # read all file name
seqence_Len = len(read_sequence) # get all file number
img_path = operatedir_video + "5.jpg"
video = cv2.imread(img_path) #read the first one to get the image size
gray_video = cv2.cvtColor(video, cv2.COLOR_BGR2GRAY)
Len_steam = 5
H, W = gray_video.shape #get size of image
H_start = 80
H_end = 200
steam = np.zeros((Len_steam, H_end - H_start, W))
steam2 = np.zeros((Len_steam, H, W))
save_sequence_num = 0 # processing iteration initial
addition_window_shift = 0 # innitial shifting parameter
Window_ki_error = 0
Window_kp_error = 0
Kp = 0 # initial shifting paramerter
for sequence_num in range(seqence_Len):
#for i in os.listdir("E:/estimagine/vs_project/PythonApplication_data_au/pic/"):
start_time = time()
# read imag for process
img_path = operatedir_video + str(sequence_num +
0) + ".jpg" # starting from 10
video = cv2.imread(img_path)
gray_video = cv2.cvtColor(video, cv2.COLOR_BGR2GRAY)
if (sequence_num < 10):
# bffer a resized one to coputer the path and cost matrix
steam = np.append(steam, [gray_video[H_start:H_end, :]],
axis=0) # save sequence
# normal beffer process
steam = np.delete(steam, 0, axis=0)
steam2 = np.append(steam2, [gray_video],
axis=0) # save sequence
steam2 = np.delete(steam2, 0, axis=0)
else:
steam = np.append(steam, [gray_video[H_start:H_end, :]],
axis=0) # save sequence
# no longer delete the fist one
steam = np.delete(steam, 1, axis=0)
steam2 = np.append(steam2, [gray_video],
axis=0) # save sequence
steam2 = np.delete(steam2, 0, axis=0)
# shifting used is zero in costmatrix caculation
#Costmatrix,shift_used = COSTMtrix.matrix_cal_corre_full_version_2(steam,0)
overall_shifting, shift_used1 = COSTMtrix.Img_fully_shifting_distance(
steam[Len_steam - 1, :, :], steam[Len_steam - 2, :, :],
addition_window_shift)
#overall_shifting0,shift_used0 = COSTMtrix.Img_fully_shifting_correlation(steam[Len_steam-1,:,:],
# steam[Len_steam-2,:,:], addition_window_shift)
#overall_shifting = overall_shifting
#Corrected_img,path,path_cost= VIDEO_PEOCESS.correct_video_with_shifting(gray_video,overall_shifting,int(sequence_num),shift_used1 )
Costmatrix = np.zeros((Window_LEN, W))
#test_show = steam2[Len_steam-2,:,:]
#cv2.imshow('correcr video',test_show.astype(np.uint8))
Costmatrix, shift_used2 = COSTMtrix.matrix_cal_corre_full_version3_2GPU(
steam2[Len_steam - 1, :, :], steam2[Len_steam - 2, :, :],
0)
###Costmatrix = Costmatrix2
#Costmatrix = cv2.blur(Costmatrix,(5,5))
Costmatrix = myfilter.gauss_filter_s(
Costmatrix) # smooth matrix
###get path and correct image
###Corrected_img,path,path_cost= VIDEO_PEOCESS.correct_video(gray_video,Costmatrix,int(i),addition_window_shift +Kp )
Corrected_img, path, path_cost = VIDEO_PEOCESS.correct_video(
gray_video, overall_shifting, Costmatrix,
int(sequence_num),
shift_used2 + Window_ki_error + Window_kp_error)
#overall_shifting3,shift_used3 = COSTMtrix.Img_fully_shifting_correlation(Corrected_img[H_start:H_end,:],
# steam[0,:,:], 0)
#Corrected_img,path,path_cost= VIDEO_PEOCESS.correct_video_with_shifting(Corrected_img,overall_shifting3,int(sequence_num),shift_used3 )
# remove the central shifting
#addition_window_shift = -0.00055*(np.mean(path)- int(Window_LEN/2))+addition_window_shift
path_mean_error = (np.mean(path) - int(Window_LEN / 2))
shift_mean_error = int(overall_shifting -
int(Overall_shiftting_WinLen / 2))
addition_window_shift = shift_mean_error + addition_window_shift
#addition_window_shift = 0
#Window_kp_error = - 0.1* path_mean_error
#Window_ki_error = -0.000005*path_mean_error+Window_ki_error
#re！！！！！Next time remenber to remove the un-corrected image from the stream
steam = np.append(steam, [Corrected_img[H_start:H_end, :]],
axis=0) # save sequence
# no longer delete the fist one
steam = np.delete(steam, 1, axis=0)
steam2 = np.append(steam2, [Corrected_img],
axis=0) # save sequence
# no longer delete the fist one
steam2 = np.delete(steam2, 0, axis=0)
if (Save_signal_flag == True):
new = np.zeros((signal_saved.DIM, 1))
new[Save_signal_enum.image_iD.value] = sequence_num
new[Save_signal_enum.additional_kp.value] = Kp
new[Save_signal_enum.additional_ki.
value] = addition_window_shift
new[Save_signal_enum.path_cost.value] = path_cost
new[Save_signal_enum.mean_path_error.
value] = path_mean_error
signal_saved.add_new_iteration_result(new, path)
signal_saved.display_and_save2(sequence_num, new)
test_time_point = time()
show1 = Costmatrix
new_frame = cv2.rotate(Corrected_img, rotateCode=2)
circular = cv2.linearPolar(
new_frame,
(new_frame.shape[1] / 2, new_frame.shape[0] / 2), 200,
cv2.WARP_INVERSE_MAP)
for i in range(len(path)):
show1[int(path[i]), i] = 254
cv2.imwrite(savedir_path + str(sequence_num) + ".jpg",
circular)
cv2.imwrite(
operatedir_matrix_unprocessed + str(sequence_num) + ".jpg",
Costmatrix)
cv2.imwrite(operatedir_matrix + str(sequence_num) + ".jpg",
show1)
cv2.imwrite(savedir_rectan_ + str(sequence_num) + ".jpg",
Corrected_img)
print("[%s] is processed. test point time is [%f] " %
(sequence_num, test_time_point - start_time))
import cv2
import numpy as np
# load image
cap = cv2.VideoCapture(0) # 0 is for 1 st webcam, if u more number strt like 1, 2 etc
while True:
ret, frame = cap.read()
width = int(cap.get(3)) #- 3 is the proeprty to get width
height = int(cap.get(4)) #- 4 is the property of height
image = np.zeros(frame.shape,np.uint8)
smaller_frame = cv2.resize(frame,(0,0),fx=0.5,fy=0.5) # shrinked to 4 peices as i shrinked heighta and width
# image[:height//2,:width//2] = smaller_frame
# image[height//2:,:width//2] = smaller_frame
# image[:height//2,width//2:] = smaller_frame
# image[height//2:,width//2:] = smaller_frame
image[:height//2,:width//2] = cv2.rotate(smaller_frame,cv2.cv2.ROTATE_180)
image[height//2:,:width//2] = smaller_frame
image[:height//2,width//2:] = cv2.rotate(smaller_frame,cv2.cv2.ROTATE_180)
image[height//2:,width//2:] = smaller_frame
cv2.imshow('image',image)
if cv2.waitKey(1) == ord('q'): # ordinal value
break
cap.release()
cv2.destroyAllWindows()
else:
print("Successfully created the directory %s " % os.getcwd() + DATASET_DIR)
# create daemon thread
t1 = threading.Thread(target=thread_function, args=(1,), daemon=True)
t1.start()
save_counter = 0
# capture frames from the camera
for frame in cap.capture_continuous(rawCapture, format="bgr", use_video_port=True):
# NumPy array representing the image
image = frame.array
# rotate frame
image = cv2.rotate(image, cv2.ROTATE_180)
# save 1/3 of images only
if save_counter == 3:
msg_lock.acquire()
image_name = datasets.datetime_st_uuid4(msg)
datasets.save_snapshot(os.getcwd() + DATASET_DIR, IMG_WIDTH, IMG_HEIGHT, image, image_name)
msg_lock.release()
save_counter = 0
else:
save_counter += 1
# show the image
# cv2.imshow("Frame", image)
# clear the stream for the next frame
piecesC = fgbg.apply(croppedC,learningRate = 0)
piecesC[piecesC==127]=0
kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE,(9,9))
piecesC = cv.morphologyEx(piecesC, cv.MORPH_OPEN, kernel)
piecesC = cv.morphologyEx(piecesC, cv.MORPH_CLOSE, kernel)
croppedC[piecesC==0] = (255,0,0)
print "done"
diff = cv.absdiff(croppedC, croppedP)
diff = cv.cvtColor(diff, cv.COLOR_BGR2GRAY)
diff = cv.threshold(diff, 25, 255, cv.THRESH_BINARY)[1]
kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE,(9,9))
diff = cv.morphologyEx(diff, cv.MORPH_OPEN, kernel)
diff = cv.morphologyEx(diff, cv.MORPH_CLOSE, kernel)
diff = cv.rotate(diff,angle)
# count number of non 0 pixels in the image
counted = locateMove(diff,squares)
sorted_by_value = sorted(counted.items(), key=operator.itemgetter(1), reverse = True)
print sorted_by_value[0][0] + sorted_by_value[1][0]
node = validateMove(board, sorted_by_value, writer, node)
print writer
isGameOver(board,writer)
cv.imshow('diff',diff)
cv.imshow('croppedC', croppedC)
cv.imshow('croppedP', croppedP)
croppedP = np.copy(croppedC)
for f in os.listdir(filePath):
with open(filePath + 'Results.txt', "a") as result:
with open(filePath + f, "rb") as fin:
image = np.fromstring(fin.read(), np.uint8)
imageDecoded = base64.encodestring(image).decode('utf-8')
baseImageData = base64.b64decode(imageDecoded)
imagePil = Image.open(io.BytesIO(baseImageData))
imageMat = np.asarray(imagePil)
print(imageMat.shape)
resized_image = []
if imageMat.shape == (768, 1024, 3):
imageRotated = cv2.rotate(imageMat, cv2.ROTATE_90_COUNTERCLOCKWISE)
resized_image = cv2.resize(imageRotated, (480, 640))
print(resized_image.shape)
else:
resized_image = imageMat
resized_image = cv2.cvtColor(resized_image, cv2.COLOR_BGR2RGB)
cv2.imwrite(filePath + "d1.jpg", resized_image)
im_pil = Image.fromarray(resized_image)
imgByteArr = io.BytesIO()
im_pil.save(imgByteArr, format='jpeg')
imgByteArr = imgByteArr.getvalue()
base64_seg_map = base64.b64encode(imgByteArr).decode('utf-8')
for sample_idx in range(len(demo_dataset)):
metadatas, bev_map, img_rgb = demo_dataset.load_bevmap_front(
sample_idx)
detections, bev_map, fps = do_detect(configs,
model,
bev_map,
is_front=True)
# Draw prediction in the image
bev_map = (bev_map.permute(1, 2, 0).numpy() * 255).astype(np.uint8)
bev_map = cv2.resize(bev_map, (cnf.BEV_WIDTH, cnf.BEV_HEIGHT))
bev_map = draw_predictions(bev_map, detections,
configs.num_classes)
# Rotate the bev_map
bev_map = cv2.rotate(bev_map, cv2.ROTATE_180)
img_path = metadatas['img_path'][0]
img_bgr = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR)
calib = Calibration(configs.calib_path)
kitti_dets = convert_det_to_real_values(detections)
if len(kitti_dets) > 0:
kitti_dets[:,
1:] = lidar_to_camera_box(kitti_dets[:,
1:], calib.V2C,
calib.R0, calib.P2)
img_bgr = show_rgb_image_with_boxes(img_bgr, kitti_dets, calib)
out_img = merge_rgb_to_bev(img_bgr,
bev_map,
output_width=configs.output_width)
duraciont = length/fps
duracionfps= duraciont/length
print( length )
print( fps )
print( duraciont )
print( duracionfps)
img_index =0
# threshold =0.8
while (cap.isOpened()):
ret, frame = cap.read()
if ret ==False:
break
img_rotate_90_clockwise = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
cv2.imwrite('/home/nicolas/Universidad/códigos/Proyecto/' + str(img_index) + '.png', img_rotate_90_clockwise)
img_index += 1
# template = cv2.imread('/home/nicolas/Universidad/códigos/Proyecto/objeto.png',0)
# face_w, face_h = template.shape[::-1]
# img_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# res = cv2.matchTemplate(img_gray,template,cv2.TM_CCOEFF_NORMED)
# loc = np.where(res >= threshold)
# for pt in zip(*loc[::-1]):
# #puting rectangle on recognized erea
# cv2.rectangle(frame, pt, (pt[0] + face_w, pt[1] + face_h), (0,0,255), 2)
# cv2.imshow("detected", frame)
def rotate(self, angle, resample=NEAREST, expand=False, center=None, translate=None, fillcolor=None):
# use original PIL code to generate affine matrix
angle = angle % 360.0
if not (center or translate):
if angle == 0:
return self.copy()
if angle == 180:
return Image(cv2.rotate(self._mat, cv2.ROTATE_180), self.mode)
if angle == 90 and expand:
return Image(cv2.rotate(self._mat, cv2.ROTATE_90_COUNTERCLOCKWISE), self.mode)
if angle == 270 and expand:
return Image(cv2.rotate(self._mat, cv2.ROTATE_90_CLOCKWISE), self.mode)
w, h = self.size
if translate is None:
post_trans = (0, 0)
else:
post_trans = translate
if center is None:
# FIXME These should be rounded to ints?
rotn_center = (w / 2.0, h / 2.0)
else:
rotn_center = center
angle = -math.radians(angle)
matrix = [
round(math.cos(angle), 15),
round(math.sin(angle), 15),
0.0,
round(-math.sin(angle), 15),
round(math.cos(angle), 15),
0.0,
]
def transform(x, y, matrix):
(a, b, c, d, e, f) = matrix
return a * x + b * y + c, d * x + e * y + f
matrix[2], matrix[5] = transform(
-rotn_center[0] - post_trans[0], -rotn_center[1] - post_trans[1], matrix
)
matrix[2] += rotn_center[0]
matrix[5] += rotn_center[1]
if expand:
# calculate output size
xx = []
yy = []
for x, y in ((0, 0), (w, 0), (w, h), (0, h)):
x, y = transform(x, y, matrix)
xx.append(x)
yy.append(y)
nw = math.ceil(max(xx)) - math.floor(min(xx))
nh = math.ceil(max(yy)) - math.floor(min(yy))
# We multiply a translation matrix from the right. Because of its
# special form, this is the same as taking the image of the
# translation vector as new translation vector.
matrix[2], matrix[5] = transform(-(nw - w) / 2.0, -(nh - h) / 2.0, matrix)
w, h = nw, nh
newmat = cv2.warpAffine(self._mat, np.array(matrix).reshape(2, 3), (w,h), flags=resample, borderMode=cv2.BORDER_CONSTANT, borderValue=fillcolor)
return Image(newmat, self.mode)
def extract_poses(filename, single_person, Notuse_tiny_yolo):
hrnet_m = 'HRNet'
hrnet_c = 32
hrnet_j = 17
hrnet_weights = "./weights/pose_hrnet_w32_256x192.pth"
image_resolution = '(384, 288)'
max_batch_size = 16
# device = None
# if device is not None:
# device = torch.device(device)
# else:
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
device = torch.device('cuda')
else:
device = torch.device('cpu')
# print(device)
dict_frametoJson = {"frames": []}
image_resolution = ast.literal_eval(image_resolution)
rotation_code = check_video_rotation(filename)
video = cv2.VideoCapture(filename)
assert video.isOpened()
# nof_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
if Notuse_tiny_yolo: # default 는 tiny yolo를 사용
yolo_model_def = "./models/detectors/yolo/config/yolov3.cfg"
yolo_class_path = "./models/detectors/yolo/data/coco.names"
yolo_weights_path = "./models/detectors/yolo/weights/yolov3.weights"
else:
yolo_model_def = "./models/detectors/yolo/config/yolov3-tiny.cfg"
yolo_class_path = "./models/detectors/yolo/data/coco.names"
yolo_weights_path = "./models/detectors/yolo/weights/yolov3-tiny.weights"
model = SimpleHRNet(hrnet_c,
hrnet_j,
hrnet_weights,
model_name=hrnet_m,
resolution=image_resolution,
multiperson=not single_person,
max_batch_size=max_batch_size,
yolo_model_def=yolo_model_def,
yolo_class_path=yolo_class_path,
yolo_weights_path=yolo_weights_path,
device=device)
index = 0
while True:
# t = time.time()
ret, frame = video.read()
if not ret:
break
if rotation_code is not None:
frame = cv2.rotate(frame, rotation_code)
pts = model.predict(frame)
for j, pt in enumerate(pts):
keypoint_byframe = {"frameNo": 0, "keyPoints": []}
keyNum = 0
row = [index, j] + pt.flatten().tolist()
if j == 0: # Person의 경우만
keypoint_byframe["frameNo"] = index
for idx in range(2, len(row), 3):
keypoint_byframe["keyPoints"].append(
makePoint(row[idx], row[idx + 1], keyNum,
row[idx + 2]))
keyNum += 1
dict_frametoJson["frames"].append(keypoint_byframe)
# fps = 1. / (time.time() - t)
# print('\rframe: % 4d / %d - framerate: %f fps ' % (index, nof_frames - 1, fps), end='')
index += 1
return json.dumps(dict_frametoJson)
def phone_autoCut(photo):
# Make photo into numpy.ndarray
print(type(photo))
if type(photo) == str:
new_img = cv2.imread(photo, cv2.IMREAD_COLOR)
elif type(photo) == Image.Image or type(
photo) == PIL.JpegImagePlugin.JpegImageFile:
new_img = np.array(photo)
elif type(photo) == np.ndarray:
new_img = copy.deepcopy(photo)
try:
# Resize & Rotate
height, width, channel = new_img.shape
new_width = 540
new_height = int(height * new_width / width)
new_img = cv2.resize(new_img,
dsize=(new_width, new_height),
interpolation=cv2.INTER_AREA)
new_img = cv2.rotate(new_img, cv2.ROTATE_90_CLOCKWISE)
except:
pass
if True:
# Grayscale
gray_img = cv2.cvtColor(new_img, cv2.COLOR_BGR2GRAY)
# cv2.imshow('gray', gray_img)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# Negative & Threshold
gray_img = cv2.bitwise_not(gray_img)
ret, gray_img = cv2.threshold(gray_img, 100, 255, cv2.THRESH_BINARY)
# cv2.imshow('gray', gray_img)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# Find Contours
img, cont, hier = cv2.findContours(gray_img, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
max_area = 0
max_index = 0
for i in range(len(cont)):
area = cv2.contourArea(cont[i])
if area > max_area:
max_area = area
max_index = i
max_cont = cont[max_index]
temp_img = copy.deepcopy(new_img)
cv2.drawContours(temp_img, [max_cont], -1, (0, 255, 0), 3)
# cv2.imshow('gray', temp_img)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# Dilation
mask = np.zeros(gray_img.shape, np.uint8)
mask = cv2.drawContours(mask, [max_cont], -1, (255, 255, 255), 5)
kernel = np.ones((30, 30), np.uint8)
dilated = cv2.dilate(mask, kernel, iterations=1)
img, cont, hier = cv2.findContours(dilated, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
max_area = 0
max_index = 0
for i in range(len(cont)):
area = cv2.contourArea(cont[i])
if area > max_area:
max_area = area
max_index = i
max_cont = cont[max_index]
# cv2.imshow('gray', dilated)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# Approx
epsilon = 0.1 * cv2.arcLength(max_cont, True)
approx = cv2.approxPolyDP(max_cont, epsilon, True)
# Convex Hull
hull = cv2.convexHull(approx)
temp_img = copy.deepcopy(new_img)
cv2.drawContours(temp_img, [hull], -1, (0, 0, 255), 3)
# cv2.imshow('gray', temp_img)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# Hull Size Calc
lines = []
for i in range(len(hull)):
lines.append(
np.linalg.norm(hull[i][0] - hull[(i + 1) % len(hull)][0]))
lines.sort()
# print(lines)
# Front View
points = []
for p in hull:
points.append(p[0])
points = np.array(points)
img_phone = transform(new_img, points,
(int(lines[1]), int(lines[3] * 1.2)))
# cv2.imshow('gray', img_phone)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
return img_phone
def rotate_image(self, bgr):
bgr = cv2.rotate(bgr)
return bgr
# --- move from PyGame to CV2 ---
color_image = pygame.surfarray.array3d(pg_img)
color_image = cv2.transpose(color_image)
color_image = cv2.cvtColor(color_image, cv2.COLOR_RGB2BGR)
# --- display CV2 image ---
cv2.imshow('Color', color_image)
cv2.waitKey(0)
cv2.destroyAllWindows()
# --- change CV2 image ---
color_image = cv2.rotate(color_image, cv2.ROTATE_90_CLOCKWISE)
gray_image = cv2.cvtColor(color_image, cv2.COLOR_BGR2GRAY)
# --- display CV2 image ---
cv2.imshow('Gray', gray_image)
cv2.waitKey(0)
cv2.destroyAllWindows()
# --- save in file in CV2 ---
cv2.imwrite('test.png', color_image)
# --- move back from CV2 to PyGame ---
gray_image = cv2.cvtColor(gray_image, cv2.COLOR_GRAY2RGB)