def test_cuda_interop(self):
npMat = (np.random.random((128, 128, 3)) * 255).astype(np.uint8)
cuMat = cv.cuda_GpuMat()
cuMat.upload(npMat)
self.assertTrue(cuMat.cudaPtr() != 0)
stream = cv.cuda_Stream()
self.assertTrue(stream.cudaPtr() != 0)
asyncstream = cv.cuda_Stream(1) # cudaStreamNonBlocking
self.assertTrue(asyncstream.cudaPtr() != 0)
def run(use_random=True, visualize=False):
cuda_stream = cv2.cuda_Stream()
backSub = cv2.cuda.createBackgroundSubtractorMOG2(history=100,
varThreshold=16,
detectShadows=False)
# backSub = cv2.createBackgroundSubtractorKNN(history=30, dist2Threshold=400.0, detectShadows=True)
warped_list = np.load("X:/frank.npy", mmap_mode='r')
N = 1000
bg_list = []
for frame_counter in range(N):
if use_random: index = random.randint(0, len(warped_list) - 1)
else: index = frame_counter % len(warped_list)
warped = warped_list[index]
warped_gpu = cv2.cuda_GpuMat()
warped_gpu.upload(warped)
fgMask = backSub.apply(warped_gpu, -1, cuda_stream)
fgMask = fgMask.download()
fgMask = cv2.morphologyEx(fgMask,
cv2.MORPH_CLOSE,
kernel=np.ones((5, 5), np.uint8))
if visualize: cv2.imshow('FG Mask', fgMask)
bg = cv2.cuda_GpuMat(warped_gpu.size(), warped_gpu.type())
backSub.getBackgroundImage(cuda_stream, bg)
bg = bg.download()
if visualize: cv2.imshow('BG', bg)
key = cv2.waitKey(1)
if key == 27:
break
cv2.imwrite("X:/final.png", bg)
if visualize: cv2.waitKey(0)
cv2.destroyAllWindows()
def run(use_random=True, visualize=False):
cuda_stream = cv2.cuda_Stream()
backSub = cv2.cuda.createBackgroundSubtractorMOG2(history=100,
varThreshold=16,
detectShadows=False)
# backSub = cv2.createBackgroundSubtractorKNN(history=30, dist2Threshold=400.0, detectShadows=True)
warped_list = np.load("X:/warped.npy", mmap_mode='r')
valid_mask_list = np.load("X:/mask.npy", mmap_mode='r')
i = 0
frame_counter = 0
mode = True
N = 1000
bg_list = []
for frame_counter in range(N):
if use_random: index = random.randint(0, len(warped_list) - 1)
else: index = frame_counter % len(warped_list)
warped = warped_list[index]
valid_mask = valid_mask_list[index]
frame_counter += 1
if frame_counter > 500 and random.randint(0, N - frame_counter) < 200:
warped_final = bg_list[random.randint(0, len(bg_list) - 1)]
else:
if frame_counter < 200:
mean_canvas = np.ones(warped.shape, dtype="uint8") * 200
mean_canvas = cv2.bitwise_or(mean_canvas,
mean_canvas,
mask=cv2.bitwise_not(valid_mask))
else:
mean_canvas = cv2.bitwise_or(bg,
bg,
mask=cv2.bitwise_not(valid_mask))
if frame_counter % 8 == 0: bg_list.append(bg)
valid_mask = cv2.bitwise_or(warped, warped, mask=valid_mask)
final = cv2.bitwise_or(mean_canvas, valid_mask)
if visualize: cv2.imshow('Passed', final)
final_gpu = cv2.cuda_GpuMat()
final_gpu.upload(final)
final = cv2.cuda_GpuMat(final)
fgMask = backSub.apply(final, -1, cuda_stream)
fgMask = fgMask.download()
# fgMask = cv2.morphologyEx(fgMask, cv2.MORPH_CLOSE, kernel=np.ones((5,5),np.uint8))
# cv2.imshow('FG Mask', fgMask)
bg = cv2.cuda_GpuMat(final_gpu.size(), final_gpu.type())
backSub.getBackgroundImage(cuda_stream, bg)
bg = bg.download()
bg_list.append(bg)
if visualize:
cv2.imshow('BG', bg)
key = cv2.waitKey(1)
if key == 27:
break
if key == ord('m'):
mode = not mode
if key == ord(' '):
cv2.imwrite(str(i) + ".jpg", warped)
i += 1
cv2.imwrite("X:/final.png", bg)
# cv2.waitKey(0)
cv2.destroyAllWindows()
def test_cuda_upload_download_stream(self):
stream = cv.cuda_Stream()
npMat = (np.random.random((128, 128, 3)) * 255).astype(np.uint8)
cuMat = cv.cuda_GpuMat(128,128, cv.CV_8UC3)
cuMat.upload(npMat, stream)
npMat2 = cuMat.download(stream=stream)
stream.waitForCompletion()
self.assertTrue(np.allclose(npMat2, npMat))
def cuda_bgsub(get_frames) -> list:
stream = cv2.cuda_Stream()
# cv2.createB
mask: cv2.cuda.createBackgroundSubtractorMOG2 = cv2.cuda.createBackgroundSubtractorMOG2(
history=3, varThreshold=100, detectShadows=False)
post_process = []
while get_frames.more():
frame = get_frames.read()
processed = mask.apply(frame, -1, stream)
post_process.append(processed)
return post_process
def apply_BGS_GPU(algorithm,
video_file: str):
"""
Apply CUDA implementation of an algorithm
"""
cuda_stream = cv2.cuda_Stream()
video_stream = video_queue(video_file, target="cuda").start()
start_time = time.time()
frame_counter = 0
while video_stream.more():
frame = video_stream.read()
frame2 = algorithm.apply(frame, 0.1, cuda_stream)
frame_counter += 1
fps = frame_counter / (time.time() - start_time)
return fps
def __init__(self):
self.rows = 1536
self.cols = 2048
self.bgmog2 = cv2.cuda.createBackgroundSubtractorMOG()
self.stream = cv2.cuda_Stream()
self.frame = PinnedMem((rows, cols, 1))
self.frame_device = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC3)
self.frame_device_r = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1)
self.frame_device_g = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1)
self.frame_device_b = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1)
self.frame_device_fg_r = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1)
self.frame_device_fg_g = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1)
self.frame_device_fg_b = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1)
self.frame_device_mask = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1)
self.fg_host = PinnedMem((rows, cols, 1))
def cuda_median(video_stream,
filter: cv2.cuda_Filter,
mask: cv2.cuda.createBackgroundSubtractorMOG2,
gaussian=False) -> list:
mask = cv2.cuda.createBackgroundSubtractorMOG2(history=3,
varThreshold=100,
detectShadows=False)
post_process = []
total = 0
cuda_stream = cv2.cuda_Stream()
if gaussian:
filter = cv2.cuda.createGaussianFilter(cv2.CV_8UC1, cv2.CV_8UC1,
(3, 3), 0.5)
while video_stream.more():
# processed = mask.apply(frame, -1, stream)
frame = video_stream.read()
if frame is None:
break
print(frame, mask, cuda_stream)
processed = mask.apply(frame, 0.009, cuda_stream)
filter.apply(processed)
post_process.append(processed)
return post_process
def cudaDetect(img,gpumat):
h,w = img.shape[:2]
img = cv2.resize(img,(int(w*0.5),int(h*0.5)),interpolation=cv2.INTER_AREA)
img = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
img = cv2.medianBlur(img,1)
img = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,9,-10)
kernel = np.ones((3,3),np.uint8)
leftimg = cv2.dilate(img[0:w//4,0:h//2],kernel,iterations=1)
rightimg = cv2.dilate(img[w//4:w//2,0:h//2],kernel,iterations=1)
#cudaコアへ転送
gpumat.upload(leftimg)
#cuda streamの生成
stream = cv2.cuda_Stream()
detector = cv2.cuda.createHoughSegmentDetector(1,np.pi/180,minLineLength=100, maxLineGap=0)
lines_gpu = detector.detect(gpumat)
leftlines = lines_gpu.download()
#cudaコアへ転送
gpumat.upload(rightimg)
#cuda streamの生成
stream = cv2.cuda_Stream()
detector = cv2.cuda.createHoughSegmentDetector(1,np.pi/180,minLineLength=100, maxLineGap=0)
lines_gpu = detector.detect(gpumat)
rightlines = lines_gpu.download()
lx1_ = 0.0
lx2_ = 0.0
ly1_ = 0.0
ly2_ = 0.0
lrad = 0.0
if leftlines is not None:
for line in leftlines[0]:
#print(line)
lx1_ += line[0]
ly1_ += line[1]
lx2_ += line[2]
ly2_ += line[3]
lrad = (ly2_-ly1_)/(lx2_-lx1_)
rx1_ = 0.0
rx2_ = 0.0
ry1_ = 0.0
ry2_ = 0.0
rrad = 0.0
if rightlines is not None:
for line in rightlines[0]:
#print(line)
rx1_ += line[0]
rx2_ += line[1]
ry1_ += line[2]
ry2_ += line[3]
rrad = (ry2_-ry1_)/(rx2_-rx1_)
print(lrad-rrad)
return lrad-rrad
0.004428959364733587, 0.06865838341764481
]]) # Humanoid
# dist = np.array([[0.02220329099612066, 0.13530759611493004, -0.0041870520396677805, 0.007599954530058233, -0.4722284261198788]]) # ESP32
rvec = np.array([0.0, 0.0, 0.0]) # float only
tvec = np.array([0.0, 0.0, 0.0]) # float only
# cap = cv2.VideoCapture(cv2.CAP_DSHOW)
# codec = 0x47504A4D # MJPG
# cap.set(cv2.CAP_PROP_FPS, 30.0)
# cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('m','j','p','g'))
# cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M','J','P','G'))
# cap.set(3, 1920)
# cap.set(4, 1080)
cap = cv2.VideoCapture("X:/out.avi")
cuda_stream = cv2.cuda_Stream()
parameters = cv2.aruco.DetectorParameters_create()
# parameters(doCornerRefinement=True)
# markerLength=0.039 # real
# markerSeparation=0.0975 # real
markerLength = 0.04
markerSeparation = 0.01
# board = cv2.aruco.GridBoard_create(markersX=10, markersY=10, markerLength=0.039, markerSeparation=0.0975, dictionary=dictionary) # real
board = cv2.aruco.GridBoard_create(markersX=10,
markersY=10,
markerLength=markerLength,
markerSeparation=markerSeparation,
dictionary=dictionary)
backSub = cv2.cuda.createBackgroundSubtractorMOG2(history=100,
varThreshold=16,
detectShadows=False)
def worker_main(args, n, workunit):
# Assign a GPU to us
assign_gpu(n)
# Load the neural network
net, nn_size = load_network(n, args)
# Open the video file
video = cv.VideoCapture(args.video)
# For historical reasons, frames are referenced according to their timestamp
# according to the following function.
#
# The frame number should be queried *before* the frame itself is captured.
#
# This was tested and matches the number given by CAP_PROP_POS_MSEC. OpenCV
# uses a frames per second (e.g., 15.0) rather than the microseconds per
# frame (e.g., 66666) that is actually encoded in the file.
framerate = video.get(cv.CAP_PROP_FPS)
get_timestamp = lambda fn: int(1e3 * fn / framerate)
# Seek to the first frame we care about
video.set(cv.CAP_PROP_POS_FRAMES, workunit[0])
assert int(video.get(cv.CAP_PROP_POS_FRAMES)) == workunit[0]
# Create the optical flow calculator
flowengine = cv.cuda_FarnebackOpticalFlow.create(
args.of_levels, args.of_pyr_scale, False, args.of_winsize,
args.of_iterations, args.of_poly_n, args.of_poly_sigma, 0)
# Create the background subtractor
bgsub = cv.cuda.createBackgroundSubtractorMOG2(
history=args.bg_history,
varThreshold=args.bg_var_threshold,
detectShadows=False)
# Create the opening filter
kernel = cv.getStructuringElement(cv.MORPH_RECT, (7, 7))
filter = cv.cuda.createMorphologyFilter(cv.MORPH_OPEN, cv.CV_8UC4, kernel)
prev = None
iterator = range(workunit[0], workunit[2])
if args.progress:
iterator = tqdm.tqdm(iterator, position=n, desc=f'G{n:02}')
# If --of-history is passed, we will use the flow from the previous frame
# to inform the flow in this frame.
last_flow = None
# Compute the prefix of the video name, for building frame filenames later
name_prefix, _ = os.path.splitext(os.path.basename(args.video))
stream = cv.cuda_Stream()
for nf in iterator:
# Read the next frame
success, frame = video.read()
assert success
# Upload the frame to the device
frame_local = frame[:]
frame = cv.cuda_GpuMat(frame.shape[0], frame.shape[1], cv.CV_8UC3)
frame.upload(frame_local, stream=stream)
# Resize the frame if necessary
if args.resize:
frame = cv.cuda.resize(frame, tuple(args.resize), stream=stream)
# Compute the output filename
out = '%s_%i' % (name_prefix, get_timestamp(nf))
# Determine whether we will want to save this image
save_this = args.frame_list is None or out in args.frame_list
# Modify the out path to use whatever directory prefix we are sorting
# into.
if args.frame_list is not None:
out = os.path.join(args.frame_list.get(out, './'), out)
# -- Foreground extraction --------------------------------------------
# Apply background subtraction to determine the mask
mask = bgsub.apply(frame, -1, stream=stream)
# Store the result in the green channel
green_channel = mask
# First exit early spot: If this purely for prepping the background
# subtractor, we don't need anything further.
if nf < workunit[1] - 1:
continue
# -- Raw image --------------------------------------------------------
# Convert the frame to grayscale and store it in the red channel
gray = cv.cuda.cvtColor(frame, cv.COLOR_BGR2GRAY, stream=stream)
red_channel = gray
# -- Optical flow -----------------------------------------------------
# Equalize the luminance histogram of the image
if args.of_equalize_luminance:
y, u, v = cv.cuda.split(cv.cuda.cvtColor(frame,
cv.COLOR_BGR2YUV,
stream=stream),
stream=stream)
y = cv.cuda.equalizeHist(y, stream=stream)
eqframe = cv.cuda_GpuMat(y.size(), cv.CV_8UC3)
cv.cuda.merge((y, u, v), eqframe, stream=stream)
eqframe = cv.cuda.cvtColor(eqframe,
cv.COLOR_YUV2RGB,
stream=stream) # no direct YUV2GRAY
eqframe = cv.cuda.cvtColor(eqframe,
cv.COLOR_RGB2GRAY,
stream=stream)
else:
eqframe = gray
# We can only compute optical flow if there is a previous frame
preveqframe, prev = prev, eqframe
if preveqframe is None:
continue
# Second early exit spot: We populated preveqframe, we do not need to
# compute the optical flow for this frame.
if nf < workunit[1] or not save_this:
continue
# Note: We may not want to exit early if args.of_history is on, or we
# may want to clear history between non-consecutive frames.
# Compute optical flow between current frame and previous
flow = flowengine.calc(preveqframe, eqframe, last_flow, stream=stream)
if args.of_history:
last_flow = flow
# Visualize the flow in color
x, y = cv.cuda.split(flow, stream=stream)
mag, ang = cv.cuda.cartToPolar(x, y, stream=stream)
c = cv.cuda_GpuMat(frame.size(), cv.CV_32FC1, 255 / (2 * np.pi))
hue = cv.cuda.multiply(c, ang, stream=stream)
sat = cv.cuda_GpuMat(frame.size(), cv.CV_32FC1, 255)
val = cv.cuda.normalize(mag, 0, 255, cv.NORM_MINMAX, -1, stream=stream)
hsv = cv.cuda_GpuMat(frame.size(), cv.CV_32FC3)
cv.cuda.merge((hue, sat, val), hsv, stream=stream)
# Convert to BGRA
bgr = cv.cuda.cvtColor(hsv, cv.COLOR_HSV2BGR, stream=stream)
bgra = cv.cuda.cvtColor(bgr, cv.COLOR_BGR2BGRA, stream=stream)
# Apply an opening operator
x = cv.cuda_GpuMat(frame.size(), cv.CV_8UC4)
bgra.convertTo(cv.CV_8UC4, x)
bgra = filter.apply(x)
# Store the result in the blue channel
bgrgray = cv.cuda.cvtColor(bgra, cv.COLOR_BGRA2GRAY, stream=stream)
blue_channel = bgrgray
# ---------------------------------------------------------------------
# Combine the channels
combined = cv.cuda_GpuMat(blue_channel.size(), cv.CV_8UC3)
cv.cuda.merge((
blue_channel,
green_channel,
red_channel,
),
combined,
stream=stream)
# Download the combined image from the device
output = combined.download(stream=stream)
# Wait for completion of the stream, after which point the finished
# image should be in the `output` array.
stream.waitForCompletion()
if args.save_original:
path = os.path.join(args.save_original, out + '_original.jpg')
cv.imwrite(path, frame_local)
if args.save_preprocessed:
path = os.path.join(args.save_preprocessed, out + '.jpg')
cv.imwrite(path, output)
# -- Neural network ---------------------------------------------------
if not net:
continue
# Create the blob to feed to the network
blob = cv.dnn.blobFromImage(np.float32(output),
1 / 255.0,
nn_size, [0, 0, 0],
True,
crop=False)
# Feed in the blob and get out the detections
net.setInput(blob)
nnouts = net.forward(net.getUnconnectedOutLayersNames())
lastlayer = net.getLayer(net.getLayerId(net.getLayerNames()[-1]))
assert lastlayer.type == 'Region'
# Interpret the network output as classification and bounding box
confidences, boxes = [], []
for nnout in nnouts:
for detection in nnout:
# Determine the classification with the highest confidence
scores = detection[5:]
classId = np.argmax(scores)
confidence = scores[classId].item()
if confidence < args.nn_threshold:
continue
confidences.append(confidence)
# Convert the bounding box to absolute coordinates
height, width, _ = output.shape
center_x = int(detection[0] * width)
center_y = int(detection[1] * height)
boxwidth = int(detection[2] * width)
boxheight = int(detection[3] * height)
left = int(center_x - boxwidth / 2)
top = int(center_y - boxheight / 2)
boxes.append([left, top, boxwidth, boxheight])
# Apply non-maximum suppression to eliminate overlapping boxes
indices = \
cv.dnn.NMSBoxes(boxes, confidences, args.nn_threshold, args.nn_nms)
indices = indices[:, 0] if len(indices) else []
boxes = [boxes[i] for i in indices]
confidences = [confidences[i] for i in indices]
# Save detection data if desired
if boxes and args.save_detection_data:
detout = {
'video': args.video,
'frame': {
'width': output.shape[1],
'height': output.shape[0],
'number': nf,
'timestamp_msec': get_timestamp(nf),
}
}
detections = detout['detections'] = []
for conf, box in zip(confidences, boxes):
detections.append({
'left': box[0],
'top': box[1],
'width': box[2],
'height': box[3],
'confidence': conf,
})
path = os.path.join(args.save_detection_data, out + '_boxes.json')
with open(path, 'w') as f:
json.dump(detout, f)
# Draw labels on an image if desired
if args.save_detection_image:
labeled = output.copy()
for conf, box in zip(confidences, boxes):
left, top = box[0], box[1]
right, bot = box[0] + box[2], box[1] + box[3]
# Draw bounding box
cv.rectangle(labeled, (left, top), (right, bot), (0, 255, 0))
# Draw label background
label = '%.2f' % conf
labelsz, baseline = \
cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, 0.5, 1)
top = max(top, labelsz[1])
cv.rectangle(labeled, (left, top - labelsz[1]),
(left + labelsz[0], top + baseline),
(255, 255, 255), cv.FILLED)
# Draw label
cv.putText(labeled, label, (left, top),
cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0))
# Write out file
path = os.path.join(args.save_detection_image,
out + '_labeled.jpg')
cv.imwrite(path, labeled)
