def test_cuda_interop(self): npMat = (np.random.random((128, 128, 3)) * 255).astype(np.uint8) cuMat = cv.cuda_GpuMat() cuMat.upload(npMat) self.assertTrue(cuMat.cudaPtr() != 0) stream = cv.cuda_Stream() self.assertTrue(stream.cudaPtr() != 0) asyncstream = cv.cuda_Stream(1) # cudaStreamNonBlocking self.assertTrue(asyncstream.cudaPtr() != 0)
def run(use_random=True, visualize=False): cuda_stream = cv2.cuda_Stream() backSub = cv2.cuda.createBackgroundSubtractorMOG2(history=100, varThreshold=16, detectShadows=False) # backSub = cv2.createBackgroundSubtractorKNN(history=30, dist2Threshold=400.0, detectShadows=True) warped_list = np.load("X:/frank.npy", mmap_mode='r') N = 1000 bg_list = [] for frame_counter in range(N): if use_random: index = random.randint(0, len(warped_list) - 1) else: index = frame_counter % len(warped_list) warped = warped_list[index] warped_gpu = cv2.cuda_GpuMat() warped_gpu.upload(warped) fgMask = backSub.apply(warped_gpu, -1, cuda_stream) fgMask = fgMask.download() fgMask = cv2.morphologyEx(fgMask, cv2.MORPH_CLOSE, kernel=np.ones((5, 5), np.uint8)) if visualize: cv2.imshow('FG Mask', fgMask) bg = cv2.cuda_GpuMat(warped_gpu.size(), warped_gpu.type()) backSub.getBackgroundImage(cuda_stream, bg) bg = bg.download() if visualize: cv2.imshow('BG', bg) key = cv2.waitKey(1) if key == 27: break cv2.imwrite("X:/final.png", bg) if visualize: cv2.waitKey(0) cv2.destroyAllWindows()
def run(use_random=True, visualize=False): cuda_stream = cv2.cuda_Stream() backSub = cv2.cuda.createBackgroundSubtractorMOG2(history=100, varThreshold=16, detectShadows=False) # backSub = cv2.createBackgroundSubtractorKNN(history=30, dist2Threshold=400.0, detectShadows=True) warped_list = np.load("X:/warped.npy", mmap_mode='r') valid_mask_list = np.load("X:/mask.npy", mmap_mode='r') i = 0 frame_counter = 0 mode = True N = 1000 bg_list = [] for frame_counter in range(N): if use_random: index = random.randint(0, len(warped_list) - 1) else: index = frame_counter % len(warped_list) warped = warped_list[index] valid_mask = valid_mask_list[index] frame_counter += 1 if frame_counter > 500 and random.randint(0, N - frame_counter) < 200: warped_final = bg_list[random.randint(0, len(bg_list) - 1)] else: if frame_counter < 200: mean_canvas = np.ones(warped.shape, dtype="uint8") * 200 mean_canvas = cv2.bitwise_or(mean_canvas, mean_canvas, mask=cv2.bitwise_not(valid_mask)) else: mean_canvas = cv2.bitwise_or(bg, bg, mask=cv2.bitwise_not(valid_mask)) if frame_counter % 8 == 0: bg_list.append(bg) valid_mask = cv2.bitwise_or(warped, warped, mask=valid_mask) final = cv2.bitwise_or(mean_canvas, valid_mask) if visualize: cv2.imshow('Passed', final) final_gpu = cv2.cuda_GpuMat() final_gpu.upload(final) final = cv2.cuda_GpuMat(final) fgMask = backSub.apply(final, -1, cuda_stream) fgMask = fgMask.download() # fgMask = cv2.morphologyEx(fgMask, cv2.MORPH_CLOSE, kernel=np.ones((5,5),np.uint8)) # cv2.imshow('FG Mask', fgMask) bg = cv2.cuda_GpuMat(final_gpu.size(), final_gpu.type()) backSub.getBackgroundImage(cuda_stream, bg) bg = bg.download() bg_list.append(bg) if visualize: cv2.imshow('BG', bg) key = cv2.waitKey(1) if key == 27: break if key == ord('m'): mode = not mode if key == ord(' '): cv2.imwrite(str(i) + ".jpg", warped) i += 1 cv2.imwrite("X:/final.png", bg) # cv2.waitKey(0) cv2.destroyAllWindows()
def test_cuda_upload_download_stream(self): stream = cv.cuda_Stream() npMat = (np.random.random((128, 128, 3)) * 255).astype(np.uint8) cuMat = cv.cuda_GpuMat(128,128, cv.CV_8UC3) cuMat.upload(npMat, stream) npMat2 = cuMat.download(stream=stream) stream.waitForCompletion() self.assertTrue(np.allclose(npMat2, npMat))
def cuda_bgsub(get_frames) -> list: stream = cv2.cuda_Stream() # cv2.createB mask: cv2.cuda.createBackgroundSubtractorMOG2 = cv2.cuda.createBackgroundSubtractorMOG2( history=3, varThreshold=100, detectShadows=False) post_process = [] while get_frames.more(): frame = get_frames.read() processed = mask.apply(frame, -1, stream) post_process.append(processed) return post_process
def apply_BGS_GPU(algorithm, video_file: str): """ Apply CUDA implementation of an algorithm """ cuda_stream = cv2.cuda_Stream() video_stream = video_queue(video_file, target="cuda").start() start_time = time.time() frame_counter = 0 while video_stream.more(): frame = video_stream.read() frame2 = algorithm.apply(frame, 0.1, cuda_stream) frame_counter += 1 fps = frame_counter / (time.time() - start_time) return fps
def __init__(self): self.rows = 1536 self.cols = 2048 self.bgmog2 = cv2.cuda.createBackgroundSubtractorMOG() self.stream = cv2.cuda_Stream() self.frame = PinnedMem((rows, cols, 1)) self.frame_device = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC3) self.frame_device_r = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1) self.frame_device_g = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1) self.frame_device_b = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1) self.frame_device_fg_r = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1) self.frame_device_fg_g = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1) self.frame_device_fg_b = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1) self.frame_device_mask = cv2.cuda_GpuMat(rows, cols, cv2.CV_8UC1) self.fg_host = PinnedMem((rows, cols, 1))
def cuda_median(video_stream, filter: cv2.cuda_Filter, mask: cv2.cuda.createBackgroundSubtractorMOG2, gaussian=False) -> list: mask = cv2.cuda.createBackgroundSubtractorMOG2(history=3, varThreshold=100, detectShadows=False) post_process = [] total = 0 cuda_stream = cv2.cuda_Stream() if gaussian: filter = cv2.cuda.createGaussianFilter(cv2.CV_8UC1, cv2.CV_8UC1, (3, 3), 0.5) while video_stream.more(): # processed = mask.apply(frame, -1, stream) frame = video_stream.read() if frame is None: break print(frame, mask, cuda_stream) processed = mask.apply(frame, 0.009, cuda_stream) filter.apply(processed) post_process.append(processed) return post_process
def cudaDetect(img,gpumat): h,w = img.shape[:2] img = cv2.resize(img,(int(w*0.5),int(h*0.5)),interpolation=cv2.INTER_AREA) img = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY) img = cv2.medianBlur(img,1) img = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,9,-10) kernel = np.ones((3,3),np.uint8) leftimg = cv2.dilate(img[0:w//4,0:h//2],kernel,iterations=1) rightimg = cv2.dilate(img[w//4:w//2,0:h//2],kernel,iterations=1) #cudaコアへ転送 gpumat.upload(leftimg) #cuda streamの生成 stream = cv2.cuda_Stream() detector = cv2.cuda.createHoughSegmentDetector(1,np.pi/180,minLineLength=100, maxLineGap=0) lines_gpu = detector.detect(gpumat) leftlines = lines_gpu.download() #cudaコアへ転送 gpumat.upload(rightimg) #cuda streamの生成 stream = cv2.cuda_Stream() detector = cv2.cuda.createHoughSegmentDetector(1,np.pi/180,minLineLength=100, maxLineGap=0) lines_gpu = detector.detect(gpumat) rightlines = lines_gpu.download() lx1_ = 0.0 lx2_ = 0.0 ly1_ = 0.0 ly2_ = 0.0 lrad = 0.0 if leftlines is not None: for line in leftlines[0]: #print(line) lx1_ += line[0] ly1_ += line[1] lx2_ += line[2] ly2_ += line[3] lrad = (ly2_-ly1_)/(lx2_-lx1_) rx1_ = 0.0 rx2_ = 0.0 ry1_ = 0.0 ry2_ = 0.0 rrad = 0.0 if rightlines is not None: for line in rightlines[0]: #print(line) rx1_ += line[0] rx2_ += line[1] ry1_ += line[2] ry2_ += line[3] rrad = (ry2_-ry1_)/(rx2_-rx1_) print(lrad-rrad) return lrad-rrad
0.004428959364733587, 0.06865838341764481 ]]) # Humanoid # dist = np.array([[0.02220329099612066, 0.13530759611493004, -0.0041870520396677805, 0.007599954530058233, -0.4722284261198788]]) # ESP32 rvec = np.array([0.0, 0.0, 0.0]) # float only tvec = np.array([0.0, 0.0, 0.0]) # float only # cap = cv2.VideoCapture(cv2.CAP_DSHOW) # codec = 0x47504A4D # MJPG # cap.set(cv2.CAP_PROP_FPS, 30.0) # cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('m','j','p','g')) # cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M','J','P','G')) # cap.set(3, 1920) # cap.set(4, 1080) cap = cv2.VideoCapture("X:/out.avi") cuda_stream = cv2.cuda_Stream() parameters = cv2.aruco.DetectorParameters_create() # parameters(doCornerRefinement=True) # markerLength=0.039 # real # markerSeparation=0.0975 # real markerLength = 0.04 markerSeparation = 0.01 # board = cv2.aruco.GridBoard_create(markersX=10, markersY=10, markerLength=0.039, markerSeparation=0.0975, dictionary=dictionary) # real board = cv2.aruco.GridBoard_create(markersX=10, markersY=10, markerLength=markerLength, markerSeparation=markerSeparation, dictionary=dictionary) backSub = cv2.cuda.createBackgroundSubtractorMOG2(history=100, varThreshold=16, detectShadows=False)
def worker_main(args, n, workunit): # Assign a GPU to us assign_gpu(n) # Load the neural network net, nn_size = load_network(n, args) # Open the video file video = cv.VideoCapture(args.video) # For historical reasons, frames are referenced according to their timestamp # according to the following function. # # The frame number should be queried *before* the frame itself is captured. # # This was tested and matches the number given by CAP_PROP_POS_MSEC. OpenCV # uses a frames per second (e.g., 15.0) rather than the microseconds per # frame (e.g., 66666) that is actually encoded in the file. framerate = video.get(cv.CAP_PROP_FPS) get_timestamp = lambda fn: int(1e3 * fn / framerate) # Seek to the first frame we care about video.set(cv.CAP_PROP_POS_FRAMES, workunit[0]) assert int(video.get(cv.CAP_PROP_POS_FRAMES)) == workunit[0] # Create the optical flow calculator flowengine = cv.cuda_FarnebackOpticalFlow.create( args.of_levels, args.of_pyr_scale, False, args.of_winsize, args.of_iterations, args.of_poly_n, args.of_poly_sigma, 0) # Create the background subtractor bgsub = cv.cuda.createBackgroundSubtractorMOG2( history=args.bg_history, varThreshold=args.bg_var_threshold, detectShadows=False) # Create the opening filter kernel = cv.getStructuringElement(cv.MORPH_RECT, (7, 7)) filter = cv.cuda.createMorphologyFilter(cv.MORPH_OPEN, cv.CV_8UC4, kernel) prev = None iterator = range(workunit[0], workunit[2]) if args.progress: iterator = tqdm.tqdm(iterator, position=n, desc=f'G{n:02}') # If --of-history is passed, we will use the flow from the previous frame # to inform the flow in this frame. last_flow = None # Compute the prefix of the video name, for building frame filenames later name_prefix, _ = os.path.splitext(os.path.basename(args.video)) stream = cv.cuda_Stream() for nf in iterator: # Read the next frame success, frame = video.read() assert success # Upload the frame to the device frame_local = frame[:] frame = cv.cuda_GpuMat(frame.shape[0], frame.shape[1], cv.CV_8UC3) frame.upload(frame_local, stream=stream) # Resize the frame if necessary if args.resize: frame = cv.cuda.resize(frame, tuple(args.resize), stream=stream) # Compute the output filename out = '%s_%i' % (name_prefix, get_timestamp(nf)) # Determine whether we will want to save this image save_this = args.frame_list is None or out in args.frame_list # Modify the out path to use whatever directory prefix we are sorting # into. if args.frame_list is not None: out = os.path.join(args.frame_list.get(out, './'), out) # -- Foreground extraction -------------------------------------------- # Apply background subtraction to determine the mask mask = bgsub.apply(frame, -1, stream=stream) # Store the result in the green channel green_channel = mask # First exit early spot: If this purely for prepping the background # subtractor, we don't need anything further. if nf < workunit[1] - 1: continue # -- Raw image -------------------------------------------------------- # Convert the frame to grayscale and store it in the red channel gray = cv.cuda.cvtColor(frame, cv.COLOR_BGR2GRAY, stream=stream) red_channel = gray # -- Optical flow ----------------------------------------------------- # Equalize the luminance histogram of the image if args.of_equalize_luminance: y, u, v = cv.cuda.split(cv.cuda.cvtColor(frame, cv.COLOR_BGR2YUV, stream=stream), stream=stream) y = cv.cuda.equalizeHist(y, stream=stream) eqframe = cv.cuda_GpuMat(y.size(), cv.CV_8UC3) cv.cuda.merge((y, u, v), eqframe, stream=stream) eqframe = cv.cuda.cvtColor(eqframe, cv.COLOR_YUV2RGB, stream=stream) # no direct YUV2GRAY eqframe = cv.cuda.cvtColor(eqframe, cv.COLOR_RGB2GRAY, stream=stream) else: eqframe = gray # We can only compute optical flow if there is a previous frame preveqframe, prev = prev, eqframe if preveqframe is None: continue # Second early exit spot: We populated preveqframe, we do not need to # compute the optical flow for this frame. if nf < workunit[1] or not save_this: continue # Note: We may not want to exit early if args.of_history is on, or we # may want to clear history between non-consecutive frames. # Compute optical flow between current frame and previous flow = flowengine.calc(preveqframe, eqframe, last_flow, stream=stream) if args.of_history: last_flow = flow # Visualize the flow in color x, y = cv.cuda.split(flow, stream=stream) mag, ang = cv.cuda.cartToPolar(x, y, stream=stream) c = cv.cuda_GpuMat(frame.size(), cv.CV_32FC1, 255 / (2 * np.pi)) hue = cv.cuda.multiply(c, ang, stream=stream) sat = cv.cuda_GpuMat(frame.size(), cv.CV_32FC1, 255) val = cv.cuda.normalize(mag, 0, 255, cv.NORM_MINMAX, -1, stream=stream) hsv = cv.cuda_GpuMat(frame.size(), cv.CV_32FC3) cv.cuda.merge((hue, sat, val), hsv, stream=stream) # Convert to BGRA bgr = cv.cuda.cvtColor(hsv, cv.COLOR_HSV2BGR, stream=stream) bgra = cv.cuda.cvtColor(bgr, cv.COLOR_BGR2BGRA, stream=stream) # Apply an opening operator x = cv.cuda_GpuMat(frame.size(), cv.CV_8UC4) bgra.convertTo(cv.CV_8UC4, x) bgra = filter.apply(x) # Store the result in the blue channel bgrgray = cv.cuda.cvtColor(bgra, cv.COLOR_BGRA2GRAY, stream=stream) blue_channel = bgrgray # --------------------------------------------------------------------- # Combine the channels combined = cv.cuda_GpuMat(blue_channel.size(), cv.CV_8UC3) cv.cuda.merge(( blue_channel, green_channel, red_channel, ), combined, stream=stream) # Download the combined image from the device output = combined.download(stream=stream) # Wait for completion of the stream, after which point the finished # image should be in the `output` array. stream.waitForCompletion() if args.save_original: path = os.path.join(args.save_original, out + '_original.jpg') cv.imwrite(path, frame_local) if args.save_preprocessed: path = os.path.join(args.save_preprocessed, out + '.jpg') cv.imwrite(path, output) # -- Neural network --------------------------------------------------- if not net: continue # Create the blob to feed to the network blob = cv.dnn.blobFromImage(np.float32(output), 1 / 255.0, nn_size, [0, 0, 0], True, crop=False) # Feed in the blob and get out the detections net.setInput(blob) nnouts = net.forward(net.getUnconnectedOutLayersNames()) lastlayer = net.getLayer(net.getLayerId(net.getLayerNames()[-1])) assert lastlayer.type == 'Region' # Interpret the network output as classification and bounding box confidences, boxes = [], [] for nnout in nnouts: for detection in nnout: # Determine the classification with the highest confidence scores = detection[5:] classId = np.argmax(scores) confidence = scores[classId].item() if confidence < args.nn_threshold: continue confidences.append(confidence) # Convert the bounding box to absolute coordinates height, width, _ = output.shape center_x = int(detection[0] * width) center_y = int(detection[1] * height) boxwidth = int(detection[2] * width) boxheight = int(detection[3] * height) left = int(center_x - boxwidth / 2) top = int(center_y - boxheight / 2) boxes.append([left, top, boxwidth, boxheight]) # Apply non-maximum suppression to eliminate overlapping boxes indices = \ cv.dnn.NMSBoxes(boxes, confidences, args.nn_threshold, args.nn_nms) indices = indices[:, 0] if len(indices) else [] boxes = [boxes[i] for i in indices] confidences = [confidences[i] for i in indices] # Save detection data if desired if boxes and args.save_detection_data: detout = { 'video': args.video, 'frame': { 'width': output.shape[1], 'height': output.shape[0], 'number': nf, 'timestamp_msec': get_timestamp(nf), } } detections = detout['detections'] = [] for conf, box in zip(confidences, boxes): detections.append({ 'left': box[0], 'top': box[1], 'width': box[2], 'height': box[3], 'confidence': conf, }) path = os.path.join(args.save_detection_data, out + '_boxes.json') with open(path, 'w') as f: json.dump(detout, f) # Draw labels on an image if desired if args.save_detection_image: labeled = output.copy() for conf, box in zip(confidences, boxes): left, top = box[0], box[1] right, bot = box[0] + box[2], box[1] + box[3] # Draw bounding box cv.rectangle(labeled, (left, top), (right, bot), (0, 255, 0)) # Draw label background label = '%.2f' % conf labelsz, baseline = \ cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, 0.5, 1) top = max(top, labelsz[1]) cv.rectangle(labeled, (left, top - labelsz[1]), (left + labelsz[0], top + baseline), (255, 255, 255), cv.FILLED) # Draw label cv.putText(labeled, label, (left, top), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0)) # Write out file path = os.path.join(args.save_detection_image, out + '_labeled.jpg') cv.imwrite(path, labeled)