def main(): print("Loading topology and model") load_model() WIDTH = 224 * 4 HEIGHT = 224 * 4 print("Starting camera") camera = USBCamera(width=WIDTH, height=HEIGHT, capture_fps=30, capture_device=0) camera.running = True cv2.namedWindow("deeplift", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("deeplift", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) # get qr code # camera.observe(read_qr_code, names='value') # camera.unobserve_all() # print("Running camera")' # spawns a thread camera.observe(execute, names='value')
def main(): # Get batch size from args batchSize = int(sys.argv[1]) # Set up USB camera (assuming device 0) cam = USBCamera(width=1920, height=1080, capture_device=0) # Set up model for yolov5s yolo = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True) device = torch.device('cuda') yolo.to(device) # Basic analytics imageCount = 0 modelTime = 0 # Continue until interrupt try: while True: # Grab as many frames as batch size imgs = [] for i in range(batchSize): imgs.append(cam.read()[:, :, ::-1]) # Convert BGR to RGB # Process with yolo t = time() results = yolo(imgs) modelTime += time() - t # Save files, janky method to change filenames and avoid overwrite for i, f in enumerate(results.files): newNumber = sum(map(int, filter(str.isdigit, f)), imageCount) newStr = str(newNumber).zfill(4) + '.jpg' results.files[i] = newStr results.save() imageCount += batchSize except KeyboardInterrupt: print( f'Ending. Processed {imageCount} images in {modelTime}s, average FPS of {imageCount/modelTime}' )
model = load_model(args["model"]) lb = pickle.loads(open(args["label_bin"], "rb").read()) # initialize the image mean for mean subtraction along with the # predictions queuesave_frames_to_dir mean = np.array([123.68, 116.779, 103.939][::1], dtype="float32") Q = deque(maxlen=args["size"]) # initialize the video stream, pointer to output video file, and # frame dimensions if use_video_as_input: vs = cv2.VideoCapture(args["input"]) else: #vs = CSICamera(width=224, height=224, capture_width=1080, capture_height=720, capture_fps=30) vs = USBCamera(capture_device=0) writer = None (W, H) = (None, None) # Start time start = time.time() # loop over frames from the video file stream count = 0 # whileTrue: for count in range(0,700): # read the next frame from the file if use_video_as_input: (grabbed, frame) = vs.read() else:
from datetime import datetime lFps_sec = 0 #current second lFPSbeg = 0 #the seccond we started to run lFPSrun = 0 #running for X secconds lFPSfnm = 0 #number of frames # lFps_c = 0 #current fps lFps_k = 0 #current frames lFps_M = 0 #max fps use_display = True # Create a VideoCapture object #capture = cv2.VideoCapture(0) #capture = CSICamera (width=1280, height=720) capture = USBCamera(width=1280, height=720, capture_width=1280, capture_height=720, capture_device=3) # Check if camera opened successfully #if (capture.isOpened() == False): # print("Unable to read camera feed") # Default resolutions of the frame are obtained.The default resolutions are system dependent. # We convert the resolutions from float to integer. w = 1280 #int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH )) h = 720 #int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT )) fps = 60 # Define the codec and create VideoWriter object.The output is stored in 'outpy.avi' file. #fourcc = cv2.VideoWriter_fourcc(*'XVID') # 9FPS #fourcc = cv2.VideoWriter_fourcc(*'X264') # 6FPS
x = torch.from_numpy(x).float() x = normalize(x) x = x.to(device) x = x[None, ...] return x #Now, let's start and display our camera. We'll also create a slider that will display the\n", #import traitlets #from IPython.display import display #import ipywidgets.widgets as widgets #from jetbot import Camera, bgr8_to_jpeg from jetcam.usb_camera import USBCamera camera = USBCamera(capture_device=0, width=224, height=224) camera.running = True #camera = Camera.instance(width=224, height=224) #image = widgets.Image(format='jpeg', width=224, height=224) #image = camera.read() #blocked_slider = widgets.FloatSlider(description='blocked', min=0.0, max=1.0, orientation='vertical') #camera_link = traitlets.dlink((camera, 'value'), (image, 'value'), transform=bgr8_to_jpeg) #camera_link = traitlets.dlink((camera, 'value'), (image, 'value')) #We'll also create our robot instance which we'll need to drive the motors." from jetbot import Robot robot = Robot() # Next, we'll create a function that will get called whenever the camera's value changes. This function will do the following steps #1. Pre-process the camera image
from datetime import datetime lFps_sec = 0 #current second lFPSbeg = 0 #the seccond we started to run lFPSrun = 0 #running for X secconds lFPSfnm = 0 #number of frames # lFps_c = 0 #current fps lFps_k = 0 #current frames lFps_M = 0 #max fps use_display = True # Create a VideoCapture object #capture = cv2.VideoCapture(0) #capture = CSICamera (width=1280, height=720) camera = USBCamera(width=1280, height=720, capture_width=1280, capture_height=720, capture_device=3) # Check if camera opened successfully #if (capture.isOpened() == False): # print("Unable to read camera feed") # Default resolutions of the frame are obtained.The default resolutions are system dependent. # We convert the resolutions from float to integer. w = 1280 #int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH )) h = 720 #int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT )) fps = 60 # Define the codec and create VideoWriter object.The output is stored in 'outpy.avi' file. #fourcc = cv2.VideoWriter_fourcc(*'XVID') # 9FPS #fourcc = cv2.VideoWriter_fourcc(*'X264') # 6FPS
tf.import_graph_def(od_graph_def, name='') TFSess = tf.compat.v1.Session(graph=detection_graph) image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0') detection_scores = detection_graph.get_tensor_by_name('detection_scores:0') detection_classes = detection_graph.get_tensor_by_name('detection_classes:0') num_detections = detection_graph.get_tensor_by_name('num_detections:0') camera = USBCamera(width=640, height=480, capture_width=640, capture_height=480, capture_device=1, capture_fps=30) frame_rate_calc = 1 freq = cv2.getTickFrequency() font = cv2.FONT_HERSHEY_SIMPLEX WIN_NAME = 'Object Detection Test' window_handle = cv2.namedWindow(WIN_NAME, cv2.WINDOW_AUTOSIZE) frameCount = 0 while True: t1 = cv2.getTickCount()
from jetcam.usb_camera import USBCamera TASK = 'thumbs' CATEGORIES = ['thumbs_up', 'thumbs_down'] DATASETS = 'A' TRANSFORMS = transforms.Compose([ transforms.ColorJitter(0.2, 0.2, 0.2, 0.2), transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) dataset = ImageClassificationDataset(TASK + '_' + DATASETS, CATEGORIES, TRANSFORMS) device = torch.device('cuda') model = torchvision.models.resnet18(pretrained=True) model.fc = torch.nn.Linear(512, len(dataset.categories)) model = model.to(device) model.load_state_dict(torch.load('./my_model.pth')) model = model.eval() camera = USBCamera(capture_device=0) while True: image = camera.read() preprocessed = preprocess(image) output = model(preprocessed) output = F.softmax(output, dim=1).detach().cpu().numpy().flatten() print('%5.2f %5.2f' % (output[0], output[1]))
device = torch.device('cuda') def preprocess(image): image = image[..., ::-1] image = PIL.Image.fromarray(image) image = transforms.functional.to_tensor(image).to(device) image.sub_(mean[:, None, None]).div_(std[:, None, None]) return image[None, ...] parse_objects = ParseObjects(topology) draw_objects = DrawObjects(topology) camera = USBCamera(capture_device=1, width=WIDTH, height=HEIGHT, capture_fps=30) cv2.namedWindow('pose', cv2.WINDOW_NORMAL) framecount = 1 indices = {idx: n for idx, n in enumerate(human_pose['keypoints'])} positions = {n: [[0, 0]] * 5 for n in human_pose['keypoints']} detected = lambda n: positions[indices[n]][-1] != [0, 0] groups = {'head': [0, 1, 2, 3, 4], 'mid': [5, 6, 17], 'bottom': [11, 12]} group_avgs = {'head': [], 'mid': [], 'bottom': []} group_dets = {'head': False, 'mid': False, 'bottom': False} angles = [] spine_angle = 0 @blynk.handle_event('read V11') def write_virtual_pin_handlerv11(pin):
def setup_camera(self): self.camera = USBCamera(width=WIDTH, height=HEIGHT, capture_device=1) self.camera.running = True
parse_objects = ParseObjects(topology) draw_objects = DrawObjects(topology) # Assuming you're using NVIDIA Jetson, you can use the jetcam package to create an easy to # use camera that will produce images in BGR8/HWC format. # # If you're not on Jetson, you may need to adapt the code below. from jetcam.usb_camera import USBCamera # from jetcam.csi_camera import CSICamera from jetcam.utils import bgr8_to_jpeg WIDTH = 224 HEIGHT = 224 camera = USBCamera(width=WIDTH, height=HEIGHT, capture_fps=30) # camera = CSICamera(width=WIDTH, height=HEIGHT, capture_fps=30) camera.running = True # Next, we'll create a widget which will be used to display the camera feed with visualizations. import ipywidgets from IPython.display import display image_w = ipywidgets.Image(format='jpeg') display(image_w) # Finally, we'll define the main execution loop. This will perform the following steps # # 1. Preprocess the camera image
def __init__(self, root): # Context variable declarations and loading self.running = False self.WIDTH = 224 self.HEIGHT = 224 self.thresh = 127 self.round = 0 self.minimum_joints = 4 self.path = './images/' self.mdelay_sec = 10 self.mtick = self.mdelay_sec self.mask = None self.calibrate = True # Flag to show calibration pose over camera feed self.calibration_pose = cv2.imread('./images/cal_pose.jpg', cv2.IMREAD_COLOR) #Loading model and model data with open('./tasks/human_pose/human_pose.json', 'r') as f: human_pose = json.load(f) self.topology = trt_pose.coco.coco_category_to_topology(human_pose) self.num_parts = len(human_pose['keypoints']) self.num_links = len(human_pose['skeleton']) self.data = torch.zeros((1, 3, self.HEIGHT, self.WIDTH)).cuda() self.OPTIMIZED_MODEL = './tasks/human_pose/resnet18_baseline_att_224x224_A_epoch_249_trt.pth' self.model_trt = TRTModule() self.model_trt.load_state_dict(torch.load(self.OPTIMIZED_MODEL)) self.mean = torch.Tensor([0.485, 0.456, 0.406]).cuda() self.std = torch.Tensor([0.229, 0.224, 0.225]).cuda() self.device = torch.device('cuda') self.parse_objects = ParseObjects(self.topology) self.draw_objects = DrawObjects(self.topology) # Start camera if USBCam: self.camera = USBCamera(width=self.WIDTH, height=self.HEIGHT, capture_fps=30) else: self.camera = CSICamera(width=self.WIDTH, height=self.HEIGHT, capture_fps=30) self.frame=Tk.Frame(root) self.root=root # Create editable title self.titleVar = Tk.StringVar() self.title= Tk.Label(root, textvariable=self.titleVar, font="Verdana 36") self.titleVar.set("Pose Estimation Game") self.title.pack(side=Tk.TOP) self.frame.pack(side=Tk.LEFT, fill=Tk.BOTH, expand=1) # Create image capture figure # Set as Frame with three possible images (live feed, mask/pose to make, image captured) # Image row self.im_row = Tk.Frame(self.frame) self.feed_label = Tk.Label(self.im_row) self.feed_label.pack(side=Tk.LEFT) self.mask_label = Tk.Label(self.im_row) self.pose_label = Tk.Label(self.im_row) # Create editable description label self.desTextVar = "Please select an option from the right" self.desText = Tk.Label(self.frame, text=self.desTextVar, font="Verdana 12") #Create Combobox for selection (Steps are currently in comments) #Grab maps from repository #Parse map names to develop choices #group map names into array self.levels = [] choices = ["Easy", "Medium", "Hard"] #Put map names in combo box self.ddVar = Tk.StringVar() self.ddVar.set('Select a Choice') self.dropDown = Tk.OptionMenu(self.frame, self.ddVar, *choices) # This function binds a function that loads all images for level upon the selection of # an option in the dropdown menu self.ddVar.trace('w', self.levels_select) # Create inital button panel self.buttonPanel = ButtonPanel(root) self.im_row.pack() self.desText.pack() self.buttonPanel.pack() self.root.after(10, self.camera_loop) MainGUI.updateToTitle(self)
import os import traitlets #import ipywidgets.widgets as widgets #from IPython.display import display #from jetbot import Camera, bgr8_to_jpeg from uuid import uuid1 from jetcam.usb_camera import USBCamera from jetcam.utils import bgr8_to_jpeg directory = '.' # Start the camera and create a video stream #camera = USBCamera(capture_device=1) camera = USBCamera(width=224, height=224, capture_width=640, capture_height=480, capture_device=0) print(camera.value.shape) # Get an bgr8 image from the camera image = camera.read() print(camera.value.shape) toJpeg = bgr8_to_jpeg(image) image_path = os.path.join(directory, str(uuid1()) + '.jpg') with open(image_path, 'wb') as f: f.write(image.value)
def to_pixels(xy, wh): return (int(xy[0] * wh[0]), int(xy[1] * wh[1])) def distance(a, b): return np.sqrt((a[0] - b[0])**2 + (a[1] - b[1])**2) # INITIALIZE cv2.namedWindow("BubbleBop", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("BubbleBop", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) camera = USBCamera(capture_device=CAPTURE_DEVICE, width=IMAGE_SHAPE[0], height=IMAGE_SHAPE[1]) pose_detector = PoseDetector(POSE_MODEL, POSE_INPUT_SHAPE) left_sampler = CircleSampler(LEFT_SAMPLE_AREA_CENTER, SAMPLE_AREA_RADIUS) right_sampler = CircleSampler(RIGHT_SAMPLE_AREA_CENTER, SAMPLE_AREA_RADIUS) left_bubble = to_int(left_sampler.sample()) right_bubble = to_int(right_sampler.sample()) left_wrist = (0, 0) right_wrist = (0, 0) bubble_radius = MIN_BUBBLE_RADIUS bubble_speed = MIN_BUBBLE_SPEED score = 0 image = np.copy(camera.read()[:, ::-1]) # RUN while True:
num_links = len(human_pose['skeleton']) # Load model # TODO: extend model to include additional joints model = models.resnet18_baseline_att(num_parts, 2 * num_links).cuda().eval() model.load_state_dict(torch.load(MODEL_WEIGHTS)) # Optimization with tensorRT # NOTE: optimization is device specific data = torch.zeros((1, 3, HEIGHT, WIDTH)).cuda() model_trt = torch2trt.torch2trt(model, [data], fp16_mode=True, max_workspace_size=1 << 25) torch.save(model_trt.state_dict(), OPTIMIZED_MODEL) # Load optimized model model_trt = TRTModule() model_trt.load_state_dict(torch.load(OPTIMIZED_MODEL)) # Setup camera and visuals parse_objects = ParseObjects(topology) draw_objects = DrawObjects(topology) camera = USBCamera(width=WIDTH, height=HEIGHT, capture_device=1) camera.running = True display(image_w) # Attach oberver to act on each new frame received camera.observe(execute, names='value')
# SQUEEZENET # model = torchvision.models.squeezenet1_1(pretrained=True) # model.classifier[1] = torch.nn.Conv2d(512, output_dim, kernel_size=1) # model.num_classes = len(dataset.categories) # RESNET 18 model = torchvision.models.resnet18(pretrained=True) model.fc = torch.nn.Linear(512, output_dim) # RESNET 34 # model = torchvision.models.resnet34(pretrained=True) # model.fc = torch.nn.Linear(512, output_dim) model = model.to(device) model.load_state_dict(torch.load('./my_xy_model.pth')) model = model.eval() print("model configured!") camera = USBCamera(width=224, height=224, capture_device=0) # confirm the capture_device number print("camera created!") while True: image = camera.read() preprocessed = preprocess(image) output = model(preprocessed).detach().cpu().numpy().flatten() for category in dataset.categories: category_index = dataset.categories.index(category) x = output[2 * category_index] y = output[2 * category_index + 1] print('{} {} {}'.format(dataset.categories[category_index], x, y)) x = int(camera.width * (x/2.0+0.5)) y = int(camera.height * (y/2.0+0.5)) color = [0,0,0]
def main(): liveDemo = True if liveDemo: # Set up USB camera (assuming device 0) cam = USBCamera(width=1920, height=1080, capture_device=0) # Set up model for yolov5s yolo = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True) device = torch.device('cuda') yolo.to(device) # Set up topology, model, and classes for ResNet with open('human_pose.json', 'r') as f: human_pose = json.load(f) topology = trt_pose.coco.coco_category_to_topology(human_pose) resnet = TRTModule() resnet.load_state_dict(torch.load('resnet_trt.pth')) parseObjects = ParseObjects(topology) drawObjects = DrawObjects(topology) # Basic analytics imageCount = 0 t = time() # Live demo on webcam if liveDemo: # Continue until interrupt try: while True: # Grab a frame img = cam.read()[:, :, ::-1] # Convert BGR to RGB print(f'got frame {imageCount}') # Process with yolo and resnet result, empty = processFrame(img, yolo, resnet, parseObjects, drawObjects) # Save file cv2.imwrite(f'imgs/{imageCount:04}.jpg', result) imageCount += 1 except KeyboardInterrupt: print('Keyboard interrupt!') finally: t = time() - t # Recorded video else: cap = cv2.VideoCapture('example_video.mpg') # Grab a frame ret, frame = cap.read() # Continue until video is done while ret: # Process and save image img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) result, empty = processFrame(img, yolo, resnet, parseObjects, drawObjects) cv2.imwrite(f'imgs/{imageCount:04}.jpg', result) # Try to grab next frame ret, frame = cap.read() imageCount += 1 t = time() - t cap.release() print( f'Ending. Processed {imageCount} images in {t}s, average FPS of {imageCount/t}' )
# Optimization with tensorRT # NOTE: optimization is device specific # data = torch.zeros((1, 3, HEIGHT, WIDTH)).cuda() # model_trt = torch2trt.torch2trt(model, [data], fp16_mode=True, max_workspace_size=1<<25) # torch.save(model_trt.state_dict(), OPTIMIZED_MODEL) # Load optimized model print("Loading optimized model") model_trt = TRTModule() model_trt.load_state_dict(torch.load(OPTIMIZED_MODEL)) # Setup camera and visuals parse_objects = ParseObjects(topology) draw_objects = DrawObjects(topology) camera = USBCamera(width=WIDTH, height=HEIGHT, capture_device=1) camera.running = True # Attach oberver to act on each new frame received # camera.observe(execute, names='value') im = plt.imshow(execute({'new': camera.value})) ani = FuncAnimation(plt.gcf(), update, interval=200) cid = plt.gcf().canvas.mpl_connect("key_press_event", close) plt.show()
from jetcam.usb_camera import USBCamera import ipywidgets from IPython.display import display from jetcam.utils import bgr8_to_jpeg import traitlets camera = USBCamera(capture_device=0) image = camera.read() print(image.shape) print(camera.value.shape) image_widget = ipywidgets.Image(format='jpeg') image_widget.value = bgr8_to_jpeg(image) display(image_widget) camera.running = True def update_image(change): image = change['new'] image_widget.value = bgr8_to_jpeg(image) camera.observe(update_image, names='value') camera.unobserve(update_image, names='value') camera_link = traitlets.dlink((camera, 'value'), (image_widget, 'value'), transform=bgr8_to_jpeg) camera_link.unlink()