def main():
print("Loading topology and model")
load_model()
WIDTH = 224 * 4
HEIGHT = 224 * 4
print("Starting camera")
camera = USBCamera(width=WIDTH,
height=HEIGHT,
capture_fps=30,
capture_device=0)
camera.running = True
cv2.namedWindow("deeplift", cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty("deeplift", cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
# get qr code
# camera.observe(read_qr_code, names='value')
# camera.unobserve_all()
# print("Running camera")'
# spawns a thread
camera.observe(execute, names='value')
def main():
# Get batch size from args
batchSize = int(sys.argv[1])
# Set up USB camera (assuming device 0)
cam = USBCamera(width=1920, height=1080, capture_device=0)
# Set up model for yolov5s
yolo = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True)
device = torch.device('cuda')
yolo.to(device)
# Basic analytics
imageCount = 0
modelTime = 0
# Continue until interrupt
try:
while True:
# Grab as many frames as batch size
imgs = []
for i in range(batchSize):
imgs.append(cam.read()[:, :, ::-1]) # Convert BGR to RGB
# Process with yolo
t = time()
results = yolo(imgs)
modelTime += time() - t
# Save files, janky method to change filenames and avoid overwrite
for i, f in enumerate(results.files):
newNumber = sum(map(int, filter(str.isdigit, f)), imageCount)
newStr = str(newNumber).zfill(4) + '.jpg'
results.files[i] = newStr
results.save()
imageCount += batchSize
except KeyboardInterrupt:
print(
f'Ending. Processed {imageCount} images in {modelTime}s, average FPS of {imageCount/modelTime}'
)
model = load_model(args["model"]) lb = pickle.loads(open(args["label_bin"], "rb").read()) # initialize the image mean for mean subtraction along with the # predictions queuesave_frames_to_dir mean = np.array([123.68, 116.779, 103.939][::1], dtype="float32") Q = deque(maxlen=args["size"]) # initialize the video stream, pointer to output video file, and # frame dimensions if use_video_as_input: vs = cv2.VideoCapture(args["input"]) else: #vs = CSICamera(width=224, height=224, capture_width=1080, capture_height=720, capture_fps=30) vs = USBCamera(capture_device=0) writer = None (W, H) = (None, None) # Start time start = time.time() # loop over frames from the video file stream count = 0 # whileTrue: for count in range(0,700): # read the next frame from the file if use_video_as_input: (grabbed, frame) = vs.read() else:
from datetime import datetime
lFps_sec = 0 #current second
lFPSbeg = 0 #the seccond we started to run
lFPSrun = 0 #running for X secconds
lFPSfnm = 0 #number of frames
#
lFps_c = 0 #current fps
lFps_k = 0 #current frames
lFps_M = 0 #max fps
use_display = True
# Create a VideoCapture object
#capture = cv2.VideoCapture(0)
#capture = CSICamera (width=1280, height=720)
capture = USBCamera(width=1280,
height=720,
capture_width=1280,
capture_height=720,
capture_device=3)
# Check if camera opened successfully
#if (capture.isOpened() == False):
# print("Unable to read camera feed")
# Default resolutions of the frame are obtained.The default resolutions are system dependent.
# We convert the resolutions from float to integer.
w = 1280 #int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH ))
h = 720 #int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT ))
fps = 60
# Define the codec and create VideoWriter object.The output is stored in 'outpy.avi' file.
#fourcc = cv2.VideoWriter_fourcc(*'XVID') # 9FPS
#fourcc = cv2.VideoWriter_fourcc(*'X264') # 6FPS
x = torch.from_numpy(x).float()
x = normalize(x)
x = x.to(device)
x = x[None, ...]
return x
#Now, let's start and display our camera. We'll also create a slider that will display the\n",
#import traitlets
#from IPython.display import display
#import ipywidgets.widgets as widgets
#from jetbot import Camera, bgr8_to_jpeg
from jetcam.usb_camera import USBCamera
camera = USBCamera(capture_device=0, width=224, height=224)
camera.running = True
#camera = Camera.instance(width=224, height=224)
#image = widgets.Image(format='jpeg', width=224, height=224)
#image = camera.read()
#blocked_slider = widgets.FloatSlider(description='blocked', min=0.0, max=1.0, orientation='vertical')
#camera_link = traitlets.dlink((camera, 'value'), (image, 'value'), transform=bgr8_to_jpeg)
#camera_link = traitlets.dlink((camera, 'value'), (image, 'value'))
#We'll also create our robot instance which we'll need to drive the motors."
from jetbot import Robot
robot = Robot()
# Next, we'll create a function that will get called whenever the camera's value changes. This function will do the following steps
#1. Pre-process the camera image
from datetime import datetime
lFps_sec = 0 #current second
lFPSbeg = 0 #the seccond we started to run
lFPSrun = 0 #running for X secconds
lFPSfnm = 0 #number of frames
#
lFps_c = 0 #current fps
lFps_k = 0 #current frames
lFps_M = 0 #max fps
use_display = True
# Create a VideoCapture object
#capture = cv2.VideoCapture(0)
#capture = CSICamera (width=1280, height=720)
camera = USBCamera(width=1280,
height=720,
capture_width=1280,
capture_height=720,
capture_device=3)
# Check if camera opened successfully
#if (capture.isOpened() == False):
# print("Unable to read camera feed")
# Default resolutions of the frame are obtained.The default resolutions are system dependent.
# We convert the resolutions from float to integer.
w = 1280 #int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH ))
h = 720 #int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT ))
fps = 60
# Define the codec and create VideoWriter object.The output is stored in 'outpy.avi' file.
#fourcc = cv2.VideoWriter_fourcc(*'XVID') # 9FPS
#fourcc = cv2.VideoWriter_fourcc(*'X264') # 6FPS
tf.import_graph_def(od_graph_def, name='')
TFSess = tf.compat.v1.Session(graph=detection_graph)
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
num_detections = detection_graph.get_tensor_by_name('num_detections:0')
camera = USBCamera(width=640,
height=480,
capture_width=640,
capture_height=480,
capture_device=1,
capture_fps=30)
frame_rate_calc = 1
freq = cv2.getTickFrequency()
font = cv2.FONT_HERSHEY_SIMPLEX
WIN_NAME = 'Object Detection Test'
window_handle = cv2.namedWindow(WIN_NAME, cv2.WINDOW_AUTOSIZE)
frameCount = 0
while True:
t1 = cv2.getTickCount()
from jetcam.usb_camera import USBCamera
TASK = 'thumbs'
CATEGORIES = ['thumbs_up', 'thumbs_down']
DATASETS = 'A'
TRANSFORMS = transforms.Compose([
transforms.ColorJitter(0.2, 0.2, 0.2, 0.2),
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
dataset = ImageClassificationDataset(TASK + '_' + DATASETS, CATEGORIES,
TRANSFORMS)
device = torch.device('cuda')
model = torchvision.models.resnet18(pretrained=True)
model.fc = torch.nn.Linear(512, len(dataset.categories))
model = model.to(device)
model.load_state_dict(torch.load('./my_model.pth'))
model = model.eval()
camera = USBCamera(capture_device=0)
while True:
image = camera.read()
preprocessed = preprocess(image)
output = model(preprocessed)
output = F.softmax(output, dim=1).detach().cpu().numpy().flatten()
print('%5.2f %5.2f' % (output[0], output[1]))
device = torch.device('cuda')
def preprocess(image):
image = image[..., ::-1]
image = PIL.Image.fromarray(image)
image = transforms.functional.to_tensor(image).to(device)
image.sub_(mean[:, None, None]).div_(std[:, None, None])
return image[None, ...]
parse_objects = ParseObjects(topology)
draw_objects = DrawObjects(topology)
camera = USBCamera(capture_device=1,
width=WIDTH,
height=HEIGHT,
capture_fps=30)
cv2.namedWindow('pose', cv2.WINDOW_NORMAL)
framecount = 1
indices = {idx: n for idx, n in enumerate(human_pose['keypoints'])}
positions = {n: [[0, 0]] * 5 for n in human_pose['keypoints']}
detected = lambda n: positions[indices[n]][-1] != [0, 0]
groups = {'head': [0, 1, 2, 3, 4], 'mid': [5, 6, 17], 'bottom': [11, 12]}
group_avgs = {'head': [], 'mid': [], 'bottom': []}
group_dets = {'head': False, 'mid': False, 'bottom': False}
angles = []
spine_angle = 0
@blynk.handle_event('read V11')
def write_virtual_pin_handlerv11(pin):
def setup_camera(self):
self.camera = USBCamera(width=WIDTH, height=HEIGHT, capture_device=1)
self.camera.running = True
parse_objects = ParseObjects(topology) draw_objects = DrawObjects(topology) # Assuming you're using NVIDIA Jetson, you can use the jetcam package to create an easy to # use camera that will produce images in BGR8/HWC format. # # If you're not on Jetson, you may need to adapt the code below. from jetcam.usb_camera import USBCamera # from jetcam.csi_camera import CSICamera from jetcam.utils import bgr8_to_jpeg WIDTH = 224 HEIGHT = 224 camera = USBCamera(width=WIDTH, height=HEIGHT, capture_fps=30) # camera = CSICamera(width=WIDTH, height=HEIGHT, capture_fps=30) camera.running = True # Next, we'll create a widget which will be used to display the camera feed with visualizations. import ipywidgets from IPython.display import display image_w = ipywidgets.Image(format='jpeg') display(image_w) # Finally, we'll define the main execution loop. This will perform the following steps # # 1. Preprocess the camera image
def __init__(self, root):
# Context variable declarations and loading
self.running = False
self.WIDTH = 224
self.HEIGHT = 224
self.thresh = 127
self.round = 0
self.minimum_joints = 4
self.path = './images/'
self.mdelay_sec = 10
self.mtick = self.mdelay_sec
self.mask = None
self.calibrate = True # Flag to show calibration pose over camera feed
self.calibration_pose = cv2.imread('./images/cal_pose.jpg', cv2.IMREAD_COLOR)
#Loading model and model data
with open('./tasks/human_pose/human_pose.json', 'r') as f:
human_pose = json.load(f)
self.topology = trt_pose.coco.coco_category_to_topology(human_pose)
self.num_parts = len(human_pose['keypoints'])
self.num_links = len(human_pose['skeleton'])
self.data = torch.zeros((1, 3, self.HEIGHT, self.WIDTH)).cuda()
self.OPTIMIZED_MODEL = './tasks/human_pose/resnet18_baseline_att_224x224_A_epoch_249_trt.pth'
self.model_trt = TRTModule()
self.model_trt.load_state_dict(torch.load(self.OPTIMIZED_MODEL))
self.mean = torch.Tensor([0.485, 0.456, 0.406]).cuda()
self.std = torch.Tensor([0.229, 0.224, 0.225]).cuda()
self.device = torch.device('cuda')
self.parse_objects = ParseObjects(self.topology)
self.draw_objects = DrawObjects(self.topology)
# Start camera
if USBCam:
self.camera = USBCamera(width=self.WIDTH, height=self.HEIGHT, capture_fps=30)
else:
self.camera = CSICamera(width=self.WIDTH, height=self.HEIGHT, capture_fps=30)
self.frame=Tk.Frame(root)
self.root=root
# Create editable title
self.titleVar = Tk.StringVar()
self.title= Tk.Label(root, textvariable=self.titleVar, font="Verdana 36")
self.titleVar.set("Pose Estimation Game")
self.title.pack(side=Tk.TOP)
self.frame.pack(side=Tk.LEFT, fill=Tk.BOTH, expand=1)
# Create image capture figure
# Set as Frame with three possible images (live feed, mask/pose to make, image captured)
# Image row
self.im_row = Tk.Frame(self.frame)
self.feed_label = Tk.Label(self.im_row)
self.feed_label.pack(side=Tk.LEFT)
self.mask_label = Tk.Label(self.im_row)
self.pose_label = Tk.Label(self.im_row)
# Create editable description label
self.desTextVar = "Please select an option from the right"
self.desText = Tk.Label(self.frame, text=self.desTextVar, font="Verdana 12")
#Create Combobox for selection (Steps are currently in comments)
#Grab maps from repository
#Parse map names to develop choices
#group map names into array
self.levels = []
choices = ["Easy", "Medium", "Hard"]
#Put map names in combo box
self.ddVar = Tk.StringVar()
self.ddVar.set('Select a Choice')
self.dropDown = Tk.OptionMenu(self.frame, self.ddVar, *choices)
# This function binds a function that loads all images for level upon the selection of
# an option in the dropdown menu
self.ddVar.trace('w', self.levels_select)
# Create inital button panel
self.buttonPanel = ButtonPanel(root)
self.im_row.pack()
self.desText.pack()
self.buttonPanel.pack()
self.root.after(10, self.camera_loop)
MainGUI.updateToTitle(self)
import os
import traitlets
#import ipywidgets.widgets as widgets
#from IPython.display import display
#from jetbot import Camera, bgr8_to_jpeg
from uuid import uuid1
from jetcam.usb_camera import USBCamera
from jetcam.utils import bgr8_to_jpeg
directory = '.'
# Start the camera and create a video stream
#camera = USBCamera(capture_device=1)
camera = USBCamera(width=224,
height=224,
capture_width=640,
capture_height=480,
capture_device=0)
print(camera.value.shape)
# Get an bgr8 image from the camera
image = camera.read()
print(camera.value.shape)
toJpeg = bgr8_to_jpeg(image)
image_path = os.path.join(directory, str(uuid1()) + '.jpg')
with open(image_path, 'wb') as f:
f.write(image.value)
def to_pixels(xy, wh):
return (int(xy[0] * wh[0]), int(xy[1] * wh[1]))
def distance(a, b):
return np.sqrt((a[0] - b[0])**2 + (a[1] - b[1])**2)
# INITIALIZE
cv2.namedWindow("BubbleBop", cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty("BubbleBop", cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
camera = USBCamera(capture_device=CAPTURE_DEVICE,
width=IMAGE_SHAPE[0],
height=IMAGE_SHAPE[1])
pose_detector = PoseDetector(POSE_MODEL, POSE_INPUT_SHAPE)
left_sampler = CircleSampler(LEFT_SAMPLE_AREA_CENTER, SAMPLE_AREA_RADIUS)
right_sampler = CircleSampler(RIGHT_SAMPLE_AREA_CENTER, SAMPLE_AREA_RADIUS)
left_bubble = to_int(left_sampler.sample())
right_bubble = to_int(right_sampler.sample())
left_wrist = (0, 0)
right_wrist = (0, 0)
bubble_radius = MIN_BUBBLE_RADIUS
bubble_speed = MIN_BUBBLE_SPEED
score = 0
image = np.copy(camera.read()[:, ::-1])
# RUN
while True:
num_links = len(human_pose['skeleton'])
# Load model
# TODO: extend model to include additional joints
model = models.resnet18_baseline_att(num_parts,
2 * num_links).cuda().eval()
model.load_state_dict(torch.load(MODEL_WEIGHTS))
# Optimization with tensorRT
# NOTE: optimization is device specific
data = torch.zeros((1, 3, HEIGHT, WIDTH)).cuda()
model_trt = torch2trt.torch2trt(model, [data],
fp16_mode=True,
max_workspace_size=1 << 25)
torch.save(model_trt.state_dict(), OPTIMIZED_MODEL)
# Load optimized model
model_trt = TRTModule()
model_trt.load_state_dict(torch.load(OPTIMIZED_MODEL))
# Setup camera and visuals
parse_objects = ParseObjects(topology)
draw_objects = DrawObjects(topology)
camera = USBCamera(width=WIDTH, height=HEIGHT, capture_device=1)
camera.running = True
display(image_w)
# Attach oberver to act on each new frame received
camera.observe(execute, names='value')
# SQUEEZENET
# model = torchvision.models.squeezenet1_1(pretrained=True)
# model.classifier[1] = torch.nn.Conv2d(512, output_dim, kernel_size=1)
# model.num_classes = len(dataset.categories)
# RESNET 18
model = torchvision.models.resnet18(pretrained=True)
model.fc = torch.nn.Linear(512, output_dim)
# RESNET 34
# model = torchvision.models.resnet34(pretrained=True)
# model.fc = torch.nn.Linear(512, output_dim)
model = model.to(device)
model.load_state_dict(torch.load('./my_xy_model.pth'))
model = model.eval()
print("model configured!")
camera = USBCamera(width=224, height=224, capture_device=0) # confirm the capture_device number
print("camera created!")
while True:
image = camera.read()
preprocessed = preprocess(image)
output = model(preprocessed).detach().cpu().numpy().flatten()
for category in dataset.categories:
category_index = dataset.categories.index(category)
x = output[2 * category_index]
y = output[2 * category_index + 1]
print('{} {} {}'.format(dataset.categories[category_index], x, y))
x = int(camera.width * (x/2.0+0.5))
y = int(camera.height * (y/2.0+0.5))
color = [0,0,0]
def main():
liveDemo = True
if liveDemo:
# Set up USB camera (assuming device 0)
cam = USBCamera(width=1920, height=1080, capture_device=0)
# Set up model for yolov5s
yolo = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True)
device = torch.device('cuda')
yolo.to(device)
# Set up topology, model, and classes for ResNet
with open('human_pose.json', 'r') as f:
human_pose = json.load(f)
topology = trt_pose.coco.coco_category_to_topology(human_pose)
resnet = TRTModule()
resnet.load_state_dict(torch.load('resnet_trt.pth'))
parseObjects = ParseObjects(topology)
drawObjects = DrawObjects(topology)
# Basic analytics
imageCount = 0
t = time()
# Live demo on webcam
if liveDemo:
# Continue until interrupt
try:
while True:
# Grab a frame
img = cam.read()[:, :, ::-1] # Convert BGR to RGB
print(f'got frame {imageCount}')
# Process with yolo and resnet
result, empty = processFrame(img, yolo, resnet, parseObjects,
drawObjects)
# Save file
cv2.imwrite(f'imgs/{imageCount:04}.jpg', result)
imageCount += 1
except KeyboardInterrupt:
print('Keyboard interrupt!')
finally:
t = time() - t
# Recorded video
else:
cap = cv2.VideoCapture('example_video.mpg')
# Grab a frame
ret, frame = cap.read()
# Continue until video is done
while ret:
# Process and save image
img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
result, empty = processFrame(img, yolo, resnet, parseObjects,
drawObjects)
cv2.imwrite(f'imgs/{imageCount:04}.jpg', result)
# Try to grab next frame
ret, frame = cap.read()
imageCount += 1
t = time() - t
cap.release()
print(
f'Ending. Processed {imageCount} images in {t}s, average FPS of {imageCount/t}'
)
# Optimization with tensorRT
# NOTE: optimization is device specific
# data = torch.zeros((1, 3, HEIGHT, WIDTH)).cuda()
# model_trt = torch2trt.torch2trt(model, [data], fp16_mode=True, max_workspace_size=1<<25)
# torch.save(model_trt.state_dict(), OPTIMIZED_MODEL)
# Load optimized model
print("Loading optimized model")
model_trt = TRTModule()
model_trt.load_state_dict(torch.load(OPTIMIZED_MODEL))
# Setup camera and visuals
parse_objects = ParseObjects(topology)
draw_objects = DrawObjects(topology)
camera = USBCamera(width=WIDTH, height=HEIGHT, capture_device=1)
camera.running = True
# Attach oberver to act on each new frame received
# camera.observe(execute, names='value')
im = plt.imshow(execute({'new': camera.value}))
ani = FuncAnimation(plt.gcf(), update, interval=200)
cid = plt.gcf().canvas.mpl_connect("key_press_event", close)
plt.show()
from jetcam.usb_camera import USBCamera
import ipywidgets
from IPython.display import display
from jetcam.utils import bgr8_to_jpeg
import traitlets
camera = USBCamera(capture_device=0)
image = camera.read()
print(image.shape)
print(camera.value.shape)
image_widget = ipywidgets.Image(format='jpeg')
image_widget.value = bgr8_to_jpeg(image)
display(image_widget)
camera.running = True
def update_image(change):
image = change['new']
image_widget.value = bgr8_to_jpeg(image)
camera.observe(update_image, names='value')
camera.unobserve(update_image, names='value')
camera_link = traitlets.dlink((camera, 'value'), (image_widget, 'value'), transform=bgr8_to_jpeg)
camera_link.unlink()
