def __init__(self, screen: Surface):
"""
Creates the game
@param screen: The screen surface (created with pygame.display.set_mode) to draw on. This screen is used both
for drawing and for creating the camera.
@note To pave our way to Rust projects, this project uses type hints in function signatures.
"""
# Load sprite sheets (aka costumes)
assets_path = os.path.dirname(os.path.realpath(__file__))
bear_spritesheet = image.load(os.path.join(
assets_path, "..", "assets", "bears.png"))
berry_spritesheet = image.load(os.path.join(
assets_path, "..", "assets", "berry.png"))
# Definition of game objects
self.world = World()
self.bear = Bear(320, 180, bear_spritesheet)
self.berry_group = Group()
for _ in range(100):
x, y = randint(0, 1000), randint(0, 1000)
berry = Berry(x, y, berry_spritesheet)
berry.rect = berry.rect.clamp(self.world.rect)
self.berry_group.add(berry)
self.screen = screen
self.camera = Camera(320, 180, screen)
self.text = TextUI(0, 0)
# Gameplay-related data
self.score = 0
self.remaining = 100
self.timer = 0
# State management. You might want to use a state machine for complex games.
self.running = True
def __init__(self, ope_depth=5, gpu_id=0, gpu_memory_limit=None, vpn_type=None, K=None):
self._session = None # tensorflow session
self._image = None # input to the 2D network
self._scoremaps_kp = None # output of the 2D network
self._scoremaps_paf = None # output of the 2D network
self._depth_vox = None # input to the 3D network
self._kp_uv = None # input to the 3D network
self._voxel_root_xyz = None # input to the 3D network
self._voxel_scale = None # input to the 3D network
self._cam_mat = None # input to the 3D network
self._kp_vox = None # output of the 3D network
# parameters
self._intermediate_scoremap_size = (100, 100) # map size used for warping 2D->3D
self.conf2d_thresh = 0.5 # minimal confidence of 2d detection
self.cam = Camera(K)
self.ope_depth = ope_depth
self.gpu_id = gpu_id
self.gpu_memory_limit = gpu_memory_limit
if vpn_type == 'fast':
self.use_fast_vpn = True
else:
self.use_fast_vpn = False
# create network (this sets some member variables)
self._setup_network() # creates the tensorflow graph)
self._init_open_pose() # loads weights
def __init__(self, game_manager):
# type: (GameManager.GameManager) -> None
self.game_manager = game_manager # type: GameManager.GameManager
# self.rendered_image = self.world_gen.get_current_room() # type: pygame.Surface
# self.current_room_rendered = self.world_gen.get_current_room() # type: RoomRenderer
self.cam = Camera.Camera(self.game_manager)
self.world_gen = WorldGenerator.WorldGenerator(
self.game_manager,
"game_data/config/world.json",
"game_data/worlds/test_world",
camera=self.cam)
self.player = Player.PlayerObject(
self.game_manager,
self.world_gen,
position=Vector2D(1280 / 2, 720 / 2),
camera=self.cam,
obj_data=self.world_gen.load_player())
self.cam.position = self.player.position.copy()
self.cam.target(self.player.position)
def draw(self, surface: Surface, camera: Camera = None):
"""
Same as bear.draw()
@note Probably should create a separate class to inherit this method rather than having it twice
"""
if camera is None:
surface.blit(self.image, self.rect)
else:
position_in_camera = camera.from_world(self.position)
rect = self.image.get_rect(center=position_in_camera)
surface.blit(self.image, rect)
def __init__(self, c1=0, camera=None):
Thread.__init__(self)
self.threadLock = Lock()
print("BackgroundSubstractor called with capture %s" % c1)
if not isinstance(camera, Camera):
self.camera = Camera(device=c1)
else:
self.camera = camera
# c1.release()re('test.avi')
self.storage = ContourStorage()
self._initialize_substractor()
def draw(self, surface: Surface, camera: Camera = None):
"""
Draws the bear onto a surface based on camera position
@param surface: the surface to draw on
@param camera: if not set, this function draws the bear onto the surface treating the *world coordinate* of the
bear's rect as its position on the surface. If set, this function uses *camera coordinate* instead.
@return: None
"""
self.image = self.costumes[self.costume_number]
if camera is None:
surface.blit(self.image, self.rect)
else:
position_in_camera = camera.from_world(self.position)
rect = self.image.get_rect(center=position_in_camera)
surface.blit(self.image, rect)
def main():
#2x2 binning
camera = Camera(sensor_width=512,
sensor_height=512,
pixel_size=0.62,
source_to_detector_distance=1200,
isocenter_distance=450)
#4x4 binning
# camera = Camera(sensor_width=620, sensor_height=480, pixel_size=0.62, source_to_detector_distance=1200,isocenter_distance=800)
####
#define the path to your dicoms here or use the simple phantom from the code above
####
CT_volume_path = r".\your_dicom_directory\\"
#save_path = r".\generated_data\test"
save_path = r"./test"
min_theta = 75
max_theta = 105
min_phi = 75
max_phi = 105
spacing_theta = 30
spacing_phi = 30
photon_count = 100000
#origin [0,0,0] corresponds to the center of the volume
origin = [0, 0, 0]
spectrum = spectrum_generator.SPECTRUM90KV_AL40
if not os.path.isdir(save_path):
os.makedirs(save_path)
generate_projections_on_sphere(CT_volume_path,
save_path,
min_theta,
max_theta,
min_phi,
max_phi,
spacing_theta,
spacing_phi,
photon_count,
camera,
spectrum,
origin=origin,
scatter=False)
def __init__(self, game_manager):
# type: (GameManager.GameManager) -> None
self.game_manager = game_manager # type: GameManager.GameManager
self.cam = Camera.Camera(self.game_manager)
self.world_gen = WorldGenerator.WorldGenerator(self.game_manager, "game_data/config/world.json",
"game_data/worlds/test_world", camera=self.cam)
self.player = Player.PlayerObject(self.game_manager,
self.world_gen,
position=Vector2D(1280 / 2,
720 / 2),
camera=self.cam,
obj_data=self.world_gen.load_player())
self.cam.position = self.player.position.copy()
self.cam.target(self.player.position)
class MainApplikacation(object):
detected = []
detected2 = []
detected3 = []
detected4 = []
detected5 = []
real = []
was_covert = []
frame_no = []
Calibrated = None
Substractor = None
input = None
camera = None
board_config_load = True
date = datetime.now().strftime("%Y-%m-%d-%H-%M")
fname = "data%s"%date
imgpoint = None
boardpoint = None
def write_data(self):
print "Writing data to file"
i = 0
fname = self.fname
fname += ".csv"
while os.path.isfile(fname):
i += 1
fname = self.fname + '-' + str(i)+".csv"
with open(fname, 'wb') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=',',dialect='excel',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(['Detected','Detected2','Detected3','Detected4','Detected5', 'Reality', 'Was Covert', 'Frameno', 'Diff', 'Diff2', 'Diff3', 'Diff4', 'Diff5'])
def calc_diff(a,b):
i = 0
diff = []
for each in a:
try:
if each == b[i]:
diff.append(True)
else:
diff.append(False)
except:
print "For some Reasion there was an error in diff calc"
i += 1
percentage = float(len([x for x in diff if x is True])) / float(len(diff))
return diff, percentage
diff, percentage = calc_diff(self.detected, self.real)
print "Tip Version 1 was %s Percent correct."%(percentage * 100)
diff2, percentage = calc_diff(self.detected2, self.real)
print "Tip Version 2 was %s Percent correct." % (percentage * 100)
diff3, percentage = calc_diff(self.detected3, self.real)
print "Tip Version 3 was %s Percent correct." % (percentage * 100)
diff4, percentage = calc_diff(self.detected4, self.real)
print "Tip Version 4 was %s Percent correct." % (percentage * 100)
diff5, percentage = calc_diff(self.detected5, self.real)
print "Tip Version 5 was %s Percent correct." % (percentage * 100)
datas = zip(self.detected, self.detected2, self.detected3, self.detected4, self.detected5, self.real,
self.was_covert, self.frame_no, diff,diff2, diff3, diff4, diff5)
for each in datas:
entry = list(each)
# if each[0] == each[1]:
# entry.append(True)
# else:
# entry.append(False)
spamwriter.writerow(entry)
def __init__(self, inp):
if isinstance(inp,str):
self.fname = inp.split('.')[0]
self.camera = Camera(device=inp, output=self.fname)
self.board = Board()
camconf = "camera_config.json"
baord_conf = "boardconfig.json"
if os.path.isfile(camconf):
self.camera.load_config(filename=camconf)
else:
self.camera.do_calibration(img=True)
self.camera.save_config(camconf)
if self.board_config_load and os.path.isfile(baord_conf):
with open(baord_conf, 'r') as bc:
imgps = json.loads(bc.readline())
self.Calibrated = BoardCalibrator(camera=self.camera, imgpts=imgps, board=self.board)
else:
self.Calibrated = BoardCalibrator(camera=self.camera)
with open("boardconfig.json", 'w') as bc:
imgps = self.Calibrated.imgpoints
bc.write(json.dumps(imgps))
self.Substractor = BackgroundSubtractor(c1=inp,camera=self.camera)
plt.ion()
self.figure = plt.figure()
self.plt1 = self.figure.add_subplot(111)
self.line1, = self.plt1.plot(range(200), [0] * 200, 'r.-')
self.plt1.axis([0, 200, 0, 10000])
cv2.namedWindow("Current", cv2.WINDOW_NORMAL)
cv2.moveWindow("Current", 20,20)
cv2.namedWindow("Original", cv2.WINDOW_NORMAL)
cv2.moveWindow("Original", 20, 500)
cv2.namedWindow("Points", cv2.WINDOW_NORMAL)
cv2.moveWindow("Current", 1000, 20)
cv2.namedWindow("Blobimg", cv2.WINDOW_NORMAL)
cv2.setMouseCallback("Points", self._click)
mixer.init()
mixer.music.load('beep.mp3')
# cv2.namedWindow("FG Substraction", cv2.WINDOW_NORMAL)
# cv2.createTrackbar("History", "Current", self.history, 1000, self._set_history)
# cv2.createTrackbar("Shadow Treshold", "Current", int(self.shad_tresh * 100), 100, self._set_shad_tresh)
# cv2.createTrackbar("VarThreshold", "Current", self.var_tresh, 100, self._set_var_tresh)
# cv2.createTrackbar("VarMax", "Current", self.var_max, 100, self._set_var_max)
# cv2.createTrackbar("VarMin", "Current", self.var_min, 100, self._set_var_min)
self.Substractor.start()
realboard = np.zeros((self.camera.height, self.camera.width, 3), np.uint8)
# self.frame = self.camera.undistort_image(img)
for i in self.Calibrated.imp:
try:
realboard[i[0][1], i[0][0]] = [0,0,255]
except IndexError:
pass
added = 0
while True:
img = self.Substractor.get_image()
if img is not None:
img = cv2.add(realboard,img)
cv2.imshow("Original", img)
cv2.imshow("Points", self.board.draw_board())
if self.Substractor.stopped:
self.write_data()
exit()
cv2.imshow("Current", self.Substractor.get_substracted())
storage, unaltered = self.Substractor.get_storage()
y = [x[2] for x in storage]
y = unaltered
self.line1.set_xdata(range(len(y)))
self.line1.set_ydata(y)
k = cv2.waitKey(1)
if k == ord('a'):
self.add_dart(frame_no=self.camera.read_frame_no)
if k == ord('s'):
self.write_data()
if k == ord('w'):
pass
self.figure.savefig(r"thesisimages/plot.jpg")
if k == ord('f'):
added = 0
self.Substractor.clear_arrows()
if k == 27:
self.Substractor.stopped = True
break
if k == 119:
print "Pressed w Key so Waiting"
cv2.waitKey(-1)
arrows = self.Substractor.get_arrows()
i = 1
for each in arrows:
tip = each.tip
frame_no = each.frame_no
points = self.Calibrated.calculate_points(tip)
if i > added:
self.add_dart(arrow=each, detected=points, frame_no=frame_no)
added += 1
i += 1
if added >= 3:
added = 0
self.Substractor.clear_arrows()
self.write_data()
def _click(self, event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONDOWN:
nx = x - self.board.size / 2
ny = self.board.size / 2 - y
print("Clicked %s " % self.board.calculate_field([[nx],[ny]]))
self.imgpoint = [x, y]
self.boardpoint = [[nx],[ny]]
# self.imgpoints.append([x, y])
def add_dart(self, arrow=None, detected="N/D", frame_no=0):
self.Substractor.paused = True
# mixer.music.play()
if arrow is not None:
print "Ratio is %s"%arrow.ratio
print [cv2.contourArea(x) for x in arrow.contours]
points = self.Calibrated.calculate_points(arrow.tip)
pimg = self.board.draw_field(points)
cv2.imshow("Points", pimg)
cv2.imshow("Blobimg", arrow.img)
k = -1
while k not in [13, 32]or self.boardpoint is None:
k = cv2.waitKey(-1)
if k == 13:
print "Enter"
print len(self.detected)
self.was_covert.append(False)
if k == 32:
self.was_covert.append(True)
self.real.append(self.board.calculate_field(self.boardpoint))
self.boardpoint = None
class PoseNet3D(object):
def __init__(self, ope_depth=5, gpu_id=0, gpu_memory_limit=None, vpn_type=None, K=None):
self._session = None # tensorflow session
self._image = None # input to the 2D network
self._scoremaps_kp = None # output of the 2D network
self._scoremaps_paf = None # output of the 2D network
self._depth_vox = None # input to the 3D network
self._kp_uv = None # input to the 3D network
self._voxel_root_xyz = None # input to the 3D network
self._voxel_scale = None # input to the 3D network
self._cam_mat = None # input to the 3D network
self._kp_vox = None # output of the 3D network
# parameters
self._intermediate_scoremap_size = (100, 100) # map size used for warping 2D->3D
self.conf2d_thresh = 0.5 # minimal confidence of 2d detection
self.cam = Camera(K)
self.ope_depth = ope_depth
self.gpu_id = gpu_id
self.gpu_memory_limit = gpu_memory_limit
if vpn_type == 'fast':
self.use_fast_vpn = True
else:
self.use_fast_vpn = False
# create network (this sets some member variables)
self._setup_network() # creates the tensorflow graph)
self._init_open_pose() # loads weights
def detect(self, image, depth_w, mask):
""" Given RGBD input it predicts 3D human pose. """
## A) 2D network on the RGB input
# 1. Preprocessing RGB: Get image to the right size
image_proc, image_s, scale_ratio = self._preproc_color(image)
#self._show_input(image_s, depth_w, block=False)
# 2. Run 2D network
scoremaps_kp_v, scoremaps_paf_v = self._session.run([self._scoremaps_kp, self._scoremaps_paf],
{self._image: image_proc})
# 3. Postprocessing: Detect keypoints and use PAF to work out person instances
keypoint_det = detect_keypoints(np.squeeze(scoremaps_kp_v, 0)) # detect keypoints in the one scoremap
paf_u, paf_v = self._split_paf_scoremap(np.squeeze(scoremaps_paf_v, 0)) # split representation
pairwise_scores = calculate_pair_scores(keypoint_det, paf_u, paf_v) # Calculate matching scores with the pafs
# upscale because we dont upsample the scoremaps anymore
keypoint_det_fs = list()
for x in keypoint_det:
x[:, :2] *= 8.0
keypoint_det_fs.append(x)
person_det = group_keypoints(keypoint_det_fs, pairwise_scores) # Use detections and pairwise scores to get final estimation
# print('Found %d persons' % len(person_det)) # coords in: person_det['person0']['kp'] = None, person_det['person1']['kp'] = (u, v)
# self._show_openpose_det(image_s, person_det, block=False)
## B) 3D network: VOXELPOSENET
coord_uv, coord2d_conf = self._trafo_dict2array(person_det)
coord_vis = coord2d_conf > self.conf2d_thresh
coord_uv_fs = coord_uv / scale_ratio
coord_xyz, det_conf = list(), list()
for pid in range(len(person_det)):
# Check if neck keypoint is visible
if coord_vis[pid, 1] == 1.0:
root_id = 1 # neck keypoint
coord2d_root = coord_uv_fs[pid, root_id, :]
#asymmetric grid
grid_size_m = np.array([[-1.1, -0.4, -1.1],
[1.1, 1.8, 1.1]])
# if not try R-hip
elif coord_vis[pid, 8] == 1.0:
root_id = 8 # R-hip keypoint
coord2d_root = coord_uv_fs[pid, root_id, :]
#symmetric grid
grid_size_m = np.array([[-1.1, -1.1, -1.1],
[1.1, 1.1, 1.1]])
# if not try L-hip
elif coord_vis[pid, 11] == 1.0:
root_id = 11 # L-hip keypoint
coord2d_root = coord_uv_fs[pid, root_id, :]
#symmetric grid
grid_size_m = np.array([[-1.1, -1.1, -1.1],
[1.1, 1.1, 1.1]])
else:
continue
# find approx. depth for root
z_value = self._get_depth_value(depth_w / 1000.0, coord2d_root[0], coord2d_root[1])
# self._show_sparse_depth(depth_w, coord_uv_fs[pid, 1, :], block=False)
if z_value == 0.0:
print("Could not extract depth value. Skipping sample.")
continue
# create voxel occupancy grid from the warped depth map
voxelgrid, coord2d_s, trafo_params = vu.voxelize_person(self.cam, depth_w, mask,
coord_uv_fs[pid, :, :], coord_vis[pid, :],
z_value, (64, 64, 64), f=1.2,
grid_size_m=grid_size_m,
root_id=root_id, coord2d_root=coord2d_root)
# 5. Run VoxelPoseNet
feed_dict_vpn = {self._depth_vox: voxelgrid, self._kp_uv: coord2d_s, self._kp_vis: coord_vis[pid, :]}
kp_scorevol_v = self._session.run(self._kp_vox, feed_dict_vpn)
# 6. Postprocessing: Detect keypoints, transform back to XYZ
keypoints_xyz_vox, det_conf_vox = self._detect_scorevol(kp_scorevol_v)
keypoints_xyz_pred = vu.trafo_vox_coords_to_xyz_new(keypoints_xyz_vox, trafo_params) # xyz from voxel
keypoints_xyz_pred_proj = self.cam.backproject(coord_uv_fs[pid, :, :], keypoints_xyz_pred[:, -1:]) # xyz from backprojected uv
# assemble solution from voxel result and backprojected solution
cond = coord2d_conf[pid, :] > det_conf_vox # use backproj only when 2d was visible and 2d/3d roughly matches
keypoints_xyz_pred[cond, :] = keypoints_xyz_pred_proj[cond, :]
coord_xyz.append(keypoints_xyz_pred)
det_conf.append(det_conf_vox)
# self._show_voxelposenet_det(voxelgrid, keypoints_xyz_vox)
# output numpy arrays
if len(coord_xyz) > 0:
coord_xyz = np.stack(coord_xyz)
else:
coord_xyz = np.zeros((0, 18, 3))
if len(det_conf) > 0:
det_conf = np.stack(det_conf)
else:
det_conf = np.zeros((0, 18))
return coord_xyz, det_conf
def _setup_network(self):
""" Creates the tensorflow graph structure. """
# input placeholder
self._image = tf.placeholder(tf.float32, (1, 376, 656, 3), 'image')
self._depth_vox = tf.placeholder(tf.float32, (64, 64, 64), 'depth_vox')
self._kp_uv = tf.placeholder(tf.float32, (18, 2), 'kp_uv')
self._kp_vis = tf.placeholder(tf.float32, (18), 'kp_vis')
self._voxel_root_xyz = tf.placeholder(tf.float32, (1, 3), 'voxel_root_xyz') # only for warped
self._voxel_scale = tf.placeholder(tf.float32, (1, 3), 'voxel_scale') # only for warped
self._cam_mat = tf.placeholder(tf.float32, (3, 3), 'cam_mat') # only for warped
evaluation = tf.placeholder_with_default(True, shape=(), name='evaluation')
# OpenPose colornet
color_net = OpenPoseCoco()
scoremaps_kp_list, scoremaps_paf_list = color_net.inference_pose(self._image, train=False, upsample=False,
gpu_id=self.gpu_id)
self._scoremaps_kp, self._scoremaps_paf = scoremaps_kp_list[self.ope_depth], scoremaps_paf_list[self.ope_depth]
# VoxelPoseNet for Person Kp
net = VoxelPoseNet(use_slim=self.use_fast_vpn)
# calculate scoremap
kp_map_uv = self._create_multiple_gaussian_map(self._kp_uv, (64, 64), 3.0, tf.expand_dims(self._kp_vis, -1))
# tile 2D scoremap into a scorevolume
det2d_vox = tf.tile(tf.expand_dims(tf.expand_dims(kp_map_uv, 0), 3), [1, 1, 1, 64, 1])
self._kp_vox = net.inference(tf.expand_dims(tf.expand_dims(self._depth_vox, 0), -1),
det2d_vox,
evaluation, gpu_id=self.gpu_id)
# start session load weights
if self.gpu_memory_limit is None:
self._session = tf.Session()
else:
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.33)
self._session = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
self._init_voxelposenet()
def _init_open_pose(self):
""" Loads weights for the OpenPose network. """
# create dict for renaming the weights
name_dict = {'conv1_1': 'conv1_1',
'conv1_2': 'conv1_2',
'conv2_1': 'conv2_1',
'conv2_2': 'conv2_2',
'conv3_1': 'conv3_1',
'conv3_2': 'conv3_2',
'conv3_3': 'conv3_3',
'conv3_4': 'conv3_4',
'conv4_1': 'conv4_1',
'conv4_2': 'conv4_2',
'conv4_3_CPM': 'conv4_3',
'conv4_4_CPM': 'conv4_4'}
for type_old, type_new in [('L1', 'paf'), ('L2', 'kp')]:
for rep_id in range(1, 8):
name_dict['conv5_%d_CPM_%s' % (rep_id, type_old)] = 'conv5_%d_%s' % (rep_id, type_new)
for stage_id in range(2, 7):
for rep_id in range(1, 8):
name_dict['Mconv%d_stage%d_%s' % (rep_id, stage_id, type_old)] = 'conv%d_%d_%s' % (stage_id + 4, rep_id, type_new)
weight_dict = dict()
with open('./weights/openpose-coco.pkl', 'rb') as fi:
if sys.version_info[0] == 3:
weight_dict_raw = pickle.load(fi, encoding='latin1') # for python3
else:
weight_dict_raw = pickle.load(fi) # for python2
for k, v in weight_dict_raw.items():
if k in name_dict.keys():
new_name = name_dict[k]
weight_dict['CocoPoseNet/' + new_name + '/weights'] = v[0]
weight_dict['CocoPoseNet/' + new_name + '/biases'] = v[1]
else:
print('Skipping: ', k)
init_op, init_feed = tf.contrib.framework.assign_from_values(weight_dict)
self._session.run(init_op, init_feed)
print('Initialized 2D network with %d variables' % len(weight_dict))
def _init_voxelposenet(self):
""" Initializes the VoxelPoseNet from a snapshot. """
if self.use_fast_vpn:
checkpoint_path = './weights/snapshots_pose_run194/'
else:
checkpoint_path = './weights/snapshots_pose_run191/'
rename_dict = {}
discard_list = ['Adam', 'global_step', 'beta']
self._load_all_variables_from_snapshot(checkpoint_path, rename_dict, discard_list)
def _load_all_variables_from_snapshot(self, checkpoint_path, rename_dict=None, discard_list=None):
""" Initializes certain tensors from a snapshot. """
last_cpt = tf.train.latest_checkpoint(checkpoint_path)
assert last_cpt is not None, "Could not locate snapshot to load."
reader = pywrap_tensorflow.NewCheckpointReader(last_cpt)
var_to_shape_map = reader.get_variable_to_shape_map() # var_to_shape_map
# for name in var_to_shape_map.keys():
# print(name, reader.get_tensor(name).shape)
# Remove everything from the discard list
num_disc = 0
var_to_shape_map_new = dict()
for k, v in var_to_shape_map.items():
good = True
for dis_str in discard_list:
if dis_str in k:
good = False
if good:
var_to_shape_map_new[k] = v
else:
num_disc += 1
var_to_shape_map = dict(var_to_shape_map_new)
print('Discarded %d items' % num_disc)
# print('Vars in checkpoint', var_to_shape_map.keys(), len(var_to_shape_map))
# rename everything according to rename_dict
num_rename = 0
if rename_dict is not None:
var_to_shape_map_new = dict()
for name in var_to_shape_map.keys():
rename = False
for rename_str in rename_dict.keys():
if rename_str in name:
new_name = name.replace(rename_str, rename_dict[rename_str])
var_to_shape_map_new[new_name] = reader.get_tensor(name)
rename = True
num_rename += 1
break
if not rename:
var_to_shape_map_new[name] = reader.get_tensor(name)
var_to_shape_map = dict(var_to_shape_map_new)
print('Renamed %d items' % num_rename)
# print('(Possibly) renamed vars', var_to_shape_map.keys(), len(var_to_shape_map))
init_op, init_feed = tf.contrib.framework.assign_from_values(var_to_shape_map)
self._session.run(init_op, init_feed)
print('Initialized %d variables from %s.' % (len(var_to_shape_map), last_cpt))
@staticmethod
def _preproc_color(image):
""" Preprocessing the color image"""
output_shape = np.array([376.0, 656.0], np.float32)
# reshape by trafo
ratio = np.min(output_shape / np.array(image.shape[:2], dtype=np.float32))
M = np.array([[ratio, 0.0, 0.0], [0.0, ratio, 0.0]])
image_s = cv2.warpAffine(image, M, (output_shape[1], output_shape[0]), flags=cv2.INTER_AREA)
# subtract mean and rgb -> bgr
image = image_s[:, :, 0:3].astype('float32')
image = image[:, :, ::-1]
image = image / 256.0 - 0.5
image = np.expand_dims(image, 0)
return image, image_s, ratio
def _detect_keypoints(self, scoremap):
"""
Takes a scoremap and finds locations for keypoints.
Returns a KxNx2 matrix with the (u, v) coordinates of the N maxima found for the K keypoints.
"""
assert len(scoremap.shape) == 3, "Needs to be a 3D scoremap."
keypoint_loc = list()
for kid in range(scoremap.shape[2]):
num_kp, maxima = self._find_maxima(scoremap[:, :, kid])
if num_kp > 0:
keypoint_loc.append(maxima)
else:
keypoint_loc.append(None)
return keypoint_loc
@staticmethod
def _find_maxima(scoremap):
"""
Takes a scoremap and detect the peaks using the local maximum filter.
Returns a Nx2 matrix with the (u, v) coordinates of the N maxima found.
"""
assert len(scoremap.shape) == 2, "Needs to be a 2D scoremap."
# apply the local maximum filter; all pixel of maximal value
local_max = maximum_filter(scoremap, size=3)
mask_max = scoremap == local_max
# mask out background
mask_bg = ((np.max(scoremap) - np.min(scoremap)) * 0.25) > scoremap
mask_max[mask_bg] = False
# find distinct objects in map
labeled, num_objects = ndimage.label(mask_max)
slices = ndimage.find_objects(labeled)
# create matrix of found objects with their location
maxima = np.zeros((num_objects, 3), dtype=np.float32)
for oid, (dy, dx) in enumerate(slices):
maxima[oid, :2] = [(dx.start + dx.stop - 1)/2, (dy.start + dy.stop - 1)/2]
u, v = int(maxima[oid, 0] + 0.5), int(maxima[oid, 1] + 0.5)
maxima[oid, 2] = scoremap[v, u]
return num_objects, maxima
@staticmethod
def _split_paf_scoremap(scoremap_paf):
""" The network outputs u1, v1, u2, v2, ... """
paf_u, paf_v = scoremap_paf[:, :, ::2], scoremap_paf[:, :, 1::2]
return paf_u, paf_v
@staticmethod
def _show_input(image, depth, block=True):
""" Shows the input to the pipeline. """
import matplotlib.pyplot as plt
fig = plt.figure()
ax1 = fig.add_subplot(121)
ax2 = fig.add_subplot(122)
ax1.imshow(image)
ax2.imshow(depth)
plt.show(block=block)
@staticmethod
def _show_sparse_depth(depth, coord_uv, block=True):
""" Shows the where from the sparse depth map we take a value. """
import matplotlib.pyplot as plt
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.imshow(depth)
ax1.plot(coord_uv[0], coord_uv[1], 'ro')
print('coord_uv', coord_uv)
plt.show(block=block)
@staticmethod
def _show_openpose_det(image_s, person_det, block=True):
""" Shows the detections of openpose in the color image. """
import matplotlib.pyplot as plt
from utils.DrawUtil import draw_person_limbs_2d_coco
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.imshow(image_s)
fmt_list = ['ro', 'go', 'co', 'mo']
for anno, fmt in zip(person_det.values(), fmt_list):
coords = np.zeros((18, 2))
vis = np.zeros((18, ))
for i, kp in enumerate(anno['kp']):
if (kp is not None) and (kp[2] > 0.5):
# ax1.plot(kp[0], kp[1], fmt)
coords[i, :] = np.array([kp[0], kp[1]])
vis[i] = 1.0
draw_person_limbs_2d_coco(ax1, coords, vis, color='sides', order='uv')
plt.show(block=block)
@staticmethod
def _show_voxelposenet_det(voxelgrid, keypoints_xyz_vox, block=True):
""" Shows the detections of VoxelPoseNet in the input voxelgrid. """
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from utils.DrawUtil import draw_person_limbs_3d_coco
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
X, Y, Z = np.where(voxelgrid)
ax.scatter(X, Y, Z, c='g')
draw_person_limbs_3d_coco(ax, keypoints_xyz_vox, color='r')
plt.show(block=block)
def _get_depth_value(self, map, u, v, crop_size=25):
""" Extracts a depth value from a map.
Checks for the closest value in a given neighborhood. """
coord = np.array([[u, v]])
while True:
# get crop
map_c, min_c, _ = self._crop_at(map, coord, crop_size)
center = coord - min_c
# find valid depths
X, Y = np.where(np.not_equal(map_c, 0.0))
if not X.shape[0] == 0:
break
crop_size *= 2 # if not successful use larger crop
# calculate distance
grid = np.stack([X, Y], 1) - center
dist = np.sqrt(np.sum(np.square(grid), 1))
# find element with minimal distance
nn_ind = np.argmin(dist)
x, y = X[nn_ind], Y[nn_ind]
z_val = map_c[x, y]
return z_val
@staticmethod
def _crop_at(map, center_coord, size):
""" Crop a given map at the given center coordinate with the crop size specified.
If the cropped area would partially reside outside of the map it is translated accordingly. """
expand = False
s = map.shape
if len(s) == 2:
map = np.expand_dims(map, 2)
expand = True
s = map.shape
assert len(s) == 3, "Map has to be of Dimension 2 or 3."
size = np.round(size).astype('int')
# make sure crop size cant exceed image dims
if s[0] <= size:
size = s[0]
elif s[1] <= size:
size = s[1]
center_coord = np.array(center_coord)
center_coord = np.reshape(center_coord, [2])
# work out the coords to actually lie in the crop
c_min = np.round(center_coord - size // 2).astype('int')
c_max = c_min + size
# check if we left map
for dim in [0, 1]:
if c_min[dim] < 0.0:
c_max[dim] -= c_min[dim]
c_min[dim] = 0.0
if c_max[dim] > s[1-dim]:
c_min[dim] -= (c_max[dim]-s[1-dim])
c_max[dim] = s[1-dim]
# perform crop
map_crop = map[c_min[1]:c_max[1], c_min[0]:c_max[0], :]
if expand:
map_crop = np.squeeze(map_crop)
return map_crop, c_min, c_max
@staticmethod
def _trafo_dict2array(person_det, kp_conf_thresh=0.25):
""" Transforms the dictionary returned from openpose into an array. """
coord_uv = np.zeros((len(person_det), 18, 2))
coord_vis = np.zeros((len(person_det), 18))
for pid, anno in enumerate(person_det.values()):
for kid, kp in enumerate(anno['kp']):
if (kp is not None) and (kp[2] > kp_conf_thresh):
coord_vis[pid, kid] = kp[2]
coord_uv[pid, kid, :] = np.array([kp[0], kp[1]])
else:
coord_vis[pid, kid] = 0.0
return coord_uv, coord_vis
@staticmethod
def _create_multiple_gaussian_map(coords_uv, output_size, sigma, valid_vec=None):
""" Creates a map of size (output_shape[0], output_shape[1]) at (center[0], center[1])
with variance sigma for multiple coordinates."""
with tf.name_scope('create_multiple_gaussian_map'):
sigma = tf.cast(sigma, tf.float32)
assert len(output_size) == 2
s = coords_uv.get_shape().as_list()
coords_uv = tf.cast(coords_uv, tf.int32)
if valid_vec is not None:
valid_vec = tf.cast(valid_vec, tf.float32)
valid_vec = tf.squeeze(valid_vec)
cond_val = tf.greater(valid_vec, 0.5)
else:
cond_val = tf.ones_like(coords_uv[:, 0], dtype=tf.float32)
cond_val = tf.greater(cond_val, 0.5)
cond_1_in = tf.logical_and(tf.less(coords_uv[:, 0], output_size[0]-1), tf.greater(coords_uv[:, 0], 0))
cond_2_in = tf.logical_and(tf.less(coords_uv[:, 1], output_size[1]-1), tf.greater(coords_uv[:, 1], 0))
cond_in = tf.logical_and(cond_1_in, cond_2_in)
cond = tf.logical_and(cond_val, cond_in)
coords_uv = tf.cast(coords_uv, tf.float32)
# create meshgrid
x_range = tf.expand_dims(tf.range(output_size[0]), 1)
y_range = tf.expand_dims(tf.range(output_size[1]), 0)
X = tf.cast(tf.tile(x_range, [1, output_size[1]]), tf.float32)
Y = tf.cast(tf.tile(y_range, [output_size[0], 1]), tf.float32)
X.set_shape((output_size[0], output_size[1]))
Y.set_shape((output_size[0], output_size[1]))
X = tf.expand_dims(X, -1)
Y = tf.expand_dims(Y, -1)
X_b = tf.tile(X, [1, 1, s[0]])
Y_b = tf.tile(Y, [1, 1, s[0]])
X_b -= coords_uv[:, 0]
Y_b -= coords_uv[:, 1]
dist = tf.square(X_b) + tf.square(Y_b)
scoremap = tf.exp(-dist / tf.square(sigma)) * tf.cast(cond, tf.float32)
return scoremap
@staticmethod
def _detect_scorevol(scorevolume):
""" Finds maximum volumewise. Tensor scorevolume is [1, D, H, W, C]. """
scorevolume = np.squeeze(scorevolume)
s = scorevolume.shape
assert len(s) == 4, "Tensor must be 4D"
coord_det = list()
coord_conf = list()
for i in range(s[3]):
max_val = np.amax(scorevolume[:, :, :, i])
ind = np.where(scorevolume[:, :, :, i] == max_val)
ind = [np.median(x) for x in ind] # this eliminates, when there are multiple maxima
ind = [int(x) for x in ind]
coord_conf.append(max_val)
coord_det.append(ind)
coord_det = np.reshape(np.array(coord_det), [-1, 3])
coord_conf = np.array(coord_conf)
return coord_det, coord_conf
# Actually, it turns out that, whatever the t's scale is, a 3D point will still end up at the same 2D location only. # TO BE PROVED # Take three cameras with any arbitrary place. # Take three more camers with same config except that their t is unit-length. # Verify if u get the distance correctly in the 2nd camera systems. # These will be the centres. c1 = np.asarray([0, 0, 0]).reshape(3, -1) c2 = np.asarray([70, 70, 70]).reshape(3, -1) c3 = np.asarray([30, 15, 40]).reshape(3, -1) c4 = np.asarray([25, 45, 25]).reshape(3, -1) # Let these be the first set of cameras Cam1 = Camera(K, Rc=utils.rotate(), center=c1) pts2d_1 = utils.hom_to_euc(utils.project(Cam1.P, pts3d_11)) Cam2 = Camera(K, Rc=utils.rotate(thetay=169), center=c2) pts2d_2 = utils.hom_to_euc(utils.project(Cam2.P, pts3d_11)) Cam3 = Camera(K, Rc=utils.rotate(thetaz=90), center=c3) pts2d_3 = utils.hom_to_euc(utils.project(Cam3.P, pts3d_11)) Cam4 = Camera(K, Rc=utils.rotate(thetax=90), center=c4) pts2d_4 = utils.hom_to_euc(utils.project(Cam4.P, pts3d_11)) # Let these be the second set of cameras at just unit-distance from world UnitCam1 = Camera(K, Rc=utils.rotate(), center=utils.norm(c1)) unit_pts2d_1 = utils.hom_to_euc(utils.project(UnitCam1.P, pts3d_11)) UnitCam2 = Camera(K, Rc=utils.rotate(thetay=169), center=utils.norm(c2)) unit_pts2d_2 = utils.hom_to_euc(utils.project(UnitCam2.P, pts3d_11)) UnitCam3 = Camera(K, Rc=utils.rotate(thetaz=90), center=utils.norm(c3)) unit_pts2d_3 = utils.hom_to_euc(utils.project(UnitCam3.P, pts3d_11))
class Game:
"""
Class representing the game.
@note Since this example has only one scene, all the timing and sprite sheets are managed in this class.
If there's a requirement for multiple scenes, you might want to create a Scene class separately, and decide which
objects are shared between scenes (for example, player's backpack) and which are in one specific scene only
(for example, timer).
"""
def __init__(self, screen: Surface):
"""
Creates the game
@param screen: The screen surface (created with pygame.display.set_mode) to draw on. This screen is used both
for drawing and for creating the camera.
@note To pave our way to Rust projects, this project uses type hints in function signatures.
"""
# Load sprite sheets (aka costumes)
assets_path = os.path.dirname(os.path.realpath(__file__))
bear_spritesheet = image.load(os.path.join(
assets_path, "..", "assets", "bears.png"))
berry_spritesheet = image.load(os.path.join(
assets_path, "..", "assets", "berry.png"))
# Definition of game objects
self.world = World()
self.bear = Bear(320, 180, bear_spritesheet)
self.berry_group = Group()
for _ in range(100):
x, y = randint(0, 1000), randint(0, 1000)
berry = Berry(x, y, berry_spritesheet)
berry.rect = berry.rect.clamp(self.world.rect)
self.berry_group.add(berry)
self.screen = screen
self.camera = Camera(320, 180, screen)
self.text = TextUI(0, 0)
# Gameplay-related data
self.score = 0
self.remaining = 100
self.timer = 0
# State management. You might want to use a state machine for complex games.
self.running = True
def update(self, ticks: int):
"""
Updates game state with timing information provided by PyGame
@param ticks: number of milliseconds passed since pygame.init() was called
(obtained with pygame.time.get_ticks())
@return: None
"""
self.bear.update(ticks)
self.bear.clamp(self.world.rect)
self.camera.move_to(self.bear.position)
self.camera.clamp(self.world.rect)
collide_list = spritecollide(self.bear, self.berry_group, True)
for _ in collide_list:
self.score += 1
self.remaining = len(self.berry_group)
if self.remaining == 0:
self.running = False
if self.running:
self.timer = ticks
self.text.set_text(
"Collected: {0}. Remaining: {1}. Elapsed time: {2:.1f} s".format(self.score, self.remaining, self.timer / 1000.0))
else:
self.text.set_text(
"You collected all berries in {0:.1f} seconds".format(self.timer / 1000.0))
def on_key_down(self, keys: List[bool]):
"""
Event listener for pygame.KEYDOWN event. All key press checks should be done in this function and on_key_up().
@bug Only checks the first pressed key in the list. Not able to handle multiple keys pressed at a same time
@param keys: a list of key status obtained with pygame.key.get_pressed().
@return: None
"""
directions = [Vector2(0, -1), Vector2(-1, 0),
Vector2(0, 1), Vector2(1, 0)]
accepted_keys = [K_UP, K_LEFT, K_DOWN, K_RIGHT, K_w, K_a, K_s, K_d]
for index, key in enumerate(accepted_keys):
if keys[key]:
self.bear.set_direction(directions[index % 4])
self.bear.set_walking(True)
def on_key_up(self, keys: List[bool]):
"""
Event listener for pygame.KEYDOWN event. All key press checks should be done in this function and on_key_down().
@note As required by Python syntax, the position argument keys must present even if it's not used anywhere in
the function.
@param keys: a list of key status obtained with pygame.key.get_pressed().
@return: None
"""
self.bear.set_walking(False)
def render(self):
"""
Draws all game objects onto the screen based on camera position.
@note Order matters.
@return: None
"""
self.world.draw(self.screen, self.camera)
for berry in self.berry_group:
if berry.rect.colliderect(self.camera.rect):
berry.draw(self.screen, self.camera)
self.bear.draw(self.screen, self.camera)
self.text.render(self.screen)
import tensorflow.keras
from PIL import Image, ImageOps
import numpy as np
import time
import cv2
import serial
from utils.Camera import *
camera = Camera()
arduino = serial.Serial(port="/dev/tty.usbmodem143101",
baudrate=9600) # create usb link
time.sleep(2) #time recommended by arduino
classes = ["verre", "plastique et metal", "carton", "rien"] #object classes
# Scientific notation desactivation
np.set_printoptions(suppress=True)
# Importe model generated with teachablemachine
model = tensorflow.keras.models.load_model('keras_model.h5')
# Create the array of the right shape to feed into the keras model
# The 'length' or number of images you can put into the array is
# determined by the first position in the shape tuple, in this case 1.
data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
def predict_object(image_cv2):
lines = []
for i in range(N_poly):
lines.extend(random_poly().lines)
world = engine.render_world(lines, 75)
save_image(os.path.join(args.output_directory, 'world.png'), world)
times = []
cam_pos = np.array((0, 0))
cam_angle = 0
max_angle_jump = np.radians(10)
data = np.empty((N_steps, 3 + cam_size))
for i in range(N_steps):
cam = Camera(cam_pos, cam_angle, cam_fov, cam_size)
start_time = time.time()
img = engine.render(lines, cam)
times.append(time.time() - start_time)
data[i, 0:2] = cam_pos
data[i, 2] = cam_angle
data[i, 3:] = img
save_image(os.path.join(args.output_directory, 'im_%03d.png' % i), img)
cam_angle += np.random.uniform(-max_angle_jump, max_angle_jump)
np.savetxt(os.path.join(args.output_directory, 'data.txt'),
data,
class BackgroundSubtractor(Thread):
###Default ######
history = 500
shad_tresh = 0.5
var_tresh = 16
var_max = 75
var_min = 4
#################
history = 200
shad_tresh = 0.55
var_tresh = 16
var_max = 75
var_min = 1
arrows = []
fgbg = None
threadLock = None
storage = None
image = None
stopped = False
paused = False
def _initialize_substractor(self):
self.fgbg = cv2.createBackgroundSubtractorMOG2(detectShadows=True)
self.fgbg.setHistory(self.history)
self.fgbg.setShadowThreshold(self.shad_tresh)
self.fgbg.setVarThreshold(self.var_tresh)
# self.fgbg.setVarMax(self.var_max)
# self.fgbg.setVarMin(self.var_min)
return self.fgbg
def get_image(self):
self.threadLock.acquire()
img = copy.copy(self.image)
self.threadLock.release()
return img
def set_image(self, img):
self.threadLock.acquire()
self.image = img
self.threadLock.release()
def get_substracted(self):
self.threadLock.acquire()
img = copy.copy(self.substracted)
self.threadLock.release()
return img
def set_substracted(self, img):
self.threadLock.acquire()
self.substracted = img
self.threadLock.release()
def _set_arrow(self, arrow):
self.threadLock.acquire()
self.arrows.append(arrow)
self.threadLock.release()
def clear_arrows(self):
self.threadLock.acquire()
self.arrows = []
self.threadLock.release()
def get_arrows(self):
self.threadLock.acquire()
return_list = list(self.arrows)
self.threadLock.release()
return return_list
def get_storage(self):
self.threadLock.acquire()
return_list = list(self.storage.storage)
unaltered = list(self.storage.unaltered)
self.threadLock.release()
return return_list, unaltered
def _add_to_storage(self, contours, f1, no_of_frame):
self.threadLock.acquire()
self.storage.add_to_storage(contours, f1, no_of_frame)
self.threadLock.release()
def __init__(self, c1=0, camera=None):
Thread.__init__(self)
self.threadLock = Lock()
print("BackgroundSubstractor called with capture %s" % c1)
if not isinstance(camera, Camera):
self.camera = Camera(device=c1)
else:
self.camera = camera
# c1.release()re('test.avi')
self.storage = ContourStorage()
self._initialize_substractor()
def run(self):
try:
self.run_substraction()
except NoImage:
self.stopped = True
def run_substraction(self):
while not self.stopped:
if not self.paused:
res = self.camera.get_image()
if res is not None:
f1,reseted = res
else:
raise NoImage()
no_of_frame = self.camera.read_frame_no
if reseted:
self._initialize_substractor()
fgmask1 = self.fgbg.apply(f1)
fgmask1 = cv2.inRange(fgmask1, 250, 255)
kernel = np.ones((6, 6), np.uint8)
closed = cv2.morphologyEx(fgmask1, cv2.MORPH_CLOSE, kernel)
kernel = np.ones((4, 4), np.uint8)
opened = cv2.morphologyEx(closed, cv2.MORPH_OPEN, kernel)
closed2 = np.array(closed)
self.set_substracted(closed)
im2, contours, hierarchy = cv2.findContours(closed2, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
colored = cv2.cvtColor(closed2, cv2.COLOR_GRAY2BGR)
self._add_to_storage(contours, f1, no_of_frame)
arrow = self.storage.get_arrow(self.history)
for a in arrow:
self._set_arrow(a)
# print str("Arrows: %s"%self.get_arrows())
stdout.flush()
for arrow in self.get_arrows():
cv2.drawContours(f1, arrow.contours, -1, (0, 255, 0), -1)
cv2.drawContours(f1, [arrow.aproximated], 0, (255, 255, 0), 2)
cv2.circle(f1, (arrow.tip[0], arrow.tip[1]), 3, [255, 0, 0], 2)
f1[arrow.tip[1], arrow.tip[0]] = [255, 0, 0]
cv2.circle(f1, (arrow.tip2[0], arrow.tip2[1]), 3, [101,8,108], 2)
f1[arrow.tip2[1], arrow.tip2[0]] = [101,8,108]
cv2.circle(f1, (arrow.tip3[0], arrow.tip3[1]), 3, [225,97,53], 2)
f1[arrow.tip2[1], arrow.tip2[0]] = [225,97,53]
cv2.circle(f1, (arrow.tip4[0], arrow.tip4[1]), 3, [81,106,37], 2)
f1[arrow.tip2[1], arrow.tip2[0]] = [81,106,37]
rows, cols = f1.shape[:2]
cv2.line(f1, (cols - 1, arrow.line[1]), (0, arrow.line[0]), (255, 255, 0), 1)
cv2.drawContours(f1, [np.int0(arrow.bbox)], 0, (0, 0, 255), 2)
self.set_image(f1)
else:
time.sleep(1)
#
# cv2.imshow("Current", closed)
# cv2.imshow("FG Substraction", colored)
# cv2.imshow("Original", f1)
#
# k = cv2.waitKey(1) & 0xFF
# if k == ord('f') or len(arrows) >= 3:
# arrows = []
# if k == 27:
# break
# if k == 119:
# cv2.waitKey(-1)
# cv2.destroyAllWindows()
def _set_history(self, val):
self.history = val
self.fgbg.setHistory(self.history)
def _set_shad_tresh(self, val):
self.shad_tresh = val/100.
self.fgbg.setShadowThreshold(self.shad_tresh)
def _set_var_tresh(self, val):
self.var_tresh = val
self.fgbg.setVarThreshold(self.var_tresh)
def _set_var_min(self, val):
self.var_min = val
self.fgbg.setVarMin(self.var_min)
def _set_var_max(self, val):
self.var_max = val
self.fgbg.setVarMax(self.var_max)
# airSphere = Sphere()
# airSphere.transform = scaling(0.5, 0.5, 0.5)
# airSphere.material = Material()
# airSphere.material.color = Color.white()
# airSphere.material.diffuse = 0
# airSphere.material.shininess = 300.0
# airSphere.material.reflective = 0.9
# airSphere.material.transparency = 0.9
# airSphere.material.ambient = 0
# airSphere.material.specular = 0.9
# airSphere.material.refractive_index = 1.0000034
world = World()
world.Objects = [Floor, _sphere, _sphere2]
world.light = PointLight(Point(-10, 10, -10), Color(1, 1, 1))
cam = Camera(width, height, pi / 3)
cam.transform = view_transform(Point(0, 1.5, -5), Point(0, 1, 0),
Vector3(0, 1, 0))
# cam.transform = view_transform(Point(0,0, -5), Point(0, 0, 0), Vector3(0, 1, 0))
def PixelRender(x, y):
global cam, world
ray = ray_for_pixel(cam, x, y)
color = color_at(world, ray)
pixel(screen, x, y, color.ConvertColor())
pygame.display.flip()
def render(camera, world):
import utils
from utils import Camera
# np.random.seed(79)
np.set_printoptions(precision=3)
thetaX = np.random.randint(0, 360)
thetaY = np.random.randint(0, 360)
thetaZ = np.random.randint(0, 360)
pts3d_11 = np.random.randint(11, 50, size=(8, 3)).astype(np.float32) # 8-points
K = np.load("../camMatrix_720p.npy")
dist = np.zeros(shape=5)
# This is the first camera with same orientation and same place as origin.
Cam11 = Camera(K)
pts2d_11 = utils.project(Cam11.P, pts3d_11)
pts2d_11 = utils.hom_to_euc(pts2d_11)
# This is second camera.
Cam12 = Camera(K, Rc=utils.rotate(thetax=thetaX), center=np.asarray([10, 25, 7]).reshape(3, -1))
Cam12_R = Cam12.R
Cam12_c = Cam12.center
Cam12_t = Cam12.t
pts3d_12 = np.matmul(Cam12.Rt, utils.euc_to_hom(pts3d_11).T).T
pts2d_12 = utils.project(Cam12.P, pts3d_11)
pts2d_12 = utils.hom_to_euc(pts2d_12)
# This is third camera.
Cam13 = Camera(K, Rc=utils.rotate(thetay=thetaY), center=np.asarray([15, 25, 10]).reshape(3, -1))
Cam13_R = Cam13.R
from flask import Flask, render_template, Response
import numpy as np
import cv2
from utils import Camera
import torch
app = Flask(__name__)
camera = Camera()
@app.route('/home', methods=['GET'])
def home():
return render_template('home.html')
@app.route('/streamCam', methods=['GET'])
def streamCam():
return Response(camera.get_frames(),
mimetype='multipart/x-mixed-replace; boundary=frame')
if __name__ == "__main__":
app.run(debug=True)
def __init__(self, inp):
if isinstance(inp,str):
self.fname = inp.split('.')[0]
self.camera = Camera(device=inp, output=self.fname)
self.board = Board()
camconf = "camera_config.json"
baord_conf = "boardconfig.json"
if os.path.isfile(camconf):
self.camera.load_config(filename=camconf)
else:
self.camera.do_calibration(img=True)
self.camera.save_config(camconf)
if self.board_config_load and os.path.isfile(baord_conf):
with open(baord_conf, 'r') as bc:
imgps = json.loads(bc.readline())
self.Calibrated = BoardCalibrator(camera=self.camera, imgpts=imgps, board=self.board)
else:
self.Calibrated = BoardCalibrator(camera=self.camera)
with open("boardconfig.json", 'w') as bc:
imgps = self.Calibrated.imgpoints
bc.write(json.dumps(imgps))
self.Substractor = BackgroundSubtractor(c1=inp,camera=self.camera)
plt.ion()
self.figure = plt.figure()
self.plt1 = self.figure.add_subplot(111)
self.line1, = self.plt1.plot(range(200), [0] * 200, 'r.-')
self.plt1.axis([0, 200, 0, 10000])
cv2.namedWindow("Current", cv2.WINDOW_NORMAL)
cv2.moveWindow("Current", 20,20)
cv2.namedWindow("Original", cv2.WINDOW_NORMAL)
cv2.moveWindow("Original", 20, 500)
cv2.namedWindow("Points", cv2.WINDOW_NORMAL)
cv2.moveWindow("Current", 1000, 20)
cv2.namedWindow("Blobimg", cv2.WINDOW_NORMAL)
cv2.setMouseCallback("Points", self._click)
mixer.init()
mixer.music.load('beep.mp3')
# cv2.namedWindow("FG Substraction", cv2.WINDOW_NORMAL)
# cv2.createTrackbar("History", "Current", self.history, 1000, self._set_history)
# cv2.createTrackbar("Shadow Treshold", "Current", int(self.shad_tresh * 100), 100, self._set_shad_tresh)
# cv2.createTrackbar("VarThreshold", "Current", self.var_tresh, 100, self._set_var_tresh)
# cv2.createTrackbar("VarMax", "Current", self.var_max, 100, self._set_var_max)
# cv2.createTrackbar("VarMin", "Current", self.var_min, 100, self._set_var_min)
self.Substractor.start()
realboard = np.zeros((self.camera.height, self.camera.width, 3), np.uint8)
# self.frame = self.camera.undistort_image(img)
for i in self.Calibrated.imp:
try:
realboard[i[0][1], i[0][0]] = [0,0,255]
except IndexError:
pass
added = 0
while True:
img = self.Substractor.get_image()
if img is not None:
img = cv2.add(realboard,img)
cv2.imshow("Original", img)
cv2.imshow("Points", self.board.draw_board())
if self.Substractor.stopped:
self.write_data()
exit()
cv2.imshow("Current", self.Substractor.get_substracted())
storage, unaltered = self.Substractor.get_storage()
y = [x[2] for x in storage]
y = unaltered
self.line1.set_xdata(range(len(y)))
self.line1.set_ydata(y)
k = cv2.waitKey(1)
if k == ord('a'):
self.add_dart(frame_no=self.camera.read_frame_no)
if k == ord('s'):
self.write_data()
if k == ord('w'):
pass
self.figure.savefig(r"thesisimages/plot.jpg")
if k == ord('f'):
added = 0
self.Substractor.clear_arrows()
if k == 27:
self.Substractor.stopped = True
break
if k == 119:
print "Pressed w Key so Waiting"
cv2.waitKey(-1)
arrows = self.Substractor.get_arrows()
i = 1
for each in arrows:
tip = each.tip
frame_no = each.frame_no
points = self.Calibrated.calculate_points(tip)
if i > added:
self.add_dart(arrow=each, detected=points, frame_no=frame_no)
added += 1
i += 1
if added >= 3:
added = 0
self.Substractor.clear_arrows()
self.write_data()
#sum_error += error
motor_speed = kp * error + kd * (error - last_error)
last_error = error
left_motor_speed = base_speed - motor_speed
right_motor_speed = base_speed + motor_speed
print(left_motor_speed, right_motor_speed)
bot.forward(left_motor_speed, right_motor_speed)
except Exception as e:
if serial_output == 'N' and prev_reading != 'N':
print("[NODE]")
node_count += 1
#else:
#print(e)
prev_reading = serial_output
cam = Camera()
cameraThread = Thread(target=cam.getArucoID)
motionThread = Thread(target=bot.right, args=(20, 20))
cameraThread.start()
motionThread.start()
cameraThread.join()
motionThread.join()
'''
serial_output = str(ser.readline())
serial_output = int(serial_output[2:-5])
while serial_output>=-1 and serial_output <=50:
bot.right(40,40)
serial_output = str(ser.readline())
color = scipy.misc.imread('./color.png') # color image
color = scipy.misc.imresize(color, (1080, 1920))
depth_w = scipy.misc.imread('./depth.png').astype(
'float32') # depth map warped into the color frame
# intrinsic calibration data
ratio = np.array([1920.0 / 512.0, 1080.0 / 424.0])
K = np.array([[
3.7132019636619111e+02 * ratio[0], 0.0,
2.5185416982679811e+02 * ratio[0]
],
[
0.0, 3.7095047063504268e+02 * ratio[1],
2.1463524817996452e+02 * ratio[1]
], [0.0, 0.0, 1.0]])
cam = Camera(K)
# create algorithm
poseNet = PoseNet3D(ope_depth=OPE_DEPTH,
vpn_type=VPN_TYPE,
gpu_id=GPU_ID,
gpu_memory_limit=GPU_MEMORY,
K=K)
# loop
mask = np.logical_not(depth_w == 0.0)
# run algorithm
coords_pred, det_conf = poseNet.detect(color, depth_w, mask)
# visualize
