def try_frame_difference():
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(0) + ".png", "depth")
start = time.time()
frame_difference = FrameDifference(depth_im / 255, rgb_im / 255, 0.3,
0.005)
print("initialization: ", time.time() - start)
for i in range(5):
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(i) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(i) + ".png",
"depth")
start = time.time()
frame_difference.current_depth = depth_im / 255
frame_difference.current_rgb = rgb_im / 255
mask = frame_difference.subtraction_mask()
mask = frame_difference.create_mask(mask)
print(time.time() - start)
mask = mask * 255
all_masks = np.zeros_like(depth_im)
all_masks = all_masks.astype(float)
for j in range(len(mask)):
all_masks += mask[j].astype(float)
image_processing.save_image(all_masks / 255,
"Results/Frame difference", i, "mask")
def create_points_cloud():
rgb_im = image_processing.load_image("falling ball and cube", "rgb_3.png")
depth_im = image_processing.load_image("falling ball and cube",
"depth_3.png", "depth")
mask_im = image_processing.load_image("Mask", "mask.png")
rgb_im, depth_im = apply_mask(rgb_im, depth_im, mask_im / 255)
return image_processing.calculate_point_cloud(rgb_im / 255, depth_im / 255)
def try_DEVB():
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(0) + ".png", "depth")
start = time.time()
devb = DEVB(rgb_im=rgb_im / 255,
depth_im=depth_im / 255,
number_of_samples=10,
time_factor=16)
print(time.time() - start)
for i in range(5):
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(i) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(i) + ".png",
"depth")
start = time.time()
devb.set_images(rgb_im / 255, depth_im / 255)
devb.set_mask()
print(time.time() - start)
mask = devb.mask
image_processing.save_image(mask, "Results/DEVB", i, "mask")
def save_point_cloud_from_images():
rgb_im = image_processing.load_image("preDiploma_PC/", "rgb_box_0.png")
depth_im = image_processing.load_image("preDiploma_PC/", "depth_box_0.png",
"depth")
points, color = image_processing.calculate_point_cloud(
rgb_im / 255, depth_im / 255)
current_object = PointsObject()
current_object.add_points(points, color)
current_object.save_all_points("preDiploma_PC/", "box")
def try_RGB_MoG():
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
start = time.time()
mog = RGB_MoG(rgb_im, number_of_gaussians=3)
print("initialization: ", time.time() - start)
for i in range(5):
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(i) + ".png")
start = time.time()
mask = mog.set_mask(rgb_im)
print("frame updating: ", time.time() - start)
image_processing.save_image(mask / 255, "Results/RGB MoG", i, "mask")
def check_data_generation():
data_generation.save_images_from_VREP()
depth_im = image_processing.load_image("3d_map/", "room_depth0.png",
"depth")
rgb_im = image_processing.load_image("3d_map/", "room_rgb0.png")
xyz, rgb = image_processing.calculate_point_cloud(rgb_im / 255,
depth_im / 255)
temp = PointsObject()
temp.add_points(xyz, rgb)
temp.save_all_points("3d_map/", "room")
visualization.visualize([temp])
def on_created(self, event):
"""
Event triggered when a new file is created inside the watched directory.
Loads the image, delets it from the temp directory, process it and insert it
in the database.
:param event: information regarding newly created file
:return:
"""
# Found file's path
path = event.src_path
# If the image is something else than an image, we stop here
extension = path.split('.')[-1]
if extension != 'png':
return
logger.info('found file: %s' % path)
# Load image from path as a PIL Image object
img = load_image(path)
self.delete_temp_image(path)
self.process_temp_image(img)
# Delete object
del img
def get_moving_mask(number_of_frame=1):
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(0) + ".png",
"depth")
mog = RGBD_MoG(rgb_im, depth_im, number_of_gaussians=3)
rgb_im = image_processing.load_image(
"falling balls and cylinder",
"rgb_" + str(number_of_frame) + ".png")
depth_im = image_processing.load_image(
"falling balls and cylinder",
"depth_" + str(number_of_frame) + ".png", "depth")
mask = mog.set_mask(rgb_im, depth_im)
return mask, depth_im, rgb_im
def ingest_tile_files(tile_files, game, dir):
ctr = 0
tag_ctr = 0
for tile_file in tile_files:
file_name = os.path.split(tile_file)[1]
tile_uuid = os.path.splitext(file_name)[0]
is_in = check_uuid_in_tiles(tile_uuid)
if is_in:
print(f'SKIPPED INGESTING TILE: {tile_uuid}')
else:
cv, encoded_png = P.load_image(tile_file)
h, w, c = cv.shape
data = encoded_png.tobytes()
result = insert_tile(tile_uuid, game, int(w), int(h), data)
# tile_id = result['tile_id']
# TODO TILE AFFORDANCES
csv_file = os.path.join(dir, game, 'tiles', 'tile_affordances.csv')
tile_entry = affords_from_csv_file(csv_file, tile_uuid)
if tile_entry is not None:
print('TILE HAD AFFORDS')
insert_tile_tag(tile_uuid, tile_entry['tagger_id'],
int(tile_entry['solid']),
int(tile_entry['movable']),
int(tile_entry['destroyable']),
int(tile_entry['dangerous']),
int(tile_entry['gettable']),
int(tile_entry['portal']),
int(tile_entry['usable']),
int(tile_entry['changeable']),
int(tile_entry['ui']))
tag_ctr += 1
ctr += 1
return ctr
def ingest_screenshot_files_with_offsets(files, game, dir):
offsets_csv = os.path.join(dir, game,
f'{game}_min_unique_lengths_offsets.csv')
ctr = 0
tag_ctr = 0
for screen_file in files:
file_name = os.path.split(screen_file)[1]
screenshot_uuid = os.path.splitext(file_name)[0]
is_in = check_uuid_in_screenshots(screenshot_uuid)
if is_in:
print(f'SKIPPED INGESTING IMAGE: {screenshot_uuid}')
else:
y_offset, x_offset = offsets_from_csv_file(offsets_csv,
screenshot_uuid)
print('offsets got for image num: {}, y:{}, x:{}'.format(
screenshot_uuid, y_offset, x_offset))
cv_image, encoded_png = P.load_image(screen_file)
h, w, *_ = cv_image.shape
data = encoded_png.tobytes()
result = insert_screenshot(screenshot_uuid, game, int(w), int(h),
y_offset, x_offset, data)
#TODO: Load known labels from numpy
labels = P.load_label(screen_file)
if labels is not None:
ingest_screenshot_tags(labels, screenshot_uuid)
tag_ctr += 1
ctr += 1
return ctr, tag_ctr
def try_fast_RGBD_MoG():
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(0) + ".png", "depth")
start = time.time()
mog = Fast_RGBD_MoG(rgb_im, depth_im, number_of_gaussians=3)
print("initialization: ", time.time() - start)
for i in range(5):
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(i) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(i) + ".png",
"depth")
start = time.time()
mask = mog.get_mask(rgb_im, depth_im)
print("frame updating: ", time.time() - start)
def check_moving_detection():
import moving_detection
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(0) + ".png", "depth")
start = time.time()
mog = moving_detection.Fast_RGBD_MoG(rgb_im,
depth_im,
number_of_gaussians=5)
print("initialization: ", time.time() - start)
for i in range(4):
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(i + 1) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(i + 1) + ".png",
"depth")
start = time.time()
mask = mog.get_mask(rgb_im, depth_im)
print("frame updating: ", time.time() - start)
visualization.show_image(mask / 255)
def try_ViBE():
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
start = time.time()
vibe = ViBЕ(rgb_im=rgb_im / 255,
number_of_samples=10,
threshold_r=20 / 255,
time_factor=16)
print(time.time() - start)
for i in range(5):
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(i) + ".png")
start = time.time()
vibe.current_rgb = rgb_im / 255
vibe.set_mask()
print(time.time() - start)
mask = vibe.mask
image_processing.save_image(mask, "Results/ViBE", i, "mask")
def make_template_images(template_filepaths, size=None):
"""Make dictionary of images from template_filepaths.
:param template_filepaths:
:param size:
"""
templates = {}
for filepath in template_filepaths:
template = imp.load_image(filepath)
if size is not None:
template = imp.resize_image(template, size)
template = imp.image_gray(template)
template = imp.image_bin_otsu(template)
template = imp.invert(template)
templates[filepath] = template
return templates
def ingest_screenshots(game, screenshots_dir):
ctr = 0
tag_ctr = 0
skip_ctr = 0
image_folders = next(os.walk(screenshots_dir))[1]
for screenshot_uuid in image_folders:
screenshot_file = os.path.join(screenshots_dir, screenshot_uuid,
f'{screenshot_uuid}.png')
# screenshot_uuid = os.path.splitext(file_name)[0]
is_in = check_uuid_in_screenshots(screenshot_uuid)
if is_in:
print(f'SKIPPED INGESTING IMAGE: {screenshot_uuid}')
skip_ctr += 1
else:
metadata = metadata_from_json(screenshots_dir, screenshot_uuid)
print('offsets got for image num: {}, y:{}, x:{}'.format(
screenshot_uuid, metadata['y_offset'], metadata['x_offset']))
cv_image, encoded_png = P.load_image(screenshot_file)
h, w, *_ = cv_image.shape
data = encoded_png.tobytes()
result = insert_screenshot(screenshot_uuid, game, int(w), int(h),
data, **metadata)
#TODO: Load known labels from numpy
label_files = glob.glob(
os.path.join(screenshots_dir, screenshot_uuid, "*.npy"))
if len(label_files) > 0:
for label_file in label_files:
tagger_npy = os.path.split(label_file)[1]
tagger = os.path.splitext(tagger_npy)[0]
has_tagged = check_tagger_tagged_screenshot(
screenshot_uuid, tagger)
if has_tagged:
print(
f'SKIPPED INGESTING Tags:{tagger} on {screenshot_uuid}'
)
else:
label = P.load_label_from_tagger(label_file)
if label is not None:
ingest_screenshot_tags(label,
screenshot_uuid,
tagger=tagger)
tag_ctr += 1
ctr += 1
return ctr, tag_ctr, skip_ctr
def make_edge_image(image, lower, upper, new_name):
img_BGR = load_image(image)
black_white = cv.cvtColor(img_BGR, cv.COLOR_BGR2GRAY)
# finding edges (new image array)
edges = cv.Canny(black_white, lower, upper)
cv.imwrite('edges_' + new_name, edges)
# contours of edge image (list of image arrays)
image, contours, hierarchy = cv.findContours(edges, cv.RETR_TREE,
cv.CHAIN_APPROX_NONE)
cv.imwrite('edge-contours_' + new_name,
cv.drawContours(img_BGR.copy(), contours, -1, (0, 255, 0), 1))
cv.imwrite(
'just_edge-contours_' + new_name,
cv.drawContours(np.zeros(img_BGR.shape), contours, -1, (0, 255, 0), 1))
def find_whole_notes(image, regions, bar_lines, clefs, time_signatures,
staff, staff_spacing, staff_distance, min_match=0.61):
note_templates = {}
for templateName in search_for_templates(["note_heads/whole", "note_heads/double_whole"]):
template = imp.load_image(templateName)
template = imp.resize_image(template, (int(round(staff_spacing)), int(round(staff_spacing))))
template = imp.image_gray(template)
template = imp.image_bin_otsu(template)
template = imp.invert(template)
note_templates[templateName] = template
rel_staff = get_rel_staff(staff, staff_distance)
# find note_heads
print("Finding whole note heads...")
notes = []
possible_regions = get_possible_whole_note_regions(regions, bar_lines, clefs, time_signatures, staff_spacing)
for region in possible_regions:
reg_top = min([r for r, c in region])
reg_height, reg_width = get_region_image(image, region).shape[:2]
if staff_spacing <= reg_width and reg_height >= staff_spacing:
min_r = rel_staff[0][-1]
line_index = 0.5
while abs(min_r - reg_top) >= staff_spacing // 2:
if min_r > reg_top:
min_r -= staff_spacing // 2
line_index -= 0.5
else:
min_r += staff_spacing // 2
line_index += 0.5
min_r = int(min_r)
max_r = max([r for r, c in region])
for start_r in range(min_r, max_r, int(staff_spacing // 2)):
sub_region = [value for value in region
if start_r + staff_spacing >= value[0] >= start_r]
if len(sub_region) > 0:
sub_region_img = get_region_image(image, sub_region)
best_match = template_match(sub_region_img,
template_images=note_templates,
print_results=False)
if min_match <= best_match[1]:
notes += [(region, line_index, best_match)]
line_index += 0.5
return notes
def predict_images(model):
img_path = sys.argv[1]
#img_path = input('image path: ')
# img = openImage(img_path)
# ret,img = cv2.threshold(img,130,255,cv2.THRESH_BINARY_INV)
img = imp.load_image(img_path)
img = ndimage.median_filter(img, 3)
# imp.show_image(img)
list_img = s._main(img)
# imp.show_image_array(list_img)
for i in range(len(list_img)):
h, w = list_img[i].shape
if h + w <= 5:
continue
img = list_img[i]
imp.show_image_array(img)
img = resize(img, 28, 28)
img = np.array(img)
#print(img)
img = img / 255
#print(img)
img = np.reshape(img, (1, 28, 28, 1))
result = model.predict_classes(img)
#print('res: ',result)
#print(labels)
kio = le.inverse_transform(result)
print(kio[0])
with open('dataaa.csv') as csvfile:
readCSV = csv.reader(csvfile, delimiter=',')
filee = open("file.txt", "a")
for row in readCSV:
if row[0] == kio[0]:
nee = row[1]
filee.write(nee)
print('UNICODE Value:', nee)
break
filee.close()
def find_vertical_notes(org_image, regions, staff, staff_spacing, staff_distance,
tolerance=None, flag_min_match=0.7, note_head_min_match=0.8,
half_note_head_min_match=0.65):
image = org_image.copy()
rel_staff = get_rel_staff(staff, staff_distance)
recognized_notes = []
unrecognized_regions = []
small_regions = []
note_heads_templates = {}
for templateName in search_for_templates(["note_heads/filled"]):
template = imp.load_image(templateName)
template = imp.resize_image(template, (int(round(staff_spacing)), int(round(staff_spacing))))
template = imp.image_gray(template)
template = imp.image_bin_otsu(template)
template = imp.invert(template)
note_heads_templates[templateName] = template
half_note_heads_templates = {}
for templateName in search_for_templates(["note_heads/half", "note_heads/whole"]):
template = imp.load_image(templateName)
template = imp.resize_image(template, (int(round(staff_spacing)), int(round(staff_spacing))))
template = imp.image_gray(template)
template = imp.image_bin_otsu(template)
template = imp.invert(template)
half_note_heads_templates[templateName] = template
for index, region in enumerate(regions):
org_reg_c = min([c for r, c in region])
org_reg_r = min([r for r, c in region])
region_image = irr.get_region_image(image, region)
if len(region_image) > 3 * staff_spacing:
img_vert_lines = imo.open_image_vertically(region_image, staff_spacing, 3)
vertical_lines = find_regions(img_vert_lines)[1]
remove_white_pixels([region_image], vertical_lines)
avg_vert_line_thickness = find_avg_thickness(vertical_lines)
if tolerance is None:
tolerance = avg_vert_line_thickness
# Find beams by searching for regions between vertical lines
print("Finding full beams...")
full_beams = []
connected_regions = []
separate_regions = []
sub_regions = find_regions(region_image)[1]
for sub_region in sub_regions:
start_vert_line = None
end_vert_line = None
lines = []
min_col = min([c for r, c in sub_region])
max_col = max([c for r, c in sub_region])
for line in vertical_lines:
min_line_col = min([c for r, c in line])
max_line_col = max([c for r, c in line])
if -tolerance <= min_col - max_line_col <= tolerance:
start_vert_line = line
lines += [line]
elif -tolerance <= min_line_col - max_col <= tolerance:
end_vert_line = line
lines += [line]
elif min_col < min_line_col < max_col:
lines += [line]
if start_vert_line is not None and end_vert_line is not None:
full_beams += [(sub_region, start_vert_line, end_vert_line, lines)]
elif len(lines) > 0:
connected_regions += [(sub_region,
lines[0])]
else:
separate_regions += [sub_region]
# Find half beams
print("Finding half beams...")
half_beams = []
for sub_region in connected_regions:
max_row = max([r for r, c in sub_region[0]])
min_row = min([r for r, c in sub_region[0]])
min_col = min([c for r, c in sub_region[0]])
max_col = max([c for r, c in sub_region[0]])
for beam in full_beams:
if sub_region[1] in beam[3]:
try:
min_beam_row = min([r for r, c in beam[0] if min_col <= c <= max_col])
max_beam_row = max([r for r, c in beam[0] if min_col <= c <= max_col])
except ValueError:
continue
distance = None
if min_row > max_beam_row:
distance = min_row - max_beam_row
elif max_row < min_beam_row:
distance = min_beam_row - max_row
if distance is not None and distance < staff_spacing:
half_beams += [sub_region]
break
for half_beam in half_beams:
connected_regions.remove(half_beam)
# find flags
print("Finding flags...")
flags = []
flag_templates = search_for_templates("flags")
for sub_region in connected_regions:
best_match = template_match(get_region_image(region_image, sub_region[0]),
template_filepaths=flag_templates,
resize=True, print_results=False)
if flag_min_match <= best_match[1]:
flags += [(sub_region, best_match)]
for flag, match in flags:
connected_regions.remove(flag)
# find note_heads
print("Finding note heads...")
note_heads = []
for connected_region in connected_regions:
min_r = rel_staff[0][-1]
line_index = 0.5
while abs(min_r - org_reg_r) >= staff_spacing // 2:
if min_r > org_reg_r:
min_r -= staff_spacing // 2
line_index -= 0.5
else:
min_r += staff_spacing // 2
line_index += 0.5
min_r -= org_reg_r
min_r = int(min_r)
max_r = max([r for r, c in connected_region[0]])
for start_r in range(min_r, max_r, int(staff_spacing // 2)):
sub_region = [value for value in connected_region[0]
if start_r + staff_spacing >= value[0] >= start_r]
if len(sub_region) > 0:
sub_region_img = get_region_image(region_image, sub_region)
best_match = template_match(sub_region_img,
template_images=note_heads_templates,
print_results=False)
if note_head_min_match <= best_match[1]:
note_heads += [(sub_region, connected_region, line_index, best_match)]
else:
best_match = template_match(sub_region_img,
template_images=half_note_heads_templates,
print_results=False)
if half_note_head_min_match <= best_match[1]:
note_heads += [(sub_region, connected_region, line_index,
("templates/note_heads/half_01", best_match[1]))]
line_index += 0.5
for note_head, connected_region, line_index, match in note_heads:
if connected_region in connected_regions:
connected_regions.remove(connected_region)
# recognize note
print("Recognizing notes' properties...")
checked_flags = []
notes = []
for note_head, connected_region, line_index, match in note_heads:
height = line_index
note_head_type = match[0].split('/')[-1].split('_')[0]
flags_and_beams = 0
line = connected_region[1]
for beam in full_beams:
if line in beam[3]:
flags_and_beams += 1
for half_beam in half_beams:
if line == half_beam[1]:
flags_and_beams += 1
for flag, flag_match in flags:
if line == flag[1]:
flags_and_beams += int(flag_match[0].split('/')[-1].split('_')[0]) / 8
checked_flags += [(flag, flag_match)]
if flags_and_beams > 0:
duration = 1 / 4.
for i in range(flags_and_beams):
duration /= 2
else:
duration = 1 / 4. if note_head_type == "filled" else 0.5
notes += [(min([c for r, c in connected_region[0]]), height, note_head_type, duration)]
for flag in flags:
if flag not in checked_flags:
connected_region = flag[0]
min_r = rel_staff[0][-1]
line_index = 0.5
while abs(min_r - org_reg_r) >= staff_spacing // 2:
if min_r > org_reg_r:
min_r -= staff_spacing // 2
line_index -= 0.5
else:
min_r += staff_spacing // 2
line_index += 0.5
min_r -= org_reg_r
min_r = int(min_r)
max_r = max([r for r, c in connected_region[0]])
for start_r in range(min_r, max_r, int(staff_spacing // 2)):
sub_region = [value for value in connected_region[0]
if start_r + staff_spacing >= value[0] >= start_r]
if len(sub_region) > 0:
best_match = template_match(get_region_image(region_image,sub_region),
template_images=half_note_heads_templates,
print_results=False)
if half_note_head_min_match <= best_match[1]:
note_heads += [(sub_region, connected_region, line_index,
("templates/note_heads/half_01", best_match[1]))]
notes += [(min([c for r, c in connected_region[0]]), line_index, "half", 0.5)]
line_index += 0.5
notes = sorted(notes)
recognized_notes += [(region, org_reg_c, notes)]
unrecognized_regions += [(region, connected_regions, separate_regions)]
if False:
# Remove flags and beams from original image
to_remove = []
to_remove_from_region_image = []
for flag, match in flags:
for r, c in flag[0]:
to_remove += [(r + org_reg_r, c + org_reg_c)]
to_remove_from_region_image += [(r, c)]
for half_beam in half_beams:
for r, c in half_beam[0]:
to_remove += [(r + org_reg_r, c + org_reg_c)]
to_remove_from_region_image += [(r, c)]
for beam in full_beams:
for r, c in beam[0]:
to_remove += [(r + org_reg_r, c + org_reg_c)]
to_remove_from_region_image += [(r, c)]
for note_head, connected_region, line_index, match in note_heads:
for r, c in note_head:
to_remove += [(r + org_reg_r, c + org_reg_c)]
to_remove_from_region_image += [(r, c)]
for line in vertical_lines:
for r, c in line:
to_remove += [(r + org_reg_r, c + org_reg_c)]
to_remove_from_region_image += [(r, c)]
remove_white_pixels([image], [to_remove])
remove_white_pixels([region_image], [to_remove_from_region_image])
else:
small_regions += [region]
# remove_white_pixels([image], [region])
recognized_notes.sort(key=lambda x: x[1])
return recognized_notes
def load_image(image_name):
print("Loading image: %s" % image_name)
return imp.load_image(image_name)
def objects_test_moving_figures_local():
number_of_comparing_points = 100
classes = {}
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(0) + ".png",
"depth")
mog = RGBD_MoG(rgb_im, depth_im, number_of_gaussians=3)
for number_of_frame in range(1, 5):
color_mask = np.zeros([rgb_im.shape[0], rgb_im.shape[1], 3])
rgb_im = image_processing.load_image(
"falling balls and cylinder",
"rgb_" + str(number_of_frame) + ".png")
depth_im = image_processing.load_image(
"falling balls and cylinder",
"depth_" + str(number_of_frame) + ".png", "depth")
mask = mog.set_mask(rgb_im, depth_im)
depth_im = depth_im * (mask / 255).astype(int)
masks = region_growing(mask / 255,
depth_im / 255,
depth_threshold=0.01,
significant_number_of_points=10)
if len(masks) == 0:
print("No moving objects in the frame")
else:
for mask in masks:
xyz_points, rgb_points = image_processing.calculate_point_cloud(
rgb_im / 255, depth_im * mask / 255)
current_object = PointsObject()
current_object.set_points(xyz_points, rgb_points)
norms = current_object.get_normals()
compared_object_descriptor = CovarianceDescriptor(
xyz_points,
rgb_points,
norms,
k_nearest_neighbours=None,
relevant_distance=0.1,
use_alpha=True,
use_beta=True,
use_ro=True,
use_theta=True,
use_psi=True,
use_rgb=True)
match_found = False
lengths = np.zeros([
len(classes),
np.amin(
[number_of_comparing_points, xyz_points.shape[0]])
])
if number_of_frame == 1:
match_found = False
else:
match_found = True
for object_number, object_class in enumerate(classes):
lengths[
object_number] = object_class.compare_descriptors(
compared_object_descriptor.
object_descriptor,
number_of_comparing_points)
print(np.sum(mask))
min_args = np.argmin(
lengths, axis=0)[np.amin(lengths, axis=0) < 0.1]
unique, counts = np.unique(min_args,
return_counts=True)
best_match = unique[np.argmax(counts)]
for object_number, object_class in enumerate(classes):
if object_number == best_match:
color_mask[:, :,
0] += mask * classes[object_class][0]
color_mask[:, :,
1] += mask * classes[object_class][1]
color_mask[:, :,
2] += mask * classes[object_class][2]
if not match_found:
classes[compared_object_descriptor] = np.random.rand(3)
color_mask[:, :, 0] += mask * classes[
compared_object_descriptor][0]
color_mask[:, :, 1] += mask * classes[
compared_object_descriptor][1]
color_mask[:, :, 2] += mask * classes[
compared_object_descriptor][2]
image_processing.save_image(color_mask,
"tracking_results",
frame_number=number_of_frame,
image_name="local_two_same")
def objects_test_moving_figures_global():
classes = {}
rgb_im = image_processing.load_image("falling balls and cylinder",
"rgb_" + str(0) + ".png")
depth_im = image_processing.load_image("falling balls and cylinder",
"depth_" + str(0) + ".png",
"depth")
mog = RGBD_MoG(rgb_im, depth_im, number_of_gaussians=3)
for number_of_frame in range(1, 5):
color_mask = np.zeros([rgb_im.shape[0], rgb_im.shape[1], 3])
rgb_im = image_processing.load_image(
"falling balls and cylinder",
"rgb_" + str(number_of_frame) + ".png")
depth_im = image_processing.load_image(
"falling balls and cylinder",
"depth_" + str(number_of_frame) + ".png", "depth")
mask = mog.set_mask(rgb_im, depth_im)
depth_im = depth_im * (mask / 255).astype(int)
masks = region_growing(mask / 255,
depth_im / 255,
depth_threshold=0.01,
significant_number_of_points=10)
if len(masks) == 0:
print("No moving objects in the frame")
else:
for mask in masks:
xyz_points, rgb_points = image_processing.calculate_point_cloud(
rgb_im / 255, depth_im * mask / 255)
current_object = PointsObject()
current_object.set_points(xyz_points, rgb_points)
norms = current_object.get_normals()
compared_object_descriptor = GlobalCovarianceDescriptor(
xyz_points,
rgb_points,
norms,
depth_im,
rgb_im,
mask,
use_xyz=True,
use_rgb=True,
use_normals=True)
match_found = False
lengths = np.zeros([len(classes)])
if number_of_frame == 1:
match_found = False
else:
match_found = True
for object_number, object_class in enumerate(classes):
lengths[
object_number] = object_class.compare_descriptors(
compared_object_descriptor.
object_descriptor)
min_arg = np.argmin(lengths)
print(lengths)
for object_number, object_class in enumerate(classes):
if object_number == min_arg:
color_mask[:, :,
0] += mask * classes[object_class][0]
color_mask[:, :,
1] += mask * classes[object_class][1]
color_mask[:, :,
2] += mask * classes[object_class][2]
if not match_found:
classes[compared_object_descriptor] = np.random.rand(3)
color_mask[:, :, 0] += mask * classes[
compared_object_descriptor][0]
color_mask[:, :, 1] += mask * classes[
compared_object_descriptor][1]
color_mask[:, :, 2] += mask * classes[
compared_object_descriptor][2]
image_processing.save_image(color_mask,
"tracking_results",
frame_number=number_of_frame,
image_name="global_two_same")
import cv2
import numpy as np
import find_marker
import image_processing
import brightness_correction
import os
import glob
# marker image folder for testing
marker_path = "../../testdata/images"
for marker in range(1,4):
marker_rgb,marker_gray=image_processing.load_image(marker_path+"/marker"+str(marker)+".jpg")
heightm, widthm, channelsm = marker_rgb.shape
for file in glob.glob(marker_path + "/marker"+str(marker)+"_*.jpg"):
image_rgb,image_gray=image_processing.load_image(file)
#brightness correction
image_rgb,image_gray = brightness_correction.BrightnessAndContrastAuto(image_rgb, 2.0)
heighti, widthi, channelsi = image_rgb.shape
hintPos=[widthi/2.0, heighti/2.0]
if marker==1:
markers, templatelocm,keypointsm=find_marker.find_marker(image_gray, [141.33, 141.33], "Circular", [2.2, 2.2], "Rectangular", [3.0, 3.0], marker_gray, 0.6, hintPos)
def openImage(path):
# img = cv2.imread(path,cv2.IMREAD_GRAYSCALE)
# ret,img = cv2.threshold(img,130,255,cv2.THRESH_BINARY)
"""using PIL"""
img = imp.load_image(path)
return img
def create_mask():
color_mask = image_processing.load_image("tracking_results",
"global_two_different3.png")
binary_mask = np.where(np.sum(color_mask, axis=2), 1, 0)
image_processing.save_image(binary_mask, "Mask", "mask")
# ------------------------------------------------------------------------------
datasets = ["train", "extra", "test"]
print("DATASET SIZES")
for dataset in datasets:
print(dataset + " : ", Y[dataset]["N"].shape[0])
# ------------------------------------------------------------------------------
# VISUALIZE SAMPLES FROM TRAIN DATA
# ------------------------------------------------------------------------------
# LOAD IMAGES INTO AN ARRAY
n_samples = 16
dimx, dimy = 50, 50
imgs = np.empty(shape=[n_samples, dimx, dimy, 3])
for i in range(n_samples):
file = Y["train"]["file"][i]
img = load_image(os.path.join(data_dir, "train", file))
img.thumbnail([dimx, dimy], ANTIALIAS)
x = (dimx - img.width) / 2
y = (dimy - img.height) / 2
img_box = Image.new('RGB', (dimx, dimy), (255, 255, 255))
img_box.paste(img, (x, y))
imgs[i] = pil2array(img_box)
# img_box.show()
# VIEW LABELLED IMAGES AS A GRID
grid_of_sample_images(
imgs,
# labels=fivers_labels ,
gridsize=(4, 4),
label_color="#000000",
title="Sample of Images from Train Dataset",
