def detect(args):
image, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
if not data.feature_index_exists(image):
mask = data.mask_as_array(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
preemptive_max = data.config.get('preemptive_max', 200)
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.image_as_array(image), data.config, mask)
if len(p_unsorted) == 0:
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
if data.config.get('matcher_type', 'FLANN') == 'FLANN':
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
def detect(args):
image, data = args
log.setup()
need_words = data.config[
'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0
need_flann = data.config['matcher_type'] == 'FLANN'
has_words = not need_words or data.words_exist(image)
has_flann = not need_flann or data.feature_index_exists(image)
has_features = data.features_exist(image)
if has_features and has_flann and has_words:
logger.info('Skip recomputing {} features for image {}'.format(
data.feature_type().upper(), image))
return
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
data.load_image(image), data.config)
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_flann:
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config['bow_words_to_match']
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config['bow_matcher_type'])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
def detect(args):
image, data = args
log.setup()
need_words = (data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if has_features and has_words:
logger.info("Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image))
return
logger.info("Extracting {} features for image {}".format(
data.feature_type().upper(), image))
start = timer()
image_array = data.load_image(image)
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image,
image_array))
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config["bow_matcher_type"])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args):
image, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
if not data.features_exist(image):
mask = data.mask_as_array(image)
if mask is not None:
print("found mask for image:%s" % image)
logger.info('Found mask to apply for image {}'.format(image))
else:
print("Not found mask for the image")
preemptive_max = data.config.get('preemptive_max', 200)
the_image = data.image_as_array(image)
save_no_mask = False
all_content = features.extract_features(the_image, data.config, mask,
save_no_mask)
if save_no_mask:
p_unsorted, f_unsorted, c_unsorted = all_content[0]
p_nomask, f_nomask, c_nomask = all_content[1]
else:
p_unsorted, f_unsorted, c_unsorted = all_content
if len(p_unsorted) == 0:
return
'''
size_nomask = p_nomask[:, 2]
order_nomask = np.argsort(size_nomask)
p_nomask = p_nomask[order_nomask, :]
f_nomask = f_nomask[order_nomask, :]
c_nomask = c_nomask[order_nomask, :]
p_nomask_pre = p_nomask[-preemptive_max:]
f_nomask_pre = f_nomask[-preemptive_max:]
data.save_features(image+'_nomask', p_nomask, f_nomask, c_nomask)
data.save_preemptive_features(image+'_nomask', p_nomask_pre, f_nomask_pre)
index_nomask = features.build_flann_index(f_nomask, data.config)
data.save_feature_index(image+'_nomask', index_nomask)
'''
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if data.config.get('preemptive_threshold', 0) > 0:
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_preemptive_features(image, p_pre, f_pre)
if data.config.get('matcher_type', "BRUTEFORCE") == "FLANN":
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
def detect(args):
image, data = args
print('detect')
print(image)
print(data)
#log.setup()
need_words = data.config[
'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if has_features and has_words:
#logger.info('Skip recomputing {} features for image {}'.format(data.feature_type().upper(), image))
return
#logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image))
#start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
data.load_image(image), data.config)
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
#logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config['bow_words_to_match']
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config['bow_matcher_type'])
data.save_words(image, closest_words)
def detect(feature_path, image_path, image, opensfm_config):
log.setup()
need_words = opensfm_config['matcher_type'] == 'WORDS' or opensfm_config[
'matching_bow_neighbors'] > 0
#has_words = not need_words or data.words_exist(image)
#has_features = data.features_exist(image)
# if has_features and has_words:
# logger.info('Skip recomputing {} features for image {}'.format(
# data.feature_type().upper(), image))
# return
#logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image))
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
load_image(image_path), opensfm_config)
#p_unmasked is points
mask_files = defaultdict(lambda: None)
fmask = load_features_mask(feature_path, image, image_path, p_unmasked,
mask_files, opensfm_config)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
#logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
save_features(feature_path, opensfm_config, image, p_sorted, f_sorted,
c_sorted)
if need_words:
bows = bow.load_bows(opensfm_config)
n_closest = opensfm_config['bow_words_to_match']
closest_words = bows.map_to_words(f_sorted, n_closest,
opensfm_config['bow_matcher_type'])
save_words(feature_path, image_path, closest_words)
def extract_features(problem, data):
"""
Extract features from all images, and save to DataSet.
"""
assert 'masks' not in data.config
for image in data.images():
if data.feature_index_exists(image):
print "{} - extracting features (cached)".format(image)
else:
print "{} - extracting features".format(image)
data.config['masks'] = problem.image2masks[image]
points, descriptors, colors = features.extract_features(
data.image_as_array(image), data.config)
del data.config['masks']
data.save_features(image, points, descriptors, colors)
index = features.build_flann_index(descriptors, data.config)
data.save_feature_index(image, index)
def detect(args):
log.setup()
image, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
if not data.feature_index_exists(image):
start = timer()
mask = data.load_combined_mask(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
preemptive_max = data.config['preemptive_max']
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.load_image(image), data.config, mask)
if len(p_unsorted) == 0:
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
if data.config['matcher_type'] == 'FLANN':
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report),
'features/{}.json'.format(image))
def detect(args):
log.setup()
image, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
if not data.feature_index_exists(image):
start = timer()
mask = data.mask_as_array(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
preemptive_max = data.config['preemptive_max']
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.image_as_array(image), data.config, mask)
if len(p_unsorted) == 0:
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
if data.config['matcher_type'] == 'FLANN':
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report),
'features/{}.json'.format(image))
def detect(args):
image, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
if not data.feature_index_exists(image):
preemptive_max = data.config.get('preemptive_max', 200)
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.image_as_array(image), data.config)
if len(p_unsorted) == 0:
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
def detect(
image: str,
image_array: np.ndarray,
segmentation_array: Optional[np.ndarray],
instances_array: Optional[np.ndarray],
data: DataSetBase,
force: bool = False,
) -> None:
log.setup()
need_words = (
data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0
)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if not force and has_features and has_words:
logger.info(
"Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image
)
)
return
logger.info(
"Extracting {} features for image {}".format(data.feature_type().upper(), image)
)
start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image, image_array)
)
# Load segmentation and bake it in the data
if data.config["features_bake_segmentation"]:
exif = data.load_exif(image)
s_unsorted, i_unsorted = bake_segmentation(
image_array, p_unmasked, segmentation_array, instances_array, exif
)
p_unsorted = p_unmasked
f_unsorted = f_unmasked
c_unsorted = c_unmasked
# Load segmentation, make a mask from it mask and apply it
else:
s_unsorted, i_unsorted = None, None
fmask = masking.load_features_mask(data, image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
if s_unsorted is not None:
semantic_data = features.SemanticData(
s_unsorted[order],
i_unsorted[order] if i_unsorted is not None else None,
data.segmentation_labels(),
)
else:
semantic_data = None
features_data = features.FeaturesData(p_sorted, f_sorted, c_sorted, semantic_data)
data.save_features(image, features_data)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(
f_sorted, n_closest, data.config["bow_matcher_type"]
)
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args: Tuple[str, DataSetBase]):
image, data = args
log.setup()
need_words = (data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if has_features and has_words:
logger.info("Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image))
return
logger.info("Extracting {} features for image {}".format(
data.feature_type().upper(), image))
start = timer()
image_array = data.load_image(image)
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image,
image_array))
# Load segmentation and bake it in the data
if data.config["features_bake_segmentation"]:
exif = data.load_exif(image)
panoptic_data = [None, None]
for i, p_data in enumerate(
[data.load_segmentation(image),
data.load_instances(image)]):
if p_data is None:
continue
new_height, new_width = p_data.shape
ps = upright.opensfm_to_upright(
p_unmasked[:, :2],
exif["width"],
exif["height"],
exif["orientation"],
new_width=new_width,
new_height=new_height,
).astype(int)
panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]]
s_unsorted, i_unsorted = panoptic_data
p_unsorted = p_unmasked
f_unsorted = f_unmasked
c_unsorted = c_unmasked
# Load segmentation, make a mask from it mask and apply it
else:
s_unsorted, i_unsorted = None, None
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
# pyre-fixme[16]: `None` has no attribute `__getitem__`.
s_sorted = s_unsorted[order] if s_unsorted is not None else None
i_sorted = i_unsorted[order] if i_unsorted is not None else None
data.save_features(image, p_sorted, f_sorted, c_sorted, s_sorted, i_sorted)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config["bow_matcher_type"])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args):
image, tags, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
DEBUG = 0
# check if features already exist
if not data.feature_index_exists(image):
mask = data.mask_as_array(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
preemptive_max = data.config.get('preemptive_max', 200)
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.image_as_array(image), data.config, mask)
if len(p_unsorted) == 0:
return
#===== prune features in tags =====#
if data.config.get('prune_features_on_tags', False):
# setup
img = cv2.imread(os.path.join(data.data_path, 'images', image))
[height, width, _] = img.shape
p_denorm = features.denormalized_image_coordinates(
p_unsorted, width, height)
# expand tag contour with grid points beyond unit square
expn = 2.0
gridpts = np.array(
[[-expn, expn], [expn, expn], [expn, -expn], [-expn, -expn]],
dtype='float32')
# find features to prune
rm_list = []
for tag in tags:
# contour from tag region
contours = np.array(tag.corners)
if DEBUG > 0:
for i in range(0, 3):
cv2.line(img, (tag.corners[i, 0].astype('int'),
tag.corners[i, 1].astype('int')),
(tag.corners[i + 1, 0].astype('int'),
tag.corners[i + 1, 1].astype('int')),
[0, 255, 0], 12)
cv2.line(img, (tag.corners[3, 0].astype('int'),
tag.corners[3, 1].astype('int')),
(tag.corners[0, 0].astype('int'),
tag.corners[0, 1].astype('int')), [0, 255, 0],
12)
# scale contour outward
H = np.array(tag.homography, dtype='float32')
contours_expanded = cv2.perspectiveTransform(
np.array([gridpts]), H)
# for each point
for pidx in range(0, len(p_unsorted)):
# point
pt = p_denorm[pidx, 0:2]
# point in contour
inout = cv2.pointPolygonTest(
contours_expanded.astype('int'), (pt[0], pt[1]), False)
# check result
if inout >= 0:
rm_list.append(pidx)
# prune features
p_unsorted = np.delete(p_unsorted, np.array(rm_list), axis=0)
f_unsorted = np.delete(f_unsorted, np.array(rm_list), axis=0)
c_unsorted = np.delete(c_unsorted, np.array(rm_list), axis=0)
# debug
if DEBUG > 0:
p_denorm = np.delete(p_denorm, np.array(rm_list), axis=0)
for pidx in range(0, len(p_denorm)):
pt = p_denorm[pidx, 0:2]
cv2.circle(img, (pt[0].astype('int'), pt[1].astype('int')),
5, [0, 0, 255], -1)
cv2.namedWindow('ShowImage', cv2.WINDOW_NORMAL)
height, width, channels = img.shape
showw = max(752, width / 4)
showh = max(480, height / 4)
cv2.resizeWindow('ShowImage', showw, showh)
cv2.imshow('ShowImage', img)
cv2.waitKey(0)
#===== prune features in tags =====#
# sort for preemptive
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
# save
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
if data.config.get('matcher_type', 'FLANN') == 'FLANN':
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
#===== tag features =====#
if data.config.get('use_apriltags', False) or data.config.get(
'use_arucotags', False) or data.config.get('use_chromatags',
False):
# setup
try:
tags_all = data.load_tag_detection()
except:
return
tags = tags_all[image]
pt = []
ft = []
ct = []
it = []
imexif = data.load_exif(image)
# for each tag in image
for tag in tags:
# normalize corners
img = cv2.imread(os.path.join(data.data_path, 'images', image))
[height, width, _] = img.shape
#print 'width = ',str(imexif['width'])
#print 'height= ',str(imexif['height'])
#print 'width2= ',str(width)
#print 'heigh2= ',str(height)
norm_tag_corners = features.normalized_image_coordinates(
tag.corners, width,
height) #imexif['width'], imexif['height'])
# for each corner of tag
for r in range(0, 4):
# tag corners
pt.append(norm_tag_corners[r, :])
# tag id
ft.append(tag.id)
# colors
ct.append(tag.colors[r, :])
# corner id (0,1,2,3)
it.append(r)
# if tag features found
if pt:
pt = np.array(pt)
ft = np.array(ft)
ct = np.array(ct)
it = np.array(it)
data.save_tag_features(image, pt, ft, it, ct)
def detect(self, args):
image, data = args
log.setup()
print()
print(data)
self.feature_pdc = {}
need_words = data.config['matcher_type'] == 'WORDS' or data.config[
'matching_bow_neighbors'] > 0
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
start = timer()
#print(image)## 1.jpg
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
data.load_image(image), data.config)
#print(p_unmasked, f_unmasked, c_unmasked)
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :] # points == numpy.ndarray
f_sorted = f_unsorted[order, :] # descriptors
c_sorted = c_unsorted[order, :] # colors
#data.save_features(image, p_sorted, f_sorted, c_sorted)
points = p_sorted.astype(np.float32)
descriptors = f_sorted.astype(np.uint8)
colors = c_sorted
OPENSFM_FEATURES_VERSION = 1
self.feature_pdc.update({'points': points})
self.feature_pdc.update({'descriptors': descriptors})
self.feature_pdc.update({'colors': colors})
self.feature_pdc.update(
{'OPENSFM_FEATURES_VERSION': OPENSFM_FEATURES_VERSION})
# self.feature_pdc.update({'points':p_sorted})
# self.feature_pdc.update({'descriptors':f_sorted})
# self.feature_pdc.update({'colors':c_sorted})
self.feature_of_images.update({image: self.feature_pdc})
print("self.feature_of_images==", hex(id(self.feature_of_images)))
print(image)
# if need_words:
# bows = bow.load_bows(data.config)
# n_closest = data.config['bow_words_to_match']
# closest_words = bows.map_to_words(
# f_sorted, n_closest, data.config['bow_matcher_type'])
# data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report_of_features(image, report)
data.save_report(io.json_dumps(report),
'features/{}.json'.format(image))
def detect(args):
id = current_process()._identity
prefix = "process %s: " % str(id[0]) if len(id) > 0 else ""
image, data = args
if data.features_exist(image):
return
print prefix + "detecting features for image %s" % image
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
# Detect features
# Retrieve image mask
mask = data.mask_as_array(image)
if mask is not None:
print prefix + "found mask for image: %s" % image
logger.info('Found mask to apply for image {}'.format(image))
# Obtain segmentation path
path_seg = data.data_path + "/images/output/results/frontend_vgg/" + os.path.splitext(
image)[0] + '.png'
else:
print prefix + "not found mask for image %s" % image
path_seg = None
preemptive_max = data.config.get('preemptive_max', 200)
the_image = data.image_as_array(image)
# Extract features
print prefix + "extracting features from image %s" % image
save_no_mask = False
all_content = features.extract_features(the_image, data.config, mask,
save_no_mask, path_seg)
if save_no_mask:
p_unsorted, f_unsorted, c_unsorted = all_content[0]
p_nomask, f_nomask, c_nomask = all_content[1]
else:
p_unsorted, f_unsorted, c_unsorted = all_content
if len(p_unsorted) == 0:
print prefix + "exit"
return
# Save features to file
print prefix + "saving features"
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if data.config.get('preemptive_threshold', 0) > 0:
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_preemptive_features(image, p_pre, f_pre)
# Prepare FLANN matching if necessary
if data.config.get('matcher_type', "BRUTEFORCE") == "FLANN":
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
# Done
print prefix + "exit"
def detect(args):
log.setup()
image, data = args
need_words = data.config['matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0
need_flann = data.config['matcher_type'] == 'FLANN'
has_words = not need_words or data.words_exist(image)
has_flann = not need_flann or data.feature_index_exists(image)
has_features = data.features_exist(image)
if has_features and has_flann and has_words:
logger.info('Skip recomputing {} features for image {}'.format(
data.feature_type().upper(), image))
return
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
start = timer()
exif = data.load_exif( image )
camera_models = data.load_camera_models()
image_camera_model = camera_models[ exif[ 'camera' ] ]
if image_camera_model.projection_type in ['equirectangular', 'spherical'] and data.config['matching_unfolded_cube']:
logger.info('Features unfolded cube.')
# For spherical cameras create an undistorted image for the purposes of
# feature finding (and later matching).
max_size = data.config.get('ai_process_size', -1)
if max_size == -1:
max_size = img.shape[1]
img = data.load_image( image )
undist_tile_size = max_size//4
undist_img = np.zeros( (max_size//2, max_size, 3 ), np.uint8 )
undist_mask = np.full( (max_size//2, max_size, 1 ), 255, np.uint8 )
undist_mask[ undist_tile_size:2*undist_tile_size, 2*undist_tile_size:3*undist_tile_size ] = 0
undist_mask[ undist_tile_size:2*undist_tile_size, undist_tile_size:2*undist_tile_size ] = 0
# The bottom mask to remove the influence of the camera person should be configurable. It depends on the forward
# direction of the camera and where the camera person positions themselves in relation to this direction. It'save_feature_index
# probably worth it to take care with this because the floor could help hold the reconstructions together.
#undist_mask[ 5*undist_tile_size//4:7*undist_tile_size//4, undist_tile_size//3:undist_tile_size ] = 0
#undist_mask[ 3*undist_tile_size//2:2*undist_tile_size, undist_tile_size//2:undist_tile_size ] = 0
spherical_shot = types.Shot()
spherical_shot.pose = types.Pose()
spherical_shot.id = image
spherical_shot.camera = image_camera_model
perspective_shots = undistort.perspective_views_of_a_panorama( spherical_shot, undist_tile_size )
for subshot in perspective_shots:
undistorted = undistort.render_perspective_view_of_a_panorama( img, spherical_shot, subshot )
subshot_id_prefix = '{}_perspective_view_'.format( spherical_shot.id )
subshot_name = subshot.id[ len(subshot_id_prefix): ] if subshot.id.startswith( subshot_id_prefix ) else subshot.id
( subshot_name, ext ) = os.path.splitext( subshot_name )
if subshot_name == 'front':
undist_img[ :undist_tile_size, :undist_tile_size ] = undistorted
#print( 'front')
elif subshot_name == 'left':
undist_img[ :undist_tile_size, undist_tile_size:2*undist_tile_size ] = undistorted
#print( 'left')
elif subshot_name == 'back':
undist_img[ :undist_tile_size, 2*undist_tile_size:3*undist_tile_size ] = undistorted
#print( 'back')
elif subshot_name == 'right':
undist_img[ :undist_tile_size, 3*undist_tile_size:4*undist_tile_size ] = undistorted
#print( 'right')
elif subshot_name == 'top':
undist_img[ undist_tile_size:2*undist_tile_size, 3*undist_tile_size:4*undist_tile_size ] = undistorted
#print( 'top')
elif subshot_name == 'bottom':
undist_img[ undist_tile_size:2*undist_tile_size, :undist_tile_size ] = undistorted
#print( 'bottom')
#data.save_undistorted_image(subshot.id, undistorted)
data.save_undistorted_image(image.split(".")[0], undist_img)
# We might consider combining a user supplied mask here as well
undist_img = resized_image(undist_img, data.config)
p_unsorted, f_unsorted, c_unsorted = features.extract_features(undist_img, data.config, undist_mask)
# Visualize the features on the unfolded cube
# --------------------------------------------------------------
if False:
h_ud, w_ud, _ = undist_img.shape
denorm_ud = denormalized_image_coordinates( p_unsorted[:, :2], w_ud, h_ud )
print( p_unsorted.shape )
print( denorm_ud.shape )
rcolors = []
for point in denorm_ud:
color = np.random.randint(0,255,(3)).tolist()
cv2.circle( undist_img, (int(point[0]),int(point[1])), 1, color, -1 )
rcolors.append( color )
data.save_undistorted_image( image + '_unfolded_cube.jpg', undist_img)
# --------------------------------------------------------------
if len(p_unsorted) > 0:
# Mask pixels that are out of valid image bounds before converting to equirectangular image coordinates
bearings = image_camera_model.unfolded_pixel_bearings( p_unsorted[:, :2] )
p_mask = np.array([ point is not None for point in bearings ])
p_unsorted = p_unsorted[ p_mask ]
f_unsorted = f_unsorted[ p_mask ]
c_unsorted = c_unsorted[ p_mask ]
p_unsorted[:, :2] = unfolded_cube_to_equi_normalized_image_coordinates( p_unsorted[:, :2], image_camera_model )
# Visualize the same features converted back to equirectangular image coordinates
# -----------------------------------------------------------------------------------------
if False:
timg = resized_image( img, data.config )
h, w, _ = timg.shape
denorm = denormalized_image_coordinates( p_unsorted[:, :2], w, h )
for ind, point in enumerate( denorm ):
cv2.circle( timg, (int(point[0]),int(point[1])), 1, rcolors[ind], -1 )
data.save_undistorted_image('original.jpg', timg)
#------------------------------------------------------------------------------------------
else:
mask = data.load_combined_mask(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.load_image(image), data.config, mask)
if len(p_unsorted) == 0:
logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_flann:
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config['bow_words_to_match']
closest_words = bows.map_to_words(
f_sorted, n_closest, data.config['bow_matcher_type'])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
