def callback(image):
global benchmarking, benchmark_convert, benchmark_detect, benchmark_threshold
if benchmarking:
convert_start = datetime.now()
image = convert(image.data, input_type='ros', output_type='cv2')
original_image = np.array(image)
if benchmarking:
benchmark_convert.append(millis(datetime.now() - convert_start))
image = threshold(image)
buttons = detect_buttons(image)
button = Button()
button.image.data = convert(image, input_type='cv2', output_type='ros')
image_publisher.publish(button.image)
for button in buttons:
x0 = int(button.x - button.w/2)
y0 = int(button.y - button.h/2)
x1 = int(button.x + button.w/2)
y1 = int(button.y + button.h/2)
thumbnail = original_image[y0:y1,x0:x1]
button.image.data = convert(cv2.resize(thumbnail, (100, 100)), input_type='cv2', output_type='ros')
#image_publisher.publish(button.image)
button_publisher.publish(button)
rospy.loginfo('Detected %d buttons' % len(buttons))
def train():
print 'Button identifier classifier'
clf = Pipeline([
('normalizer', MinMaxScaler()),
('classifier',
OneVsRestClassifier(LogisticRegression(penalty='l2', C=1e-1)))
] # LogisticRegression(penalty='l2', C=1e-4)
)
X = []
y = []
print 'Getting data'
n = 0
for button in progressor(db_buttons.find({})):
#x = np.array( button['image'], dtype=np.uint8 )
#img = cv2.imdecode(x, 1)
img = convert(button['image'], input_type='ros', output_type='cv2')
vector = get_feature_vector(img)
#display_button(img, button['label'], '')
#key = cv2.waitKey() % 256
#if(key == ord('s')):
# n += 1
# cv2.imwrite('/home/rolf/Desktop/%d.jpg' % n, img)
#print vector
X.append(vector)
y.append(process_label(button['label']))
train_save_test(clf, X, y, 'ovr_lr.pkl', labels)
print 'Button on/off classifier'
X = []
y = []
print 'Getting data'
for button in progressor(db_buttons.find()):
#x = np.array( button['img'], dtype=np.uint8 )
#img = cv2.imdecode(x, 1)
img = convert(button['image'], input_type='ros', output_type='cv2')
vector = get_onoff_feature_vector(img)
#print button.get('on')
#vector = get_onoff_feature_vector(img)
#print sum(vector)
#print len(vector)
X.append(vector)
y.append(1 if button.get('on') == True else 0)
clf = Pipeline([('normalizer', MinMaxScaler()),
('classifier',
LogisticRegression(penalty='l2',
C=1e1,
class_weight='auto'))
] #LogisticRegression(penalty='l2', C=1e0))]
)
train_save_test(clf, X, y, 'onoff_lr.pkl', [0, 1])
def train():
print 'Button identifier classifier'
clf = Pipeline([
('normalizer', MinMaxScaler()),
('classifier', OneVsRestClassifier(LogisticRegression(penalty='l2', C=1e-1)))] # LogisticRegression(penalty='l2', C=1e-4)
)
X = []
y = []
print 'Getting data'
n = 0
for button in progressor(db_buttons.find({})):
#x = np.array( button['image'], dtype=np.uint8 )
#img = cv2.imdecode(x, 1)
img = convert(button['image'], input_type='ros', output_type='cv2')
vector = get_feature_vector(img)
#display_button(img, button['label'], '')
#key = cv2.waitKey() % 256
#if(key == ord('s')):
# n += 1
# cv2.imwrite('/home/rolf/Desktop/%d.jpg' % n, img)
#print vector
X.append(vector)
y.append(process_label(button['label']))
train_save_test(clf, X, y, 'ovr_lr.pkl', labels)
print 'Button on/off classifier'
X = []
y = []
print 'Getting data'
for button in progressor(db_buttons.find()):
#x = np.array( button['img'], dtype=np.uint8 )
#img = cv2.imdecode(x, 1)
img = convert(button['image'], input_type='ros', output_type='cv2')
vector = get_onoff_feature_vector(img)
#print button.get('on')
#vector = get_onoff_feature_vector(img)
#print sum(vector)
#print len(vector)
X.append(vector)
y.append(1 if button.get('on') == True else 0)
clf = Pipeline([
('normalizer', MinMaxScaler()),
('classifier', LogisticRegression(penalty='l2', C=1e1, class_weight='auto'))]#LogisticRegression(penalty='l2', C=1e0))]
)
train_save_test(clf, X, y, 'onoff_lr.pkl', [0,1])
def callback(button):
try:
img = convert(button.image.data, input_type='ros', output_type='cv2')
vector = get_feature_vector(img)
label = clf.predict([vector])[0]
try:
p = np.max(clf.predict_proba([vector])[0])
except:
p = 1.0
#rospy.loginfo('%s: %f' % (label, p))
assign_message_label(button, label)
if button.button_type != button.BUTTON_NONE:
rospy.loginfo('Uncertain found button %s (%.2f%%) (C:%.2f)' %
(label, p, certainties[button.button_type]))
#display_button(img, label, '%f' % p)
for k, v in certainties.iteritems():
certainties[k] = max(0, certainties[k] - 1)
if button.button_type != button.BUTTON_NONE and p > 0.7:
certainties[button.button_type] = min(10, certainties[k] + 3 * p)
if certainties[button.button_type] > 2.7:
#rospy.loginfo('%s - %.3f' % (label, p))
rospy.loginfo('Found button %s' % label)
on = onoff_clf.predict(get_onoff_feature_vector(img))
button.on = True if on else False
label_str = '%s (%.0f%%)' % (label, 100.0 * p)
onoff_str = 'on' if on else 'off'
button_publisher.publish(button)
except Exception as e:
rospy.logwarn(e)
def callback(button):
try:
img = convert(button.image.data, input_type='ros', output_type='cv2')
vector = get_feature_vector(img)
label = clf.predict([vector])[0]
try:
p = np.max(clf.predict_proba([vector])[0])
except:
p = 1.0
#rospy.loginfo('%s: %f' % (label, p))
assign_message_label(button, label)
if button.button_type != button.BUTTON_NONE:
rospy.loginfo('Uncertain found button %s (%.2f%%) (C:%.2f)' % (label, p, certainties[button.button_type]))
#display_button(img, label, '%f' % p)
for k, v in certainties.iteritems():
certainties[k] = max(0, certainties[k] - 1)
if button.button_type != button.BUTTON_NONE and p > 0.7:
certainties[button.button_type] = min(10, certainties[k] + 3 * p)
if certainties[button.button_type] > 2.7:
#rospy.loginfo('%s - %.3f' % (label, p))
rospy.loginfo('Found button %s' % label)
on = onoff_clf.predict(get_onoff_feature_vector(img))
button.on = True if on else False
label_str = '%s (%.0f%%)' % (label, 100.0*p)
onoff_str = 'on' if on else 'off'
button_publisher.publish(button)
except Exception as e:
rospy.logwarn(e)
def test():
"""
Runs the test set with all possible detectors, descriptors and matchers,
"""
# Get the query images
global query_images
query_images = get_query_images()
for key, image in query_images.iteritems():
query_images[key][ 0: 18, 0:100] = [255, 255, 255]
query_images[key][82:100, 0:100] = [255, 255, 255]
query_images[key][ 0:100, 0: 18] = [255, 255, 255]
query_images[key][ 0:100, 82:100] = [255, 255, 255]
# Get the data test set
X = []
y = []
labels = []
for button in progressor(buttons.find(snapshot=True)):
try:
image = preprocess(convert(button['image'], input_type='ros', output_type='cv2'))
label = button['label'].upper() if button['label'] is not None else 'NONE'
if label not in labels:
labels.append(label)
X.append(image)
y.append(label)
except:
print 'Skipping sample'
# Set up which detectors, descriptors and matchers to use for testing
detector_formats = [""]#,"Grid","Pyramid"]
detector_types = ["HARRIS", "SIFT", "SURF", "ORB", "MSER", "GFTT"]
descriptor_types = ["SIFT", "SURF", "ORB"]
matcher_types = ["BruteForce", "FlannBased"]
# Test all combinations
for detector_format in detector_formats:
for detector_type in detector_types:
for descriptor_type in descriptor_types:
for matcher_type in matcher_types:
if descriptor_type == 'ORB' and detector_type == 'SIFT':
continue
test_technique(X, y, labels, detector_name=detector_format + detector_type, descriptor_name=descriptor_type,
matcher_name=matcher_type)
def test():
"""
Tests the button detection algorithm on our dataset
"""
db = MongoClient()
fotos = db['aidu']['elevator_fotos']
for idx, foto in enumerate(fotos.find()):
#if idx not in [143, 196, 206, 524]:
# continue
rospy.loginfo("Processing foto %d" % idx)
convert_start = datetime.now()
image = convert(foto['image'], input_type='ros', output_type='cv2')
benchmark_convert.append(millis(datetime.now() - convert_start))
original_image = np.array(image)
# Threshold
image = threshold(image)
threshold_image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
# Detect buttons
buttons = detect_buttons(image)
rospy.loginfo('Foto %d - convert: %fms - threshold: %fms - detect: %fms' % (idx, benchmark_convert[-1],
benchmark_threshold[-1],
benchmark_detect[-1]))
rospy.loginfo('Bytes: %d' % len(foto['image']))
rospy.loginfo('Detected %d buttons' % len(buttons))
# Show button
for button in buttons:
x0 = int(button.x - button.w/2)
y0 = int(button.y - button.h/2)
x1 = int(button.x + button.w/2)
y1 = int(button.y + button.h/2)
cv2.rectangle(original_image, (x0, y0), (x1, y1), (0, 0, 255), 2)
cv2.rectangle(threshold_image, (x0, y0), (x1, y1), (0, 0, 255), 2)
out = np.vstack((original_image, threshold_image))
cv2.imshow('Button detector', cv2.resize(out, (640, 720)))
cv2.waitKey()
if rospy.is_shutdown():
break