def recognize(self):
items = self.tableview_selected_items()
for tableview_item in items:
area_range = (self.tool_bar.ui.horizontalSlider.value(),
self.tool_bar.ui.horizontalSlider_2.value())
dilate_size = (self.tool_bar.ui.horizontalSlider_3.value(),
self.tool_bar.ui.horizontalSlider_3.value())
image_process = ImageProcess(tableview_item.data('qpixmap'))
edge, rects, crops = image_process.recognize_table(
area_range, dilate_size)
tableview_item.data('rects_index', 0)
tableview_item.data('edge', edge)
tableview_item.data('rects', rects)
tableview_item.data('crops', crops)
children = self.model.root().children()
setting_names = [
c.data('Settings') for c in children
if not c.data('Settings') is None
]
setting_names = [
s for s in setting_names if not s in setting_names
]
if len(items) > 0:
self.ui.image_view.update_rows(rects)
self.graphics_view_update()
def portfolio(self, image_list, horizontal):
random.shuffle(image_list)
list1 = image_list[:4]
count = 1
bg = Image.new('RGB', (1200, 1200), (255, 255, 255))
horizontal = horizontal.resize((1200, 600), Image.ANTIALIAS)
bg.paste(horizontal, (0, 0))
for image in list1:
if count == 1:
image = image.resize((400, 600), Image.ANTIALIAS)
bg.paste(image, (0, 600))
if count == 2:
image = image.resize((400, 600), Image.ANTIALIAS)
bg.paste(image, (400, 600))
if count == 3:
image = image.resize((400, 600), Image.ANTIALIAS)
bg.paste(image, (800, 600))
count += 1
timestr = ImageProcess.timestamp(self)
filename = 'collage' + str(timestr) + '.jpg'
filename = os.path.join(self.output, filename)
if self.text:
bg = ImageProcess.draw_text(self, bg, self.text, 30, 'bottom left')
bg = ImageProcess.put_logo(self, bg, 'HauteBook', 30, 'bottom right')
bg.save(filename, quality=90, optimize=True)
def __init__(self, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT # use it for csv file name
self.csv_path = csv_path
self.train_generator = None
self.valid_generator = None
self.train_hist = None
self.drive = None
#self.config = Config() #model_name)
self.data_path = data_path
#self.model_name = model_name
self.drive = DriveData(self.csv_path)
self.net_model = NetModel(data_path)
self.image_process = ImageProcess()
self.data_aug = DataAugmentation()
def __init__(self, model_path, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT
self.data_path = data_path
self.data = DriveData(csv_path)
self.test_generator = None
self.num_test_samples = 0
#self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.model_path = model_path
self.image_process = ImageProcess()
self.measurements = []
self.predictions = []
self.differences = []
self.squared_differences = []
def main():
if (len(sys.argv) != 2):
print('Give the model path.')
return
drive = DriveRun(sys.argv[1])
config = Config()
csv_fname = '/home/mir-alb/Ninad_Thesis/Test/Test.csv'
csv_header = ['image_fname', 'steering_angle']
df = pd.read_csv(csv_fname, names=csv_header, index_col=False)
num_data = len(df)
text = open('/home/mir-lab/Ninad_Thesis/Test/Salient/Salient.txt', 'w+')
bar = ProgressBar()
image_process = ImageProcess()
for i in bar(range(num_data)):
image_name = df.loc[i]['image_fname']
steering = df.loc[i]['steering_angle']
image_path = '/home/mir-lab/Ninad_Thesis/Test/' + image_name + '.jpg'
image = utils.load_img(image_path, target_size=(config.image_size[1],
config.image_size[0]))
image = image_process.process(image)
prediction = drive.run(image)
text.write(str(image_name) + '\t' + str(steering) + '\t' + str(prediction))
modifiers = [None, 'negate', 'small_values']
for i, modifier in enumerate(modifiers):
heatmap = visualize_saliency(drive.net_model.model, layer_idx=-1,
filter_indices=0, seed_input=image,
grad_modifier=modifier, keepdims=True)
final = overlay(image, heatmap, alpha=0.5)
cv2.imwrite('/home/mir-lab/Ninad_Thesis/Test/Salient/' + image_name + '_' + str(i) + '.jpg', final)
def main():
config = Config()
image_process = ImageProcess()
try:
if (len(sys.argv) != 3):
print('Use model_path image_file_name.')
return
image = cv2.imread(sys.argv[2])
image = cv2.resize(image, (config.image_size[0],
config.image_size[1]))
image = image_process.process(image)
if (len(image) == 0):
print('File open error: ', sys.argv[2])
return
drive_run = DriveRun(sys.argv[1])
measurments = drive_run.run(image)
print(measurments)
except KeyboardInterrupt:
print ('\nShutdown requested. Exiting...')
def __init__(self):
rospy.init_node('BMW_controller')
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.rate = rospy.Rate(10)
self.drive= DriveRun(sys.argv[1])
rospy.Subscriber('/image_topic_2', Image, self.controller_cb)
self.image = None
self.image_processed = False
def __init__(self):
rospy.init_node('controller')
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.rate = rospy.Rate(10)
self.drive= DriveRun(sys.argv[1])
rospy.Subscriber('/bulldogbolt/camera_left/image_raw_left', Image, self.controller_cb)
self.image = None
self.image_processed = False
def predict_from_camera(drive):
check, frame = video.read()
cropImg = frame[yMin:yMax,xMin:xMax]
newimg = cv2.resize(frame,(160,70))
image_process = ImageProcess() #normalize the image
IImage = cv2.cvtColor(newimg, cv2.COLOR_RGB2BGR)
IImage = image_process.process(IImage)
prediction= drive.run(IImage)
print(prediction)
steering_commands(prediction)
def __init__(self, model_path):
self.model = None
self.num_test_samples = 0
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
def __init__(self):
rospy.init_node('controller')
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.rate = rospy.Rate(30)
self.drive = DriveRun(sys.argv[1])
rospy.Subscriber('/bolt/front_camera/image_raw', Image,
self.controller_cb)
self.image = None
self.image_processed = False
self.config = Config()
def __init__(self, model_path):
model_path = '/home/yadav/lidar_network/balu/lidar_only'
self.test_generator = None
self.data_path = None
self.num_test_samples = 0
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
def __init__(self, weight_file_name):
rospy.init_node('run_neural')
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.rate = rospy.Rate(30)
self.drive = DriveRun(weight_file_name)
rospy.Subscriber(Config.data_collection['camera_image_topic'], Image,
self._controller_cb)
self.image = None
self.image_processed = False
#self.config = Config()
self.braking = False
def black_magic(self, image_list):
random.shuffle(image_list)
list1 = image_list[:4]
count = 1
bg = Image.new('RGB', (735, 1055), (255, 255, 255))
for image in list1:
if count == 1:
image = image.resize((400, 600), Image.ANTIALIAS)
bg.paste(image, (5, 5))
if count == 2:
image = image.resize((400, 600), Image.ANTIALIAS)
bg.paste(image, (330, 450))
if count == 3:
image = image.resize((300, 440), Image.ANTIALIAS)
bg.paste(image, (420, 5))
else:
image = image.resize((300, 440), Image.ANTIALIAS)
bg.paste(image, (15, 610))
count += 1
timestr = ImageProcess.timestamp(self)
filename = 'collage' + str(timestr) + '.jpg'
filename = os.path.join(self.output, filename)
if self.text:
bg = ImageProcess.draw_text(self, bg, self.text, 30, 'bottom left')
# bg = ImageProcess.put_logo(self, bg, 'HauteBook', 30, 'bottom right')
bg.save(filename, quality=90, optimize=True)
random.shuffle(image_list)
list1 = image_list[:4]
count = 1
bg = Image.new('RGB', (735, 1055), (255, 255, 255))
for image in list1:
if count == 1:
image = image.resize((400, 600), Image.ANTIALIAS)
bg.paste(image, (330, 5))
if count == 2:
image = image.resize((400, 600), Image.ANTIALIAS)
bg.paste(image, (5, 450))
if count == 3:
image = image.resize((300, 440), Image.ANTIALIAS)
bg.paste(image, (420, 610))
else:
image = image.resize((300, 440), Image.ANTIALIAS)
bg.paste(image, (15, 5))
count += 1
timestr = ImageProcess.timestamp(self)
filename = 'collage' + str(timestr) + '.jpg'
filename = os.path.join(self.output, filename)
if self.text:
bg = ImageProcess.draw_text(self, bg, self.text, 30, 'bottom left')
# bg = ImageProcess.put_logo(self, bg, 'HauteBook', 30, 'bottom right')
bg.save(filename, quality=90, optimize=True)
class NeuralControl:
def __init__(self):
rospy.init_node('controller')
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.rate = rospy.Rate(30)
self.drive = DriveRun(sys.argv[1])
rospy.Subscriber('/bolt/front_camera/image_raw', Image,
self.controller_cb)
self.image = None
self.image_processed = False
self.config = Config()
def controller_cb(self, image):
img = self.ic.imgmsg_to_opencv(image)
cropImg = img[self.config.capture_area[1]:self.config.capture_area[3],
self.config.capture_area[0]:self.config.capture_area[2]]
#cropImg = img[yMin:yMax,xMin:xMax]
img = cv2.resize(
cropImg, (self.config.image_size[0], self.config.image_size[1]))
self.image = self.image_process.process(img)
if self.config.typeofModel == 4 or self.config.typeofModel == 5:
self.image = np.array(self.image).reshape(
1, self.config.image_size[1], self.config.image_size[0],
self.config.image_size[2])
self.image_processed = True
def __init__(self):
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.driveC = DriveRun(sys.argv[2])
rospy.Subscriber('/usb_cam/image_raw', Image, self.recorder1)
self.imageC = None
self.camera_processed = False
class NeuralControl:
def __init__(self):
rospy.init_node('BMW_controller')
self.rate = rospy.Rate(10)
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.drive = DriveRun(sys.argv[1])
rospy.Subscriber('/image_topic_2', Image, self.controller_cb)
rospy.Subscriber('/usb_cam/image_raw', Image, self.recorder1)
self.image = None
self.image_processed = False
def recorder1(self, image):
cam_img = self.ic.imgmsg_to_opencv(image)
cropImg = cam_img[yMin:yMax, xMin:xMax]
global newimg
newimg = cv2.resize(cropImg, (160, 50))
def controller_cb(self, image):
img = self.ic.imgmsg_to_opencv(image)
img_resize = cv2.resize(img, (160, 50))
stacked_img = np.concatenate((img_resize, newimg), axis=0)
self.image = self.image_process.process(stacked_img)
self.image_processed = True
def __init__(self, path, mode):
str_list = path.split('/')
self.kind = str_list[1]
self.name = str_list[2]
self.mode = copy.deepcopy(mode)
self.image = cv2.imread(path)
self.image_proc_s = ImageProcess(
cv2.medianBlur(self.image,5), 70
)
self.image=cv2.bilateralFilter(self.image,5,50,50)
self.image_proc_m = ImageProcess(self.image, 100)
self.image_proc_l = ImageProcess(self.image, 130)
self.client = MongoClient()
self.db = self.client.object_finder
def init_image_proc_l(self):
self.image_proc_l = ImageProcess(
cv2.bilateralFilter(self.image, 5, 50, 50), 130
)
self.image_proc_l.set_classify_target_list(
self.clf_shape.predict(
self.image_proc_l.get_classify_vec_list()
)
)
def main(model_path, input):
image_file_name = input[0]
if Config.neural_net['num_inputs'] == 2:
velocity = input[1]
image = cv2.imread(image_file_name)
image_process = ImageProcess()
# show the image
fig, (ax1, ax2) = plt.subplots(1, 2)
ax1.imshow(image)
ax1.set(title='original image')
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image, (Config.neural_net['input_image_width'],
Config.neural_net['input_image_height']))
image = image_process.process(image)
drive_run = DriveRun(model_path)
if Config.neural_net['num_inputs'] == 2:
predict = drive_run.run((image, velocity))
steering_angle = predict[0][0]
throttle = predict[0][1]
fig.suptitle('pred-steering:{} pred-throttle:{}'.format(
steering_angle, throttle))
else:
predict = drive_run.run((image, ))
steering_angle = predict[0][0]
fig.suptitle('pred_steering:{}'.format(steering_angle))
ax2.imshow(image)
ax2.set(title='resize and processed')
plt.show()
def __init__(self, data_path):
model_name = data_path[data_path.rfind('/'):] # get folder name
model_name = model_name.strip('/')
csv_path = data_path + '/' + model_name + '.csv' # use it for csv file name
self.csv_path = csv_path
self.train_generator = None
self.valid_generator = None
self.train_hist = None
self.drive = None
self.config = Config() #model_name)
self.data_path = data_path
#self.model_name = model_name
self.drive = DriveData(self.csv_path)
self.net_model = NetModel(data_path)
self.image_process = ImageProcess()
def recognize(self):
mainwindow = self.mainwindow(self)
if mainwindow is None:
return
tableview_item = mainwindow.tableview_selected_item()
if tableview_item is None:
return
area_range = ( self.ui.horizontalSlider.value(), self.ui.horizontalSlider_2.value() )
dilate_size = ( self.ui.horizontalSlider_3.value(), self.ui.horizontalSlider_3.value() )
image_process = ImageProcess( tableview_item.data('qpixmap') )
edge, rects, crops = image_process.recognize_table(area_range, dilate_size)
tableview_item.data('rects_index', 0)
tableview_item.data('edge', edge)
tableview_item.data('rects', rects)
tableview_item.data('crops', crops)
self.update_rows()
self.graphics_view_update()
def start(self, app_config):
'''
Starts one or more processes to process images, monitors the configured folder paths to identify images to be
processed and add them to the work queue.
'''
active_task_bound = app_config['process_pool_size'] * 2
task_queue = Queue()
manager = Manager()
active_task_dict = manager.dict()
semaphore = manager.Semaphore(active_task_bound)
process_pool_size = app_config['process_pool_size']
for i in range(process_pool_size):
process=ImageProcess(task_queue, active_task_dict, semaphore, app_config['delete_after_processed'])
process.daemon = True
processes.append(process)
process.start()
log.info('Launched {} processes'.format(process_pool_size))
while True:
for path_item in app_config['path_items']:
os.chdir(path_item['local_path'])
for glob_pattern in path_item['glob_patterns']:
for file_name in glob.glob(glob_pattern):
semaphore.acquire()
file_path = os.path.join(path_item['local_path'], file_name)
if file_path not in active_task_dict:
active_task_dict[file_path] = True
task_queue.put((file_path, path_item['process_tasks']))
log.debug('Added item to active_task_dict, length: {}'.format(len(active_task_dict)))
time.sleep(PAUSE_INTERVAL)
def __init__(self, model_path, data_path, target_path):
# remove the last '/' in data and target path if exists
if data_path[-1] == '/':
data_path = data_path[:-1]
if target_path[-1] == '/':
target_path = target_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
data_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
data_name = data_path
csv_path = data_path + '/' + data_name + const.DATA_EXT
self.data_name = data_name
self.data_path = data_path
self.target_path = target_path + '/' + data_name + '/'
if os.path.isdir(target_path) is False:
os.mkdir(target_path)
if os.path.isdir(self.target_path) is False:
os.mkdir(self.target_path)
self.drive_data = DriveData(csv_path)
self.drive_data.read(normalize=False)
self.data_len = len(self.drive_data.image_names)
self.net_model = None
self.model_path = None
if model_path is not None:
from net_model import NetModel
self.net_model = NetModel(model_path)
self.net_model.load()
self.model_path = model_path
self.image_process = ImageProcess()
self.display = DisplaySettings()
def init_image_proc_s(self):
self.image_proc_s = ImageProcess(
cv2.medianBlur(self.image, 5), 70
)
self.image_proc_s.set_classify_target_list(
self.clf_fore.predict(
self.image_proc_s.get_classify_vec_list()
)
)
self.image_proc_s.image = cv2.bilateralFilter(self.image, 5, 50, 50)
self.image_proc_s.compute_foreground_mask()
self.image_proc_s.compute_foreground_image()
self.image_proc_s.compute_color_list()
self.image_proc_s.compute_sift_list()
def taker(data):
joy_pub = rospy.Publisher('/joy', Joy, queue_size=10)
joy_data = Joy()
img = ic.imgmsg_to_opencv(data)
config = Config()
image_process = ImageProcess()
IImage = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
IImage = cv2.resize(IImage, (config.image_size[0], config.image_size[1]))
IImage = image_process.process(IImage)
#cv2.imwrite('balu.jpg', IImage)
#drive_run = DriveRun(sys.agrv[1])
drive_run = DriveRun(
'/home/mir-lab/Desktop/Balu_Autodrive/2018-06-14-15-16-03')
prediction = drive_run.run(IImage)
#print(prediction)
#print(np.shape(prediction))
#print(type(prediction))
if (prediction[0][0] > 0.3):
#print("1")
prediction[0][0] = 1
elif (prediction[0][0] < -0.3):
#print("2")
prediction[0][0] = -1
else:
#print("3")
prediction[0][0] = 0
#new_pred = prediction.astype(np.float)
#new_predd = np.asscalar(np.array([prediction]))
#print(type(new_predd))
#print(np.shape(new_predd))
#new_predd = joy_data.axes.append(0)
#0.25 = joy_data.axes.append(3)
#print(new_predd)
#print(prediction[0][0])
joy_data.axes = [prediction[0][0], 0, 0, 0.05, 0, 0]
joy_pub.publish(joy_data)
class NeuralControl:
def __init__(self):
rospy.init_node('BMW_controller')
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.rate = rospy.Rate(10)
self.drive= DriveRun(sys.argv[1])
rospy.Subscriber('/image_topic_2', Image, self.controller_cb)
self.image = None
self.image_processed = False
def controller_cb(self, image):
img = self.ic.imgmsg_to_opencv(image)
img = cv2.resize(img,(160,70))
self.image = self.image_process.process(img)
self.image_processed = True
class NeuralControl:
def __init__(self):
rospy.init_node('controller')
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.rate = rospy.Rate(10)
self.drive= DriveRun(sys.argv[1])
rospy.Subscriber('/bulldogbolt/camera_left/image_raw_left', Image, self.controller_cb)
self.image = None
self.image_processed = False
def controller_cb(self, image):
img = self.ic.imgmsg_to_opencv(image)
cropImg = img[yMin:yMax,xMin:xMax]
img = cv2.resize(cropImg,(160,70))
self.image = self.image_process.process(img)
self.image_processed = True
class NeuralControl:
def __init__(self, weight_file_name):
rospy.init_node('run_neural')
self.ic = ImageConverter()
self.image_process = ImageProcess()
self.rate = rospy.Rate(30)
self.drive = DriveRun(weight_file_name)
rospy.Subscriber(Config.data_collection['camera_image_topic'], Image,
self._controller_cb)
self.image = None
self.image_processed = False
#self.config = Config()
self.braking = False
def _controller_cb(self, image):
img = self.ic.imgmsg_to_opencv(image)
cropped = img[Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
img = cv2.resize(
cropped,
(config['input_image_width'], config['input_image_height']))
self.image = self.image_process.process(img)
## this is for CNN-LSTM net models
if config['lstm'] is True:
self.image = np.array(self.image).reshape(
1, config['input_image_height'], config['input_image_width'],
config['input_image_depth'])
self.image_processed = True
def _timer_cb(self):
self.braking = False
def apply_brake(self):
self.braking = True
timer = threading.Timer(Config.run_neural['brake_apply_sec'],
self._timer_cb)
timer.start()
class Label:
def __init__(self, path, mode):
str_list = path.split('/')
self.kind = str_list[1]
self.name = str_list[2]
self.mode = copy.deepcopy(mode)
self.image = cv2.imread(path)
self.image_proc_s = ImageProcess(
cv2.medianBlur(self.image,5), 70
)
self.image=cv2.bilateralFilter(self.image,5,50,50)
self.image_proc_m = ImageProcess(self.image, 100)
self.image_proc_l = ImageProcess(self.image, 130)
self.client = MongoClient()
self.db = self.client.object_finder
def label_fore(self):
self.image_proc_s.label_classify_target_list(True)
coll = self.db[self.mode + '_fore']
vec_list = self.image_proc_s.get_classify_vec_list()
target_list = self.image_proc_s.get_classify_target_list()
for index in range(len(vec_list)):
coll.insert({
'vec': vec_list[index],
'target': target_list[index],
'kind': self.kind,
'name': self.name
})
def label_shape(self):
self.image_proc_m.label_classify_target_list(True)
self.image_proc_l.label_classify_target_list(True)
coll = self.db[self.mode + '_shape']
vec_list = \
self.image_proc_m.get_classify_vec_list() + self.image_proc_l.get_classify_vec_list()
target_list = \
self.image_proc_m.get_classify_target_list() + self.image_proc_l.get_classify_target_list()
for index in range(len(vec_list)):
coll.insert({
'vec': vec_list[index],
'target': target_list[index],
'kind': self.kind,
'name': self.name
})
class DriveView:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
# target_path = path/to/save/view e.g. ../target/
#
def __init__(self, model_path, data_path, target_path):
# remove the last '/' in data and target path if exists
if data_path[-1] == '/':
data_path = data_path[:-1]
if target_path[-1] == '/':
target_path = target_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
data_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
data_name = data_path
csv_path = data_path + '/' + data_name + const.DATA_EXT
self.data_name = data_name
self.data_path = data_path
self.target_path = target_path + '/' + data_name + '/'
if os.path.isdir(target_path) is False:
os.mkdir(target_path)
if os.path.isdir(self.target_path) is False:
os.mkdir(self.target_path)
self.drive_data = DriveData(csv_path)
self.drive_data.read(normalize=False)
self.data_len = len(self.drive_data.image_names)
self.net_model = None
self.model_path = None
if model_path is not None:
from net_model import NetModel
self.net_model = NetModel(model_path)
self.net_model.load()
self.model_path = model_path
self.image_process = ImageProcess()
self.display = DisplaySettings()
def _print_info(self, i, draw, input_image, steering_angle, degree_angle):
if self.net_model is not None and Config.neural_net['lstm'] is True:
images = []
lstm_time_step = 1
########################
# inference included
if self.net_model is not None:
# convert PIL image to numpy array
image = np.asarray(input_image)
# don't forget OSCAR's default color space is BGR (cv2's default)
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image,
(Config.neural_net['input_image_width'],
Config.neural_net['input_image_height']))
image = self.image_process.process(image)
######################
# infer using neural net
if Config.neural_net['lstm'] is True:
images.append(image)
if lstm_time_step >= Config.neural_net['lstm_timestep']:
trans_image = np.array(images).reshape(
-1, Config.neural_net['lstm_timestep'],
Config.neural_net['input_image_height'],
Config.neural_net['input_image_width'],
Config.neural_net['input_image_depth'])
predict = self.net_model.model.predict(trans_image)
pred_steering_angle = predict[0][0]
pred_steering_angle = pred_steering_angle / Config.neural_net[
'steering_angle_scale']
del images[0]
lstm_time_step += 1
else: # not lstm -- normal cnn
npimg = np.expand_dims(image, axis=0)
predict = self.net_model.model.predict(npimg)
pred_steering_angle = predict[0][0]
pred_steering_angle = pred_steering_angle / Config.neural_net[
'steering_angle_scale']
#####################
# display
degree_angle = pred_steering_angle * Config.data_collection[
'steering_angle_max']
rotated_img = self.display.infer_wheel.image.rotate(degree_angle)
input_image.paste(rotated_img, self.display.infer_wheel.image_pos,
rotated_img)
draw.text(self.display.infer_wheel.label_pos,
"Angle: {:.2f}".format(degree_angle),
font=self.display.font,
fill=self.display.font_color)
if self.net_model is not None:
diff = abs(pred_steering_angle -
self.drive_data.measurements[i][0])
if Config.data_collection['brake'] is True:
draw.multiline_text(
self.display.info_pos,
"Input: {}\nThrottle: {}\nBrake: {}\nSteering: {}\nPredicted: {}\nAbs Diff: {}\nVelocity: {:.2f}\nPosition: (x:{:.2f}, y:{:.2f}, z:{:.2f})"
.format(
self.drive_data.image_names[i],
self.drive_data.measurements[i][1],
self.drive_data.measurements[i][2],
# steering angle: -1 to 1 scale
self.drive_data.measurements[i][0],
pred_steering_angle,
diff,
self.drive_data.velocities[i],
self.drive_data.positions_xyz[i][0],
self.drive_data.positions_xyz[i][1],
self.drive_data.positions_xyz[i][2]),
font=self.display.font,
fill=self.display.font_color)
else:
draw.multiline_text(
self.display.info_pos,
"Input: {}\nThrottle: {}\nSteering: {}\nPredicted: {}\nAbs Diff: {}\nVelocity: {:.2f}\nPosition: (x:{:.2f}, y:{:.2f}, z:{:.2f})"
.format(
self.drive_data.image_names[i],
self.drive_data.measurements[i][1],
# steering angle: -1 to 1 scale
self.drive_data.measurements[i][0],
pred_steering_angle,
diff,
self.drive_data.velocities[i],
self.drive_data.positions_xyz[i][0],
self.drive_data.positions_xyz[i][1],
self.drive_data.positions_xyz[i][2]),
font=self.display.font,
fill=self.display.font_color)
loc_dot = self.drive_data.image_names[i].rfind('.')
target_img_name = "{}_{:.2f}_{:.2f}{}".format(
self.drive_data.image_names[i][:loc_dot], pred_steering_angle,
degree_angle, const.IMAGE_EXT)
else:
if Config.data_collection['brake'] is True:
draw.multiline_text(
self.display.info_pos,
"Input: {}\nThrottle: {}\nBrake: {}\nSteering: {}\nVelocity: {:.2f}\nPosition: (x:{:.2f}, y:{:.2f}, z:{:.2f})"
.format(
self.drive_data.image_names[i],
self.drive_data.measurements[i][1],
self.drive_data.measurements[i][2],
# steering angle: -1 to 1 scale
self.drive_data.measurements[i][0],
self.drive_data.velocities[i],
self.drive_data.positions_xyz[i][0],
self.drive_data.positions_xyz[i][1],
self.drive_data.positions_xyz[i][2]),
font=self.display.font,
fill=self.display.font_color)
else:
draw.multiline_text(
self.display.info_pos,
"Input: {}\nThrottle: {}\nSteering: {}\nVelocity: {:.2f}\nPosition: (x:{:.2f}, y:{:.2f}, z:{:.2f})"
.format(
self.drive_data.image_names[i],
self.drive_data.measurements[i][1],
# steering angle: -1 to 1 scale
self.drive_data.measurements[i][0],
self.drive_data.velocities[i],
self.drive_data.positions_xyz[i][0],
self.drive_data.positions_xyz[i][1],
self.drive_data.positions_xyz[i][2]),
font=self.display.font,
fill=self.display.font_color)
loc_dot = self.drive_data.image_names[i].rfind('.')
target_img_name = "{}_{:.2f}_{:.2f}{}".format(
self.drive_data.image_names[i][:loc_dot], steering_angle,
degree_angle, const.IMAGE_EXT)
# save it
input_image.save(self.target_path + target_img_name)
###########################################################################
#
def run(self):
bar = ProgressBar()
############################
# steering angle raw value:
# -1 to 1 (0 --> 1: left, 0 --> -1: right)
for i in bar(range(self.data_len)):
abs_path_image = self.data_path + '/' + self.drive_data.image_names[
i]
input_image = Image.open(abs_path_image)
steering_angle = self.drive_data.measurements[i][
0] # -1 to 1 scale
degree_angle = steering_angle * Config.data_collection[
'steering_angle_max']
rotated_img = self.display.label_wheel.image.rotate(degree_angle)
input_image.paste(rotated_img, self.display.label_wheel.image_pos,
rotated_img)
# logo
input_image.paste(self.display.logo.image,
self.display.logo.image_pos,
self.display.logo.image)
draw = ImageDraw.Draw(input_image)
draw.text(self.display.label_wheel.label_pos,
"Angle: {:.2f}".format(degree_angle),
font=self.display.font,
fill=self.display.font_color)
self._print_info(i, draw, input_image, steering_angle,
degree_angle)
class DriveTest:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, model_path):
self.test_generator = None
self.data_path = None
self.num_test_samples = 0
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
###########################################################################
#
def _prepare_data(self, data_path):
self.data_path = data_path
folder_name = data_path[data_path.rfind('/'):] # get folder name
folder_name = folder_name.strip('/')
csv_path = data_path + '/' + folder_name + '.csv' # use it for csv file name
self.drive = DriveData(csv_path)
self.drive.read()
self.test_data = list(
zip(self.drive.image_names, self.drive.measurements))
self.num_test_samples = len(self.test_data)
print('\nTest samples: ', self.num_test_samples)
###########################################################################
#
def _prep_generator(self):
if self.data_path == None:
raise NameError('data_path must be set.')
def _generator(samples, batch_size=self.config.batch_size):
num_samples = len(samples)
while True: # Loop forever so the generator never terminates
bar = ProgressBar()
samples = sklearn.utils.shuffle(samples)
for offset in bar(range(0, num_samples, batch_size)):
batch_samples = samples[offset:offset + batch_size]
images = []
measurements = []
for image_name, measurement in batch_samples:
image_path = self.data_path + '/' + image_name + \
self.config.fname_ext
image = cv2.imread(image_path)
image = cv2.resize(image, (self.config.image_size[0],
self.config.image_size[1]))
image = self.image_process.process(image)
images.append(image)
steering_angle, throttle = measurement
#angles.append(float(steering_angle))
#measurements.append(steering_angle)
measurements.append(steering_angle *
self.config.raw_scale)
X_train = np.array(images)
y_train = np.array(measurements)
yield sklearn.utils.shuffle(X_train, y_train)
self.test_generator = _generator(self.test_data)
###########################################################################
#
def _start_test(self):
if (self.test_generator == None):
raise NameError('Generators are not ready.')
print("\nEvaluating the model with test data sets ...")
## Note: Do not use multiprocessing or more than 1 worker.
## This will genereate threading error!!!
score = self.net_model.model.evaluate_generator(
self.test_generator,
self.num_test_samples // self.config.batch_size)
#workers=1)
print("\nLoss: ", score) #[0], "Accuracy: ", score[1])
#print("\nLoss: ", score[0], "rmse: ", score[1])
###########################################################################
#
def test(self, data_path):
self._prepare_data(data_path)
self._prep_generator()
self._start_test()
def main(model_path, image_file_path):
image_process = ImageProcess()
image = cv2.imread(image_file_path)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image, (Config.neural_net['input_image_width'],
Config.neural_net['input_image_height']))
image = image_process.process(image)
drive_run = DriveRun(model_path)
measurement = drive_run.run((image, ))
""" grad modifier doesn't work somehow
fig, axs = plt.subplots(1, 3)
fig.suptitle('Saliency Visualization' + str(measurement))
titles = ['left steering', 'right steering', 'maintain steering']
modifiers = [None, 'negate', 'small_values']
for i, modifier in enumerate(modifiers):
layer_idx = utils.find_layer_idx(drive_run.net_model.model, 'conv2d_last')
heatmap = visualize_cam(drive_run.net_model.model, layer_idx,
filter_indices=None, seed_input=image, backprop_modifier='guided',
grad_modifier=modifier)
axs[i].set(title = titles[i])
axs[i].imshow(image)
axs[i].imshow(heatmap, cmap='jet', alpha=0.3)
"""
plt.figure()
#plt.title('Saliency Visualization' + str(measurement))
plt.title('Steering Angle Prediction: ' + str(measurement[0][0]))
layer_idx = utils.find_layer_idx(drive_run.net_model.model, 'conv2d_last')
heatmap = visualize_cam(drive_run.net_model.model,
layer_idx,
filter_indices=None,
seed_input=image,
backprop_modifier='guided')
plt.imshow(image)
plt.imshow(heatmap, cmap='jet', alpha=0.5)
# file name
loc_slash = image_file_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
image_file_name = image_file_path[loc_slash + 1:]
saliency_file_path = model_path + '_' + image_file_name + '_saliency.png'
saliency_file_path_pdf = model_path + '_' + image_file_name + '_saliency.pdf'
plt.tight_layout()
# save fig
plt.savefig(saliency_file_path, dpi=150)
plt.savefig(saliency_file_path_pdf, dpi=150)
print('Saved ' + saliency_file_path + ' & .pdf')
class ObjectProcess:
def __init__(self, path, clf_fore, clf_shape):
str_list = path.split('/')
self.kind = str_list[-2]
self.name = str_list[-1]
self.clf_fore = clf_fore
self.clf_shape = clf_shape
self.image = cv2.imread(path)
self.image = cv2.resize(self.image, (400, 400))
# cv2.imshow('img',self.image)
# cv2.waitKey()
thread_s = threading.Thread(target=self.init_image_proc_s)
thread_m = threading.Thread(target=self.init_image_proc_m)
thread_l = threading.Thread(target=self.init_image_proc_l)
thread_s.start()
thread_m.start()
thread_l.start()
thread_s.join()
thread_m.join()
thread_l.join()
self.hog_list = []
hog_list = self.image_proc_m.get_hog_list() + \
self.image_proc_l.get_hog_list()
target_list = list(self.image_proc_m.get_classify_target_list()) + \
list(self.image_proc_l.get_classify_target_list())
for index in range(len(hog_list)):
if target_list[index]:
self.hog_list.append(hog_list[index])
self.client = MongoClient()
self.db = self.client.object_finder
def init_image_proc_s(self):
self.image_proc_s = ImageProcess(
cv2.medianBlur(self.image, 5), 70
)
self.image_proc_s.set_classify_target_list(
self.clf_fore.predict(
self.image_proc_s.get_classify_vec_list()
)
)
self.image_proc_s.image = cv2.bilateralFilter(self.image, 5, 50, 50)
self.image_proc_s.compute_foreground_mask()
self.image_proc_s.compute_foreground_image()
self.image_proc_s.compute_color_list()
self.image_proc_s.compute_sift_list()
def init_image_proc_m(self):
self.image_proc_m = ImageProcess(
cv2.bilateralFilter(self.image, 5, 50, 50), 100
)
self.image_proc_m.set_classify_target_list(
self.clf_shape.predict(
self.image_proc_m.get_classify_vec_list()
)
)
def init_image_proc_l(self):
self.image_proc_l = ImageProcess(
cv2.bilateralFilter(self.image, 5, 50, 50), 130
)
self.image_proc_l.set_classify_target_list(
self.clf_shape.predict(
self.image_proc_l.get_classify_vec_list()
)
)
def get_image(self):
return copy.deepcopy(self.image)
def get_fore(self):
return copy.deepcopy(self.image_proc_s.get_foreground_image())
def get_seg_image_list(self):
return [
np.array(self.image_proc_s.get_mark_image() * 255, dtype=np.uint8),
np.array(self.image_proc_m.get_mark_image() * 255, dtype=np.uint8),
np.array(self.image_proc_l.get_mark_image() * 255, dtype=np.uint8)
]
def get_pos_hog_image_list(self):
try:
return copy.deepcopy(self.hog_image_list)
except:
self.hog_image_list = []
tmp_list = self.image_proc_m.get_hog_image_list() + \
self.image_proc_l.get_hog_image_list()
target_list = list(self.image_proc_m.get_classify_target_list()) + \
list(self.image_proc_l.get_classify_target_list())
for i in range(len(target_list)):
if target_list[i]:
self.hog_image_list.append(tmp_list[i])
return copy.deepcopy(self.hog_image_list)
def get_color_list(self):
return copy.deepcopy(self.image_proc_s.get_color_list())
def get_sift_list(self):
return copy.deepcopy(self.image_proc_s.get_sift_list())
def get_hog_list(self):
return copy.deepcopy(self.hog_list)
def store_color_list(self):
coll = self.db.color_list
color_list=self.get_color_list()
try:
for color in color_list:
coll.insert({
'vec': color,
'kind': self.kind,
'name': self.name
})
except:
pass
return self
def store_sift_list(self):
coll = self.db.sift_list
sift_list = self.get_sift_list()
try:
for sift in sift_list:
coll.insert({
'vec': [int(num) for num in sift],
'kind': self.kind,
'name': self.name
})
except:
pass
return self
def store_hog_list(self):
coll = self.db.hog_list
hog_list = self.get_hog_list()
try:
for hog in hog_list:
coll.insert({
'vec': hog,
'kind': self.kind,
'name': self.name
})
except:
pass
return self
def write_fore_image(self, img_base):
cv2.imwrite(
img_base + '/' + self.kind + '/' + self.name,
self.image_proc_s.get_foreground_image()
)
def insert_into_dict(self, fit, arg_dict):
if fit['kind'] in arg_dict.keys():
if fit['name'][:-4] in arg_dict[fit['kind']].keys():
arg_dict[fit['kind']][fit['name'][:-4]] += 1
else:
arg_dict[fit['kind']][fit['name'][:-4]] = 1
else:
arg_dict[fit['kind']] = {}
arg_dict[fit['kind']][fit['name'][:-4]] = 1
def find_k_means_color_list(self):
fit_list = []
best_fit_list = []
print len(self.get_color_list())
for color in self.get_color_list():
fit, best = find_k_means(
color,
'object_finder',
'k_means_color_list',
'vec'
)
fit_list += fit
best_fit_list.append(best)
self.fit_color_dict = {}
for fit in fit_list:
self.insert_into_dict(fit, self.fit_color_dict)
self.best_fit_color_dict = {}
for best_fit in best_fit_list:
self.insert_into_dict(best_fit, self.best_fit_color_dict)
def get_fit_color_dict(self):
try:
return copy.deepcopy(self.fit_color_dict)
except:
self.find_k_means_color_list()
return copy.deepcopy(self.fit_color_dict)
def get_best_fit_color_dict(self):
try:
return copy.deepcopy(self.best_fit_color_dict)
except:
self.find_k_means_color_list()
return copy.deepcopy(self.best_fit_color_dict)
def find_k_means_sift_list(self):
fit_list = []
best_fit_list = []
try:
print len(self.get_sift_list())
for sift in self.get_sift_list():
fit, best = find_k_means(
sift,
'object_finder',
'k_means_sift_list',
'vec'
)
fit_list += fit
best_fit_list.append(best)
except:
print 0
self.fit_sift_dict = {}
for fit in fit_list:
self.insert_into_dict(fit, self.fit_sift_dict)
self.best_fit_sift_dict = {}
for best_fit in best_fit_list:
self.insert_into_dict(best_fit, self.best_fit_sift_dict)
def get_fit_sift_dict(self):
try:
return copy.deepcopy(self.fit_sift_dict)
except:
self.find_k_means_sift_list()
return copy.deepcopy(self.fit_sift_dict)
def get_best_fit_sift_dict(self):
try:
return copy.deepcopy(self.best_fit_sift_dict)
except:
self.find_k_means_sift_list()
return copy.deepcopy(self.best_fit_sift_dict)
def find_k_means_hog_list(self):
fit_list = []
best_fit_list = []
for hog in self.get_hog_list():
fit, best = find_k_means(
hog,
'object_finder',
'k_means_hog_list',
'vec'
)
fit_list += fit
best_fit_list.append(best)
self.fit_hog_dict = {}
for fit in fit_list:
self.insert_into_dict(fit, self.fit_hog_dict)
self.best_fit_hog_dict = {}
for best_fit in best_fit_list:
self.insert_into_dict(best_fit, self.best_fit_hog_dict)
def get_fit_hog_dict(self):
try:
return copy.deepcopy(self.fit_hog_dict)
except:
self.find_k_means_hog_list()
return copy.deepcopy(self.fit_hog_dict)
def get_best_fit_hog_dict(self):
try:
return copy.deepcopy(self.best_fit_hog_dict)
except:
self.find_k_means_hog_list()
return copy.deepcopy(self.best_fit_hog_dict)
class DriveTrain:
###########################################################################
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56/'
def __init__(self, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT # use it for csv file name
self.csv_path = csv_path
self.train_generator = None
self.valid_generator = None
self.train_hist = None
self.drive = None
#self.config = Config() #model_name)
self.data_path = data_path
#self.model_name = model_name
self.drive = DriveData(self.csv_path)
self.net_model = NetModel(data_path)
self.image_process = ImageProcess()
self.data_aug = DataAugmentation()
###########################################################################
#
def _prepare_data(self):
self.drive.read()
from sklearn.model_selection import train_test_split
samples = list(zip(self.drive.image_names, self.drive.measurements))
self.train_data, self.valid_data = train_test_split(samples,
test_size=Config.config['validation_rate'])
self.num_train_samples = len(self.train_data)
self.num_valid_samples = len(self.valid_data)
print('Train samples: ', self.num_train_samples)
print('Valid samples: ', self.num_valid_samples)
###########################################################################
#
def _build_model(self, show_summary=True):
def _generator(samples, batch_size=Config.config['batch_size']):
num_samples = len(samples)
while True: # Loop forever so the generator never terminates
if Config.config['lstm'] is False:
samples = sklearn.utils.shuffle(samples)
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset+batch_size]
images = []
measurements = []
for image_name, measurement in batch_samples:
image_path = self.data_path + '/' + image_name
image = cv2.imread(image_path)
image = cv2.resize(image,
(Config.config['input_image_width'],
Config.config['input_image_height']))
image = self.image_process.process(image)
images.append(image)
steering_angle, throttle = measurement
if abs(steering_angle) < Config.config['steering_angle_jitter_tolerance']:
steering_angle = 0
measurements.append(steering_angle*Config.config['steering_angle_scale'])
if Config.config['data_aug_flip'] is True:
# Flipping the image
flip_image, flip_steering = self.data_aug.flipping(image, steering_angle)
images.append(flip_image)
measurements.append(flip_steering*Config.config['steering_angle_scale'])
if Config.config['data_aug_bright'] is True:
# Changing the brightness of image
if steering_angle > Config.config['steering_angle_jitter_tolerance'] or \
steering_angle < -Config.config['steering_angle_jitter_tolerance']:
bright_image = self.data_aug.brightness(image)
images.append(bright_image)
measurements.append(steering_angle*Config.config['steering_angle_scale'])
if Config.config['data_aug_shift'] is True:
# Shifting the image
shift_image, shift_steering = self.data_aug.shift(image, steering_angle)
images.append(shift_image)
measurements.append(shift_steering*Config.config['steering_angle_scale'])
X_train = np.array(images)
y_train = np.array(measurements)
if Config.config['lstm'] is True:
X_train = np.array(images).reshape(-1, 1,
Config.config['input_image_height'],
Config.config['input_image_width'],
Config.config['input_image_depth'])
y_train = np.array(measurements).reshape(-1, 1, 1)
if Config.config['lstm'] is False:
yield sklearn.utils.shuffle(X_train, y_train)
else:
yield X_train, y_train
self.train_generator = _generator(self.train_data)
self.valid_generator = _generator(self.valid_data)
if (show_summary):
self.net_model.model.summary()
###########################################################################
#
def _start_training(self):
if (self.train_generator == None):
raise NameError('Generators are not ready.')
######################################################################
# callbacks
from keras.callbacks import ModelCheckpoint, EarlyStopping
# checkpoint
callbacks = []
#weight_filename = self.net_model.name + '_' + const.CONFIG_YAML + '_ckpt'
weight_filename = self.data_path + '_' + const.CONFIG_YAML + '_ckpt'
checkpoint = ModelCheckpoint(weight_filename+'.h5',
monitor='val_loss',
verbose=1, save_best_only=True, mode='min')
callbacks.append(checkpoint)
# early stopping
earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3,
verbose=1, mode='min')
callbacks.append(earlystop)
self.train_hist = self.net_model.model.fit_generator(
self.train_generator,
steps_per_epoch=self.num_train_samples//Config.config['batch_size'],
epochs=Config.config['num_epochs'],
validation_data=self.valid_generator,
validation_steps=self.num_valid_samples//Config.config['batch_size'],
verbose=1, callbacks=callbacks)
###########################################################################
#
def _plot_training_history(self):
print(self.train_hist.history.keys())
### plot the training and validation loss for each epoch
plt.plot(self.train_hist.history['loss'])
plt.plot(self.train_hist.history['val_loss'])
plt.ylabel('mse loss')
plt.xlabel('epoch')
plt.legend(['training set', 'validatation set'], loc='upper right')
plt.show()
###########################################################################
#
def train(self, show_summary=True):
self._prepare_data()
self._build_model(show_summary)
self._start_training()
self.net_model.save()
self._plot_training_history()
Config.summary()
class Viewer(tk.Frame):
def __init__(self, master, screnn_size):
self.svae_folder_name = None
FONT_SIZE = 13
#image
INPUT_POSITION_X = 10
INPUT_POSITION_Y = 10
INPUT_IMAGE_PLACE_Y = 30
IMAGE_WIDTH = int(screnn_size[0] / 3)
OUTPUT_PLACE_X = 2 * IMAGE_WIDTH + INPUT_POSITION_X
DEPTH_PLACE_X = IMAGE_WIDTH + INPUT_POSITION_X
#button
BUTTON_WIDTH = 10
BUTTON_HEIGHT = 2
WIDHT = int(screnn_size[0] / 2)
BUTTON_X = 3 * int(WIDHT / 3)
BUTTON_Y = 800
RUN_BUTTON_Y = INPUT_POSITION_Y + 40
VISUAL_BUTTON_Y = RUN_BUTTON_Y + 40
ON_BUTTON_Y = VISUAL_BUTTON_Y + 40
self.imgProc = ImageProcess()
dobi = ImageTk.PhotoImage(Image.open('dobi.png'))
# input image view
font = tf.Font(size=FONT_SIZE, weight='bold')
input_text = tk.Label(text='Input Image', font=font)
input_text.place(x=INPUT_POSITION_X, y=INPUT_POSITION_Y)
self.input_image = tk.Label()
self.upload_image_to_tkinter(self.input_image, dobi, INPUT_POSITION_X,
INPUT_IMAGE_PLACE_Y)
# output image view
ouput_text = tk.Label(text='Output Image', font=font)
ouput_text.place(x=OUTPUT_PLACE_X, y=INPUT_POSITION_Y)
self.ouput_image = tk.Label()
self.upload_image_to_tkinter(self.ouput_image, dobi, OUTPUT_PLACE_X,
INPUT_IMAGE_PLACE_Y)
# Depth Image view
depth_text = tk.Label(text='Depth Image', font=font)
depth_text.place(x=DEPTH_PLACE_X, y=INPUT_POSITION_Y)
self.depth_image = tk.Label()
self.upload_image_to_tkinter(self.depth_image, dobi, DEPTH_PLACE_X,
INPUT_IMAGE_PLACE_Y)
# shooting button
shooting_button = tk.Button(master,
overrelief='solid',
text='CAPTURE',
command=self.shotting,
width=BUTTON_WIDTH,
height=BUTTON_HEIGHT)
shooting_button.place(x=BUTTON_X, y=BUTTON_Y)
run_button = tk.Button(master,
overrelief='solid',
text='RUN',
command=self.run,
width=BUTTON_WIDTH,
height=BUTTON_HEIGHT)
run_button.place(x=BUTTON_X + 10 * BUTTON_WIDTH + 5, y=BUTTON_Y)
visual_button = tk.Button(master,
overrelief='solid',
text='VISUAL',
command=self.visual,
width=BUTTON_WIDTH,
height=BUTTON_HEIGHT)
visual_button.place(x=BUTTON_X + 2 * 10 * BUTTON_WIDTH + 5, y=BUTTON_Y)
save_button = tk.Button(master,
overrelief='solid',
text='SAVE',
command=self.save,
width=BUTTON_WIDTH,
height=BUTTON_HEIGHT)
save_button.place(x=BUTTON_X + 3 * 10 * BUTTON_WIDTH + 5, y=BUTTON_Y)
self.t1 = threading.Thread(target=self.imgProc.on)
self.t1.start()
def __del__(self):
self.imgProc.stream_stop = True
def save(self):
if self.svae_folder_name is None:
self.svae_folder_name = askstring(
"SAVE", "Enter the name of the folder to be saved")
self.imgProc.save(self.svae_folder_name)
def upload_image_to_tkinter(self, label, img, *place):
axis = place
label.image = img
label.configure(image=img)
if axis != ():
label.place(x=axis[0], y=axis[1])
def run(self):
self.imgProc.run()
output_ndarray = self.imgProc.output_image
output_image = ImageTk.PhotoImage(
image=Image.fromarray(output_ndarray))
self.upload_image_to_tkinter(self.ouput_image, output_image)
def shotting(self):
depth_array, image_array = self.imgProc.shotting()
depth_image = ImageTk.PhotoImage(image=Image.fromarray(depth_array))
self.upload_image_to_tkinter(self.depth_image, depth_image)
input_image = ImageTk.PhotoImage(image=Image.fromarray(image_array))
self.upload_image_to_tkinter(self.input_image, input_image)
def visual(self):
self.imgProc.visual()
