def get_loss_history_preview(loss_history, epoch, w, c): loss_history = np.array (loss_history.copy()) lh_height = 100 lh_img = np.ones ( (lh_height,w,c) ) * 0.1 loss_count = len(loss_history[0]) lh_len = len(loss_history) l_per_col = lh_len / w plist_max = [ [ max (0.0, loss_history[int(col*l_per_col)][p], *[ loss_history[i_ab][p] for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) ) ] ) for p in range(loss_count) ] for col in range(w) ] plist_min = [ [ min (plist_max[col][p], loss_history[int(col*l_per_col)][p], *[ loss_history[i_ab][p] for i_ab in range( int(col*l_per_col), int((col+1)*l_per_col) ) ] ) for p in range(loss_count) ] for col in range(w) ] plist_abs_max = np.mean(loss_history[ len(loss_history) // 5 : ]) * 2 for col in range(0, w): for p in range(0,loss_count): point_color = [1.0]*c point_color[0:3] = colorsys.hsv_to_rgb ( p * (1.0/loss_count), 1.0, 1.0 ) ph_max = int ( (plist_max[col][p] / plist_abs_max) * (lh_height-1) ) ph_max = np.clip( ph_max, 0, lh_height-1 ) ph_min = int ( (plist_min[col][p] / plist_abs_max) * (lh_height-1) ) ph_min = np.clip( ph_min, 0, lh_height-1 ) for ph in range(ph_min, ph_max+1): lh_img[ (lh_height-ph-1), col ] = point_color lh_lines = 5 lh_line_height = (lh_height-1)/lh_lines for i in range(0,lh_lines+1): lh_img[ int(i*lh_line_height), : ] = (0.8,)*c last_line_t = int((lh_lines-1)*lh_line_height) last_line_b = int(lh_lines*lh_line_height) lh_text = 'Epoch: %d' % (epoch) if epoch != 0 else '' lh_img[last_line_t:last_line_b, 0:w] += image_utils.get_text_image ( (w,last_line_b-last_line_t,c), lh_text, color=[0.8]*c ) return lh_img
def previewThread(input_queue, output_queue): previews = None loss_history = None selected_preview = 0 update_preview = False is_showing = False is_waiting_preview = False epoch = 0 while True: if not input_queue.empty(): input = input_queue.get() op = input['op'] if op == 'show': is_waiting_preview = False loss_history = input[ 'loss_history'] if 'loss_history' in input.keys() else None previews = input['previews'] if 'previews' in input.keys( ) else None epoch = input['epoch'] if 'epoch' in input.keys() else 0 if previews is not None: max_w = 0 max_h = 0 for (preview_name, preview_rgb) in previews: (h, w, c) = preview_rgb.shape max_h = max(max_h, h) max_w = max(max_w, w) max_size = 800 if max_h > max_size: max_w = int(max_w / (max_h / max_size)) max_h = max_size #make all previews size equal for preview in previews[:]: (preview_name, preview_rgb) = preview (h, w, c) = preview_rgb.shape if h != max_h or w != max_w: previews.remove(preview) previews.append( (preview_name, cv2.resize(preview_rgb, (max_w, max_h)))) selected_preview = selected_preview % len(previews) update_preview = True elif op == 'close': break if update_preview: update_preview = False selected_preview_name = previews[selected_preview][0] selected_preview_rgb = previews[selected_preview][1] (h, w, c) = selected_preview_rgb.shape # HEAD head_text_color = [0.8] * c head_lines = [ '[s]:save [enter]:exit', '[p]:update [space]:next preview', 'Preview: "%s" [%d/%d]' % (selected_preview_name, selected_preview + 1, len(previews)) ] head_line_height = 15 head_height = len(head_lines) * head_line_height head = np.ones((head_height, w, c)) * 0.1 for i in range(0, len(head_lines)): t = i * head_line_height b = (i + 1) * head_line_height head[t:b, 0:w] += image_utils.get_text_image( (w, head_line_height, c), head_lines[i], color=head_text_color) final = head if loss_history is not None: # LOSS HISTORY loss_history = np.array(loss_history) lh_height = 100 lh_img = np.ones((lh_height, w, c)) * 0.1 loss_count = len(loss_history[0]) lh_len = len(loss_history) l_per_col = lh_len / w plist_max = [[ max( 0.0, loss_history[int(col * l_per_col)][p], *[ loss_history[i_ab][p] for i_ab in range(int(col * l_per_col), int((col + 1) * l_per_col)) ]) for p in range(loss_count) ] for col in range(w)] plist_min = [[ min( plist_max[col][p], loss_history[int(col * l_per_col)][p], *[ loss_history[i_ab][p] for i_ab in range(int(col * l_per_col), int((col + 1) * l_per_col)) ]) for p in range(loss_count) ] for col in range(w)] plist_abs_max = np.mean( loss_history[len(loss_history) // 5:]) * 2 for col in range(0, w): for p in range(0, loss_count): point_color = [1.0] * c point_color[0:3] = colorsys.hsv_to_rgb( p * (1.0 / loss_count), 1.0, 1.0) ph_max = int((plist_max[col][p] / plist_abs_max) * (lh_height - 1)) ph_max = np.clip(ph_max, 0, lh_height - 1) ph_min = int((plist_min[col][p] / plist_abs_max) * (lh_height - 1)) ph_min = np.clip(ph_min, 0, lh_height - 1) for ph in range(ph_min, ph_max + 1): lh_img[(lh_height - ph - 1), col] = point_color lh_lines = 5 lh_line_height = (lh_height - 1) / lh_lines for i in range(0, lh_lines + 1): lh_img[int(i * lh_line_height), :] = (0.8, ) * c last_line_t = int((lh_lines - 1) * lh_line_height) last_line_b = int(lh_lines * lh_line_height) lh_text = 'Epoch: %d' % (epoch) if epoch != 0 else '' lh_img[last_line_t:last_line_b, 0:w] += image_utils.get_text_image( (w, last_line_b - last_line_t, c), lh_text, color=head_text_color) final = np.concatenate([final, lh_img], axis=0) final = np.concatenate([final, selected_preview_rgb], axis=0) cv2.imshow('Training preview', final) is_showing = True if is_showing: key = cv2.waitKey(100) else: time.sleep(0.1) key = 0 if key == ord('\n') or key == ord('\r'): output_queue.put({'op': 'close'}) elif key == ord('s'): output_queue.put({'op': 'save'}) elif key == ord('p'): if not is_waiting_preview: is_waiting_preview = True output_queue.put({'op': 'preview'}) elif key == ord(' '): selected_preview = (selected_preview + 1) % len(previews) update_preview = True cv2.destroyAllWindows()
def previewThread (input_queue, output_queue): previews = None loss_history = None selected_preview = 0 update_preview = False is_showing = False is_waiting_preview = False epoch = 0 while True: if not input_queue.empty(): input = input_queue.get() op = input['op'] if op == 'show': is_waiting_preview = False loss_history = input['loss_history'] if 'loss_history' in input.keys() else None previews = input['previews'] if 'previews' in input.keys() else None epoch = input['epoch'] if 'epoch' in input.keys() else 0 if previews is not None: max_w = 0 max_h = 0 for (preview_name, preview_rgb) in previews: (h, w, c) = preview_rgb.shape max_h = max (max_h, h) max_w = max (max_w, w) max_size = 800 if max_h > max_size: max_w = int( max_w / (max_h / max_size) ) max_h = max_size #make all previews size equal for preview in previews[:]: (preview_name, preview_rgb) = preview (h, w, c) = preview_rgb.shape if h != max_h or w != max_w: previews.remove(preview) previews.append ( (preview_name, cv2.resize(preview_rgb, (max_w, max_h))) ) selected_preview = selected_preview % len(previews) update_preview = True elif op == 'close': break if update_preview: update_preview = False selected_preview_name = previews[selected_preview][0] selected_preview_rgb = previews[selected_preview][1] (h,w,c) = selected_preview_rgb.shape # HEAD head_lines = [ '[s]:save [enter]:exit', '[p]:update [space]:next preview', 'Preview: "%s" [%d/%d]' % (selected_preview_name,selected_preview+1, len(previews) ) ] head_line_height = 15 head_height = len(head_lines) * head_line_height head = np.ones ( (head_height,w,c) ) * 0.1 for i in range(0, len(head_lines)): t = i*head_line_height b = (i+1)*head_line_height head[t:b, 0:w] += image_utils.get_text_image ( (w,head_line_height,c) , head_lines[i], color=[0.8]*c ) final = head if loss_history is not None: # LOSS HISTORY lh_img = models.ModelBase.get_loss_history_preview(loss_history, epoch, w, c) final = np.concatenate ( [final, lh_img], axis=0 ) final = np.concatenate ( [final, selected_preview_rgb], axis=0 ) final = np.clip(final, 0, 1) cv2.imshow ( 'Training preview', (final*255).astype(np.uint8) ) is_showing = True if is_showing: key = cv2.waitKey(100) else: time.sleep(0.1) key = 0 if key == ord('\n') or key == ord('\r'): output_queue.put ( {'op': 'close'} ) elif key == ord('s'): output_queue.put ( {'op': 'save'} ) elif key == ord('p'): if not is_waiting_preview: is_waiting_preview = True output_queue.put ( {'op': 'preview'} ) elif key == ord(' '): selected_preview = (selected_preview + 1) % len(previews) update_preview = True cv2.destroyAllWindows()
def main(args, device_args): io.log_info("Running trainer.\r\n") no_preview = args.get('no_preview', False) s2c = queue.Queue() c2s = queue.Queue() thread = threading.Thread(target=trainerThread, args=(s2c, c2s, args, device_args)) thread.start() if no_preview: while True: if not c2s.empty(): input = c2s.get() op = input.get('op', '') if op == 'close': break io.process_messages(0.1) else: wnd_name = "Training preview" io.named_window(wnd_name) io.capture_keys(wnd_name) previews = None loss_history = None selected_preview = 0 update_preview = False is_showing = False is_waiting_preview = False show_last_history_iters_count = 0 iter = 0 while True: if not c2s.empty(): input = c2s.get() op = input['op'] if op == 'show': is_waiting_preview = False loss_history = input[ 'loss_history'] if 'loss_history' in input.keys( ) else None previews = input['previews'] if 'previews' in input.keys( ) else None iter = input['iter'] if 'iter' in input.keys() else 0 if previews is not None: max_w = 0 max_h = 0 for (preview_name, preview_rgb) in previews: (h, w, c) = preview_rgb.shape max_h = max(max_h, h) max_w = max(max_w, w) max_size = 800 if max_h > max_size: max_w = int(max_w / (max_h / max_size)) max_h = max_size #make all previews size equal for preview in previews[:]: (preview_name, preview_rgb) = preview (h, w, c) = preview_rgb.shape if h != max_h or w != max_w: previews.remove(preview) previews.append( (preview_name, cv2.resize(preview_rgb, (max_w, max_h)))) selected_preview = selected_preview % len(previews) update_preview = True elif op == 'close': break if update_preview: update_preview = False selected_preview_name = previews[selected_preview][0] selected_preview_rgb = previews[selected_preview][1] (h, w, c) = selected_preview_rgb.shape # HEAD head_lines = [ '[s]:save [enter]:exit', '[p]:update [space]:next preview [l]:change history range', 'Preview: "%s" [%d/%d]' % (selected_preview_name, selected_preview + 1, len(previews)) ] head_line_height = 15 head_height = len(head_lines) * head_line_height head = np.ones((head_height, w, c)) * 0.1 for i in range(0, len(head_lines)): t = i * head_line_height b = (i + 1) * head_line_height head[t:b, 0:w] += image_utils.get_text_image( (w, head_line_height, c), head_lines[i], color=[0.8] * c) final = head if loss_history is not None: if show_last_history_iters_count == 0: loss_history_to_show = loss_history else: loss_history_to_show = loss_history[ -show_last_history_iters_count:] lh_img = models.ModelBase.get_loss_history_preview( loss_history_to_show, iter, w, c) final = np.concatenate([final, lh_img], axis=0) final = np.concatenate([final, selected_preview_rgb], axis=0) final = np.clip(final, 0, 1) io.show_image(wnd_name, (final * 255).astype(np.uint8)) is_showing = True key_events = io.get_key_events(wnd_name) key, = key_events[-1] if len(key_events) > 0 else (0, ) if key == ord('\n') or key == ord('\r'): s2c.put({'op': 'close'}) elif key == ord('s'): s2c.put({'op': 'save'}) elif key == ord('p'): if not is_waiting_preview: is_waiting_preview = True s2c.put({'op': 'preview'}) elif key == ord('l'): if show_last_history_iters_count == 0: show_last_history_iters_count = 5000 elif show_last_history_iters_count == 5000: show_last_history_iters_count = 10000 elif show_last_history_iters_count == 10000: show_last_history_iters_count = 50000 elif show_last_history_iters_count == 50000: show_last_history_iters_count = 100000 elif show_last_history_iters_count == 100000: show_last_history_iters_count = 0 update_preview = True elif key == ord(' '): selected_preview = (selected_preview + 1) % len(previews) update_preview = True io.process_messages(0.1) io.destroy_all_windows()