def _split_large_chars(chars: List[Node],
rects: List[Rectangle]) -> Iterable[Node]:
extra = 0
for i, char in enumerate(chars):
height, width = char.image.shape
if width < 32:
yield char
continue
min_x = int(width / 3)
max_x = int(width / 1.5)
center = char.image[:int(height * 0.60), min_x:max_x]
energy = center.sum(axis=0)
min_seam = np.argmin(energy)
split_point = min_seam + min_x
if energy[min_seam] < 128:
yield Node(char.image[:, :split_point],
parents=[char],
key=['split_left'])
yield Node(char.image[:, split_point:],
parents=[char],
key=['split_right'])
rects.insert(i + extra, rects[i + extra])
extra += 1
else:
yield char
def add_nodes(self, node: Node, result: TestResult):
if node is None:
return
for descendent in node.descendents_and_me():
if descendent.test_uuid is not None:
continue
new_id = uuid4()
descendent.test_uuid = new_id
descendent.test_result = result
self.id_to_node[new_id] = descendent
for ancestor in node.ancestors_and_me():
if ancestor.test_uuid is not None:
continue
new_id = uuid4()
ancestor.test_uuid = new_id
ancestor.test_result = result
self.id_to_node[new_id] = ancestor
def load_relevant_sprites() -> Iterable[Tuple[str, Node]]:
root = Path(os.environ['OBJECTS_SRC'])
for path in root.glob('**/*.png'):
img = cv2.imread(path.as_posix())
node = Node(img)
if node.width < 24 or node.height < 24:
continue
# Filter out portraits (TODO: I should delete them.)
if node.width == 48 and node.height == 32:
continue
yield path.parent.parent.name, node
def prepare_frame(frame: Node) -> Node:
gray = frame.gray
everything = gray.thumbnail32
bottom_left = gray.crop(Rectangle(40, 522, 470, 175)).thumbnail32
bottom_right = gray.crop(Rectangle(520, 522, 470, 175)).thumbnail32
effect_area = gray.crop(Rectangle(260, 94, 450, 95)).thumbnail32
image = np.block([[everything.image, effect_area.image],
[bottom_left.image, bottom_right.image]])
return Node(image,
parents=[everything, bottom_left, bottom_right, effect_area])
def node_to_tiles(frame: Node) -> Iterable[Node]:
load_classes()
width, height = frame.width, frame.height
tiles_wide = width // TILE_WIDTH
tiles_high = height // TILE_HEIGHT
for i in range(tiles_wide):
for j in range(tiles_high):
x_offset = i * TILE_WIDTH
y_offset = j * TILE_HEIGHT
rect = Rectangle(i * TILE_WIDTH, j * TILE_HEIGHT, TILE_WIDTH,
TILE_HEIGHT)
yield x_offset, y_offset, frame.crop(rect)
def load_labelled_states():
from pathlib import Path
import cv2
xs = []
ys = []
for path in Path(os.environ['STREAM_STATE_SRC']).iterdir():
if path.name[0] == '.':
continue
state = path.name
index = STREAM_STATES.index(state)
for image_path in path.glob('*.jpg'):
frame = Node(cv2.imread(image_path.as_posix()))
img = prepare_frame(frame)
xs.append(img)
ys.append(index)
return np.array(xs), np.array(ys)
def __call__(self, frame: Node) -> List[ObjectPrediction]:
import tensorflow as tf
tiles = list(
node_to_tiles(
frame.resize(STREAM_WIDTH // SCALE, STREAM_HEIGHT // SCALE)))
y_pred = self.model(
tf.stack([process_image(tile.image) for (_, _, tile) in tiles]))
pred_class = [self.classes[i] for i in np.argmax(y_pred, axis=1)]
confidence = np.max(y_pred, axis=1)
out = []
for i in range(len(tiles)):
x_offset, y_offset, tile = tiles[i]
rect = Rectangle(x_offset * SCALE, y_offset * SCALE, TILE_WIDTH,
TILE_HEIGHT)
out.append(
ObjectPrediction(pred_class[i], confidence[i], rect, tile))
return out
def run():
# TODO: Replace this with hand picked test cases instead of including everything
stream_state_model = stream_state.StreamStateModel()
for path in Path(os.environ['STREAM_STATE_SRC']).iterdir():
if path.name[0] == '.':
continue
expected = path.name
for image_path in path.glob('*.jpg'):
frame = Node(cv2.imread(image_path.as_posix()))
state = stream_state_model(frame)
actual = state.name
yield TestResult(image_path.as_posix(),
name='stream_state',
frame=frame,
data=state,
ok=actual == expected,
actual=actual,
expected=expected,
relevant_nodes=[state.node])
def run():
test_cases = json.loads(Path('data/tests/character.json').read_text())
char_model = character.CharacterModel()
char_finders = character.finders_from_model(char_model)
by_name = {finder.name: finder for finder in char_finders}
for fp, case in test_cases.items():
img = cv2.imread('data/tests/character/' + fp)
for key, expected in case.items():
frame = Node(img)
finder = by_name[key]
string = finder(frame)
actual = string.to_str()
yield TestResult(fp,
name=finder.name,
frame=frame,
data=string,
ok=actual == expected,
actual=actual,
expected=expected,
relevant_nodes=string.nodes)
def main():
configure()
birdvision.quiet.silence_tensorflow()
fps = int(os.environ['FPS'])
stop_event = threading.Event()
queue = Queue(maxsize=fps * 30)
ffmpeg_thread = threading.Thread(
target=lambda: stream_viewer.download_stream(queue, stop_event),
daemon=True)
ffmpeg_thread.start()
pygame.init()
pygame.font.init()
font = pygame.font.Font('data/pygame/RobotoCondensed-Regular.ttf', 20)
pygame.display.set_caption("Birb Brains Vision")
pygame.display.set_icon(pygame.image.load('data/pygame/icon.png'))
size = width, height = STREAM_WIDTH, STREAM_HEIGHT + 200
screen = pygame.display.set_mode(size)
black = 0, 0, 0
surface = pygame.Surface((STREAM_WIDTH, STREAM_HEIGHT))
# offsets = [(5, i * 28 + 5 + STREAM_HEIGHT) for i in range(6)] \
# + [(505, i * 28 + 5 + STREAM_HEIGHT) for i in range(6)]
clock = pygame.time.Clock()
saved_screens = 0
watcher = Watcher()
# object_model = ObjectModel()
last_state = None
while not stop_event.is_set():
for event in pygame.event.get():
if event.type == pygame.QUIT:
sys.exit()
f_start = time.monotonic()
try:
image = queue.get(block=False)
except Empty:
clock.tick(fps)
continue
jpeg_buf = np.frombuffer(image, np.uint8)
image = cv2.imdecode(jpeg_buf, flags=cv2.IMREAD_COLOR)
if image is None:
continue
frame = Node(image)
color_mapped = cv2.applyColorMap(frame.gray.image, cv2.COLORMAP_BONE)
frame_info = watcher(frame)
if frame_info != last_state and frame_info.state != stream_state.BLACK:
print(frame_info)
last_state = frame_info
color_mapped = color_mapped[..., ::-1].copy()
arr = pygame.surfarray.map_array(surface, color_mapped).swapaxes(0, 1)
pygame.surfarray.blit_array(surface, arr)
screen.blit(surface, surface.get_rect())
# if stream_state.in_game(frame_info.state):
# objects = object_model(frame)
# for obj in objects:
# if obj.kind == 'None':
# continue
# kind = font.render(obj.kind, True, (100, 255, 100))
# screen.blit(kind, obj.rect.top_left)
f_duration = time.monotonic() - f_start
status_line = f'{queue.qsize():03d} {saved_screens:05d} {f_duration * 1000:.2f}ms'
status_surf = font.render(status_line, True, (100, 255, 100))
screen.blit(status_surf, (width - 200, 25))
pygame.display.flip()
screen.fill(black)
clock.tick(fps)
def load_image(path):
image = cv2.imread(path.as_posix())
if image is None or image.size == 0:
return None
return Node(image)
def load_relevant_backgrounds() -> Iterable[Node]:
root = Path(os.environ['GENERATIVE_BGS_SRC'])
for path in root.glob('*.png'):
img = cv2.imread(path.as_posix())
node = Node(img)
yield node.resize(node.width // 2, node.height // 2)