def train_module_classifier():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(PASSWORD_LETTER_CLASSIFIER_DIR)
def module_vocab_paths():
current_module_offset = 0
for i in range(MAX_INDEX + 1):
if i % 3 == 0:
continue
file_name = "{:04d}-full-{}.png".format(
i, MODULE_NAME_FOR_OFFSET[current_module_offset])
file_path = os.path.join(unlabelled_dir, file_name)
current_module_offset = (current_module_offset +
1) % NUM_MODULE_POSITIONS
yield file_path
def image_and_features_path():
for screenshot_set in range(MAX_INDEX + 1):
if screenshot_set % 3 == 0:
continue
for module_offset in range(NUM_MODULE_POSITIONS):
file_name = "{:04d}-full-{}".format(
screenshot_set, MODULE_NAME_FOR_OFFSET[module_offset])
screenshot_path = os.path.join(unlabelled_dir,
file_name + ".png")
feature_path = os.path.join(features_dir, file_name + ".npy")
yield screenshot_path, feature_path
def solve_stored_mazes():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(MODULE_CLASSIFIER_DIR)
maze_dir = os.path.join(labelled_dir, "maze")
files_to_test = (
"/Users/danny/Dropbox (Personal)/Projects/KeepTalkingBot/module_specific_data/debug/0907.png",
)
for i, file_name in enumerate(os.listdir(maze_dir)):
# for file_name in files_to_test:
# if file_name != "0023-full-bottom-left.png":
# continue
print file_name
maze_image = cv2.imread(os.path.join(maze_dir, file_name))
lookup_key, start_coordinates, end_coordinates = get_maze_params(
maze_image)
print "lookup", lookup_key
print "start (white)", start_coordinates
print "end (red)", end_coordinates
top, right, bottom, left = get_button_locations(maze_image)
for location in (top, right, bottom, left):
cv2.circle(maze_image, location, 10, (255, 0, 0), 10)
show(maze_image)
# print top, right, bottom, left
moves = find_path_through_maze(lookup_key, start_coordinates,
end_coordinates)
print " ".join(moves)
def manually_group_images(classifier_dir):
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = get_classifier_directories(classifier_dir)
for file_path in ls(unlabelled_dir):
folder_name = chr(show(cv2.imread(file_path)))
folder = os.path.join(labelled_dir, folder_name)
if not os.path.exists(folder):
os.makedirs(folder)
dst = os.path.join(labelled_dir, folder, os.path.basename(file_path))
shutil.copyfile(file_path, dst)
def train_classifier_pipeline(classifier_dir):
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = get_classifier_directories(classifier_dir)
print "Translating data"
translate_data(labelled_dir, features_dir, svm_data_dir)
print "Training classifier"
train_classifier(svm_data_dir)
print "Testing classifier"
run_test(svm_data_dir)
print "Saving label mappings"
save_label_mappings(labelled_dir, svm_data_dir)
def manually_group_images(classifier_dir):
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = get_classifier_directories(
classifier_dir)
for file_path in ls(unlabelled_dir):
folder_name = chr(show(cv2.imread(file_path)))
folder = os.path.join(labelled_dir, folder_name)
if not os.path.exists(folder):
os.makedirs(folder)
dst = os.path.join(labelled_dir, folder, os.path.basename(file_path))
shutil.copyfile(file_path, dst)
def train_classifier_pipeline(classifier_dir):
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = get_classifier_directories(
classifier_dir)
print "Translating data"
translate_data(labelled_dir, features_dir, svm_data_dir)
print "Training classifier"
train_classifier(svm_data_dir)
print "Testing classifier"
run_test(svm_data_dir)
print "Saving label mappings"
save_label_mappings(labelled_dir, svm_data_dir)
def extract_all_letter_images_for_training():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(MODULE_CLASSIFIER_DIR)
for file_name in os.listdir(os.path.join(labelled_dir, "password")):
if file_name == ".DS_Store":
continue
file_path = os.path.join(labelled_dir, "password", file_name)
im = cv2.imread(file_path)
letters = get_letter_images(im)
without_ext, _ = os.path.splitext(file_name)
letter_file_name_template = without_ext + "-%s.png"
for i, letter in enumerate(letters):
save_path = os.path.join(MODULE_SPECIFIC_DIR, "password", "letters", "unlabelled", letter_file_name_template % i)
cv2.imwrite(save_path, letter)
def extract_all_letter_images_for_training():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(MODULE_CLASSIFIER_DIR)
for file_name in os.listdir(os.path.join(labelled_dir, "password")):
if file_name == ".DS_Store":
continue
file_path = os.path.join(labelled_dir, "password", file_name)
im = cv2.imread(file_path)
letters = get_letter_images(im)
without_ext, _ = os.path.splitext(file_name)
letter_file_name_template = without_ext + "-%s.png"
for i, letter in enumerate(letters):
save_path = os.path.join(MODULE_SPECIFIC_DIR, "password",
"letters", "unlabelled",
letter_file_name_template % i)
cv2.imwrite(save_path, letter)
def test():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(MODULE_CLASSIFIER_DIR)
for path in ls(os.path.join(labelled_dir, "password")):
im = cv2.imread(path)
letter_classifier = inflate_classifier(PASSWORD_LETTER_CLASSIFIER_DIR)
top_buttons, bottom_buttons = find_column_buttons(im)
submit_button = find_submit_button(im)
print get_letters(im, letter_classifier)
for i, b in enumerate(top_buttons):
cv2.circle(im, b, 5, (0, 0, 255))
cv2.putText(im, str(i), b, cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 0))
for i, b in enumerate(bottom_buttons):
cv2.circle(im, b, 5, (0, 255, 0))
cv2.putText(im, str(i), b, cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 0))
cv2.circle(im, submit_button, 5, (255, 0, 0))
show(im)
def test():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(MODULE_CLASSIFIER_DIR)
for path in ls(os.path.join(labelled_dir, "password")):
im = cv2.imread(path)
letter_classifier = inflate_classifier(PASSWORD_LETTER_CLASSIFIER_DIR)
top_buttons, bottom_buttons = find_column_buttons(im)
submit_button = find_submit_button(im)
print get_letters(im, letter_classifier)
for i, b in enumerate(top_buttons):
cv2.circle(im, b, 5, (0, 0, 255))
cv2.putText(im, str(i), b, cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 0))
for i, b in enumerate(bottom_buttons):
cv2.circle(im, b, 5, (0, 255, 0))
cv2.putText(im, str(i), b, cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 0))
cv2.circle(im, submit_button, 5, (255, 0, 0))
show(im)
def test():
tesseract = _get_tesseract()
classifier = inflate_classifier(WHOS_ON_FIRST_BUTTON_CLASSIFIER_DIR)
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(MODULE_CLASSIFIER_DIR)
i = 0
# for path in ["/Users/danny/Dropbox (Personal)/Projects/KeepTalkingBot/module_specific_data/whos_on_first/in_game_6.png"]:
# for path in ls(os.path.join(labelled_dir, "whos_on_first")):
for path in ls_debug(1486, 1486):
i += 1
# if i < 50:
# continue
# if i >= 50:
# break
# name = "-module-".join(os.path.basename(path).split("-full-"))
# path = os.path.join(unlabelled_dir, name)
im = cv2.imread(path)
# show(im)
screen_text = get_screen_content(im, tesseract, 9999)
def _test():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(SERIAL_IS_ZERO_CLASSIFIER_DIR)
letter_idx = 0
i = 0
group = 0
found_in_group = 0
# for path in ls(DATA_DIR + "module_classifier/unlabelled"):
for path in ls_debug(1294, 1297):
# if "-left.png" not in path:
# if "0036-edge-bottom.png" not in path:
# continue
# new_group = int(os.path.basename(path).split("-")[0])
# if new_group % 3 != 0:
# continue
# if new_group != group:
# if found_in_group != 1:
# "!!!! Found {} in group {} !!!!".format(found_in_group, group)
# found_in_group = 0
# group = new_group
i += 1
# if i == 1:
# continue
# if i > 1:
# break
print path
im = cv2.imread(path)
from sides import _extract_side
im = _extract_side(im, "-bottom" in path)
text = get_serial_number_from_side(im)
if text is None:
print "NO SERIAL NUMBER"
else:
print "-".join(text)
text_threshold = _get_cleaned_up_text_subsection(im)
if text_threshold is not None:
show(text_threshold)
def _test():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(SERIAL_IS_ZERO_CLASSIFIER_DIR)
letter_idx = 0
i = 0
group = 0
found_in_group = 0
# for path in ls(DATA_DIR + "module_classifier/unlabelled"):
for path in ls_debug(1294, 1297):
# if "-left.png" not in path:
# if "0036-edge-bottom.png" not in path:
# continue
# new_group = int(os.path.basename(path).split("-")[0])
# if new_group % 3 != 0:
# continue
# if new_group != group:
# if found_in_group != 1:
# "!!!! Found {} in group {} !!!!".format(found_in_group, group)
# found_in_group = 0
# group = new_group
i += 1
# if i == 1:
# continue
# if i > 1:
# break
print path
im = cv2.imread(path)
from sides import _extract_side
im = _extract_side(im, "-bottom" in path)
text = get_serial_number_from_side(im)
if text is None:
print "NO SERIAL NUMBER"
else:
print "-".join(text)
text_threshold = _get_cleaned_up_text_subsection(im)
if text_threshold is not None:
show(text_threshold)
def cluster_images_pipeline(classifier_dir, num_clusters):
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir =\
get_classifier_directories(classifier_dir)
def representative_image_paths():
for i, file_name in enumerate(os.listdir(unlabelled_dir)):
if file_name == ".DS_Store":
continue
# Only want some of them
# if i % 10 != 0:
# continue
yield os.path.join(unlabelled_dir, file_name)
def image_and_feature_paths():
for file_name in os.listdir(unlabelled_dir):
if file_name == ".DS_Store":
continue
without_ext, _ = os.path.splitext(file_name)
letter_path = os.path.join(unlabelled_dir, file_name)
feature_path = os.path.join(features_dir, without_ext + ".npy")
yield letter_path, feature_path
def feature_and_copy_paths():
for file_name in os.listdir(unlabelled_dir):
if file_name == ".DS_Store":
continue
without_ext, _ = os.path.splitext(file_name)
feature_path = os.path.join(features_dir, without_ext + ".npy")
src_path = os.path.join(unlabelled_dir, file_name)
dst_path_template = os.path.join(labelled_dir, "%s", file_name)
yield feature_path, src_path, dst_path_template
print "Generating vocab"
generate_vocab(representative_image_paths(), vocab_path)
print "Extracting features"
extract_features(vocab_path, image_and_feature_paths())
print "Clustering images"
cluster_features(num_clusters, feature_and_copy_paths())
def cluster_images_pipeline(classifier_dir, num_clusters):
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir =\
get_classifier_directories(classifier_dir)
def representative_image_paths():
for i, file_name in enumerate(os.listdir(unlabelled_dir)):
if file_name == ".DS_Store":
continue
# Only want some of them
# if i % 10 != 0:
# continue
yield os.path.join(unlabelled_dir, file_name)
def image_and_feature_paths():
for file_name in os.listdir(unlabelled_dir):
if file_name == ".DS_Store":
continue
without_ext, _ = os.path.splitext(file_name)
letter_path = os.path.join(unlabelled_dir, file_name)
feature_path = os.path.join(features_dir, without_ext + ".npy")
yield letter_path, feature_path
def feature_and_copy_paths():
for file_name in os.listdir(unlabelled_dir):
if file_name == ".DS_Store":
continue
without_ext, _ = os.path.splitext(file_name)
feature_path = os.path.join(features_dir, without_ext + ".npy")
src_path = os.path.join(unlabelled_dir, file_name)
dst_path_template = os.path.join(labelled_dir, "%s", file_name)
yield feature_path, src_path, dst_path_template
print "Generating vocab"
generate_vocab(representative_image_paths(), vocab_path)
print "Extracting features"
extract_features(vocab_path, image_and_feature_paths())
print "Clustering images"
cluster_features(num_clusters, feature_and_copy_paths())
def train_module_classifier():
vocab_path, unlabelled_dir, labelled_dir, features_dir, svm_data_dir = \
get_classifier_directories(PASSWORD_LETTER_CLASSIFIER_DIR)
def module_vocab_paths():
current_module_offset = 0
for i in range(MAX_INDEX + 1):
if i % 3 == 0:
continue
file_name = "{:04d}-full-{}.png".format(i, MODULE_NAME_FOR_OFFSET[current_module_offset])
file_path = os.path.join(unlabelled_dir, file_name)
current_module_offset = (current_module_offset + 1) % NUM_MODULE_POSITIONS
yield file_path
def image_and_features_path():
for screenshot_set in range(MAX_INDEX + 1):
if screenshot_set % 3 == 0:
continue
for module_offset in range(NUM_MODULE_POSITIONS):
file_name = "{:04d}-full-{}".format(screenshot_set, MODULE_NAME_FOR_OFFSET[module_offset])
screenshot_path = os.path.join(unlabelled_dir, file_name + ".png")
feature_path = os.path.join(features_dir, file_name + ".npy")
yield screenshot_path, feature_path