def get_only_random_predictions(examples): total_random_correct = 0 with open("tensorboard/random_predictions.csv", 'w') as f: for i in range(2755): total_count = 0 total_random_correct = 0 for example in examples: expected = np.argmax(example[2]) expected_tile = example[1][expected] random_prediction = random.randint( 0, SolutionChecker.get_n_nonplaced_tiles(example[1]) - 1) if np.array_equal(expected_tile, example[1][random_prediction]): total_random_correct += 1 total_count += 1 accuracy = (total_random_correct / total_count) * 100 f.write(f'{accuracy}\n') print(accuracy) return
def get_examples(given_examples, n_tiles, height, width, dg, from_file=False, return_binary_mask=False, predict_v=False, predict_move_index=True, scalar_tiles=False, shuffle_tiles_times=20): examples = [] for i, _example in enumerate(given_examples): # print(f'{i}/{len(given_examples)}') if scalar_tiles: state, tiles, solution = _example state = state.squeeze() else: state, solution = _example grid = np.zeros([height, width]) for solution_index, solution_tile in enumerate(solution): solution_copy = np.copy(solution) solution_order = np.array(solution_copy[solution_index:]) solution_tile_dims = solution_tile[:2] orig_solution_order = solution_order for i in range(shuffle_tiles_times): # tile permutations solution_order = np.copy(orig_solution_order) _tiles_ints = [x[:ORIENTATIONS] for x in solution_order] _tiles_ints = get_tiles_with_orientation(_tiles_ints) _tiles_ints = SolutionChecker.get_possible_tile_actions_given_grid( grid, _tiles_ints) np.random.shuffle(_tiles_ints) if scalar_tiles: tiles = tiles_to_np_array( SolutionChecker.pad_tiles_with_zero_scalars( _tiles_ints, ORIENTATIONS * n_tiles - len(_tiles_ints))) else: tiles = dg._transform_instance_to_matrix( _tiles_ints, only_one_orientation=True) tiles = pad_tiles_with_zero_matrices( tiles, ORIENTATIONS * n_tiles - tiles.shape[2], width, height) state = np.dstack((np.expand_dims(grid, axis=2), tiles)) pi = solution_to_solution_matrix( solution_tile, cols=width, rows=height, return_binary_mask=False).flatten() # v = N_TILES - solution_index v = 1 if solution_index == len(solution) - 1: continue if predict_move_index: n_possible_tiles = SolutionChecker.get_n_nonplaced_tiles( _tiles_ints) if n_possible_tiles == 1: # if only one action-tile placement is possible pass else: _tiles_ints = SolutionChecker.np_array_to_tiles( _tiles_ints) solution_index = _tiles_ints.index(solution_tile_dims) if scalar_tiles: if INDIVIDUAL_TILES: split_tiles = np.array(tiles) split_tiles = np.split(split_tiles, split_tiles.shape[0]) else: split_tiles = tiles example = [ grid.copy(), split_tiles, one_hot_encode(_tiles_ints, solution_tile_dims, len(tiles)) ] else: example = [ state, one_hot_encode(_tiles_ints, solution_tile_dims, state.shape[2] - 1) ] examples.append(example) # print(_tiles_ints, solution_tile_dims, one_hot_encode(_tiles_ints, solution_tile_dims, state)) else: example = [state, pi] if predict_v: example.append(v) examples.append(example) success, grid = SolutionChecker.place_element_on_grid_given_grid( solution_tile[:ORIENTATIONS], solution_tile[2], val=1, grid=grid, cols=width, rows=height) return examples
def main(options): #N_TILES = 8 #HEIGHT = 8 #WIDTH = 8 HEIGHT = 25 WIDTH = 25 N_TILES = 50 for (WIDTH, HEIGHT) in [(30, 30)]: #for N_TILES in [50]: SCALAR_TILES = True predict_move_index = True g = Game(HEIGHT, WIDTH, N_TILES) dg = DataGenerator(WIDTH, HEIGHT) # from alpha.binpack.tensorflow.NNet import NNetWrapper as nn from alpha.binpack.keras.NNet import NNetWrapper as nn nnet = nn(g, scalar_tiles=SCALAR_TILES) n_tiles, height, width = N_TILES, HEIGHT, WIDTH if options['load_model']: nnet.load_checkpoint() else: # place tiles one by one # generate pair x and y where x is stack of state + tiles print('Preparing examples') N_EXAMPLES = 1000 examples = get_n_examples(N_EXAMPLES, width, height, n_tiles, dg, scalar_tiles=SCALAR_TILES) if options['load_examples']: with open( f'models/train_examples_{N_TILES}_{HEIGHT}_{WIDTH}.pickle', 'rb') as f: train_examples = pickle.load(f) else: train_examples = get_examples(examples, N_TILES, height, width, dg, return_binary_mask=True, predict_move_index=True, scalar_tiles=SCALAR_TILES) with open( f'models/train_examples_{N_TILES}_{HEIGHT}_{WIDTH}.pickle', 'wb') as f: pickle.dump(train_examples, f) if options['load_val_examples']: with open( f'models/validation_examples_{N_TILES}_{HEIGHT}_{WIDTH}.pickle', 'rb') as f: validation_examples = pickle.load(f) else: N_EXAMPLES = 100 validation_examples = get_n_examples(N_EXAMPLES, width, height, n_tiles, dg, scalar_tiles=SCALAR_TILES) validation_examples = get_examples(validation_examples, N_TILES, height, width, dg, from_file=False, return_binary_mask=True, predict_move_index=True, scalar_tiles=SCALAR_TILES, shuffle_tiles_times=1) validation_examples = get_val_examples_not_in_overlap( train_examples, validation_examples) with open( f'models/validation_examples_{N_TILES}_{HEIGHT}_{WIDTH}.pickle', 'wb') as f: pickle.dump(validation_examples, f) if not options['load_model']: print(f'In total: {len(train_examples)} train examples') print(f'In total: {len(validation_examples)} validation examples') if options['load_model']: nnet.load_checkpoint() else: nnet.train(train_examples, validation_examples) nnet.save_checkpoint() np.set_printoptions( formatter={'float': lambda x: "{0:0.2f}".format(x)}, linewidth=115) total_correct = 0 total_random_correct = 0 total_max_col_correct = 0 total_max_row_correct = 0 total_biggest_tile_correct = 0 total_smallest_tile_correct = 0 total_most_common_tile_correct = 0 total_count = 0 n_empty_tiles_with_fails = [0] * (N_TILES + 1) if False: # overlap was 39/1868 between val and train get_overlap_between_examples(train_examples, validation_examples) return if False: get_only_random_predictions(validation_examples) return for example in validation_examples: prediction = nnet.predict([example[0], example[1]]) random_prediction = random.randint( 0, SolutionChecker.get_n_nonplaced_tiles(example[1]) - 1) output_str = '' if VISUALIZE_PREDICTIONS: output_str += f'----------------------------------------------------------' output_str += '\n' if predict_move_index: _prediction = prediction if VISUALIZE_PREDICTIONS: output_str += f'{prediction}\n' max_index = np.argmax(prediction) _prediction_index = max_index if SCALAR_TILES: expected = np.argmax(example[2]) else: expected = np.argmax(example[1]) # if not scalar_tiles: # print('grid state') # print(example[0][:, :, 0]) # print(state_to_tiles_dims(example[0], dg)) # print('expected') if SCALAR_TILES: expected_tile = example[1][expected] prediction_tile = example[1][_prediction_index] if VISUALIZE_PREDICTIONS: output_str += f'{example[1].tolist()}\n' else: expected_tile = dg.get_matrix_tile_dims( example[0][:, :, expected + 1]) prediction_tile = dg.get_matrix_tile_dims( example[0][:, :, _prediction_index + 1]) # print(expected, expected_tile) #print('prediction') # print(_prediction) #print(_prediction_index, prediction_tile) if VISUALIZE_PREDICTIONS: output_str += f'{example[0]}\n' output_str += f'expected: {expected_tile}, got: {prediction_tile}' output_str += f'random: {example[1][random_prediction]}' if SCALAR_TILES: widest_tile = example[1][0] highest_tile = example[1][0] biggest_tile = example[1][0] smallest_tile = example[1][0] counter = Counter() for i, tile in enumerate(example[1]): if tile[1] > widest_tile[1]: widest_tile = tile elif tile[1] == widest_tile[1]: if tile[0] > widest_tile[0]: widest_tile = tile if tile[0] > highest_tile[0]: highest_tile = tile elif tile[0] == highest_tile[0]: if tile[1] > highest_tile[1]: highest_tile = tile if tile[1] * tile[0] > (biggest_tile[1] * biggest_tile[0]): biggest_tile = tile if tile[1] * tile[0] < (smallest_tile[1] * smallest_tile[0]): smallest_tile = tile counter[tuple(tile.tolist())] += 1 if np.array_equal(expected_tile, widest_tile): total_max_col_correct += 1 if np.array_equal(expected_tile, highest_tile): total_max_row_correct += 1 if np.array_equal(expected_tile, biggest_tile): total_biggest_tile_correct += 1 if np.array_equal(expected_tile, smallest_tile): total_smallest_tile_correct += 1 most_common_tile = np.array(counter.most_common(1)[0][0]) if np.array_equal(most_common_tile, np.array([0, 0])): most_common_tile = np.array( counter.most_common(2)[1][0]) if np.array_equal(expected_tile, most_common_tile): total_most_common_tile_correct += 1 if VISUALIZE_PREDICTIONS: output_str += f'max_tile: {widest_tile}\n' if np.array_equal(expected_tile, prediction_tile): total_correct += 1 # visualize predictions #if not np.array_equal(expected_tile, widest_tile) and VISUALIZE_PREDICTIONS: # print(output_str) if VISUALIZE_PREDICTIONS: print(output_str) else: n_empty_tiles_with_fails[count_n_of_non_placed_tiles( example[1]) // 2] += 1 # print(example[1][random_prediction]) if np.array_equal(expected_tile, example[1][random_prediction]): total_random_correct += 1 else: if expected_tile == prediction_tile: total_correct += 1 else: n_empty_tiles_with_fails[count_n_of_non_placed_tiles( state_to_tiles_dims(example[0], dg)) // 2] += 1 total_count += 1 else: _prediction = np.reshape(prediction, (width, height)) _prediction = get_prediction_masked(_prediction, example[0][:, :, 0]) expected = np.reshape(example[1], (width, height)) if VISUALIZE_PREDICTIONS: # visualize predictions # print('-' * 50) # print('grid state') # print(example[0][:, :, 0]) # print('expected') # print(expected) # print('prediction') # print(_prediction) pass if predict_move_index: with open(f"nn_results/custom_{N_TILES}_{width}_{height}.csv", 'w') as f: output_str = ( f'{width}-{height}\n' f'In total guessed;{100*(total_correct/ total_count)}; {total_correct}/{total_count}\n' f'Random baseline; {100*(total_random_correct/total_count)}\n' f'Max col tile baseline; {100*(total_max_col_correct/total_count)}\n' f'Max row tile baseline; {100*(total_max_row_correct/total_count)}\n' f'Most common tile baseline; {100*(total_most_common_tile_correct/total_count)}\n' f'Max area tile baseline; {100*(total_biggest_tile_correct/total_count)}\n' f'Min area tile baseline; {100*(total_smallest_tile_correct/total_count)}' ) f.write(output_str) print(output_str) print(n_empty_tiles_with_fails) print('-' * 100) if not PREDICT_FULL_EXAMPLES: # return continue tiles_left = [] for example in examples: if SCALAR_TILES: state, tiles, _ = example tiles = get_tiles_with_orientation(tiles.tolist()) else: tiles, _ = example # tiles = dg.get_matrix_tile_dims(tiles) grid = np.zeros((width, height)) tiles_left.append( play_using_prediction(nnet, width, height, tiles, grid, N_TILES, dg, predict_move_index, scalar_tiles=SCALAR_TILES)) # [0, 6, 4, 2, 2, 2, 0, 4, 4, 8, 6, 2, 2, 6, 6, 8, 6, 4, 4, 4] print(tiles_left) print(np.sum(tiles_left) / len(tiles_left))