def __init__(self, datapath):
"""
Builds dictionary to store paths and create output directories in ``datapath`` if
they do not exist.
:param: datapath: path to data folder
:type: datapath: str
:raise: FileNotFoundError: if datapath is invalid
"""
if not os.path.exists(datapath):
raise FileNotFoundError(
"{} not found. "
"Please create /raw path and put chess data in there".format(
datapath))
self.paths_dict = {
'data': datapath,
'raw': oshelper.pathjoin(datapath, "raw"),
'processed': oshelper.pathjoin(datapath, "processed",
"processed.json"),
'arrays': oshelper.pathjoin(datapath, "arrays")
}
self.games = {'games': [], 'length': 0}
oshelper.create_if_not_exists(self.paths_dict['processed'],
is_file=True)
oshelper.create_if_not_exists(self.paths_dict['arrays'], is_file=False)
def current_run_directory(tmp_path, run_name='knight'):
"""
Sets correct tensorboard directory dynamically based on number of runs.
Delete tensorboard directory to start count over
"""
runpath = oshelper.pathjoin(tmp_path, run_name)
print(runpath)
oshelper.create_if_not_exists(runpath)
count = _incr_tb_run_number(runpath)
return oshelper.pathjoin(runpath, count)
def convert_to_arrays(self):
"""
Converts to two arrays, X and y.
X is the list of all chess positions in feature_list form
Y is the list of evaluations in the form [good for white, draw, good for black]
:return: X and y
"""
features, labels = [], []
color_dict = {color.white: 0, color.black: 1}
with open(self.paths_dict['processed'], 'r') as processed:
for i, line in enumerate(processed):
print("On game number {}".format(i))
if line[0:3] == "1/2": # Game is drawn
result = [0, 1, 0]
game_str = line[4:]
else: # Victor exists
result = [1, 0, 0] if int(line[0:1]) == 0 else [0, 0, 1]
game_str = line[2:]
print("Result of the game was {}".format(result))
move_list = game_str.split(' ')
print(move_list)
data_board = Board.init_default()
color_itr = itertools.cycle([color.white, color.black])
for move_str in move_list:
current_color = next(color_itr)
print(move_str)
print(data_board)
print(color_dict[current_color])
try:
move = converter.incomplete_alg(move_str, current_color)
if move is not None:
move = converter.make_legal(move, data_board)
except AttributeError as error:
print(error)
break
if move is None:
print("Move is None")
break
data_board.update(move)
features.append(extract_features(data_board))
labels.append(result)
np.save(oshelper.pathjoin(self.paths_dict['arrays'], 'features'), np.array(features))
np.save(oshelper.pathjoin(self.paths_dict['arrays'], 'labels'), np.array(labels))
return features, labels
def __init__(self, tmp_path, run_name='knight', run_tracking_file='count.txt'):
"""
Returns correct tensorboard directory which is set dynamically based on number of runs.
Delete tensorboard directory to start count over
"""
self._path = oshelper.pathjoin(tmp_path, run_name)
self._run_tracking_file_path = oshelper.pathjoin(self._path, run_tracking_file)
print(self._path)
if not os.path.exists(self._run_tracking_file_path):
oshelper.create_if_not_exists(self._run_tracking_file_path, is_file=True)
with open(self._run_tracking_file_path, 'w') as f:
f.write(str(1))
def __init__(self, datapath):
"""
Creates object that processes data and converts it to numpy arrays.
Requires original PGN files to be from FICS database and in ``/data/raw``.
Saves processed PGN in ``/data/processed`` and np arrays in ``/data/arrays``.
:param datapath: path to data folder
"""
if not os.path.exists(datapath):
raise FileNotFoundError("create /data/raw and put data in there")
self.paths_dict = {'data': datapath,
'raw': oshelper.pathjoin(datapath, "raw"),
'processed': oshelper.pathjoin(datapath, "processed", "processed.pgn"),
'arrays': oshelper.pathjoin(datapath, "arrays")}
oshelper.create_if_not_exists(self.paths_dict['processed'])
oshelper.create_if_not_exists(self.paths_dict['raw'])
def _incr_tb_run_number(runpath):
"""
Increments tensorboard count by 1 for the new run. If no runs are present, create the directory itself.
:param tbpath: path of tensorboard
:return: The number of runs plus 1 for use as the name of the new run
"""
countfilename = oshelper.pathjoin(runpath, 'count.txt')
if not os.path.exists(countfilename):
with open(countfilename, 'w') as f:
f.write('1')
return '1'
else:
with open(countfilename, 'r') as f:
current = int(f.readline())
with open(countfilename, 'w') as f:
f.write(str(current + 1))
return str(current + 1)
def __init__(self, tmp_path):
# Tensorboard Setup
self.tmp_path = tmp_path
self.save_path = oshelper.pathjoin(self.tmp_path, 'saved', 'model')
oshelper.create_if_not_exists(tmp_path)
self.tb_dir = tensorboardsetup.current_run_directory(tmp_path)
# Placholder initialization
self.X_placeholder = tf.placeholder(tf.float32,
[None, self.BOARD_SIZE],
name="X")
self.y_placeholder = tf.placeholder(tf.float32,
[None, self.NUMBER_OF_CLASSES],
name="y")
self.keep_prob_placeholder = tf.placeholder(tf.float32)
self.learning_rate = tf.placeholder(tf.float32)
# Model creation
self.optimizer = None
self.evaluate = None
self.accuracy = None
self._create_model()
def __init__(self, tmp_path):
# Tensorboard Setup
self.tmp_path = tmp_path
self.save_path = oshelper.pathjoin(self.tmp_path, 'saved',
'model.ckpt')
oshelper.create_if_not_exists(tmp_path, is_file=False)
self.tb_manager = TensorboardManager(tmp_path)
# Placholder initialization
self.X_placeholder = tf.placeholder(tf.float32,
[None, self.BOARD_SIZE],
name="X")
self.y_placeholder = tf.placeholder(tf.float32,
[None, self.NUMBER_OF_CLASSES],
name="y")
self.keep_prob_placeholder = tf.placeholder(tf.float32)
self.learning_rate = tf.placeholder(tf.float32)
# Model creation
self.optimizer = None
self.predict_op = None
self.accuracy = None
self._create_model()
def convert_to_arrays(self, label_type='material'):
"""
Converts to two arrays, X and y.
``features`` is the list of all chess positions in the form
[number of games, 64 (number of squares on the board)]
``labels`` is the list of evaluations in the form
[number of games, 3 (good for white, draw, good for black)]
``labels`` examples:
[0, 0, 1] if white is doing better
[1, 0, 0] if black is doing better
[0, 1, 0] if the game is even.
Saves output in ``data/arrays`` and returns it.
:return: features, labels
:rtype: tuple(np.array, np.array)
"""
with open(self.paths_dict['processed'], 'r') as f:
self.games = json.load(f)
features = []
labels = []
# resets every game
game_increment = 0
# decrements when Exception is raised
number_of_games = len(self.games['games'])
for i, game_dict in enumerate(self.games['games']):
data_board = Board.init_default()
print("On game number {} out of {}".format(i, number_of_games))
for move in game_dict['moves']:
if game_increment % 2 == 0:
current_color = color.white
else:
current_color = color.black
print("Raw: {}".format(move))
move = converter.incomplete_alg(move, current_color,
data_board)
move = converter.make_legal(move, data_board)
data_board.update(move)
features.append(
featurehelper.extract_features_from_position(
data_board))
if label_type == 'turn':
if current_color == color.white: # white has just moved
labels.append([1, 0, 0])
else: # black has just moved
labels.append([0, 0, 1])
elif label_type == 'result':
if int(game_dict['result']) == 0: # white wins
labels.append([1, 0, 0])
elif int(game_dict['result']) == 1: # black wins
labels.append([0, 0, 1])
else: # draw
labels.append([0, 1, 0])
elif label_type == 'material':
material_imbalance = np.sum(np.array(features[-1]))
if material_imbalance > 0: # white holds material advantage
labels.append([1, 0, 0])
elif material_imbalance < 0: # black holds material advantage
labels.append([0, 0, 1])
else: # material even
labels.append([0, 1, 0])
else:
raise ValueError(
"label_type {} is invalid "
"\nlabel_type must be \'turn\', \'result\', or \'material\'"
.format(label_type))
game_increment += 1
print(".", end='')
game_increment = 0
print()
print(data_board)
np.save(
oshelper.pathjoin(self.paths_dict['arrays'],
'features-{}'.format(label_type)),
np.array(features))
np.save(
oshelper.pathjoin(self.paths_dict['arrays'],
'labels-{}'.format(label_type)),
np.array(labels))
return np.array(features), np.array(labels)
def train_on(self,
data_or_path,
array_folder_name='arrays',
epochs=300,
batch_size=100,
learning_rate=0.5,
train_keep_prob=0.5,
test_keep_prob=1.0):
"""
Trains on data in the form of a tuple of np arrays stored as (X, y),
or the path to a folder containing 2 npy files
wih X named ``features.npy`` and y named ``labels.npy``.
:param data_or_path: training data
:type: str or np.array
:param array_folder_name: name of folder where arrays are stored. Defaults to ``arrays``
:param epochs: Number of epochs to run when training. Defaults to 300.
:param batch_size: Size of individual batches to
:param learning_rate:
:param train_keep_prob:
:param test_keep_prob:
"""
if isinstance(data_or_path, str):
features = np.load(
oshelper.pathjoin(data_or_path, array_folder_name,
'features.npy'))
labels = np.load(
oshelper.pathjoin(data_or_path, array_folder_name,
'labels.npy'))
else:
features, labels = data_or_path
train_features, test_features, train_labels, test_labels = randomly_assign_train_test(
features, labels)
saver = tf.train.Saver()
with tf.Session() as sess:
print("Session starting")
# Initialization
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(self.tb_dir)
writer.add_graph(sess.graph)
sess.run(tf.global_variables_initializer())
# Training loop
for epoch in range(epochs):
for i, (batch_X, batch_y) in \
enumerate(next_batch(train_features, train_labels, batch_size=batch_size)):
# Dict fed to train model
train_dict = {
self.X_placeholder: batch_X,
self.y_placeholder: batch_y,
self.keep_prob_placeholder: train_keep_prob,
self.learning_rate: learning_rate
}
sess.run(self.optimizer, feed_dict=train_dict)
# Write to tensorboard
if i % 5 == 0:
s = sess.run(merged_summary, feed_dict=train_dict)
writer.add_summary(s, i)
accuracy_dict = {
self.X_placeholder: train_features,
self.y_placeholder: train_labels,
self.keep_prob_placeholder: train_keep_prob
}
print("Train accuracy {}".format(
self.accuracy.eval(accuracy_dict)))
accuracy_dict[self.keep_prob_placeholder] = test_keep_prob
print("Test accuracy {}".format(
self.accuracy.eval(accuracy_dict)))
saver.save(sess, self.save_path)
with tf.Session() as sess:
saver.restore(sess, self.save_path)
# self.evaluate = tf.get_collection('evaluate')[0]
advantages = self.predict_op.eval(feed_dict={
self.X_placeholder: positions,
self.keep_prob_placeholder: 1.0
})
print("Position evaluation is {}".format(advantages))
return advantages
if __name__ == '__main__':
tmp_folder_path = os.path.join(ROOT_DIR, 'tmp')
data_path = os.path.join(ROOT_DIR, 'data')
features = np.load(os.path.join(data_path, 'arrays',
'features-result.npy'))
labels = np.load(
oshelper.pathjoin(data_path, 'arrays', 'labels-result.npy'))
train_features, test_features, train_labels, test_labels = split.randomly_assign_train_test(
features, labels)
evaluator = BoardEvaluator(tmp_folder_path)
evaluator.fit(training_data=(features, labels),
testing_data=(test_features, test_labels),
epochs=20,
learning_rate=0.01)
evaluator.predict(test_features)