def train(verbose=True,doall=False):
data=json.load(open("evals.json"))
global train_set_size
train_set_size=len(data)
fens=list(data.keys())
if not doall:
random.shuffle(fens)
totalloss=0
al=0
for fen,i in zip(fens,range(len(data))):
if i < min(MAX_TRAIN_SIZE,len(data)) or doall:
attrs=data[fen]
board.setFromFen(fen)
score=true_score(attrs["score"])
sess.run(trainjob, {X:board.inputsrow, Y:[[score]]})
actualloss=sess.run(loss, {X:board.inputsrow, Y:[[score]]})
totalloss+=actualloss
n=i+1
al=avg_loss(totalloss,n)
if verbose:
if doall and (i%100)!=0:
pass
else:
pf("{0:5d}. avg loss: {1:20f}".format(n,al))
return al
def load_config():
global config
global learning_rate
try:
config=json.load(open(CONFIG_PATH))
learning_rate=config.get("learningrate",learning_rate)
pf(json.dumps(config,indent=1))
except Exception:
pf("no config detected")
def load_model():
if Path(STORE_ENGINE_WEIGHTS_DIR).is_dir():
try:
saver.restore(sess, STORE_ENGINE_WEIGHTS_PATH)
pf("engine model loaded ok")
except Exception:
reset_model()
else:
pf("no stored engine model")
rand()
def epoch(n):
global latest_trains
for i in range(n):
t=train(False)
latest_trains=latest_trains[1:]
latest_trains.append(t)
pf("{0:4d} : {1:10.6f} [ {2:10.6f} ]".format(i,t,calc_latest_trains_moving_average()))
if model_has_nan():
break
if ((i+1)%10) == 0:
save_model()
pf("training set size {0:d}".format(train_set_size))
save_model()
def load(self):
X = []
Y = []
t0 = time.time()
for i, (x, y) in enumerate(zip(self._dask_X, self._dask_Y)):
if len(self._indices[i]) == 0:
if x.shape[0] > self._pr_category:
start_index = 0 if np.random.choice([True, False]) else x.shape[0] % self._pr_category
indices = range(start_index, x.shape[0], self._pr_category)
np.random.shuffle(indices)
self._indices[i] = indices
else:
self._indices[i] = [0]
j = self._indices[i].pop()
X.extend([ values for index, values in x[j:j+self._pr_category].compute() ])
Y.extend(y[j:j+self._pr_category])
X = np.vstack(X)
Y = np.squeeze(np.vstack(Y))
print '\t{} vectors read from disk in {} seconds.'.format(X.shape[0], pf(time.time()-t0))
return X, Y
def load(self):
X = []
Y = []
t0 = time.time()
for i, (x, y) in enumerate(zip(self._dask_X, self._dask_Y)):
if len(self._indices[i]) == 0:
if x.shape[0] > self._pr_category:
start_index = 0 if np.random.choice(
[True, False]) else x.shape[0] % self._pr_category
indices = range(start_index, x.shape[0], self._pr_category)
np.random.shuffle(indices)
self._indices[i] = indices
else:
self._indices[i] = [0]
j = self._indices[i].pop()
X.extend([
values
for index, values in x[j:j + self._pr_category].compute()
])
Y.extend(y[j:j + self._pr_category])
X = np.vstack(X)
Y = np.squeeze(np.vstack(Y))
print '\t{} vectors read from disk in {} seconds.'.format(
X.shape[0], pf(time.time() - t0))
return X, Y
def save_model():
if model_has_nan():
pf("model have nan, refused to save")
load_model()
else:
pf(saver.save(sess, STORE_ENGINE_WEIGHTS_PATH))
pf("engine model saved ok")
def test():
pf("test")
for key in model:
size=sess.run(tf.size(model[key]))
sumsq=sess.run(tf.reduce_sum(tf.square(model[key])))
pf("model {0:6s} : exp {1:10d} , act {2:10.2f} , rat {3:2.4f}".format(key,size,sumsq,sumsq/size))
pf("pos value {0:f}".format(calc_pos_value(board.fen)))
def reset_model():
pf("resetting model")
try:
shutil.rmtree(STORE_ENGINE_WEIGHTS_DIR)
pf("tree removed ok")
except Exception:
pf("tree was not found")
rand()
def play_move(data):
global smart
selsan=""
bestvalue=INFINITE
if smart==1:
for item in data:
fen=item["fen"]
actualvalue=calc_pos_value(fen)
if actualvalue < bestvalue:
selsan=item["san"]
bestvalue=actualvalue
pf("wise move {0:8s} value {1:20.10f} mat {2:d}".format(selsan,bestvalue,board.mat_balance()))
time.sleep(0.3)
else:
item=random.choice(data)
selsan=item["san"]
pf("random move "+selsan)
time.sleep(0.5)
movejsonstr=json.dumps({ "action" : "makesan" , "san" : selsan })
pf(movejsonstr)
def set_smart(s):
global smart
smart=s
pf("smart set to {0:d}".format(smart))
def save_config(): global config with open(CONFIG_PATH, 'w') as outfile: json.dump(config, outfile) pf(json.dumps(config,indent=1))
def learn(data, model_name, learning_rate, save_every, batch_size, hidden_size,
dropout, epochs, print_every, model_dir, network, filter_width,
depth, q_size, gpu_memory_ratio):
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=gpu_memory_ratio)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
q_x_in = tf.placeholder(tf.float32,
shape=[batch_size, data.train.X_features])
q_y_in = tf.placeholder(tf.int32, shape=[batch_size])
y_real = tf.placeholder(tf.int32, shape=[None])
keep_prob = tf.placeholder_with_default([1.], shape=None)
q = tf.FIFOQueue(q_size, [tf.float32, tf.int32],
shapes=[q_x_in.get_shape(),
q_y_in.get_shape()])
enqueue_op = q.enqueue([q_x_in, q_y_in])
q_x_out, q_y_out = q.dequeue()
x = tf.placeholder_with_default(q_x_out,
shape=[None, data.train.X_features],
name='input')
y_ = tf.placeholder_with_default(q_y_out, shape=[None])
if network == 'perceptron':
W = weight_variable([data.train.X_features, data.train.Y_features])
b = bias_variable([data.train.Y_features])
logits = tf.matmul(x, W) + b
if network == 'dense':
W_in = weight_variable([data.train.X_features, hidden_size])
b_in = bias_variable([hidden_size])
hidden = tf.matmul(x, W_in) + b_in
relu = tf.nn.relu(hidden)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
W_out = weight_variable([hidden_size, data.train.Y_features])
b_out = bias_variable([data.train.Y_features])
logits = tf.matmul(hidden_dropout, W_out) + b_out
if network == 'lenet':
w1 = weight_variable([1, filter_width, 1, depth])
b1 = bias_variable([depth])
x_4d = tf.expand_dims(tf.expand_dims(
x, 1), -1) # Singleton dimension height, out_channel
conv = tf.nn.conv2d(x_4d, w1, strides=[1, 1, 1, 1], padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, b1))
pool = tf.nn.max_pool(relu,
ksize=[1, 1, 2, 1],
strides=[1, 1, 2, 1],
padding='SAME')
w2 = weight_variable([1, filter_width, depth, depth * 2])
b2 = bias_variable([depth * 2])
conv = tf.nn.conv2d(pool, w2, strides=[1, 1, 1, 1], padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, b2))
pool = tf.nn.max_pool(relu,
ksize=[1, 1, 2, 1],
strides=[1, 1, 2, 1],
padding='SAME')
pool_shape = tf.shape(pool)
reshape = tf.reshape(
pool,
[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
w3 = weight_variable([(data.train.X_features / 4) * 2 * depth,
hidden_size])
b3 = bias_variable([hidden_size])
hidden = tf.matmul(reshape, w3) + b3
relu = tf.nn.relu(hidden)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
w4 = weight_variable([hidden_size, data.train.Y_features])
b4 = bias_variable([data.train.Y_features])
logits = tf.matmul(hidden_dropout, w4) + b4
# Loss & train
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=y_)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(
cross_entropy)
# Evaluation
y = tf.nn.softmax(logits)
train_correct_prediction = tf.equal(
tf.to_int32(tf.argmax(y, 1)),
y_,
)
train_accuracy = tf.reduce_mean(
tf.cast(train_correct_prediction, tf.float32))
# Applying convolutional filter to put subcategories into original categories for testing
stride = [1, 1, 1, 1]
_filter = tf.constant(data.output_filter,
dtype=tf.float32,
shape=data.output_filter.shape)
conv_in = tf.expand_dims(y, 0)
conv_in = tf.expand_dims(conv_in, -1)
conv_out = tf.nn.conv2d(conv_in, _filter, stride,
'VALID') # We don't want zero padding.
back = tf.squeeze(conv_out, squeeze_dims=[0, 2], name='output')
test_correct_prediction = tf.equal(tf.to_int32(tf.argmax(back, 1)),
y_real)
test_accuracy = tf.reduce_mean(
tf.cast(test_correct_prediction, tf.float32))
sess.run(tf.initialize_all_variables())
def load_data():
try:
while True:
next_x, next_y = data.train.next_batch(batch_size)
if next_x.shape[0] == batch_size:
sess.run(enqueue_op,
feed_dict={
q_x_in: next_x,
q_y_in: next_y
})
except Exception as error:
print(error)
print('Stopped streaming of data.')
data_thread = threading.Thread(target=load_data)
data_thread.daemon = True
data_thread.start()
last_epoch = 0
epoch = 0
t_epoch = time.time()
t_end = []
i = 0
while epoch <= epochs:
t_start = time.time()
# batch_x, batch_y = data.train.next_batch(batch_size)
# sess.run(train_step, feed_dict={keep_prob: dropout, x: batch_x, y_: batch_y})
sess.run(train_step, feed_dict={keep_prob: dropout})
t_end.append(time.time() - t_start)
if epoch > last_epoch:
if epoch % print_every == 0:
batch_x, batch_y = data.train.next_batch(batch_size)
train_accuracy_mean = train_accuracy.eval(feed_dict={
x: batch_x,
y_: batch_y
})
validation_accuracy_mean = np.mean([
test_accuracy.eval(feed_dict={
x: t_x,
y_real: t_y
}) for t_x, t_y in zip(chunks(data.test.X, batch_size),
chunks(data.test.Y, batch_size))
])
print(
'''Epoch {} train accuracy: {}, test accuracy: {}. '''
'''{} states/sec on average, {} secs/epoch.'''.format(
epoch, pf(train_accuracy_mean),
pf(validation_accuracy_mean),
pf(batch_size / np.mean(t_end)),
pf(time.time() - t_epoch)))
if epoch % save_every == 0 or epoch == epochs:
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ['input', 'output'])
with gfile.FastGFile(os.path.join(model_dir, model_name),
'w') as f:
f.write(output_graph_def.SerializeToString())
t_epoch = time.time()
t_end = []
last_epoch = epoch
i += 1
if batch_size * i > data.train.X_len:
epoch += 1
i = 0
q.close(cancel_pending_enqueues=True)
print('Trained model saved to {}'.format(
os.path.join(model_dir, model_name)))
def evaluate(model, h5_files, top_k, categories, out_file, index_to_grapheme):
h5_files = sorted(h5_files)
print('Evaluating {}'.format(model))
print('NOTE: ALL NUMBER TRIPLES ARE ON THE FORM (mean, median, standard deviation)')
model = load_model(model)#, custom_objects={'moe_loss': moe_loss})
all_accuracies = []
all_top_k_accuracy = []
all_top_level_accuracy = []
stats = []
with pd.HDFStore(out_file, mode='w', complevel=9, complib='blosc') as store:
for target, h5 in enumerate(h5_files):
text_inputs = pd.read_hdf(h5, 'text').values
text_inputs = text_inputs[:,:args.seq_len]
visual_inputs = pd.read_hdf(h5, 'visual')
predictions = model.predict_on_batch([ text_inputs, visual_inputs ])
index = visual_inputs.index
correct = np.argmax(predictions, axis=1) == target
accuracy = np.mean(correct)
category_i = os.path.basename(h5).replace('.h5','')
if 'parent' in categories[category_i]:
top_level_accuracy = np.mean([ 'parent' in categories[os.path.basename(h5_files[prediction]).replace('.h5','')] and
categories[category_i]['parent'] == categories[os.path.basename(h5_files[prediction]).replace('.h5','')]['parent']
for prediction in np.argmax(predictions, axis=1) ])
else:
top_level_accuracy = accuracy
top_k_accuracy = np.mean([ target in np.argsort(prediction)[-top_k:]
for prediction in predictions ])
correct_scores = np.max(predictions[correct], axis=1)
correct_x = np.linspace(0,1, num=len(correct_scores))
correct_confidence = np.mean(correct_scores)
correct_confidence_median = np.median(np.max(predictions[correct], axis=1))
correct_confidence_std = np.std(np.max(predictions[correct], axis=1))
correct_text = text_inputs[correct]
category = categories[category_i]['english']
sorted_correct = sorted(zip(correct_scores, index[correct], correct_text),
key=lambda x: x[0])
sorted_correct_scores, sorted_correct_paths, sorted_correct_text = zip(*sorted_correct)
sorted_correct_text = to_text(sorted_correct_text, index_to_grapheme)
df = pd.DataFrame(data=zip(sorted_correct_scores, sorted_correct_text), index=sorted_correct_paths, columns=['score', 'text'])
df.index.name = 'ad_id'
store.append('{}/correct'.format(category_i), df)
wrong_scores = np.max(predictions[~correct], axis=1)
wrong_categories_i = np.argmax(predictions[~correct], axis=1)
wrong_x = np.linspace(0,1, num=len(wrong_scores))
wrong_confidence = np.mean(wrong_scores)
wrong_confidence_median = np.median(np.max(predictions[~correct], axis=1))
wrong_confidence_std = np.std(np.max(predictions[~correct], axis=1))
wrong_text = text_inputs[~correct]
wrong_categories = [ os.path.basename(h5_files[i]).replace('.h5','')
for i in wrong_categories_i ]
sorted_wrong = sorted(zip(wrong_scores, index[~correct], wrong_categories, wrong_text),
key=lambda x: x[0])
sorted_wrong_scores, sorted_wrong_paths, sorted_wrong_categories, sorted_wrong_text = zip(*sorted_wrong)
sorted_wrong_text = to_text(sorted_wrong_text, index_to_grapheme)
df = pd.DataFrame(data=zip(sorted_wrong_scores, sorted_wrong_categories, sorted_wrong_text),
index=sorted_wrong_paths, columns=['score', 'category', 'text'])
df.index.name='ad_id'
store.append('{}/wrong/out'.format(category_i), df)
spread = defaultdict(list)
for score, path, wrong_category, wrong_text in zip(sorted_wrong_scores, sorted_wrong_paths, sorted_wrong_categories, sorted_wrong_text):
spread[wrong_category].append((path, score, wrong_text))
for wrong_category, X in spread.items():
paths, scores, text = zip(*X)
df = pd.DataFrame(data=zip(scores, [category_i]*len(paths), text), index=paths, columns=['score', 'category', 'text'])
df.index.name='ad_id'
store.append('{}/wrong/in'.format(wrong_category), df, min_itemsize={'index': 79, 'category': 12, 'text': 193})
print('Category {}, {} images. \t accuracy: {} top level accuracy: {} top_{} accuracy: {} '
'correct confidence: {}, {}, {} wrong confidence: {}, {}, {} '
'diff: {}, {}, {}'
''.format(category_i, len(index), pf(accuracy), pf(top_level_accuracy),
top_k,
pf(top_k_accuracy),
pf(correct_confidence),
pf(correct_confidence_median),
pf(correct_confidence_std),
pf(wrong_confidence),
pf(wrong_confidence_median),
pf(wrong_confidence_std),
pf(correct_confidence - wrong_confidence),
pf(correct_confidence_median - wrong_confidence_median),
pf(correct_confidence_std - wrong_confidence_std)
))
all_accuracies.append(accuracy)
all_top_k_accuracy.append(top_k_accuracy)
all_top_level_accuracy.append(top_level_accuracy)
stats.append([ category, pf(accuracy), pf(top_k_accuracy), pf(correct_confidence),
wrong_categories, wrong_scores, index[~correct] ])
results_index = [ 'test_len', 'train_len', 'accuracy', 'top_k_accuracy', 'k' ]
values = [ len(index), len(index)*4, accuracy, top_k_accuracy, top_k ]
stats = pd.DataFrame(data=values, index=results_index, columns=[category])
store.append('{}/stats'.format(category_i), stats)
mean_accuracy = pf(np.mean(all_accuracies))
top_k_accuracy = pf(np.mean(all_top_k_accuracy))
top_level_accuracy = pf(np.mean(all_top_level_accuracy))
print('Average accuracy across categories: {}'.format(mean_accuracy))
print('Average top_{} accuracy across categories: {}'.format(top_k, top_k_accuracy))
print('Average top level accuracy across categories: {}'.format(top_level_accuracy))
return mean_accuracy, top_k_accuracy, stats
def learn(train_states, test_states, learning_rate, save_every, batch_size, hidden_size, dropout, epochs,
print_every, model_dir, perceptron, dense, lenet, filter_width, depth, mem_ratio,
q_size, use_dask, in_memory, dask_chunksize):
data = read_data(train_states, test_states, use_dask, in_memory, dask_chunksize)
model_name = ('''transfer_classifier_epochs_{}_batch_{}_learning_rate_{}'''.format(
epochs, batch_size, learning_rate))
if perceptron:
model_name = '{}_perceptron.pb'.format(model_name)
if dense:
model_name = '{}_dense_dropout_{}_hidden_size_{}.pb'.format(model_name, dropout, hidden_size)
if lenet:
model_name = '{}_lenet_dropout_{}_hidden_size_{}.pb'.format(model_name, dropout, hidden_size)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
q_x_in = tf.placeholder(tf.float32, shape=[batch_size, data.train.X_features])
q_y_in = tf.placeholder(tf.int32, shape=[batch_size])
y_real = tf.placeholder(tf.int32, shape=[None])
keep_prob = tf.placeholder_with_default([1.], shape=None)
q = tf.FIFOQueue(q_size, [tf.float32, tf.int32],
shapes=[ q_x_in.get_shape(), q_y_in.get_shape()])
enqueue_op = q.enqueue([q_x_in, q_y_in])
q_x_out, q_y_out = q.dequeue()
x = tf.placeholder_with_default(q_x_out, shape=[None, data.train.X_features], name='input')
y_ = tf.placeholder_with_default(q_y_out, shape=[None])
if perceptron:
W = weight_variable([data.train.X_features, data.train.Y_features])
b = bias_variable([data.train.Y_features])
logits = tf.matmul(x, W) + b
if dense:
W_in = weight_variable([data.train.X_features, hidden_size])
b_in = bias_variable([hidden_size])
hidden = tf.matmul(x, W_in) + b_in
relu = tf.nn.relu(hidden)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
W_out = weight_variable([hidden_size,data.train.Y_features])
b_out = bias_variable([data.train.Y_features])
logits = tf.matmul(hidden_dropout, W_out) + b_out
if lenet:
w1 = weight_variable([1, filter_width, 1, depth])
b1 = bias_variable([depth])
x_4d = tf.expand_dims(tf.expand_dims(x,1),-1) # Singleton dimension height, out_channel
conv = tf.nn.conv2d(x_4d, w1, strides=[1,1,1,1], padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, b1))
pool = tf.nn.max_pool(relu, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='SAME')
w2 = weight_variable([1, filter_width, depth, depth*2])
b2 = bias_variable([depth*2])
conv = tf.nn.conv2d(pool, w2, strides=[1,1,1,1], padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, b2))
pool = tf.nn.max_pool(relu, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='SAME')
pool_shape = tf.shape(pool)
reshape = tf.reshape(pool,
[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3] ])
w3 = weight_variable([ (data.train.X_features/4)*2*depth, hidden_size])
b3 = bias_variable([hidden_size])
hidden = tf.matmul(reshape, w3) + b3
relu = tf.nn.relu(hidden)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
w4 = weight_variable([hidden_size, data.train.Y_features])
b4 = bias_variable([data.train.Y_features])
logits = tf.matmul(hidden_dropout, w4) + b4
# Loss & train
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y_)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)
# Evaluation
y = tf.nn.softmax(logits)
train_correct_prediction = tf.equal(tf.to_int32(tf.argmax(y, 1)), y_,)
train_accuracy = tf.reduce_mean(tf.cast(train_correct_prediction, tf.float32))
# Applying convolutional filter to put subcategories into original categories for testing
stride = [1,1,1,1]
_filter = tf.constant(data.output_filter, dtype=tf.float32, shape=data.output_filter.shape)
conv_in = tf.expand_dims(y, 0)
conv_in = tf.expand_dims(conv_in,-1)
conv_out = tf.nn.conv2d(conv_in, _filter, stride, 'VALID') # We don't want zero padding.
back = tf.squeeze(conv_out, squeeze_dims=[0,2], name='output')
test_correct_prediction = tf.equal(tf.to_int32(tf.argmax(back, 1)), y_real)
test_accuracy = tf.reduce_mean(tf.cast(test_correct_prediction, tf.float32))
sess.run(tf.initialize_all_variables())
def load_data():
try:
while True:
next_x, next_y = data.train.next_batch(batch_size)
if next_x.shape[0] == batch_size:
sess.run(enqueue_op, feed_dict={q_x_in: next_x, q_y_in: next_y})
except Exception as error:
print(error)
print('Stopped streaming of data.')
data_thread = threading.Thread(target=load_data)
data_thread.daemon = True
data_thread.start()
last_epoch = 0
epoch = 0
t_epoch = time.time()
t_end = []
i = 0
while epoch <= epochs:
t_start = time.time()
sess.run(train_step, feed_dict={keep_prob: dropout})
t_end.append(time.time() - t_start)
if epoch > last_epoch:
if epoch % print_every == 0:
batch_x, batch_y = data.train.next_batch(batch_size)
train_accuracy_mean = train_accuracy.eval(feed_dict={
x: batch_x,
y_: batch_y })
validation_accuracy_mean = np.mean([ test_accuracy.eval(feed_dict={x: t_x, y_real: t_y })
for t_x, t_y in zip(chunks(data.test.X, batch_size),
chunks(data.test.Y, batch_size)) ])
print('''Epoch {} train accuracy: {}, test accuracy: {}. '''
'''{} states/sec on average, {} secs/epoch.'''.format(epoch, pf(train_accuracy_mean),
pf(validation_accuracy_mean),
pf(batch_size/np.mean(t_end)),
pf(time.time() - t_epoch)))
if epoch % save_every == 0 or epoch == epochs:
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ['input', 'output'])
with gfile.FastGFile(os.path.join(model_dir, model_name), 'w') as f:
f.write(output_graph_def.SerializeToString())
t_epoch = time.time()
t_end = []
last_epoch = epoch
i += 1
if batch_size*i > data.train.X_len:
epoch += 1
i = 0
q.close(cancel_pending_enqueues=True)
print('Trained model saved to {}'.format(os.path.join(model_dir, model_name)))
import shutil
import time
from pathlib import Path
################################################
import utils
from utils import printflush as pf
from utils import INFINITE
import chess
################################################
# TensorFlow init
pf("neural initialization")
import tensorflow as tf
tf.reset_default_graph()
pf("\n------------\nneural started\n------------\n")
################################################
board = chess.Board()
################################################
config = {}
def set_learning_rate(lr):
learning_rate=lr
config["learningrate"]=learning_rate
pf("learning rate set to {0:f}".format(learning_rate))
save_config()
def rand():
pf("rand")
sess.run(init)
test()
elif(command=="savemodel"):
network.save_model()
elif(command=="resetmodel"):
network.reset_model()
elif(command=="dull"):
network.set_smart(0)
elif(command=="smart"):
network.set_smart(1)
elif(command=="trainall"):
for i in range(get_int(0,1)):
network.train(True,True)
elif(command=="lr"):
network.set_learning_rate(get_float(0,network.learning_rate))
else:
try:
c=json.loads(line)
action=c["action"]
if action=="playmove":
fens=c["fens"]
network.play_move(fens)
elif action=="epoch":
network.epoch(c["n"])
elif action=="test":
network.test()
else:
pf("unknown command")
except Exception:
pf("command error")
################################################
def evaluate(model, h5_files, top_k, categories, out_file, special,
num_images):
h5_files = sorted(h5_files)
try:
num_images = int(num_images)
except:
needle = 'images_'
substr = model[model.find(needle) + len(needle):]
num_images = substr[:substr.find('_')]
num_images = int(num_images)
with tf.Session() as sess:
print('Evaluating {}'.format(model))
print(
'NOTE: ALL NUMBER TRIPLES ARE ON THE FORM (mean, median, standard deviation)'
)
load_graph(model)
transfer_predictor = sess.graph.get_tensor_by_name('output:0')
all_accuracies = []
all_top_k_accuracy = []
all_top_level_accuracy = []
stats = []
with pd.HDFStore(out_file, mode='w', complevel=9,
complib='blosc') as store:
for target, h5 in enumerate(h5_files):
with pd.HDFStore(h5, mode='r') as in_store:
keys = sorted(in_store.keys())[:num_images]
if special:
all_predictions = defaultdict(list)
for key in keys:
X = pd.read_hdf(h5, key)
local_predictions = sess.run(transfer_predictor,
{'input:0': X})
for i, (prediction, x) in enumerate(
zip(local_predictions, X.values)):
if sum(x): # No image present equals all 0s
all_predictions[i].append(prediction)
predictions = np.vstack(
[np.mean(x, axis=0) for x in all_predictions.values()])
else:
X = np.hstack([pd.read_hdf(h5, key) for key in keys])
predictions = sess.run(transfer_predictor, {'input:0': X})
data = pd.read_hdf(
h5, key
) # We need the index, that is why we load the file again.
correct = np.argmax(predictions, axis=1) == target
accuracy = np.mean(correct)
category_i = os.path.basename(h5).replace('.h5', '')
if 'parent' in categories[category_i]:
top_level_accuracy = np.mean([
'parent' in categories[os.path.basename(
h5_files[prediction]).replace('.h5', '')]
and categories[category_i]['parent']
== categories[os.path.basename(
h5_files[prediction]).replace('.h5', '')]['parent']
for prediction in np.argmax(predictions, axis=1)
])
else:
top_level_accuracy = accuracy
top_k_accuracy = np.mean([
target in np.argsort(prediction)[-top_k:]
for prediction in predictions
])
correct_scores = np.max(predictions[correct], axis=1)
correct_x = np.linspace(0, 1, num=len(correct_scores))
correct_confidence = np.mean(correct_scores)
correct_confidence_median = np.median(
np.max(predictions[correct], axis=1))
correct_confidence_std = np.std(
np.max(predictions[correct], axis=1))
category = categories[category_i]['english']
sorted_correct = sorted(zip(correct_scores,
data.index[correct]),
key=lambda x: x[0])
sorted_correct_scores, sorted_correct_paths = zip(
*sorted_correct)
df = pd.DataFrame(data=list(sorted_correct_scores),
index=sorted_correct_paths,
columns=['score'])
df.index.name = 'ad_id'
store.append('{}/correct'.format(category_i), df)
wrong_scores = np.max(predictions[~correct], axis=1)
wrong_categories_i = np.argmax(predictions[~correct], axis=1)
wrong_x = np.linspace(0, 1, num=len(wrong_scores))
wrong_confidence = np.mean(wrong_scores)
wrong_confidence_median = np.median(
np.max(predictions[~correct], axis=1))
wrong_confidence_std = np.std(
np.max(predictions[~correct], axis=1))
wrong_categories = [
os.path.basename(h5_files[i]).replace('.h5', '')
for i in wrong_categories_i
]
sorted_wrong = sorted(zip(wrong_scores, data.index[~correct],
wrong_categories),
key=lambda x: x[0])
sorted_wrong_scores, sorted_wrong_paths, sorted_wrong_categories = zip(
*sorted_wrong)
df = pd.DataFrame(data=zip(sorted_wrong_scores,
sorted_wrong_categories),
index=sorted_wrong_paths,
columns=['score', 'category'])
df.index.name = 'ad_id'
store.append('{}/wrong/out'.format(category_i), df)
spread = defaultdict(list)
for score, path, wrong_category in sorted_wrong:
spread[wrong_category].append((path, score))
for wrong_category, X in spread.items():
paths, scores = zip(*X)
df = pd.DataFrame(data=zip(scores,
[category_i] * len(paths)),
index=paths,
columns=['score', 'category'])
df.index.name = 'ad_id'
store.append('{}/wrong/in'.format(wrong_category),
df,
min_itemsize={
'index': 79,
'category': 12
})
print(
'Category {}, {} images. \t accuracy: {} top level accuracy: {} top_{} accuracy: {} '
'correct confidence: {}, {}, {} wrong confidence: {}, {}, {} '
'diff: {}, {}, {}'
''.format(
category_i, len(data), pf(accuracy),
pf(top_level_accuracy), top_k, pf(top_k_accuracy),
pf(correct_confidence), pf(correct_confidence_median),
pf(correct_confidence_std), pf(wrong_confidence),
pf(wrong_confidence_median), pf(wrong_confidence_std),
pf(correct_confidence - wrong_confidence),
pf(correct_confidence_median -
wrong_confidence_median),
pf(correct_confidence_std - wrong_confidence_std)))
all_accuracies.append(accuracy)
all_top_k_accuracy.append(top_k_accuracy)
all_top_level_accuracy.append(top_level_accuracy)
stats.append([
category,
pf(accuracy),
pf(top_k_accuracy),
pf(correct_confidence), wrong_categories, wrong_scores,
data.index[~correct]
])
index = [
'test_len', 'train_len', 'accuracy', 'top_k_accuracy', 'k',
'num_images'
]
values = [
len(data),
len(data) * 4, accuracy, top_k_accuracy, top_k, num_images
]
stats = pd.DataFrame(data=values,
index=index,
columns=[category])
store.append('{}/stats'.format(category_i), stats)
mean_accuracy = pf(np.mean(all_accuracies))
top_k_accuracy = pf(np.mean(all_top_k_accuracy))
top_level_accuracy = pf(np.mean(all_top_level_accuracy))
print(
'Average accuracy across categories: {}'.format(mean_accuracy))
print('Average top_{} accuracy across categories: {}'.format(
top_k, top_k_accuracy))
print('Average top level accuracy across categories: {}'.format(
top_level_accuracy))
tf.reset_default_graph()
return mean_accuracy, top_k_accuracy, stats
def evaluate(model, h5_files, top_k, categories=None, out_file='stats.h5'):
h5_files = sorted(h5_files)
plotly_data = []
with tf.Session() as sess:
print('Evaluating {}'.format(model))
print('NOTE: ALL NUMBER TRIPLES ARE ON THE FORM (mean, median, standard deviation)')
load_graph(model)
transfer_predictor = sess.graph.get_tensor_by_name('output:0')
all_accuracies = []
all_top_k_accuracy = []
all_top_level_accuracy = []
stats = []
with pd.HDFStore(out_file, mode='w', complevel=9, complib='blosc') as store:
for target, h5 in enumerate(h5_files):
data = pd.read_hdf(h5)
category_i = os.path.basename(h5).replace('.h5','')
predictions = sess.run(transfer_predictor, { 'input:0': data })
top_level_accuracy = np.mean([ categories[category_i]['parent'] ==
categories[os.path.basename(h5_files[prediction]).replace('.h5','')]['parent']
for prediction in np.argmax(predictions, axis=1) ])
correct = np.argmax(predictions, axis=1) == target
accuracy = np.mean(correct)
top_k_accuracy = np.mean([ target in np.argsort(prediction)[-top_k:]
for prediction in predictions ])
correct_scores = np.max(predictions[correct], axis=1)
correct_x = np.linspace(0,1, num=len(correct_scores))
correct_confidence = np.mean(correct_scores)
correct_confidence_median = np.median(np.max(predictions[correct], axis=1))
correct_confidence_std = np.std(np.max(predictions[correct], axis=1))
category = categories[category_i]['name'] if categories else category_i
sorted_correct = sorted(zip(correct_scores, data.index[correct]),
key=lambda x: x[0])
sorted_correct_scores, sorted_correct_paths = zip(*sorted_correct)
df = pd.DataFrame(data=list(sorted_correct_scores), index=sorted_correct_paths, columns=['score'])
df.index.name = 'filename'
store.append('{}/correct'.format(category_i), df)
wrong_scores = np.max(predictions[~correct], axis=1)
wrong_categories_i = np.argmax(predictions[~correct], axis=1)
wrong_x = np.linspace(0,1, num=len(wrong_scores))
wrong_confidence = np.mean(wrong_scores)
wrong_confidence_median = np.median(np.max(predictions[~correct], axis=1))
wrong_confidence_std = np.std(np.max(predictions[~correct], axis=1))
wrong_categories = [ os.path.basename(h5_files[i]).replace('.h5','')
for i in wrong_categories_i ]
sorted_wrong = sorted(zip(wrong_scores, data.index[~correct], wrong_categories),
key=lambda x: x[0])
sorted_wrong_scores, sorted_wrong_paths, sorted_wrong_categories = zip(*sorted_wrong)
df = pd.DataFrame(data=zip(sorted_wrong_scores, sorted_wrong_categories),
index=sorted_wrong_paths, columns=['score', 'category'])
df.index.name='filename'
store.append('{}/wrong/out'.format(category_i), df)
spread = defaultdict(list)
for score, path, wrong_category in sorted_wrong:
spread[wrong_category].append((path, score))
for wrong_category, X in spread.items():
paths, scores = zip(*X)
df = pd.DataFrame(data=zip(scores, [category_i]*len(paths)), index=paths, columns=['score', 'category'])
df.index.name='filename'
store.append('{}/wrong/in'.format(wrong_category), df, min_itemsize={'index': 50})
# plotly_data.append(go.Scatter(
# x=wrong_x,
# y=sorted_wrong_scores,
# mode='markers',
# name=category,
# hoverinfo='name+y',
# text=[ json.dumps({ 'path': path, 'prediction': prediction }) for path, prediction in
# zip(sorted_wrong_paths, sorted_wrong_categories)]))
print('Category {}, {} images. \t accuracy: {} top level accuracy: {} top_{} accuracy: {} '
'correct confidence: {}, {}, {} wrong confidence: {}, {}, {} '
'diff: {}, {}, {}'
''.format(category_i, len(data), pf(accuracy), pf(top_level_accuracy),
top_k,
pf(top_k_accuracy),
pf(correct_confidence),
pf(correct_confidence_median),
pf(correct_confidence_std),
pf(wrong_confidence),
pf(wrong_confidence_median),
pf(wrong_confidence_std),
pf(correct_confidence - wrong_confidence),
pf(correct_confidence_median - wrong_confidence_median),
pf(correct_confidence_std - wrong_confidence_std)
))
all_accuracies.append(accuracy)
all_top_k_accuracy.append(top_k_accuracy)
all_top_level_accuracy.append(top_level_accuracy)
stats.append([ category, pf(accuracy), pf(top_k_accuracy), pf(correct_confidence),
wrong_categories, wrong_scores, data.index[~correct] ])
mean_accuracy = pf(np.mean(all_accuracies))
top_k_accuracy = pf(np.mean(all_top_k_accuracy))
top_level_accuracy = pf(np.mean(all_top_level_accuracy))
print('Average accuracy across categories: {}'.format(mean_accuracy))
print('Average top_{} accuracy across categories: {}'.format(top_k, top_k_accuracy))
print('Average top level accuracy across categories: {}'.format(top_level_accuracy))
tf.reset_default_graph()
return plotly_data, mean_accuracy, top_k_accuracy, stats