def __init__(self, filename):
# must have self.graph valid at end of constructor
# sqlite open file
self.conn = sqlite3.connect(filename)
fresh_file = not table_exists(self.conn,
'config') # before making ConfigDict
self.config = ConfigDict(self.conn)
datastore = kvstore.KVStore(self.conn, V2_TABLENAME)
# check for config format version
version = self.config['version']
if fresh_file:
print 'fresh file'
self.graph = model.Graph(datastore, None)
self.load_default_config()
self.commit()
else:
if self.config['version'] is None:
# if no conf, migrate by creating empty graph, creating cards
# and loading it from a v1 DataStore
self.graph = model.Graph(datastore, None)
self.import_v1()
# else, load commit
elif version == '2':
head_ptr = self.config['head']
if head_ptr is None:
raise CorruptionError('No head pointer!')
try:
self.graph = model.Graph(datastore, head_ptr)
# after this, should be all loaded
except model.Error as e:
print 'failed to open gp file:', e
raise ValueError
def acc():
g = model.Graph()
with tf.Session(graph=g.graph) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print('restore from ckpt{}'.format(ckpt))
else:
print('cannot restore')
val_feeder = utils.DataIterator(data_dir=inferFolder)
val_inputs, val_seq_len, val_labels = val_feeder.input_index_generate_batch(
)
val_feed = {
g.inputs: val_inputs,
g.labels: val_labels,
g.seq_len: np.array([27] * val_inputs.shape[0])
}
dense_decoded = sess.run(g.dense_decoded, val_feed)
# print the decode result
acc = utils.accuracy_calculation(val_feeder.labels,
dense_decoded,
ignore_value=-1,
isPrint=True)
print(acc)
def master(train_data, dev_data, utility):
#creates TF graph and calls trainer or evaluator
batch_size = utility.FLAGS.batch_size
model_dir = utility.FLAGS.output_dir + "/model" + utility.FLAGS.job_id + "/"
#create all paramters of the model
param_class = parameters.Parameters(utility)
params, global_step, init = param_class.parameters(utility)
key = "test" if (FLAGS.evaluator_job) else "train"
graph = model.Graph(utility,
batch_size,
utility.FLAGS.max_passes,
mode=key)
graph.create_graph(params, global_step)
prev_dev_error = 0.0
final_loss = 0.0
final_accuracy = 0.0
#start session
with tf.Session() as sess:
sess.run(init.name)
sess.run(graph.init_op.name)
to_save = params.copy()
saver = tf.train.Saver(to_save, max_to_keep=500)
if (FLAGS.evaluator_job):
while True:
selected_models = {}
file_list = tf.gfile.ListDirectory(model_dir)
for model_file in file_list:
if ("checkpoint" in model_file or "index" in model_file
or "meta" in model_file):
continue
if ("data" in model_file):
model_file = model_file.split(".")[0]
model_step = int(
model_file.split("_")[len(model_file.split("_")) - 1])
selected_models[model_step] = model_file
file_list = sorted(list(selected_models.items()),
key=lambda x: x[0])
if (len(file_list) > 0):
file_list = file_list[0:len(file_list) - 1]
print(("list of models: ", file_list))
for model_file in file_list:
model_file = model_file[1]
print(("restoring: ", model_file))
saver.restore(sess, (model_dir + "/" + model_file).replace(
'//', '/'))
model_step = int(
model_file.split("_")[len(model_file.split("_")) - 1])
print(("evaluating on dev ", model_file, model_step))
evaluate(sess, dev_data, batch_size, graph, model_step)
else:
ckpt = tf.train.get_checkpoint_state(model_dir)
print(("model dir: ", model_dir))
if (not (tf.gfile.IsDirectory(utility.FLAGS.output_dir))):
print(("create dir: ", utility.FLAGS.output_dir))
tf.gfile.MkDir(utility.FLAGS.output_dir)
if (not (tf.gfile.IsDirectory(model_dir))):
print(("create dir: ", model_dir))
tf.gfile.MkDir(model_dir)
Train(graph, utility, batch_size, train_data, sess, model_dir,
saver)
def generateGraph(n):
graph = m.Graph()
for i in range(n):
node = m.Node(i, r.randint(0, maxX), r.randint(0, maxY),
r.randint(0, maxD))
graph.addNode(node)
graph.countDistances()
return graph
def get_graph_snapshot(self) -> model.Graph:
g = model.Graph()
# Capture edges first, to avoid dangling references
people_skills = self.get_people_skills()
for p in self.get_people():
g.add_person(p)
for s in self.get_skills():
g.add_skill(s)
for pid, sid in people_skills:
g.link_person_to_skill(pid, sid)
return g
def get_skills_subgraph_snapshot(self,
skill_ids: Iterable[int]) -> model.Graph:
g = model.Graph()
# Capture edges first, to avoid dangling references
people_skills = self.find_people_skills_of_skills(skill_ids)
people_ids = set([pid for pid, sid in people_skills])
for p in self.find_people_by_id(people_ids):
g.add_person(p)
for s in self.find_skills_by_id(skill_ids):
g.add_skill(s)
for pid, sid in people_skills:
g.link_person_to_skill(pid, sid)
return g
def main():
g = model.Graph()
with tf.Session(graph=g.graph) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print('restore from ckpt{}'.format(ckpt))
else:
print('cannot restore')
imgStack = []
right = total = 0
for img in imgList:
#pattern = r'.*_(.*)\..*'
try:
org = img.split('_')[1]
except:
print('>>>>>>>>the img name does not match the pattern: ', img)
continue
total += 1
im = cv2.imread(img, 0).astype(np.float32)
im = cv2.resize(im, (utils.image_width, utils.image_height))
im = im.swapaxes(0, 1)
im = im[:, :, np.newaxis] / 255.
imgStack.append(im)
start = time.time()
def get_input_lens(seqs):
leghs = np.array([len(s) for s in seqs], dtype=np.int64)
return seqs, leghs
inp, seq_len = get_input_lens(np.array([im]))
feed = {g.inputs: inp, g.seq_len: np.array([27])}
d = sess.run(g.decoded[0], feed)
dense_decoded = tf.sparse_tensor_to_dense(
d, default_value=-1).eval(session=sess)
res = ''
for d in dense_decoded:
for i in d:
if i == -1:
res += ''
else:
res += utils.decode_maps[i]
print('cost time: ', time.time() - start)
if res == org: right += 1
else: print('ORG: ', org, ' decoded: ', res)
print('total accuracy: ', right * 1.0 / total)
def test(val_dir=data_dir, val_text_dir=text_dir):
g = model.Graph(is_training=True)
print('loading validation data, please wait---------------------', 'end= ')
val_feeder = utils.DataIterator2(data_dir=val_dir, text_dir=val_text_dir)
print('***************get image: ', val_feeder.size)
num_val_samples = val_feeder.size
num_val_per_epoch = int(num_val_samples / FLAGS.batch_size)
config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=False)
config.gpu_options.allow_growth = True
with tf.Session(graph=g.graph, config=config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print('restore from ckpt{}'.format(ckpt))
else:
print('cannot restore')
print(
'=============================begin testing============================='
)
if True:
if True:
if True:
acc_avg = 0.0
for cur_batch_cv in range(num_val_per_epoch):
print(num_val_per_epoch)
index_cv = []
for i in range(FLAGS.batch_size):
index_cv.append(cur_batch_cv * FLAGS.batch_size +
i)
#print("index_cv",index_cv)
val_inputs, val_seq_len, val_labels = val_feeder.input_index_generate_batch(
index_cv)
val_feed = {
g.inputs: val_inputs,
g.labels: val_labels,
g.keep_prob_cv: 1
}
predict_word_index, lr = sess.run(
[g.logits, g.learning_rate], val_feed)
print(val_labels[0], predict_word_index[0])
acc = utils.compute_acc(val_labels, predict_word_index)
acc_avg += acc
acc_avg = acc_avg / num_val_per_epoch
print("acc", acc_avg)
def acc():
acc_all = []
g = model.Graph()
with tf.Session(graph=g.graph) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print('restore from ckpt{}'.format(ckpt))
else:
print('cannot restore')
test_feeder = utils.DataIterator2(data_dir=data_dir, text_dir=text_dir)
print("total data:", test_feeder.size)
print("total image in folder", test_feeder.total_pic_read)
total_epoch = int(test_feeder.size / FLAGS.batch_size) + 1
for cur_batch in range(total_epoch):
print("cur_epoch/total_epoch", cur_batch, "/", total_epoch)
indexs = []
cur_batch_num = FLAGS.batch_size
if cur_batch == int(test_feeder.size / FLAGS.batch_size):
cur_batch_num = test_feeder.size - cur_batch * FLAGS.batch_size
for i in range(cur_batch_num):
indexs.append(cur_batch * FLAGS.batch_size + i)
test_inputs, test_seq_len, test_labels = test_feeder.input_index_generate_batch(
indexs)
cur_labels = [test_feeder.labels[i] for i in indexs]
test_feed = {
g.inputs: test_inputs,
g.labels: test_labels,
g.seq_len: np.array([g.cnn_time] * test_inputs.shape[0]),
g.keep_prob_fc: 1,
g.keep_prob_cv1: 1,
g.keep_prob_cv2: 1,
g.keep_prob_cv3: 1,
g.keep_prob_cv4: 1
}
dense_decoded, logits_before_ctc = sess.run(
[g.dense_decoded, g.logits_before_ctc], test_feed)
print("shape*****", logits_before_ctc.shape)
acc = utils.accuracy_calculation(cur_labels,
dense_decoded,
ignore_value=-1,
isPrint=False)
acc_all.append(acc)
print("cur_acc", acc)
print("$$$$$$$$$$$$$$$$$ ACC is :", acc_all, "$$$$$$$$$$$$$$$$$")
print("avg_acc:", np.array(acc_all).mean())
def predict_func():
g = model.Graph()
with tf.Session(graph=g.graph) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print('restore from ckpt{}'.format(ckpt))
else:
print('cannot restore')
test_feeder = utils.DataIterator3(data_dir=data_dir)
f = open(save_dir, mode="a")
print("total data:", test_feeder.size)
print("total image in folder", test_feeder.total_pic_read)
total_epoch = int(test_feeder.size / FLAGS.batch_size) + 1
for cur_batch in range(total_epoch):
print("cur_epoch/total_epoch", cur_batch, "/", total_epoch)
indexs = []
cur_batch_num = FLAGS.batch_size
if cur_batch == int(test_feeder.size / FLAGS.batch_size):
cur_batch_num = test_feeder.size - cur_batch * FLAGS.batch_size
for i in range(cur_batch_num):
indexs.append(cur_batch * FLAGS.batch_size + i)
test_inputs, num_batch = test_feeder.input_index_generate_batch(
indexs)
test_feed = {
g.inputs: test_inputs,
g.seq_len: np.array([g.cnn_time] * test_inputs.shape[0]),
g.keep_prob_fc: 1,
g.keep_prob_cv1: 1,
g.keep_prob_cv2: 1,
g.keep_prob_cv3: 1,
g.keep_prob_cv4: 1
}
dense_decoded, logits_before_ctc = sess.run(
[g.dense_decoded, g.logits_before_ctc], test_feed)
for encode_list, dense_list, e_num in zip(logits_before_ctc,
dense_decoded,
num_batch):
decode_string = utils.decode_function1(encode_list)
decode_string1 = utils.decode_function1(dense_list)
f.write(e_num + "," + decode_string + "," + decode_string1 +
"," + str(encode_list) + "\n")
f.close()
print("saved prediction")
def build_graph(utility):
""" Build Neural Programmer graph """
# creates TF graph and calls evaluator
batch_size = utility.FLAGS.batch_size
model_dir = utility.FLAGS.output_dir + "/model" + utility.FLAGS.job_id + "/"
# create all paramters of the model
param_class = parameters.Parameters(utility)
params, global_step, init = param_class.parameters(utility)
key = "test" #if (FLAGS.evaluator_job) else "train"
graph = model.Graph(utility,
batch_size,
utility.FLAGS.max_passes,
mode="test")
graph.create_graph(params, global_step)
sess = tf.InteractiveSession()
sess.run(init.name)
sess.run(graph.init_op.name)
return sess, graph, params
def train(train_dir=None,
val_dir=None,
train_text_dir=None,
val_text_dir=None):
g = model.Graph(is_training=True)
print('loading train data, please wait---------------------', 'end= ')
train_feeder = utils.DataIterator2(data_dir=train_dir,
text_dir=train_text_dir)
print('***************get image: ', train_feeder.size)
print('loading validation data, please wait---------------------', 'end= ')
val_feeder = utils.DataIterator2(data_dir=val_dir, text_dir=val_text_dir)
print('***************get image: ', val_feeder.size)
num_train_samples = train_feeder.size
num_batches_per_epoch = int(num_train_samples / FLAGS.batch_size)
num_val_samples = val_feeder.size
num_val_per_epoch = int(num_val_samples / FLAGS.batch_size)
config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=False)
with tf.Session(graph=g.graph, config=config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
g.graph.finalize()
if FLAGS.restore:
print("restore is true")
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print('restore from the checkpoint{0}'.format(ckpt))
print(
'=============================begin training============================='
)
val_inputs, val_seq_len, val_labels, val_labels_len = val_feeder.input_index_generate_batch(
)
val_feed = {
g.inputs: val_inputs,
g.y_: val_labels_len,
g.keep_prob_fc: 1,
g.keep_prob_cv1: 1
}
for cur_epoch in range(FLAGS.num_epochs):
shuffle_idx = np.random.permutation(num_train_samples)
for cur_batch in range(num_batches_per_epoch):
indexs = [
shuffle_idx[i % num_train_samples]
for i in range(cur_batch *
FLAGS.batch_size, (cur_batch + 1) *
FLAGS.batch_size)
]
batch_inputs, batch_seq_len, batch_labels, batch_labels_len = train_feeder.input_index_generate_batch(
indexs)
feed = {
g.inputs: batch_inputs,
g.y_: batch_labels_len,
g.keep_prob_fc: FLAGS.train_keep_prob_fc,
g.keep_prob_cv1: FLAGS.train_keep_prob_cv
}
_, step = sess.run([g.train_step, g.global_step], feed)
if (step + 1) % FLAGS.validation_steps == 0:
train_accuracy = sess.run([g.accuracy], feed)
val_accuracy = sess.run([g.accuracy], val_feed)
print("===step", step, "train_acc:", train_accuracy,
"val_acc", val_accuracy)
if (step + 1) % FLAGS.save_steps == 0:
print("save checkpoint", step)
if not os.path.isdir(FLAGS.checkpoint_dir):
os.mkdir(FLAGS.checkpoint_dir)
saver.save(sess,
os.path.join(FLAGS.checkpoint_dir, 'ocr-model'),
global_step=step)
def level_master(self, file_name: str):
"""
Метод отвечает за отображение окна с режимом игры
:param file_name:
:return:
"""
stell_graph = model.Graph(file_name)
start_3str, stop_3str = stell_graph.rnd_start_stop()
current = stell_graph.constellations[start_3str]
stop = stell_graph.constellations[stop_3str]
window = pygame.display.set_mode((800, 670))
pygame.display.set_caption('ASTROWARS')
self._win_blit(window, 'master.jpg', 'name.png', start_3str, stop_3str,
[])
pygame.display.flip()
clock = pygame.time.Clock()
finished = False
player1 = Player(True)
player2 = Player(False)
while not finished:
lasts = []
try:
lasts.append(player1.path[len(player1.path) - 1])
except IndexError:
pass
try:
lasts.append(player2.path[len(player2.path) - 1])
except IndexError:
pass
current.mark = 1
if player1.turn:
current_player = player1
else:
current_player = player2
applicant_str = self._get_text(lasts, start_3str, stop_3str)
if applicant_str == 'EXIT':
finished = True
continue
applicant = stell_graph.is_neighbours(current, applicant_str)
if applicant:
if applicant.mark:
current_player.mistakes -= 1
self._special_event(window, 'mistake.jpg')
clock.tick(1)
else:
current = applicant
current_player.path = model.np.append(
current_player.path, current.names[0])
player1.turn = not player1.turn
player2.turn = not player2.turn
print('Meow')
else:
current_player.mistakes -= 1
self._special_event(window, 'mistake.jpg')
if not current_player.mistakes:
if current_player is player1:
self._special_event(window, 'pl2win.jpg')
clock.tick(0.3)
finished = 1
else:
self._special_event(window, 'pl1win.jpg')
clock.tick(0.3)
finished = 1
if current is stop:
if current_player is player1:
self._special_event(window, 'pl1win.jpg')
clock.tick(0.3)
finished = 1
else:
self._special_event(window, 'pl2win.jpg')
clock.tick(0.3)
finished = 1
pygame.display.update()
clock.tick(FPS)
for event in pygame.event.get():
if event.type == pygame.QUIT:
finished = True
import core import model import graphs fname = "task4/task.txt" ourGraph = model.Graph(fname) graphs.levits(ourGraph)
# -*- coding: utf-8 -*-
import model
import numpy
import core
ourGraph = model.Graph("task2/task.txt")
print(ourGraph.data)
somelist = []
i = 0
for line in ourGraph.data:
j = 0
for col in line:
if col != 0:
somelist.append([i, j, col])
j += 1
i += 1
print(somelist)
somelist.sort(key=lambda somelist: somelist[2])
print(somelist)
vertex = []
for line in range(len(somelist)):
vertex.append(0)
print(vertex)
result = []
for edge in somelist:
if vertex[edge[0]] == 0:
vertex[edge[0]] = 1
def initTrees(n, t):
main = gg.generateGraph(n)
trees = []
for i in range(t):
trees.append(m.Graph())
return (main, trees)
import model import utils import time import tensorflow as tf import numpy as np import os import logging,datetime g = model.Graph(is_training=True)
# -*- coding: utf-8 -*-
import numpy
import core
import model
print("Проверим свойства отношения частичного порядка: ")
step = 1
adds = 0
ourGraph = model.Graph("task1/task.txt")
print(
str(step) + ") Проверим свойства отношения: \n ",
ourGraph.checkRelations())
step = step + 1
print(
str(step) +
") Чтобы отношение было отношением частичного порядка нужно чтобы оно было рефлексивным, антисимметричным, транзитивным:"
)
if ourGraph.checkReflexive():
print("а) Отношение рефлексивно.")
else:
print("а) Отношение не рефлексивно.")
reflex_adds = ourGraph.makeReflexive()
adds = adds + reflex_adds
print("а) При дополнении до рефлексивности добавлено " + str(reflex_adds) +
" ребер:")
print(ourGraph.data)
print("")
if ourGraph.checkTransitive():
def build_graph(data_dir):
graph = model.Graph()
tm = time.time()
#initialize header fields.
header = Header()
while True:
try:
line = raw_input()
except EOFError:
store(header, graph, data_dir)
break
#handle header
if (line.split(trace.header_delimiter)[0] == trace.header_indicator):
#log progress
t = time.time()
sys.stderr.write("%s,%s,%s\n" % (line, t, t - tm))
tm = t
#output
if (not header.is_empty):
store(header, graph, data_dir)
graph = model.Graph()
#update header fields
header.update_from_header_line(line)
continue
#handle warts line
line_dict = parse_line(line)
dst_ip = line_dict["dst_ip"]
hops = line_dict["hops"]
#print hops #debug
#handle traceroute hops.
hop_list = hops.split(trace.hop_delimiter)
#print hop_list #debug
i = 0
while (i <= len(hop_list) - 1): #ignore preceding blanks.
hop_dict = parse_hop(hop_list[i])
if (hop_dict.has_key("ip")):
break
i += 1
#add node
hop_dict = parse_hop(hop_list[i])
ip = hop_dict["ip"]
rtt_i = hop_dict["rtt"]
node_i = model.Node(ip)
ind_i = graph.add_node(node_i)
while i <= len(hop_list) - 1:
j = i + 1
is_direct = True
while (j < len(hop_list)):
hop_dict = parse_hop(hop_list[j])
if (hop_dict.has_key("ip")):
break
is_direct = False
j += 1
#print "i:%s, j:%s" % (i, j) #debug
#print "ip:%s, ind_i:%s" % (ip, ind_i) #debug
if i == len(hop_list) - 1 and dst_ip == ip: #check if hop is host.
#print "is not router" #debug
graph.nodes[ind_i].is_router = False
if j < len(hop_list):
hop_dict = parse_hop(hop_list[j])
ip = hop_dict["ip"]
rtt_j = hop_dict["rtt"]
node_j = model.Node(ip)
ind_j = graph.add_node(node_j)
#print "ip:%s, ind_i:%s" % (ip, ind_i) #debug
if j == len(
hop_list) - 1 and dst_ip == ip: #check if hop is host.
#print "is not router" #debug
graph.nodes[ind_j].is_router = False
if not is_direct:
blank = (ind_i, j - i - 1, ind_j)
node_blank = model.Node(blank)
node_blank.is_blank = True
ind_blank = graph.add_node(node_blank)
#add edge
delay = rtt_j - rtt_i
edge = model.Edge(ind_i, ind_j, delay)
edge.is_direct = is_direct
graph.add_edge(edge)
rtt_i = rtt_j
ind_i = ind_j
i = j
def train(train_dir=None,
val_dir=None,
train_text_dir=None,
val_text_dir=None):
acc_avg = 0.0
acc_best = 0.0
acc_best_step = 0
g = model.Graph(is_training=True)
print('loading train data, please wait---------------------', 'end= ')
train_feeder = utils.DataIterator2(data_dir=train_dir,
text_dir=train_text_dir)
print('***************get image: ', train_feeder.size)
print('loading validation data, please wait---------------------', 'end= ')
val_feeder = utils.DataIterator2(data_dir=val_dir, text_dir=val_text_dir)
print('***************get image: ', val_feeder.size)
num_train_samples = train_feeder.size
num_batches_per_epoch = int(num_train_samples / FLAGS.batch_size)
num_val_samples = val_feeder.size
num_val_per_epoch = int(num_val_samples / FLAGS.batch_size)
config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=False)
config.gpu_options.allow_growth = True
with tf.Session(graph=g.graph, config=config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
g.graph.finalize()
if FLAGS.restore:
print("restore is true")
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print('restore from the checkpoint{0}'.format(ckpt))
print(
'=============================begin training============================='
)
for cur_epoch in range(FLAGS.num_epochs):
shuffle_idx = np.random.permutation(num_train_samples)
train_cost = 0
for cur_batch in range(num_batches_per_epoch):
now = datetime.datetime.now()
indexs = [
shuffle_idx[i % num_train_samples]
for i in range(cur_batch *
FLAGS.batch_size, (cur_batch + 1) *
FLAGS.batch_size)
]
batch_inputs, batch_seq_len, batch_labels = train_feeder.input_index_generate_batch(
indexs)
feed = {
g.inputs: batch_inputs,
g.labels: batch_labels,
g.keep_prob_cv: FLAGS.train_keep_prob_cv
}
batch_cost_avg, step, _, predict_result = sess.run(
[g.cost_batch_avg, g.global_step, g.optimizer, g.logits],
feed)
if step % 1 == 0:
print("cur_epoch====", cur_epoch, "cur_batch----",
cur_batch, "g_step****", step, "cost",
batch_cost_avg)
if False:
print("real", batch_labels)
print("predict", predict_result)
if step % FLAGS.validation_steps == 0:
acc_avg = 0.0
for cur_batch_cv in range(num_val_per_epoch):
print(num_val_per_epoch)
index_cv = []
for i in range(FLAGS.batch_size):
index_cv.append(cur_batch_cv * FLAGS.batch_size +
i)
print("index_cv", index_cv)
val_inputs, val_seq_len, val_labels = val_feeder.input_index_generate_batch(
index_cv)
val_feed = {
g.inputs: val_inputs,
g.labels: val_labels,
g.keep_prob_cv: 1
}
predict_word_index, lr = sess.run(
[g.logits, g.learning_rate], val_feed)
acc = utils.compute_acc(val_labels, predict_word_index)
acc_avg += acc
acc_avg = acc_avg / num_val_per_epoch
if acc_avg - acc_best > 0.00001:
acc_best = acc_avg
acc_best_step = step
print("acc", acc_avg)
if Flag_Isserver:
f = open('../log/acc/acc.txt', mode="a")
log = "{}/{} {}:{}:{}, Epoch {}/{}, step=={}-->cur_acc = {:.3f}, best_step=={}-->best_acc = {:.3f}, lr={:.8f},batch_cost_avg={:.3f}\n"
f.write(
log.format(now.month, now.day, now.hour,
now.minute, now.second, cur_epoch + 1,
FLAGS.num_epochs, step, acc_avg,
acc_best_step, acc_best, lr,
batch_cost_avg))
f.close()
if step % FLAGS.save_steps == 0 or acc_avg - acc_best > 0.0001:
print("save checkpoint", step)
if not os.path.isdir(FLAGS.checkpoint_dir):
os.mkdir(FLAGS.checkpoint_dir)
saver.save(sess,
os.path.join(FLAGS.checkpoint_dir, 'ocr-model'),
global_step=step)
def master(train_data, dev_data, utility, dat):
#creates TF graph and calls trainer or evaluator
batch_size = utility.FLAGS.batch_size
model_dir = utility.FLAGS.output_dir + "/model_" + utility.FLAGS.job_id + "/"
#create all paramters of the model
param_class = parameters.Parameters(utility)
params, global_step, init = param_class.parameters(utility)
key = FLAGS.job_mode
print("Running with key " + key)
graph = model.Graph(utility, batch_size, utility.FLAGS.max_passes, mode=key)
graph.create_graph(params, global_step)
prev_dev_error = 0.0
final_loss = 0.0
final_accuracy = 0.0
#start session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init.name)
sess.run(graph.init_op.name)
to_save = params.copy()
saver = tf.train.Saver(to_save, max_to_keep=500)
if (key == 'test'):
model_file = 'model_' + utility.FLAGS.model_id
print("restoring: ", model_file)
saver.restore(sess, model_dir + model_file)
Test(graph, utility, batch_size, sess, model_dir, dat, 'data/custom/uefa.examples')
if (key == 'error-test'):
while True:
selected_models = {}
file_list = tf.gfile.ListDirectory(model_dir)
for model_file in file_list:
if ("checkpoint" in model_file or "index" in model_file or
"meta" in model_file):
continue
if ("data" in model_file):
model_file = model_file.split(".")[0]
model_step = int(
model_file.split("_")[len(model_file.split("_")) - 1])
selected_models[model_step] = model_file
file_list = sorted(selected_models.items(), key=lambda x: x[0])
if (len(file_list) > 0):
file_list = file_list[0:len(file_list) - 1]
print "list of models: ", file_list
for model_file in file_list:
model_file = model_file[1]
print "restoring: ", model_file
saver.restore(sess, model_dir + model_file)
model_step = int(
model_file.split("_")[len(model_file.split("_")) - 1])
print "evaluating on dev ", model_file, model_step
evaluate(sess, dev_data, batch_size, graph, model_step)
elif (key == 'train'):
ckpt = tf.train.get_checkpoint_state(model_dir)
print "model dir: ", model_dir
if (not (tf.gfile.IsDirectory(utility.FLAGS.output_dir))):
print "create dir: ", utility.FLAGS.output_dir
tf.gfile.MkDir(utility.FLAGS.output_dir)
if (not (tf.gfile.IsDirectory(model_dir))):
print "create dir: ", model_dir
tf.gfile.MkDir(model_dir)
Train(graph, utility, batch_size, train_data, sess, model_dir,
saver)
elif (key == 'demo'):
#create all paramters of the model
model_file = 'model_' + utility.FLAGS.model_id
print("restoring: ", model_file)
saver.restore(sess, model_dir + model_file)
if utility.FLAGS.mode == 'console':
DemoConsole(graph, utility, sess, model_dir, dat)
else:
Demo(graph, utility, sess, model_dir, dat)
print(f"{person.name} <-> {skill.name}")
#############################
# Model tests
#############################
import model
p1 = model.Person(1, "Alice")
p2 = model.Person(2, "Bob")
p3 = model.Person(3, "Charles")
s1 = model.Skill(11, "Dancing")
s2 = model.Skill(22, "Fishing")
g = model.Graph()
g.add_person(p1)
g.add_person(p2)
g.add_person(p3)
g.add_skill(s1)
g.add_skill(s2)
g.link_person_to_skill(1, 11)
g.link_person_to_skill(1, 22)
g.link_person_to_skill(2, 11)
g.link_person_to_skill(3, 22)
assert g.people[p1.id] == p1
assert g.people[p2.id] == p2
def train(train_dir=None,
val_dir=None,
train_text_dir=None,
val_text_dir=None):
g = model.Graph(is_training=True)
print('loading train data, please wait---------------------', 'end= ')
train_feeder = utils.DataIterator2(data_dir=train_dir,
text_dir=train_text_dir)
print('get image: ', train_feeder.size)
print('loading validation data, please wait---------------------', 'end= ')
val_feeder = utils.DataIterator2(data_dir=val_dir, text_dir=val_text_dir)
print('get image: ', val_feeder.size)
num_train_samples = train_feeder.size
num_batches_per_epoch = int(num_train_samples / FLAGS.batch_size)
num_val_samples = val_feeder.size
num_val_per_epoch = int(num_val_samples / FLAGS.batch_size)
config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=False)
config.gpu_options.allow_growth = True
#config.gpu_options.per_process_gpu_memory_fraction = 0.6
with tf.Session(graph=g.graph, config=config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=3)
g.graph.finalize()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
if FLAGS.restore:
print("restore is true")
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print('restore from the checkpoint{0}'.format(ckpt))
print(
'=============================begin training============================='
)
val_inputs, val_seq_len, val_labels = val_feeder.input_index_generate_batch(
)
#print(len(val_inputs))
val_feed = {
g.inputs: val_inputs,
g.labels: val_labels,
g.seq_len: np.array([g.cnn_time] * val_inputs.shape[0])
}
for cur_epoch in range(FLAGS.num_epochs):
shuffle_idx = np.random.permutation(num_train_samples)
train_cost = 0
start_time = time.time()
batch_time = time.time()
#the tracing part
for cur_batch in range(num_batches_per_epoch):
if (cur_batch + 1) % 100 == 0:
print('batch', cur_batch, ': time',
time.time() - batch_time)
batch_time = time.time()
indexs = [
shuffle_idx[i % num_train_samples]
for i in range(cur_batch *
FLAGS.batch_size, (cur_batch + 1) *
FLAGS.batch_size)
]
batch_inputs, batch_seq_len, batch_labels = train_feeder.input_index_generate_batch(
indexs)
#batch_inputs,batch_seq_len,batch_labels=utils.gen_batch(FLAGS.batch_size)
feed = {
g.inputs: batch_inputs,
g.labels: batch_labels,
g.seq_len: np.array([g.cnn_time] * batch_inputs.shape[0])
}
# if summary is needed
#batch_cost,step,train_summary,_ = sess.run([cost,global_step,merged_summay,optimizer],feed)
summary_str, batch_cost, step, _ = sess.run(
[g.merged_summay, g.cost, g.global_step, g.optimizer],
feed)
#calculate the cost
train_cost += batch_cost * FLAGS.batch_size
train_writer.add_summary(summary_str, step)
# save the checkpoint
if step % FLAGS.save_steps == 0:
print("save checkpoint", step)
if not os.path.isdir(FLAGS.checkpoint_dir):
os.mkdir(FLAGS.checkpoint_dir)
logger.info('save the checkpoint of{0}', format(step))
saver.save(sess,
os.path.join(FLAGS.checkpoint_dir, 'ocr-model'),
global_step=step)
#train_err+=the_err*FLAGS.batch_size
#do validation
if step % FLAGS.validation_steps == 0:
dense_decoded, lastbatch_err, lr = sess.run(
[g.dense_decoded, g.lerr, g.learning_rate], val_feed)
# print the decode result
acc = utils.accuracy_calculation(val_feeder.labels,
dense_decoded,
ignore_value=-1,
isPrint=True)
avg_train_cost = train_cost / (
(cur_batch + 1) * FLAGS.batch_size)
#train_err/=num_train_samples
now = datetime.datetime.now()
log = "{}/{} {}:{}:{} step==={}, Epoch {}/{}, accuracy = {:.3f},avg_train_cost = {:.3f}, lastbatch_err = {:.3f}, time = {:.3f},lr={:.8f}\n"
print(
log.format(now.month, now.day, now.hour, now.minute,
now.second, step, cur_epoch + 1,
FLAGS.num_epochs, acc, avg_train_cost,
lastbatch_err,
time.time() - start_time, lr))
if Flag_Isserver:
f = open('../log/acc/acc.txt', mode="a")
f.write(
log.format(now.month, now.day, now.hour,
now.minute, now.second, step,
cur_epoch + 1, FLAGS.num_epochs, acc,
avg_train_cost, lastbatch_err,
time.time() - start_time, lr))
f.close()
import pygame.locals
class Player:
def __init__(self, turn):
self.score = 0
self.mistakes = 3
self.turn = turn
self.path = np.array([], dtype='<U13')
pygame.init()
FPS = 20
stell_graph = model.Graph('Data.txt')
start_3str, stop_3str = stell_graph.rnd_start_stop()
current = stell_graph.constellations[start_3str]
stop = stell_graph.constellations[stop_3str]
window = pygame.display.set_mode((800, 670))
pygame.display.set_caption('ASTROWARS')
def win_blit(window, master_file_name, name_file_name, start, stop, lasts):
screen = pygame.image.load(master_file_name)
info = pygame.image.load(name_file_name)
window.blit(screen, (0, 150))
window.blit(info, (70, 0))
