def __init__(self, filename): # must have self.graph valid at end of constructor # sqlite open file self.conn = sqlite3.connect(filename) fresh_file = not table_exists(self.conn, 'config') # before making ConfigDict self.config = ConfigDict(self.conn) datastore = kvstore.KVStore(self.conn, V2_TABLENAME) # check for config format version version = self.config['version'] if fresh_file: print 'fresh file' self.graph = model.Graph(datastore, None) self.load_default_config() self.commit() else: if self.config['version'] is None: # if no conf, migrate by creating empty graph, creating cards # and loading it from a v1 DataStore self.graph = model.Graph(datastore, None) self.import_v1() # else, load commit elif version == '2': head_ptr = self.config['head'] if head_ptr is None: raise CorruptionError('No head pointer!') try: self.graph = model.Graph(datastore, head_ptr) # after this, should be all loaded except model.Error as e: print 'failed to open gp file:', e raise ValueError
def acc(): g = model.Graph() with tf.Session(graph=g.graph) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=100) ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from ckpt{}'.format(ckpt)) else: print('cannot restore') val_feeder = utils.DataIterator(data_dir=inferFolder) val_inputs, val_seq_len, val_labels = val_feeder.input_index_generate_batch( ) val_feed = { g.inputs: val_inputs, g.labels: val_labels, g.seq_len: np.array([27] * val_inputs.shape[0]) } dense_decoded = sess.run(g.dense_decoded, val_feed) # print the decode result acc = utils.accuracy_calculation(val_feeder.labels, dense_decoded, ignore_value=-1, isPrint=True) print(acc)
def master(train_data, dev_data, utility): #creates TF graph and calls trainer or evaluator batch_size = utility.FLAGS.batch_size model_dir = utility.FLAGS.output_dir + "/model" + utility.FLAGS.job_id + "/" #create all paramters of the model param_class = parameters.Parameters(utility) params, global_step, init = param_class.parameters(utility) key = "test" if (FLAGS.evaluator_job) else "train" graph = model.Graph(utility, batch_size, utility.FLAGS.max_passes, mode=key) graph.create_graph(params, global_step) prev_dev_error = 0.0 final_loss = 0.0 final_accuracy = 0.0 #start session with tf.Session() as sess: sess.run(init.name) sess.run(graph.init_op.name) to_save = params.copy() saver = tf.train.Saver(to_save, max_to_keep=500) if (FLAGS.evaluator_job): while True: selected_models = {} file_list = tf.gfile.ListDirectory(model_dir) for model_file in file_list: if ("checkpoint" in model_file or "index" in model_file or "meta" in model_file): continue if ("data" in model_file): model_file = model_file.split(".")[0] model_step = int( model_file.split("_")[len(model_file.split("_")) - 1]) selected_models[model_step] = model_file file_list = sorted(list(selected_models.items()), key=lambda x: x[0]) if (len(file_list) > 0): file_list = file_list[0:len(file_list) - 1] print(("list of models: ", file_list)) for model_file in file_list: model_file = model_file[1] print(("restoring: ", model_file)) saver.restore(sess, (model_dir + "/" + model_file).replace( '//', '/')) model_step = int( model_file.split("_")[len(model_file.split("_")) - 1]) print(("evaluating on dev ", model_file, model_step)) evaluate(sess, dev_data, batch_size, graph, model_step) else: ckpt = tf.train.get_checkpoint_state(model_dir) print(("model dir: ", model_dir)) if (not (tf.gfile.IsDirectory(utility.FLAGS.output_dir))): print(("create dir: ", utility.FLAGS.output_dir)) tf.gfile.MkDir(utility.FLAGS.output_dir) if (not (tf.gfile.IsDirectory(model_dir))): print(("create dir: ", model_dir)) tf.gfile.MkDir(model_dir) Train(graph, utility, batch_size, train_data, sess, model_dir, saver)
def generateGraph(n): graph = m.Graph() for i in range(n): node = m.Node(i, r.randint(0, maxX), r.randint(0, maxY), r.randint(0, maxD)) graph.addNode(node) graph.countDistances() return graph
def get_graph_snapshot(self) -> model.Graph: g = model.Graph() # Capture edges first, to avoid dangling references people_skills = self.get_people_skills() for p in self.get_people(): g.add_person(p) for s in self.get_skills(): g.add_skill(s) for pid, sid in people_skills: g.link_person_to_skill(pid, sid) return g
def get_skills_subgraph_snapshot(self, skill_ids: Iterable[int]) -> model.Graph: g = model.Graph() # Capture edges first, to avoid dangling references people_skills = self.find_people_skills_of_skills(skill_ids) people_ids = set([pid for pid, sid in people_skills]) for p in self.find_people_by_id(people_ids): g.add_person(p) for s in self.find_skills_by_id(skill_ids): g.add_skill(s) for pid, sid in people_skills: g.link_person_to_skill(pid, sid) return g
def main(): g = model.Graph() with tf.Session(graph=g.graph) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=100) ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from ckpt{}'.format(ckpt)) else: print('cannot restore') imgStack = [] right = total = 0 for img in imgList: #pattern = r'.*_(.*)\..*' try: org = img.split('_')[1] except: print('>>>>>>>>the img name does not match the pattern: ', img) continue total += 1 im = cv2.imread(img, 0).astype(np.float32) im = cv2.resize(im, (utils.image_width, utils.image_height)) im = im.swapaxes(0, 1) im = im[:, :, np.newaxis] / 255. imgStack.append(im) start = time.time() def get_input_lens(seqs): leghs = np.array([len(s) for s in seqs], dtype=np.int64) return seqs, leghs inp, seq_len = get_input_lens(np.array([im])) feed = {g.inputs: inp, g.seq_len: np.array([27])} d = sess.run(g.decoded[0], feed) dense_decoded = tf.sparse_tensor_to_dense( d, default_value=-1).eval(session=sess) res = '' for d in dense_decoded: for i in d: if i == -1: res += '' else: res += utils.decode_maps[i] print('cost time: ', time.time() - start) if res == org: right += 1 else: print('ORG: ', org, ' decoded: ', res) print('total accuracy: ', right * 1.0 / total)
def test(val_dir=data_dir, val_text_dir=text_dir): g = model.Graph(is_training=True) print('loading validation data, please wait---------------------', 'end= ') val_feeder = utils.DataIterator2(data_dir=val_dir, text_dir=val_text_dir) print('***************get image: ', val_feeder.size) num_val_samples = val_feeder.size num_val_per_epoch = int(num_val_samples / FLAGS.batch_size) config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=False) config.gpu_options.allow_growth = True with tf.Session(graph=g.graph, config=config) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=100) ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from ckpt{}'.format(ckpt)) else: print('cannot restore') print( '=============================begin testing=============================' ) if True: if True: if True: acc_avg = 0.0 for cur_batch_cv in range(num_val_per_epoch): print(num_val_per_epoch) index_cv = [] for i in range(FLAGS.batch_size): index_cv.append(cur_batch_cv * FLAGS.batch_size + i) #print("index_cv",index_cv) val_inputs, val_seq_len, val_labels = val_feeder.input_index_generate_batch( index_cv) val_feed = { g.inputs: val_inputs, g.labels: val_labels, g.keep_prob_cv: 1 } predict_word_index, lr = sess.run( [g.logits, g.learning_rate], val_feed) print(val_labels[0], predict_word_index[0]) acc = utils.compute_acc(val_labels, predict_word_index) acc_avg += acc acc_avg = acc_avg / num_val_per_epoch print("acc", acc_avg)
def acc(): acc_all = [] g = model.Graph() with tf.Session(graph=g.graph) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=100) ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from ckpt{}'.format(ckpt)) else: print('cannot restore') test_feeder = utils.DataIterator2(data_dir=data_dir, text_dir=text_dir) print("total data:", test_feeder.size) print("total image in folder", test_feeder.total_pic_read) total_epoch = int(test_feeder.size / FLAGS.batch_size) + 1 for cur_batch in range(total_epoch): print("cur_epoch/total_epoch", cur_batch, "/", total_epoch) indexs = [] cur_batch_num = FLAGS.batch_size if cur_batch == int(test_feeder.size / FLAGS.batch_size): cur_batch_num = test_feeder.size - cur_batch * FLAGS.batch_size for i in range(cur_batch_num): indexs.append(cur_batch * FLAGS.batch_size + i) test_inputs, test_seq_len, test_labels = test_feeder.input_index_generate_batch( indexs) cur_labels = [test_feeder.labels[i] for i in indexs] test_feed = { g.inputs: test_inputs, g.labels: test_labels, g.seq_len: np.array([g.cnn_time] * test_inputs.shape[0]), g.keep_prob_fc: 1, g.keep_prob_cv1: 1, g.keep_prob_cv2: 1, g.keep_prob_cv3: 1, g.keep_prob_cv4: 1 } dense_decoded, logits_before_ctc = sess.run( [g.dense_decoded, g.logits_before_ctc], test_feed) print("shape*****", logits_before_ctc.shape) acc = utils.accuracy_calculation(cur_labels, dense_decoded, ignore_value=-1, isPrint=False) acc_all.append(acc) print("cur_acc", acc) print("$$$$$$$$$$$$$$$$$ ACC is :", acc_all, "$$$$$$$$$$$$$$$$$") print("avg_acc:", np.array(acc_all).mean())
def predict_func(): g = model.Graph() with tf.Session(graph=g.graph) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=100) ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from ckpt{}'.format(ckpt)) else: print('cannot restore') test_feeder = utils.DataIterator3(data_dir=data_dir) f = open(save_dir, mode="a") print("total data:", test_feeder.size) print("total image in folder", test_feeder.total_pic_read) total_epoch = int(test_feeder.size / FLAGS.batch_size) + 1 for cur_batch in range(total_epoch): print("cur_epoch/total_epoch", cur_batch, "/", total_epoch) indexs = [] cur_batch_num = FLAGS.batch_size if cur_batch == int(test_feeder.size / FLAGS.batch_size): cur_batch_num = test_feeder.size - cur_batch * FLAGS.batch_size for i in range(cur_batch_num): indexs.append(cur_batch * FLAGS.batch_size + i) test_inputs, num_batch = test_feeder.input_index_generate_batch( indexs) test_feed = { g.inputs: test_inputs, g.seq_len: np.array([g.cnn_time] * test_inputs.shape[0]), g.keep_prob_fc: 1, g.keep_prob_cv1: 1, g.keep_prob_cv2: 1, g.keep_prob_cv3: 1, g.keep_prob_cv4: 1 } dense_decoded, logits_before_ctc = sess.run( [g.dense_decoded, g.logits_before_ctc], test_feed) for encode_list, dense_list, e_num in zip(logits_before_ctc, dense_decoded, num_batch): decode_string = utils.decode_function1(encode_list) decode_string1 = utils.decode_function1(dense_list) f.write(e_num + "," + decode_string + "," + decode_string1 + "," + str(encode_list) + "\n") f.close() print("saved prediction")
def build_graph(utility): """ Build Neural Programmer graph """ # creates TF graph and calls evaluator batch_size = utility.FLAGS.batch_size model_dir = utility.FLAGS.output_dir + "/model" + utility.FLAGS.job_id + "/" # create all paramters of the model param_class = parameters.Parameters(utility) params, global_step, init = param_class.parameters(utility) key = "test" #if (FLAGS.evaluator_job) else "train" graph = model.Graph(utility, batch_size, utility.FLAGS.max_passes, mode="test") graph.create_graph(params, global_step) sess = tf.InteractiveSession() sess.run(init.name) sess.run(graph.init_op.name) return sess, graph, params
def train(train_dir=None, val_dir=None, train_text_dir=None, val_text_dir=None): g = model.Graph(is_training=True) print('loading train data, please wait---------------------', 'end= ') train_feeder = utils.DataIterator2(data_dir=train_dir, text_dir=train_text_dir) print('***************get image: ', train_feeder.size) print('loading validation data, please wait---------------------', 'end= ') val_feeder = utils.DataIterator2(data_dir=val_dir, text_dir=val_text_dir) print('***************get image: ', val_feeder.size) num_train_samples = train_feeder.size num_batches_per_epoch = int(num_train_samples / FLAGS.batch_size) num_val_samples = val_feeder.size num_val_per_epoch = int(num_val_samples / FLAGS.batch_size) config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=False) with tf.Session(graph=g.graph, config=config) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=100) g.graph.finalize() if FLAGS.restore: print("restore is true") ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from the checkpoint{0}'.format(ckpt)) print( '=============================begin training=============================' ) val_inputs, val_seq_len, val_labels, val_labels_len = val_feeder.input_index_generate_batch( ) val_feed = { g.inputs: val_inputs, g.y_: val_labels_len, g.keep_prob_fc: 1, g.keep_prob_cv1: 1 } for cur_epoch in range(FLAGS.num_epochs): shuffle_idx = np.random.permutation(num_train_samples) for cur_batch in range(num_batches_per_epoch): indexs = [ shuffle_idx[i % num_train_samples] for i in range(cur_batch * FLAGS.batch_size, (cur_batch + 1) * FLAGS.batch_size) ] batch_inputs, batch_seq_len, batch_labels, batch_labels_len = train_feeder.input_index_generate_batch( indexs) feed = { g.inputs: batch_inputs, g.y_: batch_labels_len, g.keep_prob_fc: FLAGS.train_keep_prob_fc, g.keep_prob_cv1: FLAGS.train_keep_prob_cv } _, step = sess.run([g.train_step, g.global_step], feed) if (step + 1) % FLAGS.validation_steps == 0: train_accuracy = sess.run([g.accuracy], feed) val_accuracy = sess.run([g.accuracy], val_feed) print("===step", step, "train_acc:", train_accuracy, "val_acc", val_accuracy) if (step + 1) % FLAGS.save_steps == 0: print("save checkpoint", step) if not os.path.isdir(FLAGS.checkpoint_dir): os.mkdir(FLAGS.checkpoint_dir) saver.save(sess, os.path.join(FLAGS.checkpoint_dir, 'ocr-model'), global_step=step)
def level_master(self, file_name: str): """ Метод отвечает за отображение окна с режимом игры :param file_name: :return: """ stell_graph = model.Graph(file_name) start_3str, stop_3str = stell_graph.rnd_start_stop() current = stell_graph.constellations[start_3str] stop = stell_graph.constellations[stop_3str] window = pygame.display.set_mode((800, 670)) pygame.display.set_caption('ASTROWARS') self._win_blit(window, 'master.jpg', 'name.png', start_3str, stop_3str, []) pygame.display.flip() clock = pygame.time.Clock() finished = False player1 = Player(True) player2 = Player(False) while not finished: lasts = [] try: lasts.append(player1.path[len(player1.path) - 1]) except IndexError: pass try: lasts.append(player2.path[len(player2.path) - 1]) except IndexError: pass current.mark = 1 if player1.turn: current_player = player1 else: current_player = player2 applicant_str = self._get_text(lasts, start_3str, stop_3str) if applicant_str == 'EXIT': finished = True continue applicant = stell_graph.is_neighbours(current, applicant_str) if applicant: if applicant.mark: current_player.mistakes -= 1 self._special_event(window, 'mistake.jpg') clock.tick(1) else: current = applicant current_player.path = model.np.append( current_player.path, current.names[0]) player1.turn = not player1.turn player2.turn = not player2.turn print('Meow') else: current_player.mistakes -= 1 self._special_event(window, 'mistake.jpg') if not current_player.mistakes: if current_player is player1: self._special_event(window, 'pl2win.jpg') clock.tick(0.3) finished = 1 else: self._special_event(window, 'pl1win.jpg') clock.tick(0.3) finished = 1 if current is stop: if current_player is player1: self._special_event(window, 'pl1win.jpg') clock.tick(0.3) finished = 1 else: self._special_event(window, 'pl2win.jpg') clock.tick(0.3) finished = 1 pygame.display.update() clock.tick(FPS) for event in pygame.event.get(): if event.type == pygame.QUIT: finished = True
import core import model import graphs fname = "task4/task.txt" ourGraph = model.Graph(fname) graphs.levits(ourGraph)
# -*- coding: utf-8 -*- import model import numpy import core ourGraph = model.Graph("task2/task.txt") print(ourGraph.data) somelist = [] i = 0 for line in ourGraph.data: j = 0 for col in line: if col != 0: somelist.append([i, j, col]) j += 1 i += 1 print(somelist) somelist.sort(key=lambda somelist: somelist[2]) print(somelist) vertex = [] for line in range(len(somelist)): vertex.append(0) print(vertex) result = [] for edge in somelist: if vertex[edge[0]] == 0: vertex[edge[0]] = 1
def initTrees(n, t): main = gg.generateGraph(n) trees = [] for i in range(t): trees.append(m.Graph()) return (main, trees)
import model import utils import time import tensorflow as tf import numpy as np import os import logging,datetime g = model.Graph(is_training=True)
# -*- coding: utf-8 -*- import numpy import core import model print("Проверим свойства отношения частичного порядка: ") step = 1 adds = 0 ourGraph = model.Graph("task1/task.txt") print( str(step) + ") Проверим свойства отношения: \n ", ourGraph.checkRelations()) step = step + 1 print( str(step) + ") Чтобы отношение было отношением частичного порядка нужно чтобы оно было рефлексивным, антисимметричным, транзитивным:" ) if ourGraph.checkReflexive(): print("а) Отношение рефлексивно.") else: print("а) Отношение не рефлексивно.") reflex_adds = ourGraph.makeReflexive() adds = adds + reflex_adds print("а) При дополнении до рефлексивности добавлено " + str(reflex_adds) + " ребер:") print(ourGraph.data) print("") if ourGraph.checkTransitive():
def build_graph(data_dir): graph = model.Graph() tm = time.time() #initialize header fields. header = Header() while True: try: line = raw_input() except EOFError: store(header, graph, data_dir) break #handle header if (line.split(trace.header_delimiter)[0] == trace.header_indicator): #log progress t = time.time() sys.stderr.write("%s,%s,%s\n" % (line, t, t - tm)) tm = t #output if (not header.is_empty): store(header, graph, data_dir) graph = model.Graph() #update header fields header.update_from_header_line(line) continue #handle warts line line_dict = parse_line(line) dst_ip = line_dict["dst_ip"] hops = line_dict["hops"] #print hops #debug #handle traceroute hops. hop_list = hops.split(trace.hop_delimiter) #print hop_list #debug i = 0 while (i <= len(hop_list) - 1): #ignore preceding blanks. hop_dict = parse_hop(hop_list[i]) if (hop_dict.has_key("ip")): break i += 1 #add node hop_dict = parse_hop(hop_list[i]) ip = hop_dict["ip"] rtt_i = hop_dict["rtt"] node_i = model.Node(ip) ind_i = graph.add_node(node_i) while i <= len(hop_list) - 1: j = i + 1 is_direct = True while (j < len(hop_list)): hop_dict = parse_hop(hop_list[j]) if (hop_dict.has_key("ip")): break is_direct = False j += 1 #print "i:%s, j:%s" % (i, j) #debug #print "ip:%s, ind_i:%s" % (ip, ind_i) #debug if i == len(hop_list) - 1 and dst_ip == ip: #check if hop is host. #print "is not router" #debug graph.nodes[ind_i].is_router = False if j < len(hop_list): hop_dict = parse_hop(hop_list[j]) ip = hop_dict["ip"] rtt_j = hop_dict["rtt"] node_j = model.Node(ip) ind_j = graph.add_node(node_j) #print "ip:%s, ind_i:%s" % (ip, ind_i) #debug if j == len( hop_list) - 1 and dst_ip == ip: #check if hop is host. #print "is not router" #debug graph.nodes[ind_j].is_router = False if not is_direct: blank = (ind_i, j - i - 1, ind_j) node_blank = model.Node(blank) node_blank.is_blank = True ind_blank = graph.add_node(node_blank) #add edge delay = rtt_j - rtt_i edge = model.Edge(ind_i, ind_j, delay) edge.is_direct = is_direct graph.add_edge(edge) rtt_i = rtt_j ind_i = ind_j i = j
def train(train_dir=None, val_dir=None, train_text_dir=None, val_text_dir=None): acc_avg = 0.0 acc_best = 0.0 acc_best_step = 0 g = model.Graph(is_training=True) print('loading train data, please wait---------------------', 'end= ') train_feeder = utils.DataIterator2(data_dir=train_dir, text_dir=train_text_dir) print('***************get image: ', train_feeder.size) print('loading validation data, please wait---------------------', 'end= ') val_feeder = utils.DataIterator2(data_dir=val_dir, text_dir=val_text_dir) print('***************get image: ', val_feeder.size) num_train_samples = train_feeder.size num_batches_per_epoch = int(num_train_samples / FLAGS.batch_size) num_val_samples = val_feeder.size num_val_per_epoch = int(num_val_samples / FLAGS.batch_size) config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=False) config.gpu_options.allow_growth = True with tf.Session(graph=g.graph, config=config) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=100) g.graph.finalize() if FLAGS.restore: print("restore is true") ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from the checkpoint{0}'.format(ckpt)) print( '=============================begin training=============================' ) for cur_epoch in range(FLAGS.num_epochs): shuffle_idx = np.random.permutation(num_train_samples) train_cost = 0 for cur_batch in range(num_batches_per_epoch): now = datetime.datetime.now() indexs = [ shuffle_idx[i % num_train_samples] for i in range(cur_batch * FLAGS.batch_size, (cur_batch + 1) * FLAGS.batch_size) ] batch_inputs, batch_seq_len, batch_labels = train_feeder.input_index_generate_batch( indexs) feed = { g.inputs: batch_inputs, g.labels: batch_labels, g.keep_prob_cv: FLAGS.train_keep_prob_cv } batch_cost_avg, step, _, predict_result = sess.run( [g.cost_batch_avg, g.global_step, g.optimizer, g.logits], feed) if step % 1 == 0: print("cur_epoch====", cur_epoch, "cur_batch----", cur_batch, "g_step****", step, "cost", batch_cost_avg) if False: print("real", batch_labels) print("predict", predict_result) if step % FLAGS.validation_steps == 0: acc_avg = 0.0 for cur_batch_cv in range(num_val_per_epoch): print(num_val_per_epoch) index_cv = [] for i in range(FLAGS.batch_size): index_cv.append(cur_batch_cv * FLAGS.batch_size + i) print("index_cv", index_cv) val_inputs, val_seq_len, val_labels = val_feeder.input_index_generate_batch( index_cv) val_feed = { g.inputs: val_inputs, g.labels: val_labels, g.keep_prob_cv: 1 } predict_word_index, lr = sess.run( [g.logits, g.learning_rate], val_feed) acc = utils.compute_acc(val_labels, predict_word_index) acc_avg += acc acc_avg = acc_avg / num_val_per_epoch if acc_avg - acc_best > 0.00001: acc_best = acc_avg acc_best_step = step print("acc", acc_avg) if Flag_Isserver: f = open('../log/acc/acc.txt', mode="a") log = "{}/{} {}:{}:{}, Epoch {}/{}, step=={}-->cur_acc = {:.3f}, best_step=={}-->best_acc = {:.3f}, lr={:.8f},batch_cost_avg={:.3f}\n" f.write( log.format(now.month, now.day, now.hour, now.minute, now.second, cur_epoch + 1, FLAGS.num_epochs, step, acc_avg, acc_best_step, acc_best, lr, batch_cost_avg)) f.close() if step % FLAGS.save_steps == 0 or acc_avg - acc_best > 0.0001: print("save checkpoint", step) if not os.path.isdir(FLAGS.checkpoint_dir): os.mkdir(FLAGS.checkpoint_dir) saver.save(sess, os.path.join(FLAGS.checkpoint_dir, 'ocr-model'), global_step=step)
def master(train_data, dev_data, utility, dat): #creates TF graph and calls trainer or evaluator batch_size = utility.FLAGS.batch_size model_dir = utility.FLAGS.output_dir + "/model_" + utility.FLAGS.job_id + "/" #create all paramters of the model param_class = parameters.Parameters(utility) params, global_step, init = param_class.parameters(utility) key = FLAGS.job_mode print("Running with key " + key) graph = model.Graph(utility, batch_size, utility.FLAGS.max_passes, mode=key) graph.create_graph(params, global_step) prev_dev_error = 0.0 final_loss = 0.0 final_accuracy = 0.0 #start session config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init.name) sess.run(graph.init_op.name) to_save = params.copy() saver = tf.train.Saver(to_save, max_to_keep=500) if (key == 'test'): model_file = 'model_' + utility.FLAGS.model_id print("restoring: ", model_file) saver.restore(sess, model_dir + model_file) Test(graph, utility, batch_size, sess, model_dir, dat, 'data/custom/uefa.examples') if (key == 'error-test'): while True: selected_models = {} file_list = tf.gfile.ListDirectory(model_dir) for model_file in file_list: if ("checkpoint" in model_file or "index" in model_file or "meta" in model_file): continue if ("data" in model_file): model_file = model_file.split(".")[0] model_step = int( model_file.split("_")[len(model_file.split("_")) - 1]) selected_models[model_step] = model_file file_list = sorted(selected_models.items(), key=lambda x: x[0]) if (len(file_list) > 0): file_list = file_list[0:len(file_list) - 1] print "list of models: ", file_list for model_file in file_list: model_file = model_file[1] print "restoring: ", model_file saver.restore(sess, model_dir + model_file) model_step = int( model_file.split("_")[len(model_file.split("_")) - 1]) print "evaluating on dev ", model_file, model_step evaluate(sess, dev_data, batch_size, graph, model_step) elif (key == 'train'): ckpt = tf.train.get_checkpoint_state(model_dir) print "model dir: ", model_dir if (not (tf.gfile.IsDirectory(utility.FLAGS.output_dir))): print "create dir: ", utility.FLAGS.output_dir tf.gfile.MkDir(utility.FLAGS.output_dir) if (not (tf.gfile.IsDirectory(model_dir))): print "create dir: ", model_dir tf.gfile.MkDir(model_dir) Train(graph, utility, batch_size, train_data, sess, model_dir, saver) elif (key == 'demo'): #create all paramters of the model model_file = 'model_' + utility.FLAGS.model_id print("restoring: ", model_file) saver.restore(sess, model_dir + model_file) if utility.FLAGS.mode == 'console': DemoConsole(graph, utility, sess, model_dir, dat) else: Demo(graph, utility, sess, model_dir, dat)
print(f"{person.name} <-> {skill.name}") ############################# # Model tests ############################# import model p1 = model.Person(1, "Alice") p2 = model.Person(2, "Bob") p3 = model.Person(3, "Charles") s1 = model.Skill(11, "Dancing") s2 = model.Skill(22, "Fishing") g = model.Graph() g.add_person(p1) g.add_person(p2) g.add_person(p3) g.add_skill(s1) g.add_skill(s2) g.link_person_to_skill(1, 11) g.link_person_to_skill(1, 22) g.link_person_to_skill(2, 11) g.link_person_to_skill(3, 22) assert g.people[p1.id] == p1 assert g.people[p2.id] == p2
def train(train_dir=None, val_dir=None, train_text_dir=None, val_text_dir=None): g = model.Graph(is_training=True) print('loading train data, please wait---------------------', 'end= ') train_feeder = utils.DataIterator2(data_dir=train_dir, text_dir=train_text_dir) print('get image: ', train_feeder.size) print('loading validation data, please wait---------------------', 'end= ') val_feeder = utils.DataIterator2(data_dir=val_dir, text_dir=val_text_dir) print('get image: ', val_feeder.size) num_train_samples = train_feeder.size num_batches_per_epoch = int(num_train_samples / FLAGS.batch_size) num_val_samples = val_feeder.size num_val_per_epoch = int(num_val_samples / FLAGS.batch_size) config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=False) config.gpu_options.allow_growth = True #config.gpu_options.per_process_gpu_memory_fraction = 0.6 with tf.Session(graph=g.graph, config=config) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=3) g.graph.finalize() train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) if FLAGS.restore: print("restore is true") ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from the checkpoint{0}'.format(ckpt)) print( '=============================begin training=============================' ) val_inputs, val_seq_len, val_labels = val_feeder.input_index_generate_batch( ) #print(len(val_inputs)) val_feed = { g.inputs: val_inputs, g.labels: val_labels, g.seq_len: np.array([g.cnn_time] * val_inputs.shape[0]) } for cur_epoch in range(FLAGS.num_epochs): shuffle_idx = np.random.permutation(num_train_samples) train_cost = 0 start_time = time.time() batch_time = time.time() #the tracing part for cur_batch in range(num_batches_per_epoch): if (cur_batch + 1) % 100 == 0: print('batch', cur_batch, ': time', time.time() - batch_time) batch_time = time.time() indexs = [ shuffle_idx[i % num_train_samples] for i in range(cur_batch * FLAGS.batch_size, (cur_batch + 1) * FLAGS.batch_size) ] batch_inputs, batch_seq_len, batch_labels = train_feeder.input_index_generate_batch( indexs) #batch_inputs,batch_seq_len,batch_labels=utils.gen_batch(FLAGS.batch_size) feed = { g.inputs: batch_inputs, g.labels: batch_labels, g.seq_len: np.array([g.cnn_time] * batch_inputs.shape[0]) } # if summary is needed #batch_cost,step,train_summary,_ = sess.run([cost,global_step,merged_summay,optimizer],feed) summary_str, batch_cost, step, _ = sess.run( [g.merged_summay, g.cost, g.global_step, g.optimizer], feed) #calculate the cost train_cost += batch_cost * FLAGS.batch_size train_writer.add_summary(summary_str, step) # save the checkpoint if step % FLAGS.save_steps == 0: print("save checkpoint", step) if not os.path.isdir(FLAGS.checkpoint_dir): os.mkdir(FLAGS.checkpoint_dir) logger.info('save the checkpoint of{0}', format(step)) saver.save(sess, os.path.join(FLAGS.checkpoint_dir, 'ocr-model'), global_step=step) #train_err+=the_err*FLAGS.batch_size #do validation if step % FLAGS.validation_steps == 0: dense_decoded, lastbatch_err, lr = sess.run( [g.dense_decoded, g.lerr, g.learning_rate], val_feed) # print the decode result acc = utils.accuracy_calculation(val_feeder.labels, dense_decoded, ignore_value=-1, isPrint=True) avg_train_cost = train_cost / ( (cur_batch + 1) * FLAGS.batch_size) #train_err/=num_train_samples now = datetime.datetime.now() log = "{}/{} {}:{}:{} step==={}, Epoch {}/{}, accuracy = {:.3f},avg_train_cost = {:.3f}, lastbatch_err = {:.3f}, time = {:.3f},lr={:.8f}\n" print( log.format(now.month, now.day, now.hour, now.minute, now.second, step, cur_epoch + 1, FLAGS.num_epochs, acc, avg_train_cost, lastbatch_err, time.time() - start_time, lr)) if Flag_Isserver: f = open('../log/acc/acc.txt', mode="a") f.write( log.format(now.month, now.day, now.hour, now.minute, now.second, step, cur_epoch + 1, FLAGS.num_epochs, acc, avg_train_cost, lastbatch_err, time.time() - start_time, lr)) f.close()
import pygame.locals class Player: def __init__(self, turn): self.score = 0 self.mistakes = 3 self.turn = turn self.path = np.array([], dtype='<U13') pygame.init() FPS = 20 stell_graph = model.Graph('Data.txt') start_3str, stop_3str = stell_graph.rnd_start_stop() current = stell_graph.constellations[start_3str] stop = stell_graph.constellations[stop_3str] window = pygame.display.set_mode((800, 670)) pygame.display.set_caption('ASTROWARS') def win_blit(window, master_file_name, name_file_name, start, stop, lasts): screen = pygame.image.load(master_file_name) info = pygame.image.load(name_file_name) window.blit(screen, (0, 150)) window.blit(info, (70, 0))