def main(_): if not tf.gfile.Exists(FLAGS.eval_log_dir): tf.gfile.MakeDirs(FLAGS.eval_log_dir) dataset = common_flags.create_dataset(split_name=FLAGS.split_name) model = common_flags.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code) data = data_provider.get_data( dataset, FLAGS.batch_size, augment=False, central_crop_size=common_flags.get_crop_size()) print("JIBIRI!") print(data.images) endpoints = model.create_base(data.images, labels_one_hot=None) model.create_loss(data, endpoints) eval_ops = model.create_summaries(data, endpoints, dataset.charset, is_training=False) slim.get_or_create_global_step() session_config = tf.ConfigProto(device_count={"GPU": 0}) checkpoint_path = "%s/model.ckpt-90482" % FLAGS.train_log_dir eval_result = slim.evaluation.evaluate_once( master=FLAGS.master, checkpoint_path=checkpoint_path, logdir=FLAGS.eval_log_dir, eval_op=eval_ops, session_config=session_config)
def main(_): prepare_training_dir() dataset = common_flags.create_dataset(split_name=FLAGS.split_name) model = common_flags.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code) hparams = get_training_hparams() # If ps_tasks is zero, the local device is used. When using multiple # (non-local) replicas, the ReplicaDeviceSetter distributes the variables # across the different devices. device_setter = tf.train.replica_device_setter(FLAGS.ps_tasks, merge_devices=True) with tf.device(device_setter): data = data_provider.get_data( dataset, FLAGS.batch_size, augment=hparams.use_augment_input, central_crop_size=common_flags.get_crop_size()) endpoints = model.create_base(data.images, data.labels_one_hot) total_loss = model.create_loss(data, endpoints) model.create_summaries(data, endpoints, dataset.charset, is_training=True) init_fn = model.create_init_fn_to_restore(FLAGS.checkpoint, FLAGS.checkpoint_inception) if FLAGS.show_graph_stats: tf.logging.info('Total number of weights in the graph: %s', calculate_graph_metrics()) train(total_loss, init_fn, hparams)
def main(_): if not tf.gfile.Exists(FLAGS.eval_log_dir): tf.gfile.MakeDirs(FLAGS.eval_log_dir) dataset = common_flags.create_dataset(split_name=FLAGS.split_name) model = common_flags.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code) data = data_provider.get_data( dataset, FLAGS.batch_size, augment=False, central_crop_size=common_flags.get_crop_size()) endpoints = model.create_base(data.images, labels_one_hot=None) model.create_loss(data, endpoints) eval_ops = model.create_summaries( data, endpoints, dataset.charset, is_training=False) slim.get_or_create_global_step() session_config = tf.ConfigProto(device_count={"GPU": 0}) slim.evaluation.evaluation_loop( master=FLAGS.master, checkpoint_dir=FLAGS.train_log_dir, logdir=FLAGS.eval_log_dir, eval_op=eval_ops, num_evals=FLAGS.num_batches, eval_interval_secs=FLAGS.eval_interval_secs, max_number_of_evaluations=FLAGS.number_of_steps, session_config=session_config)
def main(_): if not tf.gfile.Exists(FLAGS.eval_log_dir): tf.gfile.MakeDirs(FLAGS.eval_log_dir) dataset = common_flags.create_dataset(FLAGS.dataset_name, FLAGS.dataset_split_name) model = common_flags.create_model(num_classes=FLAGS.num_classes) data = data_provider.get_data(dataset, FLAGS.model_name, FLAGS.batch_size, is_training=False, height=FLAGS.height, width=FLAGS.width) logits, endpoints = model.create_model(data.images, num_classes=FLAGS.num_classes, is_training=False) eval_ops = model.create_summary(data, logits, is_training=False) slim.get_or_create_global_step() session_config = tf.ConfigProto() session_config.gpu_options.allow_growth = True slim.evaluation.evaluation_loop( master=FLAGS.master, checkpoint_dir=FLAGS.train_dir, logdir=FLAGS.eval_log_dir, eval_op=eval_ops, num_evals=FLAGS.num_evals, eval_interval_secs=FLAGS.eval_interval_secs, max_number_of_evaluations=FLAGS.number_of_steps, session_config=session_config)
def local_train(stemmer=data_provider.NoStemmer(), text_representation='bag-of-words', create_model=dnn_model): data_provider.init_data_provider(ngrams=False) X, Y = data_provider.get_data(input_format='hot_vector' if create_model == dnn_model else 'sequential', output_format='categorical', ngrams=text_representation == 'ngrams', all_data=True) model = create_model(len(X[0]), len(Y[0])) X_train, X_val = split(X, 0.9) Y_train, Y_val = split(Y, 0.9) data_provider.STATE = data_provider.initial_state() del X, Y gc.collect() run_id = get_run_id(stemmer, text_representation, create_model) print(run_id) for i in range(40): model.fit(X_train, Y_train, n_epoch=5, validation_set=(X_val, Y_val), show_metric=True, run_id=run_id) Y_pred = get_predictions(model, X_val) compute_metrics(Y_val, Y_pred)
def initial(self): dataset = common_flags.create_dataset(split_name=FLAGS.split_name) model = common_flags.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code, charset=dataset.charset) data = data_provider.get_data( dataset, FLAGS.batch_size, augment=False, central_crop_size=common_flags.get_crop_size()) self.image_height = int(data.images.shape[1]) self.image_width = int(data.images.shape[2]) self.image_channel = int(data.images.shape[3]) self.num_of_view = dataset.num_of_views placeholder_shape = (1, self.image_height, self.image_width, self.image_channel) print placeholder_shape self.placeholder = tf.placeholder(tf.float32, shape=placeholder_shape) self.endpoint = model.create_base(self.placeholder, labels_one_hot=None) init_fn = model.create_init_fn_to_restore(FLAGS.checkpoint) self.sess = tf.Session(config=config) tf.tables_initializer().run(session=self.sess) init_fn(self.sess)
def main(_): if not tf.gfile.Exists(FLAGS.eval_log_dir): tf.gfile.MakeDirs(FLAGS.eval_log_dir) dataset = common_flags.create_dataset(FLAGS.dataset_name, FLAGS.dataset_split_name) model = common_flags.create_model(num_classes=FLAGS.num_classes) data = data_provider.get_data(dataset, FLAGS.model_name, FLAGS.batch_size, is_training=False, height=FLAGS.height, width=FLAGS.width) logits, endpoints = model.create_model(data.images, num_classes=FLAGS.num_classes, is_training=False) eval_ops = model.create_summary(data, logits, is_training=False) slim.get_or_create_global_step() session_config = tf.ConfigProto() session_config.gpu_options.allow_growth = True slim.evaluation.evaluation_loop( master=FLAGS.master, checkpoint_dir=FLAGS.train_dir, logdir=FLAGS.eval_log_dir, eval_op=eval_ops, num_evals=FLAGS.num_evals, eval_interval_secs=FLAGS.eval_interval_secs, max_number_of_evaluations=FLAGS.number_of_steps, session_config=session_config)
def main(_): if not tf.gfile.Exists(FLAGS.eval_log_dir): tf.gfile.MakeDirs(FLAGS.eval_log_dir) dataset = common_flags.create_dataset(split_name=FLAGS.split_name) model = common_flags.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code) data = data_provider.get_data( dataset, FLAGS.batch_size, augment=False, central_crop_size=common_flags.get_crop_size()) endpoints = model.create_base(data.images, labels_one_hot=None) model.create_loss(data, endpoints) eval_ops = model.create_summaries(data, endpoints, dataset.charset, is_training=False) slim.get_or_create_global_step() session_config = tf.ConfigProto(device_count={"GPU": 0}) ### session_config.gpu_options.allow_growth = True session_config.log_device_placement = False ### slim.evaluation.evaluation_loop( master=FLAGS.master, checkpoint_dir=FLAGS.train_log_dir, logdir=FLAGS.eval_log_dir, eval_op=eval_ops, num_evals=FLAGS.num_batches, eval_interval_secs=FLAGS.eval_interval_secs, max_number_of_evaluations=1, session_config=session_config)
def main(_): prepare_training_dir() logging.info('dataset_name: {}, split_name: {}'.format( FLAGS.dataset_name, FLAGS.dataset_split_name)) dataset = common_flags.create_dataset(FLAGS.dataset_name, FLAGS.dataset_split_name) model = common_flags.create_model(num_classes=FLAGS.num_classes) data = data_provider.get_data(dataset, FLAGS.model_name, batch_size=FLAGS.batch_size, is_training=True, height=FLAGS.height, width=FLAGS.width) logits, endpoints = model.create_model(data.images, num_classes=dataset.num_classes, weight_decay=FLAGS.weight_decay, is_training=True) total_loss = model.create_loss(logits, endpoints, data.labels_one_hot, FLAGS.label_smoothing) model.create_summary(data, logits, is_training=True) init_fn = model.create_init_fn_to_restore(FLAGS.checkpoint_path, FLAGS.checkpoint_inception, FLAGS.checkpoint_exclude_scopes) variables_to_train = model.get_variables_to_train(FLAGS.trainable_scopes) if FLAGS.show_graph_state: logging.info('Total number of weights in the graph: %s', utils.calculate_graph_metrics()) train(total_loss, init_fn, variables_to_train)
def run_svm_instance(actor, args={}): X, Y = data_provider.get_data('../scrapper/out/unijokes.json', input_format='hot_vector', output_format='categorical', stemmer=nltk.stem.lancaster.LancasterStemmer()) model = create_model(len(X[0]), len(Y[0])) asyncio.ensure_future(svm_instance_process(actor, args))
def main(_): prepare_training_dir() num_char_classes = 74 max_sequence_length = 37 num_of_views = 4 null_code = 2 model = common_flags.create_model(num_char_classes, max_sequence_length, num_of_views,null_code) hparams = get_training_hparams() # If ps_tasks is zero, the local device is used. When using multiple # (non-local) replicas, the ReplicaDeviceSetter distributes the variables # across the different devices. device_setter = tf.train.replica_device_setter( FLAGS.ps_tasks, merge_devices=True) with tf.device(device_setter): images_orig = tf.placeholder(tf.float32, [32, 150, 600,3], name='image_orig') labels = tf.placeholder(tf.int32, [32,37, ], name='label') data = data_provider.get_data( images_orig,labels, num_of_views, num_char_classes) #augment=hparams.use_augment_input, #central_crop_size=None) endpoints = model.create_base(data.images,data.labels_one_hot) saver = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() saver.restore(sess,'./trained/my-model-240') filelist = os.listdir('./real_train') # print(filelist[:10]) rates = list() for num in range(20): images = list() for i in range(32): image = np.resize(np.array(Image.open(os.path.join('./real_train',filelist[i+32*num]))),(150,600,3)) images.append(image) images = np.array(images) y = np.array(np.random.randint(10, size=[32,37, ]), dtype=np.int32) result = sess.run(endpoints.predicted_chars,feed_dict={images_orig:images,labels:y}) f = open('./vocab.txt') line = f.readline() f.close() rate = 0 for i in range(32): words = idx_to_words(list(result[i])) # print(filelist[i][:-4]) # print(''.join(words)) rate += distance.levenshtein(filelist[i+32*num][:-4],words)/float(len(filelist[i+32*num][:-4])) rate = 1 - float(rate) / 32 # print(rate) rates.append(rate) print(sum(rates)/20)
def test_optionally_applies_central_crop(self): batch_size = 4 data = data_provider.get_data( dataset=datasets.fsns_test.get_test_split(), batch_size=batch_size, augment=True, central_crop_size=(500, 100)) with self.test_session() as sess, queues.QueueRunners(sess): images_np = sess.run(data.images) self.assertEqual(images_np.shape, (batch_size, 100, 500, 3))
def test_provided_data_has_correct_shape(self): batch_size = 4 data = data_provider.get_data( dataset=datasets.fsns_test.get_test_split(), batch_size=batch_size, augment=True, central_crop_size=None) with self.test_session() as sess, queues.QueueRunners(sess): images_np, labels_np = sess.run([data.images, data.labels_one_hot]) self.assertEqual(images_np.shape, (batch_size, 150, 600, 3)) self.assertEqual(labels_np.shape, (batch_size, 37, 134))
def test_labels_correctly_shuffled(self): batch_size = 4 data = data_provider.get_data( dataset=datasets.fsns_test.get_test_split(), batch_size=batch_size, augment=True, central_crop_size=None) with self.test_session() as sess, queues.QueueRunners(sess): images, labels, probs, texts = sess.run( [data.images, data.labels, data.probs, data.texts]) for i in range(batch_size * batch_size): plt.imshow(images[i]) print(texts[i], probs[i], labels[i])
def test_provided_data_has_correct_shape(self): dataset_name = 'flowers' model_name = 'inception_v3' for split_name in ['train']: is_training = True if split_name == 'train' else False dataset = common_flags.create_dataset(dataset_name, split_name) batch_size = 4 data = data_provider.get_data(dataset, model_name, batch_size=batch_size, is_training=is_training, height=224, width=224) with self.test_session() as sess, slim.queues.QueueRunners(sess): images_np, labels_np = sess.run([data.images, data.labels]) self.assertEqual(images_np.shape, (batch_size, 224, 224, 3))
def train(): record_names = ["sudoku_train"] * 50 + ["mnist_train"] record_paths = ["data/{}.tfrecords".format(x) for x in record_names] # Load data in batches. images, labels = data_provider.get_data(record_paths, batch_size=FLAGS.batch_size, is_training=True) # Define network with slim.arg_scope([slim.layers.dropout, slim.batch_norm], is_training=True): predictions = model(images) # Display images to tensorboard tf.image_summary('images', images, max_images=5) # Define loss function slim.losses.softmax_cross_entropy(predictions, labels) total_loss = slim.losses.get_total_loss() tf.scalar_summary('loss', total_loss) # Create learning rate decay global_step = slim.get_or_create_global_step() learning_rate = tf.train.exponential_decay(FLAGS.initial_learning_rate, global_step=global_step, decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate) # Optimizer to use. optimizer = tf.train.AdamOptimizer(learning_rate) # Create training operation train_op = slim.learning.create_train_op(total_loss, optimizer) logging.set_verbosity(1) # Start training slim.learning.train(train_op, FLAGS.log_dir, save_summaries_secs=20, save_interval_secs=60, log_every_n_steps=100)
def local_train(stemmer=data_provider.NoStemmer(), text_representation='bag-of-words', C=1, max_iter=10000): data_provider.STATE['stemmer'] = stemmer X, Y = data_provider.get_data(input_format='hot_vector', output_format='numerical', ngrams=text_representation=='ngrams', all_data=True) model = create_model(C, max_iter) X_train, X_val = split(X, 0.9) Y_train, Y_val = split(Y, 0.9) data_provider.STATE = data_provider.initial_state() del X, Y gc.collect() #X_train, X_val, Y_train, Y_val = np.array(X_train), np.array(X_val), np.array(Y_train), np.array(Y_val) data_provider.STATE = data_provider.initial_state() print(">>> {} {} {} {} {}".format(type(stemmer).__name__, text_representation, 'svm', C, max_iter)) start = time.time() model.fit(X_train, Y_train) print(">>> TRAINING TIME: {}s".format(time.time() - start)) Y_pred = model.predict(X_val) compute_metrics(Y_val, Y_pred)
def main(_): tmp_capStore = os.path.abspath('.') + 'caption.txt' opts, args = getopt.getopt(sys.argv[1:], "hi:o:") input_file = "" output_file = tmp_capStore for op, value in opts: if op == "-i": input_file = value elif op == "-o": output_file = value elif op == "-h": usage() sys.exit() dataset = utils.create_dataset(split_name=split_name) model = utils.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code) data = data_provider.get_data(dataset, batch_size, augment=False, central_crop_size=utils.get_crop_size()) endpoints = model.create_base(data.images, labels_one_hot=None) model.create_loss(data, endpoints) eval_ops = model.create_summaries(data, endpoints, dataset.charset, is_training=False) slim.get_or_create_global_step() session_config = tf.ConfigProto(device_count={"GPU": 0}) slim.evaluation.evaluation_loop(master=master, checkpoint_dir=train_log_dir, logdir=eval_log_dir, eval_op=eval_ops, num_evals=num_batches, eval_interval_secs=eval_interval_secs, max_number_of_evaluations=number_of_steps, session_config=session_config)
def main(_): dataset = common_flags.create_dataset(split_name=FLAGS.split_name) model = common_flags.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code, charset=dataset.charset) data = data_provider.get_data( dataset, FLAGS.batch_size, augment=False, central_crop_size=common_flags.get_crop_size()) input_image = Image.open(FLAGS.input_image).convert("RGB").resize((data.images.shape[2] / dataset.num_of_views, data.images.shape[1])) input_array = np.array(input_image).astype(np.float32) #input_array = np.concatenate((input_array, input_array, input_array, input_array), axis=1) #Image.fromarray(input_array.astype(np.uint8), "RGB").save("test_input_1.jpg") input_array = np.expand_dims(input_array, axis=0) print input_array.shape print input_array.dtype #input_image.save("test_input.jpg") #return placeholder_shape = (1, data.images.shape[1], data.images.shape[2], data.images.shape[3]) print placeholder_shape image_placeholder = tf.placeholder(tf.float32, shape=placeholder_shape) endpoints = model.create_base(image_placeholder, labels_one_hot=None) init_fn = model.create_init_fn_to_restore(FLAGS.checkpoint) with tf.Session() as sess: tf.tables_initializer().run() # required by the CharsetMapper init_fn(sess) predictions = sess.run(endpoints.predicted_text, feed_dict={image_placeholder: input_array}) print("Predicted strings:") for line in predictions: print(line)
from data_provider import get_data from makiflow.trainers import SegmentatorTrainer import makiflow as mf import os if __name__ == "__main__": mf.set_main_gpu(0) Xtrain, Ytrain, num_pos, Xtest, Ytest = get_data() os.makedirs('experiments', exist_ok=True) trainer = SegmentatorTrainer('exp_params.json', 'experiments') trainer.set_test_data(Xtest, Ytest) trainer.set_train_data(Xtrain, Ytrain, num_pos) trainer.start_experiments()
def main(_): # 检查训练目录 prepare_training_dir() # 建立数据集 split_name: train test dataset = common_flags.create_dataset(split_name=FLAGS.split_name) # 建立模型 max_sequence_length: 37, num_of_views: 4, null_code:133 # 这里还没有创建模型,只是返回了模型类,和初始化了模型相关参数 model = common_flags.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code) hparams = get_training_hparams() # If ps_tasks is zero, the local device is used. When using multiple # (non-local) replicas, the ReplicaDeviceSetter distributes the variables # across the different devices. device_setter = tf.train.replica_device_setter(FLAGS.ps_tasks, merge_devices=True) with tf.device(device_setter): # 获得训练数据 data = data_provider.get_data( dataset, FLAGS.batch_size, augment=hparams.use_augment_input, central_crop_size=common_flags.get_crop_size()) # 打印 dataset 的数据,看一下 # print("#######################") # print("images:", data.images) # print("labels:", data.labels) # print(dir(data.labels)) # print("labels_one_hot:", data.labels_one_hot.shape) # print("labels_0:", data.labels[0]) # print("labels_one_host_0:", data.labels_one_hot[0]) # print("#######################") # init = tf.global_variables_initializer() # with tf.Session() as session: # session.run(init) # coord = tf.train.Coordinator() # threads = tf.train.start_queue_runners(coord=coord) # labels = session.run(data.labels) # print(labels[0]) # labels = session.run(data.labels_one_hot) # print(labels[0]) # coord.request_stop() # coord.join(threads) # return # 创建模型 endpoints = model.create_base(data.images, data.labels_one_hot) # 创建损失函数 total_loss = model.create_loss(data, endpoints) model.create_summaries(data, endpoints, dataset.charset, is_training=True) # 恢复数据 init_fn = model.create_init_fn_to_restore(FLAGS.checkpoint, FLAGS.checkpoint_inception) if FLAGS.show_graph_stats: logging.info('Total number of weights in the graph: %s', calculate_graph_metrics()) train(total_loss, init_fn, hparams)
def main(_): prepare_training_dir() dataset = common_flags.create_dataset(split_name=FLAGS.split_name) model = common_flags.create_model(dataset.num_char_classes, dataset.max_sequence_length, dataset.num_of_views, dataset.null_code) hparams = get_training_hparams() # If ps_tasks is zero, the local device is used. When using multiple # (non-local) replicas, the ReplicaDeviceSetter distributes the variables # across the different devices. #device_setter = tf.train.replica_device_setter( # FLAGS.ps_tasks, merge_devices=True) with tf.device("/cpu:0"): provider = data_provider.get_data( dataset, FLAGS.batch_size, augment=hparams.use_augment_input, central_crop_size=common_flags.get_crop_size()) batch_queue = slim.prefetch_queue.prefetch_queue([ provider.images, provider.images_orig, provider.labels, provider.labels_one_hot ], capacity=2 * FLAGS.num_clones) losses = [] for i in xrange(FLAGS.num_clones): with tf.name_scope("clone_{0}".format(i)): with tf.device("/gpu:{0}".format(i)): #if i == 1: # continue images, images_orig, labels, labels_one_hot = batch_queue.dequeue( ) if i == 0: endpoints = model.create_base(images, labels_one_hot) else: endpoints = model.create_base(images, labels_one_hot, reuse=True) init_fn = model.create_init_fn_to_restore( FLAGS.checkpoint, FLAGS.checkpoint_inception) if FLAGS.show_graph_stats: logging.info('Total number of weights in the graph: %s', calculate_graph_metrics()) data = InputEndpoints(images=images, images_orig=images_orig, labels=labels, labels_one_hot=labels_one_hot) total_loss, single_model_loss = model.create_loss( data, endpoints) losses.append((single_model_loss, i)) with tf.device("/cpu:0"): tf.summary.scalar('model_loss'.format(i), single_model_loss) model.create_summaries_multigpu(data, endpoints, dataset.charset, i, is_training=True) train_multigpu(losses, init_fn, hparams)
import tensorflow as tf import os import model import data_provider MODEL_SAVE_PATH = "path/to/model" if not os.path.exists(MODEL_SAVE_PATH): os.makedirs(MODEL_SAVE_PATH) MODEL_NAME = "face.ckpt" global_step = tf.Variable(0, trainable=False) image, label = data_provider.get_data("train") num_classes = 2 learning_rate = 0.001 batch_size = 30 TRAINING_ROUNDS = int(40000 / batch_size) capacity = 1000 + 3 * batch_size image_batch, label_batch = tf.train.batch([image, label], batch_size=batch_size, capacity=capacity, allow_smaller_final_batch=True) print(image_batch, label_batch) x = tf.placeholder(tf.float32, [None, 64 * 64]) y = tf.placeholder(tf.float32, [None, num_classes]) keep_prob = tf.placeholder(tf.float32) #dropout (keep probability)
def main(_): prepare_training_dir() num_char_classes = 77 #less than 37 max_sequence_length = 25 num_of_views = 4 # null_code = 42 null_code = 2 model = common_flags.create_model(num_char_classes, max_sequence_length, num_of_views,null_code) hparams = get_training_hparams() # If ps_tasks is zero, the local device is used. When using multiple # (non-local) replicas, the ReplicaDeviceSetter distributes the variables # across the different devices. device_setter = tf.train.replica_device_setter( FLAGS.ps_tasks, merge_devices=True) with tf.device(device_setter): images_orig = tf.placeholder(tf.float32, [64, 150, 600,3], name='image_orig') labels = tf.placeholder(tf.int32, [64,max_sequence_length, ], name='label') data = data_provider.get_data( images_orig,labels, num_of_views, num_char_classes) # augment=hparams.use_augment_input, # central_crop_size=None) endpoints = model.create_base(data.images,data.labels_one_hot) total_loss = model.create_loss(labels, endpoints) init_fn = model.create_init_fn_to_restore(FLAGS.checkpoint, FLAGS.checkpoint_inception) train_op = train(total_loss, init_fn, hparams) saver = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() images_dataset,labels_dataset = prehandle_data.get_data() print('data prepared') for epoch in range(400): ids = np.random.permutation(images_dataset.shape[0]) images_permutation = images_dataset[ids] labels_permutation = labels_dataset[ids] for step in range(images_dataset.shape[0] / 64): result, loss, _ = sess.run([endpoints.predicted_chars,total_loss,train_op],feed_dict={images_orig:images_permutation[step*64:(step+1)*64],labels:labels_permutation[step*64:(step+1)*64]}) # print(loss) if step % 200 == 0: print(loss) target = labels_permutation[step*64:(step+1)*64] leven_distance = list() count = 0 for h_shape in range(result.shape[0]): tmp_result = list(result[h_shape]) target_result = list(target[h_shape]) if count < 5: f = open('./aaa.txt','a+') f.write(''.join(idx_to_words(tmp_result))+'\n') f.write(''.join(idx_to_words(target_result))+'\n') f.write('--------------------------------------'+'\n') f.close() count += 1 target_start = target_result.index(1)+1 target_end = target_result.index(2) divide = float(target_end - target_start) if 1 not in tmp_result and 2 not in tmp_result: leven_distance.append((target_end-target_start)/divide) if 1 in tmp_result and 2 not in tmp_result: leven_distance.append(distance.levenshtein(tmp_result[tmp_result.index(1)+1:],target_result[target_start:target_end])/divide) if 1 not in tmp_result and 2 in tmp_result: leven_distance.append(distance.levenshtein(tmp_result[:tmp_result.index(2)],target_result[target_start:target_end])/divide) if 1 in tmp_result and 2 in tmp_result: if(tmp_result.index(1) < tmp_result.index(2)): leven_distance.append(distance.levenshtein(tmp_result[tmp_result.index(1)+1:tmp_result.index(2)],target_result[target_start:target_end])/divide) else: leven_distance.append((target_end-target_start)/divide) summary = sum(leven_distance) / 64 print(summary) f = open('result_1.txt','a+') f.write(str(loss)+'\n') f.write(str(summary)+'\n') f.close() if epoch % 40 == 0: saver.save(sess,'./trained/my-model',global_step=epoch)
import json import re import sys import matplotlib.pyplot as plt from collections import OrderedDict from nltk.stem import * import data_provider data_provider.init_data_provider(ngrams=True) X, Y = data_provider.get_data(all_data=True) print(len(X), len(Y)) # plt.title('unijokes raw dataset words (number of different words {})'.format(len(words_counter))) # plt.bar(range(len(words_counter)), sorted(words_counter.values(), reverse=True), align='center', alpha=0.5, color='orange') # plt.xticks(range(10), ordered.keys(), rotation='vertical') # plt.show()
def main(_): prepare_training_dir() num_char_classes = 3215 #less than 37 max_sequence_length = 37 num_of_views = 4 # null_code = 42 null_code = 2 model = common_flags.create_model(num_char_classes, max_sequence_length, num_of_views, null_code) hparams = get_training_hparams() # If ps_tasks is zero, the local device is used. When using multiple # (non-local) replicas, the ReplicaDeviceSetter distributes the variables # across the different devices. device_setter = tf.train.replica_device_setter(FLAGS.ps_tasks, merge_devices=True) with tf.device(device_setter): images_orig = tf.placeholder(tf.float32, [32, 150, 600, 3], name='image_orig') labels = tf.placeholder(tf.int32, [ 32, max_sequence_length, ], name='label') data = data_provider.get_data(images_orig, labels, num_of_views, num_char_classes) # augment=hparams.use_augment_input, # central_crop_size=None) endpoints = model.create_base(data.images, data.labels_one_hot) total_loss = model.create_loss(labels, endpoints) init_fn = model.create_init_fn_to_restore(FLAGS.checkpoint, FLAGS.checkpoint_inception) train_op = train(total_loss, init_fn, hparams) saver = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() filelist = os.listdir('./chinese_data/ad_flag') f = open('./chinese_data/chinese_vocab.txt') char_list = f.readline() f.close() count = 0 for epoch in range(10): random.shuffle(filelist) for step in range(len(filelist) / 32): count += 1 images_permutation, labels_permutation = handle_data( filelist[step * 32:(step + 1) * 32], char_list) result, loss, _ = sess.run( [endpoints.predicted_chars, total_loss, train_op], feed_dict={ images_orig: images_permutation, labels: labels_permutation }) if step % 2 == 0: print(loss) target = labels_permutation leven_distance = list() for h_shape in range(result.shape[0]): tmp_result = list(result[h_shape]) target_result = list(target[h_shape]) target_start = target_result.index(1) + 1 target_end = target_result.index(2) divide = float(target_end - target_start) if 1 not in tmp_result and 2 not in tmp_result: leven_distance.append( (target_end - target_start) / divide) if 1 in tmp_result and 2 not in tmp_result: leven_distance.append( distance.levenshtein( tmp_result[tmp_result.index(1) + 1:], target_result[target_start:target_end]) / divide) if 1 not in tmp_result and 2 in tmp_result: leven_distance.append( distance.levenshtein( tmp_result[:tmp_result.index(2)], target_result[target_start:target_end]) / divide) if 1 in tmp_result and 2 in tmp_result: if (tmp_result.index(1) < tmp_result.index(2)): leven_distance.append( distance.levenshtein( tmp_result[tmp_result.index(1) + 1:tmp_result.index(2)], target_result[ target_start:target_end]) / divide) else: leven_distance.append( (target_end - target_start) / divide) summary = sum(leven_distance) / 32 print(summary) f = open('chinese_result.txt', 'a+') f.write(str(loss) + '\n') f.write(str(summary) + '\n') f.close() if count % 10000 == 0: saver.save(sess, './chinese_trained/my-model', global_step=count)
import tensorflow as tf import os import model import data_provider TEXT_EXAMPLES = 800 num_classes = 2 global_step = tf.Variable(0, trainable=False) x = tf.placeholder(tf.float32, [None, 64 * 64]) y = tf.placeholder(tf.float32, [None, num_classes]) keep_prob = tf.placeholder(tf.float32) #dropout (keep probability) image, label = data_provider.get_data("test") image = tf.reshape(image, shape=(1, 64 * 64)) label = tf.reshape(label, shape=(1, 2)) logit = model.create_model(x, num_classes, keep_prob) correct_pred = tf.equal(tf.argmax(logit, 1), tf.argmax(y, 1)) correct_num = 0 saver = tf.train.Saver() checkpoint_dir = "path/to/model/" with tf.Session() as sess: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord)
import tensorflow as tf import data_provider import numpy as np image_orig = tf.placeholder(tf.int32, [None, 150, 360, 3], name='image_orig') label = tf.placeholder(tf.int32, [ None, ], name='label') return_data = data_provider.get_data(image_orig, label, 4, 134) with tf.Session() as sess: x = np.array(np.random.randint(10, size=[2, 150, 360, 3]), dtype=np.int32) y = np.array(np.random.randint(10, size=[ 2, ]), dtype=np.int32) result = sess.run(return_data, feed_dict={image_orig: x, label: y}) print(result)