Пример #1
0
def eval_model(model_type, attack_level, num_modified_words, percentage_attacked_samples):
    print("evaluate")
    print("%s white-box adversarial attack modifies %d words of %d%% of the instances: " % (
        attack_level, num_modified_words, percentage_attacked_samples))

    global model_conf
    if model_type == "lstm":
        import movieqa.conf_lstm as model_conf
    else:
        import movieqa.conf_cnn as model_conf

    if not tf.io.gfile.exists(data_conf.EVAL_DIR):
        tf.io.gfile.makedirs(data_conf.EVAL_DIR)

    util.save_config_values(data_conf, data_conf.TRAIN_DIR + "/data")
    util.save_config_values(model_conf, data_conf.TRAIN_DIR + "/model")

    filepath = data_conf.EVAL_RECORD_PATH + '/*'
    filenames = glob.glob(filepath)
    print("Evaluating adversarial attack on %s" % filenames)

    global_step = tf.contrib.framework.get_or_create_global_step()
    dataset = tf.contrib.data.TFRecordDataset(filenames)
    dataset = dataset.map(get_single_sample)
    batch_size = 1

    dataset = dataset.padded_batch(batch_size, padded_shapes=(
        [None], [ANSWER_COUNT, None], [None], (), [None, None], ()))

    iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)

    next_q, next_a, next_l, next_plot_ids, next_plots, next_q_types = iterator.get_next()

    _, w_atts, s_atts, _ = predict_batch(model_type, [next_q, next_a, next_plots], training=False)

    if attack_level == "sentence":
        m_p = tf.compat.v1.py_func(remove_plot_sentence, [next_plots, s_atts, next_l], [tf.int64])[0]
    elif attack_level == "word":
        m_p = tf.compat.v1.py_func(modify_plot_sentence,
                         [next_plots, w_atts, s_atts, next_l, num_modified_words, percentage_attacked_samples],
                         [tf.int64])[0]

    logits, atts, sent_atts, pl_d = predict_batch(model_type, [next_q, next_a, m_p], training=False)

    next_q_types = tf.reshape(next_q_types, ())

    probabs = model.compute_probabilities(logits=logits)
    loss_example = model.compute_batch_mean_loss(logits, next_l, model_conf.LOSS_FUNC)
    accuracy_example = tf.reduce_mean(input_tensor=model.compute_accuracies(logits=logits, labels=next_l, dim=1))

    to_restore = tf.contrib.slim.get_variables_to_restore(exclude=["embeddings"])
    saver = tf.compat.v1.train.Saver(to_restore)
    summary_writer = tf.compat.v1.summary.FileWriter(data_conf.TRAIN_DIR)

    step = 0
    total_acc = 0.0
    total_prec = 0.0
    total_rank = 0.0
    total_loss = 0.0
    type_counts = np.zeros(6, dtype=np.int32)
    type_accs = np.zeros(6)
    max_sent_atts = {}
    max_atts = {}
    p_counts = 0
    last_p = ''
    with tf.compat.v1.Session() as sess:
        init_op = tf.group(tf.compat.v1.global_variables_initializer(), tf.compat.v1.local_variables_initializer())
        sess.run(init_op)
        ckpt = tf.train.get_checkpoint_state(data_conf.TRAIN_DIR)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
        else:
            print('No checkpoint file found')
        _ = sess.run(set_embeddings_op, feed_dict={place: vectors})
        coord = tf.train.Coordinator()
        threads = tf.compat.v1.train.start_queue_runners(sess=sess, coord=coord)
        try:
            while not coord.should_stop():
                loss_val, acc_val, probs_val, gs_val, q_type_val, q_val, atts_val, sent_atts_val, labels_val, p_val, a_val, p_id_val = sess.run(
                    [loss_example, accuracy_example, probabs, global_step, next_q_types, next_q, atts, sent_atts,
                     next_l,
                     pl_d, next_a, next_plot_ids])
                type_accs[q_type_val + 1] += acc_val
                type_counts[q_type_val + 1] += 1

                predicted_probabilities = probs_val[0]
                sentence_attentions = sent_atts_val[0]

                total_loss += loss_val
                total_acc += acc_val

                pred_index = np.argmax(probs_val[0])
                labels = labels_val[0]
                gold = np.argmax(labels)

                filename = ''
                q_s = ''
                for index in q_val[0]:
                    word = (vocab[index])
                    q_s += (word + ' ')
                    filename += (word + '_')

                p_id = str(p_id_val[0].decode("utf-8"))
                path = data_conf.EVAL_DIR + "/plots/" + p_id + "/" + filename

                corr_ans = np.argmax(labels_val[0])

                max_att_val = np.argmax(sent_atts_val[0][corr_ans])

                att_row = np.max(atts_val[0][corr_ans][max_att_val], 1)

                red = np.max(atts_val[0][corr_ans][max_att_val], 1)

                att_inds = np.argsort(red)[::-1]

                if (p_id != last_p and p_counts < 20):
                    for i, a_att in enumerate(atts_val[0]):
                        a_att = np.mean(a_att, 2)
                        qa_s = q_s + "? (acc: " + str(acc_val) + ")\n "
                        for index in a_val[0][i]:
                            qa_s += (vocab[index] + ' ')
                        lv = " (label: " + str(int(labels_val[0][i])) + " - prediction: " + (
                            str("%.2f" % (probs_val[0][i] * 100))) + "%)"
                        qa_s += lv

                        a_sents = []
                        y_labels = []

                        for j, att in enumerate(a_att):
                            a_s = []
                            y_labels.append(str("%.2f" % (sent_atts_val[0][i][j] * 100)) + "%")
                            for index in p_val[0][j]:
                                a_s.append(vocab[index])
                            a_sents.append(a_s)
                    # util.plot_attention(np.array(a_att), np.array(a_sents),qa_s,y_labels,path,filename)
                    last_p = p_id
                    p_counts += 1

                m_ap = util.example_precision(probs_val[0], labels_val[0], 5)
                rank = util.example_rank(probs_val[0], labels_val[0], 5)
                total_prec += m_ap
                total_rank += rank

                print("Sample loss: " + str(loss_val))
                print("Sample acc: " + str(acc_val))
                print("Sample prec: " + str(m_ap))
                print("Sample rank: " + str(rank))

                util.print_predictions(data_conf.EVAL_DIR, step, gold, predicted_probabilities, data_conf.MODE)
                util.print_sentence_attentions(data_conf.EVAL_DIR, step, sentence_attentions)

                step += 1

                print("Total acc: " + str(total_acc / step))
                print("Total prec: " + str(total_prec / step))
                print("Total rank: " + str(total_rank / step))
                print("Local_step: " + str(step * batch_size))
                print("Global_step: " + str(gs_val))
                if attack_level == "word":
                    print("%d modified word(s)" % num_modified_words)
                print("===========================================")
        except tf.errors.OutOfRangeError:

            summary = tf.compat.v1.Summary()
            summary.value.add(tag='validation_loss', simple_value=total_loss / step)
            summary.value.add(tag='validation_accuracy', simple_value=(total_acc / step))
            summary_writer.add_summary(summary, gs_val)
            keys = util.get_question_keys()
            with open(data_conf.EVAL_DIR + "/accuracy.txt", "a") as file:
                file.write("global step: " + str(gs_val) + " - total accuracy: " + str(
                    total_acc / step) + "- total loss: " + str(total_loss / step) + str(num_modified_words) + "" "\n")
                file.write("Types (name / count / correct / accuracy):\n")
                for entry in zip(keys, type_counts, type_accs, (type_accs / type_counts)):
                    file.write(str(entry) + "\n")
                file.write("===================================================================" + "\n")
                util.save_eval_score(
                    "global step: " + str(gs_val) + " - acc : " + str(
                        total_acc / step) + " - total loss: " + str(
                        total_loss / step) + " - " + data_conf.TRAIN_DIR + "_" + str(gs_val))
        finally:
            coord.request_stop()
        coord.join(threads)
Пример #2
0
def train_model(model_type, attack_level, num_modified_words, percentage_attacked_samples):
    print("train")
    print("%s white-box adversarial attack modifies %d words of %d%% of the instances: " % (
        attack_level, num_modified_words, percentage_attacked_samples))

    global model_conf
    if model_type == "lstm":
        import movieqa.conf_lstm as model_conf
    else:
        import movieqa.conf_cnn as model_conf

    global_step = tf.contrib.framework.get_or_create_global_step()

    init = False
    if not tf.io.gfile.exists(data_conf.TRAIN_DIR):
        init = True
        print("RESTORING WEIGHTS")
        tf.io.gfile.makedirs(data_conf.TRAIN_DIR)
    util.save_config_values(data_conf, data_conf.TRAIN_DIR + "/data")
    util.save_config_values(model_conf, data_conf.TRAIN_DIR + "/model")

    filenames = glob.glob(data_conf.TRAIN_RECORD_PATH + '/*')
    print("Reading training dataset from %s" % filenames)
    dataset = tf.contrib.data.TFRecordDataset(filenames)
    dataset = dataset.map(get_single_sample)
    dataset = dataset.shuffle(buffer_size=9000)
    dataset = dataset.repeat(data_conf.NUM_EPOCHS)
    batch_size = data_conf.BATCH_SIZE

    dataset = dataset.padded_batch(data_conf.BATCH_SIZE, padded_shapes=(
        [None], [ANSWER_COUNT, None], [None], (), [None, None], ()))

    iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)

    next_q, next_a, next_l, next_plot_ids, next_plots, next_q_types = iterator.get_next()

    _, w_atts, s_atts, _ = predict_batch(model_type, [next_q, next_a, next_plots], training=True)

    if attack_level == "sentence":
        m_p = tf.compat.v1.py_func(remove_plot_sentence, [next_plots, s_atts, next_l], [tf.int64])[0]
    elif attack_level == "word":
        m_p = tf.compat.v1.py_func(modify_plot_sentence, [next_plots, w_atts, s_atts, next_l], [tf.int64])[0]

    logits, _, _, _ = predict_batch(model_type, [next_q, next_a, m_p], training=True)

    probabs = model.compute_probabilities(logits=logits)
    loss_batch = model.compute_batch_mean_loss(logits, next_l, model_conf.LOSS_FUNC)
    accuracy = model.compute_accuracies(logits=logits, labels=next_l, dim=1)
    accuracy_batch = tf.reduce_mean(input_tensor=accuracy)
    tf.compat.v1.summary.scalar("train_accuracy", accuracy_batch)
    tf.compat.v1.summary.scalar("train_loss", loss_batch)

    training_op = update_op(loss_batch, global_step, model_conf.OPTIMIZER, model_conf.INITIAL_LEARNING_RATE)
    config = tf.compat.v1.ConfigProto()
    config.gpu_options.allow_growth = True
    config.allow_soft_placement = True

    with tf.compat.v1.train.MonitoredTrainingSession(
            checkpoint_dir=data_conf.TRAIN_DIR,
            save_checkpoint_secs=60,
            save_summaries_steps=5,
            hooks=[tf.estimator.StopAtStepHook(last_step=model_conf.MAX_STEPS),
                   ], config=config) as sess:
        step = 0
        total_acc = 0.0
        if init:
            _ = sess.run(set_embeddings_op, feed_dict={place: vectors})
        while not sess.should_stop():
            _, loss_val, acc_val, probs_val, lab_val, gs_val = sess.run(
                [training_op, loss_batch, accuracy_batch, probabs, next_l, global_step])

            print(probs_val)
            print(lab_val)
            print("Batch loss: " + str(loss_val))
            print("Batch acc: " + str(acc_val))
            step += 1
            total_acc += acc_val

            print("Total acc: " + str(total_acc / step))
            print("Local_step: " + str(step * batch_size))
            print("Global_step: " + str(gs_val))
            print("===========================================")
    util.copy_model(data_conf.TRAIN_DIR, gs_val)
Пример #3
0
def run_creation(model_type, attack, model_folder, examples_folder,
                 instances_to_attack):
    print("store created examples in %s" % examples_folder)
    if model_type == "lstm":
        import movieqa.run_lstm as runner
    else:
        import movieqa.run_cnn as runner

    runner.data_conf.TRAIN_DIR = model_folder

    load = False
    check_sents = []
    check_found = []
    check_num = 0
    corr_probs = []
    if not tf.gfile.Exists(examples_folder):
        tf.gfile.MakeDirs(examples_folder)
    else:
        checkpoints = glob.glob(examples_folder + "/[!accuracies]*")
        checkpoints = sorted(checkpoints, reverse=True)
        latest = checkpoints[0]
        splitted = latest.split(".txt")[0]
        check_num = int(splitted[len(splitted) - 1]) + 1

        check = open(latest, encoding="utf8")
        for line in check:
            parts = line.replace('\n', '').split("\t")
            check_words = parts[0].split(" ")
            check_sents.append(check_words)
            last_prob = float(parts[1])
            found = parts[2]
            if found == 'True':
                b_found = True
            else:
                b_found = False
            corr_probs.append(last_prob)
            check_found.append(b_found)

        load = True

    emb_dir = runner.data_conf.EMBEDDING_DIR

    vectors, vocab = util.load_embeddings(emb_dir)
    rev_vocab = dict(zip(vocab.values(), vocab.keys()))
    # print(rev_vocab)
    filename = "adversarial_addAny/common_english.txt"
    # length of the distractor sentence
    d = 10
    # pool size of common words to sample from for each word in the distractor sentence
    poolsize = 10
    common_words = {}
    fin = open(filename, encoding="utf8")
    for line in fin:
        word = line.replace('\n', '')
        # print(word)
        if word in rev_vocab:
            common_words[word] = rev_vocab[word]
        else:
            print(
                'ERROR: word "%s" not in vocab. Run add_common_words_to_vocab.py first.'
                % word)
            exit(1)

    with open(instances_to_attack + '/val.pickle', 'rb') as handle:
        qa = pickle.load(handle)

    w_s = []
    w_choices = []
    w_found = []

    q_inds = []
    pools = []
    with open(examples_folder + "/" + str(0 + check_num) + ".txt",
              "a") as file:
        for k, question in enumerate(qa):
            # load question indices
            q_words = util.normalize_text(question.question)
            q_ind = []
            for word in q_words:
                q_ind.append(rev_vocab[word])

            a_words = []
            for i, answer in enumerate(question.answers):
                if not i == int(question.correct_index):
                    words = util.normalize_text(answer)
                    a_words.extend(words)
            w = []
            w_choice = []
            rand_sent = ""
            for i in range(0, d):
                if load:
                    c_word = check_sents[k][i]
                    w_index = rev_vocab[c_word]
                    rand_sent += (c_word + " ")

                else:
                    w_index = random.choice(list(common_words.values()))
                    rand_sent += (vocab[w_index] + " ")
                    w_found.append(False)
                w.append(w_index)
                w_choice.append(i)

            if load:
                found = check_found[k]
                w_found.append(found)
                # file.write(rand_sent+"\t"+str(corr_probs[k])+"\t"+str(found)+"\n")
            else:
                found = False
                w_found.append(found)
                file.write(rand_sent + "\t" + "1.0" + "\t" + str(found) + "\n")

            shuffle(w_choice)
            w_choices.append(w_choice)

            w_s.append(w)
            d_pools = []
            for j, dj in enumerate(w):
                pool = []
                random_common_words = np.random.choice(list(
                    common_words.values()),
                                                       poolsize,
                                                       replace=False)
                print("Adding common words")
                pool.extend(random_common_words)
                if attack == 'addQ' or attack == "addQA":
                    print("Adding question words")
                    for word in q_words:
                        pool.append(rev_vocab[word])
                if attack == "addA" or attack == "addQA":
                    print("Adding answer words")
                    for word in a_words:
                        pool.append(rev_vocab[word])

                shuffle(pool)
                d_pools.append(pool)
            pools.append(d_pools)

    filepath = instances_to_attack + "/*.tfrecords"
    filenames = glob.glob(filepath)

    global_step = tf.contrib.framework.get_or_create_global_step()
    dataset = tf.contrib.data.TFRecordDataset(filenames)
    dataset = dataset.map(runner.get_single_sample)
    dataset = dataset.repeat(poolsize * d)
    batch_size = 1

    dataset = dataset.padded_batch(batch_size,
                                   padded_shapes=([None], [5, None], [None],
                                                  (), [None, None], ()))

    iterator = dataset.make_one_shot_iterator()

    next_q, next_a, next_l, next_plot_ids, next_plots, next_q_types = iterator.get_next(
    )
    add_sent = tf.placeholder(tf.int64, shape=[None])
    # sent_exp = tf.expand_dims(add_sent,0)
    m_p = tf.py_func(add_plot_sentence, [next_plots, add_sent], [tf.int64])[0]
    # m_p = next_plots
    # m_p = tf.concat([next_plots,sent_exp],axis=0)

    logits, atts, sent_atts, _ = runner.predict_batch([next_q, next_a, m_p],
                                                      training=False)

    probabs = model.compute_probabilities(logits=logits)
    accuracy_example = tf.reduce_mean(
        model.compute_accuracies(logits=logits, labels=next_l, dim=1))

    to_restore = tf.contrib.slim.get_variables_to_restore(
        exclude=["embeddings"])
    saver = tf.train.Saver(to_restore)

    p_counts = 0
    last_p = ''
    p_id = 0
    f_counter = 0
    with tf.Session() as sess:
        init_op = tf.group(tf.global_variables_initializer(),
                           tf.local_variables_initializer())
        sess.run(init_op)
        ckpt = tf.train.get_checkpoint_state(runner.data_conf.TRAIN_DIR)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
        else:
            print('No checkpoint file found')
        _ = sess.run(runner.set_embeddings_op,
                     feed_dict={runner.place: vectors})
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)
        if not load:
            accs = np.ones(shape=(len(qa)))
        else:
            accs = corr_probs
        for w_counter in range(0, d):
            words = np.zeros(shape=(len(qa)), dtype=np.int64)
            # select next word to optimize greedily
            next_inds = []
            for k, question in enumerate(qa):
                next_word = w_choices[k].pop()
                next_inds.append(next_word)
                words[k] = w_s[k][next_word]

            # go through whole pool for every question
            next_ind = 0
            for pool_counter in range(0, poolsize):
                total_acc = 0.0
                info = ""

                for k, question in enumerate(qa):
                    w_copy = [x for x in w_s[k]]
                    print("==============")
                    next_ind = next_inds[k]
                    pool = pools[k][next_ind]

                    pool_ind = pool.pop()

                    print("setting " + str(w_s[k][next_ind]) + " to " +
                          str(pool_ind))
                    w_copy[next_ind] = pool_ind
                    info = "wordcounter: " + str(
                        w_counter) + " - poolcounter: " + str(
                            pool_counter) + " - question: " + str(k)
                    print(info)
                    acc_val, probs_val, gs_val, q_type_val, q_val, atts_val, sent_atts_val, labels_val, p_val, a_val, p_id_val = sess.run(
                        [
                            accuracy_example, probabs, global_step,
                            next_q_types, next_q, atts, sent_atts, next_l, m_p,
                            next_a, next_plot_ids
                        ],
                        feed_dict={add_sent: w_copy})
                    sent = ""
                    for word in w_copy:
                        sent += (" " + vocab[word])
                    print(sent + " - acc: " + str(acc_val))
                    corr = np.argmax(labels_val[0])
                    pred_val = probs_val[0][corr]
                    if (pred_val < accs[k]):
                        word_s = vocab[words[k]]
                        pool_s = vocab[pool_ind]
                        print(pool_s + " (" + str(pred_val) + ") < " + word_s +
                              " (" + str(accs[k]) + ")")
                        words[k] = pool_ind
                        accs[k] = pred_val
                        if acc_val == 0:
                            print("setting" + str(k) + " to true with acc" +
                                  str(acc_val) + " and pred " + str(pred_val))
                            w_found[k] = True
                            f_counter += 1

                    filename = ''
                    q_s = ''
                    for index in q_val[0]:
                        word = (vocab[index])
                        q_s += (word + ' ')
                        filename += (word + '_')
                    predicted_probabilities = probs_val[0]
                    labels = labels_val[0]

                    p_id = 'test'
                    path = runner.data_conf.EVAL_DIR + "/plots/" + p_id + "/" + filename
                    if (p_counts < 20):
                        for i, a_att in enumerate(atts_val[0]):
                            # a_att = np.max(a_att, 1)
                            qa_s = q_s + "? (acc: " + str(acc_val) + ")\n "
                            for index in a_val[0][i]:
                                qa_s += (vocab[index] + ' ')
                            lv = " (label: " + str(int(
                                labels[i])) + " - prediction: " + (str(
                                    "%.2f" %
                                    (predicted_probabilities[i] * 100))) + "%)"
                            qa_s += lv

                            a_sents = []
                            y_labels = []

                            for j, att in enumerate(a_att):
                                a_s = []
                                y_labels.append(
                                    str("%.2f" %
                                        (sent_atts_val[0][i][j] * 100)) + "%")
                                for index in p_val[0][j]:
                                    a_s.append(vocab[index])
                                a_sents.append(a_s)
                            util.plot_attention(np.array(a_att),
                                                np.array(a_sents), qa_s,
                                                y_labels, path, filename)
                        last_p = p_id
                        p_counts += 1
                    total_acc += acc_val
                    print(total_acc / (k + 1))
                with open(examples_folder + "/accuracies.txt", "a") as file:
                    file.write(info + " - " + str(total_acc / (len(qa))) +
                               "\n")

            with open(
                    examples_folder + "/" + str(w_counter + check_num + 1) +
                    ".txt", "a") as file:
                for k, question in enumerate(qa):
                    w_s[k][next_ind] = words[k]
                    sent = ""
                    for word in w_s[k]:
                        sent += (vocab[word] + " ")
                    file.write(sent + "\t" + str(accs[k]) + "\t" +
                               str(w_found[k]) + "\n")
Пример #4
0
def eval_model():
    if not tf.io.gfile.exists(data_conf.EVAL_DIR):
        tf.io.gfile.makedirs(data_conf.EVAL_DIR)

    util.save_config_values(data_conf, data_conf.EVAL_DIR + "/data_")
    util.save_config_values(model_conf, data_conf.EVAL_DIR + "/model_")

    filepath = data_conf.EVAL_RECORD_PATH + '/*'
    filenames = glob.glob(filepath)

    global_step = tf.compat.v1.train.get_or_create_global_step()
    dataset = tf.data.TFRecordDataset(filenames)
    dataset = dataset.map(get_single_sample)
    batch_size = 1

    dataset = dataset.padded_batch(batch_size,
                                   padded_shapes=([None], [ANSWER_COUNT, None],
                                                  [None], (), [None,
                                                               None], ()))

    iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)

    next_q, next_a, next_l, next_plot_ids, next_plots, next_q_types = iterator.get_next(
    )

    logits, word_atts, sent_atts, pl_d = predict_batch(
        [next_q, next_a, next_plots], training=False)

    next_q_types = tf.reshape(next_q_types, ())

    probabs = model.compute_probabilities(logits=logits)
    loss_example = model.compute_batch_mean_loss(logits, next_l,
                                                 model_conf.LOSS_FUNC)
    accuracy_example = tf.reduce_mean(input_tensor=model.compute_accuracies(
        logits=logits, labels=next_l, dim=1))

    # do not restore embeddings in case the vocabulary size has changed
    #to_restore = tf.contrib.slim.get_variables_to_restore(exclude=["embeddings"])

    saver = tf.compat.v1.train.Saver()
    summary_writer = tf.compat.v1.summary.FileWriter(data_conf.TRAIN_DIR)

    step = 0
    total_acc = 0.0
    total_loss = 0.0
    p_counts = 0
    last_p = ''
    type_counts = np.zeros(6, dtype=np.int32)
    type_accs = np.zeros(6)
    with tf.compat.v1.Session() as sess:
        init_op = tf.group(tf.compat.v1.global_variables_initializer(),
                           tf.compat.v1.local_variables_initializer())
        sess.run(init_op)
        ckpt = tf.train.get_checkpoint_state(data_conf.TRAIN_DIR)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
        else:
            print('No checkpoint file found')
        coord = tf.train.Coordinator()
        threads = tf.compat.v1.train.start_queue_runners(sess=sess,
                                                         coord=coord)
        sess.run(set_embeddings_op, feed_dict={place: vectors})
        try:
            while not coord.should_stop():
                loss_val, acc_val, probs_val, gs_val, q_type_val, q_val, labels_val, p_val, a_val, p_id_val, atts_val, \
                sent_atts_val = sess.run([loss_example, accuracy_example, probabs, global_step, next_q_types, next_q,
                                          next_l, next_plots, next_a, next_plot_ids, word_atts, sent_atts])

                total_loss += loss_val
                total_acc += acc_val

                predicted_probabilities = probs_val[0]
                sentence_attentions = sent_atts_val[0]

                pred_index = np.argmax(probs_val[0])
                labels = labels_val[0]
                gold = np.argmax(labels)

                filename = ''
                q_s = ''
                for index in q_val[0]:
                    word = (vocab[index])
                    q_s += (word + ' ')
                    filename += (word + '_')

                filename += "?"

                p_id = str(p_id_val[0].decode("utf-8"))
                path = data_conf.EVAL_DIR + "/plots/" + p_id + "_" + str(
                    step) + "/"  # + filename

                # write attention heat-map
                if (p_id != last_p and p_counts < data_conf.PLOT_SAMPLES_NUM):
                    # if True:
                    for i, a_att in enumerate(atts_val[0]):
                        # a_att = np.mean(a_att, 2)
                        qa_s = q_s + "? (acc: " + str(acc_val) + ")\n "
                        for index in a_val[0][i]:
                            word = vocab[index]
                            qa_s += (word + ' ')
                            filename += word + "_"
                        lv = " (label: " + str(int(
                            labels[i])) + " - prediction: " + (str(
                                "%.2f" %
                                (predicted_probabilities[i] * 100))) + "%)"
                        qa_s += lv

                        a_sents = []
                        y_labels = []

                        for j, att in enumerate(a_att):
                            a_s = []
                            y_labels.append(
                                str("%.2f" % (sent_atts_val[0][i][j] * 100)) +
                                "%")
                            for index in p_val[0][j]:
                                a_s.append(vocab[index])
                            a_sents.append(a_s)
                        util.plot_attention(np.array(a_att), np.array(a_sents),
                                            qa_s, y_labels, path, filename)
                    last_p = p_id
                    p_counts += 1

                print("Sample loss: " + str(loss_val))
                print("Sample labels: " + str(labels))
                print("Sample probabilities: " + str(predicted_probabilities))
                print("Sample acc: " + str(acc_val))

                util.print_predictions(data_conf.EVAL_DIR, step, gold,
                                       predicted_probabilities, data_conf.MODE)
                util.print_sentence_attentions(data_conf.EVAL_DIR, step,
                                               sentence_attentions)

                step += 1

                print("Total acc: " + str(total_acc / step))
                print("Local_step: " + str(step * batch_size))
                print("Global_step: " + str(gs_val))
                print("===========================================")
        except tf.errors.OutOfRangeError:
            summary = tf.compat.v1.Summary()
            summary.value.add(tag='validation_loss',
                              simple_value=total_loss / step)
            summary.value.add(tag='validation_accuracy',
                              simple_value=(total_acc / step))
            summary_writer.add_summary(summary, gs_val)
            keys = util.get_question_keys()
            if data_conf.MODE == "val":
                with open(data_conf.EVAL_DIR + "/val_accuracy.txt",
                          "a") as file:
                    file.write("global step: " + str(gs_val) +
                               " - total accuracy: " + str(total_acc / step) +
                               "- total loss: " + str(total_loss / step) +
                               "\n")
                    file.write("Types (name / count / correct / accuracy):\n")
                    for entry in zip(keys, type_counts, type_accs,
                                     (type_accs / type_counts)):
                        file.write(str(entry) + "\n")
                    file.write(
                        "==================================================================="
                        + "\n")
                    util.save_eval_score("global step: " + str(gs_val) +
                                         " - acc : " + str(total_acc / step) +
                                         " - total loss: " +
                                         str(total_loss / step) + " - " +
                                         data_conf.TRAIN_DIR + "_" +
                                         str(gs_val))
        finally:
            coord.request_stop()
        coord.join(threads)
Пример #5
0
def train_model():
    print("train RNN-LSTM model")
    global_step = tf.compat.v1.train.get_or_create_global_step()

    if not tf.io.gfile.exists(data_conf.TRAIN_DIR):
        print("RESTORING WEIGHTS")
        tf.io.gfile.makedirs(data_conf.TRAIN_DIR)
    util.save_config_values(data_conf, data_conf.TRAIN_DIR + "/data")
    util.save_config_values(model_conf, data_conf.TRAIN_DIR + "/model")

    filenames = glob.glob(data_conf.TRAIN_RECORD_PATH + '/*')
    print("Reading training dataset from %s" % filenames)
    dataset = tf.data.TFRecordDataset(filenames)
    dataset = dataset.map(get_single_sample)
    dataset = dataset.shuffle(buffer_size=9000)
    dataset = dataset.repeat(model_conf.NUM_EPOCHS)
    batch_size = model_conf.BATCH_SIZE

    dataset = dataset.padded_batch(model_conf.BATCH_SIZE,
                                   padded_shapes=([None], [ANSWER_COUNT, None],
                                                  [None], (), [None,
                                                               None], ()))

    iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)

    next_q, next_a, next_l, next_plot_ids, next_plots, next_q_types = iterator.get_next(
    )

    logits, _, _, _ = predict_batch([next_q, next_a, next_plots],
                                    training=True)

    probabs = model.compute_probabilities(logits=logits)
    loss_batch = model.compute_batch_mean_loss(logits, next_l,
                                               model_conf.LOSS_FUNC)
    accuracy = model.compute_accuracies(logits=logits, labels=next_l, dim=1)
    accuracy_batch = tf.reduce_mean(input_tensor=accuracy)
    tf.compat.v1.summary.scalar("train_accuracy", accuracy_batch)
    tf.compat.v1.summary.scalar("train_loss", loss_batch)

    training_op = update_op(loss_batch, global_step, model_conf.OPTIMIZER,
                            model_conf.INITIAL_LEARNING_RATE)
    config = tf.compat.v1.ConfigProto()
    config.gpu_options.allow_growth = True
    config.allow_soft_placement = True
    config.graph_options.optimizer_options.global_jit_level = tf.compat.v1.OptimizerOptions.OFF

    with tf.compat.v1.train.MonitoredTrainingSession(
            checkpoint_dir=data_conf.TRAIN_DIR,
            save_checkpoint_secs=60,
            save_summaries_steps=5,
            hooks=[
                tf.estimator.StopAtStepHook(last_step=model_conf.MAX_STEPS),
            ],
            config=config) as sess:
        step = 0
        total_acc = 0.0
        # print("Feeding embeddings %s of size %s" % (str(vectors), len(vectors)))
        sess.run(set_embeddings_op, feed_dict={place: vectors})
        while not sess.should_stop():
            _, loss_val, acc_val, probs_val, lab_val, gs_val = sess.run([
                training_op, loss_batch, accuracy_batch, probabs, next_l,
                global_step
            ])
            print(probs_val)
            print(lab_val)
            print("Batch loss: " + str(loss_val))
            print("Batch acc: " + str(acc_val))
            step += 1
            total_acc += acc_val

            print("Total acc: " + str(total_acc / step))
            print("Local_step: " + str(step * batch_size))
            print("Global_step: " + str(gs_val))
            print("===========================================")
    util.copy_model(data_conf.TRAIN_DIR, gs_val)
Пример #6
0
def eval_model():
    if not tf.gfile.Exists(data_conf.EVAL_DIR):
        tf.gfile.MakeDirs(data_conf.EVAL_DIR)

    util.save_config_values(data_conf, data_conf.EVAL_DIR + "/data")
    util.save_config_values(model_conf, data_conf.EVAL_DIR + "/model")

    filepath = data_conf.EVAL_RECORD_PATH + '/*'
    filenames = glob.glob(filepath)

    print("Evaluate model on %s" % str(filenames))

    global_step = tf.contrib.framework.get_or_create_global_step()
    dataset = tf.contrib.data.TFRecordDataset(filenames)
    dataset = dataset.map(get_single_sample)
    batch_size = 1

    dataset = dataset.padded_batch(batch_size,
                                   padded_shapes=([None], [ANSWER_COUNT, None],
                                                  [None], (), [None], ()))

    iterator = dataset.make_one_shot_iterator()

    next_q, next_a, next_l, next_plot_ids, next_plots, next_q_types = iterator.get_next(
    )

    logits = predict_batch([next_q, next_a, next_plots], training=False)

    next_q_types = tf.reshape(next_q_types, ())

    probabs = model.compute_probabilities(logits=logits)
    loss_example = model.compute_batch_mean_loss(logits, next_l,
                                                 model_conf.LOSS_FUNC)
    accuracy_example = tf.reduce_mean(
        model.compute_accuracies(logits=logits, labels=next_l, dim=1))

    saver = tf.train.Saver()
    summary_writer = tf.summary.FileWriter(data_conf.TRAIN_DIR)

    step = 0
    total_acc = 0.0
    total_loss = 0.0
    type_counts = np.zeros(6, dtype=np.int32)
    type_accs = np.zeros(6)
    with tf.Session() as sess:
        init_op = tf.group(tf.global_variables_initializer(),
                           tf.local_variables_initializer())
        sess.run(init_op)
        ckpt = tf.train.get_checkpoint_state(data_conf.TRAIN_DIR)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
        else:
            print('No checkpoint file found')
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)
        try:
            while not coord.should_stop():
                loss_val, acc_val, probs_val, gs_val, q_type_val, labels_val = sess.run(
                    [
                        loss_example, accuracy_example, probabs, global_step,
                        next_q_types, next_l
                    ])

                predicted_probabilities = probs_val[0]
                pred_index = np.argmax(probs_val[0])
                labels = labels_val[0]
                gold = np.argmax(labels)

                type_accs[q_type_val + 1] += acc_val
                type_counts[q_type_val + 1] += 1

                total_loss += loss_val
                total_acc += acc_val

                print("Sample loss: " + str(loss_val))
                print("Sample acc: " + str(acc_val))

                util.print_predictions(data_conf.EVAL_DIR, step, gold,
                                       predicted_probabilities, data_conf.MODE)

                step += 1

                print("Total acc: " + str(total_acc / step))
                print("Local_step: " + str(step * batch_size))
                print("Global_step: " + str(gs_val))
                print("===========================================")

        except tf.errors.OutOfRangeError:
            summary = tf.Summary()
            summary.value.add(tag='validation_loss',
                              simple_value=total_loss / step)
            summary.value.add(tag='validation_accuracy',
                              simple_value=(total_acc / step))
            summary_writer.add_summary(summary, gs_val)
            keys = util.get_question_keys()

            if data_conf.MODE == "val":
                with open(data_conf.EVAL_DIR + "/val_accuracy.txt",
                          "a") as file:
                    file.write("global step: " + str(gs_val) +
                               " - total accuracy: " + str(total_acc / step) +
                               "- total loss: " + str(total_loss / step) +
                               "\n")
                    file.write("Types (name / count / correct / accuracy):\n")
                    for entry in zip(keys, type_counts, type_accs,
                                     (type_accs / type_counts)):
                        file.write(str(entry) + "\n")
                    file.write(
                        "==================================================================="
                        + "\n")
                    util.save_eval_score("global step: " + str(gs_val) +
                                         " - acc : " + str(total_acc / step) +
                                         " - total loss: " +
                                         str(total_loss / step) + " - " +
                                         data_conf.TRAIN_DIR + "_" +
                                         str(gs_val))
        finally:
            coord.request_stop()
        coord.join(threads)