Example #1
0
def display_metrics (c_file, results, verbose_mode):

    filename=os.path.basename(c_file)
    n_loc=results[0]
    n_bl=results[1]
    n_com=results[2]
    per_bl= utils.percent(n_bl, n_loc)
    per_com= utils.percent(n_com, n_loc)

    #
    if verbose_mode >=1 :
        print "%s : %s LOC with %s BL(%.1f%%) and %s comments(%.1f%%)" % (filename, n_loc, n_bl, per_bl, n_com, per_com)
Example #2
0
def determiner(person=None, gender=None, number=None, function='S',
               kind=None, S=None):
    if percent(35):
        det = wf.getNominal('article', gender, number)
        next = (det.value.index, 'article')
    elif percent(50):
        det = wf.getAdjectivePronoun(gender, number, 'determiner')
        next = (det.value.index, 'adjective_pronoun')
    else:
        det = wf.getPossessivePronoun(
            gender, number, aleatory('person'), aleatory('number'))
        next = (det.value.index, 'possessive_pronoun')
    return Tree('determiner', [det], {'next': next})
Example #3
0
def nounPhrase(person=None, gender=None, number=None, function='S',
               kind=None, S=None):

    if person is None:
        if S and verify_semantics(S, 'PESSOA'):
            person = aleatory('person')
        else:
            person = '3'
    if kind is None:
        if person != '3' or percent(15):
            kind = 'personal_pronoun'
        else:
            kind = 'noun'

    L = []
    next = None
    if kind == 'noun':
        det = determiner(person, gender, number, function, kind, S)
        L.append(det)
        next = det.info['next']
    n = nounBar(person, gender, number, function, kind, S)

    if not next:
        next = n.info['next']

    L.append(n)
    return Tree('noun phrase', L, {'head': n.info['head'], 'next': next})
Example #4
0
 def __get_stats(info, field):
     l = info[field]
     total = float(info['total'])
     stats = { }
     for e in l:
         stats[e['term']] = utils.percent(float(e['count']) / total)
     return stats
Example #5
0
def adnominalAdjunct(person=None, gender=None, number=None, function='S',
                     kind=None, S=None, position='pos'):
    L = []

    if kind == 'noun':
        if percent(20):
            if percent(50):
                # menino feio
                # bola feia
                adjective = wf.getNominal('adjective', gender, number, S)
                L.append(adjective)
            else:
                if percent(40):
                    # menino que correu
                    # bola que caiu
                    L = [wf.getRelativePronoun(gender, number, None),
                         verbPhrase(person, gender, number, S=S)]
                elif percent(50):
                    # menino que a mãe ama
                    # bola que o menino chutou
                    L = [wf.getRelativePronoun(gender, number, None),
                         clause(tran='vtd', OD=S)]
                else:
                    # menino de quem a mãe gosta
                    # bola da qual o menino gosta
                    that_clause = clause(tran='vti', OI=S)
                    preposition = wf.getPreposition(
                        that_clause.info['preposition'], gender,
                        number, None)
                    L = [
                        preposition,
                        wf.getRelativePronoun(
                            gender, number, S
                        ),
                        that_clause
                    ]

    elif kind == 'personal_pronoun' and function == 'S':
        if percent(20):
            # eu que corri
            L = [wf.getRelativePronoun(gender, number, None),
                 verbPhrase(person, gender, number, S=S)]

    if L:
        return Tree('adnominal adjunct', L, {})
    else:
        return None
Example #6
0
def main():
    sess = tf.Session()
    images = tf.placeholder(tf.float32, [1, 224, 224, 3])
    vgg = vgg16.Vgg16()
    vgg.forward(images)
    while True:
        img_path = input('\nInput the path and image name: ')

        if img_path == '0':
            print('\nexiting-------------------------\n')
            exit(0)
        else:
            print('\nrunning-------------------------\n')
        img_ready = utils.load_image(img_path)

        fig = plt.figure(u"Top-5 预测结果")

        probability = sess.run(vgg.prob, feed_dict={images: img_ready})
        top5 = np.argsort(probability[0])[-1:-6:-1]
        print("top5:", top5)
        values = []
        bar_label = []
        for n, i in enumerate(top5):
            print("n:", n)
            print("i:", i)
            values.append(probability[0][i])
            bar_label.append(labels[i])
            print(i, ":", labels[i], "----", utils.percent(probability[0][i]))

        ax = fig.add_subplot(111)
        ax.bar(range(len(values)),
               values,
               tick_label=bar_label,
               width=0.5,
               fc='g')
        ax.set_ylabel(u'probabilityit')
        ax.set_title(u'Top-5')
        for a, b in zip(range(len(values)), values):
            ax.text(a,
                    b + 0.0005,
                    utils.percent(b),
                    ha='center',
                    va='bottom',
                    fontsize=7)
        plt.show()
Example #7
0
def nounBar(person=None, gender=None, number=None, function='S',
            kind=None, S=None):
    noun_ = noun(person, gender, number, function, kind, S)
    adn_adj = None
    if percent(50):
        adn_adj = adnominalAdjunct(person, gender, number, function, kind, S)
    if adn_adj:
        L = [noun_, adn_adj]
    else:
        L = [noun_, ]
    return Tree('noun-bar', L, {'head': noun_.info['head'],
                                'next': noun_.info['next']})
Example #8
0
    def __get_stats(info, field):
        """Get stats about info[field]

        Args:
            info (dict): data
            field (str): field name

        Returns:
            dict: stats for each entry
        """
        l = info[field]
        total = float(info['total'])
        stats = {}
        for e in l:
            stats[e['term']] = utils.percent(float(e['count']) / total)
        return stats
Example #9
0
    def __get_stats(info, field):
        """Get stats about info[field]

        Args:
            info (dict): data
            field (str): field name

        Returns:
            dict: stats for each entry
        """
        l = info[field]
        total = float(info['total'])
        stats = {}
        for e in l:
            stats[e['term']] = utils.percent(float(e['count']) / total)
        return stats
Example #10
0
    def __get_bt_stats(info, credentials):
        hits = info['hits']
        uuids = [ hit['uuid'] for hit in hits ]
        bt = Backtrace(uuids, fraction = 1, credentials = credentials)#0.5)
        bt_info = bt.get()
        total = 0
        rec = 0.
        weird_address = { }
        os_cpu = { }
        cpu_info = { }
        reason = { }
        cycles = { }
        if bt_info:
            recursive_bt = 0
            addrs = { }
                        
            total = float(len(bt_info))
            for v in bt_info.itervalues():
                _cycles = v['cycles']
                if _cycles:
                    recursive_bt += 1
                    cycles[_cycles] = cycles[_cycles] + 1 if _cycles in cycles else 1
                        
                addr = v['address']
                t = (addr, v['cpu_name'])
                addrs[t] = addrs[t] + 1 if t in addrs else 1
                t = (v['os'], v['cpu_name'])
                os_cpu[t] = os_cpu[t] + 1 if t in os_cpu else 1
                ci = v['cpu_info']
                cpu_info[ci] = cpu_info[ci] + 1 if ci in cpu_info else 1
                r = v['reason']
                reason[r] = reason[r] + 1 if r in reason else 1
                
            rec = utils.percent(float(recursive_bt) / total)

            for k, v in os_cpu.iteritems():
                os_cpu[k] = utils.percent(float(v) / total)

            for k, v in cpu_info.iteritems():
                cpu_info[k] = utils.percent(float(v) / total)

            for k, v in reason.iteritems():
                reason[k] = utils.percent(float(v) / total)

            for k, v in addrs.iteritems():
                percent = float(v) / total
                if Track.__is_weird_address(*k):
                    weird_address[k] = utils.percent(percent)
                elif percent >= 0.25:
                    weird_address[k] = utils.percent(percent)

            total = int(total)

        return { 'sample_size': total, 'bt_has_rec': rec, 'weird_address': weird_address, 'cycles': cycles, 'os_cpu': os_cpu, 'cpu_info': cpu_info }
def print_status_bar(stdscr, bookwnd_nav):
    if not bookwnd_nav.show_status_bar:
        return

    perc = utils.percent(bookwnd_nav.line_number,
                         bookwnd_nav.book_number_lines())

    if bookwnd_nav.show_percentage_points:
        lines_to_new_p_point = utils.lines_to_change_percentage_point(
            bookwnd_nav.line_number, bookwnd_nav.book_number_lines())
        status_text = f"{bookwnd_nav.line_number} of {bookwnd_nav.book_number_lines()}      (%{perc:.1f})  (> {lines_to_new_p_point})"
    else:
        status_text = f"{bookwnd_nav.line_number} of {bookwnd_nav.book_number_lines()}      (%{perc:.1f})"

    pos_height = bookwnd_nav.window_height - 1
    pos_width = bookwnd_nav.window_width
    stdscr.attron(curses.color_pair(STATUSBAR_COLOR_PAIRCODE))
    stdscr.addstr(pos_height, pos_width // 2, status_text)
    stdscr.attroff(curses.color_pair(1))
Example #12
0
def prepositionalPhrase(preposition=None, person=None, gender=None,
                        number=None, function='S', kind=None, S=None):

    if person is None:
        if S and verify_semantics(S, 'PESSOA'):
            person = aleatory('person')
        else:
            person = '3'
    if kind is None:
        if person != '3' or percent(15):
            kind = 'personal_pronoun'
        else:
            kind = 'noun'

    np = nounPhrase(person, gender, number, function, kind, S)
    next = np.info['next']
    L = [np]

    p = wf.getPreposition(preposition, gender, number, next)

    L = [Tree("preposition", [p])] + L

    return Tree('prepositional phrase', L, {'head': np.info['head']})
Example #13
0
fig = plt.figure(u'Top-5预测结果')

with tf.Session() as sess:
    images = tf.placeholder(tf.float32, shape=[1, 224, 224, 3])
    vgg = vgg16.vgg16()
    vgg.forward(images)
    probability = sess.run(vgg.prob, feed_dict={images: img_ready})
    top5 = np.argsort(probability[0])[-1:-6:-1]
    print('top5:', top5)
    values = []
    bar_label = []
    for n, i, in enumerate(top5):
        print('n:', n)
        print('i', i)
        values.append(probability[0][i])
        bar_label.append(labels[i])
        print(i, ':', labels[i], '----', utils.percent(probability[0][i]))

    ax = fig.add_subplot(111)
    ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g')
    ax.set_ylabel(u'probablity')
    ax.set_title(u'Top-5')
    for a, b in zip(range(len(values)), values):
        ax.text(a,
                b + 0.0005,
                utils.percent(b),
                ha='center',
                va='bottom',
                fontsize=7)
    plt.show()
Example #14
0
def main():
    main_start = time.time()
    parser = argparse.ArgumentParser()

    parser.add_argument("--train_path",
                        type=str,
                        help="path to train file",
                        default="./train.txt")
    parser.add_argument("--validation_path",
                        type=str,
                        help="path to validation file",
                        default="")
    parser.add_argument("--label_prefix",
                        type=str,
                        help="label prefix",
                        default="__label__")
    parser.add_argument(
        "--min_word_count",
        type=int,
        default=1,
        help="discard words which appear less than this number")
    parser.add_argument(
        "--min_label_count",
        type=int,
        default=1,
        help="discard labels which appear less than this number")
    parser.add_argument("--dim",
                        type=int,
                        default=100,
                        help="length of embedding vector")
    parser.add_argument("--n_epochs",
                        type=int,
                        default=5,
                        help="number of epochs")
    parser.add_argument("--word_ngrams",
                        type=int,
                        default=1,
                        help="word ngrams")
    parser.add_argument("--batch_size",
                        type=int,
                        default=1024,
                        help="batch size for train")
    parser.add_argument(
        "--batch_size_inference",
        type=int,
        default=1024,
        help=
        "batch size for inference, ignored if validation_path is not provided")
    parser.add_argument("--seed", type=int, default=17)
    parser.add_argument("--learning_rate",
                        type=float,
                        default=0.1,
                        help="learning rate")
    parser.add_argument("--learning_rate_multiplier",
                        type=float,
                        default=0.8,
                        help="learning rate multiplier after each epoch")
    parser.add_argument(
        "--data_fraction",
        type=float,
        default=1,
        help=
        "data fraction, if < 1, train (and validation) data will be randomly sampled"
    )
    parser.add_argument("--use_validation",
                        type=int,
                        default=0,
                        help="evaluate on validation data")
    parser.add_argument("--use_gpu",
                        type=int,
                        default=0,
                        help="use gpu for training")
    parser.add_argument("--gpu_fraction",
                        type=float,
                        default=0.5,
                        help="what fraction of gpu to allocate")
    parser.add_argument("--result_dir",
                        type=str,
                        help="result dir",
                        default="./results/")

    args = parser.parse_args()
    for bool_param in [args.use_validation, args.use_gpu]:
        assert bool_param in [0, 1]
    train_path = args.train_path
    validation_path = args.validation_path
    label_prefix = args.label_prefix
    min_word_count = args.min_word_count
    min_label_count = args.min_label_count
    emb_dim = args.dim
    n_epochs = args.n_epochs
    word_ngrams = args.word_ngrams
    batch_size = args.batch_size
    batch_size_inference = args.batch_size_inference
    initial_learning_rate = args.learning_rate
    learning_rate_multiplier = args.learning_rate_multiplier
    use_validation = bool(args.use_validation)
    seed = args.seed
    data_fraction = args.data_fraction
    use_gpu = bool(args.use_gpu)
    gpu_fraction = args.gpu_fraction
    result_dir = validate(args.result_dir)

    print('training with arguments:')
    print(args)
    print('\n')

    np.random.seed(seed)

    train_descs, train_labels, max_words = parse_txt(train_path,
                                                     return_max_len=True,
                                                     debug_till_row=-1,
                                                     fraction=data_fraction,
                                                     seed=seed,
                                                     label_prefix=label_prefix)

    model_params = {
        "word_ngrams":
        word_ngrams,
        "word_id_path":
        os.path.abspath(os.path.join(result_dir, "word_id.json")),
        "label_dict_path":
        os.path.abspath(os.path.join(result_dir, "label_dict.json"))
    }

    for child_dir in os.listdir(result_dir):
        dir_tmp = os.path.join(result_dir, child_dir)
        if os.path.isdir(dir_tmp):
            shutil.rmtree(dir_tmp)
        if dir_tmp.endswith(".pb"):
            os.remove(dir_tmp)

    max_words_with_ng = 1
    for ng in range(word_ngrams):
        max_words_with_ng += max_words - ng

    print("preparing dataset")
    print("total number of datapoints: {}".format(len(train_descs)))
    print("max number of words in description: {}".format(max_words))
    print("max number of words with n-grams in description: {}".format(
        max_words_with_ng))

    label_dict_path = os.path.join(result_dir, "label_dict.json")
    word_id_path = os.path.join(result_dir, "word_id.json")

    train_vocab = make_train_vocab(train_descs, word_ngrams)
    label_vocab = make_label_vocab(train_labels)

    if min_word_count > 1:
        tmp_cnt = 1
        train_vocab_thresholded = {}
        for k, v in sorted(train_vocab.items(), key=lambda t: t[0]):
            if v["cnt"] >= min_word_count:
                v["id"] = tmp_cnt
                train_vocab_thresholded[k] = v
                tmp_cnt += 1

        train_vocab = train_vocab_thresholded.copy()
        del train_vocab_thresholded

        print(
            "number of unique words and phrases after thresholding: {}".format(
                len(train_vocab)))

    print("\nnumber of labels in train: {}".format(len(set(
        label_vocab.keys()))))
    if min_label_count > 1:
        label_vocab_thresholded = {}
        tmp_cnt = 0
        for k, v in sorted(label_vocab.items(), key=lambda t: t[0]):
            if v["cnt"] >= min_label_count:
                v["id"] = tmp_cnt
                label_vocab_thresholded[k] = v
                tmp_cnt += 1

        label_vocab = label_vocab_thresholded.copy()
        del label_vocab_thresholded

        print("number of unique labels after thresholding: {}".format(
            len(label_vocab)))

    final_train_labels = set(label_vocab.keys())

    with open(label_dict_path, "w+") as outfile:
        json.dump(label_vocab, outfile)
    with open(word_id_path, "w+") as outfile:
        json.dump(train_vocab, outfile)
    with open(os.path.join(result_dir, "model_params.json"), "w+") as outfile:
        json.dump(model_params, outfile)

    num_words_in_train = len(train_vocab)
    num_labels = len(label_vocab)

    train_descs2, train_labels2 = [], []
    labels_lookup = {}

    labels_thrown, descs_thrown = 0, 0
    for train_desc, train_label in zip(tqdm(train_descs), train_labels):
        final_train_inds = [0] + [
            train_vocab[phrase]["id"]
            for phrase in get_all(train_desc, word_ngrams)
            if phrase in train_vocab
        ]
        if len(final_train_inds) == 1:
            descs_thrown += 1
            continue

        if train_label not in labels_lookup:
            if train_label in final_train_labels:
                labels_lookup[train_label] = construct_label(
                    label_vocab[train_label]["id"], num_labels)
            else:
                labels_thrown += 1
                continue

        train_labels2.append(train_label)
        train_descs2.append(train_desc)
    del train_descs, train_labels

    print("\n{} datapoints thrown because of empty description".format(
        descs_thrown))
    if min_label_count > 1:
        print("{} datapoints thrown because of label".format(labels_thrown))

    if use_validation:
        val_descs, val_labels, max_words_val = parse_txt(
            validation_path,
            return_max_len=True,
            label_prefix=label_prefix,
            seed=seed,
            fraction=data_fraction)
        max_words_with_ng_val = 1
        for ng in range(word_ngrams):
            max_words_with_ng_val += max_words_val - ng

        print("\ntotal number of val datapoints: {}".format(len(val_descs)))
        val_descs2, val_labels2 = [], []
        num_thrown_for_label = 0

        for val_desc, val_label in zip(val_descs, val_labels):
            if val_label not in labels_lookup:
                num_thrown_for_label += 1
                continue

            val_descs2.append(val_desc)
            val_labels2.append(val_label)

        val_labels_set = set(val_labels2)

        print("{} datapoints thrown because of label".format(
            num_thrown_for_label))
        print("number of val datapoints after cleaning: {}".format(
            len(val_descs2)))
        print("number of unique labels in val after cleaning: {}".format(
            len(val_labels_set)))
        initial_val_len = len(val_descs)
        del val_descs, val_labels

    if use_gpu:
        device = "/gpu:0"
        config = tf.ConfigProto(
            allow_soft_placement=True,
            gpu_options=tf.GPUOptions(
                per_process_gpu_memory_fraction=gpu_fraction,
                allow_growth=True))
    else:
        device = "/cpu:0"
        config = tf.ConfigProto(allow_soft_placement=True)

    with tf.device(device):
        with tf.Session(config=config) as sess:
            input_ph = tf.placeholder(tf.int32,
                                      shape=[None, None],
                                      name="input")
            weights_ph = tf.placeholder(tf.float32,
                                        shape=[None, None, 1],
                                        name="input_weights")
            labels_ph = tf.placeholder(tf.float32,
                                       shape=[None, num_labels],
                                       name="label")
            learning_rate_ph = tf.placeholder_with_default(
                initial_learning_rate, shape=[], name="learning_rate")

            tf.set_random_seed(seed)

            with tf.name_scope("embeddings"):
                look_up_table = tf.Variable(tf.random_uniform(
                    [num_words_in_train + 1, emb_dim]),
                                            name="embedding_matrix")

            with tf.name_scope("mean_sentece_vector"):
                gath_vecs = tf.gather(look_up_table, input_ph)
                weights_broadcasted = tf.tile(weights_ph,
                                              tf.stack([1, 1, emb_dim]))
                mean_emb = tf.reduce_sum(tf.multiply(weights_broadcasted,
                                                     gath_vecs),
                                         axis=1,
                                         name="sentence_embedding")

            logits = tf.layers.dense(
                mean_emb,
                num_labels,
                use_bias=False,
                kernel_initializer=tf.truncated_normal_initializer(),
                name="logits")
            output = tf.nn.softmax(logits, name="prediction")
            # this is not used in the training, but will be used for inference

            correctly_predicted = tf.nn.in_top_k(logits,
                                                 tf.argmax(labels_ph, axis=1),
                                                 1,
                                                 name="top_1")

            ce_loss = tf.reduce_mean(
                tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels_ph,
                                                           logits=logits),
                name="ce_loss")

            train_op = tf.train.AdamOptimizer(learning_rate_ph).minimize(
                ce_loss)
            sess.run(tf.global_variables_initializer())

            train_start = time.time()

            for epoch in range(n_epochs):
                print("\nepoch {} started".format(epoch + 1))

                end_epoch_accuracy, end_epoch_accuracy_k = [], []
                moving_loss = []
                for batch, batch_weights, batch_labels in \
                        batch_generator(train_descs2, train_labels2, batch_size, train_vocab, labels_lookup,
                                        word_ngrams, shuffle=True):
                    _, correct, batch_loss = sess.run(
                        [train_op, correctly_predicted, ce_loss],
                        feed_dict={
                            input_ph: batch,
                            weights_ph: batch_weights,
                            labels_ph: batch_labels
                        })

                    end_epoch_accuracy.extend(correct)
                    moving_loss.append(batch_loss)

                print('\ncurrent learning rate: {}'.format(
                    round(initial_learning_rate, 7)))
                print("epoch {} ended".format(epoch + 1))
                print("epoch moving mean loss: {}".format(
                    round(np.mean(moving_loss), 3)))
                print("train moving average accuracy: {}".format(
                    percent(end_epoch_accuracy)))

                initial_learning_rate *= learning_rate_multiplier

                if use_validation:
                    end_epoch_accuracy, end_epoch_accuracy_k = [], []
                    end_epoch_loss = []

                    for batch, batch_weights, batch_labels in \
                            batch_generator(val_descs2, val_labels2, batch_size_inference, train_vocab, labels_lookup,
                                            word_ngrams):
                        correct, batch_loss = sess.run(
                            [correctly_predicted, ce_loss],
                            feed_dict={
                                input_ph: batch,
                                weights_ph: batch_weights,
                                labels_ph: batch_labels
                            })

                        end_epoch_accuracy.extend(correct)
                        end_epoch_loss.append(batch_loss)

                    mean_acc = np.round(
                        100 * np.sum(end_epoch_accuracy) / initial_val_len, 2)

                    print("end epoch mean val accuracy: {}".format(mean_acc))

            freeze_save_graph(sess, result_dir,
                              "model_ep{}.pb".format(epoch + 1), "prediction")
            print("the model is stored at {}".format(result_dir))
            print("the training took {} seconds".format(
                round(time.time() - train_start, 0)))
    print("all process took {} seconds".format(
        round(time.time() - main_start, 0)))
Example #15
0
        probability = sess.run(vgg.prob, feed_dict={images:img_ready})
        #得到预测概率最大的五个索引值
		top5 = np.argsort(probability[0])[-1:-6:-1]
        print "top5:",top5
		#定义两个list-对应概率值和实际标签
        values = []
        bar_label = []
		#枚举上面取出的五个索引值
        for n, i in enumerate(top5): 
            print "n:",n
            print "i:",i
			#将索引值对应的预测概率值取出并放入value
            values.append(probability[0][i]) 
			#将索引值对应的际标签取出并放入bar_label
            bar_label.append(labels[i]) 
            print i, ":", labels[i], "----", utils.percent(probability[0][i]) 
        
		#将画布分为一行一列,并把下图放入其中
        ax = fig.add_subplot(111) 
		#绘制柱状图
        ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g')
        #设置横轴标签
		ax.set_ylabel(u'probabilityit') 
		#添加标题
        ax.set_title(u'Top-5') 
        for a,b in zip(range(len(values)), values):
			#显示预测概率值
            ax.text(a, b+0.0005, utils.percent(b), ha='center', va = 'bottom', fontsize=7)   
        #显示图像
		plt.show() 
Example #16
0
with tf.Session() as sess:
    images = tf.placeholder(tf.float32, [1, 224, 224, 3])
    vgg = vgg16.Vgg16()
    vgg.forward(images)
    probability = sess.run(vgg.prob, feed_dict={images: img_ready})
    top5 = np.argsort(probability[0])[-1:-6:-1]
    print("top5:", top5)
    values = []
    bar_label = []
    for n, i in enumerate(top5):
        print("n:" + str(n))
        print("i:" + str(i))
        values.append(probability[0][i])
        bar_label.append(labels[i])
        print(
            str(i) + ":" + labels[i] + "----" +
            utils.percent(probability[0][i]))

    ax = fig.add_subplot(111)
    ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g')
    ax.set_ylabel(u'probabilityit')
    ax.set_title(u'Top-5')
    for a, b in zip(range(len(values)), values):
        ax.text(a,
                b + 0.0005,
                utils.percent(b),
                ha='center',
                va='bottom',
                fontsize=7)
    plt.show()
Example #17
0
    'Chesapeake_Bay_retriever', 'French_bulldog', 'Newfoundland', 'pug',
    'Boston_bull', 'Norfolk_terrier', 'Saint_Bernard', 'EntleBucher',
    'Border_terrier', 'Brittany_spaniel', 'malinois', 'Border_collie',
    'Gordon_setter', 'Australian_terrier', 'Labrador_retriever', 'basset',
    'Tibetan_terrier', 'Staffordshire_bullterrier', 'Irish_setter',
    'Japanese_spaniel', 'dingo', 'Irish_terrier', 'Saluki', 'Shih', 'collie',
    'Old_English_sheepdog', 'Ibizan_hound', 'Afghan_hound', 'soft',
    'Walker_hound', 'Lhasa', 'Walker_hound', 'Lhasa'
]
print(len(labels))

gpuConfig = tf.ConfigProto(allow_soft_placement=True)
gpuConfig.gpu_options.allow_growth = True

with tf.Session(config=gpuConfig) as sess:
    # 给输入图片占位
    images = tf.placeholder(tf.float32, [1, 224, 224, 3])
    # 初始化vgg,读出保存的文件参数
    vgg = vgg16.Vgg16()
    # 调用前向传播函数,把待识别图片喂入神经网络
    vgg.forward(images)
    # 运行会话,预测分类结果
    probability = sess.run(vgg.prob, feed_dict={images: img_ready})
    top5 = np.argsort(probability[0])[-1:-6:-1]
    print("top5:", top5)
    for n, i in enumerate(top5):
        print("n:", n)
        print("i:", i)
        print(i, ":", labels[i])
        print("----", utils.percent(probability[0][i]))
Example #18
0
                                            str(BRL_AMOUNT_TRADE), True, base))
        thread_sell = threading.Thread(target=async_offer,
                                       args=(biscoint_robot, 'sell',
                                             str(BRL_AMOUNT_TRADE), True,
                                             base))
        thread_buy.start()
        thread_sell.start()
        thread_buy.join()
        thread_sell.join()

        # Get buy and sell offers of this cycle
        buy = request_orders['buy']
        sell = request_orders['sell']

        # Calculate if arbitrage is possible
        calculated_percent = percent(buy['efPrice'], sell['efPrice'])

        showCycle(cycle_count, calculated_percent)

        if calculated_percent > percent_record:
            logging.info(
                f"Percent Record Reached!! : {calculated_percent} at {dt.now()}"
            )
            percent_record = calculated_percent
        # If Arbitrage is possible, confirm offers
        if calculated_percent >= MIN_PERCENT_REQUIRED:
            logging.info(
                f"Arbitrage oportunity: buy:{buy['efPrice']}   sell:{sell['efPrice']} ({calculated_percent})"
            )
            playsound('beep.wav')
Example #19
0
                       max_mask_r=5 * beam_pxl, mask_step=1,
                       upsample_factor=1000)
    shift_value = math.sqrt(shift[0]**2 + shift[1]**2)
    shifts.append(shift)
    shifts_values.append(shift_value)
    print("Found shift : {} with value {}".format(shift, shift_value))
    shifts_mask_size[i] = list()
    print("Found shift dependence on mask size for current bootstrap sample")
    for r in range(0, 100, 5):
        sh = i15.cross_correlate(i5, region1=(i15.x_c, i15.y_c, r, None),
                                 region2=(i5.x_c, i5.y_c, r, None),
                                 upsample_factor=1000)
        sh_value = math.sqrt(sh[0]**2 + sh[1]**2)
        print("r = {}, shift = {}".format(r, sh_value))
        shifts_mask_size[i].append(sh_value)


# np.savetxt('2230_shifts_300.txt', shifts)
hist_d, edges_d = histogram(shifts_values, normed=False)
lower_d = np.resize(edges_d, len(edges_d) - 1)
fig, ax = plt.subplots(1, 1)
ax.bar(lower_d, hist_d, width=np.diff(lower_d)[0], linewidth=1, color='#4682b4')
ax.axvline(x=percent(shifts_values, perc=16), color='r')
ax.axvline(x=percent(shifts_values, perc=84), color='r')
font = {'family': 'Droid Sans', 'weight': 'normal', 'size': 18}
matplotlib.rc('font', **font)
# matplotlib.rcParams.update({'font.size': 22})
ax.set_xlabel(ur"Shift values, [pixels]")
ax.set_ylabel(ur"Number of replications")
fig.savefig("shifts_histogram.png", bbox_inches='tight', dpi=200)
Example #20
0
    probability = sess.run(vgg.prob, feed_dict={x: img_ready})
    # np.argsort 函数返回预测值(probability 的数据结构[[各预测类别的概率值]])由小到大的索引值,
    # 并取出预测概率最大的五个索引值
    top5 = np.argsort(probability[0])[-1:-6:-1]
    print("top5:", top5)

    # 定义两个 list---对应的概率值和实际标签(zebra)
    values = []
    bar_label = []
    for n, i in enumerate(top5):  # 枚举上面取出的五个索引值
        print("n:", n)
        print("i:", i)
        values.append(probability[0][i])  # 将索引值对应的预测概率值取出并放入 values
        bar_label.append(labels[i])  # 根据索引值取出对应的实际标签并放入 bar_label
        print(i, ":", labels[i], "----",
              utils.percent(probability[0][i]))  # 打印属于某个类别的概率

    ax = fig.add_subplot(111)  # 将画布划分为一行一列,并把下图放入其中
    # bar()函数绘制柱状图,参数 range(len(values)是柱子下标, values 表示柱高的列表(也就是五个预测概率值,
    # tick_label 是每个柱子上显示的标签(实际对应的标签), width 是柱子的宽度, fc 是柱子的颜色)
    ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g')
    ax.set_ylabel(u'probability')  # 设置横轴标签
    ax.set_title(u'Top-5')  # 添加标题
    for a, b in zip(range(len(values)), values):
        # 在每个柱子的顶端添加对应的预测概率值, a, b 表示坐标, b+0.0005 表示要把文本信息放置在高于每个柱子顶端
        #0.0005 的位置,
        # center 是表示文本位于柱子顶端水平方向上的的中间位置, bottom 是将文本水平放置在柱子顶端垂直方向上的底端
        #位置, fontsize 是字号
        ax.text(a,
                b + 0.0005,
                utils.percent(b),
Example #21
0
File: app.py Project: qrzbing/MOOC
    # 索引值存入Top 5。probabilty中的这5个索引值就是Nclasses.py
    # 标签字典中的键
    top5 = np.argsort(probability[0])[-1:-6:-1]
    print "top5:", top5
    # 新建 values 列表用来存储probability中元素的值
    # probability 元素的值是 top5 中5个物种出现的概率
    values = []
    # 新建bar_label列表用来存储标签字典中对应的值也就是5个物种的名称
    bar_label = []
    for n, i in enumerate(top5):
        print "n:", n
        print "i:", i
        # probability中的元素也就是每个物种出现的概率
        values.append(probability[0][i])
        bar_label.append(labels[i])
        print i, ":", labels[i], "----", utils.percent(probability[0][i])

    # 构建一行一列的子图,画出第一张子图
    ax = fig.add_subplot(111)
    ax.bar(
        range(len(values)),  # 下标
        values,  # 高度
        tick_label=bar_label,  # 柱子的标签
        width=0.5,  # 柱子的宽度
        fc='g'  # 柱子的颜色
    )
    ax.set_ylabel(u'probabilityit')
    ax.set_title(u'Top-5')
    # 在每个柱子的顶端添加对应的预测概率
    # a, b表示x, y坐标
    for a, b in zip(range(len(values)), values):
Example #22
0
    def __get_bt_stats(info):
        """Get stats about backtrace

        Args:
            info (dict): data

        Returns:
            dict: stats for each backtraces
        """
        hits = info['hits']
        uuids = [hit['uuid'] for hit in hits]
        bt_info = backtrace.get_infos(uuids, fraction=1)
        total = 0
        rec = 0.
        weird_address = {}
        os_cpu = {}
        cycles = {}
        backtraces = {}
        bts_common_part = None
        if bt_info:
            recursive_bt = 0
            addrs = {}
            backtraces, bts_common_part = Track.__get_different_bt_stats(bt_info)

            total = float(len(bt_info))
            for v in bt_info.values():
                _cycles = v['cycles']
                if _cycles:
                    recursive_bt += 1
                    cycles[_cycles] = cycles[_cycles] + 1 if _cycles in cycles else 1

                addr = v['address']
                t = (addr, v['cpu_name'])
                addrs[t] = addrs[t] + 1 if t in addrs else 1
                t = (v['os'], v['cpu_name'])
                os_cpu[t] = os_cpu[t] + 1 if t in os_cpu else 1

            rec = utils.percent(float(recursive_bt) / total)

            for k, v in os_cpu.items():
                os_cpu[k] = utils.percent(float(v) / total)

            for k, v in addrs.items():
                percent = float(v) / total
                if memory.isweird(*k):
                    weird_address[k] = utils.percent(percent)
                elif percent >= 0.25:
                    weird_address[k] = utils.percent(percent)

            _addrs = []
            for k in addrs.iterkeys():
                _addrs.append(k[0])

            total = int(total)

        return {'sample_size': total,
                'bt_has_rec': rec,
                'weird_address': weird_address,
                'cycles': cycles,
                'os_cpu': os_cpu,
                'backtraces': backtraces,
                'common_part': bts_common_part}
    def update_md5(self, spot_name, spot_path, threshold=800):

        logger = logging.getLogger(__name__)
        logging.getLogger('elasticsearch').setLevel(logging.WARNING)

        spotlog = MD5LogFile(spot_name, spot_path)
        file_list = spotlog.as_list()

        logger.debug(f"Spot: {spot_path} contains {len(spotlog)} files.")

        param_func, query_tmpl = ElasticsearchQuery.ceda_fbs()
        result = self.check_files_existence(param_func,
                                            query_tmpl,
                                            file_list,
                                            raw_resp=True,
                                            threshold=threshold)

        # return len(result["True"])

        logger.info(
            "Spot: {}. Files in index: {}. Files not in: {}. Percentage in: {}%"
            .format(spot_path, len(result["True"]), len(result["False"]),
                    utils.percent(len(spotlog), len(result["True"]))))

        files_in = result["True"]

        # Check md5s
        update_total = 0
        md5_json = ""

        try:
            for file in files_in:
                file_info = file[0]["_source"]["info"]
                filepath = os.path.join(file_info["directory"],
                                        file_info["name"])

                if file_info["md5"] != spotlog.get_md5(filepath):
                    update_total += 1

                    id = file[0]["_id"]
                    index = json.dumps(
                        {"update": {
                            "_id": id,
                            "_type": "file"
                        }}) + "\n"
                    md5_field = json.dumps({
                        "source": {
                            "doc": {
                                "info": {
                                    "md5": spotlog.get_md5(filepath)
                                }
                            }
                        }
                    }) + "\n"
                    md5_json += index + md5_field

                if update_total > threshold:
                    self.make_bulk_update(md5_json)
                    md5_json = ""
                    update_total = 0

            if md5_json:
                self.make_bulk_update(md5_json)

        except Exception as msg:
            logger.error(msg)
Example #24
0
File: app.py Project: Jing42/Python
img_ready = utils.load_image(img_path) 

fig=plt.figure(u"Top-5 预测结果") 

with tf.Session() as sess:
    images = tf.placeholder(tf.float32, [1, 224, 224, 3])
    vgg = vgg16.Vgg16() 
    vgg.forward(images) 
    probability = sess.run(vgg.prob, feed_dict={images:img_ready})
    top5 = np.argsort(probability[0])[-1:-6:-1]
    print "top5:",top5
    values = []
    bar_label = []
    for n, i in enumerate(top5): 
        print "n:",n
        print "i:",i
        values.append(probability[0][i]) 
        bar_label.append(labels[i]) 
        print i, ":", labels[i], "----", utils.percent(probability[0][i]) 
        
    ax = fig.add_subplot(111) 
    ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g')
    ax.set_ylabel(u'probabilityit') 
    ax.set_title(u'Top-5') 
    for a,b in zip(range(len(values)), values):
        ax.text(a, b+0.0005, utils.percent(b), ha='center', va = 'bottom', fontsize=7)   
    plt.show() 


    
 def test_percent(self):
     param_list = [(150, 30, 20.0)]
     for total_lines, current_index, want in param_list:
         with self.subTest():
             got = utils.percent(current_index, total_lines)
             self.assertEqual(got, want, f"got={got}, expected={want}")
Example #26
0
        probability = sess.run(vgg.prob, feed_dict={images: img_ready})
        # 得到预测概率最大的五个索引值
        top5 = np.argsort(probability[0])[-1:-6:-1]
        print("top5:", top5)
        # 定义两个list-对应概率值和实际标签
        values = []
        bar_label = []
        # 枚举上面取出的五个索引值
        for n, i in enumerate(top5):
            print("n:", n)
            print("i:", i)
            # 将索引值对应的预测概率值取出并放入value
            values.append(probability[0][i])
            # 将索引值对应的际标签取出并放入bar_label
            bar_label.append(labels[i])
            print(i, ":", labels[i], "----", utils.percent(probability[0][i]))

        # 将画布分为一行一列,并把下图放入其中
        ax = fig.add_subplot(111)
        # 绘制柱状图
        ax.bar(range(len(values)),
               values,
               tick_label=bar_label,
               width=0.5,
               fc='g')
        # 设置横轴标签
        ax.set_ylabel(u'probabilityit')
        # 添加标题
        ax.set_title(u'Top-5')
        for a, b in zip(range(len(values)), values):
            # 显示预测概率值
Example #27
0
    def __get_bt_stats(info):
        """Get stats about backtrace

        Args:
            info (dict): data

        Returns:
            dict: stats for each backtraces
        """
        hits = info['hits']
        uuids = [hit['uuid'] for hit in hits]
        bt_info = backtrace.get_infos(uuids, fraction=1)
        total = 0
        rec = 0.
        weird_address = {}
        os_cpu = {}
        cycles = {}
        backtraces = {}
        bts_common_part = None
        if bt_info:
            recursive_bt = 0
            addrs = {}
            backtraces, bts_common_part = Track.__get_different_bt_stats(
                bt_info)

            total = float(len(bt_info))
            for v in bt_info.values():
                _cycles = v['cycles']
                if _cycles:
                    recursive_bt += 1
                    cycles[_cycles] = cycles[
                        _cycles] + 1 if _cycles in cycles else 1

                addr = v['address']
                t = (addr, v['cpu_name'])
                addrs[t] = addrs[t] + 1 if t in addrs else 1
                t = (v['os'], v['cpu_name'])
                os_cpu[t] = os_cpu[t] + 1 if t in os_cpu else 1

            rec = utils.percent(float(recursive_bt) / total)

            for k, v in os_cpu.items():
                os_cpu[k] = utils.percent(float(v) / total)

            for k, v in addrs.items():
                percent = float(v) / total
                if memory.isweird(*k):
                    weird_address[k] = utils.percent(percent)
                elif percent >= 0.25:
                    weird_address[k] = utils.percent(percent)

            _addrs = []
            for k in addrs.iterkeys():
                _addrs.append(k[0])

            total = int(total)

        return {
            'sample_size': total,
            'bt_has_rec': rec,
            'weird_address': weird_address,
            'cycles': cycles,
            'os_cpu': os_cpu,
            'backtraces': backtraces,
            'common_part': bts_common_part
        }