def display_metrics (c_file, results, verbose_mode): filename=os.path.basename(c_file) n_loc=results[0] n_bl=results[1] n_com=results[2] per_bl= utils.percent(n_bl, n_loc) per_com= utils.percent(n_com, n_loc) # if verbose_mode >=1 : print "%s : %s LOC with %s BL(%.1f%%) and %s comments(%.1f%%)" % (filename, n_loc, n_bl, per_bl, n_com, per_com)
def determiner(person=None, gender=None, number=None, function='S', kind=None, S=None): if percent(35): det = wf.getNominal('article', gender, number) next = (det.value.index, 'article') elif percent(50): det = wf.getAdjectivePronoun(gender, number, 'determiner') next = (det.value.index, 'adjective_pronoun') else: det = wf.getPossessivePronoun( gender, number, aleatory('person'), aleatory('number')) next = (det.value.index, 'possessive_pronoun') return Tree('determiner', [det], {'next': next})
def nounPhrase(person=None, gender=None, number=None, function='S', kind=None, S=None): if person is None: if S and verify_semantics(S, 'PESSOA'): person = aleatory('person') else: person = '3' if kind is None: if person != '3' or percent(15): kind = 'personal_pronoun' else: kind = 'noun' L = [] next = None if kind == 'noun': det = determiner(person, gender, number, function, kind, S) L.append(det) next = det.info['next'] n = nounBar(person, gender, number, function, kind, S) if not next: next = n.info['next'] L.append(n) return Tree('noun phrase', L, {'head': n.info['head'], 'next': next})
def __get_stats(info, field): l = info[field] total = float(info['total']) stats = { } for e in l: stats[e['term']] = utils.percent(float(e['count']) / total) return stats
def adnominalAdjunct(person=None, gender=None, number=None, function='S', kind=None, S=None, position='pos'): L = [] if kind == 'noun': if percent(20): if percent(50): # menino feio # bola feia adjective = wf.getNominal('adjective', gender, number, S) L.append(adjective) else: if percent(40): # menino que correu # bola que caiu L = [wf.getRelativePronoun(gender, number, None), verbPhrase(person, gender, number, S=S)] elif percent(50): # menino que a mãe ama # bola que o menino chutou L = [wf.getRelativePronoun(gender, number, None), clause(tran='vtd', OD=S)] else: # menino de quem a mãe gosta # bola da qual o menino gosta that_clause = clause(tran='vti', OI=S) preposition = wf.getPreposition( that_clause.info['preposition'], gender, number, None) L = [ preposition, wf.getRelativePronoun( gender, number, S ), that_clause ] elif kind == 'personal_pronoun' and function == 'S': if percent(20): # eu que corri L = [wf.getRelativePronoun(gender, number, None), verbPhrase(person, gender, number, S=S)] if L: return Tree('adnominal adjunct', L, {}) else: return None
def main(): sess = tf.Session() images = tf.placeholder(tf.float32, [1, 224, 224, 3]) vgg = vgg16.Vgg16() vgg.forward(images) while True: img_path = input('\nInput the path and image name: ') if img_path == '0': print('\nexiting-------------------------\n') exit(0) else: print('\nrunning-------------------------\n') img_ready = utils.load_image(img_path) fig = plt.figure(u"Top-5 预测结果") probability = sess.run(vgg.prob, feed_dict={images: img_ready}) top5 = np.argsort(probability[0])[-1:-6:-1] print("top5:", top5) values = [] bar_label = [] for n, i in enumerate(top5): print("n:", n) print("i:", i) values.append(probability[0][i]) bar_label.append(labels[i]) print(i, ":", labels[i], "----", utils.percent(probability[0][i])) ax = fig.add_subplot(111) ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g') ax.set_ylabel(u'probabilityit') ax.set_title(u'Top-5') for a, b in zip(range(len(values)), values): ax.text(a, b + 0.0005, utils.percent(b), ha='center', va='bottom', fontsize=7) plt.show()
def nounBar(person=None, gender=None, number=None, function='S', kind=None, S=None): noun_ = noun(person, gender, number, function, kind, S) adn_adj = None if percent(50): adn_adj = adnominalAdjunct(person, gender, number, function, kind, S) if adn_adj: L = [noun_, adn_adj] else: L = [noun_, ] return Tree('noun-bar', L, {'head': noun_.info['head'], 'next': noun_.info['next']})
def __get_stats(info, field): """Get stats about info[field] Args: info (dict): data field (str): field name Returns: dict: stats for each entry """ l = info[field] total = float(info['total']) stats = {} for e in l: stats[e['term']] = utils.percent(float(e['count']) / total) return stats
def __get_bt_stats(info, credentials): hits = info['hits'] uuids = [ hit['uuid'] for hit in hits ] bt = Backtrace(uuids, fraction = 1, credentials = credentials)#0.5) bt_info = bt.get() total = 0 rec = 0. weird_address = { } os_cpu = { } cpu_info = { } reason = { } cycles = { } if bt_info: recursive_bt = 0 addrs = { } total = float(len(bt_info)) for v in bt_info.itervalues(): _cycles = v['cycles'] if _cycles: recursive_bt += 1 cycles[_cycles] = cycles[_cycles] + 1 if _cycles in cycles else 1 addr = v['address'] t = (addr, v['cpu_name']) addrs[t] = addrs[t] + 1 if t in addrs else 1 t = (v['os'], v['cpu_name']) os_cpu[t] = os_cpu[t] + 1 if t in os_cpu else 1 ci = v['cpu_info'] cpu_info[ci] = cpu_info[ci] + 1 if ci in cpu_info else 1 r = v['reason'] reason[r] = reason[r] + 1 if r in reason else 1 rec = utils.percent(float(recursive_bt) / total) for k, v in os_cpu.iteritems(): os_cpu[k] = utils.percent(float(v) / total) for k, v in cpu_info.iteritems(): cpu_info[k] = utils.percent(float(v) / total) for k, v in reason.iteritems(): reason[k] = utils.percent(float(v) / total) for k, v in addrs.iteritems(): percent = float(v) / total if Track.__is_weird_address(*k): weird_address[k] = utils.percent(percent) elif percent >= 0.25: weird_address[k] = utils.percent(percent) total = int(total) return { 'sample_size': total, 'bt_has_rec': rec, 'weird_address': weird_address, 'cycles': cycles, 'os_cpu': os_cpu, 'cpu_info': cpu_info }
def print_status_bar(stdscr, bookwnd_nav): if not bookwnd_nav.show_status_bar: return perc = utils.percent(bookwnd_nav.line_number, bookwnd_nav.book_number_lines()) if bookwnd_nav.show_percentage_points: lines_to_new_p_point = utils.lines_to_change_percentage_point( bookwnd_nav.line_number, bookwnd_nav.book_number_lines()) status_text = f"{bookwnd_nav.line_number} of {bookwnd_nav.book_number_lines()} (%{perc:.1f}) (> {lines_to_new_p_point})" else: status_text = f"{bookwnd_nav.line_number} of {bookwnd_nav.book_number_lines()} (%{perc:.1f})" pos_height = bookwnd_nav.window_height - 1 pos_width = bookwnd_nav.window_width stdscr.attron(curses.color_pair(STATUSBAR_COLOR_PAIRCODE)) stdscr.addstr(pos_height, pos_width // 2, status_text) stdscr.attroff(curses.color_pair(1))
def prepositionalPhrase(preposition=None, person=None, gender=None, number=None, function='S', kind=None, S=None): if person is None: if S and verify_semantics(S, 'PESSOA'): person = aleatory('person') else: person = '3' if kind is None: if person != '3' or percent(15): kind = 'personal_pronoun' else: kind = 'noun' np = nounPhrase(person, gender, number, function, kind, S) next = np.info['next'] L = [np] p = wf.getPreposition(preposition, gender, number, next) L = [Tree("preposition", [p])] + L return Tree('prepositional phrase', L, {'head': np.info['head']})
fig = plt.figure(u'Top-5预测结果') with tf.Session() as sess: images = tf.placeholder(tf.float32, shape=[1, 224, 224, 3]) vgg = vgg16.vgg16() vgg.forward(images) probability = sess.run(vgg.prob, feed_dict={images: img_ready}) top5 = np.argsort(probability[0])[-1:-6:-1] print('top5:', top5) values = [] bar_label = [] for n, i, in enumerate(top5): print('n:', n) print('i', i) values.append(probability[0][i]) bar_label.append(labels[i]) print(i, ':', labels[i], '----', utils.percent(probability[0][i])) ax = fig.add_subplot(111) ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g') ax.set_ylabel(u'probablity') ax.set_title(u'Top-5') for a, b in zip(range(len(values)), values): ax.text(a, b + 0.0005, utils.percent(b), ha='center', va='bottom', fontsize=7) plt.show()
def main(): main_start = time.time() parser = argparse.ArgumentParser() parser.add_argument("--train_path", type=str, help="path to train file", default="./train.txt") parser.add_argument("--validation_path", type=str, help="path to validation file", default="") parser.add_argument("--label_prefix", type=str, help="label prefix", default="__label__") parser.add_argument( "--min_word_count", type=int, default=1, help="discard words which appear less than this number") parser.add_argument( "--min_label_count", type=int, default=1, help="discard labels which appear less than this number") parser.add_argument("--dim", type=int, default=100, help="length of embedding vector") parser.add_argument("--n_epochs", type=int, default=5, help="number of epochs") parser.add_argument("--word_ngrams", type=int, default=1, help="word ngrams") parser.add_argument("--batch_size", type=int, default=1024, help="batch size for train") parser.add_argument( "--batch_size_inference", type=int, default=1024, help= "batch size for inference, ignored if validation_path is not provided") parser.add_argument("--seed", type=int, default=17) parser.add_argument("--learning_rate", type=float, default=0.1, help="learning rate") parser.add_argument("--learning_rate_multiplier", type=float, default=0.8, help="learning rate multiplier after each epoch") parser.add_argument( "--data_fraction", type=float, default=1, help= "data fraction, if < 1, train (and validation) data will be randomly sampled" ) parser.add_argument("--use_validation", type=int, default=0, help="evaluate on validation data") parser.add_argument("--use_gpu", type=int, default=0, help="use gpu for training") parser.add_argument("--gpu_fraction", type=float, default=0.5, help="what fraction of gpu to allocate") parser.add_argument("--result_dir", type=str, help="result dir", default="./results/") args = parser.parse_args() for bool_param in [args.use_validation, args.use_gpu]: assert bool_param in [0, 1] train_path = args.train_path validation_path = args.validation_path label_prefix = args.label_prefix min_word_count = args.min_word_count min_label_count = args.min_label_count emb_dim = args.dim n_epochs = args.n_epochs word_ngrams = args.word_ngrams batch_size = args.batch_size batch_size_inference = args.batch_size_inference initial_learning_rate = args.learning_rate learning_rate_multiplier = args.learning_rate_multiplier use_validation = bool(args.use_validation) seed = args.seed data_fraction = args.data_fraction use_gpu = bool(args.use_gpu) gpu_fraction = args.gpu_fraction result_dir = validate(args.result_dir) print('training with arguments:') print(args) print('\n') np.random.seed(seed) train_descs, train_labels, max_words = parse_txt(train_path, return_max_len=True, debug_till_row=-1, fraction=data_fraction, seed=seed, label_prefix=label_prefix) model_params = { "word_ngrams": word_ngrams, "word_id_path": os.path.abspath(os.path.join(result_dir, "word_id.json")), "label_dict_path": os.path.abspath(os.path.join(result_dir, "label_dict.json")) } for child_dir in os.listdir(result_dir): dir_tmp = os.path.join(result_dir, child_dir) if os.path.isdir(dir_tmp): shutil.rmtree(dir_tmp) if dir_tmp.endswith(".pb"): os.remove(dir_tmp) max_words_with_ng = 1 for ng in range(word_ngrams): max_words_with_ng += max_words - ng print("preparing dataset") print("total number of datapoints: {}".format(len(train_descs))) print("max number of words in description: {}".format(max_words)) print("max number of words with n-grams in description: {}".format( max_words_with_ng)) label_dict_path = os.path.join(result_dir, "label_dict.json") word_id_path = os.path.join(result_dir, "word_id.json") train_vocab = make_train_vocab(train_descs, word_ngrams) label_vocab = make_label_vocab(train_labels) if min_word_count > 1: tmp_cnt = 1 train_vocab_thresholded = {} for k, v in sorted(train_vocab.items(), key=lambda t: t[0]): if v["cnt"] >= min_word_count: v["id"] = tmp_cnt train_vocab_thresholded[k] = v tmp_cnt += 1 train_vocab = train_vocab_thresholded.copy() del train_vocab_thresholded print( "number of unique words and phrases after thresholding: {}".format( len(train_vocab))) print("\nnumber of labels in train: {}".format(len(set( label_vocab.keys())))) if min_label_count > 1: label_vocab_thresholded = {} tmp_cnt = 0 for k, v in sorted(label_vocab.items(), key=lambda t: t[0]): if v["cnt"] >= min_label_count: v["id"] = tmp_cnt label_vocab_thresholded[k] = v tmp_cnt += 1 label_vocab = label_vocab_thresholded.copy() del label_vocab_thresholded print("number of unique labels after thresholding: {}".format( len(label_vocab))) final_train_labels = set(label_vocab.keys()) with open(label_dict_path, "w+") as outfile: json.dump(label_vocab, outfile) with open(word_id_path, "w+") as outfile: json.dump(train_vocab, outfile) with open(os.path.join(result_dir, "model_params.json"), "w+") as outfile: json.dump(model_params, outfile) num_words_in_train = len(train_vocab) num_labels = len(label_vocab) train_descs2, train_labels2 = [], [] labels_lookup = {} labels_thrown, descs_thrown = 0, 0 for train_desc, train_label in zip(tqdm(train_descs), train_labels): final_train_inds = [0] + [ train_vocab[phrase]["id"] for phrase in get_all(train_desc, word_ngrams) if phrase in train_vocab ] if len(final_train_inds) == 1: descs_thrown += 1 continue if train_label not in labels_lookup: if train_label in final_train_labels: labels_lookup[train_label] = construct_label( label_vocab[train_label]["id"], num_labels) else: labels_thrown += 1 continue train_labels2.append(train_label) train_descs2.append(train_desc) del train_descs, train_labels print("\n{} datapoints thrown because of empty description".format( descs_thrown)) if min_label_count > 1: print("{} datapoints thrown because of label".format(labels_thrown)) if use_validation: val_descs, val_labels, max_words_val = parse_txt( validation_path, return_max_len=True, label_prefix=label_prefix, seed=seed, fraction=data_fraction) max_words_with_ng_val = 1 for ng in range(word_ngrams): max_words_with_ng_val += max_words_val - ng print("\ntotal number of val datapoints: {}".format(len(val_descs))) val_descs2, val_labels2 = [], [] num_thrown_for_label = 0 for val_desc, val_label in zip(val_descs, val_labels): if val_label not in labels_lookup: num_thrown_for_label += 1 continue val_descs2.append(val_desc) val_labels2.append(val_label) val_labels_set = set(val_labels2) print("{} datapoints thrown because of label".format( num_thrown_for_label)) print("number of val datapoints after cleaning: {}".format( len(val_descs2))) print("number of unique labels in val after cleaning: {}".format( len(val_labels_set))) initial_val_len = len(val_descs) del val_descs, val_labels if use_gpu: device = "/gpu:0" config = tf.ConfigProto( allow_soft_placement=True, gpu_options=tf.GPUOptions( per_process_gpu_memory_fraction=gpu_fraction, allow_growth=True)) else: device = "/cpu:0" config = tf.ConfigProto(allow_soft_placement=True) with tf.device(device): with tf.Session(config=config) as sess: input_ph = tf.placeholder(tf.int32, shape=[None, None], name="input") weights_ph = tf.placeholder(tf.float32, shape=[None, None, 1], name="input_weights") labels_ph = tf.placeholder(tf.float32, shape=[None, num_labels], name="label") learning_rate_ph = tf.placeholder_with_default( initial_learning_rate, shape=[], name="learning_rate") tf.set_random_seed(seed) with tf.name_scope("embeddings"): look_up_table = tf.Variable(tf.random_uniform( [num_words_in_train + 1, emb_dim]), name="embedding_matrix") with tf.name_scope("mean_sentece_vector"): gath_vecs = tf.gather(look_up_table, input_ph) weights_broadcasted = tf.tile(weights_ph, tf.stack([1, 1, emb_dim])) mean_emb = tf.reduce_sum(tf.multiply(weights_broadcasted, gath_vecs), axis=1, name="sentence_embedding") logits = tf.layers.dense( mean_emb, num_labels, use_bias=False, kernel_initializer=tf.truncated_normal_initializer(), name="logits") output = tf.nn.softmax(logits, name="prediction") # this is not used in the training, but will be used for inference correctly_predicted = tf.nn.in_top_k(logits, tf.argmax(labels_ph, axis=1), 1, name="top_1") ce_loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels_ph, logits=logits), name="ce_loss") train_op = tf.train.AdamOptimizer(learning_rate_ph).minimize( ce_loss) sess.run(tf.global_variables_initializer()) train_start = time.time() for epoch in range(n_epochs): print("\nepoch {} started".format(epoch + 1)) end_epoch_accuracy, end_epoch_accuracy_k = [], [] moving_loss = [] for batch, batch_weights, batch_labels in \ batch_generator(train_descs2, train_labels2, batch_size, train_vocab, labels_lookup, word_ngrams, shuffle=True): _, correct, batch_loss = sess.run( [train_op, correctly_predicted, ce_loss], feed_dict={ input_ph: batch, weights_ph: batch_weights, labels_ph: batch_labels }) end_epoch_accuracy.extend(correct) moving_loss.append(batch_loss) print('\ncurrent learning rate: {}'.format( round(initial_learning_rate, 7))) print("epoch {} ended".format(epoch + 1)) print("epoch moving mean loss: {}".format( round(np.mean(moving_loss), 3))) print("train moving average accuracy: {}".format( percent(end_epoch_accuracy))) initial_learning_rate *= learning_rate_multiplier if use_validation: end_epoch_accuracy, end_epoch_accuracy_k = [], [] end_epoch_loss = [] for batch, batch_weights, batch_labels in \ batch_generator(val_descs2, val_labels2, batch_size_inference, train_vocab, labels_lookup, word_ngrams): correct, batch_loss = sess.run( [correctly_predicted, ce_loss], feed_dict={ input_ph: batch, weights_ph: batch_weights, labels_ph: batch_labels }) end_epoch_accuracy.extend(correct) end_epoch_loss.append(batch_loss) mean_acc = np.round( 100 * np.sum(end_epoch_accuracy) / initial_val_len, 2) print("end epoch mean val accuracy: {}".format(mean_acc)) freeze_save_graph(sess, result_dir, "model_ep{}.pb".format(epoch + 1), "prediction") print("the model is stored at {}".format(result_dir)) print("the training took {} seconds".format( round(time.time() - train_start, 0))) print("all process took {} seconds".format( round(time.time() - main_start, 0)))
probability = sess.run(vgg.prob, feed_dict={images:img_ready}) #得到预测概率最大的五个索引值 top5 = np.argsort(probability[0])[-1:-6:-1] print "top5:",top5 #定义两个list-对应概率值和实际标签 values = [] bar_label = [] #枚举上面取出的五个索引值 for n, i in enumerate(top5): print "n:",n print "i:",i #将索引值对应的预测概率值取出并放入value values.append(probability[0][i]) #将索引值对应的际标签取出并放入bar_label bar_label.append(labels[i]) print i, ":", labels[i], "----", utils.percent(probability[0][i]) #将画布分为一行一列,并把下图放入其中 ax = fig.add_subplot(111) #绘制柱状图 ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g') #设置横轴标签 ax.set_ylabel(u'probabilityit') #添加标题 ax.set_title(u'Top-5') for a,b in zip(range(len(values)), values): #显示预测概率值 ax.text(a, b+0.0005, utils.percent(b), ha='center', va = 'bottom', fontsize=7) #显示图像 plt.show()
with tf.Session() as sess: images = tf.placeholder(tf.float32, [1, 224, 224, 3]) vgg = vgg16.Vgg16() vgg.forward(images) probability = sess.run(vgg.prob, feed_dict={images: img_ready}) top5 = np.argsort(probability[0])[-1:-6:-1] print("top5:", top5) values = [] bar_label = [] for n, i in enumerate(top5): print("n:" + str(n)) print("i:" + str(i)) values.append(probability[0][i]) bar_label.append(labels[i]) print( str(i) + ":" + labels[i] + "----" + utils.percent(probability[0][i])) ax = fig.add_subplot(111) ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g') ax.set_ylabel(u'probabilityit') ax.set_title(u'Top-5') for a, b in zip(range(len(values)), values): ax.text(a, b + 0.0005, utils.percent(b), ha='center', va='bottom', fontsize=7) plt.show()
'Chesapeake_Bay_retriever', 'French_bulldog', 'Newfoundland', 'pug', 'Boston_bull', 'Norfolk_terrier', 'Saint_Bernard', 'EntleBucher', 'Border_terrier', 'Brittany_spaniel', 'malinois', 'Border_collie', 'Gordon_setter', 'Australian_terrier', 'Labrador_retriever', 'basset', 'Tibetan_terrier', 'Staffordshire_bullterrier', 'Irish_setter', 'Japanese_spaniel', 'dingo', 'Irish_terrier', 'Saluki', 'Shih', 'collie', 'Old_English_sheepdog', 'Ibizan_hound', 'Afghan_hound', 'soft', 'Walker_hound', 'Lhasa', 'Walker_hound', 'Lhasa' ] print(len(labels)) gpuConfig = tf.ConfigProto(allow_soft_placement=True) gpuConfig.gpu_options.allow_growth = True with tf.Session(config=gpuConfig) as sess: # 给输入图片占位 images = tf.placeholder(tf.float32, [1, 224, 224, 3]) # 初始化vgg,读出保存的文件参数 vgg = vgg16.Vgg16() # 调用前向传播函数,把待识别图片喂入神经网络 vgg.forward(images) # 运行会话,预测分类结果 probability = sess.run(vgg.prob, feed_dict={images: img_ready}) top5 = np.argsort(probability[0])[-1:-6:-1] print("top5:", top5) for n, i in enumerate(top5): print("n:", n) print("i:", i) print(i, ":", labels[i]) print("----", utils.percent(probability[0][i]))
str(BRL_AMOUNT_TRADE), True, base)) thread_sell = threading.Thread(target=async_offer, args=(biscoint_robot, 'sell', str(BRL_AMOUNT_TRADE), True, base)) thread_buy.start() thread_sell.start() thread_buy.join() thread_sell.join() # Get buy and sell offers of this cycle buy = request_orders['buy'] sell = request_orders['sell'] # Calculate if arbitrage is possible calculated_percent = percent(buy['efPrice'], sell['efPrice']) showCycle(cycle_count, calculated_percent) if calculated_percent > percent_record: logging.info( f"Percent Record Reached!! : {calculated_percent} at {dt.now()}" ) percent_record = calculated_percent # If Arbitrage is possible, confirm offers if calculated_percent >= MIN_PERCENT_REQUIRED: logging.info( f"Arbitrage oportunity: buy:{buy['efPrice']} sell:{sell['efPrice']} ({calculated_percent})" ) playsound('beep.wav')
max_mask_r=5 * beam_pxl, mask_step=1, upsample_factor=1000) shift_value = math.sqrt(shift[0]**2 + shift[1]**2) shifts.append(shift) shifts_values.append(shift_value) print("Found shift : {} with value {}".format(shift, shift_value)) shifts_mask_size[i] = list() print("Found shift dependence on mask size for current bootstrap sample") for r in range(0, 100, 5): sh = i15.cross_correlate(i5, region1=(i15.x_c, i15.y_c, r, None), region2=(i5.x_c, i5.y_c, r, None), upsample_factor=1000) sh_value = math.sqrt(sh[0]**2 + sh[1]**2) print("r = {}, shift = {}".format(r, sh_value)) shifts_mask_size[i].append(sh_value) # np.savetxt('2230_shifts_300.txt', shifts) hist_d, edges_d = histogram(shifts_values, normed=False) lower_d = np.resize(edges_d, len(edges_d) - 1) fig, ax = plt.subplots(1, 1) ax.bar(lower_d, hist_d, width=np.diff(lower_d)[0], linewidth=1, color='#4682b4') ax.axvline(x=percent(shifts_values, perc=16), color='r') ax.axvline(x=percent(shifts_values, perc=84), color='r') font = {'family': 'Droid Sans', 'weight': 'normal', 'size': 18} matplotlib.rc('font', **font) # matplotlib.rcParams.update({'font.size': 22}) ax.set_xlabel(ur"Shift values, [pixels]") ax.set_ylabel(ur"Number of replications") fig.savefig("shifts_histogram.png", bbox_inches='tight', dpi=200)
probability = sess.run(vgg.prob, feed_dict={x: img_ready}) # np.argsort 函数返回预测值(probability 的数据结构[[各预测类别的概率值]])由小到大的索引值, # 并取出预测概率最大的五个索引值 top5 = np.argsort(probability[0])[-1:-6:-1] print("top5:", top5) # 定义两个 list---对应的概率值和实际标签(zebra) values = [] bar_label = [] for n, i in enumerate(top5): # 枚举上面取出的五个索引值 print("n:", n) print("i:", i) values.append(probability[0][i]) # 将索引值对应的预测概率值取出并放入 values bar_label.append(labels[i]) # 根据索引值取出对应的实际标签并放入 bar_label print(i, ":", labels[i], "----", utils.percent(probability[0][i])) # 打印属于某个类别的概率 ax = fig.add_subplot(111) # 将画布划分为一行一列,并把下图放入其中 # bar()函数绘制柱状图,参数 range(len(values)是柱子下标, values 表示柱高的列表(也就是五个预测概率值, # tick_label 是每个柱子上显示的标签(实际对应的标签), width 是柱子的宽度, fc 是柱子的颜色) ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g') ax.set_ylabel(u'probability') # 设置横轴标签 ax.set_title(u'Top-5') # 添加标题 for a, b in zip(range(len(values)), values): # 在每个柱子的顶端添加对应的预测概率值, a, b 表示坐标, b+0.0005 表示要把文本信息放置在高于每个柱子顶端 #0.0005 的位置, # center 是表示文本位于柱子顶端水平方向上的的中间位置, bottom 是将文本水平放置在柱子顶端垂直方向上的底端 #位置, fontsize 是字号 ax.text(a, b + 0.0005, utils.percent(b),
# 索引值存入Top 5。probabilty中的这5个索引值就是Nclasses.py # 标签字典中的键 top5 = np.argsort(probability[0])[-1:-6:-1] print "top5:", top5 # 新建 values 列表用来存储probability中元素的值 # probability 元素的值是 top5 中5个物种出现的概率 values = [] # 新建bar_label列表用来存储标签字典中对应的值也就是5个物种的名称 bar_label = [] for n, i in enumerate(top5): print "n:", n print "i:", i # probability中的元素也就是每个物种出现的概率 values.append(probability[0][i]) bar_label.append(labels[i]) print i, ":", labels[i], "----", utils.percent(probability[0][i]) # 构建一行一列的子图,画出第一张子图 ax = fig.add_subplot(111) ax.bar( range(len(values)), # 下标 values, # 高度 tick_label=bar_label, # 柱子的标签 width=0.5, # 柱子的宽度 fc='g' # 柱子的颜色 ) ax.set_ylabel(u'probabilityit') ax.set_title(u'Top-5') # 在每个柱子的顶端添加对应的预测概率 # a, b表示x, y坐标 for a, b in zip(range(len(values)), values):
def __get_bt_stats(info): """Get stats about backtrace Args: info (dict): data Returns: dict: stats for each backtraces """ hits = info['hits'] uuids = [hit['uuid'] for hit in hits] bt_info = backtrace.get_infos(uuids, fraction=1) total = 0 rec = 0. weird_address = {} os_cpu = {} cycles = {} backtraces = {} bts_common_part = None if bt_info: recursive_bt = 0 addrs = {} backtraces, bts_common_part = Track.__get_different_bt_stats(bt_info) total = float(len(bt_info)) for v in bt_info.values(): _cycles = v['cycles'] if _cycles: recursive_bt += 1 cycles[_cycles] = cycles[_cycles] + 1 if _cycles in cycles else 1 addr = v['address'] t = (addr, v['cpu_name']) addrs[t] = addrs[t] + 1 if t in addrs else 1 t = (v['os'], v['cpu_name']) os_cpu[t] = os_cpu[t] + 1 if t in os_cpu else 1 rec = utils.percent(float(recursive_bt) / total) for k, v in os_cpu.items(): os_cpu[k] = utils.percent(float(v) / total) for k, v in addrs.items(): percent = float(v) / total if memory.isweird(*k): weird_address[k] = utils.percent(percent) elif percent >= 0.25: weird_address[k] = utils.percent(percent) _addrs = [] for k in addrs.iterkeys(): _addrs.append(k[0]) total = int(total) return {'sample_size': total, 'bt_has_rec': rec, 'weird_address': weird_address, 'cycles': cycles, 'os_cpu': os_cpu, 'backtraces': backtraces, 'common_part': bts_common_part}
def update_md5(self, spot_name, spot_path, threshold=800): logger = logging.getLogger(__name__) logging.getLogger('elasticsearch').setLevel(logging.WARNING) spotlog = MD5LogFile(spot_name, spot_path) file_list = spotlog.as_list() logger.debug(f"Spot: {spot_path} contains {len(spotlog)} files.") param_func, query_tmpl = ElasticsearchQuery.ceda_fbs() result = self.check_files_existence(param_func, query_tmpl, file_list, raw_resp=True, threshold=threshold) # return len(result["True"]) logger.info( "Spot: {}. Files in index: {}. Files not in: {}. Percentage in: {}%" .format(spot_path, len(result["True"]), len(result["False"]), utils.percent(len(spotlog), len(result["True"])))) files_in = result["True"] # Check md5s update_total = 0 md5_json = "" try: for file in files_in: file_info = file[0]["_source"]["info"] filepath = os.path.join(file_info["directory"], file_info["name"]) if file_info["md5"] != spotlog.get_md5(filepath): update_total += 1 id = file[0]["_id"] index = json.dumps( {"update": { "_id": id, "_type": "file" }}) + "\n" md5_field = json.dumps({ "source": { "doc": { "info": { "md5": spotlog.get_md5(filepath) } } } }) + "\n" md5_json += index + md5_field if update_total > threshold: self.make_bulk_update(md5_json) md5_json = "" update_total = 0 if md5_json: self.make_bulk_update(md5_json) except Exception as msg: logger.error(msg)
img_ready = utils.load_image(img_path) fig=plt.figure(u"Top-5 预测结果") with tf.Session() as sess: images = tf.placeholder(tf.float32, [1, 224, 224, 3]) vgg = vgg16.Vgg16() vgg.forward(images) probability = sess.run(vgg.prob, feed_dict={images:img_ready}) top5 = np.argsort(probability[0])[-1:-6:-1] print "top5:",top5 values = [] bar_label = [] for n, i in enumerate(top5): print "n:",n print "i:",i values.append(probability[0][i]) bar_label.append(labels[i]) print i, ":", labels[i], "----", utils.percent(probability[0][i]) ax = fig.add_subplot(111) ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g') ax.set_ylabel(u'probabilityit') ax.set_title(u'Top-5') for a,b in zip(range(len(values)), values): ax.text(a, b+0.0005, utils.percent(b), ha='center', va = 'bottom', fontsize=7) plt.show()
def test_percent(self): param_list = [(150, 30, 20.0)] for total_lines, current_index, want in param_list: with self.subTest(): got = utils.percent(current_index, total_lines) self.assertEqual(got, want, f"got={got}, expected={want}")
probability = sess.run(vgg.prob, feed_dict={images: img_ready}) # 得到预测概率最大的五个索引值 top5 = np.argsort(probability[0])[-1:-6:-1] print("top5:", top5) # 定义两个list-对应概率值和实际标签 values = [] bar_label = [] # 枚举上面取出的五个索引值 for n, i in enumerate(top5): print("n:", n) print("i:", i) # 将索引值对应的预测概率值取出并放入value values.append(probability[0][i]) # 将索引值对应的际标签取出并放入bar_label bar_label.append(labels[i]) print(i, ":", labels[i], "----", utils.percent(probability[0][i])) # 将画布分为一行一列,并把下图放入其中 ax = fig.add_subplot(111) # 绘制柱状图 ax.bar(range(len(values)), values, tick_label=bar_label, width=0.5, fc='g') # 设置横轴标签 ax.set_ylabel(u'probabilityit') # 添加标题 ax.set_title(u'Top-5') for a, b in zip(range(len(values)), values): # 显示预测概率值
def __get_bt_stats(info): """Get stats about backtrace Args: info (dict): data Returns: dict: stats for each backtraces """ hits = info['hits'] uuids = [hit['uuid'] for hit in hits] bt_info = backtrace.get_infos(uuids, fraction=1) total = 0 rec = 0. weird_address = {} os_cpu = {} cycles = {} backtraces = {} bts_common_part = None if bt_info: recursive_bt = 0 addrs = {} backtraces, bts_common_part = Track.__get_different_bt_stats( bt_info) total = float(len(bt_info)) for v in bt_info.values(): _cycles = v['cycles'] if _cycles: recursive_bt += 1 cycles[_cycles] = cycles[ _cycles] + 1 if _cycles in cycles else 1 addr = v['address'] t = (addr, v['cpu_name']) addrs[t] = addrs[t] + 1 if t in addrs else 1 t = (v['os'], v['cpu_name']) os_cpu[t] = os_cpu[t] + 1 if t in os_cpu else 1 rec = utils.percent(float(recursive_bt) / total) for k, v in os_cpu.items(): os_cpu[k] = utils.percent(float(v) / total) for k, v in addrs.items(): percent = float(v) / total if memory.isweird(*k): weird_address[k] = utils.percent(percent) elif percent >= 0.25: weird_address[k] = utils.percent(percent) _addrs = [] for k in addrs.iterkeys(): _addrs.append(k[0]) total = int(total) return { 'sample_size': total, 'bt_has_rec': rec, 'weird_address': weird_address, 'cycles': cycles, 'os_cpu': os_cpu, 'backtraces': backtraces, 'common_part': bts_common_part }