def sliding_window_search(img, motion_img, svm, method, feature, sbox_height, sbox_width, slide=10, threshold=0.2): detections = [] det_count = 0 height, width = img.shape for i in range(1, (height - sbox_height), slide): for j in range(1, (width - sbox_width), slide): img_patch = img[i:i + sbox_height - 1, j:j + sbox_width - 1] #motion_patch = motion_img[i:i+sbox_height-1, j:j+sbox_width-1] #img_feat = extract(img_patch, motion_patch, method, feature) img_feat = extract(img_patch, None, method, feature) y = [] y.append(0) x = [] x.append(img_feat) plabel, acc, pr = svm_predict(y, x, svm) if pr[0][0] > threshold: #print "pr: " + str(pr[0][0]) detections.append( [i, i + sbox_height - 1, j, j + sbox_width - 1, pr[0][0]]) return detections
def generate_test_features(test_data_file, test_feature_file): write_header = True with open(test_data_file) as f: for line in f: url = line.split(',')[0].strip() if url: urlFeature = feature_extractor.extract(url) write_feature(urlFeature, test_feature_file, write_header) write_feature = False
def submitCallBack(): url = E1.get() urlFeature = feature_extractor.extract(url) test_file_name = 'gui_test_feature.csv' os.remove(test_file_name) main.write_feature(urlFeature, test_file_name, True) return_ans = tr.gui_caller('url_features.csv', test_file_name) a = str(return_ans).split() if int(a[1]) == 0: tkMessageBox.showinfo("CIS475", "The specified URL \'" + url + "\' is Benign") else: tkMessageBox.showinfo("CIS475", "The specified URL \'" + url + "\' is Malicious")
def generate_train_features(train_data_file, train_feature_file): write_header = True with open(train_data_file, 'rb') as csvfile: csvReader = csv.DictReader(csvfile) i = 0 for row in csvReader: url = row['url'] is_malicious = 1 if row['label'] == 'bad' else 0 if url: urlFeature = feature_extractor.extract(url) urlFeature['malicious'] = is_malicious write_feature(urlFeature, train_feature_file, write_header) write_header = False i += 1 if (i % 100) == 0: print "Processed {0} urls".format(i)
def main(_): with tf.Graph().as_default(): global_step = tf.train.get_or_create_global_step() lr = tf.train.exponential_decay(FLAGS.learning_rate, global_step=global_step, decay_steps=FLAGS.decay_steps, decay_rate=0.9995) opt = tf.train.MomentumOptimizer(lr, 0.9) with tf.device('/cpu:0'): anchors_array = anchor_generator.get_anchor( model=FLAGS.model, base_net=FLAGS.base_net, image_size=FLAGS.image_size, base_size=FLAGS.base_size) anchors = tf.constant(anchors_array, dtype=tf.float32) image, label, target, xmin, ymin, xmax, ymax, classes = input_reader.input( record_path=FLAGS.record_path, dataset_size=FLAGS.dataset_size, batch_size=FLAGS.batch_size, image_size=FLAGS.image_size) objectness_targets, xmins, ymins, xmaxes, ymaxes, weights, num_pos, num_neg = batch_process( label, target, xmin, ymin, xmax, ymax, FLAGS.num_samples) with tf.device('/gpu:0'): feature_map = feature_extractor.extract(base_net=FLAGS.base_net, image=image) obj_logits, box_logits = rpn.inference(feature_map, k=FLAGS.k) rpn_loss, cls_loss, box_loss = loss_op(cls_logits=obj_logits, box_logits=box_logits, objectness=objectness_targets, xmin=xmins, ymin=ymins, xmax=xmaxes, ymax=ymaxes, weights=weights, num_pos=num_pos, num_neg=num_neg, k=FLAGS.k, lamb=FLAGS.lamb, anchors=anchors) train_var = [ var for var in tf.global_variables() if 'Momentum' in var.name or 'rpn' in var.name or 'conv3' in var.name or 'conv4' in var.name or 'conv5' in var.name ] rpn_train = opt.minimize(rpn_loss, global_step=global_step, var_list=train_var) var_list = [ var for var in tf.global_variables() if not 'Momentum' in var.name and not 'rpn' in var.name ] restore_saver = tf.train.Saver(var_list) saver = tf.train.Saver(max_to_keep=1000) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_path, 'checkpoint') restore_saver.restore(sess, ckpt.model_checkpoint_path) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) start_time = datetime.now() iter_start_time = datetime.now() for i in range(10): _, test0, test1, test2 = sess.run( [rpn_train, rpn_loss, cls_loss, box_loss]) iter_end_time = datetime.now() print( "iteration {}: loss: {},class loss: {},regression loss: {}. duration: {} " .format(i + 1, test0, test1, test2, iter_end_time - iter_start_time)) if i % FLAGS.save_step == 0: saver.save(sess, './rpn.ckpt', global_step=global_step) if i % FLAGS.log_step == 0: log_file = open('./log_file', 'a') log_file.write('{},{},{}\n'.format(i, test1, test2)) log_file.close() iter_start_time = datetime.now() end_time = datetime.now() print('total training time: {}'.format(end_time - start_time)) coord.request_stop() coord.join(threads)
def index(): if request.method == 'POST': features = np.load('embeddings.npy') with open("img_paths.json", 'r') as f: img_paths = json.load(f) # ###### Image Embedding ######## # for image in os.listdir('./static/img/'): # if '.jpg' in image: # img_paths.append("./static/img/"+image) # feature_image = extract(img="./static/img/"+image) # features.append(feature_image) # print(len(img_paths)) # print(len(features)) # features_numpy = np.array(features) # features_numpy = features_numpy.reshape(features_numpy.shape[0]*features_numpy.shape[1],features_numpy.shape[2]) # features = features_numpy file = request.files['query_img'] # Save query image img = Image.open(file.stream) # PIL image uploaded_img_path = "static/uploaded/" + datetime.now().isoformat( ).replace(":", ".") + "_" + file.filename img.save(uploaded_img_path) # Run search query = extract(uploaded_img_path) query = query.reshape(1, 20) ### Create Cosine Function features = np.append(features, query, axis=0) img_paths.append(uploaded_img_path) ## Calculation M = features DotProducts = M.dot(M.T) # kronecker product of row norms NormKronecker = a([norm(M, axis=1)]) * a([norm(M, axis=1)]).T CosineSimilarity = DotProducts / NormKronecker import pandas as pd df_vae = pd.DataFrame(CosineSimilarity) df_vae.index = img_paths df_vae.columns = img_paths filtered_df = df_vae[[uploaded_img_path ]].sort_values(by=uploaded_img_path, ascending=False) filtered_df = filtered_df.head(30) recommended_images = filtered_df.index score_list = filtered_df[uploaded_img_path].to_list() # dists = np.linalg.norm(features-query, axis=1) # L2 distances to features # ids = np.argsort(dists)[:5] # Top 5 results scores = [(score_list[i], recommended_images[i]) for i in range(30)] print(scores) return render_template('index.html', query_path=uploaded_img_path, scores=scores) else: return render_template('index.html')
app = Flask(__name__) # Read image features model = VAE() features = [] img_paths = [] # for feature_path in Path("./static/feature").glob("*.npy"): # features.append(np.load(feature_path)) # img_paths.append(Path("./static/img") / (feature_path.stem + ".jpg")) # features = np.array(features) for image in os.listdir('./static/img/'): if '.jpg' in image: img_paths.append("./static/img/" + image) feature_image = extract(img="./static/img/" + image) features.append(feature_image) print(len(img_paths)) print(len(features)) features_numpy = np.array(features) features_numpy = features_numpy.reshape( features_numpy.shape[0] * features_numpy.shape[1], features_numpy.shape[2]) features = features_numpy ## Save List of image names with open("img_paths.json", 'w') as f: # indent=2 is not needed but makes the file human-readable json.dump(img_paths, f, indent=2)