def train(epoch): print('\nEpoch: %d' % epoch) net.train() train_loss = 0 correct = 0 total = 0 for batch_idx, (inputs, targets) in enumerate(trainloader): if use_cuda: inputs, targets = inputs.cuda(), targets.cuda() optimizer.zero_grad() inputs, targets = Variable(inputs), Variable(targets) outputs = net(inputs) a = list(net.parameters())[0] loss = criterion(outputs, targets) loss.backward() optimizer.step() b = list(net.parameters())[0] print(str(torch.equal(a.data, b.data))) keyboard() train_loss += loss.data[0] _, predicted = torch.max(outputs.data, 1) total += targets.size(0) correct += predicted.eq(targets.data).cpu().sum() progress_bar( batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' % (train_loss / (batch_idx + 1), 100. * correct / total, correct, total))
def forward(self, x): out = F.relu(self.bn1(self.conv1(x))) keyboard() out = self.layer1(out) out = self.layer2(out) out = self.layer3(out) out = self.layer4(out) out = F.avg_pool2d(out, 4) out = out.view(out.size(0), -1) out = self.linear(out) return out
S = tf.string_split([key], '/') length = tf.cast(S.dense_shape[1], tf.int32) # adjust constant value corresponding to your paths if you face issues. It should work for above format. label = S.values[length - tf.constant(2, dtype=tf.int32)] #keyboard() #label = tf.string_to_number(label,out_type=tf.int32) image = tf.image.decode_png(image_file) # Start a new session to show example output. with tf.Session() as sess: # Required to get the filename matching to run. #tf.initialize_all_variables().run() sess.run(tf.local_variables_initializer()) sess.run(tf.global_variables_initializer()) # Coordinate the loading of image files. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) for i in xrange(6): # Get an image tensor and print its value. key_val, label_val, image_tensor = sess.run([key, label, image]) print(image_tensor.shape) print(key_val) print(label_val) keyboard() # Finish off the filename queue coordinator. coord.request_stop() coord.join(threads)
def Matching(): args = getArgs() start = time.time() category_dirs = sorted(os.listdir(args.data_dir)) """ des = [] #Features labels=[] #label #Create Label volumes = [] # We will redo this part in pandas for category_dir in category_dirs: if os.path.isdir(os.path.join(args.data_dir,category_dir)): label = category_dir for file in os.listdir(os.path.join(args.data_dir, category_dir)): if file.endswith('bbox.yml'): fs = cv2.FileStorage(os.path.join(args.data_dir,category_dir,file),0) vector = fs.getNode("bbox").mat()[0].tolist() vector_normalized = [] for value in vector: vector_normalized.append(remap(value,0.45,0,0.9,0.1)) volume = vector[0]*vector[1]*vector[2] volume_normalized = remap_volume(volume,0.01,0,0.9,0.1) vector_normalized.append(volume_normalized) #volumes.append(volume) labels.append(label) #print("ClassName=%s" % (label)) des.append(vector_normalized) """ # Read in the data des=[] used_category=[] labels=[] for file in os.listdir(args.data_dir): if file.endswith(".csv"): my_dataframe = pd.read_csv(os.path.join(args.data_dir,file)) category_tmp = my_dataframe["item name"][0] if category_tmp not in used_category: used_category.append(category_tmp) x_value = remap(my_dataframe[" x"][0]) y_value = remap(my_dataframe[" y"][0]) z_value = remap(my_dataframe[" z"][0]) xyz = [x_value,y_value,z_value] xyz = sorted(xyz) des.append(xyz) labels.append(category_tmp) print 'start Grid Search' search_list=[29.5] tuned_parameters = [{'C': search_list}] linear_svc = LinearSVC()#SMN Modoul gscv = GridSearchCV(linear_svc, tuned_parameters ,cv=5) #parameters grid search print 'gscv.fit' gscv.fit(des, labels)#Training parameters tuning (Find the best parameters automatically) svm_best = gscv.best_estimator_#Choose the best tuned parameter print 'start re-learning SVM with best parameter set.' #### CalibratedClassifierCV is the calibrated classifier which can give probabilistic classifier ####sigmoid will use Platt's scaling. Refer to documentation for other methods. print 'searched result of C =', svm_best.C best_C_for_record = svm_best.C svm_best = CalibratedClassifierCV(svm_best,method='sigmoid',cv=3) #Parameter setup svm_best.fit(des, labels) #Training #Save Model joblib.dump(svm_best, './bbox_svm_parameter.pkl', compress=9) #Load Model svm_best = joblib.load('./bbox_svm_parameter.pkl') ####load test image (load test data) #label, test_image = LoadFile("crop_burts2.png", "/root/catkin_ws/src/tnp/tnp_svm/script/data/RealData") correct_count = 0 for n in range(len(des)): des_test = des[n] ###### SVM recognition classes = svm_best.classes_ #Model load label Must do #Forward Predict testResponse = svm_best.predict(des_test)[[0]] #print("testResponse="+ str(testResponse[0])) # print(classes) confidences_matrix = svm_best.predict_proba(des_test) # print(confidences_matrix) labels_class = svm_best.classes_ #read label again(You must do this or your label will changed by machine itself) #print("Real: "+str(labels[n])) sort_ascend = np.sort(confidences_matrix) #ranking the list of predicted data # print(sort_ascend) sort_index_ascend = np.argsort(confidences_matrix) #get index and sort # print(sort_index_ascend) sort_descend = sort_ascend[0][::-1] #ranked from big to small #print "sort_descend" ,sort_descend sort_index_descend = sort_index_ascend[0][::-1] #ranked from big to small (index) # print(sort_index_descend) items_ids=[] for i in range (0,len(labels_class)): items_ids.append(labels_class[sort_index_descend[i]]) #print(items_ids) if testResponse[0] == labels[n]: correct_count+=1 print("Success Rate: " + str(float(correct_count)/float(len(des)))) keyboard()
def main(): # setup data directory root_dir = dirname(realpath(__file__)) data_path = join(root_dir, '..', '..', 'data', 'amazon') # load the amazon csv with open(join(data_path, 'amazon.csv'), 'r', encoding='latin1') as f: amazon_df = pd.read_csv(f) amazon_df.rename(columns={'Unnamed: 0': 'pid'}, inplace=True) # include additional columns in amazon csv amazon_df = amazon_df.set_index('pid') amazon_df['related_title'] = '' amazon_df['related_description'] = '' # load the related text file with open(join(data_path, 'related.txt'), 'r', encoding='latin1') as f: related_text = f.read() related_lines = related_text.split('\n') # obtain product ids valid_df = amazon_df[amazon_df['title'].notna() & amazon_df['description'].notna()] pids = list(valid_df.index.values) titles = valid_df['title'].to_dict() descriptions = valid_df['description'].to_dict() related_pids = {} related_titles = {} related_descriptions = {} # loop over the rows and assign values counter = 0 for line in related_lines: # process string input item = line.split(' also purchased ') # obtain pid and related items pid = item[0] related = [ rel_pid for rel_pid in item[1].split(' ') if rel_pid != 'rights_details' ] # loop over related items for rel_pid in related: if rel_pid in pids: related_pids[pid] = rel_pid related_titles[pid] = titles[rel_pid] related_descriptions[pid] = descriptions[rel_pid] break # counter analysis counter += 1 if counter % 100 == 0: print(f'Counter: {counter}') keyboard() # write to amazon csv with open(join(data_path, 'amazon_full.csv'), 'w', encoding='latin1') as f: amazon_df.to_csv(f)
def Matching(): args = getArgs() start = time.time() category_dirs = sorted(os.listdir(args.data_dir_bbox)) weight_files = os.listdir(args.data_dir_weight) modelname = args.modelname # Read in the data des = [] used_category = [] labels = [] names = [ 'assured_face_mask', 'assured_lavender_epsom_salts', 'betty_crocker_measuring_spoons', 'bobbins_with_ring', 'casemate_black_binder', 'equate_flushable_wipes', 'five_star_red_hinged_note_cards', 'green_treat_bags', 'jute_twine', 'mini_solo_cups', 'orange_balloons', 'pink_flamingo_cup', 'r2_precision_pens', 'salad_tongs', 'scrub_buddies', 'star_wars_bubbles' ] for file in os.listdir(args.data_dir_bbox): if file.endswith(".csv"): bbox_dataframe = pd.read_csv(os.path.join(args.data_dir_bbox, file)) category_tmp = bbox_dataframe["item name"][0] # print(category_tmp) if category_tmp not in used_category: used_category.append(category_tmp) weight = -1 for weight_file in weight_files: # search for filename that if there is same as category_tmp, if it is-> add a noisy weight value and normalize it if category_tmp in weight_file: weight_dataframe = pd.read_csv( os.path.join(args.data_dir_weight, weight_file)) weight = float(weight_dataframe[" weight"][0]) weight = noisy_weight(weight) weight_normalized = remap_weight(weight) if weight == -1: print("Error! Missing " + category_tmp + " weight file.") exit() x_value = remap_xyz(bbox_dataframe[" x"][0]) y_value = remap_xyz(bbox_dataframe[" y"][0]) z_value = remap_xyz(bbox_dataframe[" z"][0]) volume = bbox_dataframe[" x"][0] * bbox_dataframe[" y"][ 0] * bbox_dataframe[" z"][0] volume_normalized = remap_volume(volume) xyz = [x_value, y_value, z_value] xyz_weight = sorted(xyz) xyz_weight.append(volume_normalized) xyz_weight.append(weight_normalized) # print(xyz_weight) # July 24, the max values were [.414, .383, .413] normalized to 0.45 des.append(xyz_weight) labels.append(category_tmp) print 'start Grid Search' search_list = np.asarray(range(1, 300)) * 0.1 search_list = search_list.tolist() tuned_parameters = [{'C': search_list}] linear_svc = LinearSVC() #SMN Modoul gscv = GridSearchCV(linear_svc, tuned_parameters, cv=5) #parameters grid search print 'gscv.fit' gscv.fit( des, labels ) #Training parameters tuning (Find the best parameters automatically) #print 'svm_best' svm_best = gscv.best_estimator_ #Choose the best tuned parameter print 'start re-learning SVM with best parameter set.' #### CalibratedClassifierCV is the calibrated classifier which can give probabilistic classifier ####sigmoid will use Platt's scaling. Refer to documentation for other methods. print 'searched result of C =', svm_best.C best_C_for_record = svm_best.C svm_best = CalibratedClassifierCV(svm_best, method='sigmoid', cv=3) #Parameter setup svm_best.fit(des, labels) #Training #Save Model joblib.dump(svm_best, modelname, compress=9) #Load Model svm_best = joblib.load(modelname) ####load test image (load test data) #label, test_image = LoadFile("crop_burts2.png", "/root/catkin_ws/src/tnp/tnp_svm/script/data/RealData") correct_count = 0 for n in range(len(des)): des_test = des[n] ###### SVM recognition classes = svm_best.classes_ #Model load label Must do #Forward Predict testResponse = svm_best.predict(des_test)[[0]] #print("testResponse="+ str(testResponse[0])) # print(classes) confidences_matrix = svm_best.predict_proba(des_test) # print(confidences_matrix) labels_class = svm_best.classes_ #read label again(You must do this or your label will changed by machine itself) #print("Real: "+str(labels[n])) sort_ascend = np.sort( confidences_matrix) #ranking the list of predicted data # print(sort_ascend) sort_index_ascend = np.argsort(confidences_matrix) #get index and sort # print(sort_index_ascend) sort_descend = sort_ascend[0][::-1] #ranked from big to small #print "sort_descend" ,sort_descend sort_index_descend = sort_index_ascend[ 0][::-1] #ranked from big to small (index) # print(sort_index_descend) items_ids = [] for i in range(0, len(labels_class)): items_ids.append(labels_class[sort_index_descend[i]]) #print(items_ids) if testResponse[0] == labels[n]: correct_count += 1 print("Success Rate: " + str(float(correct_count) / float(len(des)))) keyboard()
def Matching(): start = time.time() des = [] #Features labels=[] #label """ #Create Label for category_dir in category_dirs: for rgb_image_path in glob.glob(os.path.join(args.data_dir, group_dir, category_dir)): for file in os.listdir(os.path.join(args.data_dir, group_dir, category_dir)): if file.endswith('.png'): label, image = LoadFile(file, rgb_image_path) des1 = ComputeHOGDescriptor(image) labels.append(label) print("ClassName=%s" % (label)) des.append(des1) """ train_labels = ["hand_weight","windex","bag","meshcup","pen","Ramen"] #Create simulate data for _ in range(950): train_vectors = np.random.rand(3) #data simulation des.append(train_vectors) chose_label = random.choice(train_labels) print(chose_label) labels.append(chose_label) """ print 'start Grid Search' #### for color # tuned_parameters = [{'C': [0.6, 0.7, 0.8, 0.9]}] ### 12 12 3 3 C=11.0 32crop C= 10.0 # tuned_parameters = [{'C': [9.5, 10.0, 11.0]}] ### 8 8 1 1 paste on black C=0.5 # tuned_parameters = [{'C': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]}] ## 8 8 3 3 rotate C=5.0 32_paste:C=5.0 32_crop:C=5.0 crop:C=4.5 tuned_parameters = [{'C': [4.5, 5.0, 5.5]}] ### 8 8 4 4 C=7.0 # tuned_parameters = [{'C': [3.0, 4.0, 5.0, 6.0, 7.0, 8.0]}] ## 8 8 4 4 rotate:C=2.8 paste:C=2.8 # tuned_parameters = [{'C': [2.5, 2.6, 2.7, 2.8]}] ### 9 9 3 3 rotate C=6.0 # tuned_parameters = [{'C': [5.5, 5.7, 6.0, 6.2, 6.5]}] # tuned_parameters = [{'C': [0.1, 0.5, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]}] # parameters = {'estimator__kernel':('linear', 'rbf'), 'estimator__C':[1, 10]} linear_svc = LinearSVC()#SMN Modoul gscv = GridSearchCV(linear_svc, tuned_parameters ,cv=5) #parameters grid search print 'gscv.fit' gscv.fit(des, labels)#Training parameters tuning (Find the best parameters automatically) #print 'svm_best' svm_best = gscv.best_estimator_#Choose the best tuned parameter print 'searched result of C =', svm_best.C print 'start re-learning SVM with best parameter set.' """ #### CalibratedClassifierCV is the calibrated classifier which can give probabilistic classifier ####sigmoid will use Platt's scaling. Refer to documentation for other methods. svm_best = LinearSVC(C=4.0) svm_best = CalibratedClassifierCV(svm_best,method='sigmoid',cv=3) #Parameter setup svm_best.fit(des, labels) #Training #Save Model joblib.dump(svm_best, './bbox_svm_parameter.pkl', compress=9) #Load Model svm_best = joblib.load('./bbox_svm_parameter.pkl') ####load test image (load test data) #label, test_image = LoadFile("crop_burts2.png", "/root/catkin_ws/src/tnp/tnp_svm/script/data/RealData") des_test = des[0] ###### SVM recognition classes = svm_best.classes_ #Model load label Must do #Forward Predict testResponse = svm_best.predict(des_test) print("testResponse=", (testResponse)) # print(classes) confidences_matrix = svm_best.predict_proba(des_test) # print(confidences_matrix) labels = svm_best.classes_ #read label again(You must do this or your label will changed by machine itself) sort_ascend = np.sort(confidences_matrix) #ranking the list of predicted data # print(sort_ascend) sort_index_ascend = np.argsort(confidences_matrix) #get index and sort # print(sort_index_ascend) sort_descend = sort_ascend[0][::-1] #ranked from big to small print "sort_descend" ,sort_descend sort_index_descend = sort_index_ascend[0][::-1] #ranked from big to small (index) # print(sort_index_descend) items_ids=[] for i in range (0,len(labels)): items_ids.append(labels[sort_index_descend[i]]) print(items_ids) keyboard() i=0