def put(self, q, queue_max_size): paths = [] while True: if len(paths) == 0: paths = self.get_paths() if q.qsize() >= queue_max_size: time.sleep(0.1) continue ix = np.random.randint(0, len(paths)) path = paths.pop(ix) img = np.load(path) img = augment(img, self.image_size) q.put(img)
def put(self, q): queue_max_size = 1000 paths = [] while True: if len(paths) == 0: paths = self.get_paths() if q.qsize() >= queue_max_size: time.sleep(0.1) continue ix = np.random.randint(0, len(paths)) path = paths.pop(ix) im_path, bb_path = path npimg = np.fromfile(im_path, dtype=np.uint8) img = cv2.imdecode(npimg, cv2.IMREAD_COLOR) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) with open(bb_path) as f: lines = f.read().splitlines() boxes = [] for line in lines: ix, xmin, ymin, xmax, ymax = line.split('\t') onehot_label = np.eye(self.num_classes)[int(ix)] box = [float(xmin), float(ymin), float(xmax), float(ymax)] + onehot_label.tolist() boxes.append(box) img, boxes = augment(img, boxes, self.input_size) #debug if len(boxes) == 0: continue boxes = np.array(boxes) assignment = assign_boxes(boxes, self.priors, self.num_classes) q.put([img, assignment])
def main(args): priors = generate_priors(args.image_size) paths = [] for bb_path in glob.glob(os.path.join(args.label_dir, '*.txt')): im_path = os.path.join(args.image_dir, os.path.splitext(os.path.basename(bb_path))[0] + '.jpg') if os.path.exists(im_path): paths.append([im_path, bb_path]) for im_path, bb_path in paths: print(im_path) img = cv2.imread(im_path) img_h, img_w = img.shape[:2] with open(bb_path) as f: lines = f.read().splitlines() labels = [] for line in lines: ix, xmin, ymin, xmax, ymax = line.split('\t') onehot_label = np.eye(args.num_classes)[int(ix)] label = [float(xmin), float(ymin), float(xmax), float(ymax)] + onehot_label.tolist() labels.append(label) img0 = cv2.resize(img, (args.image_size, args.image_size)) for label in labels: xmin, ymin, xmax, ymax = [int(float(f) * args.image_size) for f in label[:4]] ix = np.argmax(label[4:]) clsid = int(ix) draw(img0, clsid, xmin, ymin, xmax, ymax) if args.augmentation: img, labels = augment(img, labels, args.image_size) img = np.array(img * 128.0 + 127.5, dtype=np.uint8) else: img = cv2.resize(img, (args.image_size, args.image_size)) labels = np.array(labels) y_true = assign_boxes(labels, priors, args.num_classes) preds = y_true[:, 4:-1] boxes = y_true[:, :4] decode_bbox = decode_box(boxes, priors) boxes = [] for box, pred in zip(decode_bbox, preds): xmin, ymin, xmax, ymax = [int(f * args.image_size) for f in box] clsid = np.argmax(pred) if clsid == 0: # in the case of background continue clsid -= 1 # decrement to skip background class boxes.append([clsid, 1.0, xmin, ymin, xmax, ymax]) if len(boxes) > 0: boxes = naive_nms(boxes, threshold=0., iou_threshold=0.8) for clsid, box in boxes.items(): for _, coord in box: left, top, right, bottom = [int(i) for i in coord] draw(img, clsid, left, top, right, bottom) print(len(labels), sum([len(j) for i, j in boxes.items()])) out = np.concatenate((img0, img), axis=1) cv2.imshow('', out) cv2.waitKey(0)
def train(num_rounds, num_edges_to_add, lam, tau, num_nodes, confident_size): memory.train() gnn.train() link_pred.train() torch_G = deepcopy(torch_G_base) memory.reset_state() # Start with a fresh memory. neighbor_loader.reset_state() # Start with an empty graph. neighbor_loader.insert(torch_G.edge_index[0], torch_G.edge_index[1]) log_metrics = log_metrics_with_G(G) T = deepcopy(C) T_random = deepcopy(C) total_loss = 0 loss_trace = [] T_trace = [] for i in range(num_rounds): print("Length of T", len(T)) torch_G = deepcopy(torch_G_base) T = deepcopy(C) T_random = deepcopy(C) subgraphs = sampleSubgraphs(torch_G, 30) for S1, S2 in combinations(subgraphs, 2): optimizer.zero_grad() # region learning_by_augmentation. S_1_aug, S_2_aug, y, y_rem, y_neg = augment( S1[1][1], S1[1][0], S2[1][1], S2[1][0], G_noisy, T) n_id = torch.cat( [S_1_aug[0], S_2_aug[0], S1[1][0], S2[1][0]]).unique() # n_id, edge_index, e_id = neighbor_loader(n_id) assoc[n_id] = torch.arange(n_id.size(0), device=device) z, last_update = memory(n_id, y_rem) # print(last_update) try: z = gnn(z, last_update, torch.cat( [assoc[S_1_aug], assoc[S_2_aug]], dim=1), i) except: print('In gnn') print([assoc[S_1_aug], assoc[S_2_aug]]) print(S1[1][1]) print(S2[1][1]) print(S_1_aug) print(S_2_aug) exit(1) try: memory.update_state(torch.cat([S_1_aug[0], S_2_aug[0]]), torch.cat( [S_1_aug[1], S_2_aug[1]]), torch.empty(S_1_aug.size(1) + S_2_aug.size(1)).fill_(i).type(torch.int64), torch.empty(S_1_aug.size(1) + S_2_aug.size(1), 0), y_rem) except: print('In memory update') print(S1[1][1]) print(S2[1][1]) print(S_1_aug) print(S_2_aug) exit(1) # assoc[n_id] = torch.arange(n_id.size(0), device=device) v_1 = torch.sum(z[assoc[S_1_aug[0]]], 0) / S_1_aug[0].size(0) v_2 = torch.sum(z[assoc[S_2_aug[0]]], 0) / S_2_aug[0].size(0) mu_prime_G = rel_subgraph(v_1, v_2) # N_1, N_2 = assoc[n_id], assoc[n_id] # assoc_s1[S1[1][0]] = torch.arange(S1[1][0].size(0), device=device) # assoc_s2[S2[1][0]] = torch.arange(S2[1][0].size(0), device=device) mu_p = torch.empty(n_id.size(0), n_id.size(0)).fill_(np.nan) truth = torch.empty(n_id.size(0), n_id.size(0)).fill_(np.nan) for n1, n2 in y: truth[assoc[n1], assoc[n2]] = 1.0 n_id_combos_src = torch.combinations( n_id, r=2, with_replacement=False) n_id_combos_dst = torch.flip(n_id_combos_src, [1]) n_id_combos = torch.cat( (n_id_combos_src, n_id_combos_dst), dim=0) n_id_A = n_id_combos[:, 0] n_id_B = n_id_combos[:, 1] z_A = z[assoc[n_id_A]] z_B = z[assoc[n_id_B]] n_id_1 = S_1_aug[1][0] n_id_2 = S_2_aug[1][0] v_A = torch.empty(z_A.shape) v_B = torch.empty(z_B.shape) mask_A1 = mask = torch.BoolTensor( [True if n in n_id_1 else False for n in n_id_A]) mask_A2 = mask = torch.BoolTensor( [True if n in n_id_2 else False for n in n_id_A]) mask_B1 = mask = torch.BoolTensor( [True if n in n_id_1 else False for n in n_id_B]) mask_B2 = mask = torch.BoolTensor( [True if n in n_id_2 else False for n in n_id_B]) v_A[mask_A1] = v_1 v_A[mask_A2] = v_2 v_B[mask_B1] = v_1 v_B[mask_B2] = v_2 mu_p_temp = rel_node(mu_prime_G*(v_A + z_A), mu_prime_G*(v_B + z_B)) # print(mu_p_temp.squeeze(1)) loss = None num_predictions = 0 for idx, edge in enumerate(n_id_combos): if tuple(edge) in y: if loss is None: loss = tau*(1.0 - mu_p_temp[idx][0])**2 else: loss += tau*(1.0 - mu_p_temp[idx][0])**2 num_predictions += 1 elif edge in torch_G.edge_index.t() and tuple(edge) not in y: if loss is None: loss = lam*(0.5 - mu_p_temp[idx][0])**2 else: loss += lam*(0.5 - mu_p_temp[idx][0])**2 if lam != 0.0: num_predictions += 1 elif tuple(edge) in y_neg: if loss is None: loss = (0.0 - mu_p_temp[idx][0])**2 else: loss += (0.0 - mu_p_temp[idx][0])**2 num_predictions += 1 if loss is None: continue loss /= num_predictions loss.backward() optimizer.step() memory.detach() # endregion n_id = torch.cat([S1[1][0], S2[1][0]]).unique() assoc[n_id] = torch.arange(n_id.size(0), device=device) z, last_update = memory(n_id, []) # print(last_update) z = gnn(z, last_update, torch.cat( [assoc[S1[1][1]], assoc[S2[1][1]]], dim=1), i) # print('updating memory') # memory.update_state(torch.cat([S1[1][1][0], S2[1][1][0]]), torch.cat( # [S1[1][1][1], S2[1][1][1]]), torch.empty(S1.size(1) + S2.size(1)).fill_(i).type(torch.int64), torch.empty(S1.size(1) + S2.size(1), 0), []) v_1 = torch.sum(z[assoc[S1[1][0]]], 0) / S1[1][0].size(0) v_2 = torch.sum(z[assoc[S2[1][0]]], 0) / S2[1][0].size(0) mu_prime_G = rel_subgraph(v_1, v_2) # N_1, N_2 = assoc[n_id], assoc[n_id] # assoc_s1[S1[1][0]] = torch.arange(S1[1][0].size(0), device=device) # assoc_s2[S2[1][0]] = torch.arange(S2[1][0].size(0), device=device) mu_p = torch.empty(n_id.size(0), n_id.size(0)).fill_(np.nan) truth = torch.empty(n_id.size(0), n_id.size(0)).fill_(np.nan) for n1, n2 in y: truth[assoc[n1], assoc[n2]] = 1.0 n_id_combos_src = torch.combinations( n_id, r=2, with_replacement=False) n_id_combos_dst = torch.flip(n_id_combos_src, [1]) n_id_combos = torch.cat( (n_id_combos_src, n_id_combos_dst), dim=0) n_id_A = n_id_combos[:, 0] n_id_B = n_id_combos[:, 1] z_A = z[assoc[n_id_A]] z_B = z[assoc[n_id_B]] n_id_1 = S1[1][0] n_id_2 = S2[1][0] v_A = torch.empty(z_A.shape) v_B = torch.empty(z_B.shape) mask_A1 = mask = torch.BoolTensor( [True if n in n_id_1 else False for n in n_id_A]) mask_A2 = mask = torch.BoolTensor( [True if n in n_id_2 else False for n in n_id_A]) mask_B1 = mask = torch.BoolTensor( [True if n in n_id_1 else False for n in n_id_B]) mask_B2 = mask = torch.BoolTensor( [True if n in n_id_2 else False for n in n_id_B]) v_A[mask_A1] = v_1 v_A[mask_A2] = v_2 v_B[mask_B1] = v_1 v_B[mask_B2] = v_2 mu_p_temp = rel_node(mu_prime_G*(v_A + z_A), mu_prime_G*(v_B + z_B)) # # print(mu_p_temp.squeeze(1)) # print(loss) # n_id_combos: all edge combinations # mu_p_temp: scores for each edge # add to set of edges top_edge_values, top_edge_indices = torch.topk( mu_p_temp.squeeze(1), min(num_edges_to_add, mu_p_temp.squeeze(1).size(0))) addEdges(torch_G, n_id_combos[top_edge_indices]) for edge in n_id_combos[top_edg # add random edges while len(T_random) < len(T): e1= random.randint(0, torch_G.num_nodes-1) e2= random.randint(0, torch_G.num_nodes-1) while e1 == e2: e2= random.randint(0, torch_G.num_nodes-1) T_random.add((e1, e2)) e_indices]: T.add(tuple(edge.tolist())) # remove from set of confident edges (without removing edges in C) bottom_edge_values, bottom_edge_indices = torch.topk( mu_p_temp.squeeze(1), min(num_edges_to_add, mu_p_temp.squeeze(1).size(0)), largest=False) edges_to_remove = [] for edge in n_id_combos[bottom_edge_indices]: if edge not in C and edge in T: T.remove(tuple(edge.tolist())) edges_to_remove.append(edge) # remove random edges while len(T_random) > len(T): edge = random.choice(tuple(T_random)) while edge in C: edge = random.choice(tuple(T_random)) T_random.remove(edge) wandb.log({'loss': loss.item(), 'T': len(T), 'round': i, 'to_add': num_edges_to_add, 'lam': lam, 'tau': tau, 'num_nodes': num_nodes, 'confident_size': confident_size }) log_metrics(T)(T_random) loss_trace.append(loss.item()) T_trace.append(len(T))
def main(args): obj = [v for k, v in table.mscoco2017.items()] sorted(obj, key=lambda x: x[0]) classes = [j for i, j in obj] colors = np.random.randint(0, 224, size=(len(classes), 3)) paths = [] for bb_path in glob.glob(os.path.join(args.label_dir, '*.txt')): im_path = os.path.join( args.image_dir, os.path.splitext(os.path.basename(bb_path))[0] + '.jpg') if os.path.exists(im_path): paths.append([im_path, bb_path]) for im_path, bb_path in paths: npimg = np.fromfile(im_path, dtype=np.uint8) img = cv2.imdecode(npimg, cv2.IMREAD_COLOR) with open(bb_path) as f: lines = f.read().splitlines() boxes = [] for line in lines: ix, xmin, ymin, xmax, ymax = line.split('\t') onehot_label = np.eye(len(classes))[int(ix)] box = [float(xmin), float(ymin), float(xmax), float(ymax)] + onehot_label.tolist() boxes.append(box) img, boxes = augment(img, boxes, args.image_size) img = np.array(img * 128.0 + 127.5, dtype=np.uint8) img_h, img_w = img.shape[:2] font = cv2.FONT_HERSHEY_SIMPLEX line_type = cv2.LINE_AA text_color = (255, 255, 255) border_size = 3 font_size = 0.4 font_scale = 1 for box in boxes: left = int(box[0] * img_w) top = int(box[1] * img_h) right = int(box[2] * img_w) bottom = int(box[3] * img_h) cls = np.argmax(box[4:]) name = classes[cls] color = tuple(colors[cls].tolist()) cv2.rectangle(img, (left, top), (right, bottom), color, border_size) (label_width, label_height), baseline = cv2.getTextSize(name, font, font_size, font_scale) cv2.rectangle(img, (left, top), (left + label_width, top + label_height), color, -1) cv2.putText(img, name, (left, top + label_height - border_size), font, font_size, text_color, font_scale, line_type) #print('{} - left: {}, top: {}, right: {}, bottom: {}'.format(name, left, top, right, bottom)) cv2.imshow('', img) cv2.waitKey(0)
def main(args): num_classes = 100 # dummpy priors = generate_priors() print(len(priors)) paths = [] label_paths = glob.glob(os.path.join(args.label_dir, '*.txt')) label_paths.sort() for bb_path in label_paths: im_path = os.path.join(args.image_dir, os.path.splitext(os.path.basename(bb_path))[0] + '.jpg') if os.path.exists(im_path): paths.append([im_path, bb_path]) for im_path, bb_path in paths: img = cv2.imread(im_path) img_h, img_w = img.shape[:2] with open(bb_path) as f: lines = f.read().splitlines() labels = [] for line in lines: ix, xmin, ymin, xmax, ymax = line.split('\t') onehot_label = np.eye(num_classes)[int(ix)] label = [float(xmin), float(ymin), float(xmax), float(ymax)] + onehot_label.tolist() labels.append(label) img0 = cv2.resize(img, (args.image_size, args.image_size)) for label in labels: xmin, ymin, xmax, ymax = [int(float(f) * args.image_size) for f in label[:4]] ix = np.argmax(label[4:]) clsid = int(ix) draw(img0, clsid, xmin, ymin, xmax, ymax) if args.augmentation: img, labels = augment(img, labels, args.image_size) if len(labels) == 0: continue img = np.array(img * 128.0 + 127.5, dtype=np.uint8) else: img = cv2.resize(img, (args.image_size, args.image_size)) labels = np.array(labels) imgs = [] num_assignment = [] for threshold in [0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.75]: img_ = img.copy() y_true = assign_boxes(labels, priors, num_classes, threshold) preds = y_true[:, 4:-1] boxes = y_true[:, :4] objectness = y_true[:, -1] decode_bbox = decode_box(boxes, priors) for box, pred, obj in zip(decode_bbox, preds, objectness): clsid = np.argmax(pred) if clsid == 0: # in the case of background continue xmin, ymin, xmax, ymax = [int(f * args.image_size) for f in box] clsid -= 1 # decrement to skip background class left = int(xmin) top = int(ymin) right = int(xmax) bottom = int(ymax) draw(img_, clsid, left, top, right, bottom) imgs.append(img_) num_assignment.append(sum(y_true[:, -1])) print(im_path, num_assignment) out = np.concatenate([img0] + imgs, axis=1) cv2.imshow('', out) cv2.waitKey(0)