def put(self, q, queue_max_size):
paths = []
while True:
if len(paths) == 0:
paths = self.get_paths()
if q.qsize() >= queue_max_size:
time.sleep(0.1)
continue
ix = np.random.randint(0, len(paths))
path = paths.pop(ix)
img = np.load(path)
img = augment(img, self.image_size)
q.put(img)
def put(self, q):
queue_max_size = 1000
paths = []
while True:
if len(paths) == 0:
paths = self.get_paths()
if q.qsize() >= queue_max_size:
time.sleep(0.1)
continue
ix = np.random.randint(0, len(paths))
path = paths.pop(ix)
im_path, bb_path = path
npimg = np.fromfile(im_path, dtype=np.uint8)
img = cv2.imdecode(npimg, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
with open(bb_path) as f:
lines = f.read().splitlines()
boxes = []
for line in lines:
ix, xmin, ymin, xmax, ymax = line.split('\t')
onehot_label = np.eye(self.num_classes)[int(ix)]
box = [float(xmin),
float(ymin),
float(xmax),
float(ymax)] + onehot_label.tolist()
boxes.append(box)
img, boxes = augment(img, boxes, self.input_size) #debug
if len(boxes) == 0:
continue
boxes = np.array(boxes)
assignment = assign_boxes(boxes, self.priors, self.num_classes)
q.put([img, assignment])
def main(args):
priors = generate_priors(args.image_size)
paths = []
for bb_path in glob.glob(os.path.join(args.label_dir, '*.txt')):
im_path = os.path.join(args.image_dir, os.path.splitext(os.path.basename(bb_path))[0] + '.jpg')
if os.path.exists(im_path):
paths.append([im_path, bb_path])
for im_path, bb_path in paths:
print(im_path)
img = cv2.imread(im_path)
img_h, img_w = img.shape[:2]
with open(bb_path) as f:
lines = f.read().splitlines()
labels = []
for line in lines:
ix, xmin, ymin, xmax, ymax = line.split('\t')
onehot_label = np.eye(args.num_classes)[int(ix)]
label = [float(xmin), float(ymin), float(xmax), float(ymax)] + onehot_label.tolist()
labels.append(label)
img0 = cv2.resize(img, (args.image_size, args.image_size))
for label in labels:
xmin, ymin, xmax, ymax = [int(float(f) * args.image_size) for f in label[:4]]
ix = np.argmax(label[4:])
clsid = int(ix)
draw(img0, clsid, xmin, ymin, xmax, ymax)
if args.augmentation:
img, labels = augment(img, labels, args.image_size)
img = np.array(img * 128.0 + 127.5, dtype=np.uint8)
else:
img = cv2.resize(img, (args.image_size, args.image_size))
labels = np.array(labels)
y_true = assign_boxes(labels, priors, args.num_classes)
preds = y_true[:, 4:-1]
boxes = y_true[:, :4]
decode_bbox = decode_box(boxes, priors)
boxes = []
for box, pred in zip(decode_bbox, preds):
xmin, ymin, xmax, ymax = [int(f * args.image_size) for f in box]
clsid = np.argmax(pred)
if clsid == 0:
# in the case of background
continue
clsid -= 1 # decrement to skip background class
boxes.append([clsid, 1.0, xmin, ymin, xmax, ymax])
if len(boxes) > 0:
boxes = naive_nms(boxes, threshold=0., iou_threshold=0.8)
for clsid, box in boxes.items():
for _, coord in box:
left, top, right, bottom = [int(i) for i in coord]
draw(img, clsid, left, top, right, bottom)
print(len(labels), sum([len(j) for i, j in boxes.items()]))
out = np.concatenate((img0, img), axis=1)
cv2.imshow('', out)
cv2.waitKey(0)
def train(num_rounds, num_edges_to_add, lam, tau, num_nodes, confident_size):
memory.train()
gnn.train()
link_pred.train()
torch_G = deepcopy(torch_G_base)
memory.reset_state() # Start with a fresh memory.
neighbor_loader.reset_state() # Start with an empty graph.
neighbor_loader.insert(torch_G.edge_index[0], torch_G.edge_index[1])
log_metrics = log_metrics_with_G(G)
T = deepcopy(C)
T_random = deepcopy(C)
total_loss = 0
loss_trace = []
T_trace = []
for i in range(num_rounds):
print("Length of T", len(T))
torch_G = deepcopy(torch_G_base)
T = deepcopy(C)
T_random = deepcopy(C)
subgraphs = sampleSubgraphs(torch_G, 30)
for S1, S2 in combinations(subgraphs, 2):
optimizer.zero_grad()
# region learning_by_augmentation.
S_1_aug, S_2_aug, y, y_rem, y_neg = augment(
S1[1][1], S1[1][0], S2[1][1], S2[1][0], G_noisy, T)
n_id = torch.cat(
[S_1_aug[0], S_2_aug[0], S1[1][0], S2[1][0]]).unique()
# n_id, edge_index, e_id = neighbor_loader(n_id)
assoc[n_id] = torch.arange(n_id.size(0), device=device)
z, last_update = memory(n_id, y_rem)
# print(last_update)
try:
z = gnn(z, last_update, torch.cat(
[assoc[S_1_aug], assoc[S_2_aug]], dim=1), i)
except:
print('In gnn')
print([assoc[S_1_aug], assoc[S_2_aug]])
print(S1[1][1])
print(S2[1][1])
print(S_1_aug)
print(S_2_aug)
exit(1)
try:
memory.update_state(torch.cat([S_1_aug[0], S_2_aug[0]]), torch.cat(
[S_1_aug[1], S_2_aug[1]]), torch.empty(S_1_aug.size(1) + S_2_aug.size(1)).fill_(i).type(torch.int64), torch.empty(S_1_aug.size(1) + S_2_aug.size(1), 0), y_rem)
except:
print('In memory update')
print(S1[1][1])
print(S2[1][1])
print(S_1_aug)
print(S_2_aug)
exit(1)
# assoc[n_id] = torch.arange(n_id.size(0), device=device)
v_1 = torch.sum(z[assoc[S_1_aug[0]]], 0) / S_1_aug[0].size(0)
v_2 = torch.sum(z[assoc[S_2_aug[0]]], 0) / S_2_aug[0].size(0)
mu_prime_G = rel_subgraph(v_1, v_2)
# N_1, N_2 = assoc[n_id], assoc[n_id]
# assoc_s1[S1[1][0]] = torch.arange(S1[1][0].size(0), device=device)
# assoc_s2[S2[1][0]] = torch.arange(S2[1][0].size(0), device=device)
mu_p = torch.empty(n_id.size(0), n_id.size(0)).fill_(np.nan)
truth = torch.empty(n_id.size(0), n_id.size(0)).fill_(np.nan)
for n1, n2 in y:
truth[assoc[n1], assoc[n2]] = 1.0
n_id_combos_src = torch.combinations(
n_id, r=2, with_replacement=False)
n_id_combos_dst = torch.flip(n_id_combos_src, [1])
n_id_combos = torch.cat(
(n_id_combos_src, n_id_combos_dst), dim=0)
n_id_A = n_id_combos[:, 0]
n_id_B = n_id_combos[:, 1]
z_A = z[assoc[n_id_A]]
z_B = z[assoc[n_id_B]]
n_id_1 = S_1_aug[1][0]
n_id_2 = S_2_aug[1][0]
v_A = torch.empty(z_A.shape)
v_B = torch.empty(z_B.shape)
mask_A1 = mask = torch.BoolTensor(
[True if n in n_id_1 else False for n in n_id_A])
mask_A2 = mask = torch.BoolTensor(
[True if n in n_id_2 else False for n in n_id_A])
mask_B1 = mask = torch.BoolTensor(
[True if n in n_id_1 else False for n in n_id_B])
mask_B2 = mask = torch.BoolTensor(
[True if n in n_id_2 else False for n in n_id_B])
v_A[mask_A1] = v_1
v_A[mask_A2] = v_2
v_B[mask_B1] = v_1
v_B[mask_B2] = v_2
mu_p_temp = rel_node(mu_prime_G*(v_A + z_A),
mu_prime_G*(v_B + z_B))
# print(mu_p_temp.squeeze(1))
loss = None
num_predictions = 0
for idx, edge in enumerate(n_id_combos):
if tuple(edge) in y:
if loss is None:
loss = tau*(1.0 - mu_p_temp[idx][0])**2
else:
loss += tau*(1.0 - mu_p_temp[idx][0])**2
num_predictions += 1
elif edge in torch_G.edge_index.t() and tuple(edge) not in y:
if loss is None:
loss = lam*(0.5 - mu_p_temp[idx][0])**2
else:
loss += lam*(0.5 - mu_p_temp[idx][0])**2
if lam != 0.0:
num_predictions += 1
elif tuple(edge) in y_neg:
if loss is None:
loss = (0.0 - mu_p_temp[idx][0])**2
else:
loss += (0.0 - mu_p_temp[idx][0])**2
num_predictions += 1
if loss is None:
continue
loss /= num_predictions
loss.backward()
optimizer.step()
memory.detach()
# endregion
n_id = torch.cat([S1[1][0], S2[1][0]]).unique()
assoc[n_id] = torch.arange(n_id.size(0), device=device)
z, last_update = memory(n_id, [])
# print(last_update)
z = gnn(z, last_update, torch.cat(
[assoc[S1[1][1]], assoc[S2[1][1]]], dim=1), i)
# print('updating memory')
# memory.update_state(torch.cat([S1[1][1][0], S2[1][1][0]]), torch.cat(
# [S1[1][1][1], S2[1][1][1]]), torch.empty(S1.size(1) + S2.size(1)).fill_(i).type(torch.int64), torch.empty(S1.size(1) + S2.size(1), 0), [])
v_1 = torch.sum(z[assoc[S1[1][0]]], 0) / S1[1][0].size(0)
v_2 = torch.sum(z[assoc[S2[1][0]]], 0) / S2[1][0].size(0)
mu_prime_G = rel_subgraph(v_1, v_2)
# N_1, N_2 = assoc[n_id], assoc[n_id]
# assoc_s1[S1[1][0]] = torch.arange(S1[1][0].size(0), device=device)
# assoc_s2[S2[1][0]] = torch.arange(S2[1][0].size(0), device=device)
mu_p = torch.empty(n_id.size(0), n_id.size(0)).fill_(np.nan)
truth = torch.empty(n_id.size(0), n_id.size(0)).fill_(np.nan)
for n1, n2 in y:
truth[assoc[n1], assoc[n2]] = 1.0
n_id_combos_src = torch.combinations(
n_id, r=2, with_replacement=False)
n_id_combos_dst = torch.flip(n_id_combos_src, [1])
n_id_combos = torch.cat(
(n_id_combos_src, n_id_combos_dst), dim=0)
n_id_A = n_id_combos[:, 0]
n_id_B = n_id_combos[:, 1]
z_A = z[assoc[n_id_A]]
z_B = z[assoc[n_id_B]]
n_id_1 = S1[1][0]
n_id_2 = S2[1][0]
v_A = torch.empty(z_A.shape)
v_B = torch.empty(z_B.shape)
mask_A1 = mask = torch.BoolTensor(
[True if n in n_id_1 else False for n in n_id_A])
mask_A2 = mask = torch.BoolTensor(
[True if n in n_id_2 else False for n in n_id_A])
mask_B1 = mask = torch.BoolTensor(
[True if n in n_id_1 else False for n in n_id_B])
mask_B2 = mask = torch.BoolTensor(
[True if n in n_id_2 else False for n in n_id_B])
v_A[mask_A1] = v_1
v_A[mask_A2] = v_2
v_B[mask_B1] = v_1
v_B[mask_B2] = v_2
mu_p_temp = rel_node(mu_prime_G*(v_A + z_A),
mu_prime_G*(v_B + z_B))
#
# print(mu_p_temp.squeeze(1))
# print(loss)
# n_id_combos: all edge combinations
# mu_p_temp: scores for each edge
# add to set of edges
top_edge_values, top_edge_indices = torch.topk(
mu_p_temp.squeeze(1), min(num_edges_to_add, mu_p_temp.squeeze(1).size(0)))
addEdges(torch_G, n_id_combos[top_edge_indices])
for edge in n_id_combos[top_edg
# add random edges
while len(T_random) < len(T):
e1= random.randint(0, torch_G.num_nodes-1)
e2= random.randint(0, torch_G.num_nodes-1)
while e1 == e2:
e2= random.randint(0, torch_G.num_nodes-1)
T_random.add((e1, e2))
e_indices]:
T.add(tuple(edge.tolist()))
# remove from set of confident edges (without removing edges in C)
bottom_edge_values, bottom_edge_indices = torch.topk(
mu_p_temp.squeeze(1), min(num_edges_to_add, mu_p_temp.squeeze(1).size(0)), largest=False)
edges_to_remove = []
for edge in n_id_combos[bottom_edge_indices]:
if edge not in C and edge in T:
T.remove(tuple(edge.tolist()))
edges_to_remove.append(edge)
# remove random edges
while len(T_random) > len(T):
edge = random.choice(tuple(T_random))
while edge in C:
edge = random.choice(tuple(T_random))
T_random.remove(edge)
wandb.log({'loss': loss.item(), 'T': len(T), 'round': i,
'to_add': num_edges_to_add, 'lam': lam, 'tau': tau, 'num_nodes': num_nodes, 'confident_size': confident_size
})
log_metrics(T)(T_random)
loss_trace.append(loss.item())
T_trace.append(len(T))
def main(args):
obj = [v for k, v in table.mscoco2017.items()]
sorted(obj, key=lambda x: x[0])
classes = [j for i, j in obj]
colors = np.random.randint(0, 224, size=(len(classes), 3))
paths = []
for bb_path in glob.glob(os.path.join(args.label_dir, '*.txt')):
im_path = os.path.join(
args.image_dir,
os.path.splitext(os.path.basename(bb_path))[0] + '.jpg')
if os.path.exists(im_path):
paths.append([im_path, bb_path])
for im_path, bb_path in paths:
npimg = np.fromfile(im_path, dtype=np.uint8)
img = cv2.imdecode(npimg, cv2.IMREAD_COLOR)
with open(bb_path) as f:
lines = f.read().splitlines()
boxes = []
for line in lines:
ix, xmin, ymin, xmax, ymax = line.split('\t')
onehot_label = np.eye(len(classes))[int(ix)]
box = [float(xmin),
float(ymin),
float(xmax),
float(ymax)] + onehot_label.tolist()
boxes.append(box)
img, boxes = augment(img, boxes, args.image_size)
img = np.array(img * 128.0 + 127.5, dtype=np.uint8)
img_h, img_w = img.shape[:2]
font = cv2.FONT_HERSHEY_SIMPLEX
line_type = cv2.LINE_AA
text_color = (255, 255, 255)
border_size = 3
font_size = 0.4
font_scale = 1
for box in boxes:
left = int(box[0] * img_w)
top = int(box[1] * img_h)
right = int(box[2] * img_w)
bottom = int(box[3] * img_h)
cls = np.argmax(box[4:])
name = classes[cls]
color = tuple(colors[cls].tolist())
cv2.rectangle(img, (left, top), (right, bottom), color,
border_size)
(label_width,
label_height), baseline = cv2.getTextSize(name, font, font_size,
font_scale)
cv2.rectangle(img, (left, top),
(left + label_width, top + label_height), color, -1)
cv2.putText(img, name, (left, top + label_height - border_size),
font, font_size, text_color, font_scale, line_type)
#print('{} - left: {}, top: {}, right: {}, bottom: {}'.format(name, left, top, right, bottom))
cv2.imshow('', img)
cv2.waitKey(0)
def main(args):
num_classes = 100 # dummpy
priors = generate_priors()
print(len(priors))
paths = []
label_paths = glob.glob(os.path.join(args.label_dir, '*.txt'))
label_paths.sort()
for bb_path in label_paths:
im_path = os.path.join(args.image_dir, os.path.splitext(os.path.basename(bb_path))[0] + '.jpg')
if os.path.exists(im_path):
paths.append([im_path, bb_path])
for im_path, bb_path in paths:
img = cv2.imread(im_path)
img_h, img_w = img.shape[:2]
with open(bb_path) as f:
lines = f.read().splitlines()
labels = []
for line in lines:
ix, xmin, ymin, xmax, ymax = line.split('\t')
onehot_label = np.eye(num_classes)[int(ix)]
label = [float(xmin), float(ymin), float(xmax), float(ymax)] + onehot_label.tolist()
labels.append(label)
img0 = cv2.resize(img, (args.image_size, args.image_size))
for label in labels:
xmin, ymin, xmax, ymax = [int(float(f) * args.image_size) for f in label[:4]]
ix = np.argmax(label[4:])
clsid = int(ix)
draw(img0, clsid, xmin, ymin, xmax, ymax)
if args.augmentation:
img, labels = augment(img, labels, args.image_size)
if len(labels) == 0:
continue
img = np.array(img * 128.0 + 127.5, dtype=np.uint8)
else:
img = cv2.resize(img, (args.image_size, args.image_size))
labels = np.array(labels)
imgs = []
num_assignment = []
for threshold in [0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.75]:
img_ = img.copy()
y_true = assign_boxes(labels, priors, num_classes, threshold)
preds = y_true[:, 4:-1]
boxes = y_true[:, :4]
objectness = y_true[:, -1]
decode_bbox = decode_box(boxes, priors)
for box, pred, obj in zip(decode_bbox, preds, objectness):
clsid = np.argmax(pred)
if clsid == 0:
# in the case of background
continue
xmin, ymin, xmax, ymax = [int(f * args.image_size) for f in box]
clsid -= 1 # decrement to skip background class
left = int(xmin)
top = int(ymin)
right = int(xmax)
bottom = int(ymax)
draw(img_, clsid, left, top, right, bottom)
imgs.append(img_)
num_assignment.append(sum(y_true[:, -1]))
print(im_path, num_assignment)
out = np.concatenate([img0] + imgs, axis=1)
cv2.imshow('', out)
cv2.waitKey(0)