def predict(imagePath):
#predict image
frame = cv2.imread(imagePath)
image = frame[...,::-1]
h, w = image.shape[:2]
inferTime = None
X, pad_up, pad_left, h_new, w_new = utils.preprocessing(image, expected_size=args.input_sz, pad_value=0)
with torch.no_grad():
if args.use_cuda:
start = time.time()
mask = model(X.cuda())
end = time.time()
inferTime = end-start
mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new]
mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0,1,...].cpu().numpy()
else:
start = time.time()
mask = model(X)
end = time.time()
inferTime = end-start
mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new]
mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0,1,...].numpy()
#mask outputSection
booleanMask = mask >= 0.5
return booleanMask, frame, inferTime
def predict(model,input,bg): image = cv2.imread(input) X, pad_up, pad_left, h_new, w_new = utils.preprocessing(image, expected_size=input_sz, pad_value=0) h,w = image.shape[:2] if bg is not None: BACKGROUND = cv2.imread(bg)[...,::-1] BACKGROUND = cv2.resize(BACKGROUND, (w,h), interpolation=cv2.INTER_LINEAR) KERNEL_SZ = 25 SIGMA = 0 else: COLOR1 = [255, 0, 0] COLOR2 = [0, 0, 255] with torch.no_grad(): mask = model(X) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].numpy() if bg is None: image_alpha = utils.draw_matting(image, mask) # image_alpha = utils.draw_transperency(image, mask, COLOR1, COLOR2) else: image_alpha = utils.draw_fore_to_back(image, mask, BACKGROUND, kernel_sz=KERNEL_SZ, sigma=SIGMA) print(image_alpha.shape) return image_alpha
def segment_humans(model, inputs):
frame = np.array(inputs["input_image"])
#image = frame[...,::-1]
h, w = frame.shape[0], frame.shape[1]
# Predict mask
X, pad_up, pad_left, h_new, w_new = utils.preprocessing(frame, expected_size=320, pad_value=0)
with torch.no_grad():
if torch.cuda.is_available():
mask = model(X.cuda())
mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new]
mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0,1,...].cpu().numpy()
else:
mask = model(X)
mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new]
mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0,1,...].numpy()
mask = 255*mask
mask = np.expand_dims(mask, axis=2)
image_alpha = np.concatenate((frame, mask), axis=2)
return image_alpha.astype(np.uint8)
def config2embedding(config, b):
data, _ = config2data_A(config, b)
adj, op = data2mat(data)
adj_, op_ = adj, op
adj, op = torch.Tensor(adj).unsqueeze(0).cuda(), torch.Tensor(op).unsqueeze(0).cuda()
adj, op, prep_reverse = preprocessing(adj, op, **cfg['prep'])
embedding = model._encoder(op, adj)[0]
return embedding, adj_, op_
def segment_background(deeplab_weights_path=None, input_images_path=None):
# Set testing device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Set up model
# model = UNet(
# backbone="mobilenetv2",
# num_classes=2,
# pretrained_backbone=None
# )
# trained_dict = torch.load('UNet_MobileNetV2.pth', map_location="cpu")['state_dict']
# model.load_state_dict(trained_dict, strict=False)
# model = model.to(device)
# model.eval()
model = DeepLabV3Plus(backbone="resnet18")
trained_dict = torch.load(deeplab_weights_path,
map_location="cpu")["state_dict"]
model.load_state_dict(trained_dict, strict=False)
model = model.to(device)
model.eval()
# Output folder to save extracted backgrounds
output_folder = Path(input_images_path).parent.joinpath(
"extracted_backgrounds")
Path(output_folder).mkdir(parents=True, exist_ok=True)
# Extract backgrounds
for img_path in Path(input_images_path).iterdir():
img = cv2.imread(str(img_path))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
print("Processing:", str(img_path))
h, w = img.shape[:2]
X, pad_up, pad_left, h_new, w_new = utils.preprocessing(
img, expected_size=320, pad_value=0)
with torch.no_grad():
mask = model(X.cuda())
mask = mask[..., pad_up:pad_up + h_new, pad_left:pad_left + w_new]
mask = F.interpolate(mask,
size=(h, w),
mode="bilinear",
align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0, 1, ...].cpu().numpy()
mask = np.where(mask > 0.9, 0, 1)
# Apply mask on original RGB image to extract background
img[:, :, 0] = mask * img[:, :, 0]
img[:, :, 1] = mask * img[:, :, 1]
img[:, :, 2] = mask * img[:, :, 2]
output_name = output_folder.joinpath(img_path.stem + img_path.suffix)
cv2.imwrite(str(output_name), cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
def __init__(self, model, ops, adj, cfg):
print('Initializing NN decoder')
t_s = time.time()
self.model = model
self.ops = ops
self.adj = adj
self.cfg = cfg
with torch.no_grad():
adj_prep, ops_prep, _ = preprocessing(self.adj, self.ops,
**self.cfg['prep'])
self.embedding = self.model.encoder(ops_prep, adj_prep)
assert len(self.embedding.shape) == 3
print('Using {} seconds to initialize NN decoder'.format(time.time() -
t_s))
def find_NN(self, ops, adj, ind, k=10):
assert len(ops.shape) == 3
ind_t1_list = []
ind_tk_list = []
with torch.no_grad():
adj_prep, ops_prep, _ = preprocessing(adj, ops, **self.cfg['prep'])
embeddings = self.model.encoder(ops_prep, adj_prep)
for e in embeddings:
dist = torch.sum((self.embedding - e)**2, dim=[1, 2])
_, ind_t1 = torch.topk(dist, 1, largest=False)
_, ind_tk = torch.topk(dist, k, largest=False)
ind_t1_list.append(ind_t1.item())
ind_tk_list.append(ind_tk.tolist())
op_recon, adj_recon = self.ops[ind_t1_list], self.adj[ind_t1_list]
op_recon_tk, adj_recon_tk = self.ops[ind_t1_list], self.adj[
ind_t1_list]
return op_recon, adj_recon, op_recon_tk, adj_recon_tk, ind_t1_list, ind_tk_list
def _reset(self, data_path, save):
if not save:
print("extract arch2vec on DARTS search space ...")
dataset = load_json(data_path)
print("length of the dataset: {}".format(len(dataset)))
self.f_path = os.path.join(self.dir_name, 'arch2vec-darts.pt')
if os.path.exists(self.f_path):
print('{} is already saved'.format(self.f_path))
exit()
print('save to {}'.format(self.f_path))
counter = 0
self.model.eval()
for k, v in dataset.items():
adj = torch.Tensor(v[0]).unsqueeze(0).cuda()
ops = torch.Tensor(one_hot_darts(v[1])).unsqueeze(0).cuda()
adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep'])
with torch.no_grad():
x, _ = self.model._encoder(ops, adj)
self.embedding[counter] = {
'feature': x.squeeze(0).mean(dim=0).cpu(),
'genotype': process(v[2])
}
print("{}/{}".format(counter, len(dataset)))
counter += 1
torch.save(self.embedding, self.f_path)
print("finished arch2vec extraction")
exit()
else:
self.f_path = os.path.join(self.dir_name, 'arch2vec-darts.pt')
print("load arch2vec from: {}".format(self.f_path))
self.embedding = torch.load(self.f_path)
for ind in range(len(self.embedding)):
self.features.append(self.embedding[ind]['feature'])
self.genotype.append(self.embedding[ind]['genotype'])
self.features = torch.stack(self.features, dim=0)
print('loading finished. pretrained embeddings shape: {}'.format(
self.features.shape))
def _reset(self, data_path, save):
if not save:
print("extract arch2vec from {}".format(os.path.join(self.dir_name, self.model_path)))
if not os.path.exists(os.path.join(self.dir_name, self.model_path)):
exit()
dataset = load_json(data_path)
self.model = Model(input_dim=5, hidden_dim=128, latent_dim=16, num_hops=5, num_mlp_layers=2, dropout=0, **cfg['GAE']).cuda()
self.model.load_state_dict(torch.load(os.path.join(self.dir_name, self.model_path).format(args.dim))['model_state'])
self.model.eval()
with torch.no_grad():
print("length of the dataset: {}".format(len(dataset)))
self.f_path = os.path.join(self.dir_name, 'arch2vec-{}'.format(self.model_path))
if os.path.exists(self.f_path):
print('{} is already saved'.format(self.f_path))
exit()
print('save to {}'.format(self.f_path))
for ind in range(len(dataset)):
adj = torch.Tensor(dataset[str(ind)]['module_adjacency']).unsqueeze(0).cuda()
ops = torch.Tensor(dataset[str(ind)]['module_operations']).unsqueeze(0).cuda()
adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep'])
test_acc = dataset[str(ind)]['test_accuracy']
valid_acc = dataset[str(ind)]['validation_accuracy']
time = dataset[str(ind)]['training_time']
x,_ = self.model._encoder(ops, adj)
self.embedding[ind] = {'feature': x.squeeze(0).mean(dim=0).cpu(), 'valid_accuracy': float(valid_acc), 'test_accuracy': float(test_acc), 'time': float(time)}
torch.save(self.embedding, self.f_path)
print("finish arch2vec extraction")
exit()
else:
self.f_path = os.path.join(self.dir_name, self.emb_path)
print("load arch2vec from: {}".format(self.f_path))
self.embedding = torch.load(self.f_path)
for ind in range(len(self.embedding)):
self.features.append(self.embedding[ind]['feature'])
self.features = torch.stack(self.features, dim=0)
print('loading finished. pretrained embeddings shape: {}'.format(self.features.shape))
csvfile = open('fcn.csv', 'w')
filewriter = csv.writer(csvfile, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
filewriter.writerow(['Name', 'Acc1','Acc2','Acc3','Acc4','Acc5','Acc6','Acc7','Acc8','Acc9','Acc10','Avg'])
flist.sort()
flist = flist[76:]
flist=['StarLightCurves']
for fname in flist:
scores=[]
x_train , x_test, y_train, y_test = train_test_splitUCR(fdir,fname)
f=open("Eye.txt","a")
a = datetime.datetime.now()
f.write("\n"+str(i)+" "+fname+ " " + str(a) +"\n")
nb_classes = len(np.unique(y_test))
x_train , x_test, y_train, y_test = preprocessing(x_train , x_test, y_train, y_test)
x_train = x_train.reshape(x_train.shape +(1,))
x_test = x_test.reshape(x_test.shape + (1,))
input_shape = x_train.shape[1:]
for i in range(10):
c =Classifier_FCN(input_shape,nb_classes)
print("\n[" +fname+"]Treinando a rede ",i+1,"° vez")
hist = c.fit(x_train,y_train,x_test,y_test)
e = str(len(hist.history['loss']))
print("Parou em "+e+" epocas")
score = c.model.evaluate(x_test,y_test)
scores.append(round(score[1]*100,2))
print("%s: %.2f%%"%(c.model.metrics_names[1],score[1]*100))
avg = round(sum(scores)/len(scores),2)
#------------------------------------------------------------------------------ # Predict frames #------------------------------------------------------------------------------ i = 0 while(cap.isOpened()): # Read frame from camera start_time = time() _, frame = cap.read() image = cv2.transpose(frame[...,::-1]) h, w = image.shape[:2] read_cam_time = time() # Predict mask X, pad_up, pad_left, h_new, w_new = utils.preprocessing(image, expected_size=args.input_sz, pad_value=0) preproc_time = time() with torch.no_grad(): if args.use_cuda: mask = model(X.cuda()) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].cpu().numpy() else: mask = model(X) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].numpy() predict_time = time()
def pretraining_gae(dataset, cfg):
""" implementation of VGAE pretraining on DARTS Search Space """
X_adj, X_ops, indices, X_adj_val, X_ops_val, indices_val = _build_dataset(dataset)
print('train set size: {}, validation set size: {}'.format(indices.shape[0], indices_val.shape[0]))
model = Model(input_dim=args.input_dim, hidden_dim=args.hidden_dim, latent_dim=args.dim,
num_hops=args.hops, num_mlp_layers=args.mlps, dropout=args.dropout, **cfg['GAE']).cuda()
optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08)
epochs = args.epochs
bs = args.bs
loss_total = []
best_graph_acc = 0
for epoch in range(0, epochs):
chunks = X_adj.shape[0] // bs
if X_adj.shape[0] % bs > 0:
chunks += 1
X_adj_split = torch.split(X_adj, bs, dim=0)
X_ops_split = torch.split(X_ops, bs, dim=0)
indices_split = torch.split(indices, bs, dim=0)
loss_epoch = []
Z = []
for i, (adj, ops, ind) in enumerate(zip(X_adj_split, X_ops_split, indices_split)):
optimizer.zero_grad()
adj, ops = adj.cuda(), ops.cuda()
# preprocessing
adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep'])
# forward
ops_recon, adj_recon, mu, logvar = model(ops, adj)
Z.append(mu)
adj_recon, ops_recon = prep_reverse(adj_recon, ops_recon)
adj, ops = prep_reverse(adj, ops)
loss = VAEReconstructed_Loss(**cfg['loss'])((ops_recon, adj_recon), (ops, adj), mu, logvar)
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 5)
optimizer.step()
loss_epoch.append(loss.item())
if i % 500 == 0:
print('epoch {}: batch {} / {}: loss: {:.5f}'.format(epoch, i, chunks, loss.item()))
Z = torch.cat(Z, dim=0)
z_mean, z_std = Z.mean(0), Z.std(0)
validity_counter = 0
buckets = {}
model.eval()
for _ in range(args.latent_points):
z = torch.randn(11, args.dim).cuda()
z = z * z_std + z_mean
op, ad = model.decoder(z.unsqueeze(0))
op = op.squeeze(0).cpu()
ad = ad.squeeze(0).cpu()
max_idx = torch.argmax(op, dim=-1)
one_hot = torch.zeros_like(op)
for i in range(one_hot.shape[0]):
one_hot[i][max_idx[i]] = 1
op_decode = to_ops_darts(max_idx)
ad_decode = (ad>0.5).int().triu(1).numpy()
ad_decode = np.ndarray.tolist(ad_decode)
if is_valid_darts(ad_decode, op_decode):
validity_counter += 1
fingerprint = graph_util.hash_module(np.array(ad_decode), one_hot.numpy().tolist())
if fingerprint not in buckets:
buckets[fingerprint] = (ad_decode, one_hot.numpy().astype('int8').tolist())
validity = validity_counter / args.latent_points
print('Ratio of valid decodings from the prior: {:.4f}'.format(validity))
print('Ratio of unique decodings from the prior: {:.4f}'.format(len(buckets) / (validity_counter+1e-8)))
acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val = get_val_acc_vae(model,cfg,X_adj_val, X_ops_val,indices_val)
print('validation set: acc_ops:{0:.2f}, mean_corr_adj:{1:.2f}, mean_fal_pos_adj:{2:.2f}, acc_adj:{3:.2f}'.format(
acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val))
#print("reconstructed adj matrix:", adj_recon[1])
#print("original adj matrix:", adj[1])
#print("reconstructed ops matrix:", ops_recon[1])
#print("original ops matrix:", ops[1])
print('epoch {}: average loss {:.5f}'.format(epoch, sum(loss_epoch)/len(loss_epoch)))
loss_total.append(sum(loss_epoch) / len(loss_epoch))
print('loss for epochs: \n', loss_total)
save_checkpoint_vae(model, optimizer, epoch, sum(loss_epoch) / len(loss_epoch), args.dim, args.name, args.dropout, args.seed)
print('loss for epochs: ', loss_total)
def pretraining_model(dataset, cfg, args):
nasbench = api.NASBench('data/nasbench_only108.tfrecord')
train_ind_list, val_ind_list = range(int(len(dataset)*0.9)), range(int(len(dataset)*0.9), len(dataset))
X_adj_train, X_ops_train, indices_train = _build_dataset(dataset, train_ind_list)
X_adj_val, X_ops_val, indices_val = _build_dataset(dataset, val_ind_list)
model = Model(input_dim=args.input_dim, hidden_dim=args.hidden_dim, latent_dim=args.dim,
num_hops=args.hops, num_mlp_layers=args.mlps, dropout=args.dropout, **cfg['GAE']).cuda()
optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08)
epochs = args.epochs
bs = args.bs
loss_total = []
for epoch in range(0, epochs):
chunks = len(train_ind_list) // bs
if len(train_ind_list) % bs > 0:
chunks += 1
X_adj_split = torch.split(X_adj_train, bs, dim=0)
X_ops_split = torch.split(X_ops_train, bs, dim=0)
indices_split = torch.split(indices_train, bs, dim=0)
loss_epoch = []
Z = []
for i, (adj, ops, ind) in enumerate(zip(X_adj_split, X_ops_split, indices_split)):
optimizer.zero_grad()
adj, ops = adj.cuda(), ops.cuda()
# preprocessing
adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep'])
# forward
ops_recon, adj_recon, mu, logvar = model(ops, adj.to(torch.long))
Z.append(mu)
adj_recon, ops_recon = prep_reverse(adj_recon, ops_recon)
adj, ops = prep_reverse(adj, ops)
loss = VAEReconstructed_Loss(**cfg['loss'])((ops_recon, adj_recon), (ops, adj), mu, logvar)
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 5)
optimizer.step()
loss_epoch.append(loss.item())
if i%1000==0:
print('epoch {}: batch {} / {}: loss: {:.5f}'.format(epoch, i, chunks, loss.item()))
Z = torch.cat(Z, dim=0)
z_mean, z_std = Z.mean(0), Z.std(0)
validity_counter = 0
buckets = {}
model.eval()
for _ in range(args.latent_points):
z = torch.randn(7, args.dim).cuda()
z = z * z_std + z_mean
op, ad = model.decoder(z.unsqueeze(0))
op = op.squeeze(0).cpu()
ad = ad.squeeze(0).cpu()
max_idx = torch.argmax(op, dim=-1)
one_hot = torch.zeros_like(op)
for i in range(one_hot.shape[0]):
one_hot[i][max_idx[i]] = 1
op_decode = transform_operations(max_idx)
ad_decode = (ad>0.5).int().triu(1).numpy()
ad_decode = np.ndarray.tolist(ad_decode)
spec = api.ModelSpec(matrix=ad_decode, ops=op_decode)
if nasbench.is_valid(spec):
validity_counter += 1
fingerprint = graph_util.hash_module(np.array(ad_decode), one_hot.numpy().tolist())
if fingerprint not in buckets:
buckets[fingerprint] = (ad_decode, one_hot.numpy().astype('int8').tolist())
validity = validity_counter / args.latent_points
print('Ratio of valid decodings from the prior: {:.4f}'.format(validity))
print('Ratio of unique decodings from the prior: {:.4f}'.format(len(buckets) / (validity_counter+1e-8)))
acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val = get_val_acc_vae(model, cfg, X_adj_val, X_ops_val, indices_val)
print('validation set: acc_ops:{0:.4f}, mean_corr_adj:{1:.4f}, mean_fal_pos_adj:{2:.4f}, acc_adj:{3:.4f}'.format(
acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val))
print('epoch {}: average loss {:.5f}'.format(epoch, sum(loss_epoch)/len(loss_epoch)))
loss_total.append(sum(loss_epoch) / len(loss_epoch))
save_checkpoint_vae(model, optimizer, epoch, sum(loss_epoch) / len(loss_epoch), args.dim, args.name, args.dropout, args.seed)
print('loss for epochs: \n', loss_total)
def video_infer(args):
cap = cv2.VideoCapture(args.video)
_, frame = cap.read()
H, W = frame.shape[:2]
fps = cap.get(cv2.CAP_PROP_FPS)
out = cv2.VideoWriter(args.output,
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), fps,
(W, H))
# Background
if args.bg is not None:
BACKGROUND = cv2.imread(args.bg)[..., ::-1]
BACKGROUND = cv2.resize(BACKGROUND, (W, H),
interpolation=cv2.INTER_LINEAR)
KERNEL_SZ = 25
SIGMA = 0
# Alpha transperency
else:
COLOR1 = [255, 0, 0]
COLOR2 = [0, 0, 255]
if args.model == 'unet':
model = UNet(backbone=args.net,
num_classes=2,
pretrained_backbone=None)
elif args.model == 'deeplabv3_plus':
model = DeepLabV3Plus(backbone=args.net,
num_classes=2,
pretrained_backbone=None)
if args.use_cuda:
model = model.cuda()
trained_dict = torch.load(args.checkpoint,
map_location="cpu")['state_dict']
model.load_state_dict(trained_dict, strict=False)
model.eval()
if W > H:
w_new = int(args.input_sz)
h_new = int(H * w_new / W)
else:
h_new = int(args.input_sz)
w_new = int(W * h_new / H)
disflow = cv2.DISOpticalFlow_create(cv2.DISOPTICAL_FLOW_PRESET_ULTRAFAST)
prev_gray = np.zeros((h_new, w_new), np.uint8)
prev_cfd = np.zeros((h_new, w_new), np.float32)
is_init = True
while (cap.isOpened()):
start_time = time()
ret, frame = cap.read()
if ret:
image = frame[..., ::-1]
h, w = image.shape[:2]
read_cam_time = time()
# Predict mask
X, pad_up, pad_left, h_new, w_new = utils.preprocessing(
image, expected_size=args.input_sz, pad_value=0)
preproc_time = time()
with torch.no_grad():
if args.use_cuda:
mask = model(X.cuda())
mask = mask[..., pad_up:pad_up + h_new,
pad_left:pad_left + w_new]
#mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0, 1, ...].cpu().numpy() #(213, 320)
else:
mask = model(X)
mask = mask[..., pad_up:pad_up + h_new,
pad_left:pad_left + w_new]
#mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0, 1, ...].numpy()
predict_time = time()
# optical tracking
cur_gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
cur_gray = cv2.resize(cur_gray, (w_new, h_new))
scoremap = 255 * mask
optflow_map = postprocess(cur_gray, scoremap, prev_gray, prev_cfd,
disflow, is_init)
optical_flow_track_time = time()
prev_gray = cur_gray.copy()
prev_cfd = optflow_map.copy()
is_init = False
optflow_map = cv2.GaussianBlur(optflow_map, (3, 3), 0)
optflow_map = threshold_mask(optflow_map,
thresh_bg=0.2,
thresh_fg=0.8)
img_matting = np.repeat(optflow_map[:, :, np.newaxis], 3, axis=2)
bg_im = np.ones_like(img_matting) * 255
re_image = cv2.resize(image, (w_new, h_new))
comb = (img_matting * re_image + (1 - img_matting) * bg_im).astype(
np.uint8)
comb = cv2.resize(comb, (W, H))
comb = comb[..., ::-1]
# Print runtime
read = read_cam_time - start_time
preproc = preproc_time - read_cam_time
pred = predict_time - preproc_time
optical = optical_flow_track_time - predict_time
total = read + preproc + pred + optical
print(
"read: %.3f [s]; preproc: %.3f [s]; pred: %.3f [s]; optical: %.3f [s]; total: %.3f [s]; fps: %.2f [Hz]"
% (read, preproc, pred, optical, total, 1 / pred))
out.write(comb)
if args.watch:
cv2.imshow('webcam', comb[..., ::-1])
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
break
cap.release()
out.release()
def video_infer(args):
cap = cv2.VideoCapture(args.video)
_, frame = cap.read()
H, W = frame.shape[:2]
fourcc = cv2.VideoWriter_fourcc(*'DIVX')
out = cv2.VideoWriter(args.output, fourcc, 30, (W,H))
font = cv2.FONT_HERSHEY_SIMPLEX
# Background
if args.bg is not None:
BACKGROUND = cv2.imread(args.bg)[...,::-1]
BACKGROUND = cv2.resize(BACKGROUND, (W,H), interpolation=cv2.INTER_LINEAR)
KERNEL_SZ = 25
SIGMA = 0
# Alpha transperency
else:
COLOR1 = [90, 140, 154]
COLOR2 = [0, 0, 0]
if args.model=='unet':
model = UNet(backbone=args.net, num_classes=2, pretrained_backbone=None)
elif args.model=='deeplabv3_plus':
model = DeepLabV3Plus(backbone=args.net, num_classes=2, pretrained_backbone=None)
elif args.model=='hrnet':
model = HighResolutionNet(num_classes=2, pretrained_backbone=None)
if args.use_cuda:
model = model.cuda()
trained_dict = torch.load(args.checkpoint, map_location="cpu")['state_dict']
model.load_state_dict(trained_dict, strict=False)
model.eval()
while(cap.isOpened()):
start_time = time()
ret, frame = cap.read()
if ret:
image = frame[...,::-1]
h, w = image.shape[:2]
read_cam_time = time()
# Predict mask
X, pad_up, pad_left, h_new, w_new = utils.preprocessing(image, expected_size=args.input_sz, pad_value=0)
preproc_time = time()
with torch.no_grad():
if args.use_cuda:
mask = model(X.cuda())
if mask.shape[1] != h_new:
mask = F.interpolate(mask, size=(args.input_sz, args.input_sz), mode='bilinear', align_corners=True)
mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new]
mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0,1,...].cpu().numpy()
else:
mask = model(X)
mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new]
mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True)
mask = F.softmax(mask, dim=1)
mask = mask[0,1,...].numpy()
predict_time = time()
# Draw result
if args.bg is None:
image_alpha = utils.draw_matting(image, mask)
#image_alpha = utils.draw_transperency(image, mask, COLOR1, COLOR2)
else:
image_alpha = utils.draw_fore_to_back(image, mask, BACKGROUND, kernel_sz=KERNEL_SZ, sigma=SIGMA)
draw_time = time()
# Print runtime
read = read_cam_time-start_time
preproc = preproc_time-read_cam_time
pred = predict_time-preproc_time
draw = draw_time-predict_time
total = read + preproc + pred + draw
fps = 1 / pred
print("read: %.3f [s]; preproc: %.3f [s]; pred: %.3f [s]; draw: %.3f [s]; total: %.3f [s]; fps: %.2f [Hz]" %
(read, preproc, pred, draw, total, fps))
# Wait for interupt
cv2.putText(image_alpha, "%.2f [fps]" % (fps), (10, 50), font, 1.5, (0, 255, 0), 2, cv2.LINE_AA)
out.write(image_alpha[..., ::-1])
if args.watch:
cv2.imshow('webcam', image_alpha[..., ::-1])
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
break
def _reset(self, data_path, save):
if not save:
print("extract arch2vec embedding table...")
dataset = load_json(data_path)
self.model = Model(input_dim=args.input_dim,
hidden_dim=args.hidden_dim,
latent_dim=args.latent_dim,
num_hops=args.hops,
num_mlp_layers=args.mlps,
dropout=args.dropout,
**cfg['GAE']).cuda()
model_ckpt_path = os.path.join(self.dir_name,
'{}'.format(args.model_path))
if not os.path.exists(model_ckpt_path):
print("File {} does not exist.".format(model_ckpt_path))
exit()
self.model.load_state_dict(
torch.load(model_ckpt_path)['model_state'])
self.model.eval()
print("length of the dataset: {}".format(len(dataset)))
self.f_path = os.path.join(
self.dir_name, '{}-arch2vec.pt'.format(args.dataset_name))
if os.path.exists(self.f_path):
print('ATTENTION!!! {} is already saved.'.format(self.f_path))
exit()
print('save to {} ...'.format(self.f_path))
for ind in range(len(dataset)):
adj = torch.Tensor(
dataset[str(ind)]['module_adjacency']).unsqueeze(0).cuda()
ops = torch.Tensor(dataset[str(ind)]
['module_operations']).unsqueeze(0).cuda()
adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep'])
test_acc = dataset[str(ind)]['test_accuracy']
valid_acc = dataset[str(ind)]['validation_accuracy']
other_info = {
'valid_accuracy_avg':
dataset[str(ind)]['validation_accuracy_avg'],
'test_accuracy_avg':
dataset[str(ind)]['test_accuracy_avg']
}
time = dataset[str(ind)]['training_time']
x, _ = self.model._encoder(ops, adj)
self.embedding[ind] = {
'feature': x.mean(dim=1).squeeze(0).cpu(),
'valid_accuracy': float(valid_acc),
'test_accuracy': float(test_acc),
'time': float(time),
'other_info': other_info
}
torch.save(self.embedding, self.f_path)
print("finished arch2vec extraction")
exit()
else:
self.f_path = os.path.join(
self.dir_name, '{}-arch2vec.pt'.format(args.dataset_name))
print("load pretrained arch2vec in path: {}".format(self.f_path))
self.embedding = torch.load(self.f_path)
random.seed(args.seed)
random.shuffle(self.embedding)
for ind in range(len(self.embedding)):
self.features.append(self.embedding[ind]['feature'])
self.features = torch.stack(self.features, dim=0)
print('loading finished. pretrained embeddings shape: {}'.format(
self.features.shape))