def predict(imagePath): #predict image frame = cv2.imread(imagePath) image = frame[...,::-1] h, w = image.shape[:2] inferTime = None X, pad_up, pad_left, h_new, w_new = utils.preprocessing(image, expected_size=args.input_sz, pad_value=0) with torch.no_grad(): if args.use_cuda: start = time.time() mask = model(X.cuda()) end = time.time() inferTime = end-start mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].cpu().numpy() else: start = time.time() mask = model(X) end = time.time() inferTime = end-start mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].numpy() #mask outputSection booleanMask = mask >= 0.5 return booleanMask, frame, inferTime
def predict(model,input,bg): image = cv2.imread(input) X, pad_up, pad_left, h_new, w_new = utils.preprocessing(image, expected_size=input_sz, pad_value=0) h,w = image.shape[:2] if bg is not None: BACKGROUND = cv2.imread(bg)[...,::-1] BACKGROUND = cv2.resize(BACKGROUND, (w,h), interpolation=cv2.INTER_LINEAR) KERNEL_SZ = 25 SIGMA = 0 else: COLOR1 = [255, 0, 0] COLOR2 = [0, 0, 255] with torch.no_grad(): mask = model(X) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].numpy() if bg is None: image_alpha = utils.draw_matting(image, mask) # image_alpha = utils.draw_transperency(image, mask, COLOR1, COLOR2) else: image_alpha = utils.draw_fore_to_back(image, mask, BACKGROUND, kernel_sz=KERNEL_SZ, sigma=SIGMA) print(image_alpha.shape) return image_alpha
def segment_humans(model, inputs): frame = np.array(inputs["input_image"]) #image = frame[...,::-1] h, w = frame.shape[0], frame.shape[1] # Predict mask X, pad_up, pad_left, h_new, w_new = utils.preprocessing(frame, expected_size=320, pad_value=0) with torch.no_grad(): if torch.cuda.is_available(): mask = model(X.cuda()) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].cpu().numpy() else: mask = model(X) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].numpy() mask = 255*mask mask = np.expand_dims(mask, axis=2) image_alpha = np.concatenate((frame, mask), axis=2) return image_alpha.astype(np.uint8)
def config2embedding(config, b): data, _ = config2data_A(config, b) adj, op = data2mat(data) adj_, op_ = adj, op adj, op = torch.Tensor(adj).unsqueeze(0).cuda(), torch.Tensor(op).unsqueeze(0).cuda() adj, op, prep_reverse = preprocessing(adj, op, **cfg['prep']) embedding = model._encoder(op, adj)[0] return embedding, adj_, op_
def segment_background(deeplab_weights_path=None, input_images_path=None): # Set testing device device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # Set up model # model = UNet( # backbone="mobilenetv2", # num_classes=2, # pretrained_backbone=None # ) # trained_dict = torch.load('UNet_MobileNetV2.pth', map_location="cpu")['state_dict'] # model.load_state_dict(trained_dict, strict=False) # model = model.to(device) # model.eval() model = DeepLabV3Plus(backbone="resnet18") trained_dict = torch.load(deeplab_weights_path, map_location="cpu")["state_dict"] model.load_state_dict(trained_dict, strict=False) model = model.to(device) model.eval() # Output folder to save extracted backgrounds output_folder = Path(input_images_path).parent.joinpath( "extracted_backgrounds") Path(output_folder).mkdir(parents=True, exist_ok=True) # Extract backgrounds for img_path in Path(input_images_path).iterdir(): img = cv2.imread(str(img_path)) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) print("Processing:", str(img_path)) h, w = img.shape[:2] X, pad_up, pad_left, h_new, w_new = utils.preprocessing( img, expected_size=320, pad_value=0) with torch.no_grad(): mask = model(X.cuda()) mask = mask[..., pad_up:pad_up + h_new, pad_left:pad_left + w_new] mask = F.interpolate(mask, size=(h, w), mode="bilinear", align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0, 1, ...].cpu().numpy() mask = np.where(mask > 0.9, 0, 1) # Apply mask on original RGB image to extract background img[:, :, 0] = mask * img[:, :, 0] img[:, :, 1] = mask * img[:, :, 1] img[:, :, 2] = mask * img[:, :, 2] output_name = output_folder.joinpath(img_path.stem + img_path.suffix) cv2.imwrite(str(output_name), cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
def __init__(self, model, ops, adj, cfg): print('Initializing NN decoder') t_s = time.time() self.model = model self.ops = ops self.adj = adj self.cfg = cfg with torch.no_grad(): adj_prep, ops_prep, _ = preprocessing(self.adj, self.ops, **self.cfg['prep']) self.embedding = self.model.encoder(ops_prep, adj_prep) assert len(self.embedding.shape) == 3 print('Using {} seconds to initialize NN decoder'.format(time.time() - t_s))
def find_NN(self, ops, adj, ind, k=10): assert len(ops.shape) == 3 ind_t1_list = [] ind_tk_list = [] with torch.no_grad(): adj_prep, ops_prep, _ = preprocessing(adj, ops, **self.cfg['prep']) embeddings = self.model.encoder(ops_prep, adj_prep) for e in embeddings: dist = torch.sum((self.embedding - e)**2, dim=[1, 2]) _, ind_t1 = torch.topk(dist, 1, largest=False) _, ind_tk = torch.topk(dist, k, largest=False) ind_t1_list.append(ind_t1.item()) ind_tk_list.append(ind_tk.tolist()) op_recon, adj_recon = self.ops[ind_t1_list], self.adj[ind_t1_list] op_recon_tk, adj_recon_tk = self.ops[ind_t1_list], self.adj[ ind_t1_list] return op_recon, adj_recon, op_recon_tk, adj_recon_tk, ind_t1_list, ind_tk_list
def _reset(self, data_path, save): if not save: print("extract arch2vec on DARTS search space ...") dataset = load_json(data_path) print("length of the dataset: {}".format(len(dataset))) self.f_path = os.path.join(self.dir_name, 'arch2vec-darts.pt') if os.path.exists(self.f_path): print('{} is already saved'.format(self.f_path)) exit() print('save to {}'.format(self.f_path)) counter = 0 self.model.eval() for k, v in dataset.items(): adj = torch.Tensor(v[0]).unsqueeze(0).cuda() ops = torch.Tensor(one_hot_darts(v[1])).unsqueeze(0).cuda() adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep']) with torch.no_grad(): x, _ = self.model._encoder(ops, adj) self.embedding[counter] = { 'feature': x.squeeze(0).mean(dim=0).cpu(), 'genotype': process(v[2]) } print("{}/{}".format(counter, len(dataset))) counter += 1 torch.save(self.embedding, self.f_path) print("finished arch2vec extraction") exit() else: self.f_path = os.path.join(self.dir_name, 'arch2vec-darts.pt') print("load arch2vec from: {}".format(self.f_path)) self.embedding = torch.load(self.f_path) for ind in range(len(self.embedding)): self.features.append(self.embedding[ind]['feature']) self.genotype.append(self.embedding[ind]['genotype']) self.features = torch.stack(self.features, dim=0) print('loading finished. pretrained embeddings shape: {}'.format( self.features.shape))
def _reset(self, data_path, save): if not save: print("extract arch2vec from {}".format(os.path.join(self.dir_name, self.model_path))) if not os.path.exists(os.path.join(self.dir_name, self.model_path)): exit() dataset = load_json(data_path) self.model = Model(input_dim=5, hidden_dim=128, latent_dim=16, num_hops=5, num_mlp_layers=2, dropout=0, **cfg['GAE']).cuda() self.model.load_state_dict(torch.load(os.path.join(self.dir_name, self.model_path).format(args.dim))['model_state']) self.model.eval() with torch.no_grad(): print("length of the dataset: {}".format(len(dataset))) self.f_path = os.path.join(self.dir_name, 'arch2vec-{}'.format(self.model_path)) if os.path.exists(self.f_path): print('{} is already saved'.format(self.f_path)) exit() print('save to {}'.format(self.f_path)) for ind in range(len(dataset)): adj = torch.Tensor(dataset[str(ind)]['module_adjacency']).unsqueeze(0).cuda() ops = torch.Tensor(dataset[str(ind)]['module_operations']).unsqueeze(0).cuda() adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep']) test_acc = dataset[str(ind)]['test_accuracy'] valid_acc = dataset[str(ind)]['validation_accuracy'] time = dataset[str(ind)]['training_time'] x,_ = self.model._encoder(ops, adj) self.embedding[ind] = {'feature': x.squeeze(0).mean(dim=0).cpu(), 'valid_accuracy': float(valid_acc), 'test_accuracy': float(test_acc), 'time': float(time)} torch.save(self.embedding, self.f_path) print("finish arch2vec extraction") exit() else: self.f_path = os.path.join(self.dir_name, self.emb_path) print("load arch2vec from: {}".format(self.f_path)) self.embedding = torch.load(self.f_path) for ind in range(len(self.embedding)): self.features.append(self.embedding[ind]['feature']) self.features = torch.stack(self.features, dim=0) print('loading finished. pretrained embeddings shape: {}'.format(self.features.shape))
csvfile = open('fcn.csv', 'w') filewriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) filewriter.writerow(['Name', 'Acc1','Acc2','Acc3','Acc4','Acc5','Acc6','Acc7','Acc8','Acc9','Acc10','Avg']) flist.sort() flist = flist[76:] flist=['StarLightCurves'] for fname in flist: scores=[] x_train , x_test, y_train, y_test = train_test_splitUCR(fdir,fname) f=open("Eye.txt","a") a = datetime.datetime.now() f.write("\n"+str(i)+" "+fname+ " " + str(a) +"\n") nb_classes = len(np.unique(y_test)) x_train , x_test, y_train, y_test = preprocessing(x_train , x_test, y_train, y_test) x_train = x_train.reshape(x_train.shape +(1,)) x_test = x_test.reshape(x_test.shape + (1,)) input_shape = x_train.shape[1:] for i in range(10): c =Classifier_FCN(input_shape,nb_classes) print("\n[" +fname+"]Treinando a rede ",i+1,"° vez") hist = c.fit(x_train,y_train,x_test,y_test) e = str(len(hist.history['loss'])) print("Parou em "+e+" epocas") score = c.model.evaluate(x_test,y_test) scores.append(round(score[1]*100,2)) print("%s: %.2f%%"%(c.model.metrics_names[1],score[1]*100)) avg = round(sum(scores)/len(scores),2)
#------------------------------------------------------------------------------ # Predict frames #------------------------------------------------------------------------------ i = 0 while(cap.isOpened()): # Read frame from camera start_time = time() _, frame = cap.read() image = cv2.transpose(frame[...,::-1]) h, w = image.shape[:2] read_cam_time = time() # Predict mask X, pad_up, pad_left, h_new, w_new = utils.preprocessing(image, expected_size=args.input_sz, pad_value=0) preproc_time = time() with torch.no_grad(): if args.use_cuda: mask = model(X.cuda()) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].cpu().numpy() else: mask = model(X) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].numpy() predict_time = time()
def pretraining_gae(dataset, cfg): """ implementation of VGAE pretraining on DARTS Search Space """ X_adj, X_ops, indices, X_adj_val, X_ops_val, indices_val = _build_dataset(dataset) print('train set size: {}, validation set size: {}'.format(indices.shape[0], indices_val.shape[0])) model = Model(input_dim=args.input_dim, hidden_dim=args.hidden_dim, latent_dim=args.dim, num_hops=args.hops, num_mlp_layers=args.mlps, dropout=args.dropout, **cfg['GAE']).cuda() optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08) epochs = args.epochs bs = args.bs loss_total = [] best_graph_acc = 0 for epoch in range(0, epochs): chunks = X_adj.shape[0] // bs if X_adj.shape[0] % bs > 0: chunks += 1 X_adj_split = torch.split(X_adj, bs, dim=0) X_ops_split = torch.split(X_ops, bs, dim=0) indices_split = torch.split(indices, bs, dim=0) loss_epoch = [] Z = [] for i, (adj, ops, ind) in enumerate(zip(X_adj_split, X_ops_split, indices_split)): optimizer.zero_grad() adj, ops = adj.cuda(), ops.cuda() # preprocessing adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep']) # forward ops_recon, adj_recon, mu, logvar = model(ops, adj) Z.append(mu) adj_recon, ops_recon = prep_reverse(adj_recon, ops_recon) adj, ops = prep_reverse(adj, ops) loss = VAEReconstructed_Loss(**cfg['loss'])((ops_recon, adj_recon), (ops, adj), mu, logvar) loss.backward() nn.utils.clip_grad_norm_(model.parameters(), 5) optimizer.step() loss_epoch.append(loss.item()) if i % 500 == 0: print('epoch {}: batch {} / {}: loss: {:.5f}'.format(epoch, i, chunks, loss.item())) Z = torch.cat(Z, dim=0) z_mean, z_std = Z.mean(0), Z.std(0) validity_counter = 0 buckets = {} model.eval() for _ in range(args.latent_points): z = torch.randn(11, args.dim).cuda() z = z * z_std + z_mean op, ad = model.decoder(z.unsqueeze(0)) op = op.squeeze(0).cpu() ad = ad.squeeze(0).cpu() max_idx = torch.argmax(op, dim=-1) one_hot = torch.zeros_like(op) for i in range(one_hot.shape[0]): one_hot[i][max_idx[i]] = 1 op_decode = to_ops_darts(max_idx) ad_decode = (ad>0.5).int().triu(1).numpy() ad_decode = np.ndarray.tolist(ad_decode) if is_valid_darts(ad_decode, op_decode): validity_counter += 1 fingerprint = graph_util.hash_module(np.array(ad_decode), one_hot.numpy().tolist()) if fingerprint not in buckets: buckets[fingerprint] = (ad_decode, one_hot.numpy().astype('int8').tolist()) validity = validity_counter / args.latent_points print('Ratio of valid decodings from the prior: {:.4f}'.format(validity)) print('Ratio of unique decodings from the prior: {:.4f}'.format(len(buckets) / (validity_counter+1e-8))) acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val = get_val_acc_vae(model,cfg,X_adj_val, X_ops_val,indices_val) print('validation set: acc_ops:{0:.2f}, mean_corr_adj:{1:.2f}, mean_fal_pos_adj:{2:.2f}, acc_adj:{3:.2f}'.format( acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val)) #print("reconstructed adj matrix:", adj_recon[1]) #print("original adj matrix:", adj[1]) #print("reconstructed ops matrix:", ops_recon[1]) #print("original ops matrix:", ops[1]) print('epoch {}: average loss {:.5f}'.format(epoch, sum(loss_epoch)/len(loss_epoch))) loss_total.append(sum(loss_epoch) / len(loss_epoch)) print('loss for epochs: \n', loss_total) save_checkpoint_vae(model, optimizer, epoch, sum(loss_epoch) / len(loss_epoch), args.dim, args.name, args.dropout, args.seed) print('loss for epochs: ', loss_total)
def pretraining_model(dataset, cfg, args): nasbench = api.NASBench('data/nasbench_only108.tfrecord') train_ind_list, val_ind_list = range(int(len(dataset)*0.9)), range(int(len(dataset)*0.9), len(dataset)) X_adj_train, X_ops_train, indices_train = _build_dataset(dataset, train_ind_list) X_adj_val, X_ops_val, indices_val = _build_dataset(dataset, val_ind_list) model = Model(input_dim=args.input_dim, hidden_dim=args.hidden_dim, latent_dim=args.dim, num_hops=args.hops, num_mlp_layers=args.mlps, dropout=args.dropout, **cfg['GAE']).cuda() optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08) epochs = args.epochs bs = args.bs loss_total = [] for epoch in range(0, epochs): chunks = len(train_ind_list) // bs if len(train_ind_list) % bs > 0: chunks += 1 X_adj_split = torch.split(X_adj_train, bs, dim=0) X_ops_split = torch.split(X_ops_train, bs, dim=0) indices_split = torch.split(indices_train, bs, dim=0) loss_epoch = [] Z = [] for i, (adj, ops, ind) in enumerate(zip(X_adj_split, X_ops_split, indices_split)): optimizer.zero_grad() adj, ops = adj.cuda(), ops.cuda() # preprocessing adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep']) # forward ops_recon, adj_recon, mu, logvar = model(ops, adj.to(torch.long)) Z.append(mu) adj_recon, ops_recon = prep_reverse(adj_recon, ops_recon) adj, ops = prep_reverse(adj, ops) loss = VAEReconstructed_Loss(**cfg['loss'])((ops_recon, adj_recon), (ops, adj), mu, logvar) loss.backward() nn.utils.clip_grad_norm_(model.parameters(), 5) optimizer.step() loss_epoch.append(loss.item()) if i%1000==0: print('epoch {}: batch {} / {}: loss: {:.5f}'.format(epoch, i, chunks, loss.item())) Z = torch.cat(Z, dim=0) z_mean, z_std = Z.mean(0), Z.std(0) validity_counter = 0 buckets = {} model.eval() for _ in range(args.latent_points): z = torch.randn(7, args.dim).cuda() z = z * z_std + z_mean op, ad = model.decoder(z.unsqueeze(0)) op = op.squeeze(0).cpu() ad = ad.squeeze(0).cpu() max_idx = torch.argmax(op, dim=-1) one_hot = torch.zeros_like(op) for i in range(one_hot.shape[0]): one_hot[i][max_idx[i]] = 1 op_decode = transform_operations(max_idx) ad_decode = (ad>0.5).int().triu(1).numpy() ad_decode = np.ndarray.tolist(ad_decode) spec = api.ModelSpec(matrix=ad_decode, ops=op_decode) if nasbench.is_valid(spec): validity_counter += 1 fingerprint = graph_util.hash_module(np.array(ad_decode), one_hot.numpy().tolist()) if fingerprint not in buckets: buckets[fingerprint] = (ad_decode, one_hot.numpy().astype('int8').tolist()) validity = validity_counter / args.latent_points print('Ratio of valid decodings from the prior: {:.4f}'.format(validity)) print('Ratio of unique decodings from the prior: {:.4f}'.format(len(buckets) / (validity_counter+1e-8))) acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val = get_val_acc_vae(model, cfg, X_adj_val, X_ops_val, indices_val) print('validation set: acc_ops:{0:.4f}, mean_corr_adj:{1:.4f}, mean_fal_pos_adj:{2:.4f}, acc_adj:{3:.4f}'.format( acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val)) print('epoch {}: average loss {:.5f}'.format(epoch, sum(loss_epoch)/len(loss_epoch))) loss_total.append(sum(loss_epoch) / len(loss_epoch)) save_checkpoint_vae(model, optimizer, epoch, sum(loss_epoch) / len(loss_epoch), args.dim, args.name, args.dropout, args.seed) print('loss for epochs: \n', loss_total)
def video_infer(args): cap = cv2.VideoCapture(args.video) _, frame = cap.read() H, W = frame.shape[:2] fps = cap.get(cv2.CAP_PROP_FPS) out = cv2.VideoWriter(args.output, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), fps, (W, H)) # Background if args.bg is not None: BACKGROUND = cv2.imread(args.bg)[..., ::-1] BACKGROUND = cv2.resize(BACKGROUND, (W, H), interpolation=cv2.INTER_LINEAR) KERNEL_SZ = 25 SIGMA = 0 # Alpha transperency else: COLOR1 = [255, 0, 0] COLOR2 = [0, 0, 255] if args.model == 'unet': model = UNet(backbone=args.net, num_classes=2, pretrained_backbone=None) elif args.model == 'deeplabv3_plus': model = DeepLabV3Plus(backbone=args.net, num_classes=2, pretrained_backbone=None) if args.use_cuda: model = model.cuda() trained_dict = torch.load(args.checkpoint, map_location="cpu")['state_dict'] model.load_state_dict(trained_dict, strict=False) model.eval() if W > H: w_new = int(args.input_sz) h_new = int(H * w_new / W) else: h_new = int(args.input_sz) w_new = int(W * h_new / H) disflow = cv2.DISOpticalFlow_create(cv2.DISOPTICAL_FLOW_PRESET_ULTRAFAST) prev_gray = np.zeros((h_new, w_new), np.uint8) prev_cfd = np.zeros((h_new, w_new), np.float32) is_init = True while (cap.isOpened()): start_time = time() ret, frame = cap.read() if ret: image = frame[..., ::-1] h, w = image.shape[:2] read_cam_time = time() # Predict mask X, pad_up, pad_left, h_new, w_new = utils.preprocessing( image, expected_size=args.input_sz, pad_value=0) preproc_time = time() with torch.no_grad(): if args.use_cuda: mask = model(X.cuda()) mask = mask[..., pad_up:pad_up + h_new, pad_left:pad_left + w_new] #mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0, 1, ...].cpu().numpy() #(213, 320) else: mask = model(X) mask = mask[..., pad_up:pad_up + h_new, pad_left:pad_left + w_new] #mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0, 1, ...].numpy() predict_time = time() # optical tracking cur_gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) cur_gray = cv2.resize(cur_gray, (w_new, h_new)) scoremap = 255 * mask optflow_map = postprocess(cur_gray, scoremap, prev_gray, prev_cfd, disflow, is_init) optical_flow_track_time = time() prev_gray = cur_gray.copy() prev_cfd = optflow_map.copy() is_init = False optflow_map = cv2.GaussianBlur(optflow_map, (3, 3), 0) optflow_map = threshold_mask(optflow_map, thresh_bg=0.2, thresh_fg=0.8) img_matting = np.repeat(optflow_map[:, :, np.newaxis], 3, axis=2) bg_im = np.ones_like(img_matting) * 255 re_image = cv2.resize(image, (w_new, h_new)) comb = (img_matting * re_image + (1 - img_matting) * bg_im).astype( np.uint8) comb = cv2.resize(comb, (W, H)) comb = comb[..., ::-1] # Print runtime read = read_cam_time - start_time preproc = preproc_time - read_cam_time pred = predict_time - preproc_time optical = optical_flow_track_time - predict_time total = read + preproc + pred + optical print( "read: %.3f [s]; preproc: %.3f [s]; pred: %.3f [s]; optical: %.3f [s]; total: %.3f [s]; fps: %.2f [Hz]" % (read, preproc, pred, optical, total, 1 / pred)) out.write(comb) if args.watch: cv2.imshow('webcam', comb[..., ::-1]) if cv2.waitKey(1) & 0xFF == ord('q'): break else: break cap.release() out.release()
def video_infer(args): cap = cv2.VideoCapture(args.video) _, frame = cap.read() H, W = frame.shape[:2] fourcc = cv2.VideoWriter_fourcc(*'DIVX') out = cv2.VideoWriter(args.output, fourcc, 30, (W,H)) font = cv2.FONT_HERSHEY_SIMPLEX # Background if args.bg is not None: BACKGROUND = cv2.imread(args.bg)[...,::-1] BACKGROUND = cv2.resize(BACKGROUND, (W,H), interpolation=cv2.INTER_LINEAR) KERNEL_SZ = 25 SIGMA = 0 # Alpha transperency else: COLOR1 = [90, 140, 154] COLOR2 = [0, 0, 0] if args.model=='unet': model = UNet(backbone=args.net, num_classes=2, pretrained_backbone=None) elif args.model=='deeplabv3_plus': model = DeepLabV3Plus(backbone=args.net, num_classes=2, pretrained_backbone=None) elif args.model=='hrnet': model = HighResolutionNet(num_classes=2, pretrained_backbone=None) if args.use_cuda: model = model.cuda() trained_dict = torch.load(args.checkpoint, map_location="cpu")['state_dict'] model.load_state_dict(trained_dict, strict=False) model.eval() while(cap.isOpened()): start_time = time() ret, frame = cap.read() if ret: image = frame[...,::-1] h, w = image.shape[:2] read_cam_time = time() # Predict mask X, pad_up, pad_left, h_new, w_new = utils.preprocessing(image, expected_size=args.input_sz, pad_value=0) preproc_time = time() with torch.no_grad(): if args.use_cuda: mask = model(X.cuda()) if mask.shape[1] != h_new: mask = F.interpolate(mask, size=(args.input_sz, args.input_sz), mode='bilinear', align_corners=True) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].cpu().numpy() else: mask = model(X) mask = mask[..., pad_up: pad_up+h_new, pad_left: pad_left+w_new] mask = F.interpolate(mask, size=(h,w), mode='bilinear', align_corners=True) mask = F.softmax(mask, dim=1) mask = mask[0,1,...].numpy() predict_time = time() # Draw result if args.bg is None: image_alpha = utils.draw_matting(image, mask) #image_alpha = utils.draw_transperency(image, mask, COLOR1, COLOR2) else: image_alpha = utils.draw_fore_to_back(image, mask, BACKGROUND, kernel_sz=KERNEL_SZ, sigma=SIGMA) draw_time = time() # Print runtime read = read_cam_time-start_time preproc = preproc_time-read_cam_time pred = predict_time-preproc_time draw = draw_time-predict_time total = read + preproc + pred + draw fps = 1 / pred print("read: %.3f [s]; preproc: %.3f [s]; pred: %.3f [s]; draw: %.3f [s]; total: %.3f [s]; fps: %.2f [Hz]" % (read, preproc, pred, draw, total, fps)) # Wait for interupt cv2.putText(image_alpha, "%.2f [fps]" % (fps), (10, 50), font, 1.5, (0, 255, 0), 2, cv2.LINE_AA) out.write(image_alpha[..., ::-1]) if args.watch: cv2.imshow('webcam', image_alpha[..., ::-1]) if cv2.waitKey(1) & 0xFF == ord('q'): break else: break
def _reset(self, data_path, save): if not save: print("extract arch2vec embedding table...") dataset = load_json(data_path) self.model = Model(input_dim=args.input_dim, hidden_dim=args.hidden_dim, latent_dim=args.latent_dim, num_hops=args.hops, num_mlp_layers=args.mlps, dropout=args.dropout, **cfg['GAE']).cuda() model_ckpt_path = os.path.join(self.dir_name, '{}'.format(args.model_path)) if not os.path.exists(model_ckpt_path): print("File {} does not exist.".format(model_ckpt_path)) exit() self.model.load_state_dict( torch.load(model_ckpt_path)['model_state']) self.model.eval() print("length of the dataset: {}".format(len(dataset))) self.f_path = os.path.join( self.dir_name, '{}-arch2vec.pt'.format(args.dataset_name)) if os.path.exists(self.f_path): print('ATTENTION!!! {} is already saved.'.format(self.f_path)) exit() print('save to {} ...'.format(self.f_path)) for ind in range(len(dataset)): adj = torch.Tensor( dataset[str(ind)]['module_adjacency']).unsqueeze(0).cuda() ops = torch.Tensor(dataset[str(ind)] ['module_operations']).unsqueeze(0).cuda() adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep']) test_acc = dataset[str(ind)]['test_accuracy'] valid_acc = dataset[str(ind)]['validation_accuracy'] other_info = { 'valid_accuracy_avg': dataset[str(ind)]['validation_accuracy_avg'], 'test_accuracy_avg': dataset[str(ind)]['test_accuracy_avg'] } time = dataset[str(ind)]['training_time'] x, _ = self.model._encoder(ops, adj) self.embedding[ind] = { 'feature': x.mean(dim=1).squeeze(0).cpu(), 'valid_accuracy': float(valid_acc), 'test_accuracy': float(test_acc), 'time': float(time), 'other_info': other_info } torch.save(self.embedding, self.f_path) print("finished arch2vec extraction") exit() else: self.f_path = os.path.join( self.dir_name, '{}-arch2vec.pt'.format(args.dataset_name)) print("load pretrained arch2vec in path: {}".format(self.f_path)) self.embedding = torch.load(self.f_path) random.seed(args.seed) random.shuffle(self.embedding) for ind in range(len(self.embedding)): self.features.append(self.embedding[ind]['feature']) self.features = torch.stack(self.features, dim=0) print('loading finished. pretrained embeddings shape: {}'.format( self.features.shape))