def predict_input_disc(args): print('Predicting with discriminative rnng.') parser = load_model(args.checkpoint) while True: sentence = input('Input a sentence: ') words = sentence.split() print('Processed:', ' '.join(parser.word_vocab.process(words))) print() print('Parse:') tree, nll = parser.parse(words) print(' {} {:.2f}'.format(tree.linearize(with_tag=False), nll.value())) print() print('Samples (alpha = {}):'.format(args.alpha, 2)) for _ in range(8): tree, nll, *rest = parser.sample(words, alpha=args.alpha) print(' {} {:.2f}'.format(tree.linearize(with_tag=False), nll.value())) print('-' * 79) print()
def predict_input_gen(args): print('Predicting with generative rnng.') joint = load_model(args.checkpoint) proposal = load_model(args.proposal_model) parser = GenerativeDecoder(model=joint, proposal=proposal, num_samples=args.num_samples, alpha=args.alpha) while True: sentence = input('Input a sentence: ') words = sentence.split() print('Processed:', ' '.join(joint.word_vocab.process(words))) print() print('Perplexity: {:.2f}'.format(parser.perplexity(words))) print() print('MAP tree:') tree, proposal_logprob, joint_logprob = parser.map_tree(words) print(' {} {:.2f} {:.2f}'.format(tree.linearize(with_tag=False), joint_logprob, proposal_logprob)) print() scored = parser.scored_samples(words) print(f'Unique samples: {len(scored)}/{args.num_samples}.') print('Highest q(y|x):') scored = sorted(scored, reverse=True, key=lambda t: t[1]) for tree, proposal_logprob, joint_logprob, count in scored[:4]: print(' {} {:.2f} {:.2f} {}'.format( tree.linearize(with_tag=False), joint_logprob, proposal_logprob, count)) print('Highest p(x,y):') scored = sorted(scored, reverse=True, key=lambda t: t[2]) for tree, proposal_logprob, joint_logprob, count in scored[:4]: print(' {} {:.2f} {:.2f} {}'.format( tree.linearize(with_tag=False), joint_logprob, proposal_logprob, count)) print('-' * 79) print()
def predict_perplexity(args): np.random.seed(args.numpy_seed) with open(args.infile, 'r') as f: lines = [line.strip() for line in f.readlines()] if is_tree(lines[0]): sentences = [fromstring(line.strip()).words() for line in lines] else: sentences = [line.strip().split() for line in lines] model = load_model(args.checkpoint) proposal = load_model(args.proposal_model) decoder = GenerativeDecoder(model=model, proposal=proposal, num_samples=args.num_samples, alpha=args.alpha) proposal_type = 'disc-rnng' if isinstance(proposal, DiscRNNG) else 'crf' filename_base = 'proposal={}_num-samples={}_temp={}_seed={}'.format( proposal_type, args.num_samples, args.alpha, args.numpy_seed) proposals_path = os.path.join(args.outdir, filename_base + '.props') result_path = os.path.join(args.outdir, filename_base + '.tsv') print('Predicting perplexity with Generative RNNG.') print(f'Loading model from `{args.checkpoint}`.') print(f'Loading proposal from `{args.proposal_model}`.') print(f'Loading lines from directory `{args.infile}`.') print(f'Writing proposals to `{proposals_path}`.') print(f'Writing predictions to `{result_path}`.') print('Sampling proposals...') decoder.generate_proposal_samples(sentences, proposals_path) print('Computing perplexity...') _, perplexity = decoder.predict_from_proposal_samples(proposals_path) with open(result_path, 'w') as f: print('\t'.join( ('proposal', 'file', 'perplexity', 'num-samples', 'temp', 'seed')), file=f) print('\t'.join( (proposal_type, os.path.basename(args.infile), str(perplexity), str(args.num_samples), str(args.alpha), str(args.numpy_seed))), file=f)
def inspect_model(args): assert args.model_type == 'disc-rnng', args.model_type print(f'Inspecting attention for sentences in `{args.infile}`.') parser = load_model(args.checkpoint) with open(args.infile, 'r') as f: lines = [line.strip() for line in f.readlines()] lines = lines[:args.max_lines] if is_tree(lines[0]): sentences = [fromstring(line).words() for line in lines] else: sentences = [line.split() for line in lines] def inspect_after_reduce(parser): subtree = parser.stack._stack[-1].subtree head = subtree.label children = [ child.label if isinstance(child, InternalNode) else child.word for child in subtree.children ] attention = parser.composer._attn gate = np.mean(parser.composer._gate) attention = [attention] if not isinstance( attention, list) else attention # in case .value() returns a float attentive = [ f'{child} ({attn:.2f})' for child, attn in zip(children, attention) ] print(' ', head, '|', ' '.join(attentive), f'[{gate:.2f}]') def parse_with_inspection(parser, words): parser.eval() nll = 0. word_ids = [parser.word_vocab.index_or_unk(word) for word in words] parser.initialize(word_ids) while not parser.stack.is_finished(): u = parser.parser_representation() action_logits = parser.f_action(u) action_id = np.argmax(action_logits.value() + parser._add_actions_mask()) nll += dy.pickneglogsoftmax(action_logits, action_id) parser.parse_step(action_id) if action_id == parser.REDUCE_ID: inspect_after_reduce(parser) tree = parser.get_tree() tree.substitute_leaves(iter(words)) # replaces UNKs with originals return tree, nll for sentence in sentences: tree, _ = parser.parse(sentence) print('>', ' '.join(sentence)) print('>', tree.linearize(with_tag=False)) parse_with_inspection(parser, sentence) print()
def sample_generative(args): print('Sampling from the generative model.') parser = load_model(args.checkpoint) print('Samples:') for i in range(args.num_samples): tree, logprob = parser.sample() print('>', tree.linearize(with_tag=False)) print()
def predict_input_crf(args): print('Predicting with crf parser.') parser = load_model(args.checkpoint) ## right_branching = fromstring( "(S (NP (@ The) (@ (@ other) (@ (@ hungry) (@ cat)))) (@ (VP meows ) (@ .)))" ).convert() left_branching = fromstring( "(S (NP (@ The) (@ (@ (@ other) (@ hungry)) (@ cat))) (@ (VP meows ) (@ .)))" ).convert() # right_branching = fromstring("(X (X (@ The) (@ (@ other) (@ (@ hungry) (@ cat)))) (@ (X meows ) (@ .)))").convert() # left_branching = fromstring("(X (X (@ The) (@ (@ (@ other) (@ hungry)) (@ cat))) (@ (X meows ) (@ .)))").convert() right_nll = parser.forward(right_branching, is_train=False) left_nll = parser.forward(left_branching, is_train=False) print('Right:', right_nll.value()) print('Left:', left_nll.value()) ## while True: sentence = input('Input a sentence: ') words = sentence.split() print('Processed:', ' '.join(parser.word_vocab.process(words))) print() print('Parse:') tree, nll = parser.parse(words) print(' {} {:.3f}'.format(tree.linearize(with_tag=False), nll.value())) print() print('Samples:') parse, parse_logprob, samples, entropy = parser.parse_sample_entropy( words, num_samples=8, alpha=1) for tree, nll in samples: print(' {} {:.3f}'.format(tree.linearize(with_tag=False), nll.value())) # print(' {} ||| {} ||| {:.3f}'.format( # tree.convert().linearize(with_tag=False), tree.un_cnf().linearize(with_tag=False), nll.value())) print() print('Parse (temperature {}):'.format(args.alpha)) print(' {} {:.3f}'.format(parse.linearize(with_tag=False), -parse_logprob.value())) print() print('Entropy:') print(' {:.3f}'.format(entropy.value())) print('-' * 79) print()
def predict_entropy(args): print( f'Predicting entropy for lines in `{args.infile}`, writing to `{args.outfile}`...' ) print(f'Loading model from `{args.checkpoint}`.') print(f'Using {args.num_samples} samples.') parser = load_model(args.checkpoint) parser.eval() with open(args.infile, 'r') as f: lines = [line.strip() for line in f.readlines()] if is_tree(lines[0]): sentences = [fromstring(line.strip()).words() for line in lines] else: sentences = [line.strip().split() for line in lines] with open(args.outfile, 'w') as f: print('id', 'entropy', 'num-samples', 'model', 'file', file=f, sep='\t') for i, words in enumerate(tqdm(sentences)): dy.renew_cg() if args.num_samples == 0: assert args.model_type == 'crf', 'exact computation only for crf.' entropy = parser.entropy(words) else: if args.model_type == 'crf': samples = parser.sample(words, num_samples=args.num_samples) if args.num_samples == 1: samples = [samples] else: samples = [ parser.sample(words, alpha=args.alpha) for _ in range(args.num_samples) ] trees, nlls = zip(*samples) entropy = dy.esum(list(nlls)) / len(nlls) print(i, entropy.value(), args.num_samples, args.model_type, args.infile, file=f, sep='\t')
def predict_text_file(args): assert os.path.exists(args.infile), 'specifiy file to parse with --infile.' print(f'Predicting trees for lines in `{args.infile}`.') with open(args.infile, 'r') as f: lines = [line.strip().split() for line in f.readlines()] if args.model_type == 'disc-rnng': print('Predicting with discriminative RNNG.') parser = load_model(args.checkpoint) elif args.model_type == 'crf': print('Predicting with CRF parser.') parser = load_model(args.checkpoint) elif args.model_type == 'gen-rnng': print('Predicting with generative RNNG.') model = load_model(args.checkpoint) parser = GenerativeDecoder(model=model) if args.proposal_model: parser.load_proposal_model(args.proposal_model) elif args.proposal_samples: parser.load_proposal_samples(args.proposal_samples) else: raise ValueError('Specify proposals.') else: raise ValueError('Specify model-type.') print(f'Predicting trees for `{args.infile}`...') trees = [] for words in tqdm(lines): dy.renew_cg() tree, *rest = parser.parse(words) trees.append(tree.linearize(with_tag=True)) print(f'Saved predicted trees in `{args.outfile}`.') with open(args.outfile, 'w') as f: print('\n'.join(trees), file=f)
def predict_perplexity_from_samples(args): print('Predicting perplexity with Generative RNNG.') print(f'Loading model from `{args.checkpoint}`.') print(f'Loading proposal samples from `{args.proposal_samples}`.') print(f'Loading lines from directory `{args.infile}`.') print(f'Writing predictions to `{args.outfile}`.') np.random.seed(args.numpy_seed) model = load_model(args.checkpoint) decoder = GenerativeDecoder(model=model, num_samples=args.num_samples, alpha=args.alpha) print('Computing perplexity...') trees, perplexity = decoder.predict_from_proposal_samples( args.proposal_samples) # Compute f-score from trees base_name = os.path.splitext(args.outfile)[0] pred_path = base_name + '.trees' result_path = base_name + '.result' with open(pred_path, 'w') as f: print('\n'.join(trees), file=f) fscore = evalb(args.evalb_dir, pred_path, args.infile, result_path) print(f'Results: {fscore} fscore, {perplexity} perplexity.') with open(args.outfile, 'w') as f: print('proposals', 'perplexity', 'fscore', 'num-samples', 'temp', 'seed', sep='\t', file=f) print(args.proposal_samples, perplexity, fscore, args.num_samples, args.alpha, args.numpy_seed, sep='\t', file=f)
def predict_tree_file(args): assert os.path.exists(args.infile), 'specifiy file to parse with --infile.' print(f'Predicting trees for lines in `{args.infile}`.') with open(args.infile, 'r') as f: lines = [ fromstring(line.strip()).words() for line in f if line.strip() ] if args.model_type == 'disc': print('Loading discriminative model...') parser = load_model(args.checkpoint) parser.eval() print('Done.') elif args.model_type == 'gen': exit('Not yet...') print('Loading generative model...') parser = GenerativeDecoder() parser.load_model(path=args.checkpoint) if args.proposal_model: parser.load_proposal_model(path=args.proposal_model) if args.proposal_samples: parser.load_proposal_samples(path=args.proposal_samples) trees = [] for line in tqdm(lines): tree, _ = parser.parse(line) trees.append(tree.linearize()) pred_path = os.path.join(args.outfile) result_path = args.outfile + '.results' # Save the predicted trees. with open(pred_path, 'w') as f: print('\n'.join(trees), file=f) # Score the trees. fscore = evalb(args.evalb_dir, pred_path, args.infile, result_path) print( f'Predictions saved in `{pred_path}`. Results saved in `{result_path}`.' ) print(f'F-score {fscore:.2f}.')
def sample_proposals(args): assert os.path.exists(args.infile), 'specifiy file to parse with --infile.' print(f'Sampling proposal trees for sentences in `{args.infile}`.') with open(args.infile, 'r') as f: lines = [line.strip() for line in f.readlines()] if is_tree(lines[0]): sentences = [fromstring(line).words() for line in lines] else: sentences = [line.split() for line in lines] parser = load_model(args.checkpoint) samples = [] if args.model_type == 'crf': for i, words in enumerate(tqdm(sentences)): dy.renew_cg() for tree, nll in parser.sample(words, num_samples=args.num_samples): samples.append(' ||| '.join((str(i), str(-nll.value()), tree.linearize(with_tag=False)))) print(' ||| '.join((str(i), str(-nll.value()), tree.linearize(with_tag=False)))) else: for i, words in enumerate(tqdm(sentences)): for _ in range(args.num_samples): dy.renew_cg() tree, nll = parser.sample(words, alpha=args.alpha) samples.append(' ||| '.join((str(i), str(-nll.value()), tree.linearize(with_tag=False)))) with open(args.outfile, 'w') as f: print('\n'.join(samples), file=f, end='')
def main(): args = get_args() torch.set_grad_enabled(False) if args.network == "mobile0.25": cfg = cfg_mnet elif args.network == "resnet50": cfg = cfg_re50 else: raise NotImplementedError( f"Only mobile0.25 and resnet50 are suppoted.") # net and model net = RetinaFace(cfg=cfg, phase="test") net = load_model(net, args.trained_model, args.cpu) net.eval() if args.fp16: net = net.half() print("Finished loading model!") cudnn.benchmark = True device = torch.device("cpu" if args.cpu else "cuda") net = net.to(device) file_paths = sorted(args.input_path.rglob("*.jpg")) if args.num_gpu is not None: start, end = split_array(len(file_paths), args.num_gpu, args.gpu_id) file_paths = file_paths[start:end] output_path = args.output_path if args.save_boxes: output_label_path = output_path / "labels" output_label_path.mkdir(exist_ok=True, parents=True) if args.save_crops: output_image_path = output_path / "images" output_image_path.mkdir(exist_ok=True, parents=True) transform = albu.Compose([ albu.Normalize( p=1, mean=(104, 117, 123), std=(1.0, 1.0, 1.0), max_pixel_value=1) ], p=1) test_loader = DataLoader( InferenceDataset(file_paths, args.origin_size, transform=transform), batch_size=args.batch_size, num_workers=args.num_workers, pin_memory=True, drop_last=False, ) with torch.no_grad(): for raw_input in tqdm(test_loader): torched_images = raw_input["torched_image"] if args.fp16: torched_images = torched_images.half() resizes = raw_input["resize"] image_paths = Path(raw_input["image_path"]) raw_images = raw_input["raw_image"] labels = [] if (args.batch_size == 1 and args.save_boxes and (output_label_path / f"{Path(image_paths[0]).stem}.json").exists()): continue loc, conf, land = net(torched_images.to(device)) # forward pass batch_size = torched_images.shape[0] image_height, image_width = torched_images.shape[2:] scale1 = torch.Tensor([ image_width, image_height, image_width, image_height, image_width, image_height, image_width, image_height, image_width, image_height, ]) scale1 = scale1.to(device) scale = torch.Tensor( [image_width, image_height, image_width, image_height]) scale = scale.to(device) priorbox = PriorBox(cfg, image_size=(image_height, image_width)) priors = priorbox.forward() priors = priors.to(device) prior_data = priors.data for batch_id in range(batch_size): image_path = image_paths[batch_id] file_id = Path(image_path).stem raw_image = raw_images[batch_id] resize = resizes[batch_id].float() boxes = decode(loc.data[batch_id], prior_data, cfg["variance"]) boxes *= scale / resize scores = conf[batch_id][:, 1] landmarks = decode_landm(land.data[batch_id], prior_data, cfg["variance"]) landmarks *= scale1 / resize # ignore low scores valid_index = torch.where( scores > args.confidence_threshold)[0] boxes = boxes[valid_index] landmarks = landmarks[valid_index] scores = scores[valid_index] order = scores.argsort(descending=True) boxes = boxes[order] landmarks = landmarks[order] scores = scores[order] # do NMS keep = nms(boxes, scores, args.nms_threshold) boxes = boxes[keep, :].int() landmarks = landmarks[keep].int() if boxes.shape[0] == 0: continue scores = scores[keep].cpu().numpy().astype(np.float64) for crop_id, bbox in enumerate(boxes): bbox = bbox.cpu().numpy() labels += [{ "crop_id": crop_id, "bbox": bbox.tolist(), "score": scores[crop_id], "landmarks": landmarks[crop_id].tolist(), }] if args.save_crops: x_min, y_min, x_max, y_max = bbox x_min = max(0, x_min) y_min = max(0, y_min) crop = raw_image[y_min:y_max, x_min:x_max].cpu().numpy() target_folder = output_image_path / f"{file_id}" target_folder.mkdir(exist_ok=True, parents=True) crop_file_path = target_folder / f"{file_id}_{crop_id}.jpg" if crop_file_path.exists(): continue cv2.imwrite( str(crop_file_path), cv2.cvtColor(crop, cv2.COLOR_BGR2RGB), [int(cv2.IMWRITE_JPEG_QUALITY), 90], ) if args.save_boxes: result = { "file_path": image_path, "file_id": file_id, "bboxes": labels, } with open(output_label_path / f"{file_id}.json", "w") as f: json.dump(result, f, indent=2)
def main(): args = get_args() torch.set_grad_enabled(False) if args.network == "mobile0.25": cfg = cfg_mnet elif args.network == "resnet50": cfg = cfg_re50 else: raise NotImplementedError(f"Only mobile0.25 and resnet50 are suppoted.") # net and model net = RetinaFace(cfg=cfg, phase="test") net = load_model(net, args.trained_model, args.cpu) net.eval() if args.fp16: net = net.half() print("Finished loading model!") cudnn.benchmark = True device = torch.device("cpu" if args.cpu else "cuda") net = net.to(device) file_paths = sorted(args.input_path.rglob("*.mp4"))[: args.num_videos] if args.num_gpu is not None: start, end = split_array(len(file_paths), args.num_gpu, args.gpu_id) file_paths = file_paths[start:end] output_path = args.output_path if args.save_boxes: output_label_path = output_path / "labels" output_label_path.mkdir(exist_ok=True, parents=True) if args.save_crops: output_image_path = output_path / "images" output_image_path.mkdir(exist_ok=True, parents=True) if args.video_decoder == "cpu": decode_device = cpu(0) elif args.video_decoder == "gpu": decode_device = gpu(0) else: raise NotImplementedError(f"Only CPU and GPU devices are supported by decard, but got {args.video_decoder}") transform = albu.Compose([albu.Normalize(p=1, mean=(104, 117, 123), std=(1.0, 1.0, 1.0), max_pixel_value=1)], p=1) with torch.no_grad(): for video_path in tqdm(file_paths): labels = [] video_id = video_path.stem with video_reader(str(video_path), ctx=decode_device) as video: len_video = len(video) if args.num_frames is None or args.num_frames == 1: frame_ids = list(range(args.num_frames)) elif args.num_frames > 1: if len_video < args.num_frames: step = 1 else: step = int(len_video / args.num_frames) frame_ids = list(range(0, len_video, step))[: args.num_frames] else: raise ValueError(f"Expect None or integer > 1 for args.num_frames, but got {args.num_frames}") frames = video.get_batch(frame_ids) if args.video_decoder == "cpu": frames = frames.asnumpy() elif args.video_decoder == "gpu": frames = dlpack.from_dlpack(frames.to_dlpack()) if args.video_decoder == "gpu": del video torch.cuda.empty_cache() gc.collect() num_frames = len(frames) image_height = frames.shape[1] image_width = frames.shape[2] scale1 = torch.Tensor( [ image_width, image_height, image_width, image_height, image_width, image_height, image_width, image_height, image_width, image_height, ] ) scale1 = scale1.to(device) scale = torch.Tensor([image_width, image_height, image_width, image_height]) scale = scale.to(device) priorbox = PriorBox(cfg, image_size=(image_height, image_width)) priors = priorbox.forward() priors = priors.to(device) prior_data = priors.data if args.resize_coeff is not None: target_size = min(args.resize_coeff) max_size = max(args.resize_coeff) image_height = frames.shape[1] image_width = frames.shape[2] image_size_min = min([image_width, image_height]) image_size_max = max([image_width, image_height]) resize = float(target_size) / float(image_size_min) if np.round(resize * image_size_max) > max_size: resize = float(max_size) / float(image_size_max) else: resize = 1 for pred_id in range(num_frames): frame = frames[pred_id] torched_image = prepare_image(frame, transform, args.video_decoder).to(device) if args.fp16: torched_image = torched_image.half() loc, conf, land = net(torched_image) # forward pass frame_id = frame_ids[pred_id] boxes = decode(loc.data[0], prior_data, cfg["variance"]) boxes *= scale / resize boxes = boxes.cpu().numpy() scores = conf[0].data.cpu().numpy()[:, 1] landmarks = decode_landm(land.data[0], prior_data, cfg["variance"]) landmarks *= scale1 / resize landmarks = landmarks.cpu().numpy() # ignore low scores valid_index = np.where(scores > args.confidence_threshold)[0] boxes = boxes[valid_index] landmarks = landmarks[valid_index] scores = scores[valid_index] # keep top-K before NMS order = scores.argsort()[::-1] # order = scores.argsort()[::-1][:args.top_k] boxes = boxes[order] landmarks = landmarks[order] scores = scores[order] # do NMS detection = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False) keep = py_cpu_nms(detection, args.nms_threshold) # keep = nms(detection, args.nms_threshold,force_cpu=args.cpu) # x_min, y_min, x_max, y_max, score detection = detection[keep, :] landmarks = landmarks[keep].astype(int) if detection.shape[0] == 0: continue bboxes = detection[:, :4].astype(int) confidence = detection[:, 4].astype(np.float64) for crop_id in range(len(detection)): bbox = bboxes[crop_id] labels += [ { "frame_id": int(frame_id), "crop_id": crop_id, "bbox": bbox.tolist(), "score": confidence[crop_id], "landmarks": landmarks[crop_id].tolist(), } ] if args.save_crops: x_min, y_min, x_max, y_max = bbox x_min = max(0, x_min) y_min = max(0, y_min) crop = frame[y_min:y_max, x_min:x_max] target_folder = output_image_path / f"{video_id}" target_folder.mkdir(exist_ok=True, parents=True) crop_file_path = target_folder / f"{frame_id}_{crop_id}.jpg" if crop_file_path.exists(): continue cv2.imwrite( str(crop_file_path), cv2.cvtColor(crop, cv2.COLOR_BGR2RGB), [int(cv2.IMWRITE_JPEG_QUALITY), 90], ) if args.save_boxes: result = { "file_path": str(video_path), "file_id": video_id, "bboxes": labels, } with open(output_label_path / f"{video_id}.json", "w") as f: json.dump(result, f, indent=2)
def main(): args = get_args() torch.set_grad_enabled(False) cfg = None if args.network == "mobile0.25": cfg = cfg_mnet elif args.network == "resnet50": cfg = cfg_re50 # net and model net = RetinaFace(cfg=cfg, phase="test") net = load_model(net, args.trained_model, args.cpu) net.eval() print("Finished loading model!") print(net) cudnn.benchmark = True device = torch.device("cpu" if args.cpu else "cuda") net = net.to(device) # testing dataset testset_folder = args.dataset_folder testset_list = args.dataset_folder[:-7] + "wider_val.txt" with open(testset_list, "r") as fr: test_dataset = fr.read().split() num_images = len(test_dataset) _t = {"forward_pass": Timer(), "misc": Timer()} # testing begin for i, img_name in enumerate(test_dataset): image_path = testset_folder + img_name img_raw = cv2.imread(image_path, cv2.IMREAD_COLOR) img = np.float32(img_raw) # testing scale target_size = 1600 max_size = 2150 im_shape = img.shape im_size_min = np.min(im_shape[0:2]) im_size_max = np.max(im_shape[0:2]) resize = float(target_size) / float(im_size_min) # prevent bigger axis from being more than max_size: if np.round(resize * im_size_max) > max_size: resize = float(max_size) / float(im_size_max) if args.origin_size: resize = 1 if resize != 1: img = cv2.resize(img, None, None, fx=resize, fy=resize, interpolation=cv2.INTER_LINEAR) im_height, im_width, _ = img.shape scale = torch.Tensor( [img.shape[1], img.shape[0], img.shape[1], img.shape[0]]) img -= (104, 117, 123) img = img.transpose(2, 0, 1) img = torch.from_numpy(img).unsqueeze(0) img = img.to(device) scale = scale.to(device) _t["forward_pass"].tic() loc, conf, landms = net(img) # forward pass _t["forward_pass"].toc() _t["misc"].tic() priorbox = PriorBox(cfg, image_size=(im_height, im_width)) priors = priorbox.forward() priors = priors.to(device) prior_data = priors.data boxes = decode(loc.data.squeeze(0), prior_data, cfg["variance"]) boxes = boxes * scale / resize boxes = boxes.cpu().numpy() scores = conf.squeeze(0).data.cpu().numpy()[:, 1] landms = decode_landm(landms.data.squeeze(0), prior_data, cfg["variance"]) scale1 = torch.Tensor([ img.shape[3], img.shape[2], img.shape[3], img.shape[2], img.shape[3], img.shape[2], img.shape[3], img.shape[2], img.shape[3], img.shape[2], ]) scale1 = scale1.to(device) landms = landms * scale1 / resize landms = landms.cpu().numpy() # ignore low scores inds = np.where(scores > args.confidence_threshold)[0] boxes = boxes[inds] landms = landms[inds] scores = scores[inds] # keep top-K before NMS order = scores.argsort()[::-1] # order = scores.argsort()[::-1][:args.top_k] boxes = boxes[order] landms = landms[order] scores = scores[order] # do NMS dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False) keep = py_cpu_nms(dets, args.nms_threshold) # keep = nms(dets, args.nms_threshold,force_cpu=args.cpu) dets = dets[keep, :] landms = landms[keep] # keep top-K faster NMS # dets = dets[:args.keep_top_k, :] # landms = landms[:args.keep_top_k, :] dets = np.concatenate((dets, landms), axis=1) _t["misc"].toc() # -------------------------------------------------------------------- save_name = args.save_folder + img_name[:-4] + ".txt" dirname = os.path.dirname(save_name) if not os.path.isdir(dirname): os.makedirs(dirname) with open(save_name, "w") as fd: bboxs = dets file_name = os.path.basename(save_name)[:-4] + "\n" bboxs_num = str(len(bboxs)) + "\n" fd.write(file_name) fd.write(bboxs_num) for box in bboxs: x = int(box[0]) y = int(box[1]) w = int(box[2]) - int(box[0]) h = int(box[3]) - int(box[1]) confidence = str(box[4]) line = str(x) + " " + str(y) + " " + str(w) + " " + str( h) + " " + confidence + " \n" fd.write(line) print("im_detect: {:d}/{:d} forward_pass_time: {:.4f}s misc: {:.4f}s". format(i + 1, num_images, _t["forward_pass"].average_time, _t["misc"].average_time)) # save image if args.save_image: for b in dets: if b[4] < args.vis_thres: continue text = "{:.4f}".format(b[4]) b = list(map(int, b)) cv2.rectangle(img_raw, (b[0], b[1]), (b[2], b[3]), (0, 0, 255), 2) cx = b[0] cy = b[1] + 12 cv2.putText(img_raw, text, (cx, cy), cv2.FONT_HERSHEY_DUPLEX, 0.5, (255, 255, 255)) # landms cv2.circle(img_raw, (b[5], b[6]), 1, (0, 0, 255), 4) cv2.circle(img_raw, (b[7], b[8]), 1, (0, 255, 255), 4) cv2.circle(img_raw, (b[9], b[10]), 1, (255, 0, 255), 4) cv2.circle(img_raw, (b[11], b[12]), 1, (0, 255, 0), 4) cv2.circle(img_raw, (b[13], b[14]), 1, (255, 0, 0), 4) # save image if not os.path.exists("./results/"): os.makedirs("./results/") name = "./results/" + str(i) + ".jpg" cv2.imwrite(name, img_raw)
def syneval_rnn(args): model = load_model(args.checkpoint) model.eval() files = SHORT if args.syneval_short else ALL outdir = os.path.join(args.checkpoint, 'output') if not os.path.exists(outdir): os.makedirs(outdir) result_filename = 'syneval_results_caps.tsv' if args.capitalize else 'syneval_results.tsv' outpath = os.path.join(outdir, result_filename) print('Predicting syneval with RNN language model.') print(f'Loading model from `{args.checkpoint}`.') print(f'Loading syneval examples from directory `{args.indir}`.') print(f'Writing predictions to `{outpath}`.') with open(outpath, 'w') as outfile: print('\t'.join( ('name', 'index', 'pos-perplexity', 'neg-perplexity', 'correct', 'pos-sentence-processed', 'neg-sentence-processed')), file=outfile) print('Predicting syneval for:', '\n', '\n '.join(files)) for fname in files: print(f'Predicting `{fname}`...') inpath = os.path.join(args.indir, fname) with open(inpath + '.pos') as f: pos_sents = [line.strip() for line in f.readlines()] if args.capitalize: pos_sents = [sent.capitalize() for sent in pos_sents] if args.add_period: pos_sents = [sent + ' .' for sent in pos_sents] with open(inpath + '.neg') as f: neg_sents = [line.strip() for line in f.readlines()] if args.capitalize: neg_sents = [sent.capitalize() for sent in neg_sents] if args.add_period: neg_sents = [sent + ' .' for sent in neg_sents] pos_sents = [sent.split() for sent in pos_sents] neg_sents = [sent.split() for sent in neg_sents] assert len(pos_sents) == len(neg_sents) if args.syneval_max_lines != -1 and len( pos_sents) > args.syneval_max_lines: subsampled_ids = np.random.choice(len(pos_sents), args.syneval_max_lines, replace=False) pos_sents = [pos_sents[i] for i in subsampled_ids] neg_sents = [neg_sents[i] for i in subsampled_ids] num_correct = 0 for i, (pos, neg) in enumerate(tqdm(list(zip(pos_sents, neg_sents)))): dy.renew_cg() pos_pp = np.exp(model.forward(pos).value() / len(pos)) neg_pp = np.exp(model.forward(neg).value() / len(neg)) correct = pos_pp < neg_pp num_correct += correct # see which words are unked during prediction pos = model.word_vocab.process(pos) neg = model.word_vocab.process(neg) result = '\t'.join( (fname, str(i), str(round(pos_pp, 2)), str(round(neg_pp, 2)), str(int(correct)), ' '.join(pos), ' '.join(neg))) print(result, file=outfile) print( f'{fname}: {num_correct}/{len(pos_sents)} = {num_correct / len(pos_sents):.2%} correct', '\n')
def main(): args = get_args() torch.set_grad_enabled(False) cfg = None if args.network == "mobile0.25": cfg = cfg_mnet elif args.network == "resnet50": cfg = cfg_re50 # net and model net = RetinaFace(cfg=cfg, phase="test") net = load_model(net, args.trained_model, args.cpu) net.eval() print("Finished loading model!") print(net) cudnn.benchmark = True device = torch.device("cpu" if args.cpu else "cuda") net = net.to(device) args.save_folder.mkdir(exist_ok=True) fw = open(os.path.join(args.save_folder, args.dataset + "_dets.txt"), "w") # testing dataset testset_folder = os.path.join("data", args.dataset, "images/") testset_list = os.path.join("data", args.dataset, "img_list.txt") with open(testset_list, "r") as fr: test_dataset = fr.read().split() num_images = len(test_dataset) # testing scale resize = 1 _t = {"forward_pass": Timer(), "misc": Timer()} # testing begin for i, img_name in enumerate(test_dataset): image_path = testset_folder + img_name + ".jpg" img_raw = cv2.imread(image_path, cv2.IMREAD_COLOR) img = np.float32(img_raw) if resize != 1: img = cv2.resize(img, None, None, fx=resize, fy=resize, interpolation=cv2.INTER_LINEAR) im_height, im_width, _ = img.shape scale = torch.Tensor( [img.shape[1], img.shape[0], img.shape[1], img.shape[0]]) img -= (104, 117, 123) img = img.transpose(2, 0, 1) img = torch.from_numpy(img).unsqueeze(0) img = img.to(device) scale = scale.to(device) _t["forward_pass"].tic() loc, conf, landms = net(img) # forward pass _t["forward_pass"].toc() _t["misc"].tic() priorbox = PriorBox(cfg, image_size=(im_height, im_width)) priors = priorbox.forward() priors = priors.to(device) prior_data = priors.data boxes = decode(loc.data.squeeze(0), prior_data, cfg["variance"]) boxes = boxes * scale / resize boxes = boxes.cpu().numpy() scores = conf.squeeze(0).data.cpu().numpy()[:, 1] landms = decode_landm(landms.data.squeeze(0), prior_data, cfg["variance"]) scale1 = torch.Tensor([ img.shape[3], img.shape[2], img.shape[3], img.shape[2], img.shape[3], img.shape[2], img.shape[3], img.shape[2], img.shape[3], img.shape[2], ]) scale1 = scale1.to(device) landms = landms * scale1 / resize landms = landms.cpu().numpy() # ignore low scores inds = np.where(scores > args.confidence_threshold)[0] boxes = boxes[inds] landms = landms[inds] scores = scores[inds] # keep top-K before NMS # order = scores.argsort()[::-1][:args.top_k] order = scores.argsort()[::-1] boxes = boxes[order] landms = landms[order] scores = scores[order] # do NMS dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False) keep = py_cpu_nms(dets, args.nms_threshold) dets = dets[keep, :] landms = landms[keep] # keep top-K faster NMS # dets = dets[:args.keep_top_k, :] # landms = landms[:args.keep_top_k, :] dets = np.concatenate((dets, landms), axis=1) _t["misc"].toc() # save dets if args.dataset == "FDDB": fw.write("{:s}\n".format(img_name)) fw.write("{:.1f}\n".format(dets.shape[0])) for k in range(dets.shape[0]): xmin = dets[k, 0] ymin = dets[k, 1] xmax = dets[k, 2] ymax = dets[k, 3] score = dets[k, 4] w = xmax - xmin + 1 h = ymax - ymin + 1 # fw.write('{:.3f} {:.3f} {:.3f} {:.3f} {:.10f}\n'.format(xmin, ymin, w, h, score)) fw.write("{:d} {:d} {:d} {:d} {:.10f}\n".format( int(xmin), int(ymin), int(w), int(h), score)) print("im_detect: {:d}/{:d} forward_pass_time: {:.4f}s misc: {:.4f}s". format(i + 1, num_images, _t["forward_pass"].average_time, _t["misc"].average_time)) # show image if args.save_image: for b in dets: if b[4] < args.vis_thres: continue text = "{:.4f}".format(b[4]) b = list(map(int, b)) cv2.rectangle(img_raw, (b[0], b[1]), (b[2], b[3]), (0, 0, 255), 2) cx = b[0] cy = b[1] + 12 cv2.putText(img_raw, text, (cx, cy), cv2.FONT_HERSHEY_DUPLEX, 0.5, (255, 255, 255)) # landms cv2.circle(img_raw, (b[5], b[6]), 1, (0, 0, 255), 4) cv2.circle(img_raw, (b[7], b[8]), 1, (0, 255, 255), 4) cv2.circle(img_raw, (b[9], b[10]), 1, (255, 0, 255), 4) cv2.circle(img_raw, (b[11], b[12]), 1, (0, 255, 0), 4) cv2.circle(img_raw, (b[13], b[14]), 1, (255, 0, 0), 4) # save image if not os.path.exists("./results/"): os.makedirs("./results/") name = "./results/" + str(i) + ".jpg" cv2.imwrite(name, img_raw) fw.close()
def process_video_files( network: str, trained_model: str, decode_gpu: bool, is_fp16: bool, file_paths: list, num_gpu: Optional[int], gpu_id: int, output_path: Path, is_save_boxes: bool, is_save_crops: bool, num_frames: int, resize_coeff: Optional[Tuple], confidence_threshold: float, num_workers: int, nms_threshold: float, batch_size: int, resize_scale: float, min_size: int, keep_top_k: int, ) -> None: torch.set_grad_enabled(False) if network == "mobile0.25": cfg = cfg_mnet_test elif network == "resnet50": cfg = cfg_re50_test else: raise NotImplementedError( f"Only mobile0.25 and resnet50 are suppoted, but we got {network}") if min_size < 0: raise ValueError( f"Min size should be positive, but we got {min_size}.") # net and model net = RetinaFace(cfg=cfg, phase="test") net = load_model(net, trained_model, load_to_cpu=False) net.eval() if is_fp16: net = net.half() device = torch.device("cuda") net.to(device) print("Finished loading model!") cudnn.benchmark = True transform = albu.Compose([ albu.Normalize( p=1, mean=(104, 117, 123), std=(1.0, 1.0, 1.0), max_pixel_value=1) ], p=1) if num_gpu is not None: start, end = split_array(len(file_paths), num_gpu, gpu_id) file_paths = file_paths[start:end] with torch.no_grad(): func = partial(get_frames, num_frames=num_frames, resize_coeff=resize_coeff, transform=transform, decode_gpu=decode_gpu) with torch.no_grad(): with concurrent.futures.ProcessPoolExecutor( num_workers) as executor: for result in tqdm(executor.map(func, file_paths), total=len(file_paths), leave=False, desc="Loading data files"): if len(result) != 0: result["is_fp16"] = is_fp16 result["device"] = device result["batch_size"] = batch_size result["cfg"] = cfg result["nms_threshold"] = nms_threshold result["confidence_threshold"] = confidence_threshold result["is_save_crops"] = is_save_crops result["is_save_boxes"] = is_save_boxes result["output_path"] = output_path result["net"] = net result["min_size"] = min_size result["resize_scale"] = resize_scale result["keep_top_k"] = keep_top_k process_frames(**result)
def syneval_parser(args): model = load_model(args.checkpoint) files = SHORT if args.syneval_short else ALL outdir = os.path.join(args.checkpoint, 'output') if not os.path.exists(outdir): os.makedirs(outdir) result_filename = 'syneval_results_caps.tsv' if args.capitalize else 'syneval_results.tsv' outpath = os.path.join(outdir, result_filename) print('Predicting syneval with discriminative parser.') print(f'Loading model from `{args.checkpoint}`.') print(f'Loading syneval examples from directory `{args.indir}`.') print(f'Writing predictions to `{outpath}`.') with open(outpath, 'w') as outfile: if args.num_samples == 1: # predict with logprob of predicted parse print('\t'.join(('name', 'index', 'pos-logprob', 'neg-logprob', 'correct', 'pos-tree', 'neg-tree')), file=outfile) else: print('\t'.join(('name', 'index', 'pos-entropy', 'neg-entropy', 'correct', 'pos-tree', 'neg-tree')), file=outfile) print('Predicting syneval for:', '\n', '\n '.join(files)) for fname in files: print(f'Predicting `{fname}`...') inpath = os.path.join(args.indir, fname) with open(inpath + '.pos') as f: pos_sents = [line.strip() for line in f.readlines()] if args.capitalize: pos_sents = [sent.capitalize() for sent in pos_sents] if args.add_period: pos_sents = [sent + ' .' for sent in pos_sents] with open(inpath + '.neg') as f: neg_sents = [line.strip() for line in f.readlines()] if args.capitalize: neg_sents = [sent.capitalize() for sent in neg_sents] if args.add_period: neg_sents = [sent + ' .' for sent in neg_sents] pos_sents = [sent.split() for sent in pos_sents] neg_sents = [sent.split() for sent in neg_sents] assert len(pos_sents) == len(neg_sents) if args.syneval_max_lines != -1 and len( pos_sents) > args.syneval_max_lines: subsampled_ids = np.random.choice(len(pos_sents), args.syneval_max_lines, replace=False) pos_sents = [pos_sents[i] for i in subsampled_ids] neg_sents = [neg_sents[i] for i in subsampled_ids] num_correct = 0 for i, (pos, neg) in enumerate(tqdm(list(zip(pos_sents, neg_sents)))): dy.renew_cg() if args.num_samples == 1: # predict with logprob of predicted parse pos_tree, pos_nll = model.parse(pos) neg_tree, neg_nll = model.parse(neg) pos_logprob = -pos_nll.value() neg_logprob = -neg_nll.value() pos_tree = pos_tree.linearize(with_tag=False) neg_tree = neg_tree.linearize(with_tag=False) correct = pos_logprob > neg_logprob num_correct += correct result = '\t'.join( (fname, str(i), str(round(pos_logprob, 4)), str(round(neg_logprob, 4)), str(int(correct)), pos_tree, neg_tree)) print(result, file=outfile) else: if args.exact_entropy: pos_tree, pos_entropy = model.parse_entropy(pos) neg_tree, neg_entropy = model.parse_entropy(neg) pos_entropy = pos_entropy.value() neg_entropy = neg_entropy.value() pos_tree = pos_tree.linearize(with_tag=False) neg_tree = neg_tree.linearize(with_tag=False) else: # predict with the parser's entropy pos_trees, pos_nlls = zip(*[ model.sample(pos) for _ in range(args.num_samples) ]) neg_trees, neg_nlls = zip(*[ model.sample(neg) for _ in range(args.num_samples) ]) pos_entropy = np.mean( [nll.value() for nll in pos_nlls]) neg_entropy = np.mean( [nll.value() for nll in neg_nlls]) pos_tree = Counter([ tree.linearize(with_tag=False) for tree in pos_trees ]).most_common(1)[0][0] neg_tree = Counter([ tree.linearize(with_tag=False) for tree in neg_trees ]).most_common(1)[0][0] correct = pos_entropy < neg_entropy num_correct += correct result = '\t'.join( (fname, str(i), str(round(pos_entropy, 4)), str(round(neg_entropy, 4)), str(int(correct)), pos_tree, neg_tree)) print(result, file=outfile) print( f'{fname}: {num_correct}/{len(pos_sents)} = {num_correct / len(pos_sents):.2%} correct', '\n')