def predict_input_disc(args):
print('Predicting with discriminative rnng.')
parser = load_model(args.checkpoint)
while True:
sentence = input('Input a sentence: ')
words = sentence.split()
print('Processed:', ' '.join(parser.word_vocab.process(words)))
print()
print('Parse:')
tree, nll = parser.parse(words)
print(' {} {:.2f}'.format(tree.linearize(with_tag=False),
nll.value()))
print()
print('Samples (alpha = {}):'.format(args.alpha, 2))
for _ in range(8):
tree, nll, *rest = parser.sample(words, alpha=args.alpha)
print(' {} {:.2f}'.format(tree.linearize(with_tag=False),
nll.value()))
print('-' * 79)
print()
def predict_input_gen(args):
print('Predicting with generative rnng.')
joint = load_model(args.checkpoint)
proposal = load_model(args.proposal_model)
parser = GenerativeDecoder(model=joint,
proposal=proposal,
num_samples=args.num_samples,
alpha=args.alpha)
while True:
sentence = input('Input a sentence: ')
words = sentence.split()
print('Processed:', ' '.join(joint.word_vocab.process(words)))
print()
print('Perplexity: {:.2f}'.format(parser.perplexity(words)))
print()
print('MAP tree:')
tree, proposal_logprob, joint_logprob = parser.map_tree(words)
print(' {} {:.2f} {:.2f}'.format(tree.linearize(with_tag=False),
joint_logprob, proposal_logprob))
print()
scored = parser.scored_samples(words)
print(f'Unique samples: {len(scored)}/{args.num_samples}.')
print('Highest q(y|x):')
scored = sorted(scored, reverse=True, key=lambda t: t[1])
for tree, proposal_logprob, joint_logprob, count in scored[:4]:
print(' {} {:.2f} {:.2f} {}'.format(
tree.linearize(with_tag=False), joint_logprob,
proposal_logprob, count))
print('Highest p(x,y):')
scored = sorted(scored, reverse=True, key=lambda t: t[2])
for tree, proposal_logprob, joint_logprob, count in scored[:4]:
print(' {} {:.2f} {:.2f} {}'.format(
tree.linearize(with_tag=False), joint_logprob,
proposal_logprob, count))
print('-' * 79)
print()
def predict_perplexity(args):
np.random.seed(args.numpy_seed)
with open(args.infile, 'r') as f:
lines = [line.strip() for line in f.readlines()]
if is_tree(lines[0]):
sentences = [fromstring(line.strip()).words() for line in lines]
else:
sentences = [line.strip().split() for line in lines]
model = load_model(args.checkpoint)
proposal = load_model(args.proposal_model)
decoder = GenerativeDecoder(model=model,
proposal=proposal,
num_samples=args.num_samples,
alpha=args.alpha)
proposal_type = 'disc-rnng' if isinstance(proposal, DiscRNNG) else 'crf'
filename_base = 'proposal={}_num-samples={}_temp={}_seed={}'.format(
proposal_type, args.num_samples, args.alpha, args.numpy_seed)
proposals_path = os.path.join(args.outdir, filename_base + '.props')
result_path = os.path.join(args.outdir, filename_base + '.tsv')
print('Predicting perplexity with Generative RNNG.')
print(f'Loading model from `{args.checkpoint}`.')
print(f'Loading proposal from `{args.proposal_model}`.')
print(f'Loading lines from directory `{args.infile}`.')
print(f'Writing proposals to `{proposals_path}`.')
print(f'Writing predictions to `{result_path}`.')
print('Sampling proposals...')
decoder.generate_proposal_samples(sentences, proposals_path)
print('Computing perplexity...')
_, perplexity = decoder.predict_from_proposal_samples(proposals_path)
with open(result_path, 'w') as f:
print('\t'.join(
('proposal', 'file', 'perplexity', 'num-samples', 'temp', 'seed')),
file=f)
print('\t'.join(
(proposal_type, os.path.basename(args.infile), str(perplexity),
str(args.num_samples), str(args.alpha), str(args.numpy_seed))),
file=f)
def inspect_model(args):
assert args.model_type == 'disc-rnng', args.model_type
print(f'Inspecting attention for sentences in `{args.infile}`.')
parser = load_model(args.checkpoint)
with open(args.infile, 'r') as f:
lines = [line.strip() for line in f.readlines()]
lines = lines[:args.max_lines]
if is_tree(lines[0]):
sentences = [fromstring(line).words() for line in lines]
else:
sentences = [line.split() for line in lines]
def inspect_after_reduce(parser):
subtree = parser.stack._stack[-1].subtree
head = subtree.label
children = [
child.label if isinstance(child, InternalNode) else child.word
for child in subtree.children
]
attention = parser.composer._attn
gate = np.mean(parser.composer._gate)
attention = [attention] if not isinstance(
attention, list) else attention # in case .value() returns a float
attentive = [
f'{child} ({attn:.2f})'
for child, attn in zip(children, attention)
]
print(' ', head, '|', ' '.join(attentive), f'[{gate:.2f}]')
def parse_with_inspection(parser, words):
parser.eval()
nll = 0.
word_ids = [parser.word_vocab.index_or_unk(word) for word in words]
parser.initialize(word_ids)
while not parser.stack.is_finished():
u = parser.parser_representation()
action_logits = parser.f_action(u)
action_id = np.argmax(action_logits.value() +
parser._add_actions_mask())
nll += dy.pickneglogsoftmax(action_logits, action_id)
parser.parse_step(action_id)
if action_id == parser.REDUCE_ID:
inspect_after_reduce(parser)
tree = parser.get_tree()
tree.substitute_leaves(iter(words)) # replaces UNKs with originals
return tree, nll
for sentence in sentences:
tree, _ = parser.parse(sentence)
print('>', ' '.join(sentence))
print('>', tree.linearize(with_tag=False))
parse_with_inspection(parser, sentence)
print()
def sample_generative(args):
print('Sampling from the generative model.')
parser = load_model(args.checkpoint)
print('Samples:')
for i in range(args.num_samples):
tree, logprob = parser.sample()
print('>', tree.linearize(with_tag=False))
print()
def predict_input_crf(args):
print('Predicting with crf parser.')
parser = load_model(args.checkpoint)
##
right_branching = fromstring(
"(S (NP (@ The) (@ (@ other) (@ (@ hungry) (@ cat)))) (@ (VP meows ) (@ .)))"
).convert()
left_branching = fromstring(
"(S (NP (@ The) (@ (@ (@ other) (@ hungry)) (@ cat))) (@ (VP meows ) (@ .)))"
).convert()
# right_branching = fromstring("(X (X (@ The) (@ (@ other) (@ (@ hungry) (@ cat)))) (@ (X meows ) (@ .)))").convert()
# left_branching = fromstring("(X (X (@ The) (@ (@ (@ other) (@ hungry)) (@ cat))) (@ (X meows ) (@ .)))").convert()
right_nll = parser.forward(right_branching, is_train=False)
left_nll = parser.forward(left_branching, is_train=False)
print('Right:', right_nll.value())
print('Left:', left_nll.value())
##
while True:
sentence = input('Input a sentence: ')
words = sentence.split()
print('Processed:', ' '.join(parser.word_vocab.process(words)))
print()
print('Parse:')
tree, nll = parser.parse(words)
print(' {} {:.3f}'.format(tree.linearize(with_tag=False),
nll.value()))
print()
print('Samples:')
parse, parse_logprob, samples, entropy = parser.parse_sample_entropy(
words, num_samples=8, alpha=1)
for tree, nll in samples:
print(' {} {:.3f}'.format(tree.linearize(with_tag=False),
nll.value()))
# print(' {} ||| {} ||| {:.3f}'.format(
# tree.convert().linearize(with_tag=False), tree.un_cnf().linearize(with_tag=False), nll.value()))
print()
print('Parse (temperature {}):'.format(args.alpha))
print(' {} {:.3f}'.format(parse.linearize(with_tag=False),
-parse_logprob.value()))
print()
print('Entropy:')
print(' {:.3f}'.format(entropy.value()))
print('-' * 79)
print()
def predict_entropy(args):
print(
f'Predicting entropy for lines in `{args.infile}`, writing to `{args.outfile}`...'
)
print(f'Loading model from `{args.checkpoint}`.')
print(f'Using {args.num_samples} samples.')
parser = load_model(args.checkpoint)
parser.eval()
with open(args.infile, 'r') as f:
lines = [line.strip() for line in f.readlines()]
if is_tree(lines[0]):
sentences = [fromstring(line.strip()).words() for line in lines]
else:
sentences = [line.strip().split() for line in lines]
with open(args.outfile, 'w') as f:
print('id',
'entropy',
'num-samples',
'model',
'file',
file=f,
sep='\t')
for i, words in enumerate(tqdm(sentences)):
dy.renew_cg()
if args.num_samples == 0:
assert args.model_type == 'crf', 'exact computation only for crf.'
entropy = parser.entropy(words)
else:
if args.model_type == 'crf':
samples = parser.sample(words,
num_samples=args.num_samples)
if args.num_samples == 1:
samples = [samples]
else:
samples = [
parser.sample(words, alpha=args.alpha)
for _ in range(args.num_samples)
]
trees, nlls = zip(*samples)
entropy = dy.esum(list(nlls)) / len(nlls)
print(i,
entropy.value(),
args.num_samples,
args.model_type,
args.infile,
file=f,
sep='\t')
def predict_text_file(args):
assert os.path.exists(args.infile), 'specifiy file to parse with --infile.'
print(f'Predicting trees for lines in `{args.infile}`.')
with open(args.infile, 'r') as f:
lines = [line.strip().split() for line in f.readlines()]
if args.model_type == 'disc-rnng':
print('Predicting with discriminative RNNG.')
parser = load_model(args.checkpoint)
elif args.model_type == 'crf':
print('Predicting with CRF parser.')
parser = load_model(args.checkpoint)
elif args.model_type == 'gen-rnng':
print('Predicting with generative RNNG.')
model = load_model(args.checkpoint)
parser = GenerativeDecoder(model=model)
if args.proposal_model:
parser.load_proposal_model(args.proposal_model)
elif args.proposal_samples:
parser.load_proposal_samples(args.proposal_samples)
else:
raise ValueError('Specify proposals.')
else:
raise ValueError('Specify model-type.')
print(f'Predicting trees for `{args.infile}`...')
trees = []
for words in tqdm(lines):
dy.renew_cg()
tree, *rest = parser.parse(words)
trees.append(tree.linearize(with_tag=True))
print(f'Saved predicted trees in `{args.outfile}`.')
with open(args.outfile, 'w') as f:
print('\n'.join(trees), file=f)
def predict_perplexity_from_samples(args):
print('Predicting perplexity with Generative RNNG.')
print(f'Loading model from `{args.checkpoint}`.')
print(f'Loading proposal samples from `{args.proposal_samples}`.')
print(f'Loading lines from directory `{args.infile}`.')
print(f'Writing predictions to `{args.outfile}`.')
np.random.seed(args.numpy_seed)
model = load_model(args.checkpoint)
decoder = GenerativeDecoder(model=model,
num_samples=args.num_samples,
alpha=args.alpha)
print('Computing perplexity...')
trees, perplexity = decoder.predict_from_proposal_samples(
args.proposal_samples)
# Compute f-score from trees
base_name = os.path.splitext(args.outfile)[0]
pred_path = base_name + '.trees'
result_path = base_name + '.result'
with open(pred_path, 'w') as f:
print('\n'.join(trees), file=f)
fscore = evalb(args.evalb_dir, pred_path, args.infile, result_path)
print(f'Results: {fscore} fscore, {perplexity} perplexity.')
with open(args.outfile, 'w') as f:
print('proposals',
'perplexity',
'fscore',
'num-samples',
'temp',
'seed',
sep='\t',
file=f)
print(args.proposal_samples,
perplexity,
fscore,
args.num_samples,
args.alpha,
args.numpy_seed,
sep='\t',
file=f)
def predict_tree_file(args):
assert os.path.exists(args.infile), 'specifiy file to parse with --infile.'
print(f'Predicting trees for lines in `{args.infile}`.')
with open(args.infile, 'r') as f:
lines = [
fromstring(line.strip()).words() for line in f if line.strip()
]
if args.model_type == 'disc':
print('Loading discriminative model...')
parser = load_model(args.checkpoint)
parser.eval()
print('Done.')
elif args.model_type == 'gen':
exit('Not yet...')
print('Loading generative model...')
parser = GenerativeDecoder()
parser.load_model(path=args.checkpoint)
if args.proposal_model:
parser.load_proposal_model(path=args.proposal_model)
if args.proposal_samples:
parser.load_proposal_samples(path=args.proposal_samples)
trees = []
for line in tqdm(lines):
tree, _ = parser.parse(line)
trees.append(tree.linearize())
pred_path = os.path.join(args.outfile)
result_path = args.outfile + '.results'
# Save the predicted trees.
with open(pred_path, 'w') as f:
print('\n'.join(trees), file=f)
# Score the trees.
fscore = evalb(args.evalb_dir, pred_path, args.infile, result_path)
print(
f'Predictions saved in `{pred_path}`. Results saved in `{result_path}`.'
)
print(f'F-score {fscore:.2f}.')
def sample_proposals(args):
assert os.path.exists(args.infile), 'specifiy file to parse with --infile.'
print(f'Sampling proposal trees for sentences in `{args.infile}`.')
with open(args.infile, 'r') as f:
lines = [line.strip() for line in f.readlines()]
if is_tree(lines[0]):
sentences = [fromstring(line).words() for line in lines]
else:
sentences = [line.split() for line in lines]
parser = load_model(args.checkpoint)
samples = []
if args.model_type == 'crf':
for i, words in enumerate(tqdm(sentences)):
dy.renew_cg()
for tree, nll in parser.sample(words,
num_samples=args.num_samples):
samples.append(' ||| '.join((str(i), str(-nll.value()),
tree.linearize(with_tag=False))))
print(' ||| '.join((str(i), str(-nll.value()),
tree.linearize(with_tag=False))))
else:
for i, words in enumerate(tqdm(sentences)):
for _ in range(args.num_samples):
dy.renew_cg()
tree, nll = parser.sample(words, alpha=args.alpha)
samples.append(' ||| '.join((str(i), str(-nll.value()),
tree.linearize(with_tag=False))))
with open(args.outfile, 'w') as f:
print('\n'.join(samples), file=f, end='')
def main():
args = get_args()
torch.set_grad_enabled(False)
if args.network == "mobile0.25":
cfg = cfg_mnet
elif args.network == "resnet50":
cfg = cfg_re50
else:
raise NotImplementedError(
f"Only mobile0.25 and resnet50 are suppoted.")
# net and model
net = RetinaFace(cfg=cfg, phase="test")
net = load_model(net, args.trained_model, args.cpu)
net.eval()
if args.fp16:
net = net.half()
print("Finished loading model!")
cudnn.benchmark = True
device = torch.device("cpu" if args.cpu else "cuda")
net = net.to(device)
file_paths = sorted(args.input_path.rglob("*.jpg"))
if args.num_gpu is not None:
start, end = split_array(len(file_paths), args.num_gpu, args.gpu_id)
file_paths = file_paths[start:end]
output_path = args.output_path
if args.save_boxes:
output_label_path = output_path / "labels"
output_label_path.mkdir(exist_ok=True, parents=True)
if args.save_crops:
output_image_path = output_path / "images"
output_image_path.mkdir(exist_ok=True, parents=True)
transform = albu.Compose([
albu.Normalize(
p=1, mean=(104, 117, 123), std=(1.0, 1.0, 1.0), max_pixel_value=1)
],
p=1)
test_loader = DataLoader(
InferenceDataset(file_paths, args.origin_size, transform=transform),
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
drop_last=False,
)
with torch.no_grad():
for raw_input in tqdm(test_loader):
torched_images = raw_input["torched_image"]
if args.fp16:
torched_images = torched_images.half()
resizes = raw_input["resize"]
image_paths = Path(raw_input["image_path"])
raw_images = raw_input["raw_image"]
labels = []
if (args.batch_size == 1 and args.save_boxes
and (output_label_path /
f"{Path(image_paths[0]).stem}.json").exists()):
continue
loc, conf, land = net(torched_images.to(device)) # forward pass
batch_size = torched_images.shape[0]
image_height, image_width = torched_images.shape[2:]
scale1 = torch.Tensor([
image_width,
image_height,
image_width,
image_height,
image_width,
image_height,
image_width,
image_height,
image_width,
image_height,
])
scale1 = scale1.to(device)
scale = torch.Tensor(
[image_width, image_height, image_width, image_height])
scale = scale.to(device)
priorbox = PriorBox(cfg, image_size=(image_height, image_width))
priors = priorbox.forward()
priors = priors.to(device)
prior_data = priors.data
for batch_id in range(batch_size):
image_path = image_paths[batch_id]
file_id = Path(image_path).stem
raw_image = raw_images[batch_id]
resize = resizes[batch_id].float()
boxes = decode(loc.data[batch_id], prior_data, cfg["variance"])
boxes *= scale / resize
scores = conf[batch_id][:, 1]
landmarks = decode_landm(land.data[batch_id], prior_data,
cfg["variance"])
landmarks *= scale1 / resize
# ignore low scores
valid_index = torch.where(
scores > args.confidence_threshold)[0]
boxes = boxes[valid_index]
landmarks = landmarks[valid_index]
scores = scores[valid_index]
order = scores.argsort(descending=True)
boxes = boxes[order]
landmarks = landmarks[order]
scores = scores[order]
# do NMS
keep = nms(boxes, scores, args.nms_threshold)
boxes = boxes[keep, :].int()
landmarks = landmarks[keep].int()
if boxes.shape[0] == 0:
continue
scores = scores[keep].cpu().numpy().astype(np.float64)
for crop_id, bbox in enumerate(boxes):
bbox = bbox.cpu().numpy()
labels += [{
"crop_id": crop_id,
"bbox": bbox.tolist(),
"score": scores[crop_id],
"landmarks": landmarks[crop_id].tolist(),
}]
if args.save_crops:
x_min, y_min, x_max, y_max = bbox
x_min = max(0, x_min)
y_min = max(0, y_min)
crop = raw_image[y_min:y_max,
x_min:x_max].cpu().numpy()
target_folder = output_image_path / f"{file_id}"
target_folder.mkdir(exist_ok=True, parents=True)
crop_file_path = target_folder / f"{file_id}_{crop_id}.jpg"
if crop_file_path.exists():
continue
cv2.imwrite(
str(crop_file_path),
cv2.cvtColor(crop, cv2.COLOR_BGR2RGB),
[int(cv2.IMWRITE_JPEG_QUALITY), 90],
)
if args.save_boxes:
result = {
"file_path": image_path,
"file_id": file_id,
"bboxes": labels,
}
with open(output_label_path / f"{file_id}.json", "w") as f:
json.dump(result, f, indent=2)
def main():
args = get_args()
torch.set_grad_enabled(False)
if args.network == "mobile0.25":
cfg = cfg_mnet
elif args.network == "resnet50":
cfg = cfg_re50
else:
raise NotImplementedError(f"Only mobile0.25 and resnet50 are suppoted.")
# net and model
net = RetinaFace(cfg=cfg, phase="test")
net = load_model(net, args.trained_model, args.cpu)
net.eval()
if args.fp16:
net = net.half()
print("Finished loading model!")
cudnn.benchmark = True
device = torch.device("cpu" if args.cpu else "cuda")
net = net.to(device)
file_paths = sorted(args.input_path.rglob("*.mp4"))[: args.num_videos]
if args.num_gpu is not None:
start, end = split_array(len(file_paths), args.num_gpu, args.gpu_id)
file_paths = file_paths[start:end]
output_path = args.output_path
if args.save_boxes:
output_label_path = output_path / "labels"
output_label_path.mkdir(exist_ok=True, parents=True)
if args.save_crops:
output_image_path = output_path / "images"
output_image_path.mkdir(exist_ok=True, parents=True)
if args.video_decoder == "cpu":
decode_device = cpu(0)
elif args.video_decoder == "gpu":
decode_device = gpu(0)
else:
raise NotImplementedError(f"Only CPU and GPU devices are supported by decard, but got {args.video_decoder}")
transform = albu.Compose([albu.Normalize(p=1, mean=(104, 117, 123), std=(1.0, 1.0, 1.0), max_pixel_value=1)], p=1)
with torch.no_grad():
for video_path in tqdm(file_paths):
labels = []
video_id = video_path.stem
with video_reader(str(video_path), ctx=decode_device) as video:
len_video = len(video)
if args.num_frames is None or args.num_frames == 1:
frame_ids = list(range(args.num_frames))
elif args.num_frames > 1:
if len_video < args.num_frames:
step = 1
else:
step = int(len_video / args.num_frames)
frame_ids = list(range(0, len_video, step))[: args.num_frames]
else:
raise ValueError(f"Expect None or integer > 1 for args.num_frames, but got {args.num_frames}")
frames = video.get_batch(frame_ids)
if args.video_decoder == "cpu":
frames = frames.asnumpy()
elif args.video_decoder == "gpu":
frames = dlpack.from_dlpack(frames.to_dlpack())
if args.video_decoder == "gpu":
del video
torch.cuda.empty_cache()
gc.collect()
num_frames = len(frames)
image_height = frames.shape[1]
image_width = frames.shape[2]
scale1 = torch.Tensor(
[
image_width,
image_height,
image_width,
image_height,
image_width,
image_height,
image_width,
image_height,
image_width,
image_height,
]
)
scale1 = scale1.to(device)
scale = torch.Tensor([image_width, image_height, image_width, image_height])
scale = scale.to(device)
priorbox = PriorBox(cfg, image_size=(image_height, image_width))
priors = priorbox.forward()
priors = priors.to(device)
prior_data = priors.data
if args.resize_coeff is not None:
target_size = min(args.resize_coeff)
max_size = max(args.resize_coeff)
image_height = frames.shape[1]
image_width = frames.shape[2]
image_size_min = min([image_width, image_height])
image_size_max = max([image_width, image_height])
resize = float(target_size) / float(image_size_min)
if np.round(resize * image_size_max) > max_size:
resize = float(max_size) / float(image_size_max)
else:
resize = 1
for pred_id in range(num_frames):
frame = frames[pred_id]
torched_image = prepare_image(frame, transform, args.video_decoder).to(device)
if args.fp16:
torched_image = torched_image.half()
loc, conf, land = net(torched_image) # forward pass
frame_id = frame_ids[pred_id]
boxes = decode(loc.data[0], prior_data, cfg["variance"])
boxes *= scale / resize
boxes = boxes.cpu().numpy()
scores = conf[0].data.cpu().numpy()[:, 1]
landmarks = decode_landm(land.data[0], prior_data, cfg["variance"])
landmarks *= scale1 / resize
landmarks = landmarks.cpu().numpy()
# ignore low scores
valid_index = np.where(scores > args.confidence_threshold)[0]
boxes = boxes[valid_index]
landmarks = landmarks[valid_index]
scores = scores[valid_index]
# keep top-K before NMS
order = scores.argsort()[::-1]
# order = scores.argsort()[::-1][:args.top_k]
boxes = boxes[order]
landmarks = landmarks[order]
scores = scores[order]
# do NMS
detection = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)
keep = py_cpu_nms(detection, args.nms_threshold)
# keep = nms(detection, args.nms_threshold,force_cpu=args.cpu)
# x_min, y_min, x_max, y_max, score
detection = detection[keep, :]
landmarks = landmarks[keep].astype(int)
if detection.shape[0] == 0:
continue
bboxes = detection[:, :4].astype(int)
confidence = detection[:, 4].astype(np.float64)
for crop_id in range(len(detection)):
bbox = bboxes[crop_id]
labels += [
{
"frame_id": int(frame_id),
"crop_id": crop_id,
"bbox": bbox.tolist(),
"score": confidence[crop_id],
"landmarks": landmarks[crop_id].tolist(),
}
]
if args.save_crops:
x_min, y_min, x_max, y_max = bbox
x_min = max(0, x_min)
y_min = max(0, y_min)
crop = frame[y_min:y_max, x_min:x_max]
target_folder = output_image_path / f"{video_id}"
target_folder.mkdir(exist_ok=True, parents=True)
crop_file_path = target_folder / f"{frame_id}_{crop_id}.jpg"
if crop_file_path.exists():
continue
cv2.imwrite(
str(crop_file_path),
cv2.cvtColor(crop, cv2.COLOR_BGR2RGB),
[int(cv2.IMWRITE_JPEG_QUALITY), 90],
)
if args.save_boxes:
result = {
"file_path": str(video_path),
"file_id": video_id,
"bboxes": labels,
}
with open(output_label_path / f"{video_id}.json", "w") as f:
json.dump(result, f, indent=2)
def main():
args = get_args()
torch.set_grad_enabled(False)
cfg = None
if args.network == "mobile0.25":
cfg = cfg_mnet
elif args.network == "resnet50":
cfg = cfg_re50
# net and model
net = RetinaFace(cfg=cfg, phase="test")
net = load_model(net, args.trained_model, args.cpu)
net.eval()
print("Finished loading model!")
print(net)
cudnn.benchmark = True
device = torch.device("cpu" if args.cpu else "cuda")
net = net.to(device)
# testing dataset
testset_folder = args.dataset_folder
testset_list = args.dataset_folder[:-7] + "wider_val.txt"
with open(testset_list, "r") as fr:
test_dataset = fr.read().split()
num_images = len(test_dataset)
_t = {"forward_pass": Timer(), "misc": Timer()}
# testing begin
for i, img_name in enumerate(test_dataset):
image_path = testset_folder + img_name
img_raw = cv2.imread(image_path, cv2.IMREAD_COLOR)
img = np.float32(img_raw)
# testing scale
target_size = 1600
max_size = 2150
im_shape = img.shape
im_size_min = np.min(im_shape[0:2])
im_size_max = np.max(im_shape[0:2])
resize = float(target_size) / float(im_size_min)
# prevent bigger axis from being more than max_size:
if np.round(resize * im_size_max) > max_size:
resize = float(max_size) / float(im_size_max)
if args.origin_size:
resize = 1
if resize != 1:
img = cv2.resize(img,
None,
None,
fx=resize,
fy=resize,
interpolation=cv2.INTER_LINEAR)
im_height, im_width, _ = img.shape
scale = torch.Tensor(
[img.shape[1], img.shape[0], img.shape[1], img.shape[0]])
img -= (104, 117, 123)
img = img.transpose(2, 0, 1)
img = torch.from_numpy(img).unsqueeze(0)
img = img.to(device)
scale = scale.to(device)
_t["forward_pass"].tic()
loc, conf, landms = net(img) # forward pass
_t["forward_pass"].toc()
_t["misc"].tic()
priorbox = PriorBox(cfg, image_size=(im_height, im_width))
priors = priorbox.forward()
priors = priors.to(device)
prior_data = priors.data
boxes = decode(loc.data.squeeze(0), prior_data, cfg["variance"])
boxes = boxes * scale / resize
boxes = boxes.cpu().numpy()
scores = conf.squeeze(0).data.cpu().numpy()[:, 1]
landms = decode_landm(landms.data.squeeze(0), prior_data,
cfg["variance"])
scale1 = torch.Tensor([
img.shape[3],
img.shape[2],
img.shape[3],
img.shape[2],
img.shape[3],
img.shape[2],
img.shape[3],
img.shape[2],
img.shape[3],
img.shape[2],
])
scale1 = scale1.to(device)
landms = landms * scale1 / resize
landms = landms.cpu().numpy()
# ignore low scores
inds = np.where(scores > args.confidence_threshold)[0]
boxes = boxes[inds]
landms = landms[inds]
scores = scores[inds]
# keep top-K before NMS
order = scores.argsort()[::-1]
# order = scores.argsort()[::-1][:args.top_k]
boxes = boxes[order]
landms = landms[order]
scores = scores[order]
# do NMS
dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32,
copy=False)
keep = py_cpu_nms(dets, args.nms_threshold)
# keep = nms(dets, args.nms_threshold,force_cpu=args.cpu)
dets = dets[keep, :]
landms = landms[keep]
# keep top-K faster NMS
# dets = dets[:args.keep_top_k, :]
# landms = landms[:args.keep_top_k, :]
dets = np.concatenate((dets, landms), axis=1)
_t["misc"].toc()
# --------------------------------------------------------------------
save_name = args.save_folder + img_name[:-4] + ".txt"
dirname = os.path.dirname(save_name)
if not os.path.isdir(dirname):
os.makedirs(dirname)
with open(save_name, "w") as fd:
bboxs = dets
file_name = os.path.basename(save_name)[:-4] + "\n"
bboxs_num = str(len(bboxs)) + "\n"
fd.write(file_name)
fd.write(bboxs_num)
for box in bboxs:
x = int(box[0])
y = int(box[1])
w = int(box[2]) - int(box[0])
h = int(box[3]) - int(box[1])
confidence = str(box[4])
line = str(x) + " " + str(y) + " " + str(w) + " " + str(
h) + " " + confidence + " \n"
fd.write(line)
print("im_detect: {:d}/{:d} forward_pass_time: {:.4f}s misc: {:.4f}s".
format(i + 1, num_images, _t["forward_pass"].average_time,
_t["misc"].average_time))
# save image
if args.save_image:
for b in dets:
if b[4] < args.vis_thres:
continue
text = "{:.4f}".format(b[4])
b = list(map(int, b))
cv2.rectangle(img_raw, (b[0], b[1]), (b[2], b[3]), (0, 0, 255),
2)
cx = b[0]
cy = b[1] + 12
cv2.putText(img_raw, text, (cx, cy), cv2.FONT_HERSHEY_DUPLEX,
0.5, (255, 255, 255))
# landms
cv2.circle(img_raw, (b[5], b[6]), 1, (0, 0, 255), 4)
cv2.circle(img_raw, (b[7], b[8]), 1, (0, 255, 255), 4)
cv2.circle(img_raw, (b[9], b[10]), 1, (255, 0, 255), 4)
cv2.circle(img_raw, (b[11], b[12]), 1, (0, 255, 0), 4)
cv2.circle(img_raw, (b[13], b[14]), 1, (255, 0, 0), 4)
# save image
if not os.path.exists("./results/"):
os.makedirs("./results/")
name = "./results/" + str(i) + ".jpg"
cv2.imwrite(name, img_raw)
def syneval_rnn(args):
model = load_model(args.checkpoint)
model.eval()
files = SHORT if args.syneval_short else ALL
outdir = os.path.join(args.checkpoint, 'output')
if not os.path.exists(outdir):
os.makedirs(outdir)
result_filename = 'syneval_results_caps.tsv' if args.capitalize else 'syneval_results.tsv'
outpath = os.path.join(outdir, result_filename)
print('Predicting syneval with RNN language model.')
print(f'Loading model from `{args.checkpoint}`.')
print(f'Loading syneval examples from directory `{args.indir}`.')
print(f'Writing predictions to `{outpath}`.')
with open(outpath, 'w') as outfile:
print('\t'.join(
('name', 'index', 'pos-perplexity', 'neg-perplexity', 'correct',
'pos-sentence-processed', 'neg-sentence-processed')),
file=outfile)
print('Predicting syneval for:', '\n', '\n '.join(files))
for fname in files:
print(f'Predicting `{fname}`...')
inpath = os.path.join(args.indir, fname)
with open(inpath + '.pos') as f:
pos_sents = [line.strip() for line in f.readlines()]
if args.capitalize:
pos_sents = [sent.capitalize() for sent in pos_sents]
if args.add_period:
pos_sents = [sent + ' .' for sent in pos_sents]
with open(inpath + '.neg') as f:
neg_sents = [line.strip() for line in f.readlines()]
if args.capitalize:
neg_sents = [sent.capitalize() for sent in neg_sents]
if args.add_period:
neg_sents = [sent + ' .' for sent in neg_sents]
pos_sents = [sent.split() for sent in pos_sents]
neg_sents = [sent.split() for sent in neg_sents]
assert len(pos_sents) == len(neg_sents)
if args.syneval_max_lines != -1 and len(
pos_sents) > args.syneval_max_lines:
subsampled_ids = np.random.choice(len(pos_sents),
args.syneval_max_lines,
replace=False)
pos_sents = [pos_sents[i] for i in subsampled_ids]
neg_sents = [neg_sents[i] for i in subsampled_ids]
num_correct = 0
for i, (pos,
neg) in enumerate(tqdm(list(zip(pos_sents, neg_sents)))):
dy.renew_cg()
pos_pp = np.exp(model.forward(pos).value() / len(pos))
neg_pp = np.exp(model.forward(neg).value() / len(neg))
correct = pos_pp < neg_pp
num_correct += correct
# see which words are unked during prediction
pos = model.word_vocab.process(pos)
neg = model.word_vocab.process(neg)
result = '\t'.join(
(fname, str(i), str(round(pos_pp,
2)), str(round(neg_pp, 2)),
str(int(correct)), ' '.join(pos), ' '.join(neg)))
print(result, file=outfile)
print(
f'{fname}: {num_correct}/{len(pos_sents)} = {num_correct / len(pos_sents):.2%} correct',
'\n')
def main():
args = get_args()
torch.set_grad_enabled(False)
cfg = None
if args.network == "mobile0.25":
cfg = cfg_mnet
elif args.network == "resnet50":
cfg = cfg_re50
# net and model
net = RetinaFace(cfg=cfg, phase="test")
net = load_model(net, args.trained_model, args.cpu)
net.eval()
print("Finished loading model!")
print(net)
cudnn.benchmark = True
device = torch.device("cpu" if args.cpu else "cuda")
net = net.to(device)
args.save_folder.mkdir(exist_ok=True)
fw = open(os.path.join(args.save_folder, args.dataset + "_dets.txt"), "w")
# testing dataset
testset_folder = os.path.join("data", args.dataset, "images/")
testset_list = os.path.join("data", args.dataset, "img_list.txt")
with open(testset_list, "r") as fr:
test_dataset = fr.read().split()
num_images = len(test_dataset)
# testing scale
resize = 1
_t = {"forward_pass": Timer(), "misc": Timer()}
# testing begin
for i, img_name in enumerate(test_dataset):
image_path = testset_folder + img_name + ".jpg"
img_raw = cv2.imread(image_path, cv2.IMREAD_COLOR)
img = np.float32(img_raw)
if resize != 1:
img = cv2.resize(img,
None,
None,
fx=resize,
fy=resize,
interpolation=cv2.INTER_LINEAR)
im_height, im_width, _ = img.shape
scale = torch.Tensor(
[img.shape[1], img.shape[0], img.shape[1], img.shape[0]])
img -= (104, 117, 123)
img = img.transpose(2, 0, 1)
img = torch.from_numpy(img).unsqueeze(0)
img = img.to(device)
scale = scale.to(device)
_t["forward_pass"].tic()
loc, conf, landms = net(img) # forward pass
_t["forward_pass"].toc()
_t["misc"].tic()
priorbox = PriorBox(cfg, image_size=(im_height, im_width))
priors = priorbox.forward()
priors = priors.to(device)
prior_data = priors.data
boxes = decode(loc.data.squeeze(0), prior_data, cfg["variance"])
boxes = boxes * scale / resize
boxes = boxes.cpu().numpy()
scores = conf.squeeze(0).data.cpu().numpy()[:, 1]
landms = decode_landm(landms.data.squeeze(0), prior_data,
cfg["variance"])
scale1 = torch.Tensor([
img.shape[3],
img.shape[2],
img.shape[3],
img.shape[2],
img.shape[3],
img.shape[2],
img.shape[3],
img.shape[2],
img.shape[3],
img.shape[2],
])
scale1 = scale1.to(device)
landms = landms * scale1 / resize
landms = landms.cpu().numpy()
# ignore low scores
inds = np.where(scores > args.confidence_threshold)[0]
boxes = boxes[inds]
landms = landms[inds]
scores = scores[inds]
# keep top-K before NMS
# order = scores.argsort()[::-1][:args.top_k]
order = scores.argsort()[::-1]
boxes = boxes[order]
landms = landms[order]
scores = scores[order]
# do NMS
dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32,
copy=False)
keep = py_cpu_nms(dets, args.nms_threshold)
dets = dets[keep, :]
landms = landms[keep]
# keep top-K faster NMS
# dets = dets[:args.keep_top_k, :]
# landms = landms[:args.keep_top_k, :]
dets = np.concatenate((dets, landms), axis=1)
_t["misc"].toc()
# save dets
if args.dataset == "FDDB":
fw.write("{:s}\n".format(img_name))
fw.write("{:.1f}\n".format(dets.shape[0]))
for k in range(dets.shape[0]):
xmin = dets[k, 0]
ymin = dets[k, 1]
xmax = dets[k, 2]
ymax = dets[k, 3]
score = dets[k, 4]
w = xmax - xmin + 1
h = ymax - ymin + 1
# fw.write('{:.3f} {:.3f} {:.3f} {:.3f} {:.10f}\n'.format(xmin, ymin, w, h, score))
fw.write("{:d} {:d} {:d} {:d} {:.10f}\n".format(
int(xmin), int(ymin), int(w), int(h), score))
print("im_detect: {:d}/{:d} forward_pass_time: {:.4f}s misc: {:.4f}s".
format(i + 1, num_images, _t["forward_pass"].average_time,
_t["misc"].average_time))
# show image
if args.save_image:
for b in dets:
if b[4] < args.vis_thres:
continue
text = "{:.4f}".format(b[4])
b = list(map(int, b))
cv2.rectangle(img_raw, (b[0], b[1]), (b[2], b[3]), (0, 0, 255),
2)
cx = b[0]
cy = b[1] + 12
cv2.putText(img_raw, text, (cx, cy), cv2.FONT_HERSHEY_DUPLEX,
0.5, (255, 255, 255))
# landms
cv2.circle(img_raw, (b[5], b[6]), 1, (0, 0, 255), 4)
cv2.circle(img_raw, (b[7], b[8]), 1, (0, 255, 255), 4)
cv2.circle(img_raw, (b[9], b[10]), 1, (255, 0, 255), 4)
cv2.circle(img_raw, (b[11], b[12]), 1, (0, 255, 0), 4)
cv2.circle(img_raw, (b[13], b[14]), 1, (255, 0, 0), 4)
# save image
if not os.path.exists("./results/"):
os.makedirs("./results/")
name = "./results/" + str(i) + ".jpg"
cv2.imwrite(name, img_raw)
fw.close()
def process_video_files(
network: str,
trained_model: str,
decode_gpu: bool,
is_fp16: bool,
file_paths: list,
num_gpu: Optional[int],
gpu_id: int,
output_path: Path,
is_save_boxes: bool,
is_save_crops: bool,
num_frames: int,
resize_coeff: Optional[Tuple],
confidence_threshold: float,
num_workers: int,
nms_threshold: float,
batch_size: int,
resize_scale: float,
min_size: int,
keep_top_k: int,
) -> None:
torch.set_grad_enabled(False)
if network == "mobile0.25":
cfg = cfg_mnet_test
elif network == "resnet50":
cfg = cfg_re50_test
else:
raise NotImplementedError(
f"Only mobile0.25 and resnet50 are suppoted, but we got {network}")
if min_size < 0:
raise ValueError(
f"Min size should be positive, but we got {min_size}.")
# net and model
net = RetinaFace(cfg=cfg, phase="test")
net = load_model(net, trained_model, load_to_cpu=False)
net.eval()
if is_fp16:
net = net.half()
device = torch.device("cuda")
net.to(device)
print("Finished loading model!")
cudnn.benchmark = True
transform = albu.Compose([
albu.Normalize(
p=1, mean=(104, 117, 123), std=(1.0, 1.0, 1.0), max_pixel_value=1)
],
p=1)
if num_gpu is not None:
start, end = split_array(len(file_paths), num_gpu, gpu_id)
file_paths = file_paths[start:end]
with torch.no_grad():
func = partial(get_frames,
num_frames=num_frames,
resize_coeff=resize_coeff,
transform=transform,
decode_gpu=decode_gpu)
with torch.no_grad():
with concurrent.futures.ProcessPoolExecutor(
num_workers) as executor:
for result in tqdm(executor.map(func, file_paths),
total=len(file_paths),
leave=False,
desc="Loading data files"):
if len(result) != 0:
result["is_fp16"] = is_fp16
result["device"] = device
result["batch_size"] = batch_size
result["cfg"] = cfg
result["nms_threshold"] = nms_threshold
result["confidence_threshold"] = confidence_threshold
result["is_save_crops"] = is_save_crops
result["is_save_boxes"] = is_save_boxes
result["output_path"] = output_path
result["net"] = net
result["min_size"] = min_size
result["resize_scale"] = resize_scale
result["keep_top_k"] = keep_top_k
process_frames(**result)
def syneval_parser(args):
model = load_model(args.checkpoint)
files = SHORT if args.syneval_short else ALL
outdir = os.path.join(args.checkpoint, 'output')
if not os.path.exists(outdir):
os.makedirs(outdir)
result_filename = 'syneval_results_caps.tsv' if args.capitalize else 'syneval_results.tsv'
outpath = os.path.join(outdir, result_filename)
print('Predicting syneval with discriminative parser.')
print(f'Loading model from `{args.checkpoint}`.')
print(f'Loading syneval examples from directory `{args.indir}`.')
print(f'Writing predictions to `{outpath}`.')
with open(outpath, 'w') as outfile:
if args.num_samples == 1:
# predict with logprob of predicted parse
print('\t'.join(('name', 'index', 'pos-logprob', 'neg-logprob',
'correct', 'pos-tree', 'neg-tree')),
file=outfile)
else:
print('\t'.join(('name', 'index', 'pos-entropy', 'neg-entropy',
'correct', 'pos-tree', 'neg-tree')),
file=outfile)
print('Predicting syneval for:', '\n', '\n '.join(files))
for fname in files:
print(f'Predicting `{fname}`...')
inpath = os.path.join(args.indir, fname)
with open(inpath + '.pos') as f:
pos_sents = [line.strip() for line in f.readlines()]
if args.capitalize:
pos_sents = [sent.capitalize() for sent in pos_sents]
if args.add_period:
pos_sents = [sent + ' .' for sent in pos_sents]
with open(inpath + '.neg') as f:
neg_sents = [line.strip() for line in f.readlines()]
if args.capitalize:
neg_sents = [sent.capitalize() for sent in neg_sents]
if args.add_period:
neg_sents = [sent + ' .' for sent in neg_sents]
pos_sents = [sent.split() for sent in pos_sents]
neg_sents = [sent.split() for sent in neg_sents]
assert len(pos_sents) == len(neg_sents)
if args.syneval_max_lines != -1 and len(
pos_sents) > args.syneval_max_lines:
subsampled_ids = np.random.choice(len(pos_sents),
args.syneval_max_lines,
replace=False)
pos_sents = [pos_sents[i] for i in subsampled_ids]
neg_sents = [neg_sents[i] for i in subsampled_ids]
num_correct = 0
for i, (pos,
neg) in enumerate(tqdm(list(zip(pos_sents, neg_sents)))):
dy.renew_cg()
if args.num_samples == 1:
# predict with logprob of predicted parse
pos_tree, pos_nll = model.parse(pos)
neg_tree, neg_nll = model.parse(neg)
pos_logprob = -pos_nll.value()
neg_logprob = -neg_nll.value()
pos_tree = pos_tree.linearize(with_tag=False)
neg_tree = neg_tree.linearize(with_tag=False)
correct = pos_logprob > neg_logprob
num_correct += correct
result = '\t'.join(
(fname, str(i), str(round(pos_logprob, 4)),
str(round(neg_logprob,
4)), str(int(correct)), pos_tree, neg_tree))
print(result, file=outfile)
else:
if args.exact_entropy:
pos_tree, pos_entropy = model.parse_entropy(pos)
neg_tree, neg_entropy = model.parse_entropy(neg)
pos_entropy = pos_entropy.value()
neg_entropy = neg_entropy.value()
pos_tree = pos_tree.linearize(with_tag=False)
neg_tree = neg_tree.linearize(with_tag=False)
else:
# predict with the parser's entropy
pos_trees, pos_nlls = zip(*[
model.sample(pos) for _ in range(args.num_samples)
])
neg_trees, neg_nlls = zip(*[
model.sample(neg) for _ in range(args.num_samples)
])
pos_entropy = np.mean(
[nll.value() for nll in pos_nlls])
neg_entropy = np.mean(
[nll.value() for nll in neg_nlls])
pos_tree = Counter([
tree.linearize(with_tag=False)
for tree in pos_trees
]).most_common(1)[0][0]
neg_tree = Counter([
tree.linearize(with_tag=False)
for tree in neg_trees
]).most_common(1)[0][0]
correct = pos_entropy < neg_entropy
num_correct += correct
result = '\t'.join(
(fname, str(i), str(round(pos_entropy, 4)),
str(round(neg_entropy,
4)), str(int(correct)), pos_tree, neg_tree))
print(result, file=outfile)
print(
f'{fname}: {num_correct}/{len(pos_sents)} = {num_correct / len(pos_sents):.2%} correct',
'\n')
