def main():
setup_default_logging()
args, args_text = parse_args()
if args.smoke_test:
ray.init(num_cpus=int(args.ray_num_workers))
else:
ray.init(address=args.ray_address)
CustomTrainingOperator = TrainingOperator.from_creators(
model_creator=model_creator,
optimizer_creator=optimizer_creator,
data_creator=data_creator,
loss_creator=loss_creator)
trainer = TorchTrainer(training_operator_cls=CustomTrainingOperator,
use_tqdm=True,
use_fp16=args.amp,
apex_args={"opt_level": "O1"},
config={
"args": args,
BATCH_SIZE: args.batch_size
},
num_workers=args.ray_num_workers)
if args.smoke_test:
args.epochs = 1
pbar = trange(args.epochs, unit="epoch")
for i in pbar:
trainer.train(num_steps=1 if args.smoke_test else None)
val_stats = trainer.validate(num_steps=1 if args.smoke_test else None)
pbar.set_postfix(dict(acc=val_stats["val_accuracy"]))
trainer.shutdown()
def main(args):
setup_default_logging()
logging.info('Running validation on {}'.format(args.dataset))
net_config = json.load(open(args.model))
if 'img_size' in net_config:
img_size = net_config['img_size']
else:
img_size = args.img_size
test_loader = get_dataloader(
dataset=args.dataset, data=args.data, test_batch_size=args.batch_size,
n_worker=args.workers, image_size=img_size).test
model = NATNet.build_from_config(net_config, pretrained=args.pretrained)
param_count = sum([m.numel() for m in model.parameters()])
logging.info('Model created, param count: %d' % param_count)
model = model.cuda()
criterion = torch.nn.CrossEntropyLoss().cuda()
validate(model, test_loader, criterion)
return
def main(args):
setup_default_logging()
logging.info('Running validation on {}'.format(args.dataset))
net_config = json.load(open(args.model))
if 'img_size' in net_config:
img_size = net_config['img_size']
else:
img_size = args.img_size
run_config = get_run_config(dataset=args.dataset,
data_path=args.data,
image_size=img_size,
n_epochs=0,
train_batch_size=args.batch_size,
test_batch_size=args.batch_size,
n_worker=args.workers,
valid_size=None)
model = NSGANetV2.build_from_config(net_config)
param_count = sum([m.numel() for m in model.parameters()])
logging.info('Model created, param count: %d' % param_count)
model = model.cuda()
criterion = torch.nn.CrossEntropyLoss().cuda()
validate(model, run_config.test_loader, criterion)
return
def main():
args = parser.parse_args()
model_cfgs = []
model_names = []
if os.path.isdir(args.checkpoint):
# validate all checkpoints in a path with same model
checkpoints = glob.glob(args.checkpoint + '/*.pth.tar')
checkpoints += glob.glob(args.checkpoint + '/*.pth')
model_names = list_models(args.model)
model_cfgs = [(args.model, c) for c in sorted(checkpoints, key=natural_key)]
else:
if args.model == 'all':
# validate all models in a list of names with pretrained checkpoints
args.pretrained = True
model_names = list_models(pretrained=True, exclude_filters=['*in21k'])
model_cfgs = [(n, '') for n in model_names]
elif not is_model(args.model):
# model name doesn't exist, try as wildcard filter
model_names = list_models(args.model)
model_cfgs = [(n, '') for n in model_names]
setup_default_logging(log_path=os.path.join(os.path.dirname(args.checkpoint) , "eval_print_log.txt"))
if len(model_cfgs):
#results_file = args.results_file or './results-all.csv'
results_file = args.results_file or os.path.join(os.path.dirname(args.checkpoint) , "eval_results.csv.txt")
_logger.info('Running bulk validation on these pretrained models: {}'.format(', '.join(model_names)))
results = []
try:
start_batch_size = args.batch_size
for m, c in model_cfgs:
batch_size = start_batch_size
args.model = m
args.checkpoint = c
result = OrderedDict(model=args.model)
r = {}
while not r and batch_size >= args.num_gpu:
torch.cuda.empty_cache()
try:
args.batch_size = batch_size
print('Validating with batch size: %d' % args.batch_size)
r = validate(args)
except RuntimeError as e:
if batch_size <= args.num_gpu:
print("Validation failed with no ability to reduce batch size. Exiting.")
raise e
batch_size = max(batch_size // 2, args.num_gpu)
print("Validation failed, reducing batch size by 50%")
result.update(r)
if args.checkpoint:
result['checkpoint'] = args.checkpoint
results.append(result)
except KeyboardInterrupt as e:
pass
results = sorted(results, key=lambda x: x['top1'], reverse=True)
if len(results):
write_results(results_file, results)
else:
validate(args)
def main():
setup_default_logging()
args = parser.parse_args()
model_cfgs = []
model_names = []
if os.path.isdir(args.checkpoint):
# validate all checkpoints in a path with same model
checkpoints = glob.glob(args.checkpoint + '/*.pth.tar')
checkpoints += glob.glob(args.checkpoint + '/*.pth')
model_names = list_models(args.model)
model_cfgs = [(args.model, c)
for c in sorted(checkpoints, key=natural_key)]
else:
if args.model == 'all':
# validate all models in a list of names with pretrained checkpoints
args.pretrained = True
model_names = list_models(
pretrained=True,
exclude_filters=['*_in21k', '*_in22k', '*_dino'])
model_cfgs = [(n, '') for n in model_names]
elif not is_model(args.model):
# model name doesn't exist, try as wildcard filter
model_names = list_models(args.model)
model_cfgs = [(n, '') for n in model_names]
if not model_cfgs and os.path.isfile(args.model):
with open(args.model) as f:
model_names = [line.rstrip() for line in f]
model_cfgs = [(n, None) for n in model_names if n]
if len(model_cfgs):
results_file = args.results_file or './results-all.csv'
_logger.info(
'Running bulk validation on these pretrained models: {}'.format(
', '.join(model_names)))
results = []
try:
initial_batch_size = args.batch_size
for m, c in model_cfgs:
args.model = m
args.checkpoint = c
r = _try_run(args, initial_batch_size)
if 'error' in r:
continue
if args.checkpoint:
r['checkpoint'] = args.checkpoint
results.append(r)
except KeyboardInterrupt as e:
pass
results = sorted(results, key=lambda x: x['top1'], reverse=True)
if len(results):
write_results(results_file, results)
else:
results = validate(args)
# output results in JSON to stdout w/ delimiter for runner script
print(f'--result\n{json.dumps(results, indent=4)}')
def main():
setup_default_logging()
args = parser.parse_args()
model_cfgs = []
model_names = []
if args.model_list:
args.model = ''
with open(args.model_list) as f:
model_names = [line.rstrip() for line in f]
model_cfgs = [(n, None) for n in model_names]
elif args.model == 'all':
# validate all models in a list of names with pretrained checkpoints
args.pretrained = True
model_names = list_models(pretrained=True, exclude_filters=['*in21k'])
model_cfgs = [(n, None) for n in model_names]
elif not is_model(args.model):
# model name doesn't exist, try as wildcard filter
model_names = list_models(args.model)
model_cfgs = [(n, None) for n in model_names]
if len(model_cfgs):
results_file = args.results_file or './benchmark.csv'
_logger.info(
'Running bulk validation on these pretrained models: {}'.format(
', '.join(model_names)))
results = []
try:
for m, _ in model_cfgs:
if not m:
continue
args.model = m
r = benchmark(args)
if r:
results.append(r)
time.sleep(10)
except KeyboardInterrupt as e:
pass
sort_key = 'infer_samples_per_sec'
if 'train' in args.bench:
sort_key = 'train_samples_per_sec'
elif 'profile' in args.bench:
sort_key = 'infer_gmacs'
results = filter(lambda x: sort_key in x, results)
results = sorted(results, key=lambda x: x[sort_key], reverse=True)
if len(results):
write_results(results_file, results)
else:
results = benchmark(args)
# output results in JSON to stdout w/ delimiter for runner script
print(f'--result\n{json.dumps(results, indent=4)}')
def main():
setup_default_logging()
args = parser.parse_args()
model_cfgs = []
model_names = []
if os.path.isdir(args.checkpoint):
# validate all checkpoints in a path with same model
checkpoints = glob.glob(args.checkpoint + '/*.pth.tar')
checkpoints += glob.glob(args.checkpoint + '/*.pth')
model_names = list_models(args.model)
model_cfgs = [(args.model, c)
for c in sorted(checkpoints, key=natural_key)]
else:
if args.model == 'all':
# validate all models in a list of names with pretrained checkpoints
args.pretrained = True
model_names = list_models(pretrained=True)
model_cfgs = [(n, '') for n in model_names]
elif not is_model(args.model):
# model name doesn't exist, try as wildcard filter
model_names = list_models(args.model)
model_cfgs = [(n, '') for n in model_names]
if len(model_cfgs):
results_file = args.results_file or './results-all.csv'
logging.info(
'Running bulk validation on these pretrained models: {}'.format(
', '.join(model_names)))
results = []
try:
for m, c in model_cfgs:
args.model = m
args.checkpoint = c
result = OrderedDict(model=args.model)
r = validate(args)
result.update(r)
if args.checkpoint:
result['checkpoint'] = args.checkpoint
results.append(result)
except KeyboardInterrupt as e:
pass
results = sorted(results, key=lambda x: x['top1'], reverse=True)
if len(results):
write_results(results_file, results)
else:
validate(args)
def main():
setup_default_logging()
args = parser.parse_args()
model_cfgs = []
model_names = []
if os.path.isdir(args.checkpoint):
# validate all checkpoints in a path with same model
checkpoints = glob.glob(args.checkpoint + '/*.pth.tar')
checkpoints += glob.glob(args.checkpoint + '/*.pth')
model_names = list_models(args.model)
model_cfgs = [(args.model, c)
for c in sorted(checkpoints, key=natural_key)]
else:
if args.model == 'all':
# validate all models in a list of names with pretrained checkpoints
args.pretrained = True
model_names = list_models(pretrained=True)
model_cfgs = [(n, '') for n in model_names]
elif not is_model(args.model):
# model name doesn't exist, try as wildcard filter
model_names = list_models(args.model)
model_cfgs = [(n, '') for n in model_names]
if len(model_cfgs):
logging.info(
'Running bulk validation on these pretrained models: {}'.format(
', '.join(model_names)))
header_written = False
with open('./results-all.csv', mode='w') as cf:
for m, c in model_cfgs:
args.model = m
args.checkpoint = c
result = OrderedDict(model=args.model)
r = validate(args)
result.update(r)
if args.checkpoint:
result['checkpoint'] = args.checkpoint
dw = csv.DictWriter(cf, fieldnames=result.keys())
if not header_written:
dw.writeheader()
header_written = True
dw.writerow(result)
cf.flush()
else:
validate(args)
def main():
args = parser.parse_args()
setup_default_logging(log_path=args.log_path)
_logger = logging.getLogger('inference')
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# create model
model = create_model(args.model,
num_classes=args.num_classes,
in_chans=3,
checkpoint_path=args.checkpoint)
data_config = resolve_data_config(vars(args),
model=model) # 默认配置 mean, std
img_transform = create_transform(data_config['input_size'],
is_training=False,
mean=data_config['mean'],
std=data_config['std'],
crop_pct=data_config['crop_pct'])
model = model.to(device)
model.eval()
# confusion matrix
conf_matrix = np.zeros((args.num_classes, args.num_classes))
# predict
class_dir = os.listdir(args.data)
for cl in class_dir:
imgs = os.listdir(os.path.join(args.data, cl))
for img in imgs:
im = Image.open(os.path.join(args.data, cl, img))
im1 = img_transform(im).unsqueeze(0)
im1 = im1.to(device)
outputs = model(im1)
idx = torch.max(outputs, 1)[1].tolist()[0]
probs = torch.nn.functional.softmax(outputs, dim=1)[0].tolist()
_logger.info('%s: %d, %.2f' %
(os.path.join(args.data, cl, img), idx, probs[idx]))
conf_matrix[int(cl), idx] += 1
_logger.info(conf_matrix)
def main():
setup_default_logging()
args = parser.parse_args()
model_cfgs = []
model_names = []
if os.path.isdir(args.checkpoint):
# validate all checkpoints in a path with same model
checkpoints = glob.glob(args.checkpoint + '/*.pth.tar')
checkpoints += glob.glob(args.checkpoint + '/*.pth')
model_names = list_models(args.model)
model_cfgs = [(args.model, c)
for c in sorted(checkpoints, key=natural_key)]
else:
if args.model == 'all':
# validate all models in a list of names with pretrained checkpoints
args.pretrained = True
model_names = list_models(pretrained=True)
model_cfgs = [(n, '') for n in model_names]
elif not is_model(args.model):
# model name doesn't exist, try as wildcard filter
model_names = list_models(args.model)
model_cfgs = [(n, '') for n in model_names]
validate(args)
def validate(args):
setup_default_logging()
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
if args.no_redundant_bias is None:
args.redundant_bias = None
else:
args.redundant_bias = not args.no_redundant_bias
# create model
bench = create_model(
args.model,
bench_task='predict',
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'image_info_{args.anno}.json')
image_dir = 'test2017'
else:
annotation_path = os.path.join(args.data, 'annotations',
f'instances_{args.anno}.json')
image_dir = args.anno
dataset = CocoDetection(os.path.join(args.data, image_dir),
annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
pin_mem=args.pin_mem)
img_ids = []
results = []
bench.eval()
batch_time = AverageMeter()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output = bench(input, target['img_scale'], target['img_size'])
output = output.cpu()
sample_ids = target['img_id'].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
for det in sample:
score = float(det[4])
if score < .001: # stop when below this threshold, scores in descending order
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]))
img_ids.append(image_id)
results.append(coco_det)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
json.dump(results, open(args.results, 'w'), indent=4)
if 'test' not in args.anno:
coco_results = dataset.coco.loadRes(args.results)
coco_eval = COCOeval(dataset.coco, coco_results, 'bbox')
coco_eval.params.imgIds = img_ids # score only ids we've used
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return results
def validate(args):
setup_default_logging()
def setthresh():
if args.checkpoint.split("/")[-1].split(
"_")[0] in getthresholds.keys():
return getthresholds[args.checkpoint.split("/")[-1].split("_")[0]]
else:
a = []
[a.append(args.threshold) for x in range(4)]
return a
threshs = setthresh()
print(threshs)
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
# create model
bench = create_model(
args.model,
bench_task='predict',
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'image_info_{args.anno}.json')
image_dir = args.anno
else:
annotation_path = os.path.join(args.data, 'annotations',
f'instances_{args.anno}.json')
image_dir = args.anno
print(os.path.join(args.data, image_dir), annotation_path)
dataset = CocoDetection(os.path.join(args.data, image_dir),
annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
pin_mem=args.pin_mem,
mean=args.mean,
std=args.std)
if 'test' in args.anno:
threshold = float(args.threshold)
else:
threshold = .001
img_ids = []
results = []
writetofilearrtay = []
bench.eval()
batch_time = AverageMeter()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output = bench(input, target['img_scale'], target['img_size'])
output = output.cpu()
# print(target['img_id'])
sample_ids = target['img_id'].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
for det in sample:
score = float(det[4])
if score < threshold: # stop when below this threshold, scores in descending order
coco_det = dict(image_id=image_id, category_id=-1)
img_ids.append(image_id)
results.append(coco_det)
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]),
sizes=target['img_size'].tolist()[0])
img_ids.append(image_id)
results.append(coco_det)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
if 'test' in args.anno:
from itertools import groupby
results.sort(key=lambda x: x['image_id'])
f = open(
str(args.model) + "-" + str(args.anno) + "-" + str(min(threshs)) +
".txt", "w+")
# for item in tqdm(writetofilearrtay):
xxx = 0
for k, v in tqdm(groupby(results, key=lambda x: x['image_id'])):
xxx += 1
f.write(getimageNamefromid(k) +
",") #print(getimageNamefromid(k),", ")
for i in v:
if i['category_id'] > 0:
if (i['category_id'] ==1 and i['score'] >= threshs[0] ) or (i['category_id'] ==2 and i['score'] >= threshs[1] ) or \
(i['category_id'] ==3 and i['score'] >= threshs[2] ) or (i['category_id'] ==4 and i['score'] >= threshs[3] ) :
f.write(
str(round(i['category_id'])) + " " +
str(round(i['bbox'][0])) + " " +
str(round(i['bbox'][1])) + " " + str(
round(
float(i['bbox'][0]) +
float(i['bbox'][2]))) + " " + str(
round(
float(i['bbox'][1]) +
float(i['bbox'][3]))) + " ")
f.write('\n')
# print(i['category_id']," ",i['bbox'][0]," ",i['bbox'][1]," ",i['bbox'][2]," ",i['bbox'][3]," ")
print("generated lines:", xxx)
f.close()
# f.close()
if 'test' not in args.anno:
array_of_dm = []
array_of_gt = []
i = 0
# if 'test' in args.anno :
for _, item in tqdm(dataset):
# if item["img_id"] == "1000780" :
# print(item)
for i in range(len(item['cls'])):
# print(str(item["img_id"]),)
array_of_gt.append(
BoundingBox(imageName=str(item["img_id"]),
classId=item["cls"][i],
x=item["bbox"][i][1] * item['img_scale'],
y=item["bbox"][i][0] * item['img_scale'],
w=item["bbox"][i][3] * item['img_scale'],
h=item["bbox"][i][2] * item['img_scale'],
typeCoordinates=CoordinatesType.Absolute,
bbType=BBType.GroundTruth,
format=BBFormat.XYX2Y2,
imgSize=(item['img_size'][0],
item['img_size'][1])))
for item in tqdm(results):
if item["category_id"] >= 0:
array_of_dm.append(
BoundingBox(imageName=str(item["image_id"]),
classId=item["category_id"],
classConfidence=item["score"],
x=item['bbox'][0],
y=item['bbox'][1],
w=item['bbox'][2],
h=item['bbox'][3],
typeCoordinates=CoordinatesType.Absolute,
bbType=BBType.Detected,
format=BBFormat.XYWH,
imgSize=(item['sizes'][0], item['sizes'][1])))
myBoundingBoxes = BoundingBoxes()
# # # # Add all bounding boxes to the BoundingBoxes object:
for box in (array_of_gt):
myBoundingBoxes.addBoundingBox(box)
for dm in array_of_dm:
myBoundingBoxes.addBoundingBox(dm)
evaluator = Evaluator()
f1res = []
f1resd0 = []
f1resd10 = []
f1resd20 = []
f1resd40 = []
for conf in tqdm(range(210, 600, 1)):
metricsPerClass = evaluator.GetPascalVOCMetrics(
myBoundingBoxes, IOUThreshold=0.5, ConfThreshold=conf / 1000.0)
totalTP = 0
totalp = 0
totalFP = 0
tp = []
fp = []
ta = []
# print('-------')
for mc in metricsPerClass:
tp.append(mc['total TP'])
fp.append(mc['total FP'])
ta.append(mc['total positives'])
totalFP = totalFP + mc['total FP']
totalTP = totalTP + mc['total TP']
totalp = totalp + (mc['total positives'])
# print(totalTP," ",totalFP," ",totalp)
if totalTP + totalFP == 0:
p = -1
else:
p = totalTP / (totalTP + totalFP)
if totalp == 0:
r = -1
else:
r = totalTP / (totalp)
f1_dict = dict(tp=totalTP,
fp=totalFP,
totalp=totalp,
conf=conf / 1000.0,
prec=p,
rec=r,
f1score=(2 * p * r) / (p + r))
f1res.append(f1_dict)
#must clean these parts
f1resd0.append(
dict(tp=tp[0],
fp=fp[0],
totalp=ta[0],
conf=conf / 1000.0,
prec=tp[0] / (tp[0] + fp[0]),
rec=tp[0] / ta[0],
f1score=(2 * (tp[0] / (tp[0] + fp[0])) *
(tp[0] / ta[0])) / ((tp[0] / (tp[0] + fp[0])) +
(tp[0] / ta[0]))))
f1resd10.append(
dict(tp=tp[1],
fp=fp[1],
totalp=ta[1],
conf=conf / 1000.0,
prec=tp[1] / (tp[1] + fp[1]),
rec=tp[1] / ta[1],
f1score=(2 * (tp[1] / (tp[1] + fp[1])) *
(tp[1] / ta[1])) / ((tp[1] / (tp[1] + fp[1])) +
(tp[1] / ta[1]))))
f1resd20.append(
dict(tp=tp[2],
fp=fp[2],
totalp=ta[2],
conf=conf / 1000.0,
prec=tp[2] / (tp[2] + fp[2]),
rec=tp[2] / ta[2],
f1score=(2 * (tp[2] / (tp[2] + fp[2])) *
(tp[2] / ta[2])) / ((tp[2] / (tp[2] + fp[2])) +
(tp[2] / ta[2]))))
f1resd40.append(
dict(tp=tp[3],
fp=fp[3],
totalp=ta[3],
conf=conf / 1000.0,
prec=tp[3] / (tp[3] + fp[3]),
rec=tp[3] / ta[3],
f1score=(2 * (tp[3] / (tp[3] + fp[3])) *
(tp[3] / ta[3])) / ((tp[3] / (tp[3] + fp[3])) +
(tp[3] / ta[3]))))
sortedf1 = sorted(f1res, key=lambda k: k['f1score'], reverse=True)
f1resd0 = sorted(f1resd0, key=lambda k: k['f1score'], reverse=True)
f1resd10 = sorted(f1resd10, key=lambda k: k['f1score'], reverse=True)
f1resd20 = sorted(f1resd20, key=lambda k: k['f1score'], reverse=True)
f1resd40 = sorted(f1resd40, key=lambda k: k['f1score'], reverse=True)
print(sortedf1[0])
print("\n\n")
print(f1resd0[0])
print(f1resd10[0])
print(f1resd20[0])
print(f1resd40[0])
# sortedf1 = sorted(f1res, key=lambda k: k['f1score'],reverse=True)
# print(sortedf1[0:2])
# json.dump(results, open(args.results, 'w'), indent=4)
json.dump(results, open(args.results, 'w'), indent=4)
# coco_results = dataset.coco.loadRes(args.results)
# coco_eval = COCOeval(dataset.coco, coco_results, 'bbox')
# coco_eval.params.imgIds = img_ids # score only ids we've used
# coco_eval.evaluate()
# coco_eval.accumulate()
# coco_eval.summarize()
# print(coco_eval.eval['params'])
json.dump(results, open(args.results, 'w'), indent=4)
return results
def main():
setup_default_logging()
args, args_text = _parse_args()
if args.log_wandb:
if has_wandb:
wandb.init(project=args.experiment, config=args)
else:
_logger.warning(
"You've requested to log metrics to wandb but package not found. "
"Metrics not being logged to wandb, try `pip install wandb`")
args.prefetcher = not args.no_prefetcher
args.distributed = False
if 'WORLD_SIZE' in os.environ:
args.distributed = int(os.environ['WORLD_SIZE']) > 1
args.device = 'cuda:0'
args.world_size = 1
args.rank = 0 # global rank
if args.distributed:
args.device = 'cuda:%d' % args.local_rank
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend='nccl',
init_method='env://')
args.world_size = torch.distributed.get_world_size()
args.rank = torch.distributed.get_rank()
_logger.info(
'Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.'
% (args.rank, args.world_size))
else:
_logger.info('Training with a single process on 1 GPUs.')
assert args.rank >= 0
# resolve AMP arguments based on PyTorch / Apex availability
use_amp = None
if args.amp:
# `--amp` chooses native amp before apex (APEX ver not actively maintained)
if has_native_amp:
args.native_amp = True
elif has_apex:
args.apex_amp = True
if args.apex_amp and has_apex:
use_amp = 'apex'
elif args.native_amp and has_native_amp:
use_amp = 'native'
elif args.apex_amp or args.native_amp:
_logger.warning(
"Neither APEX or native Torch AMP is available, using float32. "
"Install NVIDA apex or upgrade to PyTorch 1.6")
random_seed(args.seed, args.rank)
if args.fuser:
set_jit_fuser(args.fuser)
model = create_model(
args.model,
pretrained=args.pretrained,
num_classes=args.num_classes,
drop_rate=args.drop,
drop_connect_rate=args.drop_connect, # DEPRECATED, use drop_path
drop_path_rate=args.drop_path,
drop_block_rate=args.drop_block,
global_pool=args.gp,
bn_momentum=args.bn_momentum,
bn_eps=args.bn_eps,
scriptable=args.torchscript,
checkpoint_path=args.initial_checkpoint)
if args.num_classes is None:
assert hasattr(
model, 'num_classes'
), 'Model must have `num_classes` attr if not set on cmd line/config.'
args.num_classes = model.num_classes # FIXME handle model default vs config num_classes more elegantly
if args.grad_checkpointing:
model.set_grad_checkpointing(enable=True)
if args.local_rank == 0:
_logger.info(
f'Model {safe_model_name(args.model)} created, param count:{sum([m.numel() for m in model.parameters()])}'
)
data_config = resolve_data_config(vars(args),
model=model,
verbose=args.local_rank == 0)
# setup augmentation batch splits for contrastive loss or split bn
num_aug_splits = 0
if args.aug_splits > 0:
assert args.aug_splits > 1, 'A split of 1 makes no sense'
num_aug_splits = args.aug_splits
# enable split bn (separate bn stats per batch-portion)
if args.split_bn:
assert num_aug_splits > 1 or args.resplit
model = convert_splitbn_model(model, max(num_aug_splits, 2))
# move model to GPU, enable channels last layout if set
model.cuda()
if args.channels_last:
model = model.to(memory_format=torch.channels_last)
# setup synchronized BatchNorm for distributed training
if args.distributed and args.sync_bn:
assert not args.split_bn
if has_apex and use_amp == 'apex':
# Apex SyncBN preferred unless native amp is activated
model = convert_syncbn_model(model)
else:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
if args.local_rank == 0:
_logger.info(
'Converted model to use Synchronized BatchNorm. WARNING: You may have issues if using '
'zero initialized BN layers (enabled by default for ResNets) while sync-bn enabled.'
)
if args.torchscript:
assert not use_amp == 'apex', 'Cannot use APEX AMP with torchscripted model'
assert not args.sync_bn, 'Cannot use SyncBatchNorm with torchscripted model'
model = torch.jit.script(model)
optimizer = create_optimizer_v2(model, **optimizer_kwargs(cfg=args))
# setup automatic mixed-precision (AMP) loss scaling and op casting
amp_autocast = suppress # do nothing
loss_scaler = None
if use_amp == 'apex':
model, optimizer = amp.initialize(model, optimizer, opt_level='O1')
loss_scaler = ApexScaler()
if args.local_rank == 0:
_logger.info('Using NVIDIA APEX AMP. Training in mixed precision.')
elif use_amp == 'native':
amp_autocast = torch.cuda.amp.autocast
loss_scaler = NativeScaler()
if args.local_rank == 0:
_logger.info(
'Using native Torch AMP. Training in mixed precision.')
else:
if args.local_rank == 0:
_logger.info('AMP not enabled. Training in float32.')
# optionally resume from a checkpoint
resume_epoch = None
if args.resume:
resume_epoch = resume_checkpoint(
model,
args.resume,
optimizer=None if args.no_resume_opt else optimizer,
loss_scaler=None if args.no_resume_opt else loss_scaler,
log_info=args.local_rank == 0)
# setup exponential moving average of model weights, SWA could be used here too
model_ema = None
if args.model_ema:
# Important to create EMA model after cuda(), DP wrapper, and AMP but before DDP wrapper
model_ema = ModelEmaV2(
model,
decay=args.model_ema_decay,
device='cpu' if args.model_ema_force_cpu else None)
if args.resume:
load_checkpoint(model_ema.module, args.resume, use_ema=True)
# setup distributed training
if args.distributed:
if has_apex and use_amp == 'apex':
# Apex DDP preferred unless native amp is activated
if args.local_rank == 0:
_logger.info("Using NVIDIA APEX DistributedDataParallel.")
model = ApexDDP(model, delay_allreduce=True)
else:
if args.local_rank == 0:
_logger.info("Using native Torch DistributedDataParallel.")
model = NativeDDP(model,
device_ids=[args.local_rank],
broadcast_buffers=not args.no_ddp_bb)
# NOTE: EMA model does not need to be wrapped by DDP
# setup learning rate schedule and starting epoch
lr_scheduler, num_epochs = create_scheduler(args, optimizer)
start_epoch = 0
if args.start_epoch is not None:
# a specified start_epoch will always override the resume epoch
start_epoch = args.start_epoch
elif resume_epoch is not None:
start_epoch = resume_epoch
if lr_scheduler is not None and start_epoch > 0:
lr_scheduler.step(start_epoch)
if args.local_rank == 0:
_logger.info('Scheduled epochs: {}'.format(num_epochs))
# create the train and eval datasets
dataset_train = create_dataset(args.dataset,
root=args.data_dir,
split=args.train_split,
is_training=True,
class_map=args.class_map,
download=args.dataset_download,
batch_size=args.batch_size,
repeats=args.epoch_repeats)
dataset_eval = create_dataset(args.dataset,
root=args.data_dir,
split=args.val_split,
is_training=False,
class_map=args.class_map,
download=args.dataset_download,
batch_size=args.batch_size)
# setup mixup / cutmix
collate_fn = None
mixup_fn = None
mixup_active = args.mixup > 0 or args.cutmix > 0. or args.cutmix_minmax is not None
if mixup_active:
mixup_args = dict(mixup_alpha=args.mixup,
cutmix_alpha=args.cutmix,
cutmix_minmax=args.cutmix_minmax,
prob=args.mixup_prob,
switch_prob=args.mixup_switch_prob,
mode=args.mixup_mode,
label_smoothing=args.smoothing,
num_classes=args.num_classes)
if args.prefetcher:
assert not num_aug_splits # collate conflict (need to support deinterleaving in collate mixup)
collate_fn = FastCollateMixup(**mixup_args)
else:
mixup_fn = Mixup(**mixup_args)
# wrap dataset in AugMix helper
if num_aug_splits > 1:
dataset_train = AugMixDataset(dataset_train, num_splits=num_aug_splits)
# create data loaders w/ augmentation pipeiine
train_interpolation = args.train_interpolation
if args.no_aug or not train_interpolation:
train_interpolation = data_config['interpolation']
loader_train = create_loader(
dataset_train,
input_size=data_config['input_size'],
batch_size=args.batch_size,
is_training=True,
use_prefetcher=args.prefetcher,
no_aug=args.no_aug,
re_prob=args.reprob,
re_mode=args.remode,
re_count=args.recount,
re_split=args.resplit,
scale=args.scale,
ratio=args.ratio,
hflip=args.hflip,
vflip=args.vflip,
color_jitter=args.color_jitter,
auto_augment=args.aa,
num_aug_repeats=args.aug_repeats,
num_aug_splits=num_aug_splits,
interpolation=train_interpolation,
mean=data_config['mean'],
std=data_config['std'],
num_workers=args.workers,
distributed=args.distributed,
collate_fn=collate_fn,
pin_memory=args.pin_mem,
use_multi_epochs_loader=args.use_multi_epochs_loader,
worker_seeding=args.worker_seeding,
)
loader_eval = create_loader(
dataset_eval,
input_size=data_config['input_size'],
batch_size=args.validation_batch_size or args.batch_size,
is_training=False,
use_prefetcher=args.prefetcher,
interpolation=data_config['interpolation'],
mean=data_config['mean'],
std=data_config['std'],
num_workers=args.workers,
distributed=args.distributed,
crop_pct=data_config['crop_pct'],
pin_memory=args.pin_mem,
)
# setup loss function
if args.jsd_loss:
assert num_aug_splits > 1 # JSD only valid with aug splits set
train_loss_fn = JsdCrossEntropy(num_splits=num_aug_splits,
smoothing=args.smoothing)
elif mixup_active:
# smoothing is handled with mixup target transform which outputs sparse, soft targets
if args.bce_loss:
train_loss_fn = BinaryCrossEntropy(
target_threshold=args.bce_target_thresh)
else:
train_loss_fn = SoftTargetCrossEntropy()
elif args.smoothing:
if args.bce_loss:
train_loss_fn = BinaryCrossEntropy(
smoothing=args.smoothing,
target_threshold=args.bce_target_thresh)
else:
train_loss_fn = LabelSmoothingCrossEntropy(
smoothing=args.smoothing)
else:
train_loss_fn = nn.CrossEntropyLoss()
train_loss_fn = train_loss_fn.cuda()
validate_loss_fn = nn.CrossEntropyLoss().cuda()
# setup checkpoint saver and eval metric tracking
eval_metric = args.eval_metric
best_metric = None
best_epoch = None
saver = None
output_dir = None
if args.rank == 0:
if args.experiment:
exp_name = args.experiment
else:
exp_name = '-'.join([
datetime.now().strftime("%Y%m%d-%H%M%S"),
safe_model_name(args.model),
str(data_config['input_size'][-1])
])
output_dir = get_outdir(
args.output if args.output else './output/train', exp_name)
decreasing = True if eval_metric == 'loss' else False
saver = CheckpointSaver(model=model,
optimizer=optimizer,
args=args,
model_ema=model_ema,
amp_scaler=loss_scaler,
checkpoint_dir=output_dir,
recovery_dir=output_dir,
decreasing=decreasing,
max_history=args.checkpoint_hist)
with open(os.path.join(output_dir, 'args.yaml'), 'w') as f:
f.write(args_text)
try:
for epoch in range(start_epoch, num_epochs):
if args.distributed and hasattr(loader_train.sampler, 'set_epoch'):
loader_train.sampler.set_epoch(epoch)
train_metrics = train_one_epoch(epoch,
model,
loader_train,
optimizer,
train_loss_fn,
args,
lr_scheduler=lr_scheduler,
saver=saver,
output_dir=output_dir,
amp_autocast=amp_autocast,
loss_scaler=loss_scaler,
model_ema=model_ema,
mixup_fn=mixup_fn)
if args.distributed and args.dist_bn in ('broadcast', 'reduce'):
if args.local_rank == 0:
_logger.info(
"Distributing BatchNorm running means and vars")
distribute_bn(model, args.world_size, args.dist_bn == 'reduce')
eval_metrics = validate(model,
loader_eval,
validate_loss_fn,
args,
amp_autocast=amp_autocast)
if model_ema is not None and not args.model_ema_force_cpu:
if args.distributed and args.dist_bn in ('broadcast',
'reduce'):
distribute_bn(model_ema, args.world_size,
args.dist_bn == 'reduce')
ema_eval_metrics = validate(model_ema.module,
loader_eval,
validate_loss_fn,
args,
amp_autocast=amp_autocast,
log_suffix=' (EMA)')
eval_metrics = ema_eval_metrics
if lr_scheduler is not None:
# step LR for next epoch
lr_scheduler.step(epoch + 1, eval_metrics[eval_metric])
if output_dir is not None:
update_summary(epoch,
train_metrics,
eval_metrics,
os.path.join(output_dir, 'summary.csv'),
write_header=best_metric is None,
log_wandb=args.log_wandb and has_wandb)
if saver is not None:
# save proper checkpoint with eval metric
save_metric = eval_metrics[eval_metric]
best_metric, best_epoch = saver.save_checkpoint(
epoch, metric=save_metric)
except KeyboardInterrupt:
pass
if best_metric is not None:
_logger.info('*** Best metric: {0} (epoch {1})'.format(
best_metric, best_epoch))
def main():
start_endpoint = "http://localhost:3000/start"
stop_endpoint = "http://localhost:3000/stop"
setup_default_logging()
args = parser.parse_args()
# might as well try to do something useful...
args.pretrained = args.pretrained or not args.checkpoint
output_dir = args.checkpoint.split('/')
output_dir.pop(-1)
output_dir = ('/').join(output_dir)
# create model
model = create_model(
args.model,
num_classes=args.num_classes,
in_chans=3,
pretrained=args.pretrained,
checkpoint_path=args.checkpoint)
logging.info('Model %s created, param count: %d' %
(args.model, sum([m.numel() for m in model.parameters()])))
# config = resolve_data_config(vars(args), model=model)
# model, test_time_pool = apply_test_time_pool(model, config, args)
if args.num_gpu > 1:
model = torch.nn.DataParallel(
model, device_ids=list(range(args.num_gpu))).cuda()
else:
model = model.cuda()
dataset_eval = torchvision.datasets.CIFAR100(
root='./data', train=False, download=True)
data_config = resolve_data_config(vars(args), model=model)
# #CIFAR_100_MEAN = (0.5071, 0.4865, 0.4409)
# #CIFAR_100_STD = (0.2673, 0.2564, 0.2762)
data_config['mean'] = (0.5071, 0.4865, 0.4409)
data_config['std'] = (0.2673, 0.2564, 0.2762)
loader = create_loader(
dataset_eval,
input_size=data_config['input_size'],
batch_size=args.batch_size,
is_training=False,
interpolation=data_config['interpolation'],
mean=data_config['mean'],
std=data_config['std'],
num_workers=args.workers,
crop_pct=data_config['crop_pct']
)
model.eval()
batch_time = AverageMeter()
with torch.no_grad():
idle_power = requests.post(url=start_endpoint)
idle_json = idle_power.json()
for batch_idx, (input, _) in enumerate(loader):
input = input.cuda()
tstart = time.time()
output = model(input)
tend = time.time()
if batch_idx != 0:
batch_time.update(tend - tstart)
if batch_idx % args.log_freq == 0:
print('Predict: [{0}/{1}] Time {batch_time.val:.6f} ({batch_time.avg:.6f})'.format(
batch_idx, len(loader), batch_time=batch_time), end='\r')
load_power = requests.post(url=stop_endpoint)
load_json = load_power.json()
fps = 1 / batch_time.avg
inference_power = float(load_json['load']) - float(idle_json['idle'])
stats = [{'FPS': [float(fps)]},
{'Total_Power': [float(inference_power)]}]
with open(os.path.join(output_dir, '{}_fps_cifar.yaml'.format(args.model)), 'w') as f:
yaml.safe_dump(stats, f)
def main():
setup_default_logging()
args = parser.parse_args()
# might as well try to do something useful...
args.pretrained = args.pretrained or not args.checkpoint
# create model
model = create_model(args.model,
num_classes=args.num_classes,
in_chans=3,
pretrained=args.pretrained,
checkpoint_path=args.checkpoint)
_logger.info('Model %s created, param count: %d' %
(args.model, sum([m.numel() for m in model.parameters()])))
config = resolve_data_config(vars(args), model=model)
model, test_time_pool = (
model,
False) if args.no_test_pool else apply_test_time_pool(model, config)
if args.num_gpu > 1:
model = torch.nn.DataParallel(model,
device_ids=list(range(
args.num_gpu))).cuda()
else:
model = model.cuda()
loader = create_loader(
Dataset(args.data, train_mode='test', fold_num=-1),
input_size=config['input_size'],
batch_size=args.batch_size,
use_prefetcher=True,
interpolation=config['interpolation'],
mean=config['mean'],
std=config['std'],
num_workers=args.workers,
crop_pct=1.0 if test_time_pool else config['crop_pct'])
model.eval()
# k = min(args.topk, args.num_classes)
batch_time = AverageMeter()
end = time.time()
topk_ids = []
name_list = []
sig_list = []
logits_list = []
m = torch.nn.Sigmoid()
with torch.no_grad():
for batch_idx, (
input,
_,
) in enumerate(loader):
input = input.cuda()
labels = model(input)
logits_list.append(labels)
sigmoided = m(labels)
sig_list.append(
np.expand_dims(sigmoided[:, 1].cpu().numpy(), axis=1))
# topk = labels.topk(k)[1]
# topk_ids.append(topk.cpu().numpy())
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % args.log_freq == 0:
_logger.info(
'Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})'
.format(batch_idx, len(loader), batch_time=batch_time))
# topk_ids = np.concatenate(topk_ids, axis=0).squeeze()
# logits = torch.cat(logits_list).cuda()
# temperature = nn.Parameter(torch.ones(1) * args.te).to(torch.device('cuda') ).detach().requires_grad_(False)
# logits = logits/temperature.unsqueeze(1).expand(logits.size(0), logits.size(1))
# temp_sigmoided = m(logits)[:,1].detach().cpu().numpy()
sig_list = np.vstack(sig_list)
name_list = loader.dataset.filenames(basename=True)
real_sigmoid = sig_list.squeeze()
# real_sigmoid = temp_sigmoided
real_pred = ((sig_list >= args.thresh) * 1).squeeze()
name_pred_dict = {}
for idx in range(len(name_list)):
name_pred_dict[name_list[idx]] = (real_pred[idx], real_sigmoid[idx])
args.output_dir = args.checkpoint.replace(
args.checkpoint.split('/')[-1], "")
with open(os.path.join(args.output_dir, './prediction.tsv'),
'w') as out_file:
# filenames_int = [int(f.split('.')[0]) for f in filenames]
# for name, topk in zip(filenames_int, topk_ids):
# print(name,topk)
# i = i+1
# if i == 10:
# break
# idx = np.argsort(filenames_int)
# topk_ids = topk_ids[idx]
for name in name_list:
out_file.write('{}\n'.format(str(name_pred_dict[name][0])))
with open(os.path.join(args.output_dir, './probability.tsv'),
'w') as out_file:
for name in name_list:
out_file.write('{}\n'.format(name_pred_dict[name][1]))
copyfile(
os.path.join(args.output_dir, './prediction.tsv'),
'/home/workspace/user-workspace/prediction/' + 'prediction_153_' +
args.checkpoint.split('/')[-2] + '.tsv')
from timm.models import create_model
from timm.data import resolve_data_config
import torch
import torch.nn as nn
import argparse
from external.utils_pruning import compute_flops, load_module_from_ckpt, measure_time
from external.hyperml import HypermlDownloader
from timm.utils import setup_default_logging
from timm.models import load_checkpoint
import logging
setup_default_logging()
parser = argparse.ArgumentParser(description='PyTorch ImageNet Validation')
parser.add_argument(
'--model',
'-m',
metavar='MODEL',
default='efficientnet_b2_pruned',
help='model architecture (default: efficientnet_b2_pruned)')
parser.add_argument('--batch_size', default=512, type=int, help='batch size)')
parser.add_argument('--fp16',
action='store_true',
default=False,
help='Use half precision (fp16)')
parser.add_argument('--use_eca',
action='store_true',
default=False,
help='Use eca bn for efficientNet')
def validate(args):
setup_default_logging()
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
if args.no_redundant_bias is None:
args.redundant_bias = None
else:
args.redundant_bias = not args.no_redundant_bias
# create model
bench2 = pt.detection_models.__dict__[args.model](
match_tf_same_padding=True, encoder_norm_act="swish_hard")
bench2 = bench2.eval().requires_grad_(False).cuda()
input_size2 = bench2.pretrained_settings["input_size"]
param_count = sum([m.numel() for m in bench2.parameters()])
print("Model %s created, param count: %d" % (args.model, param_count))
if has_amp:
print("Using AMP mixed precision.")
bench2 = amp.initialize(bench2, opt_level="O1")
else:
print("AMP not installed, running network in FP32.")
if "test" in args.anno:
annotation_path = os.path.join(args.data, "annotations",
f"image_info_{args.anno}.json")
image_dir = "test2017"
else:
annotation_path = os.path.join(args.data, "annotations",
f"instances_{args.anno}.json")
image_dir = args.anno
# loader = create_loader(
# dataset,
# input_size=input_size,
# batch_size=args.batch_size,
# use_prefetcher=args.prefetcher,
# interpolation=args.interpolation,
# fill_color=args.fill_color,
# num_workers=args.workers,
# pin_mem=args.pin_mem,
# )
print(f"Input size: {input_size2[0]}")
loader = DaliLoader(False, args.batch_size, args.workers, input_size2[0])
img_ids = []
results = []
batch_time = AverageMeter()
end = time.time()
start_time = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output2 = bench2.predict(input)
_, batch_ids, ratios = target
target = {"img_scale": ratios, "img_id": batch_ids}
# rescale to image size and clip
output2[..., :4] *= target["img_scale"].view(-1, 1, 1).to(output2)
# works even without clipping
# output2[..., :4] = box_utils.clip_bboxes_batch(output2[..., :4], target["img_size"][..., [1, 0]])
# xyxy => xywh
output2[..., 2:4] = output2[..., 2:4] - output2[..., :2]
output = output2
output = output.cpu()
sample_ids = target["img_id"].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
for det in sample:
score = float(det[4])
if score < 0.001: # stop when below this threshold, scores in descending order
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]))
img_ids.append(image_id)
results.append(coco_det)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
"Test: [{0:>4d}/{1}] "
"Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) "
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
# if i > 10:
# break
print(f"Full eval took: {time.time() - start_time:.2f}s")
json.dump(results, open(args.results, "w"), indent=4)
if "test" not in args.anno:
coco_api = COCO("data/annotations/instances_val2017.json")
coco_results = coco_api.loadRes(args.results)
coco_eval = COCOeval(coco_api, coco_results, "bbox")
coco_eval.params.imgIds = img_ids # score only ids we've used
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return results
def validate(args):
setup_default_logging()
if args.amp:
if has_apex:
args.apex_amp = True
elif has_native_amp:
args.native_amp = True
assert not args.apex_amp or not args.native_amp, "Only one AMP mode should be set."
args.pretrained = args.pretrained or not args.checkpoint # might as well try to validate something
args.prefetcher = not args.no_prefetcher
# create model
with set_layer_config(scriptable=args.torchscript):
bench = create_model(
args.model,
bench_task='predict',
num_classes=args.num_classes,
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
soft_nms=args.soft_nms,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
model_config = bench.config
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
amp_autocast = suppress
if args.apex_amp:
bench = amp.initialize(bench, opt_level='O1')
print('Using NVIDIA APEX AMP. Validating in mixed precision.')
elif args.native_amp:
amp_autocast = torch.cuda.amp.autocast
print('Using native Torch AMP. Validating in mixed precision.')
else:
print('AMP not enabled. Validating in float32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
dataset = create_dataset(args.dataset, args.root, args.split)
input_config = resolve_input_config(args, model_config)
loader = create_loader(dataset,
input_size=input_config['input_size'],
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=input_config['interpolation'],
fill_color=input_config['fill_color'],
mean=input_config['mean'],
std=input_config['std'],
num_workers=args.workers,
pin_mem=args.pin_mem)
evaluator = create_evaluator(args.dataset, dataset, pred_yxyx=False)
bench.eval()
batch_time = AverageMeter()
end = time.time()
last_idx = len(loader) - 1
imgs = []
with torch.no_grad():
for i, (input, target) in enumerate(loader):
for b in range(input.shape[0]):
imgs.append(input[b].cpu().numpy())
# targets.append(target[b].cpu().numpy())
with amp_autocast():
output = bench(input, img_info=target)
evaluator.add_predictions(output, target)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0 or i == last_idx:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg))
mean_ap = 0.
if dataset.parser.has_labels:
preds = [p[:2, :] for p in evaluator.predictions]
anns = evaluator.coco_api.imgToAnns
targets = [
np.asarray((anns[k][0]['bbox'], anns[k][1]['bbox']))
for k in range(len(imgs))
]
mean_ap = evaluator.evaluate()
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
for i, img in enumerate(imgs):
img = imgs[i]
img_m = np.mean(img, axis=0)
for c in range(3):
img[c] = img_m
img_ = img.transpose(1, 2, 0)
m = img_.min()
M = img_.max()
img_ = ((img_ - m) / (M - m) * 255).astype('uint8').copy()
img_ = draw_bbox(img_, preds[i], targets[i])
cv2.imwrite(os.path.join(args.out_dir, '%d.jpg' % i), img_)
else:
evaluator.save(args.results)
return mean_ap
def main():
setup_default_logging()
args = parser.parse_args()
# might as well try to do something useful...
args.pretrained = args.pretrained or not args.checkpoint
# create model
model = create_model(args.model,
num_classes=args.num_classes,
in_chans=3,
pretrained=args.pretrained,
checkpoint_path=args.checkpoint)
logging.info('Model %s created, param count: %d' %
(args.model, sum([m.numel() for m in model.parameters()])))
config = resolve_data_config(vars(args), model=model)
model, test_time_pool = apply_test_time_pool(model, config, args)
if args.num_gpu > 1:
model = torch.nn.DataParallel(model,
device_ids=list(range(
args.num_gpu))).cuda()
else:
model = model.cuda()
loader = create_loader(
Dataset(args.data),
input_size=config['input_size'],
batch_size=args.batch_size,
use_prefetcher=True,
interpolation=config['interpolation'],
mean=config['mean'],
std=config['std'],
num_workers=args.workers,
crop_pct=1.0 if test_time_pool else config['crop_pct'])
model.eval()
k = min(args.topk, args.num_classes)
batch_time = AverageMeter()
end = time.time()
topk_ids = []
with torch.no_grad():
for batch_idx, (input, _) in enumerate(loader):
input = input.cuda()
labels = model(input)
topk = labels.topk(k)[1]
topk_ids.append(topk.cpu().numpy())
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % args.log_freq == 0:
logging.info(
'Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})'
.format(batch_idx, len(loader), batch_time=batch_time))
topk_ids = np.concatenate(topk_ids, axis=0).squeeze()
savebase = "classification_result/"
os.makedirs(savebase, exist_ok=True)
classfile = "labels.txt"
classpath = os.path.join(os.getcwd(), classfile)
classlist = {}
with open(classpath) as f:
for idx, line in enumerate(f):
val = line.split('\n')[0]
classlist[idx] = val
filenames = loader.dataset.filenames()
for filepath, label in zip(filenames, topk_ids):
filename = os.path.basename(filepath)
prediction = classlist[label[0]]
savedir = savebase + prediction
savepath = savedir + "/" + filename
os.makedirs(savedir, exist_ok=True)
copyfile(filepath, savepath)
print('{0} : {1}'.format(filename, prediction))
def predict(INP_DIR, BATCH_SIZE, MODEL_PATH):
print("[INFO] Predicting")
setup_default_logging()
args = parser.parse_args()
# might as well try to do something useful...
args.model = 'tf_efficientnet_b8'
args.data = INP_DIR
args.num_classes = 8
args.checkpoint = MODEL_PATH
args.batch_size = BATCH_SIZE
args.pretrained = args.pretrained or not args.checkpoint
# create model
model = create_model(
args.model,
num_classes=args.num_classes,
in_chans=3,
pretrained=args.pretrained,
checkpoint_path=args.checkpoint)
_logger.info('Model %s created, param count: %d' %
(args.model, sum([m.numel() for m in model.parameters()])))
config = resolve_data_config(vars(args), model=model)
model, test_time_pool = (model, False) if args.no_test_pool else apply_test_time_pool(model, config)
if args.num_gpu > 1:
model = torch.nn.DataParallel(model, device_ids=list(range(args.num_gpu))).cuda()
else:
model = model.cuda()
loader = create_loader(
Dataset(args.data),
input_size=config['input_size'],
batch_size=args.batch_size,
use_prefetcher=True,
interpolation=config['interpolation'],
mean=config['mean'],
std=config['std'],
num_workers=args.workers,
crop_pct=1.0 if test_time_pool else config['crop_pct'])
model.eval()
k = min(args.topk, args.num_classes)
batch_time = AverageMeter()
end = time.time()
topk_ids = []
topk_prob = []
with torch.no_grad():
for batch_idx, (input, _) in enumerate(loader):
try:
input = input.cuda()
labels = model(input)
topk = labels.topk(k)[0]
topk = topk.cpu().numpy()
# print(topk)
topk = np.exp(topk) / np.sum(np.exp(topk), axis=-1)[:, np.newaxis]
# print(topk)
topk_prob.append(topk)
topk = labels.topk(k)[1]
topk_ids.append(topk.cpu().numpy())
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % args.log_freq == 0:
_logger.info('Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})'.format(
batch_idx, len(loader), batch_time=batch_time))
except Exception:
pass
topk_ids = np.concatenate(topk_ids, axis=0).squeeze()
topk_prob = np.concatenate(topk_prob, axis=0).squeeze()
print("topk_ids: ", topk_ids)
print("topk_prob: ", topk_prob)
# out_path = os.path.join(args.output_dir, 'submission_{}.txt'.format(args.model))
out_path = os.path.join('hackathon_test', "output.txt")
with open(out_path, 'w') as out_file:
filenames = loader.dataset.filenames(basename=True)
for filename, label, prob in zip(filenames, topk_ids, topk_prob):
# out_file.write(("{}" + "\t{}\t{:.4f}"*5 + "\n").format(
# filename, *chain(*zip(label, prob))))
out_file.write(("{}" + " {} {:.4f}"*5 + "\n").format(
filename, *chain(*zip(label, prob))))
print("label: {}, prob: {}".format(label, prob))
return out_path
def main():
setup_default_logging()
args = parser.parse_args()
# might as well try to do something useful...
args.pretrained = args.pretrained or not args.checkpoint
# create model
model = create_model(
args.model,
num_classes=args.num_classes,
in_chans=3,
pretrained=args.pretrained,
checkpoint_path=args.checkpoint)
logging.info('Model %s created, param count: %d' %
(args.model, sum([m.numel() for m in model.parameters()])))
config = resolve_data_config(vars(args), model=model)
test_time_pool = False
if args.num_gpu > 1:
model = torch.nn.DataParallel(model, device_ids=list(range(args.num_gpu))).cuda()
else:
model = model.cuda()
test_dataset = Dataset(args.data)
class_mapper = {v:k for k,v in test_dataset.class_to_idx.items()}
loader = create_loader(
test_dataset,
input_size=config['input_size'],
batch_size=args.batch_size,
use_prefetcher=False,
interpolation=config['interpolation'],
mean=config['mean'],
std=config['std'],
num_workers=args.workers,
crop_pct=1.0 if test_time_pool else config['crop_pct'])
model.eval()
k = min(args.topk, args.num_classes)
batch_time = AverageMeter()
end = time.time()
topk_ids = []
results = []
with torch.no_grad():
for batch_idx, (input, _, path) in enumerate(loader):
input = input.cuda()
labels = model(input)
topk = labels.topk(k)[1]
topk_ids.append(topk.cpu().numpy())
label_ = list(topk.cpu().numpy())[0][0]
results.append([path, class_mapper[label_]])
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % args.log_freq == 0:
logging.info('Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})'.format(
batch_idx, len(loader), batch_time=batch_time))
topk_ids = np.concatenate(topk_ids, axis=0).squeeze()
with open(os.path.join(args.output_dir, './topk_ids.csv'), 'w') as out_file:
filenames = loader.dataset.filenames()
for filename, label in zip(filenames, topk_ids):
filename = os.path.basename(filename)
out_file.write('{0},{1},{2},{3}\n'.format(
filename, label[0], label[1], label[2]))
if args.result_dir:
if not os.path.isdir(args.result_dir):
os.system("mkdir "+args.result_dir)
else:
os.system("rm -rf "+args.result_dir)
os.system("mkdir "+args.result_dir)
for k,v in class_mapper.items():
os.system("mkdir -p "+os.path.join(args.result_dir,v))
for image in results:
path = image[0][0]
result = image[1]
gt = path.split('/')[-2]
print("\tPath : %s \tResult : %s \tGT : %s"%(path, result, gt))
os.system("cp "+path+" "+os.path.join(args.result_dir,result))
def validate_det(args):
setup_default_logging()
# might as well try to validate something
args.pretrained = args.checkpoint
args.prefetcher = not args.no_prefetcher
# create model
bench = create_model(
args.model,
bench_task='predict',
num_classes=args.num_classes,
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
model_config = bench.config
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
#print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
'''if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')'''
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
'''if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json')
image_dir = 'test2017'''
'''annotation_path = os.path.join(args.data, 'Highway_drop_inlets.v1-vdot_coco.coco/coco_and_dropinlets_annotations/test_annotations', f'{args.anno}.json')
image_dir = os.path.join(args.data, 'Highway_drop_inlets.v1-vdot_coco.coco/coco_and_dropinlets/test/')
dataset = CocoDetection(image_dir, annotation_path)'''
annotation_path = os.path.join(args.data, f'{args.anno}.json')
image_dir = os.path.join(args.data, 'others_mix_set')
dataset = VdotTestDataset(image_dir, annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batches_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
pin_mem=args.pin_mem)
img_ids = []
results = []
bench.eval()
#example_input = torch.randn((1, 3, 512, 512), requires_grad=True)
#bench(example_input.cuda())
'''tracingModelInput = torch.ones(1,3,512,512)
torch.onnx._export(bench.model, tracingModelInput.cuda(), './effdet0_checkonly.onnx', opset_version=11, export_params=True)
print('\nDone exporting ONNX model!')
onnx_model='./effdet0_checkonly.onnx'
onnx.checker.check_model(onnx_model)'''
'''dummy_input = torch.randn(1, 3, 512, 512, device='cuda')
input =["input"]
output=["class_1","class_2","class_3","class_4", "class_5", "box_1", "box_2", "box_3","box_4","box_5"]
#dynamic_axes = {"actual_input_1":{input:"batch_size"}, "output1":{output:"batch_size"}}
torch.onnx.export(bench.model, dummy_input, "effdet0_upsamplenormal.onnx", verbose=True, opset_version=11,
input_names=input, output_names=output)'''
#dynamic_axes=dynamic_axes)
batch_time = AverageMeter()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output = bench(input, img_info=target)
output = output.cpu()
sample_ids = target['img_id'].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
for det in sample:
score = float(det[4])
if score < 0.36: # stop when below this threshold, scores in descending order(perfect 0.5 for 91 classes)
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]))
img_ids.append(image_id)
results.append(coco_det)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
json.dump(results, open(args.results, 'w'), indent=4)
'''if 'test' not in args.anno:
coco_results = dataset.coco.loadRes(args.results)
coco_eval = COCOeval(dataset.coco, coco_results, 'bbox')
coco_eval.params.imgIds = img_ids # score only ids we've used
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()'''
return results
def validate(args):
setup_default_logging()
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
# create model
bench = create_model(args.model,
bench_task='predict',
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema)
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'image_info_{args.anno}.json')
image_dir = args.anno
else:
annotation_path = os.path.join(args.data, 'annotations',
f'instances_{args.anno}.json')
image_dir = args.anno
dataset = CocoDetection(os.path.join(args.data, image_dir),
annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
mean=args.mean,
std=args.std,
pin_mem=args.pin_mem)
img_ids = []
results = []
bench.eval()
for i, (input, target) in enumerate(loader, 1):
dumm_inp = input
tisc = target['img_scale']
tisz = target['img_size']
break
starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(
enable_timing=True)
# repetitions = 300
# timings=np.zeros((repetitions,1))
#GPU-WARM-UP
# print(enumerate())
for _ in range(10):
_ = bench(dumm_inp, tisc, tisz)
# MEASURE PERFORMANCE
# dummy_input = torch.randn(1, 3,bench.config.image_size,bench.config.image_size,dtype=torch.float).to("cuda")
print("starting")
batch_time = AverageMeter()
# end = time.time()
with torch.no_grad():
for _ in range(2000):
starter.record()
_ = bench(dumm_inp, tisc, tisz)
ender.record()
# measure elapsed time
torch.cuda.synchronize()
curr_time = starter.elapsed_time(ender)
batch_time.update(curr_time)
# print(curr_time)
# end = time.time()
# if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}ms ({batch_time.avg:.3f}ms, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=dumm_inp.size(0) / batch_time.avg,
))
# json.dump(results, open(args.results, 'w'), indent=4)
# if 'test' not in args.anno:
# coco_results = dataset.coco.loadRes(args.results)
# coco_eval = COCOeval(dataset.coco, coco_results, 'bbox')
# coco_eval.params.imgIds = img_ids # score only ids we've used
# coco_eval.evaluate()
# coco_eval.accumulate()
# coco_eval.summarize()
return results
def main():
setup_default_logging()
args = parser.parse_args()
# might as well try to do something useful...
args.pretrained = args.pretrained or not args.checkpoint
# create model
model = create_model(args.model,
num_classes=args.num_classes,
in_chans=3,
pretrained=args.pretrained,
checkpoint_path=args.checkpoint)
_logger.info('Model %s created, param count: %d' %
(args.model, sum([m.numel() for m in model.parameters()])))
config = resolve_data_config(vars(args), model=model)
model, test_time_pool = (
model,
False) if args.no_test_pool else apply_test_time_pool(model, config)
if args.num_gpu > 1:
model = torch.nn.DataParallel(model,
device_ids=list(range(
args.num_gpu))).cuda()
else:
model = model.cuda()
loader = create_loader(
ImageDataset(args.data),
input_size=config['input_size'],
batch_size=args.batch_size,
use_prefetcher=True,
interpolation=config['interpolation'],
mean=config['mean'],
std=config['std'],
num_workers=args.workers,
crop_pct=1.0 if test_time_pool else config['crop_pct'])
model.eval()
k = min(args.topk, args.num_classes)
batch_time = AverageMeter()
end = time.time()
topk_ids = []
with torch.no_grad():
for batch_idx, (input, _) in enumerate(loader):
input = input.cuda()
labels = model(input)
topk = labels.topk(k)[1]
topk_ids.append(topk.cpu().numpy())
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % args.log_freq == 0:
_logger.info(
'Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})'
.format(batch_idx, len(loader), batch_time=batch_time))
topk_ids = np.concatenate(topk_ids, axis=0).squeeze()
with open(os.path.join(args.output_dir, 'topk_ids.csv'), 'w') as out_file:
filenames = loader.dataset.filenames(basename=True)
for filename, label in zip(filenames, topk_ids):
out_file.write('{0},{1},{2},{3},{4},{5}\n'.format(
filename, label[0], label[1], label[2], label[3], label[4]))
def main():
setup_default_logging()
args = parser.parse_args()
validate(args)
def main():
setup_default_logging()
args, args_text = _parse_args()
# args = parser.parse_args()
torch.manual_seed(args.seed) # ! mostly for aug on test
random.seed(args.seed)
np.random.seed(args.seed)
model_cfgs = []
model_names = []
if os.path.isdir(
args.checkpoint): # ! can pass in a directory of checkpoints
# validate all checkpoints in a path with same model
checkpoints = glob.glob(args.checkpoint + '/*.pth.tar')
checkpoints += glob.glob(args.checkpoint + '/*.pth')
model_names = list_models(args.model)
model_cfgs = [(args.model, c)
for c in sorted(checkpoints, key=natural_key)]
else:
if args.model == 'all':
# validate all models in a list of names with pretrained checkpoints
args.pretrained = True
model_names = list_models(pretrained=True)
model_cfgs = [(n, '') for n in model_names]
elif not is_model(args.model):
# model name doesn't exist, try as wildcard filter
model_names = list_models(args.model)
model_cfgs = [(n, '') for n in model_names]
results_file = args.results_file or './results-all.csv'
if len(model_cfgs):
_logger.info(
'Running bulk validation on these pretrained models: {}'.format(
', '.join(model_names)))
results = []
try:
start_batch_size = args.batch_size
for m, c in model_cfgs:
batch_size = start_batch_size
args.model = m
args.checkpoint = c
result = OrderedDict(model=args.model)
r = {}
while not r and batch_size >= args.num_gpu:
torch.cuda.empty_cache()
try:
args.batch_size = batch_size
print('Validating with batch size: %d' %
args.batch_size)
r, prediction, true_label = validate(args)
except RuntimeError as e:
if batch_size <= args.num_gpu:
print(
"Validation failed with no ability to reduce batch size. Exiting."
)
raise e
batch_size = max(batch_size // 2, args.num_gpu)
print("Validation failed, reducing batch size by 50%")
result.update(r)
if args.checkpoint:
result['checkpoint'] = args.checkpoint
results.append(result)
except KeyboardInterrupt as e:
pass
results = sorted(results, key=lambda x: x['top1'], reverse=True)
if len(results):
write_results(results_file, results)
else:
# ! eval one single model
results_file = re.sub(r'\.csv', '', results_file) + '-standard.csv'
_, prediction, true_label = validate(args)
from HAM10000 import helper
prediction = helper.softmax(
prediction) # softmax convert to range 0-1, sum to 1
if args.has_eval_label:
from sklearn.metrics import accuracy_score, balanced_accuracy_score
true_label_onehot = np.identity(
args.num_classes)[true_label] # array into 1 hot
# https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html
_logger.info(
' * sklearn multilabel probabilities accuracy_score {:.3f} '.
format(accuracy_score(true_label_onehot,
np.round(prediction))))
_logger.info(
' * sklearn max probabilities balanced_accuracy_score {:.3f} '.
format(
balanced_accuracy_score(
true_label,
helper.convert_max_1_other_0(prediction).argmax(
axis=1))))
# output csv, need to reorder columns
helper.save_output_csv(
prediction, obs_name=[],
output_name=results_file) # no name for each observation, use []
def validate(args):
setup_default_logging()
def setthresh():
if args.checkpoint.split("/")[-1].split(
"_")[0] in getthresholds.keys():
return getthresholds[args.checkpoint.split("/")[-1].split("_")[0]]
else:
a = []
[a.append(args.threshold) for x in range(4)]
return a
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
# create model
bench = create_model(
args.model,
bench_task='predict',
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'image_info_{args.anno}.json')
image_dir = args.anno
elif 'val' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'instances_{args.anno}.json')
image_dir = args.anno
# else:
# annotation_path = os.path.join(args.data, f'{args.anno}.json')
# image_dir = args.anno
print(os.path.join(args.data, image_dir), annotation_path)
dataset = CocoDetection(os.path.join(args.data, image_dir),
annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
pin_mem=args.pin_mem,
mean=args.mean,
std=args.std)
if 'test' in args.anno:
threshold = float(args.threshold)
# elif 'detector' in args.anno:
# threshold = min(getthresholds['d0'])
else:
threshold = .001
img_ids = []
results = []
writetofilearrtay = []
bench.eval()
batch_time = AverageMeter()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output = bench(input, target['img_scale'], target['img_size'])
output = output.cpu()
# print(target['img_id'])
sample_ids = target['img_id'].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
# if 'test' in args.anno :
# tempWritetoFile = []
# tempWritetoFile.append(getimageNamefromid(image_id))
for det in sample:
score = float(det[4])
if score < threshold: # stop when below this threshold, scores in descending order
coco_det = dict(image_id=image_id, category_id=-1)
img_ids.append(image_id)
results.append(coco_det)
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]),
sizes=target['img_size'].tolist()[0])
img_ids.append(image_id)
results.append(coco_det)
# exit()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
# if 'test' in args.anno :
if not os.path.exists(args.tosave):
os.makedirs(args.tosave)
from itertools import groupby
results.sort(key=lambda x: x['image_id'])
count = 0
for k, v in tqdm(groupby(results, key=lambda x: x['image_id'])):
# print(args.data +"/" + str(getimageNamefromid(k)))
img = drawonimage(
os.path.join(args.data, image_dir, str(getimageNamefromid(k))), v,
setthresh())
cv2.imwrite(args.tosave + "/" + str(getimageNamefromid(k)), img)
count += 1
# print(i['category_id']," ",i['bbox'][0]," ",i['bbox'][1]," ",i['bbox'][2]," ",i['bbox'][3]," ")
print("generated predictions for ", count, " images.")
return results
def validate(args):
setup_default_logging()
if args.amp:
if has_apex:
args.apex_amp = True
elif has_native_amp:
args.native_amp = True
assert not args.apex_amp or not args.native_amp, "Only one AMP mode should be set."
args.pretrained = args.pretrained or not args.checkpoint # might as well try to validate something
args.prefetcher = not args.no_prefetcher
# create model
with set_layer_config(scriptable=args.torchscript):
bench = create_model(
args.model,
bench_task='predict',
num_classes=args.num_classes,
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
soft_nms=args.soft_nms,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
model_config = bench.config
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
amp_autocast = suppress
if args.apex_amp:
bench = amp.initialize(bench, opt_level='O1')
print('Using NVIDIA APEX AMP. Validating in mixed precision.')
elif args.native_amp:
amp_autocast = torch.cuda.amp.autocast
print('Using native Torch AMP. Validating in mixed precision.')
else:
print('AMP not enabled. Validating in float32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
dataset = create_dataset(args.dataset, args.root, args.split)
input_config = resolve_input_config(args, model_config)
loader = create_loader(dataset,
input_size=input_config['input_size'],
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=input_config['interpolation'],
fill_color=input_config['fill_color'],
mean=input_config['mean'],
std=input_config['std'],
num_workers=args.workers,
pin_mem=args.pin_mem)
evaluator = create_evaluator(args.dataset, dataset, pred_yxyx=False)
bench.eval()
batch_time = AverageMeter()
end = time.time()
last_idx = len(loader) - 1
with torch.no_grad():
for i, (input, target) in enumerate(loader):
with amp_autocast():
output = bench(input, img_info=target)
evaluator.add_predictions(output, target)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0 or i == last_idx:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg))
mean_ap = 0.
if dataset.parser.has_labels:
mean_ap = evaluator.evaluate()
else:
evaluator.save(args.results)
return mean_ap
def main():
setup_default_logging()
args, args_text = _parse_args()
args.device = 'cuda:0'
args.world_size = 1
args.rank = 0 # global rank
_logger.info('Training with a single process on %d GPUs.' % args.num_gpu)
torch.manual_seed(args.seed + args.rank)
# prepare model
model = create_model(args.model,
args.encoder,
pretrained=args.pretrained,
num_classes=args.num_classes,
checkpoint_path=args.initial_checkpoint)
# prepare optimizer
optimizer = create_optimizer(args, model)
# prepare scheduler
lr_scheduler, num_epochs = create_scheduler(args, optimizer)
_logger.info('Scheduled epochs: {}'.format(num_epochs))
# prepare dataset
folder = args.data_folder
train_fold = args.train_fold
images = np.load(f'{folder}/images/fold{train_fold}_images.npy')
masks = np.load(f'{folder}/masks/fold{train_fold}_masks.npy')
types = np.load(f'{folder}/types/fold{train_fold}_types.npy')
valid_fold = args.valid_fold
images_val = np.load(f'{folder}/images/fold{valid_fold}_images.npy')
masks_val = np.load(f'{folder}/masks/fold{valid_fold}_masks.npy')
types_val = np.load(f'{folder}/types/fold{valid_fold}_types.npy')
if args.no_aug:
train_dataset = PanNukeDataset(images, masks, types,
get_valid_transforms())
else:
train_dataset = PanNukeDataset(images, masks, types,
get_training_trasnforms(args.aug_type))
val_dataset = PanNukeDataset(images_val, masks_val, types_val,
get_valid_transforms())
loaders = {
'train':
DataLoader(train_dataset,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True,
shuffle=True),
'valid':
DataLoader(val_dataset,
batch_size=args.batch_size *
args.validation_batch_size_multiplier,
num_workers=args.workers,
pin_memory=True,
shuffle=False)
}
# save config
output_dir = ''
output_base = args.output if args.output else './logs'
exp_name = '-'.join([
datetime.now().strftime("%Y%m%d-%H%M%S"), args.model, args.encoder,
args.aug_type,
args.opt.lower()
])
output_dir = get_outdir(output_base, 'train', exp_name)
with open(os.path.join(output_dir, 'args.yaml'), 'w') as f:
f.write(args_text)
criterion, criterion_names = create_criterion(args)
callbacks = create_callbacks(args, criterion_names)
eval_metric = args.eval_metric
minimize_metric = True if eval_metric == 'loss' else False
runner = SupervisedRunner(input_key=args.input_key,
input_target_key=args.input_target_key)
# set fp16
if args.fp16:
fp16_params = dict(opt_level="O1") # params for FP16
_logger.info('Using fp16 O1')
else:
fp16_params = None
_logger.info('Not using fp16 O1')
runner.train(
model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=lr_scheduler,
loaders=loaders,
callbacks=callbacks,
logdir=output_dir,
num_epochs=num_epochs,
main_metric=eval_metric,
minimize_metric=minimize_metric,
verbose=True,
fp16=fp16_params,
)
