def train(
epoch: int,
dataset: tf.data.Dataset,
model: tf.keras.models.Model,
loss_fn: LossFunction,
optimizer: tf.optimizers.Optimizer,
args: dict,
metrics: dict,
):
#for batch_idx, data in enumerate(dataset):
train_iter = dataset.take(10000 // hvd.size())
compress = args.get('fp16_allreduce', True)
log_interval = args.get('log_interval', 10)
for batch_idx, data in enumerate(train_iter):
first_batch = (epoch == 0 and batch_idx == 0)
loss, output = train_step(data,
model,
loss_fn,
optimizer,
first_batch,
compress=compress)
metrics['train_accuracy'].update_state(data[1], output)
if batch_idx % log_interval == 0:
metrics_ = {
'epoch': epoch,
'loss': loss,
'accuracy': metrics['train_accuracy'].result(),
}
io.print_metrics(metrics_, pre=f'[{hvd.rank()}] ', logger=logger)
def evaluate_mcmot_seq(gt_path, res_path):
"""
"""
if not (os.path.isfile(gt_path) and os.path.isfile(gt_path)):
print('[Err]: invalid file path.')
return
# metric_name2id = defaultdict(int)
# metric_id2name = defaultdict(str)
# for id, name in enumerate(metric_names):
# metric_id2name[id] = name
# metric_name2id[name] = id
# read txt file
trackDB = read_txt_to_struct(res_path)
gtDB = read_txt_to_struct(gt_path)
# compute for each object class
metrics = np.zeros((len(id2cls.keys()), len(metric_names)), dtype=float)
for cls_id in id2cls.keys():
selected = np.where(cls_id == gtDB[:, 7])[0]
cls_gtDB = gtDB[selected]
print('gt: {:d} items for object class {:s}'.format(
len(cls_gtDB), id2cls[cls_id]))
if len(cls_gtDB) == 0:
continue
selected = np.where(cls_id == trackDB[:, 7])[0]
cls_resDB = trackDB[selected]
print('res: {:d} items for object class {:s}'.format(
len(cls_resDB), id2cls[cls_id]))
if len(cls_resDB) == 0:
continue
# ---------- main function to do evaluation
cls_metrics, cls_extra_info = evaluate_seq(cls_resDB,
cls_gtDB,
distractor_ids=None)
metrics[cls_id] = cls_metrics
# ----------
print_metrics('Seq evaluation for class {:s}'.format(id2cls[cls_id]),
cls_metrics)
# ---------- mean of the metrics
mean_metrics = metrics.mean(axis=0) # mean value of each column
print_metrics('Seq evaluation mean metrics:', mean_metrics)
def evaluate_tracking(sequences, track_dir, gt_dir):
all_info = []
for seqname in sequences:
track_res = os.path.join(track_dir, seqname, 'res.txt')
gt_file = os.path.join(gt_dir, seqname, 'gt.txt')
assert os.path.exists(track_res) and os.path.exists(
gt_file
), 'Either tracking result or groundtruth directory does not exist'
trackDB = read_txt_to_struct(track_res)
gtDB = read_txt_to_struct(gt_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
metrics, extra_info = evaluate_sequence(trackDB, gtDB, distractor_ids)
print_metrics(seqname + ' Evaluation', metrics)
all_info.append(extra_info)
all_metrics = evaluate_bm(all_info)
print_metrics('Summary Evaluation', all_metrics)
def evaluate_seqs(seqs, track_dir, gt_dir):
all_info = []
for seq_name in seqs: # process every seq
track_res = os.path.join(track_dir, seq_name, 'res.txt')
gt_file = os.path.join(gt_dir, seq_name, 'gt.txt')
assert os.path.exists(track_res) and os.path.exists(gt_file), \
'Either tracking result {} or ' \
'groundtruth directory {} does not exist'.format(track_res, gt_file)
trackDB = read_txt_to_struct(track_res) # track result
gtDB = read_txt_to_struct(gt_file) # ground truth
# filtering for specific class id
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
# ---------- main function to do evaluation
metrics, extra_info = evaluate_seq(trackDB, gtDB, distractor_ids)
# ----------
print_metrics(seq_name + ' Evaluation', metrics)
all_info.append(extra_info)
all_metrics = evaluate_bm(all_info)
print_metrics('Summary Evaluation', all_metrics)
def train(
epoch: int,
data: DistributedDataObject,
device: torch.device,
rank: int,
model: nn.Module,
loss_fn: LossFunction,
optimizer: optim.Optimizer,
args: dict,
scaler: GradScaler = None,
):
model.train()
# Horovod: set epoch to sampler for shuffling
data.sampler.set_epoch(epoch)
running_loss = torch.tensor(0.0)
training_acc = torch.tensor(0.0)
if torch.cuda.is_available():
running_loss = running_loss.to(device)
training_acc = training_acc.to(device)
for batch_idx, (batch, target) in enumerate(data.loader):
if torch.cuda.is_available():
batch, target = batch.to(device), target.to(device)
optimizer.zero_grad()
output = model(batch)
loss = loss_fn(output, target)
if scaler is not None:
scaler.scale(loss).backward()
scaler.step(optimizer)
scaler.update()
else:
loss.backward()
optimizer.step()
pred = output.data.max(1, keepdim=True)[1]
acc = pred.eq(target.data.view_as(pred)).cpu().float().sum()
training_acc += acc
running_loss += loss.item()
if batch_idx % args.log_interval == 0:
metrics_ = {
'epoch': epoch,
'batch_loss': loss.item() / args.batch_size,
'running_loss': running_loss / len(data.sampler),
'batch_acc': acc.item() / args.batch_size,
'training_acc': training_acc / len(data.sampler),
}
jdx = batch_idx * len(batch)
frac = 100. * batch_idx / len(data.loader)
pre = [
f'[{rank}]',
f'[{jdx:>5}/{len(data.sampler):<5} ({frac:>03.1f}%)]'
]
io.print_metrics(metrics_, pre=pre, logger=logger)
running_loss = running_loss / len(data.sampler)
training_acc = training_acc / len(data.sampler)
loss_avg = metric_average(running_loss)
training_acc = metric_average(training_acc)
if rank == 0:
logger.log(f'training set; avg loss: {loss_avg:.4g}, '
f'accuracy: {training_acc * 100:.2f}%')