def prefetch_test(opt): os.environ['CUDA_VISIBLE_DEVICES'] = str(opt.gpus) opt = opts().update_dataset_info_and_set_heads(opt, COCO) print(opt) Logger(opt) split = 'val' if not opt.trainval else 'test' dataset = COCO(opt, split) detector = CtdetDetector(opt) data_loader = torch.utils.data.DataLoader(PrefetchDataset(opt, dataset, pre_process), batch_size=1, shuffle=False, num_workers=1, pin_memory=True) results = {} num_iters = len(dataset) bar = Bar(f'{opt.exp_id}', max=num_iters) time_stats = ['tot', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} for i, (img_id, images) in enumerate(data_loader): ret = detector.run(images) results[img_id.numpy().astype(np.int32)[0]] = ret['results'] bar.suffix = f'{i}/{num_iters}|Elapsed: {bar.elapsed_td} |ETA: {bar.eta_td} ' for t in avg_time_stats: avg_time_stats[t].update(ret[t]) bar.suffix = bar.suffix + '|{} {tm.val:.3f} ({tm.avg:.3f}) '.format(t, tm=avg_time_stats[t]) bar.next() bar.finish() dataset.run_eval(results, opt.save_dir)
def fit(self, train_loader, valid_loader, start_epoch=0, max_epochs=200): best_acc = 0. bar = IncrementalBar(max=max_epochs - start_epoch) for e in range(start_epoch, max_epochs): bar.message = '{:>5.2f}%%'.format(bar.percent) bar.suffix = '{}/{} [{}<{}\t{:.2f}it/s]'.format( bar.index, bar.max, bar.elapsed_td, bar.eta_td, bar.avg) bar.next() if e == self.milestones[0]: schedule_lr(self.optimizer) # update learning rate once if e == self.milestones[1]: schedule_lr(self.optimizer) if e == self.milestones[2]: schedule_lr(self.optimizer) self.train(train_loader, self.model, self.criterion, self.optimizer, e) accuracy, best_threshold, roc_curve_tensor = self.evaluate( self.conf, *valid_loader['agedb_30']) self.board_val('agedb_30', accuracy, best_threshold, roc_curve_tensor, e) if accuracy > best_acc: best_acc = accuracy save_checkpoint(self.model, self.optimizer, self.conf, best_acc, e) bar.finish()
def load(self, fp): self.size = 0 file_size = os.fstat(fp.fileno()).st_size nblocks = 1 + (file_size - 1) // self.blocksize bar = IncrementalBar('Computing', max=nblocks) bar.suffix = '%(percent).1f%% - %(eta)ds' for block in bar.iter(file_read_iterator(fp, self.blocksize)): self.append(self._hash_block(block)) self.size += len(block)
def get_progress_bar(**kwargs): bar = IncrementalBar(**kwargs) bar.suffix = "%(index)d / %(max)d (%(elapsed_td)s)" return bar
def _run_epoch(self, model, dataloader, optimize=False, save_activations=False, reweight=None, bit_pretrained=False, adv_metrics=False): """Runs the model on a given dataloader. Note: The latter item in the returned tuple is what is necessary to run GEORGECluster.train and GEORGECluster.evaluate. Args: model(nn.Module): A PyTorch model. dataloader(DataLoader): The dataloader. The dataset within must subclass GEORGEDataset. optimize(bool, optional): If True, the model is trained on self.criterion. save_activations(bool, optional): If True, saves the activations in `outputs`. Default is False. bit_pretrained(bool, optional): If True, assumes bit_pretrained and does not evaluate performance metrics Returns: metrics(Dict[str, Any]) A dictionary object that stores the metrics defined in self.config['metric_types']. outputs(Dict[str, Any]) A dictionary object that stores artifacts necessary for model analysis, including labels, activations, and predictions. """ dataset = dataloader.dataset self._check_dataset(dataset) type_to_num_classes = { label_type: dataset.get_num_classes(label_type) for label_type in LABEL_TYPES if label_type in dataset.Y_dict.keys() } outputs = { 'metrics': None, 'activations': [], 'superclass': [], 'subclass': [], 'true_subclass': [], 'alt_subclass': [], 'targets': [], 'probs': [], 'preds': [], 'losses': [], 'reweight': [], } activations_handle = self._init_activations_hook( model, outputs['activations']) if optimize: progress_prefix = 'Training' model.train() else: progress_prefix = 'Evaluation' model.eval() per_class_meters = self._init_per_class_meters(type_to_num_classes) metric_meters = { k: AverageMeter() for k in ['loss', 'acc', 'loss_rw', 'acc_rw'] } progress = self.config['show_progress'] if progress: bar = ProgressBar(progress_prefix, max=len(dataloader), width=50) for batch_idx, (inputs, targets) in enumerate(dataloader): batch_size = len(inputs) if self.use_cuda: inputs, targets = move_to_device([inputs, targets], device=self.device) type_to_labels = {} for label_type in type_to_num_classes.keys(): type_to_labels[label_type] = targets[label_type] outputs[label_type].append(targets[label_type]) if optimize and not bit_pretrained: logits = model(inputs) loss_targets = targets['superclass'] co = self.criterion(logits, loss_targets, targets['subclass']) loss, (losses, corrects), _ = co self.optimizer.zero_grad() loss.backward() self.optimizer.step() else: with torch.no_grad(): logits = model(inputs) loss_targets = targets['superclass'] if bit_pretrained: if progress: bar.suffix = PROGRESS_BAR_SUFFIX.format( batch=batch_idx + 1, size=len(dataloader), total=format_timedelta(bar.elapsed_td), eta=format_timedelta(bar.eta_td), **{k: 0 for k in prog_metric_names}) bar.next() continue co = self.criterion(logits, loss_targets, targets['subclass']) loss, (losses, corrects), _ = co if not save_activations: outputs['activations'].pop() # delete activations reweight_vec = None if reweight is None else reweight[ targets['true_subclass']] metrics = self._compute_progress_metrics(losses, corrects, type_to_labels, type_to_num_classes, per_class_meters, reweight=reweight_vec) acc, preds = compute_accuracy(logits.data, loss_targets.data, return_preds=True) outputs['probs'].append( F.softmax(logits, dim=1).detach().cpu()[:, 1]) outputs['preds'].append(preds) outputs['losses'].append(losses.detach().cpu()) outputs['targets'].append(loss_targets.detach().cpu()) if reweight_vec is not None: outputs['reweight'].append(reweight_vec.cpu()) self._update_metrics(metric_meters, acc, loss, losses, corrects, batch_size, reweight_vec) PROGRESS_BAR_STR = PROGRESS_BAR_SUFFIX if self.compute_auroc: sub_map = dataloader.dataset.get_class_map('subclass') assert (set(sub_map.keys()) == {0, 1}) # must be a binary problem targets_cat, probs_cat = torch.cat( outputs['targets']), torch.cat(outputs['probs']) auroc = compute_roc_auc(targets_cat, probs_cat) metrics['auroc'] = auroc has_alt_subclass = 'alt_subclass' in dataloader.dataset.Y_dict for key in ['subclass', 'true_subclass' ] + ['alt_subclass'] * has_alt_subclass: sub_map = dataloader.dataset.get_class_map(key) neg_subclasses = sub_map[0] pos_subclasses = sub_map[1] if len(neg_subclasses) == len(pos_subclasses) == 1: # only one subclass in each superclass rob_auroc = auroc else: subclass_labels = torch.cat(outputs[key]) paired_aurocs = [] for neg_subclass in neg_subclasses: for pos_subclass in pos_subclasses: inds = ((subclass_labels == neg_subclass) | (subclass_labels == pos_subclass)).cpu() subset_pair_auroc = compute_roc_auc( targets_cat[inds], probs_cat[inds]) paired_aurocs.append(subset_pair_auroc) rob_auroc = min(paired_aurocs) metrics[f'{key}_rob_auroc'] = rob_auroc if not has_alt_subclass: metrics[alt_subclass_rob_auroc] = auroc PROGRESS_BAR_STR += ' | AUROC: {auroc:.4f} | R AUROC: {subclass_rob_auroc:.4f} | ' \ 'TR AUROC: {true_subclass_rob_auroc:.4f} | AR AUROC: {alt_subclass_rob_auroc:.4f}' if progress: bar.suffix = PROGRESS_BAR_STR.format( batch=batch_idx + 1, size=len(dataloader), total=format_timedelta(bar.elapsed_td), eta=format_timedelta(bar.eta_td), **{ **metrics, **{k: v.avg for k, v in metric_meters.items()} }) bar.next() if progress: bar.finish() if activations_handle: activations_handle.remove() for k, v in outputs.items(): if type(v) == list and len(v) > 0: outputs[k] = concatenate_iterable(v) if bit_pretrained: return outputs['metrics'], outputs outputs['metrics'] = metrics outputs['metrics'].update( {k: float(v.avg) for k, v in metric_meters.items()}) outputs['metrics'].update(self._compute_aggregate_metrics(outputs)) self._print_output_metrics(outputs) if adv_metrics: scaa = np.mean([ ga.avg * 100 for ga in np.array(per_class_meters[f'per_true_subclass_accs']) ]) self.logger.info( f'All accs: {[ga.avg * 100 for ga in np.array(per_class_meters[f"per_true_subclass_accs"])]}' ) self.logger.info(f'SCAA: {scaa:.3f}') ap = sklearn.metrics.average_precision_score( outputs['targets'], outputs['probs'], sample_weight=outputs['reweight'] if reweight_vec is not None else None) self.logger.info(f'MaP: {ap:.4f}') return outputs['metrics'], outputs
def train_torch(model, loader, loss_fn, optimizer, epochs, use_cuda, classification=True, name='Train'): model.train() epoch = 0 top1_accuracy = None top5_accuracy = None if classification: top1_accuracy = utils.AverageMeter() top5_accuracy = utils.AverageMeter() average_loss = utils.AverageMeter() bar = None if logger.getEffectiveLevel() == logging.INFO: bar = IncrementalBar(name, max=epochs * len(loader)) for i in range(epochs): if bar is not None: bar.suffix = 'Epoch {}/{}'.format(i + 1, epochs) for images, targets in loader: torch_images = torch.from_numpy(images) torch_targets = torch.from_numpy(targets) if classification: # Torch uses long ints for the labels torch_targets = torch_targets.long() torch_images.requires_grad_() if use_cuda: torch_images = torch_images.cuda() torch_targets = torch_targets.cuda() # Compute the outputs outputs = model(torch_images) loss = loss_fn(outputs, torch_targets) optimizer.zero_grad() loss.backward() optimizer.step() # Compute the statistics outputs = outputs.detach().cpu().numpy() loss = loss.detach().cpu().numpy() average_loss.update(loss, len(images)) if classification: top1_count = utils.top_k_count(outputs, targets, k=1) top5_count = utils.top_k_count(outputs, targets, k=5) top1_accuracy.update(1, top1_count) top1_accuracy.update(0, len(images) - top1_count) top5_accuracy.update(1, top5_count) top5_accuracy.update(0, len(images) - top5_count) if bar is not None: bar.next() logger.debug('\n=========') logger.debug('Epoch {}'.format(epoch)) logger.debug('=========\n') logger.debug('Average Loss: {:2.2e}'.format(average_loss.avg)) if classification: logger.debug('Top-1 Accuracy: {:2.2f}%'.format(top1_accuracy.avg * 100.0)) logger.debug('Top-5 Accuracy: {:2.2f}%'.format(top5_accuracy.avg * 100.0)) if bar is not None: bar.finish() if classification: top1_accuracy = top1_accuracy.avg top5_accuracy = top5_accuracy.avg return average_loss.avg, top1_accuracy, top5_accuracy
except Fault, fault: if fault.status != 409: raise else: return v if isinstance(fault.data, types.StringType): missing = json.loads(fault.data) elif isinstance(fault.data, types.ListType): missing = fault.data if '' in missing: del missing[missing.index(''):] bar = IncrementalBar('Uploading', max=len(missing)) bar.suffix = '%(percent).1f%% - %(eta)ds' with open(path) as fp: for hash in missing: offset = hashes.index(unhexlify(hash)) * blocksize fp.seek(offset) block = fp.read(blocksize) client.update_container_data(container, StringIO(block)) bar.next() bar.finish() return client.create_object_by_hashmap(container, object, map, **kwargs) def download(client, container, object, path): res = client.retrieve_object_hashmap(container, object)
def test(test_loader, model, optimizer, epoch, config, classes_name, segmentation_enable): # switch to evaluate mode model.eval() n_classes = config['yolo']['classes'] end = time.time() #bar = Bar('Validating', max=len(test_loader)) bar = IncrementalBar('Validating', max=len(test_loader), width=32) #for batch_idx, (inputs, targets) in enumerate(testloader): n_gt = [0] * n_classes correct = [0] * n_classes n_pred = [0] * n_classes n_iou = [0] * n_classes n_images = 0 det_boxes = list() det_labels = list() det_scores = list() true_boxes = list() true_labels = list() true_difficulties = list() gt_box = 0 pred_box = 0 for batch_idx, (images, targets) in enumerate(test_loader): images = images.to(device) # (batch_size (N), 3, H, W) labels = [torch.Tensor(l).to(device) for l in targets] bs = len(labels) # compute output with torch.no_grad(): if segmentation_enable: detections, _ = model( images ) # (N, num_defaultBoxes, 4), (N, num_defaultBoxes, n_classes) else: detections = model( images ) # (N, num_defaultBoxes, 4), (N, num_defaultBoxes, n_classes) for sample_i in range(bs): # Get labels for sample where width is not zero (dummies) # print(len(labels[0]),labels[sample_i]) target_sample = labels[sample_i] gt_box = gt_box + len(target_sample) tx1, tx2 = torch.unsqueeze( (target_sample[..., 1] - target_sample[..., 3] / 2), 1), torch.unsqueeze( (target_sample[..., 1] + target_sample[..., 3] / 2), 1) ty1, ty2 = torch.unsqueeze( (target_sample[..., 2] - target_sample[..., 4] / 2), 1), torch.unsqueeze( (target_sample[..., 2] + target_sample[..., 4] / 2), 1) box = torch.cat((tx1, ty1, tx2, ty2), 1) size = target_sample.size(0) true_boxes.append(box) true_labels.append(target_sample[..., 0]) true_difficulties.append(torch.zeros(size, requires_grad=False)) #print(detections[0][sample_i].shape,detections[1][sample_i].shape) preds = detections[sample_i] pred_box = pred_box + len(preds) if preds is not None: det_boxes.append(preds[..., :4]) det_labels.append((preds[..., 6] + 1).to(device)) conf = (preds[..., 4] * preds[..., 5]).to(device) det_scores.append(conf) else: empty = torch.empty(0).to(device) det_boxes.append(empty) det_labels.append(empty) det_scores.append(empty) n_images = n_images + 1 # measure elapsed time sum_gt = sum(n_gt) sum_n_pred = sum(n_pred) # plot progress bar.suffix = '({batch}/{size}) | Total: {total:} | ETA: {eta:}| n_img: {n_img:} | gt_box: {gt_box:} | pred_box: {pred_box:}'.format( batch=batch_idx + 1, size=len(test_loader), total=bar.elapsed_td, eta=bar.eta_td, n_img=n_images, gt_box=gt_box, pred_box=pred_box) bar.next() #if batch_idx == 50: # break bar.finish() print("\nVal conf. is %f\n" % (model.yolo_losses[0].val_conf)) model.yolo_losses[0].val_conf = adjust_confidence( gt_box, pred_box, model.yolo_losses[0].val_conf) model.yolo_losses[1].val_conf = adjust_confidence( gt_box, pred_box, model.yolo_losses[1].val_conf) # Calculate mAP APs, mAP, TP, FP = calculate_mAP(det_boxes, det_labels, det_scores, true_boxes, true_labels, true_difficulties, classes_name) pp.pprint(APs) print('\nMean Average Precision (mAP): %.3f' % mAP) return mAP
def train(train_loader, model, optimizer, epoch, sampler, segmentation_enable): model.train() bar = IncrementalBar('Training', max=len(sampler), width=12) #batch_time = AverageMeter() #data_time = AverageMeter() losses = AverageMeter() recall = [AverageMeter(), AverageMeter()] iou = [AverageMeter(), AverageMeter()] obj = [AverageMeter(), AverageMeter()] no_obj = [AverageMeter(), AverageMeter()] conf_loss = [AverageMeter(), AverageMeter()] cls_loss = [AverageMeter(), AverageMeter()] cls_score = [AverageMeter(), AverageMeter()] count = [AverageMeter(), AverageMeter()] seg_obj = AverageMeter() seg_no_obj = AverageMeter() #end = time.time() for batch_idx, (images, targets, total_num, seg_maps) in enumerate(train_loader): #print('\n1-',sum(sampler.get_mosaic_array()),'\n') #print('1-',sampler.mosaic_array,'\n') #print(targets) #data_time.update(time.time() - end) bs = images.size(0) #print(images.shape) #print(i,targets[0]) optimizer.zero_grad() images = images.to(device) # (batch_size (N), 3, H, W) if segmentation_enable: seg_maps = seg_maps.to( device) # (batch_size (N), H, W, num seg class) outputs, seg_out = model(images, targets, seg_maps) else: outputs = model(images, targets, seg_maps) #losses0 = yolo_losses[0](outputs[0],targets) #losses1 = yolo_losses[1](outputs[1],targets) t_loss = list() for i, l in enumerate(outputs): #print(l[0]) t_loss.append(l[0]) recall[i].update(l[1]) iou[i].update(l[2]) obj[i].update(l[3]) no_obj[i].update(l[4]) cls_score[i].update(l[5]) count[i].update(l[6]) #conf_loss.update(l[5]) #cls_loss.update(l[6]) loss = sum(t_loss) if segmentation_enable: seg_obj.update(seg_out[1]) seg_no_obj.update(seg_out[2]) loss += seg_out[0] losses.update(loss.item(), bs) loss.backward() optimizer.step() # measure elapsed time #batch_time.update(time.time() - end) #end = time.time() if segmentation_enable: bar.suffix = \ '%(percent)3d%% | {total:} | {loss:.4f} | {cnt:2.1f} | {iou:.3f} | {obj:.3f} | {no_obj:.4f} | {cls:.3f} | {rec:.4f} | {seg_obj:.3f} | {seg_no_obj:.6f} |'\ .format( total=bar.elapsed_td, loss=losses.avg, cnt=(count[0].avg+count[1].avg), iou=(iou[0].avg+iou[1].avg)/2., obj=(obj[0].avg+obj[1].avg)/2., no_obj=(no_obj[0].avg+no_obj[1].avg)/2., cls=(cls_score[0].avg+cls_score[1].avg)/2., rec=(recall[0].avg+recall[1].avg)/2., seg_obj=seg_obj.avg, seg_no_obj = seg_no_obj.avg ) else: bar.suffix = \ '%(percent)3d%% | {total:} | {loss:.4f} | {cnt1:2.1f} | {iou1:.3f} | {obj1:.3f} | {no_obj1:.4f} | {cls1:.3f} | {rec1:.3f} | {cnt2:2.1f} | {iou2:.3f} | {obj2:.3f} | {no_obj2:.4f} | {cls2:.3f} | {rec2:.3f} |'\ .format( #batch=batch_idx + 1, #size=len(train_loader), #data=data_time.avg, #bt=batch_time.avg, total=bar.elapsed_td, loss=losses.avg, #loss1=losses[0].avg, #loss2=losses[1].avg, cnt1=(count[0].avg), cnt2=(count[1].avg), #recall=recall.avg, iou1=iou[0].avg, iou2=iou[1].avg, obj1=obj[0].avg, no_obj1=no_obj[0].avg, cls1=cls_score[0].avg, obj2=obj[1].avg, no_obj2=no_obj[1].avg, cls2=cls_score[1].avg, rec1=recall[0].avg, rec2=recall[1].avg, #cls=cls_loss.avg, ) bar.next(total_num) bar.finish() return losses.avg, (iou[0].avg + iou[1].avg) / 2
from progress.bar import IncrementalBar def to_color(s, c): return f'\x1b[{c}m{s}\x1b[0m' if use_color else s def out_line(s='', color=37, indent=0, thread_name=None): t = '' if thread_name is not None and show_threads: t = to_color(f'<{thread_name}> ', thread_color) print(t + ' ' * indent + to_color(s, color)) def out(s='', color=37, indent=0, thread_name=None): for line in s.split('\n'): out_line(line, color, indent, thread_name) use_color = True show_threads = True thread_color = 34 # stuff for progress bars export_task_count = 0 filtered_export_task_count = 0 bar = IncrementalBar() bar.suffix = "%(index)d / %(max)d (%(elapsed_td)s)"