def to_dict(self): d = asdict(self) return reduce( lambda acc, f: update_in(acc, f[1], lambda _: d[f[0].name]), to_fields, {}, )
def or_bitfields(bitfields: List[Bitfield]) -> Bitfield: byte_slices = zip_longest(*bitfields) if len(set((len(b) for b in bitfields))) != 1: raise ValueError("The bitfield sizes are different") return Bitfield(bytes(map(reduce(operator.or_), byte_slices)))
def createIndexConverter( self, indexMap: IndexMap, inShape: Shape, outShape: Shape, boole: bool = False ) -> IndexConverter: if indexMap == {}: return NullIndexConverter() if boole and set(indexMap.values()) == {(0,)}: return UnitIndexConverter(len(self.outAst.shape), 1) try: indiceConverters = list(reduce( lambda ls, i: ls + [self.getIndiceConverter(valmap(lambda v, i=i: v[i], indexMap), ls, boole)], range(len(inShape)), [[(0, 0, 0, 1)] * len(outShape)]))[1:] if not indiceConverters: return ZeroIndexConverter() genCoeffs = lambda func, func0=identity: [func0([func(c) for c in l]) for l in indiceConverters] linearCoeffs = genCoeffs(nth(0)) bs = genCoeffs(nth(1), sum) indexModCoeffs = genCoeffs(nth(2)) indexModValues = genCoeffs(nth(3)) if boole: return LinearIndiceConverter(linearCoeffs[0], bs[0], inShape[0], indexModCoeffs[0], indexModValues[0]) else: return LinearIndexConverter(linearCoeffs, bs, inShape, indexModCoeffs, indexModValues) except LinearError: if all([s != -1 for s in outShape]): return FixIndexConverter(indexMap) else: raise TransformError("Fail to find a transform.")
def to_dict(self, convert_values: bool = False) -> MutableMapping[str, Any]: to_fields = curried.pipe( fields(self.__class__), curried.map(lambda a: (a, curried.get_in([to_key], a.metadata))), curried.filter(lambda f: f[1]), list, ) if convert_values: d = asdict(self) else: d = { a.name: getattr(self, a.name) for a in fields(self.__class__) } if not to_fields: return d return curried.reduce( lambda acc, f: curried.update_in(acc, f[1], lambda _: d[f[0]. name]), to_fields, {}, )
def validate(predicts, dataset, batch_size): loader = DataLoader( dataset, batch_size=batch_size, pin_memory=True, shuffle=False, ) y_preds = np.array(predicts).mean(axis=0).argmax(axis=1) y_trues = pipe( loader, map(lambda x: x['label'].cpu().detach().tolist()), reduce(lambda x, y: x + y), np.array, ) score = iou( y_preds, y_trues, ) tn, fp, fn, tp = confusion_matrix(y_trues, y_preds).ravel() return { 'TPR': tp / (tp + fn), 'FNR': fn / (tp + fn), 'FPR': fp / (fp + tn), 'acc': (tp + tn) / (tp + tn + fp + fn), 'pre': tp / (tp + fp), 'iou': tp / (fn + tp + fp), }
def predict( model_dirs, dataset, out_path, batch_size=512, ): device = torch.device("cuda") models = pipe(model_dirs, map(lambda x: os.path.join(x, '*.pt')), map(glob.glob), concat, map(torch.load), map(lambda x: x.eval().to(device)), list) loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, pin_memory=True, ) rows = [] y_preds = [] y_ids = [] with torch.no_grad(): for sample in loader: ids = sample['id'] palser_x = sample['palsar'].to(device) normal_outputs = pipe( models, map(lambda x: x(palser_x)[0]), list, ) output = pipe( [*normal_outputs], map(lambda x: x.softmax(dim=1)), reduce(lambda x, y: (x + y) / 2), lambda x: x.argmax(dim=1), ) y_ids += ids y_preds += output.cpu().detach().tolist() rows = pipe(zip(y_ids, y_preds), map(lambda x: { 'id': x[0], 'lable': x[1] }), list) df = pd.DataFrame(rows) df.to_csv(out_path, sep='\t', header=False, index=False) return out_path
def validate( models, loader, ): models = pipe(models, map(lambda x: x.eval()), list) device = torch.device("cuda") y_preds = [] y_trues = [] sum_loss = 0 batch_len = 0 for sample in loader: with torch.no_grad(): palsar_x = sample['palsar'].to(device) landsat_y = sample['landsat'].to(device) labels = sample['label'].to(device) label_preds = pipe(models, map(lambda x: x(palsar_x)[0].softmax(dim=1)), reduce(lambda x, y: (x + y) / 2)) y_preds += label_preds.argmax(dim=1).cpu().detach().tolist() y_trues += labels.cpu().detach().tolist() batch_len += 1 score = iou( y_preds, y_trues, ) tn, fp, fn, tp = confusion_matrix(y_trues, y_preds).ravel() return { 'TPR': tp / (tp + fn), 'FNR': fn / (tp + fn), 'FPR': fp / (fp + tn), 'acc': (tp + tn) / (tp + tn + fp + fn), 'pre': tp / (tp + fp), 'iou': tp / (fn + tp + fp), }
def test_reduce(): assert reduce(add)((1, 2, 3)) == 6
def predict( model_paths, dataset, log_dir, hdf5_path, log_interval=100, ): dataset.df.sort_index(inplace=True) loader = DataLoader(dataset, batch_size=1, shuffle=False) device = torch.device('cpu') if torch.cuda.is_available(): device = torch.device("cuda") models = pipe(model_paths, map(torch.load), map(lambda x: x.to(device)), list) for m in models: m.eval() df = pd.DataFrame() sample_ids = [] rle_masks = [] scores = [] n_itr = 0 images = [] ids = [] with torch.no_grad(): for sample in loader: sample_id = sample['id'][0] image = sample['image'].to(device) ids.append(str(sample_id)) normal_outputs = pipe( models, map(lambda x: x(image)[0]), list, ) images.append(normal_outputs[0][0, 1, :, :]) fliped_outputs = pipe( models, map(lambda x: x(image.flip([3]))[0].flip([3])), list, ) output = pipe([*normal_outputs, *fliped_outputs], map(lambda x: x.softmax(dim=1)), reduce(lambda x, y: x + y / 2), lambda x: F.softmax(x, dim=1), lambda x: x.argmax(dim=1).float()) sample_ids.append(sample_id) rle_masks.append(rl_enc(output.cpu().numpy().reshape(101, 101))) log_images = [image[0], output] if 'mask' in sample.keys(): mask = sample['mask'].to(device)[0] log_images.append(mask) score = iou(output.cpu().numpy(), mask.cpu().numpy()) scores.append(score) if n_itr % log_interval == 0: with SummaryWriter(log_dir) as w: w.add_image(f"predict", vutils.make_grid(log_images, scale_each=True), n_itr) n_itr += 1 df['id'] = sample_ids df['rle_mask'] = rle_masks if len(scores) > 0: df['score'] = scores score = df['score'].mean() with SummaryWriter(log_dir) as w: w.add_text('score', f'score: {score}') df = df.set_index('id') ids = pipe(ids, map(lambda x: x.encode('ascii', 'ignore')), list) images = torch.stack(images).cpu().numpy().astype(np.float16) with h5py.File(hdf5_path, 'w') as f: f.create_dataset('id', data=ids) f.create_dataset('mask', data=images) return df
def train_multi( model_dir, sets, model_type, model_kwargs, epochs, batch_size, log_dir, landsat_weight, lr, num_ensamble, neg_scale, ): model_dir = Path(model_dir) model_dir.mkdir() device = torch.device("cuda") Model = getattr(mdl, model_type) models = pipe(range(num_ensamble), map(lambda _: Model(**model_kwargs).to(device).train()), list) optimizers = pipe( models, map(lambda x: optim.Adam(x.parameters(), amsgrad=True, lr=lr)), list, ) model_paths = pipe( range(num_ensamble), map(lambda x: model_dir / f'{x}.pt'), list, ) check_model_paths = pipe( range(num_ensamble), map(lambda x: model_dir / f'{x}_check.pt'), list, ) pos_set = pipe(range(neg_scale), map(lambda _: sets['train_pos']), reduce(lambda x, y: x + y)) train_pos_loader = DataLoader( pos_set, batch_size=batch_size // 2, shuffle=True, pin_memory=True, ) train_neg_loaders = pipe( range(num_ensamble), map(lambda x: DataLoader( sets['train_neg'], batch_size=batch_size // 2, pin_memory=True, sampler=ChunkSampler( epoch_size=len(pos_set), len_indices=len(sets['train_neg']), shuffle=True, start_at=x, ), )), list, ) val_set = sets['val_neg'] + sets['val_pos'] val_loader = DataLoader( val_set, batch_size=batch_size, pin_memory=True, shuffle=False, ) batch_len = len(train_pos_loader) max_val_score = 0 max_iou_train = 0 min_vial_loss = 0 mean_train_pos_loss = 0 mean_train_neg_loss = 0 mean_val_pos_loss = 0 mean_val_neg_loss = 0 min_train_pos_loss = 1 for epoch in range(epochs): sum_train_loss = 0 sum_val_loss = 0 sum_train_score = 0 sum_val_score = 0 val_probs = [] val_labels = [] train_probs = [] train_labels = [] traineds = pipe( zip(models, optimizers, train_neg_loaders), map(lambda x: train_epoch( model=x[0], optimizer=x[1], neg_loader=x[2], pos_loader=train_pos_loader, criterion=criterion(landsat_weight), device=device, )), list, ) train_loss = pipe(traineds, map(lambda x: x[1]), list, np.mean) models = pipe( traineds, map(lambda x: x[0]), list, ) metrics = validate( models=models, loader=val_loader, ) with SummaryWriter(log_dir) as w: w.add_scalars('loss', { 'train': train_loss, }, epoch) w.add_scalars('score', {**metrics}, epoch) if max_val_score <= metrics['iou']: max_val_score = metrics['iou'] w.add_text('iou', f"val: {metrics['iou']}, epoch: {epoch}", epoch) pipe(zip(models, model_paths), map(lambda x: torch.save(x[0], x[1])), list) return model_dir