def objective(trial: optuna.trial.Trial) -> float:
model = nn.Sequential(nn.Linear(20, 1), nn.Sigmoid())
learn = Learner(
data_bunch,
model,
metrics=[accuracy],
callback_fns=[
partial(FastAIPruningCallback,
trial=trial,
monitor="valid_loss")
],
)
learn.fit(1)
return 1.0
def __init__(self,
data,
scales=None,
ratios=None,
backbone=None,
pretrained_path=None):
# Set default backbone to be 'resnet50'
if backbone is None:
backbone = models.resnet50
super().__init__(data, backbone)
# Check if a backbone provided is compatible, use resnet50 as default
if not self._check_backbone_support(backbone):
raise Exception(
f"Enter only compatible backbones from {', '.join(self.supported_backbones)}"
)
self.name = "RetinaNet"
self._code = code
self.scales = ifnone(scales, [1, 2**(-1 / 3), 2**(-2 / 3)])
self.ratios = ifnone(ratios, [1 / 2, 1, 2])
self._n_anchors = len(self.scales) * len(self.ratios)
self._data = data
self._chip_size = (data.chip_size, data.chip_size)
# Cut-off the backbone before the penultimate layer
self._encoder = create_body(self._backbone, -2)
# Initialize the model, loss function and the Learner object
self._model = RetinaNetModel(self._encoder,
n_classes=data.c - 1,
final_bias=-4,
chip_size=self._chip_size,
n_anchors=self._n_anchors)
self._loss_f = RetinaNetFocalLoss(sizes=self._model.sizes,
scales=self.scales,
ratios=self.ratios)
self.learn = Learner(data, self._model, loss_func=self._loss_f)
self.learn.split([self._model.encoder[6], self._model.c5top5])
self.learn.freeze()
if pretrained_path is not None:
self.load(str(pretrained_path))
def change_path_to_permanent(experiment_run: ExperimentRun, learner: Learner) -> None:
old_full_path: str = str(learner.path / learner.model_dir)
new_full_path: str = experiment_run.perm_path_to_model
new_path, new_model_dir = os.path.split(new_full_path)
if str(learner.path) != new_path:
raise AssertionError(f"When changing model path to permanent, path to models must remain the same. "
f"However trying to rename: {old_full_path} -> {new_full_path}")
learner.model_dir = new_model_dir
os.rename(old_full_path, new_full_path)
def main():
model = PSMNet(args.maxdisp, args.mindisp).cuda()
if args.load_model is not None:
if args.load is not None:
warn('args.load is not None. load_model will be covered by load.')
ckpt = torch.load(args.load_model, 'cpu')
if 'model' in ckpt.keys():
pretrained = ckpt['model']
elif 'state_dict' in ckpt.keys():
pretrained = ckpt['state_dict']
else:
raise RuntimeError()
pretrained = {
k.replace('module.', ''): v
for k, v in pretrained.items()
}
model.load_state_dict(pretrained)
train_dl = DataLoader(KITTIRoiDataset(args.data_dir, 'train',
args.resolution, args.maxdisp,
args.mindisp),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers)
val_dl = DataLoader(KITTIRoiDataset(args.data_dir, 'val', args.resolution,
args.maxdisp, args.mindisp),
batch_size=args.batch_size,
num_workers=args.workers)
loss_fn = PSMLoss()
databunch = DataBunch(train_dl, val_dl, device='cuda')
learner = Learner(databunch,
model,
loss_func=loss_fn,
model_dir=args.model_dir)
learner.callbacks = [
DistributedSaveModelCallback(learner),
TensorBoardCallback(learner)
]
if num_gpus > 1:
learner.to_distributed(get_rank())
if args.load is not None:
learner.load(args.load)
if args.mode == 'train':
learner.fit(args.epochs, args.maxlr)
elif args.mode == 'train_oc':
fit_one_cycle(learner, args.epochs, args.maxlr)
else:
raise ValueError('args.mode not supported.')
def get_list_from_model(learn:Learner, ds_type:DatasetType, batch:Tuple)->[]:
"Factory method to convert a batch of model images to a list of ModelImageSet."
image_sets = []
x,y = batch[0],batch[1]
preds = learn.pred_batch(ds_type=ds_type, batch=(x,y), reconstruct=True)
for orig_px, real_px, gen in zip(x,y,preds):
orig, real = Image(px=orig_px), Image(px=real_px)
image_set = ModelImageSet(orig=orig, real=real, gen=gen)
image_sets.append(image_set)
return image_sets
def colorize_crit_learner(
data: ImageDataBunch,
loss_critic=AdaptiveLoss(nn.BCEWithLogitsLoss()),
nf: int = 256,
) -> Learner:
return Learner(
data,
custom_gan_critic(nf=nf),
metrics=accuracy_thresh_expand,
loss_func=loss_critic,
wd=1e-3,
)
def __init__(self, data, model_conf, backbone=None, pretrained_path=None):
super().__init__(data, backbone)
self.model_conf = model_conf()
self.model_conf_class = model_conf
self._backend = 'pytorch'
model = self.model_conf.get_model(data, backbone)
if self._is_multispectral:
model = _change_tail(model, data)
if not _isnotebook() and os.name == 'posix':
_set_ddp_multigpu(self)
if self._multigpu_training:
self.learn = Learner(
data, model,
loss_func=self.model_conf.loss).to_distributed(
self._rank_distributed)
else:
self.learn = Learner(data,
model,
loss_func=self.model_conf.loss)
else:
self.learn = Learner(data, model, loss_func=self.model_conf.loss)
self.learn.callbacks.append(
self.train_callback(self.learn, self.model_conf.on_batch_begin))
self._code = code
self._arcgis_init_callback() # make first conv weights learnable
if pretrained_path is not None:
self.load(pretrained_path)
def run_alexnet(input_path, output_path, batch_size, epochs, learning_rate):
# Load image databunch
print("[INFO] Loading Data")
data = load_catsvsdog(input_path, batch_size)
# Defining the learner
alexnet_learner = Learner(
data=data,
model=ALEXNet(n_class=data.c),
loss_func=nn.CrossEntropyLoss(),
metrics=accuracy,
)
# Training the model
print("[INFO] Training started.")
alexnet_learner.fit_one_cycle(epochs, learning_rate)
# Validation accuracy
val_acc = int(
np.round(alexnet_learner.recorder.metrics[-1][0].numpy().tolist(), 3) *
1000)
# Saving the model
print("[INFO] Saving model weights.")
alexnet_learner.save("alexnet_catsvsdog_stg_1_" + str(val_acc))
# Evaluation
print("[INFO] Evaluating Network.")
evaluate_model(alexnet_learner, output_path, plot=True)
class Sequential:
def __init__(self, model=None):
self.layers = []
self.last_dim = None
self.model = model
self.device = torch.device('cpu')
if torch.cuda.is_available():
self.device = torch.device('cuda')
def add(self, layer):
layer = layer.get_layer(self.last_dim)
self.last_dim = layer['output_dim']
self.layers.extend(layer['layers'])
def compile(self, loss, optimizer=None):
if len(self.layers) > 0:
self.model = nn.Sequential(*self.layers)
self.loss = loss
def fit(self, x, y, bs, epochs, lr=1e-3, one_cycle=True, get_lr=True):
db = create_db(x, y, bs=bs)
self.learn = Learner(db, self.model, loss_func=self.loss)
if one_cycle:
self.learn.fit_one_cycle(epochs, lr)
else:
self.learn.fit(epochs, lr)
def lr_find(self, x, y, bs):
db = create_db(x, y, bs=bs)
learn = Learner(db, self.model, loss_func=self.loss)
learn.lr_find()
clear_output()
learn.recorder.plot(suggestion=True)
def predict(self, x):
self.learn.model.eval()
with torch.no_grad():
y_preds = self.learn.model(torch.Tensor(x).to(device))
return y_preds.cpu().numpy()
def compute_feature(im_or_path: Image, learn: Learner,
embedding_layer: Module) -> List[float]:
"""Compute features for a single image
Args:
im_or_path: Image or path to image
learn: Trained model to use as featurizer
embedding_layer: Number of columns on which to display the images
Returns: DNN feature of the provided image.
"""
if isinstance(im_or_path, str):
im = open_image(im_or_path, convert_mode="RGB")
else:
im = im_or_path
featurizer = SaveFeatures(embedding_layer)
featurizer.features = None
learn.predict(im)
feats = featurizer.features[0][:]
assert len(feats) > 1
featurizer.features = None
return feats
def do_train(
cfg,
model,
train_dl,
valid_dl,
optimizer,
loss_fn,
metrics=[],
callbacks: list = [],
):
log_period = cfg.SOLVER.LOG_PERIOD
checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD
output_dir = cfg.OUTPUT_DIR
device = cfg.MODEL.DEVICE
epochs = cfg.SOLVER.MAX_EPOCHS
data_bunch = DataBunch(train_dl, valid_dl)
learn = Learner(data_bunch, model, loss_func=loss_fn)
callbacks.append(LoggingLog(learn, "template_model.train"))
if metrics:
learn.metrics = metrics
learn.fit_one_cycle(epochs, cfg.SOLVER.BASE_LR)
def main(test, s3_data, batch, debug):
"""Train a semantic segmentation FPN model on the CamVid-Tiramisu dataset."""
if batch:
run_on_batch(test, debug)
# Setup options
batch_sz = 8
num_workers = 4
num_epochs = 20
lr = 1e-4
backbone_arch = 'resnet18'
sample_pct = 1.0
if test:
batch_sz = 1
num_workers = 0
num_epochs = 2
sample_pct = 0.01
# Setup data
tmp_dir_obj = tempfile.TemporaryDirectory()
tmp_dir = tmp_dir_obj.name
output_dir = local_output_uri
make_dir(output_dir)
data_dir = download_data(s3_data, tmp_dir)
data = get_databunch(data_dir,
sample_pct=sample_pct,
batch_sz=batch_sz,
num_workers=num_workers)
print(data)
plot_data(data, output_dir)
# Setup and train model
num_classes = data.c
model = SegmentationFPN(backbone_arch, num_classes)
metrics = [acc_camvid]
learn = Learner(data,
model,
metrics=metrics,
loss_func=SegmentationFPN.loss,
path=output_dir)
learn.unfreeze()
callbacks = [
SaveModelCallback(learn, monitor='valid_loss'),
CSVLogger(learn, filename='log'),
]
learn.fit_one_cycle(num_epochs, lr, callbacks=callbacks)
# Plot predictions and sync
plot_preds(data, learn, output_dir)
if s3_data:
sync_to_dir(output_dir, remote_output_uri)
def create_learner(cls,
k=None,
data=None,
model=None,
opt_func=None,
loss_func=None,
metrics=None,
**kargs):
"""opt_func should pass seprately
Args:
k: (Default value = None)
data: (Default value = None)
model: (Default value = None)
opt_func: (Default value = None)
loss_func: (Default value = None)
metrics: (Default value = None)
**kargs:
Returns:
"""
if k is not None:
data = k.data if hasattr(k,
'data') and k.data is not None else data
model = k.model if hasattr(
k, 'model') and k.model is not None else model
loss_func = k.model_loss if hasattr(
k, 'model_loss') and k.model_loss is not None else loss_func
metrics = k.model_metrics if hasattr(
k,
'model_metrics') and k.model_metrics is not None else metrics
assert data is not None
assert opt_func is not None
learner = Learner(
data,
model,
opt_func,
loss_func=loss_func,
metrics=metrics,
bn_wd=False,
**kargs) # opt_func postitional parameter is before loss_func
return learner
def compute_features_learner(
data, dataset_type: DatasetType, learn: Learner,
embedding_layer: Module) -> List[Dict[str, np.array]]:
"""Compute features for multiple image using mini-batching.
Use this function to featurize the training or test set of a learner
Args:
dataset_type: Specify train, valid or test set.
learn: Trained model to use as featurizer
embedding_layer: Number of columns on which to display the images
Note: this function processes each image at a time and is hence slower
compared to using mini-batches of >1.
Returns: DNN feature of the provided image.
"""
# Note: In Fastai, for DatasetType.Train, only the output of complete minibatches is computed. Ie if one has 101 images,
# and uses a minibatch size of 16, then len(feats) is 96 and not 101. For DatasetType.Valid this is not the case,
# and len(feats) is as expected 101. A way around this is to use DatasetType.Fix instead when referring to the training set.
# See e.g. issue: https://forums.fast.ai/t/get-preds-returning-less-results-than-length-of-original-dataset/34148
if dataset_type == DatasetType.Train or dataset_type == DatasetType.Fix:
dataset_type = (
DatasetType.Fix
) # Training set without shuffeling and no dropping of last batch. See note above.
label_list = list(data.train_ds.items)
elif dataset_type == DatasetType.Valid:
label_list = list(data.valid_ds.items)
elif dataset_type == DatasetType.Test:
label_list = list(data.test_ds.items)
else:
raise Exception(
"Dataset_type needs to be of type DatasetType.Train, DatasetType.Valid, DatasetType.Test or DatasetType.Fix."
)
# Compute features
featurizer = SaveFeatures(embedding_layer)
_ = learn.get_preds(dataset_type)
feats = featurizer.features[:]
# Get corresponding image paths
assert len(feats) == len(label_list)
im_paths = [str(x) for x in label_list]
return dict(zip(im_paths, feats))
def show_prediction_vs_actual(sample_idx: int, learn: Learner) -> ImageSegment:
"""Return predicted mask, additionally print input image and tile-level label"""
sample = learn.data.valid_ds[sample_idx]
image, label = sample
print("Label: " + str(label.__repr__()))
image.show()
batch = learn.data.one_item(image)
pred = learn.pred_batch(batch=batch).squeeze(dim=0)
img = pred.argmax(dim=0, keepdim=True)
predicted_colors = torch.zeros(len(ALL_CLASSES))
for i in img.unique():
predicted_colors[i] = 1
print("Predicted colors: " + str(predicted_colors))
image_segment = ImageSegment(img)
return image_segment
def run_mnist(input_path,
output_path,
batch_size,
epochs,
learning_rate,
model=Mnist_NN()):
path = Path(input_path)
## Defining transformation
ds_tfms = get_transforms(
do_flip=False,
flip_vert=False,
max_rotate=15,
max_zoom=1.1,
max_lighting=0.2,
max_warp=0.2,
)
## Creating Databunch
data = (ImageItemList.from_folder(path, convert_mode="L").split_by_folder(
train="training", valid="testing").label_from_folder().transform(
tfms=ds_tfms, size=28).databunch(bs=batch_size))
## Defining the learner
mlp_learner = Learner(data=data,
model=model,
loss_func=nn.CrossEntropyLoss(),
metrics=accuracy)
# Training the model
mlp_learner.fit_one_cycle(epochs, learning_rate)
val_acc = int(
np.round(mlp_learner.recorder.metrics[-1][0].numpy().tolist(), 3) *
1000)
## Saving the model
mlp_learner.save("mlp_mnist_stg_1_" + str(val_acc))
## Evaluation
print("Evaluating Network..")
interp = ClassificationInterpretation.from_learner(mlp_learner)
print(classification_report(interp.y_true, interp.pred_class))
## Plotting train and validation loss
mlp_learner.recorder.plot_losses()
plt.savefig(output_path + "/loss.png")
mlp_learner.recorder.plot_metrics()
plt.savefig(output_path + "/metric.png")
def __init__(self, data, pretrained_path=None, *args, **kwargs):
super().__init__(data, None)
if not HAS_FASTAI:
raise_fastai_import_error(import_exception=import_exception, message="This model requires module 'torch_geometric' to be installed.", installation_steps=' ')
self._backbone = None
self.sample_point_num = kwargs.get('sample_point_num', data.max_point)
self.learn = Learner(data,
PointCNNSeg(self.sample_point_num, data.c, data.extra_dim, kwargs.get('encoder_params', None), kwargs.get('dropout', None)),
loss_func=CrossEntropyPC(data.c),
metrics=[AverageMetric(accuracy)],
callback_fns=[partial(SamplePointsCallback, sample_point_num=self.sample_point_num)])
self.encoder_params = self.learn.model.encoder_params
self.learn.model = self.learn.model.to(self._device)
if pretrained_path is not None:
self.load(pretrained_path)
def compute_features_learner(
data, dataset_type: DatasetType, learn: Learner,
embedding_layer: Module) -> List[Dict[str, np.array]]:
"""Compute features for multiple image using mini-batching.
Use this function to featurize the training or test set of a learner
Args:
dataset_type: Specify train, valid or test set.
learn: Trained model to use as featurizer
embedding_layer: Number of columns on which to display the images
Note: this function processes each image at a time and is hence slower
compared to using mini-batches of >1.
Returns: DNN feature of the provided image.
"""
if dataset_type == DatasetType.Train:
label_list = list(data.train_ds.items)
elif dataset_type == DatasetType.Valid:
label_list = list(data.valid_ds.items)
elif dataset_type == DatasetType.Test:
label_list = list(data.test_ds.items)
else:
raise Exception(
"Dataset_type needs to be of type DatasetType.Train, DatasetType.Valid or DatasetType.Test."
)
featurizer = SaveFeatures(embedding_layer)
_ = learn.get_preds(dataset_type)
feats = featurizer.features[:]
# Get corresponding image paths
im_paths = [str(x) for x in label_list]
assert len(feats) == len(im_paths)
return dict(zip(im_paths, feats))
def run_shallownet(input_path, output_path, batch_size, epochs, learning_rate):
path = Path(input_path)
# Creating Databunch
data = (
ImageItemList.from_folder(path)
.split_by_folder(train="train", valid="test")
.label_from_folder()
.transform(tfms=None, size=32)
.databunch(bs=batch_size)
)
# Defining the learner
sn_learner = Learner(
data=data,
model=ShallowNet(n_class=data.c, size=32, in_channels=3),
loss_func=nn.CrossEntropyLoss(),
metrics=accuracy,
)
# Training the model
sn_learner.fit_one_cycle(epochs, learning_rate)
val_acc = int(
np.round(sn_learner.recorder.metrics[-1][0].numpy().tolist(), 3) * 1000
)
# Saving the model
sn_learner.save("sn_cifar10_stg_1_" + str(val_acc))
# Evaluation
print("Evaluating Network..")
interp = ClassificationInterpretation.from_learner(sn_learner)
print(classification_report(interp.y_true, interp.pred_class))
# Plotting train and validation loss
sn_learner.recorder.plot_losses()
plt.savefig(output_path + "/loss.png")
sn_learner.recorder.plot_metrics()
plt.savefig(output_path + "/metric.png")
def unet_learner_wide(
data: DataBunch,
arch: Callable,
pretrained: bool = True,
blur_final: bool = True,
norm_type: Optional[NormType] = NormType,
split_on: Optional[SplitFuncOrIdxList] = None,
blur: bool = False,
self_attention: bool = False,
y_range: Optional[Tuple[float, float]] = None,
last_cross: bool = True,
bottle: bool = False,
nf_factor: int = 1,
**kwargs: Any
) -> Learner:
"Build Unet learner from `data` and `arch`."
meta = cnn_config(arch)
body = create_body(arch, pretrained)
model = to_device(
DynamicUnetWide(
body,
n_classes=data.c,
blur=blur,
blur_final=blur_final,
self_attention=self_attention,
y_range=y_range,
norm_type=norm_type,
last_cross=last_cross,
bottle=bottle,
nf_factor=nf_factor,
),
data.device,
)
learn = Learner(data, model, **kwargs)
learn.split(ifnone(split_on, meta['split']))
if pretrained:
learn.freeze()
apply_init(model[2], nn.init.kaiming_normal_)
return learn
def run_ner(
lang: str = 'eng',
log_dir: str = 'logs',
task: str = NER,
batch_size: int = 1,
epochs: int = 1,
dataset: str = 'data/conll-2003/',
loss: str = 'cross',
max_seq_len: int = 128,
do_lower_case: bool = False,
warmup_proportion: float = 0.1,
rand_seed: int = None,
ds_size: int = None,
data_bunch_path: str = 'data/conll-2003/db',
tuned_learner: str = None,
do_train: str = False,
do_eval: str = False,
save: bool = False,
nameX: str = 'ner',
mask: tuple = ('s', 's'),
):
name = "_".join(
map(str, [
nameX, task, lang, mask[0], mask[1], loss, batch_size, max_seq_len,
do_train, do_eval
]))
log_dir = Path(log_dir)
log_dir.mkdir(parents=True, exist_ok=True)
init_logger(log_dir, name)
if rand_seed:
random.seed(rand_seed)
np.random.seed(rand_seed)
torch.manual_seed(rand_seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(rand_seed)
trainset = dataset + lang + '/train.txt'
devset = dataset + lang + '/dev.txt'
testset = dataset + lang + '/test.txt'
bert_model = 'bert-base-cased' if lang == 'eng' else 'bert-base-multilingual-cased'
print(f'Lang: {lang}\nModel: {bert_model}\nRun: {name}')
model = BertForTokenClassification.from_pretrained(bert_model,
num_labels=len(VOCAB),
cache_dir='bertm')
if tuned_learner:
print('Loading pretrained learner: ', tuned_learner)
model.bert.load_state_dict(torch.load(tuned_learner))
model = torch.nn.DataParallel(model)
model_lr_group = bert_layer_list(model)
layers = len(model_lr_group)
kwargs = {'max_seq_len': max_seq_len, 'ds_size': ds_size, 'mask': mask}
train_dl = DataLoader(dataset=NerDataset(trainset,
bert_model,
train=True,
**kwargs),
batch_size=batch_size,
shuffle=True,
collate_fn=partial(pad, train=True))
dev_dl = DataLoader(dataset=NerDataset(devset, bert_model, **kwargs),
batch_size=batch_size,
shuffle=False,
collate_fn=pad)
test_dl = DataLoader(dataset=NerDataset(testset, bert_model, **kwargs),
batch_size=batch_size,
shuffle=False,
collate_fn=pad)
data = DataBunch(train_dl=train_dl,
valid_dl=dev_dl,
test_dl=test_dl,
collate_fn=pad,
path=Path(data_bunch_path))
train_opt_steps = int(len(train_dl.dataset) / batch_size) * epochs
optim = BertAdam(model.parameters(),
lr=0.01,
warmup=warmup_proportion,
t_total=train_opt_steps)
loss_fun = ner_loss_func if loss == 'cross' else partial(ner_loss_func,
zero=True)
metrics = [Conll_F1()]
learn = Learner(
data,
model,
BertAdam,
loss_func=loss_fun,
metrics=metrics,
true_wd=False,
layer_groups=model_lr_group,
path='learn' + nameX,
)
learn.opt = OptimWrapper(optim)
lrm = 1.6
# select set of starting lrs
lrs_eng = [0.01, 5e-4, 3e-4, 3e-4, 1e-5]
lrs_deu = [0.01, 5e-4, 5e-4, 3e-4, 2e-5]
startlr = lrs_eng if lang == 'eng' else lrs_deu
results = [['epoch', 'lr', 'f1', 'val_loss', 'train_loss', 'train_losses']]
if do_train:
learn.freeze()
learn.fit_one_cycle(1, startlr[0], moms=(0.8, 0.7))
learn.freeze_to(-3)
lrs = learn.lr_range(slice(startlr[1] / (1.6**15), startlr[1]))
learn.fit_one_cycle(1, lrs, moms=(0.8, 0.7))
learn.freeze_to(-6)
lrs = learn.lr_range(slice(startlr[2] / (1.6**15), startlr[2]))
learn.fit_one_cycle(1, lrs, moms=(0.8, 0.7))
learn.freeze_to(-12)
lrs = learn.lr_range(slice(startlr[3] / (1.6**15), startlr[3]))
learn.fit_one_cycle(1, lrs, moms=(0.8, 0.7))
learn.unfreeze()
lrs = learn.lr_range(slice(startlr[4] / (1.6**15), startlr[4]))
learn.fit_one_cycle(1, lrs, moms=(0.8, 0.7))
if do_eval:
res = learn.validate(test_dl, metrics=metrics)
met_res = [f'{m.__name__}: {r}' for m, r in zip(metrics, res[1:])]
print(f'Validation on TEST SET:\nloss {res[0]}, {met_res}')
results.append(['val', '-', res[1], res[0], '-', '-'])
with open(log_dir / (name + '.csv'), 'a') as resultFile:
wr = csv.writer(resultFile)
wr.writerows(results)
num_workers=0)
v_data = DLDataLoader(v_chain,
collate_fn=dlc.gdf_col,
pin_memory=False,
num_workers=0)
databunch = DataBunch(t_data, v_data, collate_fn=dlc.gdf_col, device="cuda")
t_final = time() - start
print(t_final)
print("Creating model")
start = time()
model = TabularModel(emb_szs=embeddings,
n_cont=len(cont_names),
out_sz=2,
layers=[512, 256])
learn = Learner(databunch, model, metrics=[accuracy])
learn.loss_func = torch.nn.CrossEntropyLoss()
t_final = time() - start
print(t_final)
print("Finding learning rate")
start = time()
learn.lr_find()
learn.recorder.plot(show_moms=True, suggestion=True)
learning_rate = 1.32e-2
epochs = 1
t_final = time() - start
print(t_final)
print("Running Training")
start = time()
learn.fit_one_cycle(epochs, learning_rate)
t_final = time() - start
#siamese = SiameseNetwork(arch=arch)
#siamese = SiameseNet(emb_len=emb_len, arch=arch, forward_type='similarity', drop_rate=0.5)
siamese = SiameseNet(emb_len=emb_len,
arch=arch,
forward_type='distance',
drop_rate=0.5)
#siamese = SiameseNetwork2(arch=arch)
siamese.to(device)
# In[11]:
learn = Learner(
data,
siamese,
#enable_validate=True,
path=learn_path,
#loss_func=BCEWithLogitsFlat(),
loss_func=ContrastiveLoss(margin=contrastive_neg_margin),
metrics=[avg_pos_dist, avg_neg_dist]
#metrics=[lambda preds, targs: accuracy_thresh(preds.squeeze(), targs, sigmoid=False)]
)
learn.load(f'{name}_80')
dist_mat, val_target, _ = cal_mat(learn.model,
data.valid_dl,
data.fix_dl,
ds_with_target1=True,
ds_with_target2=True)
for threshold in np.linspace(1, 9, 30):
top5_matrix, map5 = cal_mapk(dist_mat,
def main(args):
if args.deterministic:
set_seed(42)
# Set device
if args.device is None:
if torch.cuda.is_available():
args.device = 'cuda:0'
else:
args.device = 'cpu'
defaults.device = torch.device(args.device)
# Aggregate path and labels into list for fastai ImageDataBunch
fnames, labels, is_valid = [], [], []
dataset = OpenFire(root=args.data_path,
train=True,
download=True,
img_folder=args.img_folder)
for sample in dataset.data:
fnames.append(
dataset._images.joinpath(sample['name']).relative_to(dataset.root))
labels.append(sample['target'])
is_valid.append(False)
dataset = OpenFire(root=args.data_path, train=False, download=True)
for sample in dataset.data:
fnames.append(
dataset._images.joinpath(sample['name']).relative_to(dataset.root))
labels.append(sample['target'])
is_valid.append(True)
df = pd.DataFrame.from_dict(
dict(name=fnames, label=labels, is_valid=is_valid))
# Split train and valid sets
il = vision.ImageList.from_df(
df, path=args.data_path).split_from_df('is_valid')
# Encode labels
il = il.label_from_df(cols='label',
label_cls=FloatList if args.binary else CategoryList)
# Set transformations
il = il.transform(vision.get_transforms(), size=args.resize)
# Create the Databunch
data = il.databunch(bs=args.batch_size,
num_workers=args.workers).normalize(
vision.imagenet_stats)
# Metric
metric = partial(vision.accuracy_thresh,
thresh=0.5) if args.binary else vision.error_rate
# Create model
model = models.__dict__[args.model](imagenet_pretrained=args.pretrained,
num_classes=data.c,
lin_features=args.lin_feats,
concat_pool=args.concat_pool,
bn_final=args.bn_final,
dropout_prob=args.dropout_prob)
# Create learner
learner = Learner(data,
model,
wd=args.weight_decay,
loss_func=CustomBCELogitsLoss()
if args.binary else nn.CrossEntropyLoss(),
metrics=metric)
# Form layer group for optimization
meta = model_meta.get(args.model, _default_meta)
learner.split(meta['split'])
# Freeze model's head
if args.pretrained:
learner.freeze()
if args.resume:
learner.load(args.resume)
if args.unfreeze:
learner.unfreeze()
learner.fit_one_cycle(args.epochs,
max_lr=slice(None, args.lr, None),
div_factor=args.div_factor,
final_div=args.final_div_factor)
learner.save(args.checkpoint)
# I think this checks to see which gpu is availabe for use
torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
# In[5]:
# path to the CIFAR10 data
path = untar_data(URLs.CIFAR)
# In[6]:
number_of_epochs = 100
data = ImageDataBunch.from_folder(path, valid="test",
bs=128).normalize(cifar_stats)
learn = Learner(
data, model, silent=True
) #,metrics=accuracy)# silent prevents the training metrics from printing
# In[ ]:
#learn.save('simple_model')
# In[ ]:
learn.lr_find()
# In[ ]:
learn.recorder.plot(suggestion=True)
# In[ ]:
def __init__(self, data=None, pretrained_path=None, **kwargs):
if data is None:
data = create_coco_data()
else:
#Removing normalization because YOLO ingests images with values in range 0-1
data.remove_tfm(data.norm)
data.norm, data.denorm = None, None
super().__init__(data)
#Creating a dummy class for the backbone because this model does not use a torchvision backbone
class DarkNet53():
def __init__(self):
self.name = "DarkNet53"
self._backbone = DarkNet53
self._code = code
self._data = data
self.config_model = {}
if getattr(data, "_is_coco", "") == True:
self.config_model = coco_config()
else:
anchors = kwargs.get('anchors', None)
self.config_model[
'ANCHORS'] = anchors if anchors is not None else generate_anchors(
num_anchor=9, hw=data.height_width)
self.config_model['ANCH_MASK'] = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
self.config_model[
'N_CLASSES'] = data.c - 1 # Subtract 1 for the background class
n_bands = kwargs.get('n_bands', None)
self.config_model[
'N_BANDS'] = n_bands if n_bands is not None else data.x[
0].data.shape[0]
self._model = YOLOv3_Model(self.config_model)
pretrained = kwargs.get('pretrained_backbone', True)
if pretrained:
# Download (if required) and load YOLOv3 weights pretrained on COCO dataset
weights_path = os.path.join(Path.home(), '.cache', 'weights')
if not os.path.exists(weights_path): os.makedirs(weights_path)
weights_file = os.path.join(weights_path, 'yolov3.weights')
if not os.path.exists(weights_file):
try:
download_yolo_weights(weights_path)
extract_zipfile(weights_path, 'yolov3.zip', remove=True)
except Exception as e:
print(e)
print(
"[INFO] Can't download and extract COCO pretrained weights for YOLOv3.\nProceeding without pretrained weights."
)
if os.path.exists(weights_file):
parse_yolo_weights(self._model, weights_file)
from IPython.display import clear_output
clear_output()
self._loss_f = YOLOv3_Loss()
self.learn = Learner(data, self._model, loss_func=self._loss_f)
self.learn.split([self._model.module_list[11]
]) #Splitting the model at Darknet53 backbone
self.learn.freeze()
if pretrained_path is not None:
self.load(str(pretrained_path))
# make first conv weights learnable and use _show_results_multispectral when using multispectral data
self._arcgis_init_callback()
# Set a default flag to toggle appending labels with images before passing images through the model
self.learn.predicting = False
self.learn.callbacks.append(AppendLabelsCallback(self.learn))
class YOLOv3(ArcGISModel):
"""
Creates a YOLOv3 object detector.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
data Required fastai Databunch. Returned data object from
`prepare_data` function.
--------------------- -------------------------------------------
pretrained_path Optional string. Path where pre-trained model is
saved.
===================== ===========================================
:returns: `YOLOv3` Object
"""
def __init__(self, data=None, pretrained_path=None, **kwargs):
if data is None:
data = create_coco_data()
else:
#Removing normalization because YOLO ingests images with values in range 0-1
data.remove_tfm(data.norm)
data.norm, data.denorm = None, None
super().__init__(data)
#Creating a dummy class for the backbone because this model does not use a torchvision backbone
class DarkNet53():
def __init__(self):
self.name = "DarkNet53"
self._backbone = DarkNet53
self._code = code
self._data = data
self.config_model = {}
if getattr(data, "_is_coco", "") == True:
self.config_model = coco_config()
else:
anchors = kwargs.get('anchors', None)
self.config_model[
'ANCHORS'] = anchors if anchors is not None else generate_anchors(
num_anchor=9, hw=data.height_width)
self.config_model['ANCH_MASK'] = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
self.config_model[
'N_CLASSES'] = data.c - 1 # Subtract 1 for the background class
n_bands = kwargs.get('n_bands', None)
self.config_model[
'N_BANDS'] = n_bands if n_bands is not None else data.x[
0].data.shape[0]
self._model = YOLOv3_Model(self.config_model)
pretrained = kwargs.get('pretrained_backbone', True)
if pretrained:
# Download (if required) and load YOLOv3 weights pretrained on COCO dataset
weights_path = os.path.join(Path.home(), '.cache', 'weights')
if not os.path.exists(weights_path): os.makedirs(weights_path)
weights_file = os.path.join(weights_path, 'yolov3.weights')
if not os.path.exists(weights_file):
try:
download_yolo_weights(weights_path)
extract_zipfile(weights_path, 'yolov3.zip', remove=True)
except Exception as e:
print(e)
print(
"[INFO] Can't download and extract COCO pretrained weights for YOLOv3.\nProceeding without pretrained weights."
)
if os.path.exists(weights_file):
parse_yolo_weights(self._model, weights_file)
from IPython.display import clear_output
clear_output()
self._loss_f = YOLOv3_Loss()
self.learn = Learner(data, self._model, loss_func=self._loss_f)
self.learn.split([self._model.module_list[11]
]) #Splitting the model at Darknet53 backbone
self.learn.freeze()
if pretrained_path is not None:
self.load(str(pretrained_path))
# make first conv weights learnable and use _show_results_multispectral when using multispectral data
self._arcgis_init_callback()
# Set a default flag to toggle appending labels with images before passing images through the model
self.learn.predicting = False
self.learn.callbacks.append(AppendLabelsCallback(self.learn))
def __str__(self):
return self.__repr__()
def __repr__(self):
return '<%s>' % (type(self).__name__)
@property
def supported_backbones(self):
""" Supported backbones for this model. """
return ['DarkNet53']
@property
def _model_metrics(self):
if getattr(self._data, "_is_coco", "") == True:
return {'accuracy': {'IoU': 0.50, 'AP': 0.558}}
return {'accuracy': self.average_precision_score(show_progress=False)}
def _analyze_pred(self,
pred,
thresh=0.1,
nms_overlap=0.1,
ret_scores=True,
device=None):
""" """
return postprocess(pred,
chip_size=self.learn.data.chip_size,
conf_thre=thresh,
nms_thre=nms_overlap)
def show_results(self, rows=5, thresh=0.1, nms_overlap=0.1):
"""
Displays the results of a trained model on a part of the validation set.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
rows Optional int. Number of rows of results
to be displayed.
--------------------- -------------------------------------------
thresh Optional float. The probabilty above which
a detection will be considered valid.
--------------------- -------------------------------------------
nms_overlap Optional float. The intersection over union
threshold with other predicted bounding
boxes, above which the box with the highest
score will be considered a true positive.
===================== ===========================================
"""
self._check_requisites()
if rows > len(self._data.valid_ds):
rows = len(self._data.valid_ds)
self.learn.predicting = True
self.learn.show_results(rows=rows,
thresh=thresh,
nms_overlap=nms_overlap,
model=self)
def _show_results_multispectral(self,
rows=5,
thresh=0.3,
nms_overlap=0.1,
alpha=1,
**kwargs):
self.learn.predicting = True
ax = show_results_multispectral(self,
nrows=rows,
thresh=thresh,
nms_overlap=nms_overlap,
alpha=alpha,
**kwargs)
def predict(self,
image_path,
threshold=0.1,
nms_overlap=0.1,
return_scores=True,
visualize=False,
resize=False):
"""
Predicts and displays the results of a trained model on a single image.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
image_path Required. Path to the image file to make the
predictions on.
--------------------- -------------------------------------------
thresh Optional float. The probabilty above which
a detection will be considered valid.
--------------------- -------------------------------------------
nms_overlap Optional float. The intersection over union
threshold with other predicted bounding
boxes, above which the box with the highest
score will be considered a true positive.
--------------------- -------------------------------------------
return_scores Optional boolean.
Will return the probability scores of the
bounding box predictions if True.
--------------------- -------------------------------------------
visualize Optional boolean. Displays the image with
predicted bounding boxes if True.
--------------------- -------------------------------------------
resize Optional boolean. Resizes the image to the same size
(chip_size parameter in prepare_data) that the model was trained on,
before detecting objects.
Note that if resize_to parameter was used in prepare_data,
the image is resized to that size instead.
By default, this parameter is false and the detections are run
in a sliding window fashion by applying the model on cropped sections
of the image (of the same size as the model was trained on).
===================== ===========================================
:returns: 'List' of xmin, ymin, width, height of predicted bounding boxes on the given image
"""
if not HAS_OPENCV:
raise Exception(
"This function requires opencv 4.0.1.24. Install it using pip install opencv-python==4.0.1.24"
)
if not HAS_PIL:
raise Exception(
"This function requires PIL. Please install it via pip or conda"
)
if isinstance(image_path, str):
image = cv2.imread(image_path)
else:
image = image_path
orig_height, orig_width, _ = image.shape
orig_frame = image.copy()
if resize and self._data.resize_to is None\
and self._data.chip_size is not None:
image = cv2.resize(image,
(self._data.chip_size, self._data.chip_size))
if self._data.resize_to is not None:
if isinstance(self._data.resize_to, tuple):
image = cv2.resize(image, self._data.resize_to)
else:
image = cv2.resize(
image, (self._data.resize_to, self._data.resize_to))
height, width, _ = image.shape
if self._data.chip_size is not None:
chips = _get_image_chips(image, self._data.chip_size)
else:
chips = [{
'width': width,
'height': height,
'xmin': 0,
'ymin': 0,
'chip': image,
'predictions': []
}]
valid_tfms = self._data.valid_ds.tfms
self._data.valid_ds.tfms = []
include_pad_detections = False
if len(chips) == 1:
include_pad_detections = True
for chip in chips:
frame = Image(
pil2tensor(PIL.Image.fromarray(
cv2.cvtColor(chip['chip'], cv2.COLOR_BGR2RGB)),
dtype=np.float32).div_(255))
self.learn.predicting = True
bbox = self.learn.predict(frame,
thresh=threshold,
nms_overlap=nms_overlap,
ret_scores=True,
model=self)[0]
if bbox:
scores = bbox.scores
bboxes, lbls = bbox._compute_boxes()
bboxes.add_(1).mul_(
torch.tensor([
chip['height'] / 2, chip['width'] / 2,
chip['height'] / 2, chip['width'] / 2
])).long()
for index, bbox in enumerate(bboxes):
if lbls is not None:
label = lbls[index]
else:
label = 'Default'
data = bb2hw(bbox)
if include_pad_detections or not _exclude_detection(
(data[0], data[1], data[2], data[3]), chip['width'],
chip['height']):
chip['predictions'].append({
'xmin':
data[0],
'ymin':
data[1],
'width':
data[2],
'height':
data[3],
'score':
float(scores[index]),
'label':
label
})
self._data.valid_ds.tfms = valid_tfms
predictions, labels, scores = _get_transformed_predictions(chips)
# Scale the predictions to original image and clip the predictions to image dims
y_ratio = orig_height / height
x_ratio = orig_width / width
for index, prediction in enumerate(predictions):
prediction[0] = prediction[0] * x_ratio
prediction[1] = prediction[1] * y_ratio
prediction[2] = prediction[2] * x_ratio
prediction[3] = prediction[3] * y_ratio
# Clip xmin
if prediction[0] < 0:
prediction[2] = prediction[2] + prediction[0]
prediction[0] = 1
# Clip width when xmax greater than original width
if prediction[0] + prediction[2] > orig_width:
prediction[2] = (prediction[0] + prediction[2]) - orig_width
# Clip ymin
if prediction[1] < 0:
prediction[3] = prediction[3] + prediction[1]
prediction[1] = 1
# Clip height when ymax greater than original height
if prediction[1] + prediction[3] > orig_height:
prediction[3] = (prediction[1] + prediction[3]) - orig_height
predictions[index] = [
prediction[0], prediction[1], prediction[2], prediction[3]
]
if visualize:
image = _draw_predictions(orig_frame,
predictions,
labels,
color=(255, 0, 0),
fontface=2,
thickness=1)
import matplotlib.pyplot as plt
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if getattr(self._data, "_is_coco", "") == True:
figsize = (20, 20)
else:
figsize = (4, 4)
fig, ax = plt.subplots(1, 1, figsize=figsize)
ax.imshow(image)
if return_scores:
return predictions, labels, scores
else:
return predictions, labels
def predict_video(self,
input_video_path,
metadata_file,
threshold=0.5,
nms_overlap=0.1,
track=False,
visualize=False,
output_file_path=None,
multiplex=False,
multiplex_file_path=None,
tracker_options={
'assignment_iou_thrd': 0.3,
'vanish_frames': 40,
'detect_frames': 10
},
visual_options={
'show_scores': True,
'show_labels': True,
'thickness': 2,
'fontface': 0,
'color': (255, 255, 255)
},
resize=False):
"""
Runs prediction on a video and appends the output VMTI predictions in the metadata file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
input_video_path Required. Path to the video file to make the
predictions on.
--------------------- -------------------------------------------
metadata_file Required. Path to the metadata csv file where
the predictions will be saved in VMTI format.
--------------------- -------------------------------------------
threshold Optional float. The probability above which
a detection will be considered.
--------------------- -------------------------------------------
nms_overlap Optional float. The intersection over union
threshold with other predicted bounding
boxes, above which the box with the highest
score will be considered a true positive.
--------------------- -------------------------------------------
track Optional bool. Set this parameter as True to
enable object tracking.
--------------------- -------------------------------------------
visualize Optional boolean. If True a video is saved
with prediction results.
--------------------- -------------------------------------------
output_file_path Optional path. Path of the final video to be saved.
If not supplied, video will be saved at path input_video_path
appended with _prediction.
--------------------- -------------------------------------------
multiplex Optional boolean. Runs Multiplex using the VMTI detections.
--------------------- -------------------------------------------
multiplex_file_path Optional path. Path of the multiplexed video to be saved.
By default a new file with _multiplex.MOV extension is saved
in the same folder.
--------------------- -------------------------------------------
tracking_options Optional dictionary. Set different parameters for
object tracking. assignment_iou_thrd parameter is used
to assign threshold for assignment of trackers,
vanish_frames is the number of frames the object should
be absent to consider it as vanished, detect_frames
is the number of frames an object should be detected
to track it.
--------------------- -------------------------------------------
visual_options Optional dictionary. Set different parameters for
visualization.
show_scores boolean, to view scores on predictions,
show_labels boolean, to view labels on predictions,
thickness integer, to set the thickness level of box,
fontface integer, fontface value from opencv values,
color tuple (B, G, R), tuple containing values between
0-255.
--------------------- -------------------------------------------
resize Optional boolean. Resizes the video frames to the same size
(chip_size parameter in prepare_data) that the model was trained on,
before detecting objects.
Note that if resize_to parameter was used in prepare_data,
the video frames are resized to that size instead.
By default, this parameter is false and the detections are run
in a sliding window fashion by applying the model on cropped sections
of the frame (of the same size as the model was trained on).
===================== ===========================================
"""
VideoUtils.predict_video(self, input_video_path, metadata_file,
threshold, nms_overlap, track, visualize,
output_file_path, multiplex,
multiplex_file_path, tracker_options,
visual_options, resize)
def average_precision_score(self,
detect_thresh=0.5,
iou_thresh=0.1,
mean=False,
show_progress=True):
"""
Computes average precision on the validation set for each class.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
detect_thresh Optional float. The probabilty above which
a detection will be considered for computing
average precision.
--------------------- -------------------------------------------
iou_thresh Optional float. The intersection over union
threshold with the ground truth labels, above
which a predicted bounding box will be
considered a true positive.
--------------------- -------------------------------------------
mean Optional bool. If False returns class-wise
average precision otherwise returns mean
average precision.
===================== ===========================================
:returns: `dict` if mean is False otherwise `float`
"""
self._check_requisites()
aps = compute_class_AP(self,
self._data.valid_dl,
self._data.c - 1,
show_progress,
detect_thresh=detect_thresh,
iou_thresh=iou_thresh)
if mean:
return statistics.mean(aps)
else:
return dict(zip(self._data.classes[1:], aps))
def _get_emd_params(self):
class_data = {}
_emd_template = {}
_emd_template["Framework"] = "arcgis.learn.models._inferencing"
_emd_template["InferenceFunction"] = "ArcGISObjectDetector.py"
_emd_template["ModelConfiguration"] = "_yolov3_inference"
_emd_template["ModelType"] = "ObjectDetection"
_emd_template["ExtractBands"] = [0, 1, 2]
_emd_template['ModelParameters'] = {}
_emd_template['ModelParameters']['anchors'] = self.config_model[
'ANCHORS']
_emd_template['ModelParameters']['n_bands'] = self.config_model[
'N_BANDS']
_emd_template['Classes'] = []
if self._data is not None:
for i, class_name in enumerate(
self._data.classes[1:]): # 0th index is background
inverse_class_mapping = {
v: k
for k, v in self._data.class_mapping.items()
}
class_data["Value"] = inverse_class_mapping[class_name]
class_data["Name"] = class_name
color = [random.choice(range(256)) for i in range(3)]
class_data["Color"] = color
_emd_template['Classes'].append(class_data.copy())
else:
for k, i in coco_class_mapping().items():
class_data['Value'] = k
class_data['Name'] = i
color = [random.choice(range(256)) for i in range(3)]
class_data["Color"] = color
_emd_template['Classes'].append(class_data.copy())
return _emd_template
@classmethod
def from_model(cls, emd_path, data=None):
"""
Creates a YOLOv3 Object Detector from an Esri Model Definition (EMD) file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
emd_path Required string. Path to Esri Model Definition
file.
--------------------- -------------------------------------------
data Required fastai Databunch or None. Returned data
object from `prepare_data` function or None for
inferencing.
===================== ===========================================
:returns: `YOLOv3` Object
"""
if not HAS_FASTAI:
_raise_fastai_import_error(import_exception=import_exception)
emd_path = Path(emd_path)
emd = json.load(open(emd_path))
model_file = Path(emd['ModelFile'])
chip_size = emd["ImageWidth"]
if not model_file.is_absolute():
model_file = emd_path.parent / model_file
class_mapping = {i['Value']: i['Name'] for i in emd['Classes']}
resize_to = emd.get('resize_to')
if isinstance(resize_to, list):
resize_to = (resize_to[0], resize_to[1])
data_passed = True
# Create an image databunch for when loading the model using emd (without training data)
if data is None:
data_passed = False
train_tfms = []
val_tfms = []
ds_tfms = (train_tfms, val_tfms)
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
sd = ImageList([],
path=emd_path.parent.parent).split_by_idx([])
data = sd.label_const(
0,
label_cls=ObjectDetectionCategoryList,
classes=list(class_mapping.values())).transform(
ds_tfms).databunch().normalize(imagenet_stats)
data.chip_size = chip_size
data.class_mapping = class_mapping
data.classes = ['background'] + list(class_mapping.values())
data = get_multispectral_data_params_from_emd(data, emd)
# Add 1 for background class
data.c += 1
data._is_empty = True
data.emd_path = emd_path
data.emd = emd
data.resize_to = resize_to
ret = cls(data, **emd['ModelParameters'], pretrained_path=model_file)
if not data_passed:
ret.learn.data.single_ds.classes = ret._data.classes
ret.learn.data.single_ds.y.classes = ret._data.classes
return ret
num_workers=NUM_WORKERS,
normalization=STATISTICS,
)
# init model
swa_model = MODEL(num_classes=N_CLASSES, dropout_p=DROPOUT)
model = MODEL(num_classes=N_CLASSES, dropout_p=DROPOUT)
# nullify all swa model parameters
swa_params = swa_model.parameters()
for swa_param in swa_params:
swa_param.data = torch.zeros_like(swa_param.data)
# average model
n_swa = len(os.listdir(MODELS_FOLDER))
print(f"Averaging {n_swa} models")
for file in os.listdir(MODELS_FOLDER):
model.load_state_dict(torch.load(f'{MODELS_FOLDER}/{file}')['model'])
model_params = model.parameters()
for model_param, swa_param in zip(model_params, swa_params):
swa_param.data += model_param.data / n_swa
# fix batch norm
print("Fixing batch norm")
swa_model.to(DEVICE)
learn = Learner(data, model, model_dir=MODELS_FOLDER, loss_func=CRITERION, opt_func=OPTIMIZER, wd=WD)
learn.model = convert_model(learn.model)
learn.model = nn.DataParallel(learn.model).to(DEVICE)
fix_batchnorm(learn.model, learn.data.train_dl)
learn.save('swa_model')
def __init__(self, learn):
super().__init__(learn)
def on_epoch_end(self, **kwargs):
for _ in train_eval_loader:
pass
# ---
databunch = DataBunch(train_loader, val_loader)
opt_func = partial(SGD, lr=0.1, momentum=0.9, weight_decay=5e-4)
learner = Learner(data=databunch,
model=model,
opt_func=opt_func,
loss_func=criterion,
metrics=[accuracy],
true_wd=False)
learner.unfreeze()
# ---
callback_on_train_begin = MakeRandomizerConsistentOnTrainBegin(learner)
callback_on_epoch_end = MakeRandomizerConsistentOnEpochEnd(learner)
learner.fit(epochs=150,
lr=0.1,
wd=5e-4,
callbacks=[callback_on_train_begin, callback_on_epoch_end])
learner.fit(epochs=100, lr=0.01, wd=5e-4, callbacks=[callback_on_epoch_end])
learner.fit(epochs=100, lr=0.001, wd=5e-4, callbacks=[callback_on_epoch_end])
def main():
parser = ArgumentParser()
parser.add_argument('--pregenerated_data', type=Path, required=True)
parser.add_argument('--output_dir', type=Path, required=True)
parser.add_argument("--bert_model", type=str, required=True,
choices=["bert-base-uncased", "bert-large-uncased", "bert-base-cased",
"bert-base-multilingual-cased", "bert-base-chinese"])
parser.add_argument("--do_lower_case", action="store_true")
parser.add_argument("--reduce_memory", action="store_true",
help="Store training data as on-disc memmaps to massively reduce memory usage")
parser.add_argument("--epochs", type=int, default=3, help="Number of epochs to train for")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument('--gradient_accumulation_steps',
type=int,
default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.")
parser.add_argument("--train_batch_size",
default=16,
type=int,
help="Total batch size for training.")
parser.add_argument('--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--loss_scale',
type=float, default=0,
help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument("--warmup_proportion",
default=0.1,
type=float,
help="Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--learning_rate",
default=3e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument('--seed',
type=int,
default=None,
help="random seed for initialization")
args = parser.parse_args()
assert args.pregenerated_data.is_dir(), \
"--pregenerated_data should point to the folder of files made by pregenerate_training_data.py!"
samples_per_epoch = []
for i in range(args.epochs):
epoch_file = args.pregenerated_data / f"epoch_{i}.json"
metrics_file = args.pregenerated_data / f"epoch_{i}_metrics.json"
if epoch_file.is_file() and metrics_file.is_file():
metrics = json.loads(metrics_file.read_text())
samples_per_epoch.append(metrics['num_training_examples'])
else:
if i == 0:
exit("No training data was found!")
print(f"Warning! There are fewer epochs of pregenerated data ({i}) than training epochs ({args.epochs}).")
print("This script will loop over the available data, but training diversity may be negatively impacted.")
num_data_epochs = i
break
else:
num_data_epochs = args.epochs
print(samples_per_epoch)
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
if args.seed:
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if args.output_dir.is_dir() and list(args.output_dir.iterdir()):
logging.warning(f"Output directory ({args.output_dir}) already exists and is not empty!")
args.output_dir.mkdir(parents=True, exist_ok=True)
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
total_train_examples = 0
for i in range(args.epochs):
# The modulo takes into account the fact that we may loop over limited epochs of data
total_train_examples += samples_per_epoch[i % len(samples_per_epoch)]
num_train_optimization_steps = int(
total_train_examples / args.train_batch_size / args.gradient_accumulation_steps)
# Prepare model
model = BertForPreTraining.from_pretrained(args.bert_model)
model = torch.nn.DataParallel(model)
# Prepare optimizer
optimizer = BertAdam
train_dataloader = DataLoader(
PregeneratedData(args.pregenerated_data, tokenizer,args.epochs, args.train_batch_size),
batch_size=args.train_batch_size,
)
data = DataBunch(train_dataloader,train_dataloader)
global_step = 0
logging.info("***** Running training *****")
logging.info(f" Num examples = {total_train_examples}")
logging.info(" Batch size = %d", args.train_batch_size)
logging.info(" Num steps = %d", num_train_optimization_steps)
def loss(x, y):
return x.mean()
learn = Learner(data, model, optimizer,
loss_func=loss,
true_wd=False,
path='learn',
layer_groups=bert_layer_list(model),
)
lr= args.learning_rate
layers = len(bert_layer_list(model))
lrs = learn.lr_range(slice(lr/(2.6**4), lr))
for epoch in range(args.epochs):
learn.fit_one_cycle(1, lrs, wd=0.01)
# save model at half way point
if epoch == args.epochs//2:
savem = learn.model.module.bert if hasattr(learn.model, 'module') else learn.model.bert
output_model_file = args.output_dir / (f"pytorch_fastai_model_{args.bert_model}_{epoch}.bin")
torch.save(savem.state_dict(), str(output_model_file))
print(f'Saved bert to {output_model_file}')
savem = learn.model.module.bert if hasattr(learn.model, 'module') else learn.model.bert
output_model_file = args.output_dir / (f"pytorch_fastai_model_{args.bert_model}_{args.epochs}.bin")
torch.save(savem.state_dict(), str(output_model_file))
print(f'Saved bert to {output_model_file}')
