def __init__(self,
model,
criterion=None,
optimizer=None,
name=DEFAULT_NAME,
weights_dir=WEIGHTS_DIR,
reports_dir=REPORTS_DIR,
write_report=WRITE_REPORT,
force_cpu=False):
self.device = h.get_device(force_cpu)
self.model = model.to(self.device)
self.name = name
self.weights_dir = weights_dir
self.reports_dir = reports_dir
self.write_report = write_report
self.callbacks = False
self.best_weights_path = None
if criterion and optimizer:
self.compile(criterion=criterion, optimizer=optimizer)
def run(self,
dev=IS_DEV,
dry_run=DRY_RUN,
noise_reducer=NOISE_REDUCER,
print_summary=PRINT_SUMMARY):
if dev: dry_run = False
# parse config
train_loader, valid_loader = self._get('loaders', dev=dev)
model = self._get('model')
accuracy_activation = self.config.get('criterion',
{}).get('accuracy_activation')
criterion = self._get('criterion')
optimizer = self._get('optimizer')
lrs = self.config.get('lrs', DEFAULT_LRS)
if dev:
nb_epochs = DEV_NB_EPOCHS
name = f'DEV---{self.name}'
else:
nb_epochs = self.config.get('nb_epochs', NB_EPOCHS)
name = self.name
patience = self.config.get('patience', PATIENCE)
mask_value = self.config.get('mask_value')
weights = self.config.get('weights')
pred_argmax = (self._model_config().get('out_ch', FORCE_PRED_ARGMAX) >
1)
# run
trainer = train.Trainer(model=model, name=name)
trainer.set_callbacks(save=True,
silent=False,
noise_reducer=noise_reducer,
name=trainer.name)
if weights:
weights_dir = self.config.get('weights_dir')
if weights_dir is None:
weights_dir = DEFAULT_WEIGHTS_DIR
if weights_dir:
weights = f'{weights_dir}/{weights}'
trainer.load_weights(weights)
if self.config.get('freeze'):
model = self._get('freeze', model=model)
if print_summary:
mcfig = self._model_config()
size = mcfig.get('size', SIZE)
in_ch = mcfig['in_ch']
summary(model.to(h.get_device()), (in_ch, size, size))
print('\n' * 4)
print('*' * 100)
print('*' * 100)
print('*' * 100)
print()
print(f"NAME: {self.name}")
if weights:
print(f"INIT WEIGHTS: {weights}")
print(f"LRS: {lrs}")
if (valid_loader is None) or (valid_loader is False):
nb_valid = ' --- '
else:
nb_valid = len(valid_loader)
print(f"NB_EPOCHS: {nb_epochs}")
print(f"NB_BATCHES:", len(train_loader), nb_valid)
print(f"PATIENCE: {patience}")
print()
print('=' * 100)
print()
pprint(self.config)
print()
print('*' * 100)
print('*' * 100)
print('*' * 100)
print('\n' * 4)
for i, lr in enumerate(lrs):
print()
print('-' * 100)
print(f'RUN: {i+1}/{len(lrs)}, LR: {lr}')
if i: trainer.load_weights()
if not dry_run:
trainer.compile(criterion=criterion,
optimizer=optimizer(trainer.model.parameters(),
lr=lr))
trainer.fit(accuracy_activation=accuracy_activation,
accuracy_method=metrics.batch_accuracy(
pred_argmax=pred_argmax,
round_prediction=(not pred_argmax),
mask_value=mask_value),
nb_epochs=nb_epochs,
train_loader=train_loader,
valid_loader=valid_loader,
early_stopping=True,
patience=patience,
patience_start=0)
print('RUN:', MODEL_NAME)
print(datetime.now().strftime(TS_FMT))
#
# DATA
#
df = pd.read_csv(PATHS_CSV)
df = df[df.black_pixel_count < MAX_BLACK_PIXEL]
paths = df.path.tolist()[:LIMIT]
print('NB_IMAGES:', len(paths))
#
# MODEL
#
DEVICE = H.get_device()
model = load.model(MODEL_NAME,
file=MODEL_PYFILE,
init_weights=f'{PROJECT_DIR}/cli/weights/{WEIGHTS}')
model = model.eval().to(DEVICE)
#
# METHODS
#
def get_input(path):
im, p = io.read(path, return_profile=True)
im = hand.process_input(im,
rotate=False,
flip=False,
input_bands=INPUT_BANDS,