def fit(self, x_train, y_train, validation_split=0., epochs=1, verbose=0):
y_train = utils.one_hot_encode(y_train, self.num_classes)
x_train, x_val, y_train, y_val = train_test_split(
x_train,
y_train,
test_size=validation_split,
)
if validation_split > 0.:
validation_data = self.data_generator.flow(x_val,
y_val,
batch_size=32)
callbacks = [EarlyStopping(patience=3)]
else:
validation_data = None
callbacks = None
with self.graph.as_default():
self.data_generator.fit(x_train)
results = self.model.fit_generator(
self.data_generator.flow(x_train, y_train, batch_size=32),
epochs=epochs,
validation_data=validation_data,
callbacks=callbacks,
verbose=verbose,
)
return (results.history['loss'][0], results.history['acc'][0])
def test(args, model, loader, prefix='', verbose=True):
"""
This function does a single pass through the testing set
and evaluates the average dice_score, and average loss.
params: args are the run parameters, model is the model being tested,
loader is the pytorch test loader, prefix is a name string,
verbose flag is for printing metrics
return: dictionary of metric values
"""
print("train: Beginning test")
metrics = defaultdict(list)
t = tqdm(loader)
with torch.no_grad():
for (img, seg) in t:
img = img.to(args.device)
seg = seg_utils.map_segmentation(seg, args.num_classes).long()
seg_hot = seg_utils.one_hot_encode(seg, args.num_classes).to(args.device)
seg = seg.to(args.device)
seg_hat = model(img)
if args.loss_func=='dice':
dice_loss = loss_fns.DiceLoss()
loss = dice_loss(seg_hat, seg_hot)
elif args.loss_func=='crossentropy':
log_loss = nn.CrossEntropyLoss()
loss = log_loss(seg_hat, seg)
t.set_postfix_str(s='loss: %f'% loss.item())
try:
metrics['loss'].append(loss.item())
except ValueError:
print(metrics)
for k, v in get_metrics(seg_hot, seg_hat).items():
metrics[k].append(v)
for k in replace_metric_by_mean:
metrics[k] = np.mean(metrics[k])
# Print!
if verbose:
start_string = '#### {} evaluation ####'.format(prefix)
print(start_string)
for k, v in metrics.items():
print('#### {} = {}'.format(k, v))
print(''.join(['#' for _ in range(len(start_string))]))
return metrics
def _score_classification(self, x_test, y_test):
loss, acc = self.model.evaluate(x_test, y_test)
num_classes = len(self.label_helper.classes)
if acc < (1. / num_classes * ACCURACY_RATIO):
self.logger.debug(
"Need at least {}% accuracy improvement over naive baseline to start labelling"
.format(int((ACCURACY_RATIO - 1.) * 100)))
return 0
self.logger.debug("Scoring items with model labeller.")
y_test = utils.one_hot_encode(y_test, num_classes)
y_pred = self.model.score(x_test)
threshold = Evaluator.threshold_for_precision(
y_test,
y_pred,
target_precision=TARGET_PRECISION,
)
unlabelled, ids = self.dataset.model_labelling_set()
if len(unlabelled) == 0:
self.logger.info("Model labelling done!")
return 0
scores = self.model.score(unlabelled)
# if scores is 2 dimentional: (batch x classes)
# This assumes only classification :(
# Renormalize scores just in case.
dist = scores / np.expand_dims(scores.sum(axis=1), -1)
idxs = np.argmax(dist, -1)
num_scored = 0
for _id, (idx, score) in list(
zip(ids, zip(idxs, dist[np.arange(len(idxs)), idxs]))):
if score > threshold:
num_scored += 1
self.dataset.add_label(
_id,
idx,
stage=Dataset.MODEL_LABELLED,
user=Dataset.USER_MODEL_LABELLER,
is_labelled=False,
save=True,
)
return num_scored
def encode_object_data(pitch_data):
print('Encoding pitch dataframe of shape {}...'.format(pitch_data.shape))
# Split label column from rest of pitch dataframe then encode
Y_all = pitch_data.loc[:, 'p1_pitch_type'].copy()
Y_all = utils.encode_simple_pitch_types(Y_all)
# Drop label colum from pitch dataframe, then one-hot-encode object columns
pitch_data = pitch_data.drop('p1_pitch_type', axis=1)
pitch_data = utils.one_hot_encode(pitch_data, False)
# Insert label data back into pitch dataframe
pitch_data['p1_pitch_type'] = Y_all.copy()
print('Pitch dataframe encoding complete. New shape: {}'.format(
pitch_data.shape))
return pitch_data
def representation_learning(
self,
x_texts,
epochs=1,
bptt=100,
batch_size=32,
verbose=False,
num_gpus=1,
on_epoch_done=None,
):
# Unfreeze language model.
utils.unfreeze_layers(self.language_model)
all_chunks = self._create_bptt_data(x_texts, bptt)
batches = [all_chunks[i:i + batch_size] for i in range(0, len(all_chunks), batch_size)]
total_losses = []
if num_gpus > 1:
batch_size = batch_size * num_gpus
model = multi_gpu_model(self.language_model, gpus=num_gpus)
model.compile('adam', 'categorical_crossentropy', metrics=['accuracy'])
print("Set batch size to {}".format(batch_size))
else:
model = self.language_model
with self.graph.as_default():
for epoch in range(epochs):
iterable = tqdm.tqdm(batches) if verbose else batches
total_loss = 0.
for tokens_batch in iterable:
# Compute x_batch and y_batch
x_text = pad_sequences(self.lang._sequence_ids(tokens_batch))
x_train = x_text[:, :-1]
y_train = x_text[:, 1:]
target = utils.one_hot_encode(y_train, self.vocab_size)
# Train language model.
result = model.fit(x_train, target, batch_size=batch_size, verbose=0)
total_loss += 1 / len(iterable) * result.history['loss'][-1]
if verbose: print("Epoch: {} | Loss: {}".format(epoch, total_loss))
total_losses.append(total_loss / len(tokens_batch))
if on_epoch_done: on_epoch_done(self)
return (total_losses, 0.)
def fit(self, x_texts, y_train, validation_split=0, epochs=1):
# Freeze language model
y_train = utils.one_hot_encode(y_train, self.num_classes)
x_train = self.vectorize_text(x_texts)
if validation_split > 0.:
callbacks = [EarlyStopping(patience=3)]
else:
callbacks = None
with self.graph.as_default():
history = self.model.fit(
x_train,
y_train,
validation_split=validation_split,
callbacks=callbacks,
epochs=epochs,
verbose=0,
)
return history.history['loss'][0], history.history['acc'][0]
def test(args, model, loader, prefix='', verbose=True):
print("train: Beginning test")
metrics = defaultdict(list)
t = tqdm(loader)
with torch.no_grad():
for (img, seg) in t:
img = img.to(args.device)
seg = map_segmentation(seg, args.num_classes).long()
seg_hot = seg_utils.one_hot_encode(seg, args.num_classes).to(args.device)
seg = seg.to(args.device)
seg_hat = model(img)
if args.loss_func=='dice':
dice_loss = loss_fns.DiceLoss()
loss = dice_loss(seg_hat, seg_hot)
elif args.loss_func=='crossentropy':
log_loss = nn.CrossEntropyLoss()
loss = log_loss(seg_hat, seg)
t.set_postfix_str(s='loss: %f'% loss.item())
try:
metrics['loss'].append(loss.item())
except ValueError:
print(metrics)
for k, v in get_metrics(seg_hot, seg_hat).items():
metrics[k].append(v)
for k in replace_metric_by_mean:
metrics[k] = np.mean(metrics[k])
# Print!
if verbose:
start_string = '#### {} evaluation ####'.format(prefix)
print(start_string)
for k, v in metrics.items():
print('#### {} = {}'.format(k, v))
print(''.join(['#' for _ in range(len(start_string))]))
return metrics
def train_unet_v0(args, dataset, train_loader, test_loader):
"""
This function defines the model, optimizer and output directories.
It performs training, and prints metrics every eval_every epochs.
Most importantly, this function also saves the model every eval_every
epochs.
params: args are run parameters, dataset is a pytorch dataset of the images,
train_loader and test_loader are the pytorch dataloaders.
return: model and metrics
"""
output_dir = args.base_output
model = load_model(args, unet.FRUNetAuto)
if args.cuda:
model = model.cuda()
if args.dataparallel:
model = nn.DataParallel(model)
params = list(model.parameters())
optimizer = optim.Adam(
params,
lr=args.lr,
weight_decay=args.weight_decay
)
metrics = defaultdict(list)
print("Getting Class balances... ")
# For weighted version of loss functions
class_weights = None
if args.class_weights:
class_weights = [dataset.class_weights[0], dataset.class_weights[1:8].sum()]
if args.num_classes > 2:
class_weights.append(dataset.class_weights[-1])
class_weights = 1 - torch.Tensor(class_weights)
class_weights = class_weights.to(args.device)
print("Class balance: %s"%str(class_weights))
print("train: Beginning train")
model.train()
for epoch_idx in range(args.epochs):
print('Starting epoch {}'.format(epoch_idx))
epoch_metrics = defaultdict(list)
tic = time.time()
t = tqdm(train_loader)
for (img, seg) in t:
img = img.to(args.device)
seg = seg_utils.map_segmentation(seg, args.num_classes).long()
seg_hot = seg_utils.one_hot_encode(seg, args.num_classes).to(args.device)
seg = seg.to(args.device)
seg_hat = model(img)
if args.loss_func=='dice':
dice_loss = loss_fns.DiceLoss(class_weights)
loss = dice_loss(seg_hat, seg_hot)
elif args.loss_func=='crossentropy':
log_loss = nn.CrossEntropyLoss(weight=class_weights)
loss = log_loss(seg_hat, seg)
epoch_metrics['loss'].append(loss.item())
for k, v in get_metrics(seg_hot, seg_hat).items():
epoch_metrics[k].append(v)
t.set_postfix_str(s='loss: %f, dice: %f'%(loss.item(),epoch_metrics['dice'][-1]))
optimizer.zero_grad()
loss.backward()
optimizer.step()
for k, v in epoch_metrics.items():
metrics[k].append(np.mean(np.array(v), axis=0))
epoch_metrics[k].append(np.mean(np.array(v), axis=0))
print('#### [{:.2f}] Epoch {} ####'.format(time.time() - tic, epoch_idx))
for k, v in epoch_metrics.items():
print('#### {}: {}'.format(k, v[-1]))
print('#####################')
# Eval and save if necessary.
if general_utils.periodic_integer_delta(epoch_idx, args.eval_every):
model = model.eval()
test_metrics = test(args, model, test_loader, prefix='Test Dataset, Epoch {}'.format(epoch_idx))
model = model.train()
for k, v in test_metrics.items():
metrics['test_{}_epoch{}'.format(k, epoch_idx)] = v
if general_utils.periodic_integer_delta(epoch_idx, args.save_every):
checkpoint_path = os.path.join(output_dir, "last.checkpoint")
print('Saving model to {}'.format(checkpoint_path))
chk = general_utils.make_checkpoint(model, optimizer, epoch_idx)
torch.save(chk, checkpoint_path)
np.save(output_dir+'/losses/loss%d.npy'%epoch_idx, epoch_metrics['loss'])
return model, metrics
def evaluate(self, x_texts, y_test):
y_test = utils.one_hot_encode(y_test, self.num_classes)
x_test = self.vectorize_text(x_texts)
with self.graph.as_default():
return self.model.evaluate(x_test, y_test)
def evaluate(self, x_test, y_test, verbose=0):
y_test = utils.one_hot_encode(y_test, self.num_classes)
with self.graph.as_default():
results = self.model.evaluate(x_test, y_test, verbose=verbose)
return results[0], results[1]
def start(self):
while True:
# Put this at the top of the while loop to prevent thread timing issues.
# If we want to solve this properly, check here: https://github.com/keras-team/keras/issues/5223
time.sleep(
min(self.exponential_backoff_factor * self.interval,
MAX_INTERVAL_TIME))
x_test, y_test = self.dataset.test_set
num_classes = len(self.label_helper.classes)
# TODO(classification_only)
# TODO(revisit)
if len(x_test) < MIN_TEST_EXAMPLES:
self.logger.debug(
"Need at least {} labels to start labelling".format(
MIN_TEST_EXAMPLES))
self.exponential_backoff_factor += 1
continue
loss, acc = self.model.evaluate(x_test, y_test)
if acc < (1. / num_classes * ACCURACY_RATIO):
self.logger.debug(
"Need at least {}% accuracy improvement over naive baseline to start labelling"
.format(int((ACCURACY_RATIO - 1.) * 100)))
self.exponential_backoff_factor += 1
continue
self.logger.debug("Scoring items with model labeller.")
y_test = utils.one_hot_encode(y_test, num_classes)
evaluator = Evaluator(self.model, x_test, y_test)
threshold = evaluator.threshold_for_precision(THRESHOLD)
unlabelled, ids = self.dataset.model_labelling_set()
if len(unlabelled) == 0:
self.logger.info("Model labelling done!")
scores = self.model.score(unlabelled)
# This assumes only classification :(
# Renormalize scores just in case.
dist = scores / np.expand_dims(scores.sum(axis=1), -1)
idxs = np.argmax(dist, -1)
past_threshold = 0
for _id, (idx, score) in list(
zip(ids, zip(idxs, dist[np.arange(len(idxs)), idxs]))):
if score > threshold:
past_threshold += 1
self.dataset.add_label(
_id,
idx,
stage=Dataset.MODEL_LABELLED,
user=Dataset.USER_MODEL_LABELLER,
is_labelled=False,
save=True,
)
if past_threshold > 0:
self.exponential_backoff_factor = 0
else:
self.exponential_backoff_factor += 1
self.logger.debug("{} / {} labelled.".format(
past_threshold, len(scores)))
