def prepare(self, priming_data):
if self._prepared:
raise Exception('You can only call "prepare" once for a given encoder.')
no_null_sentences = [x if x is not None else '' for x in priming_data]
estimated_time = 1/937*self._train_iters*len(no_null_sentences)
log_every = math.ceil(self._train_iters/100)
log.info('We will train an encoder for this text, on a CPU it will take about {min} minutes'.format(
min=estimated_time))
self._input_lang = Lang('input')
self._output_lang = self._input_lang
for row in no_null_sentences:
if row is not None:
self._input_lang.addSentence(row)
max_length = max(map(len, no_null_sentences))
hidden_size = self._encoded_vector_size
self._encoder = EncoderRNN(self._input_lang.n_words, hidden_size).to(device)
self._decoder = DecoderRNN(hidden_size, self._output_lang.n_words).to(device)
trainIters(self._encoder, self._decoder, self._input_lang, self._output_lang, no_null_sentences, no_null_sentences, self._train_iters, int(log_every), self._learning_rate, self._stop_on_error,
max_length)
self._prepared = True
def _default_on_iter(self, epoch, train_error, test_error, delta_mean,
accuracy):
test_error_rounded = round(test_error, 4)
for col in accuracy:
value = accuracy[col]['value']
if accuracy[col]['function'] == 'r2_score':
value_rounded = round(value, 3)
log.info(
f'We\'ve reached training epoch nr {epoch} with an r2 score of {value_rounded} on the testing dataset'
)
else:
value_pct = round(value * 100, 2)
log.info(
f'We\'ve reached training epoch nr {epoch} with an accuracy of {value_pct}% on the testing dataset'
)
def predict(self, when_data=None, when=None):
"""
Predict given when conditions.
:param when_data: pandas.DataFrame
:param when: dict
:return: pandas.DataFrame
"""
device, _available_devices = get_devices()
log.info(f'Computing device used: {device}')
if when is not None:
when_dict = {key: [when[key]] for key in when}
when_data = pandas.DataFrame(when_dict)
when_data_ds = DataSource(when_data, self.config, prepare_encoders=False)
when_data_ds.eval()
kwargs = {'include_extra_data': self.config.get('include_extra_data', False)}
return self._mixer.predict(when_data_ds, **kwargs)
def learn(self, from_data, test_data=None):
"""
Train and save a model (you can use this to retrain model from data).
:param from_data: DataFrame or DataSource
The data to learn from
:param test_data: DataFrame or DataSource
The data to test accuracy and learn_error from
"""
device, _available_devices = get_devices()
log.info(f'Computing device used: {device}')
# generate the configuration and set the order for the input and output columns
if self._generate_config is True:
self._input_columns = [col for col in from_data if col not in self._output_columns]
self.config = {
'input_features': [{'name': col, 'type': self._type_map(from_data, col)} for col in self._input_columns],
'output_features': [{'name': col, 'type': self._type_map(from_data, col)} for col in self._output_columns]
}
self.config = predictor_config_schema.validate(self.config)
log.info('Automatically generated a configuration')
log.info(self.config)
else:
self._output_columns = [col['name'] for col in self.config['output_features']]
self._input_columns = [col['name'] for col in self.config['input_features']]
if isinstance(from_data, pandas.DataFrame):
train_ds = DataSource(from_data, self.config)
elif isinstance(from_data, DataSource):
train_ds = from_data
else:
raise TypeError(':from_data: must be either DataFrame or DataSource')
nr_subsets = 3 if len(train_ds) > 100 else 1
if test_data is None:
test_ds = train_ds.subset(0.1)
elif isinstance(test_data, pandas.DataFrame):
test_ds = train_ds.make_child(test_data)
elif isinstance(test_data, DataSource):
test_ds = test_data
else:
raise TypeError(':test_data: must be either DataFrame or DataSource')
train_ds.create_subsets(nr_subsets)
test_ds.create_subsets(nr_subsets)
train_ds.train()
test_ds.train()
mixer_class = self.config['mixer']['class']
mixer_kwargs = self.config['mixer']['kwargs']
self._mixer = mixer_class(**mixer_kwargs)
self._mixer.fit(train_ds=train_ds, test_ds=test_ds)
self.train_accuracy = self._mixer.calculate_accuracy(test_ds)
return self
def encode(self, column_data):
encoded_audio_arr = []
for path in column_data:
if path.startswith('http'):
response = requests.get(path)
with open(path.split('/')[-1], 'wb') as f:
f.write(response.content)
try:
audio = AudioSegment.from_file(path.split('/')[-1])
except Exception as e:
print(e)
finally:
os.remove(path.split('/')[-1])
else:
audio = AudioSegment.from_file(path)
# For now convert all (usually will be stereo) to mono by adding up and averging the amplitudes
audio = audio.set_channels(1)
original_frame_rate = audio.frame_rate
new_frame_rate = int(
original_frame_rate /
(len(audio.get_array_of_samples()) / self._max_samples))
if new_frame_rate < original_frame_rate:
audio = audio.set_frame_rate(new_frame_rate)
log.info(
f'Lowering audio frame rate from {original_frame_rate} to {new_frame_rate} for ease of processing !'
)
audio_arr = list(np.array(audio.get_array_of_samples()))
encoded_audio = self._ts_encoder.encode([audio_arr])
encoded_audio_arr.append(encoded_audio[0])
return torch.Tensor(encoded_audio_arr)
def __init__(self,
dynamic_parameters,
input_size=None,
output_size=None,
nr_outputs=None,
shape=None,
dropout=None,
pretrained_net=None):
self.input_size = input_size
self.output_size = output_size
self.nr_outputs = nr_outputs
self.max_variance = None
# How many devices we can train this network on
self.available_devices = 1
self.device, _ = get_devices()
self.dynamic_parameters = dynamic_parameters
"""
Here we define the basic building blocks of our model,
in forward we define how we put it all together along with an input
"""
super(DefaultNet, self).__init__()
if shape is None and pretrained_net is None:
shape = [
self.input_size,
max([self.input_size * 2, self.output_size * 2, 400]),
self.output_size
]
if pretrained_net is None:
log.info(f'Building network of shape: {shape}')
rectifier = torch.nn.SELU #alternative: torch.nn.ReLU
layers = []
for ind in range(len(shape) - 1):
if (dropout is not None) and (0 < ind < len(shape)):
layers.append(torch.nn.Dropout(p=dropout))
linear_function = PLinear if CONFIG.USE_PROBABILISTIC_LINEAR else torch.nn.Linear
layers.append(linear_function(shape[ind], shape[ind + 1]))
if ind < len(shape) - 2:
layers.append(rectifier())
self.net = torch.nn.Sequential(*layers)
else:
self.net = pretrained_net
for layer in self.net:
if isinstance(layer, torch.nn.Linear):
if self.input_size is None:
self.input_size = layer.in_features
self.output_size = layer.out_features
self.net = self.net.to(self.device)
if 'cuda' in str(self.device):
self.available_devices = torch.cuda.device_count()
else:
self.available_devices = 1
if self.available_devices > 1:
self._foward_net = torch.nn.DataParallel(self.net)
else:
self._foward_net = self.net
def prepare(self, priming_data):
random.seed(len(priming_data))
if self._prepared:
raise Exception(
'You can only call "prepare" once for a given encoder.')
self.onehot_encoder.prepare(priming_data)
input_len = self.onehot_encoder._lang.n_words
self.use_autoencoder = self.max_encoded_length is not None and input_len > self.max_encoded_length
if self.use_autoencoder:
log.info(
'Preparing a categorical autoencoder, this might take a while')
embeddings_layer_len = self.max_encoded_length
self.net = DefaultNet(
dynamic_parameters={},
shape=[input_len, embeddings_layer_len, input_len])
criterion = torch.nn.CrossEntropyLoss()
optimizer = Ranger(self.net.parameters())
gym = Gym(model=self.net,
optimizer=optimizer,
scheduler=None,
loss_criterion=criterion,
device=self.net.device,
name=self.name,
input_encoder=self.onehot_encoder.encode,
output_encoder=self._encoder_targets)
batch_size = min(200, int(len(priming_data) / 50))
priming_data_str = [str(x) for x in priming_data]
train_data_loader = DataLoader(list(
zip(priming_data_str, priming_data_str)),
batch_size=batch_size,
shuffle=True)
test_data_loader = None
best_model, error, training_time = gym.fit(
train_data_loader,
test_data_loader,
desired_error=self.desired_error,
max_time=self.max_training_time,
callback=self._train_callback,
eval_every_x_epochs=1,
max_unimproving_models=5)
self.net = best_model.to(self.net.device)
modules = [
module for module in self.net.modules()
if type(module) != torch.nn.Sequential
and type(module) != DefaultNet
]
self.encoder = torch.nn.Sequential(*modules[0:2]).eval()
self.decoder = torch.nn.Sequential(*modules[2:3]).eval()
log.info('Categorical autoencoder ready')
self._prepared = True
def _train_callback(self, error, real_buff, predicted_buff):
log.info(f'{self.name} reached a loss of {error} while training !')
def _iter_fit(self, ds, subset_id=None, max_epochs=120000):
if self._nonpersistent['sampler'] is None:
data_loader = DataLoader(ds,
batch_size=self.batch_size,
shuffle=True,
num_workers=0)
else:
data_loader = DataLoader(ds,
batch_size=self.batch_size,
num_workers=0,
sampler=self._nonpersistent['sampler'])
for epoch in range(max_epochs): # loop over the dataset multiple times
running_loss = 0.0
error = 0
for i, data in enumerate(data_loader, 0):
if self.start_selfaware_training and not self.is_selfaware:
log.info(
'Starting to train selfaware network for better confidence determination !'
)
self.is_selfaware = True
if self.stop_selfaware_training and self.is_selfaware:
log.info(
'Cannot train selfaware network, will fallback to using simpler confidence models !'
)
self.is_selfaware = False
self.total_iterations += 1
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data
labels = labels.to(self.net.device)
inputs = inputs.to(self.net.device)
# zero the parameter gradients
self.optimizer.zero_grad()
self.selfaware_optimizer.zero_grad()
# forward + backward + optimize
with LightwoodAutocast():
outputs = self.net(inputs)
if self.is_selfaware:
with LightwoodAutocast():
awareness = self.selfaware_net(inputs.detach(),
outputs.detach())
loss = None
for k, criterion in enumerate(self.criterion_arr):
with LightwoodAutocast():
target_loss = criterion(
outputs[:,
ds.out_indexes[k][0]:ds.out_indexes[k][1]],
labels[:,
ds.out_indexes[k][0]:ds.out_indexes[k][1]])
if loss is None:
loss = target_loss
else:
loss += target_loss
awareness_loss = None
if self.is_selfaware:
unreduced_losses = []
for k, criterion in enumerate(
self.unreduced_criterion_arr):
target_loss = criterion(
outputs[:,
ds.out_indexes[k][0]:ds.out_indexes[k][1]],
labels[:,
ds.out_indexes[k][0]:ds.out_indexes[k][1]])
target_loss = target_loss.tolist()
if type(target_loss[0]) == type([]):
target_loss = [np.mean(x) for x in target_loss]
for i, value in enumerate(target_loss):
if len(unreduced_losses) <= i:
unreduced_losses.append([])
unreduced_losses[i].append(value)
unreduced_losses = torch.Tensor(unreduced_losses).to(
self.net.device)
awareness_loss = self.awareness_criterion(
awareness,
unreduced_losses) * self.awareness_scale_factor
if CONFIG.MONITORING['batch_loss']:
self.monitor.plot_loss(awareness_loss.item(),
self.total_iterations,
'Awreness Batch Loss')
if CONFIG.MONITORING['batch_loss']:
self.monitor.plot_loss(loss.item(), self.total_iterations,
'Targets Batch Loss')
if awareness_loss is not None:
awareness_loss.backward(retain_graph=True)
running_loss += loss.item()
loss.backward()
# @NOTE: Decrease 900 if you want to plot gradients more often, I find it's too expensive to do so
if CONFIG.MONITORING['network_heatmap'] and random.randint(
0, 1000) > 900:
weights = []
gradients = []
layer_name = []
for index, layer in enumerate(self.net.net):
if 'Linear' in str(type(layer)):
weights.append(
list(layer.weight.cpu().detach().numpy().ravel(
)))
gradients.append(
list(layer.weight.grad.cpu().detach().numpy().
ravel()))
layer_name.append(f'Layer {index}-{index+1}')
self.monitor.weight_map(layer_name, weights,
'Predcitive network weights')
self.monitor.weight_map(layer_name, weights,
'Predictive network gradients')
if self.is_selfaware:
weights = []
gradients = []
layer_name = []
for index, layer in enumerate(self.selfaware_net.net):
if 'Linear' in str(type(layer)):
weights.append(
list(layer.weight.cpu().detach().numpy().
ravel()))
gradients.append(
list(layer.weight.grad.cpu().detach().
numpy().ravel()))
layer_name.append(f'Layer {index}-{index+1}')
self.monitor.weight_map(layer_name, weights,
'Awareness network weights')
self.monitor.weight_map(layer_name, weights,
'Awareness network gradients')
# now that we have run backward in both losses, optimize()
# (review: we may need to optimize for each step)
self.optimizer.step()
if self.is_selfaware and self.start_selfaware_training:
self.selfaware_optimizer.step()
error = running_loss / (i + 1)
if CONFIG.MONITORING['batch_loss']:
#self.monitor.plot_loss(total_loss.item(), self.total_iterations, 'Total Batch Loss')
self.monitor.plot_loss(error, self.total_iterations,
'Mean Total Running Loss')
if CONFIG.MONITORING['epoch_loss']:
self.monitor.plot_loss(error, self.total_iterations,
'Train Epoch Error')
self.monitor.plot_loss(
error, self.total_iterations,
f'Train Epoch Error - Subset {subset_id}')
yield error
def _predict(self, when_data_source, include_extra_data=False):
data_loader = DataLoader(when_data_source,
batch_size=self.batch_size,
shuffle=False,
num_workers=0)
# set model into evaluation mode in order to skip things such as Dropout
self.net = self.net.eval()
outputs = []
awareness_arr = []
loss_confidence_arr = [[] for _ in when_data_source.out_indexes]
for i, data in enumerate(data_loader, 0):
inputs, _ = data
inputs = inputs.to(self.net.device)
with torch.no_grad():
if self.is_selfaware:
self.selfaware_net.eval()
output = self.net(inputs)
awareness = self.selfaware_net(inputs, output)
awareness = awareness.to('cpu')
awareness_arr.extend(awareness.tolist())
else:
output = self.net(inputs)
awareness_arr = None
for k, criterion in enumerate(self.criterion_arr):
try:
max_conf = 1
if len(self.max_confidence_per_output) >= (
k - 1
) and self.max_confidence_per_output[k] is not None:
max_conf = self.max_confidence_per_output[k]
confidences = criterion.estimate_confidence(
output[:, when_data_source.out_indexes[k][0]:
when_data_source.out_indexes[k][1]],
max_conf)
loss_confidence_arr[k].extend(confidences)
except Exception:
loss_confidence_arr[k] = None
output = output.to('cpu')
outputs.extend(output)
output_trasnformed_vectors = {}
confidence_trasnformed_vectors = {}
for i in range(len(outputs)):
output_vector = outputs[i]
transformed_output_vectors = when_data_source.transformer.revert(
output_vector, feature_set='output_features')
for feature in transformed_output_vectors:
if feature not in output_trasnformed_vectors:
output_trasnformed_vectors[feature] = []
output_trasnformed_vectors[feature].append(
transformed_output_vectors[feature])
predictions = {}
for k, output_column in enumerate(
list(output_trasnformed_vectors.keys())):
decoded_predictions = when_data_source.get_decoded_column_data(
output_column,
torch.Tensor(output_trasnformed_vectors[output_column]))
predictions[output_column] = {
'predictions': decoded_predictions['predictions']
}
if awareness_arr is not None:
predictions[output_column]['selfaware_confidences'] = [
1 / abs(x[k]) if x[k] != 0 else 1 / 0.000001
for x in awareness_arr
]
if loss_confidence_arr[k] is not None:
predictions[output_column][
'loss_confidences'] = loss_confidence_arr[k]
if include_extra_data:
predictions[output_column][
'encoded_predictions'] = output_trasnformed_vectors[
output_column]
if 'class_distribution' in decoded_predictions:
predictions[output_column][
'class_distribution'] = decoded_predictions[
'class_distribution']
predictions[output_column][
'class_labels'] = decoded_predictions['class_labels']
log.info('Model predictions and decoding completed')
return predictions
def _fit(self, train_ds, test_ds, stop_training_after_seconds=None):
"""
:param stop_training_after_seconds: int
"""
if stop_training_after_seconds is None:
stop_training_after_seconds = self.stop_training_after_seconds
input_sample, output_sample = train_ds[0]
self.net = self.nn_class(self.dynamic_parameters,
input_size=len(input_sample),
output_size=len(output_sample),
nr_outputs=len(train_ds.out_types),
dropout=self.dropout_p)
self.net = self.net.train()
self.selfaware_net = SelfAware(input_size=len(input_sample),
output_size=len(output_sample),
nr_outputs=len(train_ds.out_types))
self.selfaware_net = self.selfaware_net.train()
if self.batch_size < self.net.available_devices:
self.batch_size = self.net.available_devices
self.awareness_criterion = torch.nn.MSELoss()
if self.criterion_arr is None:
self.criterion_arr = []
self.unreduced_criterion_arr = []
if train_ds.output_weights is not None and train_ds.output_weights is not False:
output_weights = torch.Tensor(train_ds.output_weights).to(
self.net.device)
else:
output_weights = None
for k, output_type in enumerate(train_ds.out_types):
if output_type in (COLUMN_DATA_TYPES.CATEGORICAL,
COLUMN_DATA_TYPES.MULTIPLE_CATEGORICAL):
if output_weights is None:
weights_slice = None
else:
# account for numerical features, not included in the output_weights
s_idx = train_ds.out_indexes[k][
0] - train_ds.output_weights_offset[k]
e_idx = train_ds.out_indexes[k][
1] - train_ds.output_weights_offset[k]
weights_slice = output_weights[s_idx:e_idx]
if output_type == COLUMN_DATA_TYPES.CATEGORICAL:
self.criterion_arr.append(
TransformCrossEntropyLoss(weight=weights_slice))
self.unreduced_criterion_arr.append(
TransformCrossEntropyLoss(weight=weights_slice,
reduction='none'))
elif output_type == COLUMN_DATA_TYPES.MULTIPLE_CATEGORICAL:
self.criterion_arr.append(
torch.nn.BCEWithLogitsLoss(weight=weights_slice))
self.unreduced_criterion_arr.append(
torch.nn.BCEWithLogitsLoss(weight=weights_slice,
reduction='none'))
# Note: MSELoss works great for numeric, for the other types it's more of a placeholder
else:
self.criterion_arr.append(torch.nn.MSELoss())
self.unreduced_criterion_arr.append(
torch.nn.MSELoss(reduction='none'))
self.optimizer_class = Ranger
if self.optimizer_args is None:
self.optimizer_args = {'lr': 0.0005}
if 'beta1' in self.dynamic_parameters:
self.optimizer_args['betas'] = (self.dynamic_parameters['beta1'],
0.999)
for optimizer_arg_name in ['lr', 'k', 'N_sma_threshold']:
if optimizer_arg_name in self.dynamic_parameters:
self.optimizer_args[
optimizer_arg_name] = self.dynamic_parameters[
optimizer_arg_name]
self.optimizer = self.optimizer_class(self.net.parameters(),
**self.optimizer_args)
self.selfaware_optimizer_args = copy.deepcopy(self.optimizer_args)
self.selfaware_optimizer_args['lr'] = self.selfaware_optimizer_args[
'lr'] * self.selfaware_lr_factor
self.selfaware_optimizer = self.optimizer_class(
self.selfaware_net.parameters(), **self.optimizer_args)
if stop_training_after_seconds is None:
stop_training_after_seconds = round(train_ds.data_frame.shape[0] *
train_ds.data_frame.shape[1] /
5)
if self.stop_model_building_after_seconds is None:
self.stop_model_building_after_seconds = stop_training_after_seconds * 3
if self.param_optimizer is not None:
input_size = len(train_ds[0][0])
training_data_length = len(train_ds)
while True:
training_time_per_iteration = stop_model_building_after_seconds / self.param_optimizer.total_trials
# Some heuristics...
if training_time_per_iteration > input_size:
if training_time_per_iteration > min(
(training_data_length /
(4 * input_size)), 16 * input_size):
break
self.param_optimizer.total_trials = self.param_optimizer.total_trials - 1
if self.param_optimizer.total_trials < 8:
self.param_optimizer.total_trials = 8
break
training_time_per_iteration = stop_model_building_after_seconds / self.param_optimizer.total_trials
self.dynamic_parameters = self.param_optimizer.evaluate(
lambda dynamic_parameters: self.evaluate(
from_data_ds,
test_data_ds,
dynamic_parameters,
max_training_time=training_time_per_iteration,
max_epochs=None))
log.info('Using hyperparameter set: ', best_parameters)
else:
self.dynamic_parameters = {}
started = time.time()
log_reasure = time.time()
stop_training = False
for subset_iteration in [1, 2]:
if stop_training:
break
subset_id_arr = [*train_ds.subsets.keys()]
for subset_id in subset_id_arr:
started_subset = time.time()
if stop_training:
break
subset_train_ds = train_ds.subsets[subset_id]
subset_test_ds = test_ds.subsets[subset_id]
lowest_error = None
last_test_error = None
last_subset_test_error = None
test_error_delta_buff = []
subset_test_error_delta_buff = []
best_model = None
#iterate over the iter_fit and see what the epoch and mixer error is
for epoch, training_error in enumerate(
self._iter_fit(subset_train_ds, subset_id=subset_id)):
# Log this every now and then so that the user knows it's running
if (int(time.time()) - log_reasure) > 30:
log_reasure = time.time()
log.info(
f'Lightwood training, iteration {epoch}, training error {training_error}'
)
# Prime the model on each subset for a bit
if subset_iteration == 1:
break
# Once the training error is getting smaller, enable dropout to teach the network to predict without certain features
if subset_iteration > 1 and training_error < 0.4 and not train_ds.enable_dropout:
self.eval_every_x_epochs = max(
1, int(self.eval_every_x_epochs / 2))
log.info('Enabled dropout !')
train_ds.enable_dropout = True
lowest_error = None
last_test_error = None
last_subset_test_error = None
test_error_delta_buff = []
subset_test_error_delta_buff = []
continue
# If the selfaware network isn't able to train, go back to the original network
if subset_iteration > 1 and (
np.isnan(training_error)
or np.isinf(training_error) or training_error >
pow(10, 5)) and not self.stop_selfaware_training:
self.start_selfaware_training = False
self.stop_selfaware_training = True
lowest_error = None
last_test_error = None
last_subset_test_error = None
test_error_delta_buff = []
subset_test_error_delta_buff = []
continue
# Once we are past the priming/warmup period, start training the selfaware network
if subset_iteration > 1 and not self.is_selfaware and self.selfaware and not self.stop_selfaware_training and training_error < 0.35:
log.info('Started selfaware training !')
self.start_selfaware_training = True
lowest_error = None
last_test_error = None
last_subset_test_error = None
test_error_delta_buff = []
subset_test_error_delta_buff = []
continue
if epoch % self.eval_every_x_epochs == 0:
test_error = self._error(test_ds)
subset_test_error = self._error(subset_test_ds,
subset_id=subset_id)
log.info(
f'Subtest test error: {subset_test_error} on subset {subset_id}, overall test error: {test_error}'
)
if lowest_error is None or test_error < lowest_error:
lowest_error = test_error
best_model = self._get_model_copy()
if last_subset_test_error is None:
pass
else:
subset_test_error_delta_buff.append(
last_subset_test_error - subset_test_error)
last_subset_test_error = subset_test_error
if last_test_error is None:
pass
else:
test_error_delta_buff.append(last_test_error -
test_error)
last_test_error = test_error
delta_mean = np.mean(test_error_delta_buff[-5:]
) if test_error_delta_buff else 0
subset_delta_mean = np.mean(
subset_test_error_delta_buff[-5:]
) if subset_test_error_delta_buff else 0
if self._nonpersistent['callback'] is not None:
self._nonpersistent['callback'](
epoch, training_error, test_error, delta_mean,
self.calculate_accuracy(test_ds))
else:
self._default_on_iter(
epoch, training_error, test_error, delta_mean,
self.calculate_accuracy(test_ds))
self.net.train()
# Stop if we're past the time limit allocated for training
if (time.time() -
started) > stop_training_after_seconds:
stop_training = True
# If the trauining subset is overfitting on it's associated testing subset
if (subset_delta_mean <= 0
and len(subset_test_error_delta_buff) > 4
) or (time.time() - started_subset
) > stop_training_after_seconds / len(
train_ds.subsets.keys(
)) or subset_test_error < 0.001:
log.info(
'Finished fitting on {subset_id} of {no_subsets} subset'
.format(subset_id=subset_id,
no_subsets=len(
train_ds.subsets.keys())))
self._update_model(best_model)
if subset_id == subset_id_arr[-1]:
stop_training = True
elif not stop_training:
break
if stop_training:
self._update_model(best_model)
if self.is_selfaware:
self._build_confidence_normalization_data(
train_ds)
self._adjust(test_ds)
self._iter_fit(test_ds, max_epochs=1)
self.encoders = train_ds.encoders
log.info('Finished training model !')
break
def predict(self, when_data_source, include_extra_data = False):
"""
:param when_data_source:
:return:
"""
when_data_source.transformer = self.transformer
when_data_source.encoders = self.encoders
data_loader = DataLoader(when_data_source, batch_size=self.batch_size, shuffle=False, num_workers=0)
# set model into evaluation mode in order to skip things such as Dropout
self.net = self.net.eval()
outputs = []
awareness_arr = []
for i, data in enumerate(data_loader, 0):
inputs, _ = data
inputs = inputs.to(self.net.device)
with torch.no_grad():
if CONFIG.SELFAWARE:
output, awareness = self.net(inputs)
awareness = awareness.to('cpu')
awareness_arr.extend(awareness)
else:
output = self.net(inputs)
awareness_arr = None
output = output.to('cpu')
outputs.extend(output)
output_trasnformed_vectors = {}
confidence_trasnformed_vectors = {}
for i in range(len(outputs)):
if awareness_arr is not None:
confidence_vector = awareness_arr[i]
transformed_confidence_vectors = when_data_source.transformer.revert(confidence_vector,feature_set = 'output_features')
for feature in transformed_confidence_vectors:
if feature not in confidence_trasnformed_vectors:
confidence_trasnformed_vectors[feature] = []
# @TODO: Very simple algorithm to get a confidence from the awareness, not necessarily what we want for the final version
confidence_trasnformed_vectors[feature].append(
[1 - sum(np.abs(transformed_confidence_vectors[feature])) / len(transformed_confidence_vectors[feature])]
)
output_vector = outputs[i]
transformed_output_vectors = when_data_source.transformer.revert(output_vector,feature_set = 'output_features')
for feature in transformed_output_vectors:
if feature not in output_trasnformed_vectors:
output_trasnformed_vectors[feature] = []
output_trasnformed_vectors[feature].append(transformed_output_vectors[feature])
predictions = {}
for output_column in output_trasnformed_vectors:
decoded_predictions = when_data_source.get_decoded_column_data(output_column, torch.Tensor(output_trasnformed_vectors[output_column]))
predictions[output_column] = {'predictions': decoded_predictions}
if awareness_arr is not None:
predictions[output_column]['confidences'] = confidence_trasnformed_vectors[output_column]
if include_extra_data:
predictions[output_column]['encoded_predictions'] = output_trasnformed_vectors[output_column]
log.info('Model predictions and decoding completed')
return predictions
def predict(self, when_data_source, include_extra_data=False):
"""
:param when_data_source:
:return:
"""
when_data_source.transformer = self.transformer
when_data_source.encoders = self.encoders
data_loader = DataLoader(when_data_source,
batch_size=len(when_data_source),
shuffle=False,
num_workers=0)
self.net.eval()
data = next(iter(data_loader))
inputs, _ = data
inputs = inputs.to(self.net.device)
sampled_models = [
self.pyro_guide(None, None)
for _ in range(CONFIG.NUMBER_OF_PROBABILISTIC_MODELS)
]
# A tensor of tensors representing the whole batch of predictions for each "model" pyro built
out_hats = [model(inputs).data for model in sampled_models]
'''
histo_exp = []
print(output_vectors)
for i in range(len(outputs_mean[0])):
for j in range(len(output_vectors)):
histo_exp.append(np.exp(output_vectors[i][j]))
meadian_probability = np.percentile(histo_exp, 50)
print(meadian_probability)
print(histo_exp)
'''
predictions_arr_dict = {}
for outputs in out_hats:
output_trasnformed_vectors = {}
for output_vector in outputs:
transformed_output_vectors = when_data_source.transformer.revert(
output_vector, feature_set='output_features')
for feature in transformed_output_vectors:
if feature not in output_trasnformed_vectors:
output_trasnformed_vectors[feature] = []
output_trasnformed_vectors[feature].append(
transformed_output_vectors[feature])
for output_column in output_trasnformed_vectors:
decoded_predictions = when_data_source.get_decoded_column_data(
output_column,
torch.Tensor(output_trasnformed_vectors[output_column]))
if output_column not in predictions_arr_dict:
predictions_arr_dict[output_column] = []
predictions_arr_dict[output_column].append(decoded_predictions)
predictions_dict = {}
for output_column in predictions_arr_dict:
final_predictions = []
final_predictions_confidence = []
possible_predictions = []
possible_predictions_confidence = []
predictions_arr = predictions_arr_dict[output_column]
predictions_arr_r = [[]] * len(predictions_arr[0])
for i in range(len(predictions_arr)):
for j in range(len(predictions_arr[i])):
predictions_arr_r[j].append(predictions_arr[i][j])
for predictions in predictions_arr_r:
if when_data_source.get_column_config(
output_column)['type'] == 'categorical':
occurance_dict = dict(Counter(predictions))
final_prediction = max(occurance_dict,
key=occurance_dict.get)
final_predictions.append(final_prediction)
final_predictions_confidence.append(
occurance_dict[final_prediction] / len(predictions))
possible_predictions.append(list(occurance_dict.keys()))
possible_predictions_confidence.append([
x / len(predictions)
for x in list(occurance_dict.values())
])
else:
predictions = [
x if x is not None else 0 for x in predictions
]
p_hist = np.histogram(np.array(predictions), 10)
final_predictions_confidence.append(
max(p_hist[0]) / sum(p_hist[0]))
final_predictions.append(
p_hist[1][np.where(p_hist[0] == max(p_hist[0]))][0])
possible_predictions_confidence.append(
[x / sum(p_hist[0]) for x in p_hist[0]])
possible_predictions.append(list(p_hist[1]))
predictions_dict[output_column] = {}
predictions_dict[output_column]['predictions'] = final_predictions
predictions_dict[output_column][
'confidence'] = final_predictions_confidence
predictions_dict[output_column][
'possible_predictions'] = possible_predictions
predictions_dict[output_column][
'possible_predictions_confidence'] = possible_predictions_confidence
log.info('Model predictions and decoding completed')
return predictions_dict
