def prepare(self, priming_data): if self._prepared: raise Exception('You can only call "prepare" once for a given encoder.') no_null_sentences = [x if x is not None else '' for x in priming_data] estimated_time = 1/937*self._train_iters*len(no_null_sentences) log_every = math.ceil(self._train_iters/100) log.info('We will train an encoder for this text, on a CPU it will take about {min} minutes'.format( min=estimated_time)) self._input_lang = Lang('input') self._output_lang = self._input_lang for row in no_null_sentences: if row is not None: self._input_lang.addSentence(row) max_length = max(map(len, no_null_sentences)) hidden_size = self._encoded_vector_size self._encoder = EncoderRNN(self._input_lang.n_words, hidden_size).to(device) self._decoder = DecoderRNN(hidden_size, self._output_lang.n_words).to(device) trainIters(self._encoder, self._decoder, self._input_lang, self._output_lang, no_null_sentences, no_null_sentences, self._train_iters, int(log_every), self._learning_rate, self._stop_on_error, max_length) self._prepared = True
def _default_on_iter(self, epoch, train_error, test_error, delta_mean, accuracy): test_error_rounded = round(test_error, 4) for col in accuracy: value = accuracy[col]['value'] if accuracy[col]['function'] == 'r2_score': value_rounded = round(value, 3) log.info( f'We\'ve reached training epoch nr {epoch} with an r2 score of {value_rounded} on the testing dataset' ) else: value_pct = round(value * 100, 2) log.info( f'We\'ve reached training epoch nr {epoch} with an accuracy of {value_pct}% on the testing dataset' )
def predict(self, when_data=None, when=None): """ Predict given when conditions. :param when_data: pandas.DataFrame :param when: dict :return: pandas.DataFrame """ device, _available_devices = get_devices() log.info(f'Computing device used: {device}') if when is not None: when_dict = {key: [when[key]] for key in when} when_data = pandas.DataFrame(when_dict) when_data_ds = DataSource(when_data, self.config, prepare_encoders=False) when_data_ds.eval() kwargs = {'include_extra_data': self.config.get('include_extra_data', False)} return self._mixer.predict(when_data_ds, **kwargs)
def learn(self, from_data, test_data=None): """ Train and save a model (you can use this to retrain model from data). :param from_data: DataFrame or DataSource The data to learn from :param test_data: DataFrame or DataSource The data to test accuracy and learn_error from """ device, _available_devices = get_devices() log.info(f'Computing device used: {device}') # generate the configuration and set the order for the input and output columns if self._generate_config is True: self._input_columns = [col for col in from_data if col not in self._output_columns] self.config = { 'input_features': [{'name': col, 'type': self._type_map(from_data, col)} for col in self._input_columns], 'output_features': [{'name': col, 'type': self._type_map(from_data, col)} for col in self._output_columns] } self.config = predictor_config_schema.validate(self.config) log.info('Automatically generated a configuration') log.info(self.config) else: self._output_columns = [col['name'] for col in self.config['output_features']] self._input_columns = [col['name'] for col in self.config['input_features']] if isinstance(from_data, pandas.DataFrame): train_ds = DataSource(from_data, self.config) elif isinstance(from_data, DataSource): train_ds = from_data else: raise TypeError(':from_data: must be either DataFrame or DataSource') nr_subsets = 3 if len(train_ds) > 100 else 1 if test_data is None: test_ds = train_ds.subset(0.1) elif isinstance(test_data, pandas.DataFrame): test_ds = train_ds.make_child(test_data) elif isinstance(test_data, DataSource): test_ds = test_data else: raise TypeError(':test_data: must be either DataFrame or DataSource') train_ds.create_subsets(nr_subsets) test_ds.create_subsets(nr_subsets) train_ds.train() test_ds.train() mixer_class = self.config['mixer']['class'] mixer_kwargs = self.config['mixer']['kwargs'] self._mixer = mixer_class(**mixer_kwargs) self._mixer.fit(train_ds=train_ds, test_ds=test_ds) self.train_accuracy = self._mixer.calculate_accuracy(test_ds) return self
def encode(self, column_data): encoded_audio_arr = [] for path in column_data: if path.startswith('http'): response = requests.get(path) with open(path.split('/')[-1], 'wb') as f: f.write(response.content) try: audio = AudioSegment.from_file(path.split('/')[-1]) except Exception as e: print(e) finally: os.remove(path.split('/')[-1]) else: audio = AudioSegment.from_file(path) # For now convert all (usually will be stereo) to mono by adding up and averging the amplitudes audio = audio.set_channels(1) original_frame_rate = audio.frame_rate new_frame_rate = int( original_frame_rate / (len(audio.get_array_of_samples()) / self._max_samples)) if new_frame_rate < original_frame_rate: audio = audio.set_frame_rate(new_frame_rate) log.info( f'Lowering audio frame rate from {original_frame_rate} to {new_frame_rate} for ease of processing !' ) audio_arr = list(np.array(audio.get_array_of_samples())) encoded_audio = self._ts_encoder.encode([audio_arr]) encoded_audio_arr.append(encoded_audio[0]) return torch.Tensor(encoded_audio_arr)
def __init__(self, dynamic_parameters, input_size=None, output_size=None, nr_outputs=None, shape=None, dropout=None, pretrained_net=None): self.input_size = input_size self.output_size = output_size self.nr_outputs = nr_outputs self.max_variance = None # How many devices we can train this network on self.available_devices = 1 self.device, _ = get_devices() self.dynamic_parameters = dynamic_parameters """ Here we define the basic building blocks of our model, in forward we define how we put it all together along with an input """ super(DefaultNet, self).__init__() if shape is None and pretrained_net is None: shape = [ self.input_size, max([self.input_size * 2, self.output_size * 2, 400]), self.output_size ] if pretrained_net is None: log.info(f'Building network of shape: {shape}') rectifier = torch.nn.SELU #alternative: torch.nn.ReLU layers = [] for ind in range(len(shape) - 1): if (dropout is not None) and (0 < ind < len(shape)): layers.append(torch.nn.Dropout(p=dropout)) linear_function = PLinear if CONFIG.USE_PROBABILISTIC_LINEAR else torch.nn.Linear layers.append(linear_function(shape[ind], shape[ind + 1])) if ind < len(shape) - 2: layers.append(rectifier()) self.net = torch.nn.Sequential(*layers) else: self.net = pretrained_net for layer in self.net: if isinstance(layer, torch.nn.Linear): if self.input_size is None: self.input_size = layer.in_features self.output_size = layer.out_features self.net = self.net.to(self.device) if 'cuda' in str(self.device): self.available_devices = torch.cuda.device_count() else: self.available_devices = 1 if self.available_devices > 1: self._foward_net = torch.nn.DataParallel(self.net) else: self._foward_net = self.net
def prepare(self, priming_data): random.seed(len(priming_data)) if self._prepared: raise Exception( 'You can only call "prepare" once for a given encoder.') self.onehot_encoder.prepare(priming_data) input_len = self.onehot_encoder._lang.n_words self.use_autoencoder = self.max_encoded_length is not None and input_len > self.max_encoded_length if self.use_autoencoder: log.info( 'Preparing a categorical autoencoder, this might take a while') embeddings_layer_len = self.max_encoded_length self.net = DefaultNet( dynamic_parameters={}, shape=[input_len, embeddings_layer_len, input_len]) criterion = torch.nn.CrossEntropyLoss() optimizer = Ranger(self.net.parameters()) gym = Gym(model=self.net, optimizer=optimizer, scheduler=None, loss_criterion=criterion, device=self.net.device, name=self.name, input_encoder=self.onehot_encoder.encode, output_encoder=self._encoder_targets) batch_size = min(200, int(len(priming_data) / 50)) priming_data_str = [str(x) for x in priming_data] train_data_loader = DataLoader(list( zip(priming_data_str, priming_data_str)), batch_size=batch_size, shuffle=True) test_data_loader = None best_model, error, training_time = gym.fit( train_data_loader, test_data_loader, desired_error=self.desired_error, max_time=self.max_training_time, callback=self._train_callback, eval_every_x_epochs=1, max_unimproving_models=5) self.net = best_model.to(self.net.device) modules = [ module for module in self.net.modules() if type(module) != torch.nn.Sequential and type(module) != DefaultNet ] self.encoder = torch.nn.Sequential(*modules[0:2]).eval() self.decoder = torch.nn.Sequential(*modules[2:3]).eval() log.info('Categorical autoencoder ready') self._prepared = True
def _train_callback(self, error, real_buff, predicted_buff): log.info(f'{self.name} reached a loss of {error} while training !')
def _iter_fit(self, ds, subset_id=None, max_epochs=120000): if self._nonpersistent['sampler'] is None: data_loader = DataLoader(ds, batch_size=self.batch_size, shuffle=True, num_workers=0) else: data_loader = DataLoader(ds, batch_size=self.batch_size, num_workers=0, sampler=self._nonpersistent['sampler']) for epoch in range(max_epochs): # loop over the dataset multiple times running_loss = 0.0 error = 0 for i, data in enumerate(data_loader, 0): if self.start_selfaware_training and not self.is_selfaware: log.info( 'Starting to train selfaware network for better confidence determination !' ) self.is_selfaware = True if self.stop_selfaware_training and self.is_selfaware: log.info( 'Cannot train selfaware network, will fallback to using simpler confidence models !' ) self.is_selfaware = False self.total_iterations += 1 # get the inputs; data is a list of [inputs, labels] inputs, labels = data labels = labels.to(self.net.device) inputs = inputs.to(self.net.device) # zero the parameter gradients self.optimizer.zero_grad() self.selfaware_optimizer.zero_grad() # forward + backward + optimize with LightwoodAutocast(): outputs = self.net(inputs) if self.is_selfaware: with LightwoodAutocast(): awareness = self.selfaware_net(inputs.detach(), outputs.detach()) loss = None for k, criterion in enumerate(self.criterion_arr): with LightwoodAutocast(): target_loss = criterion( outputs[:, ds.out_indexes[k][0]:ds.out_indexes[k][1]], labels[:, ds.out_indexes[k][0]:ds.out_indexes[k][1]]) if loss is None: loss = target_loss else: loss += target_loss awareness_loss = None if self.is_selfaware: unreduced_losses = [] for k, criterion in enumerate( self.unreduced_criterion_arr): target_loss = criterion( outputs[:, ds.out_indexes[k][0]:ds.out_indexes[k][1]], labels[:, ds.out_indexes[k][0]:ds.out_indexes[k][1]]) target_loss = target_loss.tolist() if type(target_loss[0]) == type([]): target_loss = [np.mean(x) for x in target_loss] for i, value in enumerate(target_loss): if len(unreduced_losses) <= i: unreduced_losses.append([]) unreduced_losses[i].append(value) unreduced_losses = torch.Tensor(unreduced_losses).to( self.net.device) awareness_loss = self.awareness_criterion( awareness, unreduced_losses) * self.awareness_scale_factor if CONFIG.MONITORING['batch_loss']: self.monitor.plot_loss(awareness_loss.item(), self.total_iterations, 'Awreness Batch Loss') if CONFIG.MONITORING['batch_loss']: self.monitor.plot_loss(loss.item(), self.total_iterations, 'Targets Batch Loss') if awareness_loss is not None: awareness_loss.backward(retain_graph=True) running_loss += loss.item() loss.backward() # @NOTE: Decrease 900 if you want to plot gradients more often, I find it's too expensive to do so if CONFIG.MONITORING['network_heatmap'] and random.randint( 0, 1000) > 900: weights = [] gradients = [] layer_name = [] for index, layer in enumerate(self.net.net): if 'Linear' in str(type(layer)): weights.append( list(layer.weight.cpu().detach().numpy().ravel( ))) gradients.append( list(layer.weight.grad.cpu().detach().numpy(). ravel())) layer_name.append(f'Layer {index}-{index+1}') self.monitor.weight_map(layer_name, weights, 'Predcitive network weights') self.monitor.weight_map(layer_name, weights, 'Predictive network gradients') if self.is_selfaware: weights = [] gradients = [] layer_name = [] for index, layer in enumerate(self.selfaware_net.net): if 'Linear' in str(type(layer)): weights.append( list(layer.weight.cpu().detach().numpy(). ravel())) gradients.append( list(layer.weight.grad.cpu().detach(). numpy().ravel())) layer_name.append(f'Layer {index}-{index+1}') self.monitor.weight_map(layer_name, weights, 'Awareness network weights') self.monitor.weight_map(layer_name, weights, 'Awareness network gradients') # now that we have run backward in both losses, optimize() # (review: we may need to optimize for each step) self.optimizer.step() if self.is_selfaware and self.start_selfaware_training: self.selfaware_optimizer.step() error = running_loss / (i + 1) if CONFIG.MONITORING['batch_loss']: #self.monitor.plot_loss(total_loss.item(), self.total_iterations, 'Total Batch Loss') self.monitor.plot_loss(error, self.total_iterations, 'Mean Total Running Loss') if CONFIG.MONITORING['epoch_loss']: self.monitor.plot_loss(error, self.total_iterations, 'Train Epoch Error') self.monitor.plot_loss( error, self.total_iterations, f'Train Epoch Error - Subset {subset_id}') yield error
def _predict(self, when_data_source, include_extra_data=False): data_loader = DataLoader(when_data_source, batch_size=self.batch_size, shuffle=False, num_workers=0) # set model into evaluation mode in order to skip things such as Dropout self.net = self.net.eval() outputs = [] awareness_arr = [] loss_confidence_arr = [[] for _ in when_data_source.out_indexes] for i, data in enumerate(data_loader, 0): inputs, _ = data inputs = inputs.to(self.net.device) with torch.no_grad(): if self.is_selfaware: self.selfaware_net.eval() output = self.net(inputs) awareness = self.selfaware_net(inputs, output) awareness = awareness.to('cpu') awareness_arr.extend(awareness.tolist()) else: output = self.net(inputs) awareness_arr = None for k, criterion in enumerate(self.criterion_arr): try: max_conf = 1 if len(self.max_confidence_per_output) >= ( k - 1 ) and self.max_confidence_per_output[k] is not None: max_conf = self.max_confidence_per_output[k] confidences = criterion.estimate_confidence( output[:, when_data_source.out_indexes[k][0]: when_data_source.out_indexes[k][1]], max_conf) loss_confidence_arr[k].extend(confidences) except Exception: loss_confidence_arr[k] = None output = output.to('cpu') outputs.extend(output) output_trasnformed_vectors = {} confidence_trasnformed_vectors = {} for i in range(len(outputs)): output_vector = outputs[i] transformed_output_vectors = when_data_source.transformer.revert( output_vector, feature_set='output_features') for feature in transformed_output_vectors: if feature not in output_trasnformed_vectors: output_trasnformed_vectors[feature] = [] output_trasnformed_vectors[feature].append( transformed_output_vectors[feature]) predictions = {} for k, output_column in enumerate( list(output_trasnformed_vectors.keys())): decoded_predictions = when_data_source.get_decoded_column_data( output_column, torch.Tensor(output_trasnformed_vectors[output_column])) predictions[output_column] = { 'predictions': decoded_predictions['predictions'] } if awareness_arr is not None: predictions[output_column]['selfaware_confidences'] = [ 1 / abs(x[k]) if x[k] != 0 else 1 / 0.000001 for x in awareness_arr ] if loss_confidence_arr[k] is not None: predictions[output_column][ 'loss_confidences'] = loss_confidence_arr[k] if include_extra_data: predictions[output_column][ 'encoded_predictions'] = output_trasnformed_vectors[ output_column] if 'class_distribution' in decoded_predictions: predictions[output_column][ 'class_distribution'] = decoded_predictions[ 'class_distribution'] predictions[output_column][ 'class_labels'] = decoded_predictions['class_labels'] log.info('Model predictions and decoding completed') return predictions
def _fit(self, train_ds, test_ds, stop_training_after_seconds=None): """ :param stop_training_after_seconds: int """ if stop_training_after_seconds is None: stop_training_after_seconds = self.stop_training_after_seconds input_sample, output_sample = train_ds[0] self.net = self.nn_class(self.dynamic_parameters, input_size=len(input_sample), output_size=len(output_sample), nr_outputs=len(train_ds.out_types), dropout=self.dropout_p) self.net = self.net.train() self.selfaware_net = SelfAware(input_size=len(input_sample), output_size=len(output_sample), nr_outputs=len(train_ds.out_types)) self.selfaware_net = self.selfaware_net.train() if self.batch_size < self.net.available_devices: self.batch_size = self.net.available_devices self.awareness_criterion = torch.nn.MSELoss() if self.criterion_arr is None: self.criterion_arr = [] self.unreduced_criterion_arr = [] if train_ds.output_weights is not None and train_ds.output_weights is not False: output_weights = torch.Tensor(train_ds.output_weights).to( self.net.device) else: output_weights = None for k, output_type in enumerate(train_ds.out_types): if output_type in (COLUMN_DATA_TYPES.CATEGORICAL, COLUMN_DATA_TYPES.MULTIPLE_CATEGORICAL): if output_weights is None: weights_slice = None else: # account for numerical features, not included in the output_weights s_idx = train_ds.out_indexes[k][ 0] - train_ds.output_weights_offset[k] e_idx = train_ds.out_indexes[k][ 1] - train_ds.output_weights_offset[k] weights_slice = output_weights[s_idx:e_idx] if output_type == COLUMN_DATA_TYPES.CATEGORICAL: self.criterion_arr.append( TransformCrossEntropyLoss(weight=weights_slice)) self.unreduced_criterion_arr.append( TransformCrossEntropyLoss(weight=weights_slice, reduction='none')) elif output_type == COLUMN_DATA_TYPES.MULTIPLE_CATEGORICAL: self.criterion_arr.append( torch.nn.BCEWithLogitsLoss(weight=weights_slice)) self.unreduced_criterion_arr.append( torch.nn.BCEWithLogitsLoss(weight=weights_slice, reduction='none')) # Note: MSELoss works great for numeric, for the other types it's more of a placeholder else: self.criterion_arr.append(torch.nn.MSELoss()) self.unreduced_criterion_arr.append( torch.nn.MSELoss(reduction='none')) self.optimizer_class = Ranger if self.optimizer_args is None: self.optimizer_args = {'lr': 0.0005} if 'beta1' in self.dynamic_parameters: self.optimizer_args['betas'] = (self.dynamic_parameters['beta1'], 0.999) for optimizer_arg_name in ['lr', 'k', 'N_sma_threshold']: if optimizer_arg_name in self.dynamic_parameters: self.optimizer_args[ optimizer_arg_name] = self.dynamic_parameters[ optimizer_arg_name] self.optimizer = self.optimizer_class(self.net.parameters(), **self.optimizer_args) self.selfaware_optimizer_args = copy.deepcopy(self.optimizer_args) self.selfaware_optimizer_args['lr'] = self.selfaware_optimizer_args[ 'lr'] * self.selfaware_lr_factor self.selfaware_optimizer = self.optimizer_class( self.selfaware_net.parameters(), **self.optimizer_args) if stop_training_after_seconds is None: stop_training_after_seconds = round(train_ds.data_frame.shape[0] * train_ds.data_frame.shape[1] / 5) if self.stop_model_building_after_seconds is None: self.stop_model_building_after_seconds = stop_training_after_seconds * 3 if self.param_optimizer is not None: input_size = len(train_ds[0][0]) training_data_length = len(train_ds) while True: training_time_per_iteration = stop_model_building_after_seconds / self.param_optimizer.total_trials # Some heuristics... if training_time_per_iteration > input_size: if training_time_per_iteration > min( (training_data_length / (4 * input_size)), 16 * input_size): break self.param_optimizer.total_trials = self.param_optimizer.total_trials - 1 if self.param_optimizer.total_trials < 8: self.param_optimizer.total_trials = 8 break training_time_per_iteration = stop_model_building_after_seconds / self.param_optimizer.total_trials self.dynamic_parameters = self.param_optimizer.evaluate( lambda dynamic_parameters: self.evaluate( from_data_ds, test_data_ds, dynamic_parameters, max_training_time=training_time_per_iteration, max_epochs=None)) log.info('Using hyperparameter set: ', best_parameters) else: self.dynamic_parameters = {} started = time.time() log_reasure = time.time() stop_training = False for subset_iteration in [1, 2]: if stop_training: break subset_id_arr = [*train_ds.subsets.keys()] for subset_id in subset_id_arr: started_subset = time.time() if stop_training: break subset_train_ds = train_ds.subsets[subset_id] subset_test_ds = test_ds.subsets[subset_id] lowest_error = None last_test_error = None last_subset_test_error = None test_error_delta_buff = [] subset_test_error_delta_buff = [] best_model = None #iterate over the iter_fit and see what the epoch and mixer error is for epoch, training_error in enumerate( self._iter_fit(subset_train_ds, subset_id=subset_id)): # Log this every now and then so that the user knows it's running if (int(time.time()) - log_reasure) > 30: log_reasure = time.time() log.info( f'Lightwood training, iteration {epoch}, training error {training_error}' ) # Prime the model on each subset for a bit if subset_iteration == 1: break # Once the training error is getting smaller, enable dropout to teach the network to predict without certain features if subset_iteration > 1 and training_error < 0.4 and not train_ds.enable_dropout: self.eval_every_x_epochs = max( 1, int(self.eval_every_x_epochs / 2)) log.info('Enabled dropout !') train_ds.enable_dropout = True lowest_error = None last_test_error = None last_subset_test_error = None test_error_delta_buff = [] subset_test_error_delta_buff = [] continue # If the selfaware network isn't able to train, go back to the original network if subset_iteration > 1 and ( np.isnan(training_error) or np.isinf(training_error) or training_error > pow(10, 5)) and not self.stop_selfaware_training: self.start_selfaware_training = False self.stop_selfaware_training = True lowest_error = None last_test_error = None last_subset_test_error = None test_error_delta_buff = [] subset_test_error_delta_buff = [] continue # Once we are past the priming/warmup period, start training the selfaware network if subset_iteration > 1 and not self.is_selfaware and self.selfaware and not self.stop_selfaware_training and training_error < 0.35: log.info('Started selfaware training !') self.start_selfaware_training = True lowest_error = None last_test_error = None last_subset_test_error = None test_error_delta_buff = [] subset_test_error_delta_buff = [] continue if epoch % self.eval_every_x_epochs == 0: test_error = self._error(test_ds) subset_test_error = self._error(subset_test_ds, subset_id=subset_id) log.info( f'Subtest test error: {subset_test_error} on subset {subset_id}, overall test error: {test_error}' ) if lowest_error is None or test_error < lowest_error: lowest_error = test_error best_model = self._get_model_copy() if last_subset_test_error is None: pass else: subset_test_error_delta_buff.append( last_subset_test_error - subset_test_error) last_subset_test_error = subset_test_error if last_test_error is None: pass else: test_error_delta_buff.append(last_test_error - test_error) last_test_error = test_error delta_mean = np.mean(test_error_delta_buff[-5:] ) if test_error_delta_buff else 0 subset_delta_mean = np.mean( subset_test_error_delta_buff[-5:] ) if subset_test_error_delta_buff else 0 if self._nonpersistent['callback'] is not None: self._nonpersistent['callback']( epoch, training_error, test_error, delta_mean, self.calculate_accuracy(test_ds)) else: self._default_on_iter( epoch, training_error, test_error, delta_mean, self.calculate_accuracy(test_ds)) self.net.train() # Stop if we're past the time limit allocated for training if (time.time() - started) > stop_training_after_seconds: stop_training = True # If the trauining subset is overfitting on it's associated testing subset if (subset_delta_mean <= 0 and len(subset_test_error_delta_buff) > 4 ) or (time.time() - started_subset ) > stop_training_after_seconds / len( train_ds.subsets.keys( )) or subset_test_error < 0.001: log.info( 'Finished fitting on {subset_id} of {no_subsets} subset' .format(subset_id=subset_id, no_subsets=len( train_ds.subsets.keys()))) self._update_model(best_model) if subset_id == subset_id_arr[-1]: stop_training = True elif not stop_training: break if stop_training: self._update_model(best_model) if self.is_selfaware: self._build_confidence_normalization_data( train_ds) self._adjust(test_ds) self._iter_fit(test_ds, max_epochs=1) self.encoders = train_ds.encoders log.info('Finished training model !') break
def predict(self, when_data_source, include_extra_data = False): """ :param when_data_source: :return: """ when_data_source.transformer = self.transformer when_data_source.encoders = self.encoders data_loader = DataLoader(when_data_source, batch_size=self.batch_size, shuffle=False, num_workers=0) # set model into evaluation mode in order to skip things such as Dropout self.net = self.net.eval() outputs = [] awareness_arr = [] for i, data in enumerate(data_loader, 0): inputs, _ = data inputs = inputs.to(self.net.device) with torch.no_grad(): if CONFIG.SELFAWARE: output, awareness = self.net(inputs) awareness = awareness.to('cpu') awareness_arr.extend(awareness) else: output = self.net(inputs) awareness_arr = None output = output.to('cpu') outputs.extend(output) output_trasnformed_vectors = {} confidence_trasnformed_vectors = {} for i in range(len(outputs)): if awareness_arr is not None: confidence_vector = awareness_arr[i] transformed_confidence_vectors = when_data_source.transformer.revert(confidence_vector,feature_set = 'output_features') for feature in transformed_confidence_vectors: if feature not in confidence_trasnformed_vectors: confidence_trasnformed_vectors[feature] = [] # @TODO: Very simple algorithm to get a confidence from the awareness, not necessarily what we want for the final version confidence_trasnformed_vectors[feature].append( [1 - sum(np.abs(transformed_confidence_vectors[feature])) / len(transformed_confidence_vectors[feature])] ) output_vector = outputs[i] transformed_output_vectors = when_data_source.transformer.revert(output_vector,feature_set = 'output_features') for feature in transformed_output_vectors: if feature not in output_trasnformed_vectors: output_trasnformed_vectors[feature] = [] output_trasnformed_vectors[feature].append(transformed_output_vectors[feature]) predictions = {} for output_column in output_trasnformed_vectors: decoded_predictions = when_data_source.get_decoded_column_data(output_column, torch.Tensor(output_trasnformed_vectors[output_column])) predictions[output_column] = {'predictions': decoded_predictions} if awareness_arr is not None: predictions[output_column]['confidences'] = confidence_trasnformed_vectors[output_column] if include_extra_data: predictions[output_column]['encoded_predictions'] = output_trasnformed_vectors[output_column] log.info('Model predictions and decoding completed') return predictions
def predict(self, when_data_source, include_extra_data=False): """ :param when_data_source: :return: """ when_data_source.transformer = self.transformer when_data_source.encoders = self.encoders data_loader = DataLoader(when_data_source, batch_size=len(when_data_source), shuffle=False, num_workers=0) self.net.eval() data = next(iter(data_loader)) inputs, _ = data inputs = inputs.to(self.net.device) sampled_models = [ self.pyro_guide(None, None) for _ in range(CONFIG.NUMBER_OF_PROBABILISTIC_MODELS) ] # A tensor of tensors representing the whole batch of predictions for each "model" pyro built out_hats = [model(inputs).data for model in sampled_models] ''' histo_exp = [] print(output_vectors) for i in range(len(outputs_mean[0])): for j in range(len(output_vectors)): histo_exp.append(np.exp(output_vectors[i][j])) meadian_probability = np.percentile(histo_exp, 50) print(meadian_probability) print(histo_exp) ''' predictions_arr_dict = {} for outputs in out_hats: output_trasnformed_vectors = {} for output_vector in outputs: transformed_output_vectors = when_data_source.transformer.revert( output_vector, feature_set='output_features') for feature in transformed_output_vectors: if feature not in output_trasnformed_vectors: output_trasnformed_vectors[feature] = [] output_trasnformed_vectors[feature].append( transformed_output_vectors[feature]) for output_column in output_trasnformed_vectors: decoded_predictions = when_data_source.get_decoded_column_data( output_column, torch.Tensor(output_trasnformed_vectors[output_column])) if output_column not in predictions_arr_dict: predictions_arr_dict[output_column] = [] predictions_arr_dict[output_column].append(decoded_predictions) predictions_dict = {} for output_column in predictions_arr_dict: final_predictions = [] final_predictions_confidence = [] possible_predictions = [] possible_predictions_confidence = [] predictions_arr = predictions_arr_dict[output_column] predictions_arr_r = [[]] * len(predictions_arr[0]) for i in range(len(predictions_arr)): for j in range(len(predictions_arr[i])): predictions_arr_r[j].append(predictions_arr[i][j]) for predictions in predictions_arr_r: if when_data_source.get_column_config( output_column)['type'] == 'categorical': occurance_dict = dict(Counter(predictions)) final_prediction = max(occurance_dict, key=occurance_dict.get) final_predictions.append(final_prediction) final_predictions_confidence.append( occurance_dict[final_prediction] / len(predictions)) possible_predictions.append(list(occurance_dict.keys())) possible_predictions_confidence.append([ x / len(predictions) for x in list(occurance_dict.values()) ]) else: predictions = [ x if x is not None else 0 for x in predictions ] p_hist = np.histogram(np.array(predictions), 10) final_predictions_confidence.append( max(p_hist[0]) / sum(p_hist[0])) final_predictions.append( p_hist[1][np.where(p_hist[0] == max(p_hist[0]))][0]) possible_predictions_confidence.append( [x / sum(p_hist[0]) for x in p_hist[0]]) possible_predictions.append(list(p_hist[1])) predictions_dict[output_column] = {} predictions_dict[output_column]['predictions'] = final_predictions predictions_dict[output_column][ 'confidence'] = final_predictions_confidence predictions_dict[output_column][ 'possible_predictions'] = possible_predictions predictions_dict[output_column][ 'possible_predictions_confidence'] = possible_predictions_confidence log.info('Model predictions and decoding completed') return predictions_dict