def __init__(self, is_target=False, aim=ENCODER_AIM.BALANCE):
super().__init__(is_target)
self.name = 'Text Transformer Encoder'
self._tokenizer = None
self._model = None
self._pad_id = None
self._max_len = None
self._max_ele = None
self._model_type = None
self.desired_error = 0.01
self.max_training_time = 7200
self._head = None
# Possible: speed, balance, accuracy
self.aim = aim
if self.aim == ENCODER_AIM.SPEED:
# uses more memory, takes very long to train and outputs weird debugging statements to the command line,
# consider waiting until it gets better or try to investigate why this happens
# (changing the pretrained model doesn't seem to help)
self._classifier_model_class = AlbertForSequenceClassification
self._embeddings_model_class = AlbertModel
self._tokenizer_class = AlbertTokenizer
self._pretrained_model_name = 'albert-base-v2'
if self.aim == ENCODER_AIM.BALANCE:
self._classifier_model_class = DistilBertForSequenceClassification
self._embeddings_model_class = DistilBertModel
self._tokenizer_class = DistilBertTokenizer
self._pretrained_model_name = 'distilbert-base-uncased'
if self.aim == ENCODER_AIM.ACCURACY:
self._classifier_model_class = DistilBertForSequenceClassification
self._embeddings_model_class = DistilBertModel
self._tokenizer_class = DistilBertTokenizer
self._pretrained_model_name = 'distilbert-base-uncased'
self.device, _ = get_devices()
def __init__(self, in_features, out_features, bias=True):
"""
:param in_features: as name suggests
:param out_features: this essentially the number of neurons
:param bias: if you want a specific bias
"""
super(PLinear, self).__init__()
self.in_features = in_features
self.out_features = out_features
# these are the matrices that we will optimize for
self.sigma = Parameter(torch.Tensor(out_features, in_features))
self.mean = Parameter(torch.Tensor(out_features, in_features))
# there can be various ways to sample, given various distributions,
# we will stick with discrete normal as it is way faster
self.w_sampler = self.w_discrete_normal
if bias:
self.bias = Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
self.reset_parameters()
# make sure that we tell the graph that these two need to be optimized
self.sigma.requiresGrad = True # set requiresGrad to true!
self.mean.requiresGrad = True # set requiresGrad to true!
self.device, _ = get_devices()
def __init__(self, input_size, output_size, nr_outputs):
super(SelfAware, self).__init__()
self.input_size = input_size
self.output_size = output_size
self.nr_outputs = nr_outputs
awareness_layers = []
awareness_net_shape = [
(self.input_size + self.output_size),
max([int((self.input_size + self.output_size) * 1.5),
300]), self.nr_outputs
]
for ind in range(len(awareness_net_shape) - 1):
rectifier = torch.nn.SELU
awareness_layers.append(
torch.nn.Linear(awareness_net_shape[ind],
awareness_net_shape[ind + 1]))
if ind < len(awareness_net_shape) - 2:
awareness_layers.append(rectifier())
self.net = torch.nn.Sequential(*awareness_layers)
for layer in self.net:
if hasattr(layer, 'weight'):
torch.nn.init.normal_(layer.weight,
mean=0.,
std=1 / math.sqrt(layer.out_features))
if hasattr(layer, 'bias'):
torch.nn.init.normal_(layer.bias, mean=0., std=0.1)
self.device, self.available_devices = get_devices()
self.to(self.device, self.available_devices)
def to(self, device=None, available_devices=None):
if device is None or available_devices is None:
device, available_devices = get_devices()
self.net = self.net.to(device)
if self.selfaware:
self.awareness_net = self.awareness_net.to(device)
available_devices = 1
if 'cuda' in str(device):
available_devices = torch.cuda.device_count()
if available_devices > 1:
self._foward_net = torch.nn.DataParallel(self.net)
if self.selfaware:
self._foward_awareness_net = torch.nn.DataParallel(self.awareness_net)
else:
self._foward_net = self.net
if self.selfaware:
self._foward_awareness_net = self.awareness_net
self.device = device
self.available_devices = available_devices
return self
def __init__(self, config):
super(InferSent, self).__init__()
self.bsize = config['bsize']
self.word_emb_dim = config['word_emb_dim']
self.enc_lstm_dim = config['enc_lstm_dim']
self.pool_type = config['pool_type']
self.dpout_model = config['dpout_model']
self.version = 1 if 'version' not in config else config['version']
self.enc_lstm = nn.LSTM(self.word_emb_dim,
self.enc_lstm_dim,
1,
bidirectional=True,
dropout=self.dpout_model)
self.device, _ = get_devices()
self.enc_lstm = self.enc_lstm.to(self.device)
assert self.version in [1, 2]
if self.version == 1:
self.bos = '<s>'
self.eos = '</s>'
self.max_pad = True
self.moses_tok = False
elif self.version == 2:
self.bos = '<p>'
self.eos = '</p>'
self.max_pad = False
self.moses_tok = True
def learn(self, from_data, test_data=None):
"""
Train and save a model (you can use this to retrain model from data).
:param from_data: DataFrame or DataSource
The data to learn from
:param test_data: DataFrame or DataSource
The data to test accuracy and learn_error from
"""
device, _available_devices = get_devices()
log.info(f'Computing device used: {device}')
# generate the configuration and set the order for the input and output columns
if self._generate_config is True:
self._input_columns = [col for col in from_data if col not in self._output_columns]
self.config = {
'input_features': [{'name': col, 'type': self._type_map(from_data, col)} for col in self._input_columns],
'output_features': [{'name': col, 'type': self._type_map(from_data, col)} for col in self._output_columns]
}
self.config = predictor_config_schema.validate(self.config)
log.info('Automatically generated a configuration')
log.info(self.config)
else:
self._output_columns = [col['name'] for col in self.config['output_features']]
self._input_columns = [col['name'] for col in self.config['input_features']]
if isinstance(from_data, pandas.DataFrame):
train_ds = DataSource(from_data, self.config)
elif isinstance(from_data, DataSource):
train_ds = from_data
else:
raise TypeError(':from_data: must be either DataFrame or DataSource')
nr_subsets = 3 if len(train_ds) > 100 else 1
if test_data is None:
test_ds = train_ds.subset(0.1)
elif isinstance(test_data, pandas.DataFrame):
test_ds = train_ds.make_child(test_data)
elif isinstance(test_data, DataSource):
test_ds = test_data
else:
raise TypeError(':test_data: must be either DataFrame or DataSource')
train_ds.create_subsets(nr_subsets)
test_ds.create_subsets(nr_subsets)
train_ds.train()
test_ds.train()
mixer_class = self.config['mixer']['class']
mixer_kwargs = self.config['mixer']['kwargs']
self._mixer = mixer_class(**mixer_kwargs)
self._mixer.fit(train_ds=train_ds, test_ds=test_ds)
self.train_accuracy = self._mixer.calculate_accuracy(test_ds)
return self
def convert_to_device(self, device_str=None):
if hasattr(self._mixer, 'to') and callable(self._mixer.to):
if device_str is not None:
device = torch.device(device_str)
available_devices = 1
if device_str == 'cuda':
available_devices = torch.cuda.device_count()
else:
device, available_devices = get_devices()
self._mixer.to(device, available_devices)
def __init__(self, model):
""" Img2Vec
:param model: name of the model to use
"""
super(Img2Vec, self).__init__()
self.device, _ = get_devices()
self.model_name = model
self.model = self._get_model()
self.model = self.model.to(self.device)
def convert_to_device(self, device_str=None):
if device_str is not None:
device = torch.device(device_str)
available_devices = 1
if device_str == 'cuda':
available_devices = torch.cuda.device_count()
else:
device, available_devices = get_devices()
self._mixer.to(device, available_devices)
for e in self._mixer.encoders:
if hasattr(self._mixer.encoders[e], 'to'):
self._mixer.encoders[e].to(device, available_devices)
def __init__(self,
encoded_vector_size=4,
train_iters=75000,
stop_on_error=0.8,
learning_rate=0.01,
is_target=False):
self._stop_on_error = stop_on_error
self._learning_rate = learning_rate
self._encoded_vector_size = encoded_vector_size
self._train_iters = train_iters
self._pytorch_wrapper = torch.FloatTensor
self._encoder = None
self._prepared = False
self.device, _ = get_devices()
def __init__(self, model='resnet-18', layer='default', layer_output_size=512):
""" Img2Vec
:param cuda: If set to True, will run forward pass on GPU
:param model: String name of requested model
:param layer: String or Int depending on model. See more docs: https://github.com/christiansafka/img2vec.git
:param layer_output_size: Int depicting the output size of the requested layer
"""
self.device, _ = get_devices()
self.layer_output_size = layer_output_size
self.model_name = model
self.model, self.extraction_layer = self._get_model_and_layer(model, layer)
self.model = self.model.to(self.device)
self.scaler = transforms.Scale((224, 224))
self.normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
self.to_tensor = transforms.ToTensor()
def __init__(self, enabled=True):
self.major = 0 # GPU major version
torch_version = [int(i) for i in torch.__version__.split('.')[:-1]]
if not enabled or not torch.cuda.is_available(
) or torch_version[0] < 1 or torch_version[1] < 6:
self._enabled = False
else:
device, _ = get_devices()
if device.type == 'cuda':
# tensor cores only exist from 7 onwards
# if this is not the case, then AMP is unnecessary overhead
self.major, _ = torch.cuda.get_device_capability(device)
self._enabled = enabled if self.major > 6 else False
else:
self._enabled = False # gpu is available but cpu is forced
self.prev = self._enabled # necessary reference to exit
def __init__(self, encoded_vector_size=128, train_iters=100, stop_on_error=0.01, learning_rate=0.01,
is_target=False, ts_n_dims=1, encoder_class=EncoderRNNNumerical):
super().__init__(is_target)
self.device, _ = get_devices()
self.encoder_class = encoder_class
self._stop_on_error = stop_on_error
self._learning_rate = learning_rate
self._encoded_vector_size = encoded_vector_size
self._transformer_hidden_size = None
self._epochs = train_iters # training epochs
self._pytorch_wrapper = torch.FloatTensor
self._prepared = False
self._is_setup = False
self._max_ts_length = 0
self._sos = 0.0 # start of sequence for decoding
self._eos = 0.0 # end of input sequence -- padding value for batches
self._n_dims = ts_n_dims
self._normalizer = None
self._target_ar_normalizers = []
def predict(self, when_data=None, when=None):
"""
Predict given when conditions.
:param when_data: pandas.DataFrame
:param when: dict
:return: pandas.DataFrame
"""
device, _available_devices = get_devices()
log.info(f'Computing device used: {device}')
if when is not None:
when_dict = {key: [when[key]] for key in when}
when_data = pandas.DataFrame(when_dict)
when_data_ds = DataSource(when_data, self.config, prepare_encoders=False)
when_data_ds.eval()
kwargs = {'include_extra_data': self.config.get('include_extra_data', False)}
return self._mixer.predict(when_data_ds, **kwargs)
def __init__(self, dynamic_parameters, input_size=None, output_size=None, nr_outputs=None, shape=None, selfaware=False, size_parameters={}, pretrained_net=None, deterministic=False):
self.input_size = input_size
self.output_size = output_size
self.nr_outputs = nr_outputs
self.selfaware = selfaware
# How many devices we can train this network on
self.available_devices = 1
self.max_variance = None
self.device, _ = get_devices()
if deterministic:
'''
Seed that always has the same value on the same dataset plus setting the bellow CUDA options
In order to make sure pytorch randomly generate number will be the same every time
when training on the same dataset
'''
torch.manual_seed(66)
if 'cuda' in str(self.device):
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
self.available_devices = torch.cuda.device_count()
else:
self.available_devices = 1
self.dynamic_parameters = dynamic_parameters
"""
Here we define the basic building blocks of our model,
in forward we define how we put it all together along with an input
"""
super(DefaultNet, self).__init__()
if shape is None and pretrained_net is None:
shape = [self.input_size, max([self.input_size*2,self.output_size*2,400]), self.output_size]
if pretrained_net is None:
logging.info(f'Building network of shape: {shape}')
rectifier = torch.nn.SELU #alternative: torch.nn.ReLU
layers = []
for ind in range(len(shape) - 1):
linear_function = PLinear if CONFIG.USE_PROBABILISTIC_LINEAR else torch.nn.Linear
layers.append(linear_function(shape[ind],shape[ind+1]))
if ind < len(shape) - 2:
layers.append(rectifier())
self.net = torch.nn.Sequential(*layers)
else:
self.net = pretrained_net
for layer in self.net:
if isinstance(layer, torch.nn.Linear):
if self.input_size is None:
self.input_size = layer.in_features
self.output_size = layer.out_features
if self.selfaware:
awareness_net_shape = [(self.input_size + self.output_size), max([int((self.input_size + self.output_size) * 1.5), 300]), self.nr_outputs]
awareness_layers = []
for ind in range(len(awareness_net_shape) - 1):
rectifier = torch.nn.SELU #alternative: torch.nn.ReLU
awareness_layers.append(torch.nn.Linear(awareness_net_shape[ind], awareness_net_shape[ind + 1]))
if ind < len(awareness_net_shape) - 2:
awareness_layers.append(rectifier())
self.awareness_net = torch.nn.Sequential(*awareness_layers)
if deterministic and pretrained_net is None: # set initial weights based on a specific distribution if we have deterministic enabled
# lambda function so that we can do this for either awareness layer or the internal layers of net
def reset_layer_params(layer):
if isinstance(layer, torch.nn.Linear):
torch.nn.init.normal_(layer.weight, mean=0., std=1 / math.sqrt(layer.out_features))
torch.nn.init.normal_(layer.bias, mean=0., std=0.1)
elif isinstance(layer, PLinear):
torch.nn.init.normal_(layer.mean, mean=0., std=1 / math.sqrt(layer.out_features))
torch.nn.init.normal_(layer.bias, mean=0., std=0.1)
if self.selfaware:
for layer in self.awareness_net:
reset_layer_params(layer)
for layer in self.net:
reset_layer_params(layer)
self.net = self.net.to(self.device)
if self.available_devices > 1:
self._foward_net = torch.nn.DataParallel(self.net)
else:
self._foward_net = self.net
if self.selfaware:
self.awareness_net = self.awareness_net.to(self.device)
if self.available_devices > 1:
self._foward_awareness_net = torch.nn.DataParallel(self.awareness_net)
else:
self._foward_awareness_net = self.awareness_net
def __init__(self,
dynamic_parameters,
input_size=None,
output_size=None,
nr_outputs=None,
shape=None,
dropout=None,
pretrained_net=None):
self.input_size = input_size
self.output_size = output_size
self.nr_outputs = nr_outputs
self.max_variance = None
# How many devices we can train this network on
self.available_devices = 1
self.device, _ = get_devices()
self.dynamic_parameters = dynamic_parameters
"""
Here we define the basic building blocks of our model,
in forward we define how we put it all together along with an input
"""
super(DefaultNet, self).__init__()
if shape is None and pretrained_net is None:
shape = [
self.input_size,
max([self.input_size * 2, self.output_size * 2, 400]),
self.output_size
]
if pretrained_net is None:
log.info(f'Building network of shape: {shape}')
rectifier = torch.nn.SELU #alternative: torch.nn.ReLU
layers = []
for ind in range(len(shape) - 1):
if (dropout is not None) and (0 < ind < len(shape)):
layers.append(torch.nn.Dropout(p=dropout))
linear_function = PLinear if CONFIG.USE_PROBABILISTIC_LINEAR else torch.nn.Linear
layers.append(linear_function(shape[ind], shape[ind + 1]))
if ind < len(shape) - 2:
layers.append(rectifier())
self.net = torch.nn.Sequential(*layers)
else:
self.net = pretrained_net
for layer in self.net:
if isinstance(layer, torch.nn.Linear):
if self.input_size is None:
self.input_size = layer.in_features
self.output_size = layer.out_features
self.net = self.net.to(self.device)
if 'cuda' in str(self.device):
self.available_devices = torch.cuda.device_count()
else:
self.available_devices = 1
if self.available_devices > 1:
self._foward_net = torch.nn.DataParallel(self.net)
else:
self._foward_net = self.net
import torchvision
import numpy as np
import torch
import sys
if sys.version_info < (3, 6):
sys.exit('Sorry, For Lightwood Python < 3.6 is not supported')
from lightwood.__about__ import __package_name__ as name, __version__
from lightwood.api.predictor import Predictor
import lightwood.model_building
import lightwood.constants.lightwood as CONST
from lightwood.helpers.device import get_devices
COLUMN_DATA_TYPES = CONST.COLUMN_DATA_TYPES
# fix random seed for reproducibility
seed = 42
np.random.seed(seed)
torch.manual_seed(seed)
if 'cuda' in str(get_devices()[0]):
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False