def __init__( self, input_features: Dict[str, "InputFeature"], config: ComparatorCombinerConfig = None, **kwargs, ): super().__init__(input_features) self.name = "ComparatorCombiner" logger.debug(f"Entering {self.name}") self.entity_1 = config.entity_1 self.entity_2 = config.entity_2 self.required_inputs = set(config.entity_1 + config.entity_2) self.fc_size = config.fc_size self.fc_stack = None # todo future: this may be redundant, check fc_layers = config.fc_layers if fc_layers is None and config.num_fc_layers is not None: fc_layers = [] for i in range(config.num_fc_layers): fc_layers.append({"fc_size": config.fc_size}) if fc_layers is not None: logger.debug("Setting up FCStack") self.e1_fc_stack = FCStack( self.get_entity_shape(config.entity_1)[-1], layers=fc_layers, num_layers=config.num_fc_layers, default_fc_size=config.fc_size, default_use_bias=config.use_bias, default_weights_initializer=config.weights_initializer, default_bias_initializer=config.bias_initializer, default_norm=config.norm, default_norm_params=config.norm_params, default_activation=config.activation, default_dropout=config.dropout, ) self.e2_fc_stack = FCStack( self.get_entity_shape(config.entity_2)[-1], layers=fc_layers, num_layers=config.num_fc_layers, default_fc_size=config.fc_size, default_use_bias=config.use_bias, default_weights_initializer=config.weights_initializer, default_bias_initializer=config.bias_initializer, default_norm=config.norm, default_norm_params=config.norm_params, default_activation=config.activation, default_dropout=config.dropout, ) self.last_fc_layer_fc_size = fc_layers[-1]["fc_size"] # todo: set initializer and regularization self.register_buffer( "bilinear_weights", torch.randn([self.last_fc_layer_fc_size, self.last_fc_layer_fc_size], dtype=torch.float32), )
def test_fc_stack(first_layer_input_size: Optional[int], layers: Optional[List], num_layers: Optional[int]): if first_layer_input_size is None: first_layer_input_size = layers[0]["input_size"] fc_stack = FCStack(first_layer_input_size=first_layer_input_size, layers=layers, num_layers=num_layers).to(DEVICE) input_tensor = torch.randn(BATCH_SIZE, first_layer_input_size, device=DEVICE) output_tensor = fc_stack(input_tensor) assert output_tensor.shape[1:] == fc_stack.output_shape
def __init__( self, vocab, representation='dense', embedding_size=50, embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=False, fc_layers=None, num_fc_layers=0, fc_size=10, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout_rate=0.0, reduce_output='sum', **kwargs): super(SetSparseEncoder, self).__init__() logger.debug(' {}'.format(self.name)) logger.debug(' EmbedSparse') self.embed_sparse = EmbedSparse( vocab, embedding_size, representation=representation, embeddings_trainable=embeddings_trainable, pretrained_embeddings=pretrained_embeddings, embeddings_on_cpu=embeddings_on_cpu, dropout_rate=dropout_rate, initializer=weights_initializer, regularizer=weights_regularizer, reduce_output=reduce_output, ) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout_rate=dropout_rate, )
def __init__(self, input_features: Dict[str, "InputFeature"] = None, config: ConcatCombinerConfig = None, **kwargs): super().__init__(input_features) self.name = "ConcatCombiner" logger.debug(f" {self.name}") self.flatten_inputs = config.flatten_inputs self.fc_stack = None # todo future: this may be redundant, check fc_layers = config.fc_layers if fc_layers is None and config.num_fc_layers is not None: fc_layers = [] for i in range(config.num_fc_layers): fc_layers.append({"fc_size": config.fc_size}) self.fc_layers = fc_layers if self.fc_layers is not None: logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=self.concatenated_shape[-1], layers=config.fc_layers, num_layers=config.num_fc_layers, default_fc_size=config.fc_size, default_use_bias=config.use_bias, default_weights_initializer=config.weights_initializer, default_bias_initializer=config.bias_initializer, default_norm=config.norm, default_norm_params=config.norm_params, default_activation=config.activation, default_dropout=config.dropout, residual=config.residual, ) if input_features and len(input_features) == 1 and self.fc_layers is None: self.supports_masking = True
def __init__( self, input_size, layers=None, num_layers=1, fc_size=256, use_bias=True, weights_initializer="xavier_uniform", bias_initializer="zeros", norm=None, norm_params=None, activation="relu", dropout=0, **kwargs, ): super().__init__() logger.debug(f" {self.name}") self.input_size = input_size logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=input_size, layers=layers, num_layers=num_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__(self, input_features: Dict[str, "InputFeature"] = None, config: TransformerCombinerConfig = None, **kwargs): super().__init__(input_features) self.name = "TransformerCombiner" logger.debug(f" {self.name}") self.reduce_output = config.reduce_output self.reduce_sequence = SequenceReducer( reduce_mode=config.reduce_output, max_sequence_length=len(self.input_features), encoding_size=config.hidden_size, ) if self.reduce_output is None: self.supports_masking = True # sequence size for Transformer layer is number of input features self.sequence_size = len(self.input_features) logger.debug(" Projectors") self.projectors = ModuleList( # regardless of rank-2 or rank-3 input, torch.prod() calculates size # after flattening the encoder output tensor [ Linear( torch.prod( torch.Tensor([*input_features[inp].output_shape ])).type(torch.int32), config.hidden_size) for inp in input_features ]) logger.debug(" TransformerStack") self.transformer_stack = TransformerStack( input_size=config.hidden_size, sequence_size=self.sequence_size, hidden_size=config.hidden_size, num_heads=config.num_heads, output_size=config.transformer_output_size, num_layers=config.num_layers, dropout=config.dropout, ) if self.reduce_output is not None: logger.debug(" FCStack") self.fc_stack = FCStack( self.transformer_stack.output_shape[-1], layers=config.fc_layers, num_layers=config.num_fc_layers, default_output_size=config.output_size, default_use_bias=config.use_bias, default_weights_initializer=config.weights_initializer, default_bias_initializer=config.bias_initializer, default_norm=config.norm, default_norm_params=config.norm_params, default_activation=config.fc_activation, default_dropout=config.fc_dropout, fc_residual=config.fc_residual, )
def __init__( self, input_features=None, fc_layers=None, num_fc_layers=None, fc_size=256, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout=0, flatten_inputs=False, residual=False, **kwargs ): super().__init__() logger.debug(' {}'.format(self.name)) self.flatten_inputs = flatten_inputs self.fc_stack = None # todo future: this may be redundant, check if fc_layers is None and \ num_fc_layers is not None: fc_layers = [] for i in range(num_fc_layers): fc_layers.append({'fc_size': fc_size}) if fc_layers is not None: logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, residual=residual, ) if input_features and len(input_features) == 1 and fc_layers is None: self.supports_masking = True
def __init__(self, feature, *args, **kwargs): super().__init__(*args, feature=feature, **kwargs) self.reduce_input = None self.reduce_dependencies = None self.dependencies = [] self.fc_layers = None self.num_fc_layers = 0 self.fc_size = 256 self.use_bias = True self.weights_initializer = 'glorot_uniform' self.bias_initializer = 'zeros' self.weights_regularizer = None self.bias_regularizer = None self.activity_regularizer = None # self.weights_constraint=None # self.bias_constraint=None self.norm = None self.norm_params = None self.activation = 'relu' self.dropout = 0 self.overwrite_defaults(feature) logger.debug(' output feature fully connected layers') logger.debug(' FCStack') self.fc_stack = FCStack( layers=self.fc_layers, num_layers=self.num_fc_layers, default_fc_size=self.fc_size, default_use_bias=self.use_bias, default_weights_initializer=self.weights_initializer, default_bias_initializer=self.bias_initializer, default_weights_regularizer=self.weights_regularizer, default_bias_regularizer=self.bias_regularizer, default_activity_regularizer=self.activity_regularizer, # default_weights_constraint=self.weights_constraint, # default_bias_constraint=self.bias_constraint, default_norm=self.norm, default_norm_params=self.norm_params, default_activation=self.activation, default_dropout=self.dropout, ) # set up two sequence reducers, one for inputs and other for dependencies self.reduce_sequence_input = SequenceReducer( reduce_mode=self.reduce_input ) if self.dependencies: self.dependency_reducers = {} for dependency in self.dependencies: self.dependency_reducers[dependency] = SequenceReducer( reduce_mode=self.reduce_dependencies )
def __init__( self, vocab: List[str], representation: str = "dense", embedding_size: int = 50, embeddings_trainable: bool = True, pretrained_embeddings: Optional[str] = None, embeddings_on_cpu: bool = False, fc_layers=None, num_fc_layers: int = 0, output_size: int = 10, use_bias: bool = True, weights_initializer: str = "xavier_uniform", bias_initializer: str = "zeros", norm: Optional[str] = None, norm_params: Optional[Dict[str, Any]] = None, activation: str = "relu", dropout: float = 0.0, **kwargs, ): super().__init__() logger.debug(f" {self.name}") self.vocab_size = len(vocab) logger.debug(" Embed") self.embed = EmbedSet( vocab, embedding_size, representation=representation, embeddings_trainable=embeddings_trainable, pretrained_embeddings=pretrained_embeddings, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" FCStack") # TODO(shreya): Make sure this is updated when FCStack is updated self.fc_stack = FCStack( first_layer_input_size=self.embed.output_shape[-1], layers=fc_layers, num_layers=num_fc_layers, default_output_size=output_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__(self, input_features: Dict[str, "InputFeature"] = None, config: ProjectAggregateCombinerConfig = None, **kwargs): super().__init__(input_features) self.name = "ProjectAggregateCombiner" logger.debug(f" {self.name}") logger.debug(" Projectors") self.projectors = ModuleList( # regardless of rank-2 or rank-3 input, torch.prod() calculates size # after flattening the encoder output tensor [ Linear( torch.prod( torch.Tensor([*input_features[inp].output_shape ])).type(torch.int32), config.projection_size, ) for inp in input_features ]) self.fc_stack = None # todo future: this may be redundant, check fc_layers = config.fc_layers if fc_layers is None and config.num_fc_layers is not None: fc_layers = [] for i in range(config.num_fc_layers): fc_layers.append({"output_size": config.output_size}) self.fc_layers = fc_layers if self.fc_layers is not None: logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=config.projection_size, layers=config.fc_layers, num_layers=config.num_fc_layers, default_output_size=config.output_size, default_use_bias=config.use_bias, default_weights_initializer=config.weights_initializer, default_bias_initializer=config.bias_initializer, default_norm=config.norm, default_norm_params=config.norm_params, default_activation=config.activation, default_dropout=config.dropout, residual=config.residual, ) if input_features and len( input_features) == 1 and self.fc_layers is None: self.supports_masking = True
def test_fc_stack( first_layer_input_size: Optional[int], layers: Optional[List], num_layers: Optional[int], ): set_random_seed(RANDOM_SEED) fc_stack = FCStack(first_layer_input_size=first_layer_input_size, layers=layers, num_layers=num_layers).to(DEVICE) input_tensor = torch.randn(BATCH_SIZE, first_layer_input_size, device=DEVICE) output_tensor = fc_stack(input_tensor) assert output_tensor.shape[1:] == fc_stack.output_shape
def test_fc_stack_input_size_mismatch_fails(): first_layer_input_size = 10 layers = [{"input_size": 2, "output_size": 4}, {"output_size": 8}] fc_stack = FCStack( first_layer_input_size=first_layer_input_size, layers=layers, ).to(DEVICE) input_tensor = torch.randn(BATCH_SIZE, first_layer_input_size, device=DEVICE) with pytest.raises(RuntimeError): fc_stack(input_tensor)
def __init__(self, feature, *args, **kwargs): super().__init__(*args, feature=feature, **kwargs) self.loss = None self.train_loss_function = None self.eval_loss_function = None self.metric_functions = OrderedDict() self.reduce_input = None self.reduce_dependencies = None self.dependencies = [] self.fc_layers = None self.num_fc_layers = 0 self.fc_size = 256 self.use_bias = True self.weights_initializer = 'glorot_uniform' self.bias_initializer = 'zeros' self.weights_regularizer = None self.bias_regularizer = None self.activity_regularizer = None # self.weights_constraint=None # self.bias_constraint=None self.norm = None self.norm_params = None self.activation = 'relu' self.dropout = 0 self.overwrite_defaults(feature) logger.debug(' output feature fully connected layers') logger.debug(' FCStack') self.fc_stack = FCStack( layers=self.fc_layers, num_layers=self.num_fc_layers, default_fc_size=self.fc_size, default_use_bias=self.use_bias, default_weights_initializer=self.weights_initializer, default_bias_initializer=self.bias_initializer, default_weights_regularizer=self.weights_regularizer, default_bias_regularizer=self.bias_regularizer, default_activity_regularizer=self.activity_regularizer, # default_weights_constraint=self.weights_constraint, # default_bias_constraint=self.bias_constraint, default_norm=self.norm, default_norm_params=self.norm_params, default_activation=self.activation, default_dropout=self.dropout, )
def __init__(self, feature, *args, **kwargs): super().__init__(*args, feature=feature, **kwargs) self.reduce_input = None self.reduce_dependencies = None self.dependencies = [] self.fc_layers = None self.num_fc_layers = 0 self.fc_size = 256 self.use_bias = True self.weights_initializer = "xavier_uniform" self.bias_initializer = "zeros" self.norm = None self.norm_params = None self.activation = "relu" self.dropout = 0 self.input_size = None self.overwrite_defaults(feature) logger.debug(" output feature fully connected layers") logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=self.input_size, layers=self.fc_layers, num_layers=self.num_fc_layers, default_fc_size=self.fc_size, default_use_bias=self.use_bias, default_weights_initializer=self.weights_initializer, default_bias_initializer=self.bias_initializer, default_norm=self.norm, default_norm_params=self.norm_params, default_activation=self.activation, default_dropout=self.dropout, ) # set up two sequence reducers, one for inputs and other for dependencies self.reduce_sequence_input = SequenceReducer( reduce_mode=self.reduce_input) if self.dependencies: self.dependency_reducers = torch.nn.ModuleDict() # todo: re-evaluate need for separate handling of `attention` reducer # currently this code does not support `attention` for dependency in self.dependencies: self.dependency_reducers[dependency] = SequenceReducer( reduce_mode=self.reduce_dependencies)
def test_fc_stack_no_layers_behaves_like_passthrough(): first_layer_input_size = 10 layers = None num_layers = 0 output_size = 15 fc_stack = FCStack( first_layer_input_size=first_layer_input_size, layers=layers, num_layers=num_layers, default_output_size=output_size, ).to(DEVICE) input_tensor = torch.randn(BATCH_SIZE, first_layer_input_size, device=DEVICE) output_tensor = fc_stack(input_tensor) assert list(output_tensor.shape[1:]) == [first_layer_input_size] assert output_tensor.shape[1:] == fc_stack.output_shape assert np.all(np.isclose(input_tensor, output_tensor))
def __init__( self, layers=None, num_layers=1, fc_size=256, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout_rate=0, **kwargs): super(DenseEncoder, self).__init__() logger.debug(' {}'.format(self.name)) logger.debug(' FCStack') self.fc_stack = FCStack( layers=layers, num_layers=num_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout_rate=dropout_rate, )
def __init__( self, fc_layers=None, num_fc_layers=0, fc_size=10, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout=0, **kwargs): """ :param fc_layers: list of dictionaries containing the parameters of all the fully connected layers :type fc_layers: List :param num_fc_layers: Number of stacked fully connected layers :type num_fc_layers: Integer :param fc_size: Size of each layer :type fc_size: Integer :param use_bias: bool determines where to use a bias vector :type use_bias: bool :param weights_initializer: Initializer for the weights (aka kernel) matrix :type weights_initializer: string :param bias_initializer: Initializer for the bias vector :type bias_initializer: string :param weights_regularizer: regularizer applied to the weights (kernal) matrix :type weights_regularizer: string :param bias_regularizer: reguralizer function applied to biase vector. :type bias_regularizer: string :param activity_regularizer: Regularizer applied to the output of the layer (activation) :type activity_regularizer: string :param norm: type of normalization to use 'batch' or 'layer' :type norm: string, default None :param norm_params: parameters to pass to normalization function :type norm_params: dictionary :param activation: Activation function to use. :type activation: string :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: float """ super().__init__() logger.debug(' {}'.format(self.name)) logger.debug(' year FCStack') self.year_fc = FCStack( num_layers=1, default_fc_size=1, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=None, default_norm_params=None, default_activation=None, default_dropout=dropout, ) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, embedding_size: int = 10, embeddings_on_cpu: bool = False, fc_layers: Optional[List[Dict]] = None, num_fc_layers: int = 0, output_size: int = 10, use_bias: bool = True, weights_initializer: str = "xavier_uniform", bias_initializer: str = "zeros", norm: Optional[str] = None, norm_params: Optional[Dict] = None, activation: str = "relu", dropout: float = 0, **kwargs, ): """ :param embedding_size: The maximum embedding size, the actual size will be `min(vocabulary_size, embedding_size)` for `dense` representations and exactly `vocabulary_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param fc_layers: list of dictionaries containing the parameters of all the fully connected layers. :type fc_layers: List :param num_fc_layers: Number of stacked fully connected layers. :type num_fc_layers: Integer :param output_size: Size of each layer. :type output_size: Integer :param use_bias: bool determines where to use a bias vector. :type use_bias: bool :param weights_initializer: Initializer for the weights (aka kernel) matrix. :type weights_initializer: string :param bias_initializer: Initializer for the bias vector. :type bias_initializer: string :param norm: type of normalization to use 'batch' or 'layer'. :type norm: string, default None :param norm_params: parameters to pass to normalization function. :type norm_params: dictionary :param activation: Activation function to use. :type activation: string :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: float """ super().__init__() logger.debug(f" {self.name}") logger.debug(" year FCStack") self.year_fc = FCStack( first_layer_input_size=1, num_layers=1, default_output_size=1, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=None, default_norm_params=None, default_activation=None, default_dropout=dropout, ) logger.debug(" month Embed") self.embed_month = Embed( [str(i) for i in range(12)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" day Embed") self.embed_day = Embed( [str(i) for i in range(31)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" weekday Embed") self.embed_weekday = Embed( [str(i) for i in range(7)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" yearday Embed") self.embed_yearday = Embed( [str(i) for i in range(366)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" hour Embed") self.embed_hour = Embed( [str(i) for i in range(24)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" minute Embed") self.embed_minute = Embed( [str(i) for i in range(60)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" second Embed") self.embed_second = Embed( [str(i) for i in range(60)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) # Summed sizes of all of the embeddings. fc_layer_input_size = ( self.year_fc.output_shape[0] + self.embed_month.output_shape[0] + self.embed_day.output_shape[0] + self.embed_weekday.output_shape[0] + self.embed_yearday.output_shape[0] + self.embed_hour.output_shape[0] + self.embed_minute.output_shape[0] + self.embed_second.output_shape[0] + 1 # for periodic_second_of_day. ) logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=fc_layer_input_size, layers=fc_layers, num_layers=num_fc_layers, default_output_size=output_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, fc_layers: Optional[List[FCStack]] = None, num_fc_layers: int = 1, output_size: int = 10, use_bias: bool = True, weights_initializer: str = "xavier_uniform", bias_initializer: str = "zeros", norm: Optional[str] = None, norm_params: Optional[Dict] = None, activation: str = "relu", dropout: int = 0, **kwargs, ): """ :param fc_layers: list of dictionaries containing the parameters of all the fully connected layers. :type fc_layers: List :param num_fc_layers: Number of stacked fully connected layers. :type num_fc_layers: Integer :param output_size: Size of each layer. :type output_size: Integer :param use_bias: bool determines where to use a bias vector. :type use_bias: bool :param weights_initializer: Initializer for the weights (aka kernel) matrix. :type weights_initializer: string :param bias_initializer: Initializer for the bias vector. :type bias_initializer: string :param norm: type of normalization to use 'batch' or 'layer'. :type norm: string, default None :param norm_params: parameters to pass to normalization function. :type norm_params: dictionary :param activation: Activation function to use. :type activation: string :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: float """ super().__init__() logger.debug(f" {self.name}") logger.debug(" year FCStack") self.year_fc = FCStack( first_layer_input_size=1, num_layers=1, default_output_size=1, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=None, default_norm_params=None, default_activation=None, default_dropout=dropout, ) # Summed sizes of all of the embeddings. # Additional 8 for periodic_[month, day, ..., second_of_day]. fc_layer_input_size = self.year_fc.output_shape[0] + 8 logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=fc_layer_input_size, layers=fc_layers, num_layers=num_fc_layers, default_output_size=output_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, embedding_size=10, embeddings_on_cpu=False, fc_layers=None, num_fc_layers=0, fc_size=10, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout=0, reduce_output='sum', **kwargs): """ :param embedding_size: it is the maximum embedding size, the actual size will be `min(vocaularyb_size, embedding_size)` for `dense` representations and exacly `vocaularyb_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embedings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memroy and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: Boolean :param initializer: the initializer to use. If `None`, the default initialized of each variable is used (`glorot_uniform` in most cases). Options are: `constant`, `identity`, `zeros`, `ones`, `orthogonal`, `normal`, `uniform`, `truncated_normal`, `variance_scaling`, `glorot_normal`, `glorot_uniform`, `xavier_normal`, `xavier_uniform`, `he_normal`, `he_uniform`, `lecun_normal`, `lecun_uniform`. Alternatively it is possible to specify a dictionary with a key `type` that identifies the type of initialzier and other keys for its parameters, e.g. `{type: normal, mean: 0, stddev: 0}`. To know the parameters of each initializer, please refer to TensorFlow's documentation. :type initializer: str :param regularize: if `True` the embedding wieghts are added to the set of weights that get reularized by a regularization loss (if the `regularization_lambda` in `training` is greater than 0). :type regularize: Boolean """ super(H3Embed, self).__init__() logger.debug(' {}'.format(self.name)) self.embedding_size = embedding_size self.reduce_output = reduce_output self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output) logger.debug(' mode Embed') self.embed_mode = Embed([str(i) for i in range(3)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' edge Embed') self.embed_edge = Embed([str(i) for i in range(7)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' resolution Embed') self.embed_resolution = Embed( [str(i) for i in range(16)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' base cell Embed') self.embed_base_cell = Embed([str(i) for i in range(122)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' cells Embed') self.embed_cells = Embed([str(i) for i in range(8)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, height: int, width: int, conv_layers: Optional[List[Dict]] = None, num_conv_layers: Optional[int] = None, num_channels: int = None, out_channels: int = 32, kernel_size: Union[int, Tuple[int]] = 3, stride: Union[int, Tuple[int]] = 1, padding: Union[int, Tuple[int], str] = "valid", dilation: Union[int, Tuple[int]] = 1, conv_use_bias: bool = True, padding_mode: str = "zeros", conv_norm: Optional[str] = None, conv_norm_params: Optional[Dict[str, Any]] = None, conv_activation: str = "relu", conv_dropout: int = 0, pool_function: str = "max", pool_kernel_size: Union[int, Tuple[int]] = 2, pool_stride: Union[int, Tuple[int]] = None, pool_padding: Union[int, Tuple[int]] = 0, pool_dilation: Union[int, Tuple[int]] = 1, groups: int = 1, fc_layers: Optional[List[Dict]] = None, num_fc_layers: Optional[int] = 1, output_size: int = 128, fc_use_bias: bool = True, fc_weights_initializer: str = "xavier_uniform", fc_bias_initializer: str = "zeros", fc_norm: Optional[str] = None, fc_norm_params: Optional[Dict[str, Any]] = None, fc_activation: str = "relu", fc_dropout: float = 0, **kwargs, ): super().__init__() logger.debug(f" {self.name}") # map parameter input feature config names to internal names img_height = height img_width = width first_in_channels = num_channels self._input_shape = (first_in_channels, img_height, img_width) if first_in_channels is None: raise ValueError("first_in_channels must not be None.") logger.debug(" Conv2DStack") self.conv_stack_2d = Conv2DStack( img_height=img_height, img_width=img_width, layers=conv_layers, num_layers=num_conv_layers, first_in_channels=first_in_channels, default_out_channels=out_channels, default_kernel_size=kernel_size, default_stride=stride, default_padding=padding, default_dilation=dilation, default_groups=groups, default_use_bias=conv_use_bias, default_padding_mode=padding_mode, default_norm=conv_norm, default_norm_params=conv_norm_params, default_activation=conv_activation, default_dropout=conv_dropout, default_pool_function=pool_function, default_pool_kernel_size=pool_kernel_size, default_pool_stride=pool_stride, default_pool_padding=pool_padding, default_pool_dilation=pool_dilation, ) out_channels, img_height, img_width = self.conv_stack_2d.output_shape first_fc_layer_input_size = out_channels * img_height * img_width self.flatten = torch.nn.Flatten() logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=first_fc_layer_input_size, layers=fc_layers, num_layers=num_fc_layers, default_output_size=output_size, default_use_bias=fc_use_bias, default_weights_initializer=fc_weights_initializer, default_bias_initializer=fc_bias_initializer, default_norm=fc_norm, default_norm_params=fc_norm_params, default_activation=fc_activation, default_dropout=fc_dropout, )
def __init__( self, input_features: Dict[str, "InputFeature"] = None, config: TabTransformerCombinerConfig = None, **kwargs ): super().__init__(input_features) self.name = "TabTransformerCombiner" logger.debug(f"Initializing {self.name}") if config.reduce_output is None: raise ValueError("TabTransformer requires the `reduce_output` " "parameter") self.reduce_output = config.reduce_output self.reduce_sequence = SequenceReducer(reduce_mode=config.reduce_output) self.supports_masking = True self.embed_input_feature_name = config.embed_input_feature_name if self.embed_input_feature_name: vocab = [ i_f for i_f in input_features if input_features[i_f].type != NUMERICAL or input_features[i_f].type != BINARY ] if self.embed_input_feature_name == "add": self.embed_i_f_name_layer = Embed(vocab, config.hidden_size, force_embedding_size=True) projector_size = config.hidden_size elif isinstance(self.embed_input_feature_name, int): if self.embed_input_feature_name > config.hidden_size: raise ValueError( "TabTransformer parameter " "`embed_input_feature_name` " "specified integer value ({}) " "needs to be smaller than " "`hidden_size` ({}).".format(self.embed_input_feature_name, config.hidden_size) ) self.embed_i_f_name_layer = Embed( vocab, self.embed_input_feature_name, force_embedding_size=True, ) projector_size = config.hidden_size - self.embed_input_feature_name else: raise ValueError( "TabTransformer parameter " "`embed_input_feature_name` " "should be either None, an integer or `add`, " "the current value is " "{}".format(self.embed_input_feature_name) ) else: projector_size = config.hidden_size logger.debug(" Projectors") self.unembeddable_features = [] self.embeddable_features = [] for i_f in input_features: if input_features[i_f].type in {NUMERICAL, BINARY}: self.unembeddable_features.append(input_features[i_f].name) else: self.embeddable_features.append(input_features[i_f].name) self.projectors = ModuleList() for i_f in self.embeddable_features: flatten_size = self.get_flatten_size(input_features[i_f].output_shape) self.projectors.append(Linear(flatten_size[0], projector_size)) # input to layer_norm are the encoder outputs for unembeddable features, # which are numerical or binary features. These should be 2-dim # tensors. Size should be concatenation of these tensors. concatenated_unembeddable_encoders_size = 0 for i_f in self.unembeddable_features: concatenated_unembeddable_encoders_size += input_features[i_f].output_shape[0] self.layer_norm = torch.nn.LayerNorm(concatenated_unembeddable_encoders_size) logger.debug(" TransformerStack") self.transformer_stack = TransformerStack( input_size=config.hidden_size, sequence_size=len(self.embeddable_features), hidden_size=config.hidden_size, # todo: can we just use projector_size? # hidden_size, num_heads=config.num_heads, fc_size=config.transformer_fc_size, num_layers=config.num_layers, dropout=config.dropout, ) logger.debug(" FCStack") transformer_hidden_size = self.transformer_stack.layers[-1].output_shape[-1] # determine input size to fully connected layer based on reducer if config.reduce_output == "concat": num_embeddable_features = len(self.embeddable_features) fc_input_size = num_embeddable_features * transformer_hidden_size else: fc_input_size = transformer_hidden_size if len(self.embeddable_features) > 0 else 0 self.fc_stack = FCStack( fc_input_size + concatenated_unembeddable_encoders_size, layers=config.fc_layers, num_layers=config.num_fc_layers, default_fc_size=config.fc_size, default_use_bias=config.use_bias, default_weights_initializer=config.weights_initializer, default_bias_initializer=config.bias_initializer, default_norm=config.norm, default_norm_params=config.norm_params, default_activation=config.fc_activation, default_dropout=config.fc_dropout, fc_residual=config.fc_residual, ) # Create empty tensor of shape [1, 0] to use as hidden in case there are no category or numeric/binary features. self.register_buffer("empty_hidden", torch.empty([1, 0])) self.register_buffer("embeddable_features_indices", torch.arange(0, len(self.embeddable_features)))
def __init__( self, entity_1: List[str], entity_2: List[str], # fc_layers=None, num_fc_layers=1, fc_size=256, use_bias=True, weights_initializer="glorot_uniform", bias_initializer="zeros", weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation="relu", dropout=0, **kwargs, ): super().__init__() logger.debug(" {}".format(self.name)) self.fc_stack = None # todo future: this may be redundant, check # if fc_layers is None and num_fc_layers is not None: fc_layers = [] for i in range(num_fc_layers): fc_layers.append({"fc_size": fc_size}) if fc_layers is not None: logger.debug(" FCStack") self.e1_fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, ) self.e2_fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, ) # todo: this should actually be the size of the last fc layer, # not just fc_size # todo: set initializer and regularization self.bilinear_weights = tf.random.normal([fc_size, fc_size], dtype=tf.float32) self.entity_1 = entity_1 self.entity_2 = entity_2 self.required_inputs = set(entity_1 + entity_2) self.fc_size = fc_size
def __init__( self, input_features=None, embed_input_feature_name=None, # None or embedding size or "add" num_layers=1, hidden_size=256, num_heads=8, transformer_fc_size=256, dropout=0.1, fc_layers=None, num_fc_layers=0, fc_size=256, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, fc_activation='relu', fc_dropout=0, fc_residual=False, reduce_output='concat', **kwargs ): super().__init__() logger.debug(' {}'.format(self.name)) if reduce_output is None: raise ValueError("TabTransformer requires the `resude_output` " "parametr") self.reduce_output = reduce_output self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output) self.supports_masking = True self.layer_norm = LayerNormalization() self.embed_input_feature_name = embed_input_feature_name if self.embed_input_feature_name: vocab = [i_f for i_f in input_features if i_f[TYPE] != NUMERICAL or i_f[TYPE] != BINARY] if self.embed_input_feature_name == 'add': self.embed_i_f_name_layer = Embed(vocab, hidden_size, force_embedding_size=True) projector_size = hidden_size elif isinstance(self.embed_input_feature_name, int): if self.embed_input_feature_name > hidden_size: raise ValueError( "TabTransformer parameter " "`embed_input_feature_name` " "specified integer value ({}) " "needs to be smaller than " "`hidden_size` ({}).".format( self.embed_input_feature_name, hidden_size )) self.embed_i_f_name_layer = Embed( vocab, self.embed_input_feature_name, force_embedding_size=True, ) projector_size = hidden_size - self.embed_input_feature_name else: raise ValueError("TabTransformer parameter " "`embed_input_feature_name` " "should be either None, an integer or `add`, " "the current value is " "{}".format(self.embed_input_feature_name)) else: projector_size = hidden_size logger.debug(' Projectors') self.projectors = [Dense(projector_size) for i_f in input_features if i_f[TYPE] != NUMERICAL and i_f[TYPE] != BINARY] self.skip_features = [i_f[NAME] for i_f in input_features if i_f[TYPE] == NUMERICAL or i_f[TYPE] == BINARY] logger.debug(' TransformerStack') self.transformer_stack = TransformerStack( hidden_size=hidden_size, num_heads=num_heads, fc_size=transformer_fc_size, num_layers=num_layers, dropout=dropout ) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=fc_activation, default_dropout=fc_dropout, fc_residual=fc_residual, )
def __init__(self, feature: Dict[str, Any], other_output_features: Dict[str, "OutputFeature"], *args, **kwargs): """Defines defaults, overwrites them based on the feature dictionary, and sets up dependencies. Any output feature can depend on one or more other output features. The `other_output_features` input dictionary should contain entries for any dependent output features, which is accomplished by constructing output features in topographically sorted order. Attributes of any dependent output features are used to properly initialize this feature's sizes. """ super().__init__(*args, feature=feature, **kwargs) self.reduce_input = None self.reduce_dependencies = None # List of feature names that this output feature is dependent on. self.dependencies = [] self.fc_layers = None self.num_fc_layers = 0 self.output_size = 256 self.use_bias = True self.weights_initializer = "xavier_uniform" self.bias_initializer = "zeros" self.norm = None self.norm_params = None self.activation = "relu" self.dropout = 0 self.input_size = None self.overwrite_defaults(feature) logger.debug(" output feature fully connected layers") logger.debug(" FCStack") self.input_size = get_input_size_with_dependencies( self.input_size, self.dependencies, other_output_features) feature["input_size"] = self.input_size # needed for future overrides self.fc_stack = FCStack( first_layer_input_size=self.input_size, layers=self.fc_layers, num_layers=self.num_fc_layers, default_output_size=self.output_size, default_use_bias=self.use_bias, default_weights_initializer=self.weights_initializer, default_bias_initializer=self.bias_initializer, default_norm=self.norm, default_norm_params=self.norm_params, default_activation=self.activation, default_dropout=self.dropout, ) self._calibration_module = self.create_calibration_module(feature) self._prediction_module = self.create_predict_module() # set up two sequence reducers, one for inputs and other for dependencies self.reduce_sequence_input = SequenceReducer( reduce_mode=self.reduce_input) if self.dependencies: self.dependency_reducers = torch.nn.ModuleDict() # todo: re-evaluate need for separate handling of `attention` reducer # currently this code does not support `attention` for dependency in self.dependencies: self.dependency_reducers[dependency] = SequenceReducer( reduce_mode=self.reduce_dependencies)
def __init__( self, embedding_size=10, embeddings_on_cpu=False, fc_layers=None, num_fc_layers=0, fc_size=10, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout=0, **kwargs): """ :param embedding_size: it is the maximum embedding size, the actual size will be `min(vocaularyb_size, embedding_size)` for `dense` representations and exacly `vocaularyb_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embedings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memroy and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param fc_layers: list of dictionaries containing the parameters of all the fully connected layers :type fc_layers: List :param num_fc_layers: Number of stacked fully connected layers :type num_fc_layers: Integer :param fc_size: Size of each layer :type fc_size: Integer :param use_bias: bool determines where to use a bias vector :type use_bias: bool :param weights_initializer: Initializer for the weights (aka kernel) matrix :type weights_initializer: string :param bias_initializer: Initializer for the bias vector :type bias_initializer: string :param weights_regularizer: regularizer applied to the weights (kernal) matrix :type weights_regularizer: string :param bias_regularizer: reguralizer function applied to biase vector. :type bias_regularizer: string :param activity_regularizer: Regularizer applied to the output of the layer (activation) :type activity_regularizer: string :param norm: type of normalization to use 'batch' or 'layer' :type norm: string, default None :param norm_params: parameters to pass to normalization function :type norm_params: dictionary :param activation: Activation function to use. :type activation: string :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: float """ super().__init__() logger.debug(' {}'.format(self.name)) logger.debug(' year FCStack') self.year_fc = FCStack( num_layers=1, default_fc_size=1, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=None, default_norm_params=None, default_activation=None, default_dropout=dropout, ) logger.debug(' month Embed') self.embed_month = Embed([str(i) for i in range(12)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' day Embed') self.embed_day = Embed([str(i) for i in range(31)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' weekday Embed') self.embed_weekday = Embed([str(i) for i in range(7)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' yearday Embed') self.embed_yearday = Embed([str(i) for i in range(366)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' hour Embed') self.embed_hour = Embed([str(i) for i in range(24)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' minute Embed') self.embed_minute = Embed([str(i) for i in range(60)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' second Embed') self.embed_second = Embed([str(i) for i in range(60)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, resnet_size=50, num_filters=16, kernel_size=3, conv_stride=1, first_pool_size=None, first_pool_stride=None, batch_norm_momentum=0.9, batch_norm_epsilon=0.001, fc_layers=None, num_fc_layers=1, fc_size=256, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout=0, **kwargs): super(ResNetEncoder, self).__init__() logger.debug(' {}'.format(self.name)) if resnet_size < 50: bottleneck = False else: bottleneck = True block_sizes = get_resnet_block_sizes(resnet_size) block_strides = [1, 2, 2, 2][:len(block_sizes)] logger.debug(' ResNet2') self.resnet = ResNet2(resnet_size, bottleneck, num_filters, kernel_size, conv_stride, first_pool_size, first_pool_stride, block_sizes, block_strides, batch_norm_momentum, batch_norm_epsilon) self.flatten = Flatten() logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=fc_weights_constraint, # default_bias_constraint=fc_bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, conv_layers=None, num_conv_layers=None, filter_size=3, num_filters=32, strides=(1, 1), padding='valid', dilation_rate=(1, 1), conv_use_bias=True, conv_weights_initializer='glorot_uniform', conv_bias_initializer='zeros', conv_weights_regularizer=None, conv_bias_regularizer=None, conv_activity_regularizer=None, # conv_weights_constraint=None, # conv_bias_constraint=None, conv_norm=None, conv_norm_params=None, conv_activation='relu', conv_dropout=0, pool_size=(2, 2), pool_strides=None, fc_layers=None, num_fc_layers=1, fc_size=128, fc_use_bias=True, fc_weights_initializer='glorot_uniform', fc_bias_initializer='zeros', fc_weights_regularizer=None, fc_bias_regularizer=None, fc_activity_regularizer=None, # fc_weights_constraint=None, # fc_bias_constraint=None, fc_norm=None, fc_norm_params=None, fc_activation='relu', fc_dropout=0, **kwargs): super(Stacked2DCNN, self).__init__() logger.debug(' {}'.format(self.name)) logger.debug(' Conv2DStack') self.conv_stack_2d = Conv2DStack( layers=conv_layers, num_layers=num_conv_layers, default_num_filters=num_filters, default_filter_size=filter_size, default_strides=strides, default_padding=padding, default_dilation_rate=dilation_rate, default_use_bias=conv_use_bias, default_weights_initializer=conv_weights_initializer, defaultbias_initializer=conv_bias_initializer, default_weights_regularizer=conv_weights_regularizer, default_bias_regularizer=conv_bias_regularizer, default_activity_regularizer=conv_activity_regularizer, # default_weights_constraint=conv_weights_constraint, # default_bias_constraint=conv_bias_constraint, default_norm=conv_norm, default_norm_params=conv_norm_params, default_activation=conv_activation, default_dropout=conv_dropout, default_pool_size=pool_size, default_pool_strides=pool_strides, ) logger.debug(' FCStacl') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=fc_use_bias, default_weights_initializer=fc_weights_initializer, default_bias_initializer=fc_bias_initializer, default_weights_regularizer=fc_weights_regularizer, default_bias_regularizer=fc_bias_regularizer, default_activity_regularizer=fc_activity_regularizer, # default_weights_constraint=fc_weights_constraint, # default_bias_constraint=fc_bias_constraint, default_norm=fc_norm, default_norm_params=fc_norm_params, default_activation=fc_activation, default_dropout=fc_dropout, )
def __init__( self, input_features=None, num_layers=1, hidden_size=256, num_heads=8, transformer_fc_size=256, dropout=0.1, fc_layers=None, num_fc_layers=0, fc_size=256, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, fc_activation='relu', fc_dropout=0, fc_residual=False, reduce_output='mean', **kwargs ): super().__init__() logger.debug(' {}'.format(self.name)) self.reduce_output = reduce_output self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output) if self.reduce_output is None: self.supports_masking = True logger.debug(' Projectors') self.projectors = [Dense(hidden_size) for _ in input_features] logger.debug(' TransformerStack') self.transformer_stack = TransformerStack( hidden_size=hidden_size, num_heads=num_heads, fc_size=transformer_fc_size, num_layers=num_layers, dropout=dropout ) if self.reduce_output is not None: logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=fc_activation, default_dropout=fc_dropout, fc_residual=fc_residual, )
def __init__( self, height: int, width: int, resnet_size: int = 50, num_channels: int = 3, out_channels: int = 16, kernel_size: Union[int, Tuple[int]] = 3, conv_stride: Union[int, Tuple[int]] = 1, first_pool_kernel_size: Union[int, Tuple[int]] = None, first_pool_stride: Union[int, Tuple[int]] = None, batch_norm_momentum: float = 0.1, batch_norm_epsilon: float = 0.001, fc_layers: Optional[List[Dict]] = None, num_fc_layers: Optional[int] = 1, output_size: int = 256, use_bias: bool = True, weights_initializer: str = "xavier_uniform", bias_initializer: str = "zeros", norm: Optional[str] = None, norm_params: Optional[Dict[str, Any]] = None, activation: str = "relu", dropout: float = 0, **kwargs, ): super().__init__() logger.debug(f" {self.name}") # map parameter input feature config names to internal names img_height = height img_width = width first_in_channels = num_channels self._input_shape = (first_in_channels, img_height, img_width) logger.debug(" ResNet") self.resnet = ResNet( img_height=img_height, img_width=img_width, first_in_channels=first_in_channels, out_channels=out_channels, resnet_size=resnet_size, kernel_size=kernel_size, conv_stride=conv_stride, first_pool_kernel_size=first_pool_kernel_size, first_pool_stride=first_pool_stride, batch_norm_momentum=batch_norm_momentum, batch_norm_epsilon=batch_norm_epsilon, ) first_fc_layer_input_size = self.resnet.output_shape[0] logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=first_fc_layer_input_size, layers=fc_layers, num_layers=num_fc_layers, default_output_size=output_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )