def __init__(self, input_dim, output_dim, hidden_dims=[]):
super().__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.hidden_dims = utils.make_iterable(hidden_dims)
self.weights = [
lbann.Weights(initializer=lbann.HeNormalInitializer())
for _ in range(len(self.hidden_dims))
]
self.weights.append(
lbann.Weights(initializer=lbann.HeNormalInitializer()))
def __init__(self,
size,
bias=False,
weights=[],
activation=None,
transpose=False,
name=None,
parallel_strategy={}):
"""Initalize channelwise fully connected module
Args:
size (int or list): Dimension of the output tensor
bias (bool): Whether to apply bias after linearity.
transpose (bool): Whether to apply transpose of weights
matrix.
weights (`Weights` or iterator of `Weights`): Weights in
fully-connected layer. There are at most two: the
matrix and the bias. If weights are not provided, the
matrix will be initialized with He normal
initialization and the bias with zeros.
activation (type): Layer class for activation function.
name (str): Default name is in the form 'channelwisefc<index>'.
parallel_strategy (dict): Data partitioning scheme.
"""
super().__init__()
ChannelwiseFullyConnectedModule.global_count += 1
self.instance = 0
self.size = size
self.bias = bias
self.transpose = transpose
self.parallel_strategy = parallel_strategy
self.name = (name if name else 'channelwisefc{0}'.format(
ChannelwiseFullyConnectedModule.global_count))
self.data_layout = 'data_parallel'
self.weights = list(make_iterable(weights))
if len(self.weights) > 2:
raise ValueError('`FullyConnectedModule` has '
'at most two weights, '
'but got {0}'.format(len(self.weights)))
if len(self.weights) == 0:
self.weights.append(
lbann.Weights(initializer=lbann.HeNormalInitializer(),
name=self.name + '_matrix'))
if self.bias and len(self.weights) == 1:
self.weights.append(
lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name=self.name + '_bias'))
self.activation = None
if activation:
if isinstance(activation, type):
self.activation = activation
else:
self.activation = type(activation)
if not issubclass(self.activation, lbann.Layer):
raise ValueError('activation must be a layer')
def __init__(self, size, bias=True, weights=[], activation=None,
name=None, data_layout='data_parallel'):
"""Initialize fully-connected module.
Args:
size (int): Size of output tensor.
activation (type): Layer class for activation function.
bias (bool): Whether to apply bias after linearity.
weights (`Weights` or iterator of `Weights`): Weights in
fully-connected layer. There are at most two: the
matrix and the bias. If weights are not provided, the
matrix will be initialized with He normal
initialization and the bias with zeros.
name (str): Default name is in the form 'fcmodule<index>'.
data_layout (str): Data layout.
"""
super().__init__()
FullyConnectedModule.global_count += 1
self.instance = 0
self.size = size
self.bias = bias
self.name = (name
if name
else 'fcmodule{0}'.format(FullyConnectedModule.global_count))
self.data_layout = data_layout
# Initialize weights
# Note: If weights are not provided, matrix weights are
# initialized with He normal scheme and bias weights are
# initialized with zeros.
self.weights = list(make_iterable(weights))
if len(self.weights) > 2:
raise ValueError('`FullyConnectedModule` has '
'at most two weights, '
'but got {0}'.format(len(self.weights)))
if len(self.weights) == 0:
self.weights.append(
lbann.Weights(initializer=lbann.HeNormalInitializer(),
name=self.name+'_matrix'))
if len(self.weights) == 1:
self.weights.append(
lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name=self.name+'_bias'))
# Initialize activation layer
self.activation = None
if activation:
if isinstance(activation, type):
self.activation = activation
else:
self.activation = type(activation)
if not issubclass(self.activation, lbann.Layer):
raise ValueError('activation must be a layer')
def __init__(
self,
embed_dim=512,
num_heads=8,
feedforward_dim=2048,
dropout=0.1,
name=None,
):
TransformerDecoderLayer.global_count += 1
self.instance = 0
self.embed_dim = embed_dim
self.feedforward_dim = feedforward_dim
self.dropout_prob = dropout
# Module name
self.name = name
if not self.name:
self.name = f'transformerdecoderlayer{TransformerDecoderLayer.global_count}'
# Layer modules
self.attention1 = lbann.modules.transformer.MultiheadAttention(
embed_dim, num_heads, name=f'{self.name}_attention1')
self.attention2 = lbann.modules.transformer.MultiheadAttention(
embed_dim, num_heads, name=f'{self.name}_attention2')
# Weights for fully-connected layers
self.fc1_weights = [
lbann.Weights(initializer=lbann.HeNormalInitializer(),
name=f'{self.name}_fc1_matrix'),
lbann.Weights(initializer=lbann.ConstantInitializer(value=0),
name=f'{self.name}_fc1_bias'),
]
self.fc2_weights = [
lbann.Weights(initializer=lbann.GlorotNormalInitializer(),
name=f'{self.name}_fc2_matrix'),
lbann.Weights(initializer=lbann.ConstantInitializer(value=0),
name=f'{self.name}_fc2_bias'),
]
def __init__(self,
neuron_dims=[1000, 1000, 1000],
activation=lbann.Relu,
keep_prob=0.95,
name=None):
self.instance = 0
self.name = (name if name else 'combo{0}'.format(Combo.global_count))
#shared weights for drug 1 and 2 tracks
shared_w = []
for i in range(len(neuron_dims)):
shared_w.append(
lbann.Weights(initializer=lbann.HeNormalInitializer(),
name='drug_matrix' + str(i)))
shared_w.append(
lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name='drug_bias' + str(i)))
print("SHARED W ", type(shared_w))
self.geneT = TrackModule(neuron_dims,
activation,
keep_prob,
name=self.name + 'gene_track')
self.drug1T = TrackModule(neuron_dims,
activation,
keep_prob,
shared_w,
name=self.name + 'drug1_track')
self.drug2T = TrackModule(neuron_dims,
activation,
keep_prob,
shared_w,
name=self.name + 'drug2_track')
self.concatT = TrackModule(neuron_dims,
activation,
keep_prob,
name=self.name + 'concat_track')
def __init__(self,
num_dims,
out_channels,
kernel_size,
stride=1,
padding=0,
dilation=1,
groups=1,
bias=True,
weights=[],
activation=None,
name=None,
transpose=False,
parallel_strategy={}):
"""Initialize convolution module.
Args:
num_dims (int): Number of dimensions.
out_channels (int): Number of output channels, i.e. number
of filters.
kernel_size (int): Size of convolution kernel.
stride (int): Convolution stride.
padding (int): Convolution padding.
dilation (int): Convolution dilation.
groups (int): Number of convolution groups.
bias (bool): Whether to apply channel-wise bias after
convolution.
weights (`Weights` or iterator of `Weights`): Weights in
convolution layer. There are at most two: the kernel
and the bias. If weights are not provided, the kernel
will be initialized with He normal initialization and
the bias with zeros.
name (str): Default name is in the form 'convmodule<index>'.
transpose (bool): If true call deconvolution (or convolution
transpose)
parallel_strategy dict): Data partitioning scheme.
"""
super().__init__()
ConvolutionModule.global_count += 1
self.instance = 0
self.num_dims = num_dims
self.out_channels = out_channels
self.kernel_size = kernel_size
self.stride = stride
self.padding = padding
self.dilation = dilation
self.groups = groups
self.bias = bias
self.weights = list(make_iterable(weights))
self.name = (name if name else 'convmodule{0}'.format(
ConvolutionModule.global_count))
self.transpose = transpose
self.parallel_strategy = parallel_strategy
# Initialize weights
# Note: If weights are not provided, kernel weights are
# initialized with He normal scheme and bias weights are
# initialized with zeros.
self.weights = list(make_iterable(weights))
if len(self.weights) > 2:
raise ValueError('`ConvolutionModule` has '
'at most two weights, '
'but got {0}'.format(len(self.weights)))
if len(self.weights) == 0:
self.weights.append(
lbann.Weights(initializer=lbann.HeNormalInitializer(),
name=self.name + '_kernel'))
if len(self.weights) == 1:
self.weights.append(
lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name=self.name + '_bias'))
# Initialize activation layer
self.activation = None
if activation:
if isinstance(activation, type):
self.activation = activation
else:
self.activation = type(activation)
if not issubclass(self.activation, lbann.Layer):
raise ValueError('activation must be a layer')