def forward(self, logits, targets, mask):
"""
Calculates loss accounting for different numbers of output per sample.
:param logits: Logits being output by the model. [batch, sequence, element_size]
:param targets: LongTensor targets [batch, sequence, element_size]
:param mask: ByteTensor mask [batch, sequence]
"""
# Calculate the loss per element in the sequence
loss_per_element = self.loss_function(logits, targets)
# if the loss has one extra dimenison then you need an extra unit dimension
# to multiply element by element
mask_float = mask.type(AppState().dtype)
if len(mask.shape) < len(loss_per_element.shape):
mask_float = mask_float.unsqueeze(-1)
# Set the loss per element to zero for unneeded output
masked_loss_per = mask_float * loss_per_element
# obtain the number of non-zero elements in the mask.
# nonzero() returns the indices so you have to divide by the number of dimensions
# The mask lacks the last dimension of the targets so needs to be
# scaled up
size = mask.nonzero().numel() / len(mask.shape) * logits.shape[-1]
loss = torch.sum(masked_loss_per) / size
return loss
def masked_accuracy(self, logits, targets, mask):
""" Calculate accuracy equal to mean difference between outputs and targets.
WARNING: Applies mask (from aux_tuple) to both logits and targets!
:param logits: Logits being output by the model. [batch, sequence, element_size]
:param targets: LongTensor targets [batch, sequence, element_size]
:param mask: ByteTensor mask [batch, sequence]
"""
# calculate the accuracy per bit in the sequences
acc_per = 1 - torch.abs(torch.round(F.sigmoid(logits)) - targets)
mask_float = mask.type(AppState().dtype)
if len(mask.shape) < len(logits.shape):
mask_float = mask_float.unsqueeze(-1)
# The mask lacks the last dimension of the targets so needs to be
# scaled up
size = mask.nonzero().numel() / len(mask.shape) * logits.shape[-1]
masked_acc_per = mask_float * acc_per
accuracy = masked_acc_per.sum().item() / size
return accuracy
def init_state(self, memory_address_size, batch_size):
"""
Returns 'zero' (initial) state tuple.
:param memory_address_size: The number of memory addresses
:param batch_size: Size of the batch in given iteraction/epoch.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
dtype = AppState().dtype
# Read attention weights [BATCH_SIZE x MEMORY_SIZE]
read_attention = torch.ones((batch_size, self._num_reads,
memory_address_size)).type(dtype) * 1e-6
# Write attention weights [BATCH_SIZE x MEMORY_SIZE]
write_attention = torch.ones((batch_size, self._num_writes,
memory_address_size)).type(dtype) * 1e-6
# Usage of memory cells [BATCH_SIZE x MEMORY_SIZE]
usage = self.mem_usage.init_state(memory_address_size, batch_size)
# temporal links tuple
link_tuple = self.temporal_linkage.init_state(memory_address_size,
batch_size)
return InterfaceStateTuple(read_attention, write_attention, usage,
link_tuple)
def init_state(self, memory_address_size, batch_size):
"""
Returns 'zero' (initial) state:
* memory is reset to random values.
* read & write weights (and read vector) are set to 1e-6.
:param batch_size: Size of the batch in given iteraction/epoch.
:param num_memory_adresses: Number of memory addresses.
"""
dtype = AppState().dtype
# Initialize controller state.
ctrl_init_state = self.control_params.init_state(batch_size)
# Initialize interface state.
interface_init_state = self.interface.init_state(
memory_address_size, batch_size)
# Memory [BATCH_SIZE x MEMORY_BITS x MEMORY_SIZE]
init_memory_BxMxA = torch.zeros(
batch_size, self.num_memory_bits, memory_address_size).type(dtype)
# Read vector [BATCH_SIZE x MEMORY_SIZE]
read_vector_BxM = self.interface.read(
interface_init_state, init_memory_BxMxA)
# Pack and return a tuple.
return NTMCellStateTuple(
ctrl_init_state,
interface_init_state,
init_memory_BxMxA,
read_vector_BxM)
def forward(self, encoded_image, encoded_question):
"""
Apply stacked attention.
:param encoded_image: output of the image encoding (CNN + FC layer), should be of shape \
[batch_size, width * height, num_channels_encoded_image]
:type encoded_image: torch.tensor
:param encoded_question: Last hidden layer of the LSTM, of shape [batch_size, question_encoding_size]
:type encoded_question: torch.tensor
:return: u: attention [batch_size, num_channels_encoded_image]
"""
for att_layer in self.san:
u, attention_prob = att_layer(encoded_image, encoded_question)
if AppState().visualize:
if self.visualize_attention is None:
self.visualize_attention = attention_prob
# Concatenate output
else:
self.visualize_attention = torch.cat(
[self.visualize_attention, attention_prob], dim=-1)
return u
def normalize(x):
"""
Normalizes the input torch tensor along the last dimension using the max of
the one norm The normalization is "fuzzy" to prevent divergences.
:param x: input of shape [batch_size, A, A1 ..An] if the input is the weight vector x'sahpe (batch_size, num_heads, memory_size)
:return: normalized x of shape [batch_size, A, A1 ..An]
"""
dtype = AppState().dtype
return x / torch.max(torch.sum(x, dim=-1, keepdim=True),
torch.Tensor([1e-12]).type(dtype))
def init_state(self, batch_size):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:returns: Initial state tuple - object of RNNStateTuple class.
"""
# Initialize LSTM hidden state [BATCH_SIZE x CTRL_HIDDEN_SIZE].
dtype = AppState().dtype
hidden_state = torch.zeros((batch_size, self.ctrl_hidden_state_size),
requires_grad=False).type(dtype)
return RNNStateTuple(hidden_state)
def init_state(self, memory_address_size, batch_size):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
dtype = AppState().dtype
self._memory_size = memory_address_size
usage = torch.zeros((batch_size, memory_address_size)).type(dtype)
return usage
def init_state(self, memory_addresses_size, batch_size):
dtype = AppState().dtype
# Initialize controller state.
tuple_ctrl_init_state = self.controller.init_state(batch_size)
# Initialize interface state.
tuple_interface_init_state = self.interface.init_state(
memory_addresses_size, batch_size)
# Initialize memory
mem_init = (torch.ones(
(batch_size, self.M, memory_addresses_size)) * 0.01).type(dtype)
return DWMCellStateTuple(tuple_ctrl_init_state,
tuple_interface_init_state, mem_init)
def exclusive_cumprod_temp(self, sorted_usage, dim=1):
"""
Applies the exclusive cumultative product (at the moment it assumes the
shape of the input)
:param sorted_usage: tensor of shape `[batch_size, memory_size]` indicating current memory usage sorted in ascending order.
:returns: Tensor of shape `[batch_size, memory_size]` that is exclusive pruduct of the sorted usage i.e. = [1, u1, u1*u2, u1*u2*u3, ....]
"""
# TODO: expand this so it works for any dim
dtype = AppState().dtype
a = torch.ones((sorted_usage.shape[0], 1)).type(dtype)
b = torch.cat((a, sorted_usage), dim=dim).type(dtype)
prod_sorted_usage = torch.cumprod(b, dim=dim)[:, :-1]
return prod_sorted_usage
def init_state(self, memory_address_size, batch_size):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
dtype = AppState().dtype
self._memory_size = memory_address_size
link = torch.ones((batch_size, self._num_writes, memory_address_size,
memory_address_size)).type(dtype) * 1e-6
precendence_weights = torch.ones(
(batch_size, self._num_writes,
memory_address_size)).type(dtype) * 1e-6
return TemporalLinkageState(link, precendence_weights)
def init_state(self, batch_size, num_memory_addresses):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:param num_memory_addresses: Number of memory addresses.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
dtype = AppState().dtype
# Add read head states - one for each read head.
read_state_tuples = []
# Initial attention weights [BATCH_SIZE x MEMORY_ADDRESSES x 1]
# Initialize attention: to address 0.
zh_attention = torch.zeros(
batch_size, num_memory_addresses, 1).type(dtype)
zh_attention[:, 0, 0] = 1
# Initialize gating: to previous attention (i.e. zero-hard).
init_gating = torch.ones(batch_size, 1, 1).type(dtype)
# Initialize shift - to zero.
init_shift = torch.zeros(
batch_size, self.interface_shift_size, 1).type(dtype)
init_shift[:, 1, 0] = 1
for i in range(self.interface_num_read_heads):
read_ht = HeadStateTuple(
zh_attention, zh_attention, init_gating, init_shift)
# Single read head tuple.
read_state_tuples.append(read_ht)
# Single write head tuple.
write_state_tuple = HeadStateTuple(
zh_attention, zh_attention, init_gating, init_shift)
# Return tuple.
interface_state = InterfaceStateTuple(
read_state_tuples, write_state_tuple)
return interface_state
def init_state(self, memory_addresses_size, batch_size):
"""
Returns 'zero' (initial) state of Interface tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:param memory_addresses_size: size of the memory
:returns: Initial state tuple - object of InterfaceStateTuple class: (head_weight_init, snapshot_weight_init)
"""
dtype = AppState().dtype
# initial attention vector
head_weight_init = torch.zeros(
(batch_size, self.num_heads, memory_addresses_size)).type(dtype)
head_weight_init[:, 0:self.num_heads, 0] = 1.0
# bookmark
snapshot_weight_init = head_weight_init
return InterfaceStateTuple(head_weight_init, snapshot_weight_init)
def init_state(self, batch_size):
"""
Initialize the state of a ``ThalNet`` module.
:param batch_size: batch size
:type batch_size: int
:return: center_state_per_module, tuple_controller_states
"""
dtype = AppState().dtype
# module state initialisation
tuple_controller_states = self.controller.init_state(batch_size)
# center state initialisation
center_state_per_module = torch.randn(
(batch_size, self.center_size_per_module)).type(dtype)
return center_state_per_module, tuple_controller_states
def masked_accuracy(self, logits, targets, mask):
"""
Calculates accuracy equal to mean number of correct predictions in a \
given batch.
.. warning::
Applies ``mask`` to both ``logits`` and ``targets``.
:param logits: Logits being output by the model. [batch, classes, sequence].
:type logits: torch.tensor.
:param targets: Targets [batch, sequence].
:type targets: torch.LongTensor
:param mask: Mask [batch, sequence].
:type mask: torch.ByteTensor
:return: accuracy value.
"""
# calculate the accuracy per bit in the sequences
acc_per = 1 - torch.abs(
torch.round(torch.nn.functional.sigmoid(logits)) - targets)
mask_float = mask.type(AppState().dtype)
if len(mask.shape) < len(logits.shape):
mask_float = mask_float.unsqueeze(-1)
# The mask lacks the last dimension of the targets so needs to be
# scaled up
size = mask.nonzero().numel() / len(mask.shape) * logits.shape[-1]
masked_acc_per = mask_float * acc_per
accuracy = masked_acc_per.sum().item() / size
return accuracy
def forward(self, logits, targets, mask):
"""
Calculates loss accounting for different numbers of output per sample.
:param logits: Logits being output by the model. [batch, classes, sequence].
:type logits: torch.tensor.
:param targets: Targets [batch, sequence].
:type targets: torch.LongTensor
:param mask: Mask [batch, sequence].
:type mask: torch.ByteTensor
:return: loss value.
"""
# Calculate the loss per element in the sequence
loss_per_element = self.loss_function(logits, targets)
# Have to convert the mask to floats to multiply by the loss
mask_float = mask.type(AppState().dtype)
# if the loss has one extra dimenison then you need an extra unit dimension
# to multiply element by element
if len(mask.shape) < len(loss_per_element.shape):
mask_float = mask_float.unsqueeze(-1)
# Set the loss per element to zero for unneeded output
masked_loss_per = mask_float * loss_per_element
# obtain the number of non-zero elements in the mask.
# nonzero() returns the indices so you have to divide by the number of
# dimensions
size = mask.nonzero().numel() / len(mask.shape)
# add up the loss scaling by only the needed outputs
loss = torch.sum(masked_loss_per) / size
return loss
def init_state(self, batch_size, num_memory_addresses,
final_encoder_attention_BxAx1):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:param num_memory_addresses: Number of memory addresses.
:param final_encoder_attention_BxAx1: final attention of the encoder [BATCH_SIZE x MEMORY_ADDRESSES x 1]
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
# Get dtype.
dtype = AppState().dtype
# Initial attention weights [BATCH_SIZE x MEMORY_ADDRESSES x 1]
# Zero-hard attention.
zh_attention = torch.zeros(batch_size, num_memory_addresses,
1).type(dtype)
# Initialize as "hard attention on 0 address"
zh_attention[:, 0, 0] = 1
# Gating [BATCH x 3 x 1]
init_gating = torch.zeros(batch_size, 3, 1).type(dtype)
init_gating[:, 0, 0] = 1 # Initialize as "prev attention"
# Shift [BATCH x SHIFT_SIZE x 1]
init_shift = torch.zeros(batch_size, self.interface_shift_size,
1).type(dtype)
init_shift[:, 1, 0] = 1 # Initialize as "0 shift".
# Remember zero-hard attention.
self.zero_hard_attention_BxAx1 = zh_attention
# Remember final attention of encoder.
self.final_encoder_attention_BxAx1 = final_encoder_attention_BxAx1
# Return tuple.
return MASInterfaceStateTuple(zh_attention,
self.final_encoder_attention_BxAx1,
init_gating, init_shift)
def init_state(self, batch_size, num_memory_addresses):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:param num_memory_addresses: Number of memory addresses.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
# Get dtype.
dtype = AppState().dtype
# Zero-hard attention.
zh_attention = torch.zeros(batch_size, num_memory_addresses,
1).type(dtype)
zh_attention[:, 0, 0] = 1
# Init gating.
init_shift = torch.zeros(batch_size, self.interface_shift_size,
1).type(dtype)
init_shift[:, 1, 0] = 1
# Return tuple.
return MAEInterfaceStateTuple(zh_attention, init_shift)
# print("--- %s seconds ---" % (time.time() - start_time))
# Plot figure and list of frames.
self.plotWindow.update(fig, frames)
return self.plotWindow.is_closed
if __name__ == "__main__":
# Set logging level.
logger = logging.getLogger('NTM-Module')
logging.basicConfig(level=logging.DEBUG)
# Set visualization.
from miprometheus.utils.app_state import AppState
AppState().visualize = True
# "Loaded parameters".
from miprometheus.utils.param_interface import ParamInterface
params = ParamInterface()
params.add_default_params({
# controller parameters
'controller': {
'name': 'GRUController',
'hidden_state_size': 5,
'num_layers': 1,
'non_linearity': 'none',
'output_size': 5
},
# interface parameters
'interface': {
def __init__(self, params, problem_default_values_={}):
"""
Initializes a Model object.
:param params: Parameters read from configuration file.
:type params: ``miprometheus.utils.ParamInterface``
:param problem_default_values_: dict of parameters values coming from the problem class. One example of such \
parameter value is the size of the vocabulary set in a translation problem.
:type problem_default_values_: dict
This constructor:
- stores a pointer to ``params``:
>>> self.params = params
- sets a default problem name:
>>> self.name = 'Model'
- initializes the logger.
>>> self.logger = logging.Logger(self.name)
- tries to parse the values coming from ``problem_default_values_``:
>>> try:
>>> for key in problem_default_values_.keys():
>>> self.params.add_custom_params({key: problem_default_values_[key]})
>>> except BaseException:
>>> self.logger.info('No parameter value was parsed from problem_default_values_')
- initializes the data definitions:
.. note::
This dict contains information about the expected inputs and produced outputs of the current model class.
This object will be used during handshaking between the model and the problem class to ensure that the model
can accept the batches produced by the problem and that the problem can accept the predictions of the model
to compute the loss and accuracy.
This dict should be defined using self.params.
This dict should at least contains the `targets` field:
>>> self.data_definitions = {'inputs': {'size': [-1, -1], 'type': [torch.Tensor]},
>>> 'targets': {'size': [-1, 1], 'type': [torch.Tensor]}
>>> }
- sets the access to ``AppState``: for dtype, visualization flag etc.
>>> self.app_state = AppState()
- initialize the best model loss (to select which model to save) to ``np.inf``:
>>> self.best_loss = np.inf
"""
# Call base class constructor here.
super(Model, self).__init__()
# Store pointer to params.
self.params = params
# "Default" model name.
self.name = 'Model'
# initialize the logger
self.logger = logging.getLogger(self.name)
# Flag indicating whether intermediate checkpoints should be saved or
# not (DEFAULT: False).
params.add_default_params({"save_intermediate": False})
self.save_intermediate = params["save_intermediate"]
# process all params from configuration file and problem_default_values_ here
try:
for key in problem_default_values_.keys():
self.params.add_custom_params(
{key: problem_default_values_[key]})
except Exception:
self.logger.warning(
'No parameter value was parsed from problem_default_values_')
# --> We assume from here that the model class has all parameters values needed (either from params or
# problem_default_values_) to be correctly instantiated and contained in self.params.
# We can then define a dict that contains a description of the expected (and mandatory) inputs for this model.
# This dict should be defined using self.params.
self.data_definitions = {
'inputs': {
'size': [-1, -1],
'type': [torch.Tensor]
},
'targets': {
'size': [-1, 1],
'type': [torch.Tensor]
}
}
# --> The remaining parameters should be hardcoded values.
# Initialize app state.
self.app_state = AppState()
# Window in which the data will be plotted.
self.plotWindow = None
# Initialization of best loss - as INF.
self.best_loss = np.inf
self.best_status = "Unknown"
def __init__(self, name="GridWorker", use_gpu=False):
"""
Base constructor for all grid workers:
- Initializes the AppState singleton:
>>> self.app_state = AppState()
- Initializes the Parameter Registry:
>>> self.params = ParamInterface()
- Defines the logger:
>>> self.logger = logging.getLogger(name=self.name)
- Creates parser and adds default worker command line arguments (you can display them with ``--h``).
:param name: Name of the worker (DEFAULT: "GridWorker").
:type name: str
:param use_gpu: Indicates whether the worker should use GPU or not. Value coming from the subclasses \
(e.g. ``GridTrainerCPU`` vs ``GridTrainerGPU``) (DEFAULT: False).
:type use_gpu: bool
"""
# Call base constructor.
super(GridWorker, self).__init__()
# Set worker name.
self.name = name
# Initialize the application state singleton.
self.app_state = AppState()
self.app_state.use_CUDA = use_gpu
# Initialize parameter interface/registry.
self.params = ParamInterface()
# Load the default logger configuration.
logger_config = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format':
'[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'
}
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'INFO',
'formatter': 'simple',
'stream': 'ext://sys.stdout'
}
},
'root': {
'level': 'DEBUG',
'handlers': ['console']
}
}
logging.config.dictConfig(logger_config)
# Create the Logger, set its label and logging level.
self.logger = logging.getLogger(name=self.name)
# Create parser with a list of runtime arguments.
self.parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
# Add arguments to the specific parser.
# These arguments will be shared by all grid workers.
self.parser.add_argument(
'--outdir',
dest='outdir',
type=str,
default="./experiments",
help=
'Path to the global output directory where the experiments folders '
'will be / are stored. Affects all grid experiments.'
' (DEFAULT: ./experiments)')
self.parser.add_argument(
'--savetag',
dest='savetag',
type=str,
default='',
help='Additional tag for the global output directory.')
self.parser.add_argument(
'--ll',
action='store',
dest='log_level',
type=str,
default='INFO',
choices=[
'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'
],
help="Log level for the experiments. (Default: INFO)")
self.parser.add_argument(
'--li',
dest='logging_interval',
default=100,
type=int,
help=
'Statistics logging interval. Will impact logging to the logger and exporting to '
'TensorBoard for the experiments. Do not affect the grid worker. '
'Writing to the csv file is not impacted (interval of 1).'
' (Default: 100, i.e. logs every 100 episodes).')
self.parser.add_argument(
'--agree',
dest='confirm',
action='store_true',
help='Request user confirmation before starting the grid experiment.'
' (Default: False)')
def __init__(self, name="Worker"):
"""
Base constructor for all workers:
- Initializes the AppState singleton:
>>> self.app_state = AppState()
- Initializes the Parameter Registry:
>>> self.params = ParamInterface()
- Defines the logger:
>>> self.logger = logging.getLogger(name=self.name)
- Creates parser and adds default worker command line arguments.
:param name: Name of the worker (DEFAULT: "Worker").
"""
# Call base constructor.
super(Worker, self).__init__()
# Set worker name.
self.name = name
# Initialize the application state singleton.
self.app_state = AppState()
# Initialize parameter interface/registry.
self.params = ParamInterface()
# Load the default logger configuration.
logger_config = {'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'}},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'INFO',
'formatter': 'simple',
'stream': 'ext://sys.stdout'}},
'root': {'level': 'DEBUG',
'handlers': ['console']}}
logging.config.dictConfig(logger_config)
# Create the Logger, set its label and logging level.
self.logger = logging.getLogger(name=self.name)
# Create parser with a list of runtime arguments.
self.parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
# Add arguments to the specific parser.
# These arguments will be shared by all basic workers.
self.parser.add_argument('--config',
dest='config',
type=str,
default='',
help='Name of the configuration file(s) to be loaded. '
'If specifying more than one file, they must be separated with coma ",".')
self.parser.add_argument('--model',
type=str,
default='',
dest='model',
help='Path to the file containing the saved parameters'
' of the model to load (model checkpoint, should end with a .pt extension.)')
self.parser.add_argument('--gpu',
dest='use_gpu',
action='store_true',
help='The current worker will move the computations on GPU devices, if available in '
'the system. (Default: False)')
self.parser.add_argument('--outdir',
dest='outdir',
type=str,
default="./experiments",
help='Path to the output directory where the experiment(s) folders will be stored.'
' (DEFAULT: ./experiments)')
self.parser.add_argument('--savetag',
dest='savetag',
type=str,
default='',
help='Tag for the save directory')
self.parser.add_argument('--ll',
action='store',
dest='log_level',
type=str,
default='INFO',
choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'],
help="Log level. (Default: INFO)")
self.parser.add_argument('--li',
dest='logging_interval',
default=100,
type=int,
help='Statistics logging interval. Will impact logging to the logger and exporting to '
'TensorBoard. Writing to the csv file is not impacted (interval of 1).'
' (Default: 100, i.e. logs every 100 episodes).')
self.parser.add_argument('--agree',
dest='confirm',
action='store_true',
help='Request user confirmation just after loading the settings, '
'before starting training. (Default: False)')
x = self.conv3(x)
x = self.batchNorm3(x)
x = F.relu(x)
x = self.conv4(x)
x = self.batchNorm4(x)
x = F.relu(x)
return x
if __name__ == '__main__':
"""
Unit Test for the ``ConvInputModel``.
"""
# "Image" - batch x channels x width x height
batch_size = 64
img_size = 128
input_np = np.random.binomial(1, 0.5, (batch_size, 3, img_size, img_size))
image = torch.from_numpy(input_np).type(AppState().dtype)
cnn = ConvInputModel()
feature_maps = cnn(image)
print('feature_maps:', feature_maps.shape)
print('Computed output height, width:',
cnn.get_output_shape(img_size, img_size))
x = torch.nn.functional.relu(x)
x = self.f_fc2(x)
x = torch.nn.functional.relu(x)
x = torch.nn.functional.dropout(x, p=0.5)
x = self.f_fc3(x)
return x
if __name__ == '__main__':
"""
Unit Tests for g_theta & f_phi.
"""
input_size = (24 + 2) * 2 + 13
batch_size = 64
inputs = np.random.binomial(1, 0.5, (batch_size, 3, input_size))
inputs = torch.from_numpy(inputs).type(AppState().dtype)
g_theta = PairwiseRelationNetwork(input_size=input_size)
g_outputs = g_theta(inputs)
print('g_outputs:', g_outputs.shape)
output_size = 10
f_phi = SumOfPairsAnalysisNetwork(output_size=output_size)
f_outputs = f_phi(g_outputs)
print('f_outputs:', f_outputs.shape)
def __init__(self, name, add_default_parser_args=True):
"""
Base constructor for all workers:
- Initializes the AppState singleton:
>>> self.app_state = AppState()
- Initializes the Parameter Registry:
>>> self.params = ParamInterface()
- Defines the logger:
>>> self.logger = logging.getLogger(name=self.name)
- Creates parser and adds default worker command line arguments.
:param name: Name of the worker.
:type name: str
:param add_default_parser_args: If set, adds default parser arguments (DEFAULT: True).
:type add_default_parser_args: bool
"""
# Call base constructor.
super(Worker, self).__init__()
# Set worker name.
self.name = name
# Initialize the application state singleton.
self.app_state = AppState()
# Initialize parameter interface/registry.
self.params = ParamInterface()
# Initialize logger using the configuration.
self.initialize_logger()
# Create parser with a list of runtime arguments.
self.parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
# Add arguments to the specific parser.
if add_default_parser_args:
# These arguments will be shared by all basic workers.
self.parser.add_argument(
'--config',
dest='config',
type=str,
default='',
help='Name of the configuration file(s) to be loaded. '
'If specifying more than one file, they must be separated with coma ",".'
)
self.parser.add_argument(
'--model',
type=str,
default='',
dest='model',
help='Path to the file containing the saved parameters'
' of the model to load (model checkpoint, should end with a .pt extension.)'
)
self.parser.add_argument(
'--gpu',
dest='use_gpu',
action='store_true',
help=
'The current worker will move the computations on GPU devices, if available '
'in the system. (Default: False)')
self.parser.add_argument(
'--expdir',
dest='expdir',
type=str,
default="./experiments",
help=
'Path to the directory where the experiment(s) folders are/will be stored.'
' (DEFAULT: ./experiments)')
self.parser.add_argument('--savetag',
dest='savetag',
type=str,
default='',
help='Tag for the save directory.')
self.parser.add_argument('--ll',
action='store',
dest='log_level',
type=str,
default='INFO',
choices=[
'CRITICAL', 'ERROR', 'WARNING',
'INFO', 'DEBUG', 'NOTSET'
],
help="Log level. (Default: INFO)")
self.parser.add_argument(
'--li',
dest='logging_interval',
default=100,
type=int,
help=
'Statistics logging interval. Will impact logging to the logger and '
'exporting to TensorBoard. Writing to the csv file is not impacted '
'(interval of 1).(Default: 100, i.e. logs every 100 episodes).'
)
self.parser.add_argument(
'--agree',
dest='confirm',
action='store_true',
help=
'Request user confirmation just after loading the settings, '
'before starting training. (Default: False)')
def circular_convolution(self, attention_BxAx1, shift_BxSx1,
prev_memory_BxAxC):
"""
Performs circular convoution, i.e. shitfts the attention accodring to
given shift vector (convolution mask).
:param attention_BxAx1: Current attention [BATCH_SIZE x ADDRESS_SIZE x 1]
:param shift_BxSx1: soft shift maks (convolutional kernel) [BATCH_SIZE x SHIFT_SIZE x 1]
:param prev_memory_BxAxC: tensor containing memory before update [BATCH_SIZE x MEMORY_ADDRESSES x CONTENT_BITS]
:returns: attention vector of size [BATCH_SIZE x ADDRESS_SIZE x 1]
"""
def circular_index(idx, num_addr):
"""
Calculates the index, taking into consideration the number of
addresses in memory.
:param idx: index (single element)
:param num_addr: number of addresses in memory
"""
if idx < 0:
return num_addr + idx
elif idx >= num_addr:
return idx - num_addr
else:
return idx
# Check whether inputs are already on GPU or not.
long_dtype = AppState().LongTensor
# Get number of memory addresses and batch size.
batch_size = prev_memory_BxAxC.size(0)
num_addr = prev_memory_BxAxC.size(1)
shift_size = self.interface_shift_size
#logger.debug("shift_BxSx1 {}: {}".format(shift_BxSx1, shift_BxSx1.size()))
# Create an extended list of indices indicating what elements of the
# sequence will be where.
ext_indices_tensor = torch.Tensor([
circular_index(shift, num_addr)
for shift in range(-shift_size // 2 + 1, num_addr +
shift_size // 2)
]).type(long_dtype)
#logger.debug("ext_indices {}:\n {}".format(ext_indices_tensor.size(), ext_indices_tensor))
# Use indices for creation of an extended attention vector.
ext_attention_BxEAx1 = torch.index_select(attention_BxAx1,
dim=1,
index=ext_indices_tensor)
#logger.debug("ext_attention_BxEAx1 {}:\n {}".format(ext_attention_BxEAx1.size(), ext_attention_BxEAx1))
# Transpose inputs to convolution.
ext_att_trans_Bx1xEA = torch.transpose(ext_attention_BxEAx1, 1, 2)
shift_trans_Bx1xS = torch.transpose(shift_BxSx1, 1, 2)
# Perform convolution for every batch-filter pair.
tmp_attention_list = []
for b in range(batch_size):
tmp_attention_list.append(
torch.nn.functional.conv1d(
ext_att_trans_Bx1xEA.narrow(0, b, 1),
shift_trans_Bx1xS.narrow(0, b, 1)))
# Concatenate list into a single tensor.
shifted_attention_BxAx1 = torch.transpose(
torch.cat(tmp_attention_list, dim=0), 1, 2)
#logger.debug("shifted_attention_BxAx1 {}:\n {}".format(shifted_attention_BxAx1.size(), shifted_attention_BxAx1))
return shifted_attention_BxAx1
plt.imshow(image.permute(1, 2, 0),
interpolation='nearest',
aspect='auto')
# Plot!
plt.show()
exit()
if __name__ == '__main__':
""" Tests CNN_LSTM on SortOfCLEVR"""
# "Loaded parameters".
from miprometheus.utils.param_interface import ParamInterface
from miprometheus.utils.app_state import AppState
app_state = AppState()
app_state.visualize = True
from miprometheus.problems.image_text_to_class.sort_of_clevr import SortOfCLEVR
problem_params = ParamInterface()
problem_params.add_config_params({
'data_folder': '~/data/sort-of-clevr/',
'split': 'train',
'regenerate': False,
'dataset_size': 10000,
'img_size': 128
})
# create problem
sortofclevr = SortOfCLEVR(problem_params)
batch_size = 64
def __init__(self, params_, name_='Problem'):
"""
Initializes problem object.
:param params_: Dictionary of parameters (read from the configuration ``.yaml`` file).
:type params_: :py:class:`miprometheus.utils.ParamInterface`
:param name_: Problem name (DEFAULT: 'Problem').
:type name_: str
This constructor:
- stores a pointer to ``params``:
>>> self.params = params_
- sets a problem name:
>>> self.name = name_
- sets a default loss function:
>>> self.loss_function = None
- initializes the size of the dataset:
>>> self.length = None
- initializes the logger.
>>> self.logger = logging.Logger(self.name)
- initializes the data definitions: this is used for defining the ``DataDict`` keys.
.. note::
This dict contains information about the DataDict produced by the current problem class.
This object will be used during handshaking between the model and the problem class to ensure that the model
can accept the batches produced by the problem.
This dict should at least contains the `targets` field:
>>> self.data_definitions = {'targets': {'size': [-1, 1], 'type': [torch.Tensor]}}
- initializes the default values: this is used to pass missing parameters values to the model.
.. note::
It is likely to encounter a case where the model needs a parameter value only known when the problem has been
instantiated, like the size of a vocabulary set or the number of marker bits.
The user can fill in those values in this dict, which will be passed to the model in its `__init__` . The
model will then be able to fill it its missing parameters values, either from params or this dict.
>>> self.default_values = {}
- sets the access to ``AppState``: for dtype, visualization flag etc.
>>> self.app_state = AppState()
"""
# Store pointer to params.
self.params = params_
# Problem name.
self.name = name_
# Empty curriculum learning params - for now.
self.curriculum_params = {}
# Set default loss function.
self.loss_function = None
# Size of the dataset
self.length = None
# Initialize the logger.
self.logger = logging.getLogger(self.name)
# data_definitions: this is used for defining the DataDict keys.
# This dict contains information about the DataDict produced by the current problem class.
# This object will be used during handshaking between the model and the problem class to ensure that the model
# can accept the batches produced by the problem.
self.data_definitions = {}
# default_values: this is used to pass missing parameters values to the model.
# It is likely to encounter a case where the model needs a parameter value only known when the problem has been
# instantiated, like the size of a vocabulary set or the number of marker bits.
# The user can fill in those values in this dict, which will be passed to the model in its `__init__` . The
# model will then be able to fill it its missing parameters values, either from params or this dict.
self.default_values = {}
# Get access to AppState: for dtype, visualization flag etc.
self.app_state = AppState()
