def init_state(self, memory_address_size, batch_size):
"""
Returns 'zero' (initial) state tuple.
:param memory_address_size: The number of memory addresses
:param batch_size: Size of the batch in given iteraction/epoch.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
dtype = AppState().dtype
# Read attention weights [BATCH_SIZE x MEMORY_SIZE]
read_attention = torch.ones((batch_size, self._num_reads,
memory_address_size)).type(dtype) * 1e-6
# Write attention weights [BATCH_SIZE x MEMORY_SIZE]
write_attention = torch.ones((batch_size, self._num_writes,
memory_address_size)).type(dtype) * 1e-6
# Usage of memory cells [BATCH_SIZE x MEMORY_SIZE]
usage = self.mem_usage.init_state(memory_address_size, batch_size)
# temporal links tuple
link_tuple = self.temporal_linkage.init_state(memory_address_size,
batch_size)
return InterfaceStateTuple(read_attention, write_attention, usage,
link_tuple)
def forward(self, logits, targets, mask):
"""
Calculates loss accounting for different numbers of output per sample.
:param logits: Logits being output by the model. [batch, sequence, element_size]
:param targets: LongTensor targets [batch, sequence, element_size]
:param mask: ByteTensor mask [batch, sequence]
"""
# Calculate the loss per element in the sequence
loss_per_element = self.loss_function(logits, targets)
# if the loss has one extra dimenison then you need an extra unit dimension
# to multiply element by element
mask_float = mask.type(AppState().dtype)
if len(mask.shape) < len(loss_per_element.shape):
mask_float = mask_float.unsqueeze(-1)
# Set the loss per element to zero for unneeded output
masked_loss_per = mask_float * loss_per_element
# obtain the number of non-zero elements in the mask.
# nonzero() returns the indices so you have to divide by the number of dimensions
# The mask lacks the last dimension of the targets so needs to be
# scaled up
size = mask.nonzero().numel() / len(mask.shape) * logits.shape[-1]
loss = torch.sum(masked_loss_per) / size
return loss
def init_state(self, memory_address_size, batch_size):
"""
Returns 'zero' (initial) state:
* memory is reset to random values.
* read & write weights (and read vector) are set to 1e-6.
:param batch_size: Size of the batch in given iteraction/epoch.
:param num_memory_adresses: Number of memory addresses.
"""
dtype = AppState().dtype
# Initialize controller state.
ctrl_init_state = self.control_params.init_state(batch_size)
# Initialize interface state.
interface_init_state = self.interface.init_state(
memory_address_size, batch_size)
# Memory [BATCH_SIZE x MEMORY_BITS x MEMORY_SIZE]
init_memory_BxMxA = torch.zeros(
batch_size, self.num_memory_bits, memory_address_size).type(dtype)
# Read vector [BATCH_SIZE x MEMORY_SIZE]
read_vector_BxM = self.interface.read(
interface_init_state, init_memory_BxMxA)
# Pack and return a tuple.
return NTMCellStateTuple(
ctrl_init_state,
interface_init_state,
init_memory_BxMxA,
read_vector_BxM)
def masked_accuracy(self, logits, targets, mask):
""" Calculate accuracy equal to mean difference between outputs and targets.
WARNING: Applies mask (from aux_tuple) to both logits and targets!
:param logits: Logits being output by the model. [batch, sequence, element_size]
:param targets: LongTensor targets [batch, sequence, element_size]
:param mask: ByteTensor mask [batch, sequence]
"""
# calculate the accuracy per bit in the sequences
acc_per = 1 - torch.abs(torch.round(F.sigmoid(logits)) - targets)
mask_float = mask.type(AppState().dtype)
if len(mask.shape) < len(logits.shape):
mask_float = mask_float.unsqueeze(-1)
# The mask lacks the last dimension of the targets so needs to be
# scaled up
size = mask.nonzero().numel() / len(mask.shape) * logits.shape[-1]
masked_acc_per = mask_float * acc_per
accuracy = masked_acc_per.sum().item() / size
return accuracy
def forward(self, encoded_image, encoded_question):
"""
Apply stacked attention.
:param encoded_image: output of the image encoding (CNN + FC layer), should be of shape \
[batch_size, width * height, num_channels_encoded_image]
:type encoded_image: torch.tensor
:param encoded_question: Last hidden layer of the LSTM, of shape [batch_size, question_encoding_size]
:type encoded_question: torch.tensor
:return: u: attention [batch_size, num_channels_encoded_image]
"""
for att_layer in self.san:
u, attention_prob = att_layer(encoded_image, encoded_question)
if AppState().visualize:
if self.visualize_attention is None:
self.visualize_attention = attention_prob
# Concatenate output
else:
self.visualize_attention = torch.cat(
[self.visualize_attention, attention_prob], dim=-1)
return u
def normalize(x):
"""
Normalizes the input torch tensor along the last dimension using the max of
the one norm The normalization is "fuzzy" to prevent divergences.
:param x: input of shape [batch_size, A, A1 ..An] if the input is the weight vector x'sahpe (batch_size, num_heads, memory_size)
:return: normalized x of shape [batch_size, A, A1 ..An]
"""
dtype = AppState().dtype
return x / torch.max(torch.sum(x, dim=-1, keepdim=True),
torch.Tensor([1e-12]).type(dtype))
def init_state(self, batch_size):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:returns: Initial state tuple - object of RNNStateTuple class.
"""
# Initialize LSTM hidden state [BATCH_SIZE x CTRL_HIDDEN_SIZE].
dtype = AppState().dtype
hidden_state = torch.zeros((batch_size, self.ctrl_hidden_state_size),
requires_grad=False).type(dtype)
return RNNStateTuple(hidden_state)
def init_state(self, memory_address_size, batch_size):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
dtype = AppState().dtype
self._memory_size = memory_address_size
usage = torch.zeros((batch_size, memory_address_size)).type(dtype)
return usage
def init_state(self, memory_addresses_size, batch_size):
dtype = AppState().dtype
# Initialize controller state.
tuple_ctrl_init_state = self.controller.init_state(batch_size)
# Initialize interface state.
tuple_interface_init_state = self.interface.init_state(
memory_addresses_size, batch_size)
# Initialize memory
mem_init = (torch.ones(
(batch_size, self.M, memory_addresses_size)) * 0.01).type(dtype)
return DWMCellStateTuple(tuple_ctrl_init_state,
tuple_interface_init_state, mem_init)
def exclusive_cumprod_temp(self, sorted_usage, dim=1):
"""
Applies the exclusive cumultative product (at the moment it assumes the
shape of the input)
:param sorted_usage: tensor of shape `[batch_size, memory_size]` indicating current memory usage sorted in ascending order.
:returns: Tensor of shape `[batch_size, memory_size]` that is exclusive pruduct of the sorted usage i.e. = [1, u1, u1*u2, u1*u2*u3, ....]
"""
# TODO: expand this so it works for any dim
dtype = AppState().dtype
a = torch.ones((sorted_usage.shape[0], 1)).type(dtype)
b = torch.cat((a, sorted_usage), dim=dim).type(dtype)
prod_sorted_usage = torch.cumprod(b, dim=dim)[:, :-1]
return prod_sorted_usage
def init_state(self, memory_address_size, batch_size):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
dtype = AppState().dtype
self._memory_size = memory_address_size
link = torch.ones((batch_size, self._num_writes, memory_address_size,
memory_address_size)).type(dtype) * 1e-6
precendence_weights = torch.ones(
(batch_size, self._num_writes,
memory_address_size)).type(dtype) * 1e-6
return TemporalLinkageState(link, precendence_weights)
def init_state(self, batch_size, num_memory_addresses):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:param num_memory_addresses: Number of memory addresses.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
dtype = AppState().dtype
# Add read head states - one for each read head.
read_state_tuples = []
# Initial attention weights [BATCH_SIZE x MEMORY_ADDRESSES x 1]
# Initialize attention: to address 0.
zh_attention = torch.zeros(
batch_size, num_memory_addresses, 1).type(dtype)
zh_attention[:, 0, 0] = 1
# Initialize gating: to previous attention (i.e. zero-hard).
init_gating = torch.ones(batch_size, 1, 1).type(dtype)
# Initialize shift - to zero.
init_shift = torch.zeros(
batch_size, self.interface_shift_size, 1).type(dtype)
init_shift[:, 1, 0] = 1
for i in range(self.interface_num_read_heads):
read_ht = HeadStateTuple(
zh_attention, zh_attention, init_gating, init_shift)
# Single read head tuple.
read_state_tuples.append(read_ht)
# Single write head tuple.
write_state_tuple = HeadStateTuple(
zh_attention, zh_attention, init_gating, init_shift)
# Return tuple.
interface_state = InterfaceStateTuple(
read_state_tuples, write_state_tuple)
return interface_state
def init_state(self, memory_addresses_size, batch_size):
"""
Returns 'zero' (initial) state of Interface tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:param memory_addresses_size: size of the memory
:returns: Initial state tuple - object of InterfaceStateTuple class: (head_weight_init, snapshot_weight_init)
"""
dtype = AppState().dtype
# initial attention vector
head_weight_init = torch.zeros(
(batch_size, self.num_heads, memory_addresses_size)).type(dtype)
head_weight_init[:, 0:self.num_heads, 0] = 1.0
# bookmark
snapshot_weight_init = head_weight_init
return InterfaceStateTuple(head_weight_init, snapshot_weight_init)
def init_state(self, batch_size):
"""
Initialize the state of a ``ThalNet`` module.
:param batch_size: batch size
:type batch_size: int
:return: center_state_per_module, tuple_controller_states
"""
dtype = AppState().dtype
# module state initialisation
tuple_controller_states = self.controller.init_state(batch_size)
# center state initialisation
center_state_per_module = torch.randn(
(batch_size, self.center_size_per_module)).type(dtype)
return center_state_per_module, tuple_controller_states
def masked_accuracy(self, logits, targets, mask):
"""
Calculates accuracy equal to mean number of correct predictions in a \
given batch.
.. warning::
Applies ``mask`` to both ``logits`` and ``targets``.
:param logits: Logits being output by the model. [batch, classes, sequence].
:type logits: torch.tensor.
:param targets: Targets [batch, sequence].
:type targets: torch.LongTensor
:param mask: Mask [batch, sequence].
:type mask: torch.ByteTensor
:return: accuracy value.
"""
# calculate the accuracy per bit in the sequences
acc_per = 1 - torch.abs(
torch.round(torch.nn.functional.sigmoid(logits)) - targets)
mask_float = mask.type(AppState().dtype)
if len(mask.shape) < len(logits.shape):
mask_float = mask_float.unsqueeze(-1)
# The mask lacks the last dimension of the targets so needs to be
# scaled up
size = mask.nonzero().numel() / len(mask.shape) * logits.shape[-1]
masked_acc_per = mask_float * acc_per
accuracy = masked_acc_per.sum().item() / size
return accuracy
def forward(self, logits, targets, mask):
"""
Calculates loss accounting for different numbers of output per sample.
:param logits: Logits being output by the model. [batch, classes, sequence].
:type logits: torch.tensor.
:param targets: Targets [batch, sequence].
:type targets: torch.LongTensor
:param mask: Mask [batch, sequence].
:type mask: torch.ByteTensor
:return: loss value.
"""
# Calculate the loss per element in the sequence
loss_per_element = self.loss_function(logits, targets)
# Have to convert the mask to floats to multiply by the loss
mask_float = mask.type(AppState().dtype)
# if the loss has one extra dimenison then you need an extra unit dimension
# to multiply element by element
if len(mask.shape) < len(loss_per_element.shape):
mask_float = mask_float.unsqueeze(-1)
# Set the loss per element to zero for unneeded output
masked_loss_per = mask_float * loss_per_element
# obtain the number of non-zero elements in the mask.
# nonzero() returns the indices so you have to divide by the number of
# dimensions
size = mask.nonzero().numel() / len(mask.shape)
# add up the loss scaling by only the needed outputs
loss = torch.sum(masked_loss_per) / size
return loss
def init_state(self, batch_size, num_memory_addresses,
final_encoder_attention_BxAx1):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:param num_memory_addresses: Number of memory addresses.
:param final_encoder_attention_BxAx1: final attention of the encoder [BATCH_SIZE x MEMORY_ADDRESSES x 1]
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
# Get dtype.
dtype = AppState().dtype
# Initial attention weights [BATCH_SIZE x MEMORY_ADDRESSES x 1]
# Zero-hard attention.
zh_attention = torch.zeros(batch_size, num_memory_addresses,
1).type(dtype)
# Initialize as "hard attention on 0 address"
zh_attention[:, 0, 0] = 1
# Gating [BATCH x 3 x 1]
init_gating = torch.zeros(batch_size, 3, 1).type(dtype)
init_gating[:, 0, 0] = 1 # Initialize as "prev attention"
# Shift [BATCH x SHIFT_SIZE x 1]
init_shift = torch.zeros(batch_size, self.interface_shift_size,
1).type(dtype)
init_shift[:, 1, 0] = 1 # Initialize as "0 shift".
# Remember zero-hard attention.
self.zero_hard_attention_BxAx1 = zh_attention
# Remember final attention of encoder.
self.final_encoder_attention_BxAx1 = final_encoder_attention_BxAx1
# Return tuple.
return MASInterfaceStateTuple(zh_attention,
self.final_encoder_attention_BxAx1,
init_gating, init_shift)
def init_state(self, batch_size, num_memory_addresses):
"""
Returns 'zero' (initial) state tuple.
:param batch_size: Size of the batch in given iteraction/epoch.
:param num_memory_addresses: Number of memory addresses.
:returns: Initial state tuple - object of InterfaceStateTuple class.
"""
# Get dtype.
dtype = AppState().dtype
# Zero-hard attention.
zh_attention = torch.zeros(batch_size, num_memory_addresses,
1).type(dtype)
zh_attention[:, 0, 0] = 1
# Init gating.
init_shift = torch.zeros(batch_size, self.interface_shift_size,
1).type(dtype)
init_shift[:, 1, 0] = 1
# Return tuple.
return MAEInterfaceStateTuple(zh_attention, init_shift)
# print("--- %s seconds ---" % (time.time() - start_time))
# Plot figure and list of frames.
self.plotWindow.update(fig, frames)
return self.plotWindow.is_closed
if __name__ == "__main__":
# Set logging level.
logger = logging.getLogger('NTM-Module')
logging.basicConfig(level=logging.DEBUG)
# Set visualization.
from miprometheus.utils.app_state import AppState
AppState().visualize = True
# "Loaded parameters".
from miprometheus.utils.param_interface import ParamInterface
params = ParamInterface()
params.add_default_params({
# controller parameters
'controller': {
'name': 'GRUController',
'hidden_state_size': 5,
'num_layers': 1,
'non_linearity': 'none',
'output_size': 5
},
# interface parameters
'interface': {
def __init__(self, name="GridWorker", use_gpu=False):
"""
Base constructor for all grid workers:
- Initializes the AppState singleton:
>>> self.app_state = AppState()
- Initializes the Parameter Registry:
>>> self.params = ParamInterface()
- Defines the logger:
>>> self.logger = logging.getLogger(name=self.name)
- Creates parser and adds default worker command line arguments (you can display them with ``--h``).
:param name: Name of the worker (DEFAULT: "GridWorker").
:type name: str
:param use_gpu: Indicates whether the worker should use GPU or not. Value coming from the subclasses \
(e.g. ``GridTrainerCPU`` vs ``GridTrainerGPU``) (DEFAULT: False).
:type use_gpu: bool
"""
# Call base constructor.
super(GridWorker, self).__init__()
# Set worker name.
self.name = name
# Initialize the application state singleton.
self.app_state = AppState()
self.app_state.use_CUDA = use_gpu
# Initialize parameter interface/registry.
self.params = ParamInterface()
# Load the default logger configuration.
logger_config = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format':
'[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'
}
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'INFO',
'formatter': 'simple',
'stream': 'ext://sys.stdout'
}
},
'root': {
'level': 'DEBUG',
'handlers': ['console']
}
}
logging.config.dictConfig(logger_config)
# Create the Logger, set its label and logging level.
self.logger = logging.getLogger(name=self.name)
# Create parser with a list of runtime arguments.
self.parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
# Add arguments to the specific parser.
# These arguments will be shared by all grid workers.
self.parser.add_argument(
'--outdir',
dest='outdir',
type=str,
default="./experiments",
help=
'Path to the global output directory where the experiments folders '
'will be / are stored. Affects all grid experiments.'
' (DEFAULT: ./experiments)')
self.parser.add_argument(
'--savetag',
dest='savetag',
type=str,
default='',
help='Additional tag for the global output directory.')
self.parser.add_argument(
'--ll',
action='store',
dest='log_level',
type=str,
default='INFO',
choices=[
'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'
],
help="Log level for the experiments. (Default: INFO)")
self.parser.add_argument(
'--li',
dest='logging_interval',
default=100,
type=int,
help=
'Statistics logging interval. Will impact logging to the logger and exporting to '
'TensorBoard for the experiments. Do not affect the grid worker. '
'Writing to the csv file is not impacted (interval of 1).'
' (Default: 100, i.e. logs every 100 episodes).')
self.parser.add_argument(
'--agree',
dest='confirm',
action='store_true',
help='Request user confirmation before starting the grid experiment.'
' (Default: False)')
plt.xlabel('Q: {} )'.format(question))
print(type(image))
plt.imshow(image.permute(1, 2, 0),
interpolation='nearest', aspect='auto')
# Plot!
plt.show()
if __name__ == '__main__':
""" Tests MultiHopsStackedAttentionNetwork on ShapeColorQuery"""
# "Loaded parameters".
from miprometheus.utils.param_interface import ParamInterface
from miprometheus.utils.app_state import AppState
app_state = AppState()
app_state.visualize = False
from miprometheus.problems import ShapeColorQuery
problem_params = ParamInterface()
problem_params.add_config_params({'data_folder': '~/data/shape-color-query/',
'split': 'train',
'regenerate': False,
'dataset_size': 10000,
'img_size': 128})
# create problem
shapecolorquery = ShapeColorQuery(problem_params)
batch_size = 64
# wrap DataLoader on top of this Dataset subclass
def __init__(self, params_, name_='Problem'):
"""
Initializes problem object.
:param params_: Dictionary of parameters (read from the configuration ``.yaml`` file).
:type params_: :py:class:`miprometheus.utils.ParamInterface`
:param name_: Problem name (DEFAULT: 'Problem').
:type name_: str
This constructor:
- stores a pointer to ``params``:
>>> self.params = params_
- sets a problem name:
>>> self.name = name_
- sets a default loss function:
>>> self.loss_function = None
- initializes the size of the dataset:
>>> self.length = None
- initializes the logger.
>>> self.logger = logging.Logger(self.name)
- initializes the data definitions: this is used for defining the ``DataDict`` keys.
.. note::
This dict contains information about the DataDict produced by the current problem class.
This object will be used during handshaking between the model and the problem class to ensure that the model
can accept the batches produced by the problem.
This dict should at least contains the `targets` field:
>>> self.data_definitions = {'targets': {'size': [-1, 1], 'type': [torch.Tensor]}}
- initializes the default values: this is used to pass missing parameters values to the model.
.. note::
It is likely to encounter a case where the model needs a parameter value only known when the problem has been
instantiated, like the size of a vocabulary set or the number of marker bits.
The user can fill in those values in this dict, which will be passed to the model in its `__init__` . The
model will then be able to fill it its missing parameters values, either from params or this dict.
>>> self.default_values = {}
- sets the access to ``AppState``: for dtype, visualization flag etc.
>>> self.app_state = AppState()
"""
# Store pointer to params.
self.params = params_
# Problem name.
self.name = name_
# Empty curriculum learning params - for now.
self.curriculum_params = {}
# Set default loss function.
self.loss_function = None
# Size of the dataset
self.length = None
# Initialize the logger.
self.logger = logging.getLogger(self.name)
# data_definitions: this is used for defining the DataDict keys.
# This dict contains information about the DataDict produced by the current problem class.
# This object will be used during handshaking between the model and the problem class to ensure that the model
# can accept the batches produced by the problem.
self.data_definitions = {}
# default_values: this is used to pass missing parameters values to the model.
# It is likely to encounter a case where the model needs a parameter value only known when the problem has been
# instantiated, like the size of a vocabulary set or the number of marker bits.
# The user can fill in those values in this dict, which will be passed to the model in its `__init__` . The
# model will then be able to fill it its missing parameters values, either from params or this dict.
self.default_values = {}
# Get access to AppState: for dtype, visualization flag etc.
self.app_state = AppState()
x = torch.nn.functional.relu(x)
x = self.f_fc2(x)
x = torch.nn.functional.relu(x)
x = torch.nn.functional.dropout(x, p=0.5)
x = self.f_fc3(x)
return x
if __name__ == '__main__':
"""
Unit Tests for g_theta & f_phi.
"""
input_size = (24 + 2) * 2 + 13
batch_size = 64
inputs = np.random.binomial(1, 0.5, (batch_size, 3, input_size))
inputs = torch.from_numpy(inputs).type(AppState().dtype)
g_theta = PairwiseRelationNetwork(input_size=input_size)
g_outputs = g_theta(inputs)
print('g_outputs:', g_outputs.shape)
output_size = 10
f_phi = SumOfPairsAnalysisNetwork(output_size=output_size)
f_outputs = f_phi(g_outputs)
print('f_outputs:', f_outputs.shape)
def __init__(self, name, add_default_parser_args=True):
"""
Base constructor for all workers:
- Initializes the AppState singleton:
>>> self.app_state = AppState()
- Initializes the Parameter Registry:
>>> self.params = ParamInterface()
- Defines the logger:
>>> self.logger = logging.getLogger(name=self.name)
- Creates parser and adds default worker command line arguments.
:param name: Name of the worker.
:type name: str
:param add_default_parser_args: If set, adds default parser arguments (DEFAULT: True).
:type add_default_parser_args: bool
"""
# Call base constructor.
super(Worker, self).__init__()
# Set worker name.
self.name = name
# Initialize the application state singleton.
self.app_state = AppState()
# Initialize parameter interface/registry.
self.params = ParamInterface()
# Initialize logger using the configuration.
self.initialize_logger()
# Create parser with a list of runtime arguments.
self.parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
# Add arguments to the specific parser.
if add_default_parser_args:
# These arguments will be shared by all basic workers.
self.parser.add_argument(
'--config',
dest='config',
type=str,
default='',
help='Name of the configuration file(s) to be loaded. '
'If specifying more than one file, they must be separated with coma ",".'
)
self.parser.add_argument(
'--model',
type=str,
default='',
dest='model',
help='Path to the file containing the saved parameters'
' of the model to load (model checkpoint, should end with a .pt extension.)'
)
self.parser.add_argument(
'--gpu',
dest='use_gpu',
action='store_true',
help=
'The current worker will move the computations on GPU devices, if available '
'in the system. (Default: False)')
self.parser.add_argument(
'--expdir',
dest='expdir',
type=str,
default="./experiments",
help=
'Path to the directory where the experiment(s) folders are/will be stored.'
' (DEFAULT: ./experiments)')
self.parser.add_argument('--savetag',
dest='savetag',
type=str,
default='',
help='Tag for the save directory.')
self.parser.add_argument('--ll',
action='store',
dest='log_level',
type=str,
default='INFO',
choices=[
'CRITICAL', 'ERROR', 'WARNING',
'INFO', 'DEBUG', 'NOTSET'
],
help="Log level. (Default: INFO)")
self.parser.add_argument(
'--li',
dest='logging_interval',
default=100,
type=int,
help=
'Statistics logging interval. Will impact logging to the logger and '
'exporting to TensorBoard. Writing to the csv file is not impacted '
'(interval of 1).(Default: 100, i.e. logs every 100 episodes).'
)
self.parser.add_argument(
'--agree',
dest='confirm',
action='store_true',
help=
'Request user confirmation just after loading the settings, '
'before starting training. (Default: False)')
class Worker(object):
"""
Base abstract class for the workers.
All base workers should subclass it and override the relevant methods.
"""
def __init__(self, name, add_default_parser_args=True):
"""
Base constructor for all workers:
- Initializes the AppState singleton:
>>> self.app_state = AppState()
- Initializes the Parameter Registry:
>>> self.params = ParamInterface()
- Defines the logger:
>>> self.logger = logging.getLogger(name=self.name)
- Creates parser and adds default worker command line arguments.
:param name: Name of the worker.
:type name: str
:param add_default_parser_args: If set, adds default parser arguments (DEFAULT: True).
:type add_default_parser_args: bool
"""
# Call base constructor.
super(Worker, self).__init__()
# Set worker name.
self.name = name
# Initialize the application state singleton.
self.app_state = AppState()
# Initialize parameter interface/registry.
self.params = ParamInterface()
# Initialize logger using the configuration.
self.initialize_logger()
# Create parser with a list of runtime arguments.
self.parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
# Add arguments to the specific parser.
if add_default_parser_args:
# These arguments will be shared by all basic workers.
self.parser.add_argument(
'--config',
dest='config',
type=str,
default='',
help='Name of the configuration file(s) to be loaded. '
'If specifying more than one file, they must be separated with coma ",".'
)
self.parser.add_argument(
'--model',
type=str,
default='',
dest='model',
help='Path to the file containing the saved parameters'
' of the model to load (model checkpoint, should end with a .pt extension.)'
)
self.parser.add_argument(
'--gpu',
dest='use_gpu',
action='store_true',
help=
'The current worker will move the computations on GPU devices, if available '
'in the system. (Default: False)')
self.parser.add_argument(
'--expdir',
dest='expdir',
type=str,
default="./experiments",
help=
'Path to the directory where the experiment(s) folders are/will be stored.'
' (DEFAULT: ./experiments)')
self.parser.add_argument('--savetag',
dest='savetag',
type=str,
default='',
help='Tag for the save directory.')
self.parser.add_argument('--ll',
action='store',
dest='log_level',
type=str,
default='INFO',
choices=[
'CRITICAL', 'ERROR', 'WARNING',
'INFO', 'DEBUG', 'NOTSET'
],
help="Log level. (Default: INFO)")
self.parser.add_argument(
'--li',
dest='logging_interval',
default=100,
type=int,
help=
'Statistics logging interval. Will impact logging to the logger and '
'exporting to TensorBoard. Writing to the csv file is not impacted '
'(interval of 1).(Default: 100, i.e. logs every 100 episodes).'
)
self.parser.add_argument(
'--agree',
dest='confirm',
action='store_true',
help=
'Request user confirmation just after loading the settings, '
'before starting training. (Default: False)')
def initialize_logger(self):
"""
Initializes the logger, with a specific configuration:
>>> logger_config = {'version': 1,
>>> 'disable_existing_loggers': False,
>>> 'formatters': {
>>> 'simple': {
>>> 'format': '[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',
>>> 'datefmt': '%Y-%m-%d %H:%M:%S'}},
>>> 'handlers': {
>>> 'console': {
>>> 'class': 'logging.StreamHandler',
>>> 'level': 'INFO',
>>> 'formatter': 'simple',
>>> 'stream': 'ext://sys.stdout'}},
>>> 'root': {'level': 'DEBUG',
>>> 'handlers': ['console']}}
"""
# Load the default logger configuration.
logger_config = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format':
'[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'
}
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'INFO',
'formatter': 'simple',
'stream': 'ext://sys.stdout'
}
},
'root': {
'level': 'DEBUG',
'handlers': ['console']
}
}
logging.config.dictConfig(logger_config)
# Create the Logger, set its label and logging level.
self.logger = logging.getLogger(name=self.name)
def display_parsing_results(self):
"""
Displays the properly & improperly parsed arguments (if any).
"""
# Log the parsed flags.
flags_str = 'Properly parsed command line arguments: \n'
flags_str += '=' * 80 + '\n'
for arg in vars(self.flags):
flags_str += "{}= {} \n".format(arg, getattr(self.flags, arg))
flags_str += '=' * 80 + '\n'
self.logger.info(flags_str)
# Log the unparsed flags if any.
if self.unparsed:
flags_str = 'Invalid command line arguments: \n'
flags_str += '=' * 80 + '\n'
for arg in self.unparsed:
flags_str += "{} \n".format(arg)
flags_str += '=' * 80 + '\n'
self.logger.warning(flags_str)
def setup_experiment(self):
"""
Setups a specific experiment.
Base method:
- Parses command line arguments.
- Sets the 3 default sections (training / validation / test) and sets their dataloaders params.
.. note::
Child classes should override this method, but still call its parent to draw the basic functionality \
implemented here.
"""
# Parse arguments.
self.flags, self.unparsed = self.parser.parse_known_args()
# Set logger depending on the settings.
self.logger.setLevel(
getattr(logging, self.flags.log_level.upper(), None))
# add empty sections
self.params.add_default_params(
{"training": {
'terminal_conditions': {}
}})
self.params.add_default_params({"validation": {}})
self.params.add_default_params({"testing": {}})
# set a default configuration section for the DataLoaders
dataloader_config = {
'dataloader': {
'shuffle': True, # shuffle set by default.
'batch_sampler': None,
'num_workers':
0, # Do not use multiprocessing by default - for now.
'pin_memory': False,
'drop_last': False,
'timeout': 0
},
'sampler': {}, # not using sampler by default
}
self.params["training"].add_default_params(dataloader_config)
self.params["validation"].add_default_params(dataloader_config)
self.params["testing"].add_default_params(dataloader_config)
def build_problem_sampler_loader(self, params, section_name):
"""
Builds and returns the Problem class, alongside its DataLoader.
Also builds the sampler if required.
:param params: 'ParamInterface' object, referring to one of main sections (training/validation/testing).
:type params: miprometheus.utils.ParamInterface
:param section_name: name of the section that will be used by logger for display.
:return: Problem instance & DataLoader instance.
"""
# Build the problem.
problem = ProblemFactory.build(params['problem'])
# Try to build the sampler.
sampler = SamplerFactory.build(problem, params['sampler'])
if sampler is not None:
# Set shuffle to False - REQUIRED as those two are exclusive.
params['dataloader'].add_config_params({'shuffle': False})
# build the DataLoader on top of the validation problem
loader = DataLoader(
dataset=problem,
batch_size=params['problem']['batch_size'],
shuffle=params['dataloader']['shuffle'],
sampler=sampler,
batch_sampler=params['dataloader']['batch_sampler'],
num_workers=params['dataloader']['num_workers'],
collate_fn=problem.collate_fn,
pin_memory=params['dataloader']['pin_memory'],
drop_last=params['dataloader']['drop_last'],
timeout=params['dataloader']['timeout'],
worker_init_fn=problem.worker_init_fn)
# Display sizes.
self.logger.info("Problem for '{}' loaded (size: {})".format(
section_name, len(problem)))
if (sampler is not None):
self.logger.info("Sampler for '{}' created (size: {})".format(
section_name, len(sampler)))
# Return sampler - even if it is none :]
return problem, sampler, loader
def get_epoch_size(self, problem, sampler, batch_size, drop_last):
"""
Compute the number of iterations ('episodes') to run given the size of the dataset and the batch size to cover
the entire dataset once.
Takes into account whether one used sampler or not.
:param problem: Object derived from the ''Problem'' class
:param sampler: Sampler (may be None)
:param batch_size: Batch size.
:type batch_size: int
:param drop_last: If True then last batch (if incomplete) will not be counted
:type drop_last: bool
.. note::
If the last batch is incomplete we are counting it in when ``drop_last`` in ``DataLoader()`` is set to Ttrue.
.. warning::
Leaving this method 'just in case', in most cases one might simply use ''len(dataloader)''.
:return: Number of iterations to perform to go though the entire dataset once.
"""
# "Estimate" dataset size.
if (sampler is not None):
problem_size = len(sampler)
else:
problem_size = len(problem)
# If problem_size is a multiciplity of batch_size OR drop last is set.
if (problem_size % batch_size) == 0 or drop_last:
return problem_size // batch_size
else:
return (problem_size // batch_size) + 1
def export_experiment_configuration(self, log_dir, filename, user_confirm):
"""
Dumps the configuration to ``yaml`` file.
:param log_dir: Directory used to host log files (such as the collected statistics).
:type log_dir: str
:param filename: Name of the ``yaml`` file to write to.
:type filename: str
:param user_confirm: Whether to request user confirmation.
:type user_confirm: bool
"""
# -> At this point, all configuration for experiment is complete.
# Display results of parsing.
self.display_parsing_results()
# Log the resulting training configuration.
conf_str = 'Final parameter registry configuration:\n'
conf_str += '=' * 80 + '\n'
conf_str += yaml.safe_dump(self.params.to_dict(),
default_flow_style=False)
conf_str += '=' * 80 + '\n'
self.logger.info(conf_str)
# Save the resulting configuration into a .yaml settings file, under log_dir
with open(log_dir + filename, 'w') as yaml_backup_file:
yaml.dump(self.params.to_dict(),
yaml_backup_file,
default_flow_style=False)
# Ask for confirmation - optional.
if user_confirm:
try:
input('Press <Enter> to confirm and start the experiment\n')
except KeyboardInterrupt:
exit(0)
def add_statistics(self, stat_col):
"""
Adds most elementary shared statistics to ``StatisticsCollector``: episode and loss.
:param stat_col: ``StatisticsCollector``.
"""
# Add default statistics with formatting.
stat_col.add_statistic('loss', '{:12.10f}')
stat_col.add_statistic('episode', '{:06d}')
def add_aggregators(self, stat_agg):
"""
Adds basic statistical aggregators to ``StatisticsAggregator``: episode, \
episodes_aggregated and loss derivatives.
:param stat_agg: ``StatisticsAggregator``.
"""
# add 'aggregators' for the episode.
stat_agg.add_aggregator('episode', '{:06d}')
# Number of aggregated episodes.
stat_agg.add_aggregator('episodes_aggregated', '{:06d}')
# Add default statistical aggregators for the loss (indicating a formatting).
# Represents the average loss, but stying with loss for TensorBoard "variable compatibility".
stat_agg.add_aggregator('loss', '{:12.10f}')
stat_agg.add_aggregator('loss_min', '{:12.10f}')
stat_agg.add_aggregator('loss_max', '{:12.10f}')
stat_agg.add_aggregator('loss_std', '{:12.10f}')
def aggregate_statistics(self, stat_col, stat_agg):
"""
Aggregates the default statistics collected by the ``StatisticsCollector``.
.. note::
Only computes the min, max, mean, std of the loss as these are basic statistical aggregator by default.
Given that the ``StatisticsAggregator`` uses the statistics collected by the ``StatisticsCollector``, \
It should be ensured that these statistics are correctly collected (i.e. use of ``self.add_statistics()`` \
and ``collect_statistics()``).
:param stat_col: ``StatisticsCollector``
:param stat_agg: ``StatisticsAggregator``
"""
# By default, copy the last value for all variables have matching names.
# (will work well for e.g. episode or epoch)
for k, v in stat_col.items():
if k in stat_agg.aggregators:
# Copy last collected value.
stat_agg.aggregators[k] = v[-1]
# Get loss values.
loss_values = stat_col['loss']
# Calculate default aggregates.
stat_agg.aggregators['loss'] = torch.mean(torch.tensor(loss_values))
stat_agg.aggregators['loss_min'] = min(loss_values)
stat_agg.aggregators['loss_max'] = max(loss_values)
stat_agg.aggregators['loss_std'] = 0.0 if len(
loss_values) <= 1 else torch.std(torch.tensor(loss_values))
stat_agg.aggregators['episodes_aggregated'] = len(loss_values)
@abstractmethod
def run_experiment(self):
"""
Main function of the worker which executes a specific experiment.
.. note::
Abstract. Should be implemented in the subclasses.
"""
def add_file_handler_to_logger(self, logfile):
"""
Add a ``logging.FileHandler`` to the logger of the current ``Worker``.
Specifies a ``logging.Formatter``:
>>> logging.Formatter(fmt='[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',
>>> datefmt='%Y-%m-%d %H:%M:%S')
:param logfile: File used by the ``FileHandler``.
"""
# create file handler which logs even DEBUG messages
fh = logging.FileHandler(logfile)
# set logging level for this file
fh.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter(
fmt='[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
# add the handler to the logger
self.logger.addHandler(fh)
def recurrent_config_parse(self, configs: str, configs_parsed: list):
"""
Parses names of configuration files in a recursive manner, i.e. \
by looking for ``default_config`` sections and trying to load and parse those \
files one by one.
:param configs: String containing names of configuration files (with paths), separated by comas.
:type configs: str
:param configs_parsed: Configurations that were already parsed (so we won't parse them many times).
:type configs_parsed: list
:return: list of parsed configuration files.
"""
# Split and remove spaces.
configs_to_parse = configs.replace(" ", "").split(',')
# Terminal condition.
while len(configs_to_parse) > 0:
# Get config.
config = configs_to_parse.pop(0)
# Skip empty names (after lose comas).
if config == '':
continue
print("Info: Parsing the {} configuration file".format(config))
# Check if it was already loaded.
if config in configs_parsed:
print(
'Warning: Configuration file {} already parsed - skipping'.
format(config))
continue
# Check if file exists.
if not os.path.isfile(config):
print('Error: Configuration file {} does not exist'.format(
config))
exit(-1)
try:
# Open file and get parameter dictionary.
with open(config, 'r') as stream:
param_dict = yaml.safe_load(stream)
except yaml.YAMLError as e:
print(
"Error: Couldn't properly parse the {} configuration file".
format(config))
print('yaml.YAMLERROR:', e)
exit(-1)
# Remember that we loaded that config.
configs_parsed.append(config)
# Check if there are any default configs to load.
if 'default_configs' in param_dict:
# If there are - recursion!
configs_parsed = self.recurrent_config_parse(
param_dict['default_configs'], configs_parsed)
# Done, return list of loaded configs.
return configs_parsed
def recurrent_config_load(self, configs_to_load):
for config in reversed(configs_to_load):
# Load params from YAML file.
self.params.add_config_params_from_yaml(config)
print('Loaded configuration from file {}'.format(config))
def check_and_set_cuda(self, use_gpu):
"""
Enables computations on CUDA if GPU is available.
Sets the default data types.
:param use_gpu: Command line flag indicating whether use GPU/CUDA or not.
"""
# Determine if GPU/CUDA is available.
if torch.cuda.is_available():
if use_gpu:
self.app_state.convert_cuda_types()
self.logger.info(
'Running computations on GPU using CUDA enabled')
elif use_gpu:
self.logger.warning(
'GPU flag is enabled but there are no available GPU devices, using CPU instead'
)
else:
self.logger.warning('GPU flag is disabled, using CPU.')
def predict_evaluate_collect(self,
model,
problem,
data_dict,
stat_col,
episode,
epoch=None):
"""
Function that performs the following:
- passes samples through the model,
- computes loss using the problem
- collects problem and model statistics,
:param model: trainable model.
:type model: ``models.model.Model`` or a subclass
:param problem: problem generating samples.
:type problem: ``problems.problem.problem`` or a subclass
:param data_dict: contains the batch of samples to pass to the model.
:type data_dict: ``DataDict``
:param stat_col: statistics collector used for logging accuracy etc.
:type stat_col: ``StatisticsCollector``
:param episode: current episode index
:type episode: int
:param epoch: current epoch index.
:type epoch: int, optional
:return:
- logits,
- loss
"""
# Convert to CUDA.
if self.app_state.use_CUDA:
data_dict = data_dict.cuda()
# Perform forward calculation.
logits = model(data_dict)
# Evaluate loss function.
loss = problem.evaluate_loss(data_dict, logits)
# Collect "elementary" statistics - episode and loss.
if ('epoch' in stat_col) and (epoch is not None):
stat_col['epoch'] = epoch
stat_col['episode'] = episode
# Collect loss as float.
stat_col['loss'] = loss
# Collect other (potential) statistics from problem & model.
problem.collect_statistics(stat_col, data_dict, logits)
model.collect_statistics(stat_col, data_dict, logits)
# Return tuple: logits, loss.
return logits, loss
def export_statistics(self, stat_obj, tag='', export_to_log=True):
"""
Export the statistics/aggregations to logger, csv and TB.
:param stat_obj: ``StatisticsCollector`` or ``StatisticsAggregato`` object.
:param tag: Additional tag that will be added to string exported to logger, optional (DEFAULT = '').
:type tag: str
:param export_to_log: If True, exports statistics to logger (DEFAULT: True)
:type export_to_log: bool
"""
# Log to logger
if export_to_log:
self.logger.info(stat_obj.export_to_string(tag))
# Export to csv
stat_obj.export_to_csv()
# Export to TensorBoard.
stat_obj.export_to_tensorboard()
def aggregate_and_export_statistics(self,
problem,
model,
stat_col,
stat_agg,
episode,
tag='',
export_to_log=True):
"""
Aggregates the collected statistics. Exports the aggregations to logger, csv and TB. \
Empties statistics collector for the next episode.
:param model: trainable model.
:type model: ``models.model.Model`` or a subclass
:param problem: problem generating samples.
:type problem: ``problems.problem.problem`` or a subclass
:param stat_col: ``StatisticsCollector`` object.
:param stat_agg: ``StatisticsAggregator`` object.
:param tag: Additional tag that will be added to string exported to logger, optional (DEFAULT = '').
:type tag: str
:param export_to_log: If True, exports statistics to logger (DEFAULT: True)
:type export_to_log: bool
"""
# Aggregate statistics.
self.aggregate_statistics(stat_col, stat_agg)
problem.aggregate_statistics(stat_col, stat_agg)
model.aggregate_statistics(stat_col, stat_agg)
# Set episode, so the datapoint will appear in the right place in TB.
stat_agg["episode"] = episode
# Export to logger, cvs and TB.
self.export_statistics(stat_agg, tag, export_to_log)
def cycle(self, iterable):
"""
Cycle an iterator to prevent its exhaustion.
This function is used in the (online) trainer to reuse the same ``DataLoader`` for a number of episodes\
> len(dataset)/batch_size.
:param iterable: iterable.
:type iterable: iter
"""
while True:
for x in iterable:
yield x
def set_random_seeds(self, params, section_name):
"""
Set ``torch`` & ``NumPy`` random seeds from the ``ParamRegistry``: \
If one was indicated, use it, or set a random one.
:param params: Section in config/param registry that will be changed \
("training" or "testing" only will be taken into account.)
:param section_name: Name of the section (for logging purposes only).
:type section_name: str
"""
# Set the random seeds: either from the loaded configuration or a default randomly selected one.
params.add_default_params({"seed_numpy": -1})
if params["seed_numpy"] == -1:
seed = randrange(0, 2**32)
# Overwrite the config param!
params.add_config_params({"seed_numpy": seed})
self.logger.info("Setting numpy random seed in {} to: {}".format(
section_name, params["seed_numpy"]))
np.random.seed(params["seed_numpy"])
params.add_default_params({"seed_torch": -1})
if params["seed_torch"] == -1:
seed = randrange(0, 2**32)
# Overwrite the config param!
params.add_config_params({"seed_torch": seed})
self.logger.info("Setting torch random seed in {} to: {}".format(
section_name, params["seed_torch"]))
torch.manual_seed(params["seed_torch"])
torch.cuda.manual_seed_all(params["seed_torch"])
def circular_convolution(self, attention_BxAx1, shift_BxSx1,
prev_memory_BxAxC):
"""
Performs circular convoution, i.e. shitfts the attention accodring to
given shift vector (convolution mask).
:param attention_BxAx1: Current attention [BATCH_SIZE x ADDRESS_SIZE x 1]
:param shift_BxSx1: soft shift maks (convolutional kernel) [BATCH_SIZE x SHIFT_SIZE x 1]
:param prev_memory_BxAxC: tensor containing memory before update [BATCH_SIZE x MEMORY_ADDRESSES x CONTENT_BITS]
:returns: attention vector of size [BATCH_SIZE x ADDRESS_SIZE x 1]
"""
def circular_index(idx, num_addr):
"""
Calculates the index, taking into consideration the number of
addresses in memory.
:param idx: index (single element)
:param num_addr: number of addresses in memory
"""
if idx < 0:
return num_addr + idx
elif idx >= num_addr:
return idx - num_addr
else:
return idx
# Check whether inputs are already on GPU or not.
long_dtype = AppState().LongTensor
# Get number of memory addresses and batch size.
batch_size = prev_memory_BxAxC.size(0)
num_addr = prev_memory_BxAxC.size(1)
shift_size = self.interface_shift_size
#logger.debug("shift_BxSx1 {}: {}".format(shift_BxSx1, shift_BxSx1.size()))
# Create an extended list of indices indicating what elements of the
# sequence will be where.
ext_indices_tensor = torch.Tensor([
circular_index(shift, num_addr)
for shift in range(-shift_size // 2 + 1, num_addr +
shift_size // 2)
]).type(long_dtype)
#logger.debug("ext_indices {}:\n {}".format(ext_indices_tensor.size(), ext_indices_tensor))
# Use indices for creation of an extended attention vector.
ext_attention_BxEAx1 = torch.index_select(attention_BxAx1,
dim=1,
index=ext_indices_tensor)
#logger.debug("ext_attention_BxEAx1 {}:\n {}".format(ext_attention_BxEAx1.size(), ext_attention_BxEAx1))
# Transpose inputs to convolution.
ext_att_trans_Bx1xEA = torch.transpose(ext_attention_BxEAx1, 1, 2)
shift_trans_Bx1xS = torch.transpose(shift_BxSx1, 1, 2)
# Perform convolution for every batch-filter pair.
tmp_attention_list = []
for b in range(batch_size):
tmp_attention_list.append(
torch.nn.functional.conv1d(
ext_att_trans_Bx1xEA.narrow(0, b, 1),
shift_trans_Bx1xS.narrow(0, b, 1)))
# Concatenate list into a single tensor.
shifted_attention_BxAx1 = torch.transpose(
torch.cat(tmp_attention_list, dim=0), 1, 2)
#logger.debug("shifted_attention_BxAx1 {}:\n {}".format(shifted_attention_BxAx1.size(), shifted_attention_BxAx1))
return shifted_attention_BxAx1
return self.plotWindow.is_closed
if __name__ == '__main__':
dim = 512
embed_hidden = 300
max_step = 12
self_attention = True
memory_gate = True
nb_classes = 28
dropout = 0.15
from miprometheus.utils.app_state import AppState
from miprometheus.utils.param_interface import ParamInterface
from torch.utils.data import DataLoader
app_state = AppState()
from miprometheus.problems import CLEVR
problem_params = ParamInterface()
problem_params.add_config_params({
'settings': {
'data_folder': '~/Downloads/CLEVR_v1.0',
'set': 'train',
'dataset_variant': 'CLEVR'
},
'images': {
'raw_images': False,
'feature_extractor': {
'cnn_model': 'resnet101',
'num_blocks': 4
}
def __init__(self, name="Worker"):
"""
Base constructor for all workers:
- Initializes the AppState singleton:
>>> self.app_state = AppState()
- Initializes the Parameter Registry:
>>> self.params = ParamInterface()
- Defines the logger:
>>> self.logger = logging.getLogger(name=self.name)
- Creates parser and adds default worker command line arguments.
:param name: Name of the worker (DEFAULT: "Worker").
"""
# Call base constructor.
super(Worker, self).__init__()
# Set worker name.
self.name = name
# Initialize the application state singleton.
self.app_state = AppState()
# Initialize parameter interface/registry.
self.params = ParamInterface()
# Load the default logger configuration.
logger_config = {'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'}},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'INFO',
'formatter': 'simple',
'stream': 'ext://sys.stdout'}},
'root': {'level': 'DEBUG',
'handlers': ['console']}}
logging.config.dictConfig(logger_config)
# Create the Logger, set its label and logging level.
self.logger = logging.getLogger(name=self.name)
# Create parser with a list of runtime arguments.
self.parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
# Add arguments to the specific parser.
# These arguments will be shared by all basic workers.
self.parser.add_argument('--config',
dest='config',
type=str,
default='',
help='Name of the configuration file(s) to be loaded. '
'If specifying more than one file, they must be separated with coma ",".')
self.parser.add_argument('--model',
type=str,
default='',
dest='model',
help='Path to the file containing the saved parameters'
' of the model to load (model checkpoint, should end with a .pt extension.)')
self.parser.add_argument('--gpu',
dest='use_gpu',
action='store_true',
help='The current worker will move the computations on GPU devices, if available in '
'the system. (Default: False)')
self.parser.add_argument('--outdir',
dest='outdir',
type=str,
default="./experiments",
help='Path to the output directory where the experiment(s) folders will be stored.'
' (DEFAULT: ./experiments)')
self.parser.add_argument('--savetag',
dest='savetag',
type=str,
default='',
help='Tag for the save directory')
self.parser.add_argument('--ll',
action='store',
dest='log_level',
type=str,
default='INFO',
choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'],
help="Log level. (Default: INFO)")
self.parser.add_argument('--li',
dest='logging_interval',
default=100,
type=int,
help='Statistics logging interval. Will impact logging to the logger and exporting to '
'TensorBoard. Writing to the csv file is not impacted (interval of 1).'
' (Default: 100, i.e. logs every 100 episodes).')
self.parser.add_argument('--agree',
dest='confirm',
action='store_true',
help='Request user confirmation just after loading the settings, '
'before starting training. (Default: False)')
plt.imshow(image.permute(1, 2, 0),
interpolation='nearest',
aspect='auto')
# Plot!
plt.show()
exit()
if __name__ == '__main__':
""" Tests CNN_LSTM on SortOfCLEVR"""
# "Loaded parameters".
from miprometheus.utils.param_interface import ParamInterface
from miprometheus.utils.app_state import AppState
app_state = AppState()
app_state.visualize = True
from miprometheus.problems.image_text_to_class.sort_of_clevr import SortOfCLEVR
problem_params = ParamInterface()
problem_params.add_config_params({
'data_folder': '~/data/sort-of-clevr/',
'split': 'train',
'regenerate': False,
'dataset_size': 10000,
'img_size': 128
})
# create problem
sortofclevr = SortOfCLEVR(problem_params)
batch_size = 64
x = self.conv3(x)
x = self.batchNorm3(x)
x = F.relu(x)
x = self.conv4(x)
x = self.batchNorm4(x)
x = F.relu(x)
return x
if __name__ == '__main__':
"""
Unit Test for the ``ConvInputModel``.
"""
# "Image" - batch x channels x width x height
batch_size = 64
img_size = 128
input_np = np.random.binomial(1, 0.5, (batch_size, 3, img_size, img_size))
image = torch.from_numpy(input_np).type(AppState().dtype)
cnn = ConvInputModel()
feature_maps = cnn(image)
print('feature_maps:', feature_maps.shape)
print('Computed output height, width:',
cnn.get_output_shape(img_size, img_size))
