def __init__(self,
base_lr,
smoothing=0.9,
target=1e-3,
ylims=None,
backend='tb'):
subscription = Subscription(
self, ['weight_updates_weights_ratio', 'batch_started'],
tag=None,
subsample=1)
super().__init__(
[subscription],
get_default_bus(), {
'title': 'learning_rate',
'ylabel': 'Adjusted learning rate',
'ylims': ylims,
'xlabel': 'Train step'
},
backend=backend)
self._ratios = defaultdict(float)
self._smoothing = smoothing
self._target = target
self._factor = 1
self._base_lr = base_lr
self._add_publication('learning_rate', type='META')
def __init__(self,
kind,
message_bus=get_default_bus(),
tag='default',
subsample=1,
ylims=None,
backend='tb',
order=2):
if not isinstance(kind, str):
raise ValueError('NormSubscriber only accepts 1 kind')
title = f'{kind}_norm{order}'
ylabel = f'L{order} Norm'
xlabel = 'Train step'
subscription = Subscription(self, [kind], tag=tag, subsample=subsample)
super().__init__([subscription],
message_bus, {
'title': title,
'ylabel': ylabel,
'ylims': ylims,
'xlabel': xlabel
},
backend=backend)
self._order = order
self._add_publication(f'{kind}_norm{order}', type='DATA')
def __init__(self, kind, message_bus=get_default_bus(), tag='default', subsample=1, ylims=None,
backend='tb'):
'''
Parameters
----------
kind : str
Message kind to compute condition number norm on. Not sure if it makes sense
for non-2d matrices, which have to be reshaped to 2-d
non-matrix type.
For other parameters, see :class:`~ikkuna.export.subscriber.PlotSubscriber`
'''
if not isinstance(kind, str):
raise ValueError(f'{self.__class__.__name__} only accepts 1 kind')
subscription = Subscription(self, [kind], tag=tag, subsample=subsample)
title = f'{kind}_condition_number'
xlabel = 'Step'
ylabel = 'Condition number'
super().__init__([subscription],
message_bus,
{'title': title,
'xlabel': xlabel,
'ylims': ylims,
'ylabel': ylabel},
backend=backend)
self._add_publication(f'{kind}_condition_number', type='DATA')
self.u = dict()
self.u_inv = dict()
def __init__(self,
dataset_meta,
n,
forward_fn,
freeze_at=10,
batch_size=256,
message_bus=get_default_bus(),
tag='default',
subsample=1,
ylims=None,
backend='tb'):
'''
Parameters
----------
dataset_meta : ikkuna.utils.DatasetMeta
Dataset to load data from for retrieving activations
n : int
Number of datapoints to randomly sample
freeze_at : float
Similarity threshold after which a layer is frozen. Values >= 1 will turn
off freezing
batch_size : int
Batch size to use for forward passes. Can be set to some value if the entire
data at once would be too larger. Otherwise use :class:`SVCCASubscriber`
'''
self._forward_fn = forward_fn
self._previous_acts = ChunkedDict(n)
self._current_acts = ChunkedDict(n)
indices = np.random.randint(0, dataset_meta.size, size=n)
dataset = Subset(dataset_meta.dataset, indices)
self._loader = DataLoader(dataset,
batch_size=batch_size,
shuffle=False,
pin_memory=True)
# cache tensors so we don't repeatedly deserialize and copy
self._input_cache = []
self._freeze_at = freeze_at if isinstance(freeze_at, float) else 10.
self._ignore_modules = set()
title = f'self_similarity'
ylabel = 'Similarity'
xlabel = 'Train step'
subscription1 = Subscription(self, ['batch_finished'],
tag=tag,
subsample=subsample)
subscription2 = Subscription(self, ['activations'],
tag='svcca_testing',
subsample=1)
super().__init__([subscription1, subscription2],
message_bus, {
'title': title,
'ylabel': ylabel,
'ylims': ylims,
'xlabel': xlabel
},
backend=backend)
self._add_publication(f'self_similarity', type='DATA')
def __init__(self,
lr,
beta1,
beta2,
eps,
message_bus=get_default_bus(),
tag=None,
subsample=40,
ylims=None,
backend='tb'):
title = 'gradient_moments'
ylabel = 'Gradient Moments'
xlabel = 'Train step'
subscription = Subscription(self, ['weight_gradients'], tag, subsample)
super().__init__([subscription],
message_bus, {
'title': title,
'ylabel': ylabel,
'ylims': ylims,
'xlabel': xlabel
},
backend=backend)
self._lr = lr
self._beta1 = beta1
self._beta2 = beta2
self._eps = eps
self._means = dict()
self._vars = dict()
for pub_name in {
'biased_grad_mean_estimate_mean',
'biased_grad_mean_estimate_median',
'biased_grad_mean_estimate_var',
'biased_grad_var_estimate_mean',
'biased_grad_var_estimate_median',
'biased_grad_var_estimate_var',
'biased_grad_mean_estimate_norm',
'biased_grad_var_estimate_norm',
'grad_mean_estimate_mean',
'grad_mean_estimate_median',
'grad_mean_estimate_var',
'grad_var_estimate_mean',
'grad_var_estimate_median',
'grad_var_estimate_var',
'grad_mean_estimate_norm',
'grad_var_estimate_norm',
'effective_lr_mean',
'effective_lr_median',
'effective_lr_var',
'effective_lr_norm',
}:
self._add_publication(pub_name, type='DATA')
def __init__(self, message_bus=get_default_bus(), tag='default', subsample=1, ylims=None,
backend='tb'):
title = 'Loss'
ylabel = 'loss'
xlabel = 'Train step'
subscription = Subscription(self, ['loss'], tag=tag, subsample=subsample)
super().__init__([subscription], message_bus,
{'title': title,
'ylabel': ylabel,
'ylims': ylims,
'xlabel': xlabel},
backend=backend)
def __init__(self,
dataset_meta,
forward_fn,
batch_size,
message_bus=get_default_bus(),
frequency=100,
ylims=None,
tag='default',
subsample=1,
backend='tb'):
'''
Parameters
----------
dataset_meta : ikkuna.utils.DatasetMeta
Test dataset
forward_fn : function
Bound version of :meth:`torch.nn.Module.forward()`. This could be obtained
from the :attr:`train.Trainer.model` property.
batch_size : int
Batch size to use for pushing the test data through the model. This doesn't
have to be the training batch size, but must be selected so that the forward
pass fits into memory.
frequency : int
Inverse of the frequency with which to compute the accuracy
'''
kinds = ['batch_finished']
title = f'test_accuracy'
ylabel = 'Accuracy'
xlabel = 'Train step'
subscription = Subscription(self, kinds, tag=tag, subsample=subsample)
super().__init__(
[subscription],
message_bus, {
'title': title,
'ylabel': ylabel,
'ylims': ylims,
'xlabel': xlabel,
'redraw_interval': 1
},
backend=backend)
self._dataset_meta = dataset_meta
self._data_loader = DataLoader(dataset_meta.dataset,
batch_size=batch_size,
shuffle=False,
pin_memory=True)
self._frequency = frequency
self._forward_fn = forward_fn
self._add_publication('test_accuracy', type='META')
def __init__(self,
kinds,
callback,
message_bus=get_default_bus(),
tag='default',
subsample=1):
'''
Parameters
----------
subscription : Subscription
message_bus : ikkuna.export.messages.MessageBus
callback : function
A function that accepts as many parameters as the ``Subscription`` delivers
messages at once.
'''
subscription = SynchronizedSubscription(self, kinds, tag, subsample)
super().__init__([subscription], message_bus)
self._callback = callback
def __init__(self, forward_fn, loss_fn, data_loader, batch_size, frequency=1, num_eig=1,
power_steps=20, tag='default', message_bus=get_default_bus(), ylims=None,
backend='tb'):
'''
Parameters
----------
forward_fn : function
Function to obtain predictions. You probably want to pass the model's
``forward()`` routine here
loss_fn : torch.nn.Module
Loss function (such as :class:`torch.nn.CrossEntropyLoss`)
data_loader : torch.utils.data.DataLoader
Loader for the dataset to compute gradients over
batch_finished : int
Number of samples to compute gradients for in one step of power iteration
More should lead to a better estimate
frequency : int
How often to compute the eigenvalues (after every nth batch)
num_eig : int
Number of top eigenvalues to compute
power_iter_steps : int
Number of steps in the power iteration for computing a singular value.
The total number of batches read is then ``power_iter_steps * num_eig``
More steps should lead to a better estimate.
'''
title = f'Top hessian Eigenvalues'
ylabel = 'tbd'
xlabel = 'Train step'
subscription = Subscription(self, ['batch_finished', 'activations'], tag=tag)
super().__init__([subscription],
message_bus,
{'title': title,
'ylabel': ylabel,
'ylims': ylims,
'xlabel': xlabel},
backend=backend)
self._forward_fn = forward_fn
self._loss_fn = loss_fn
self._power_steps = power_steps
self._dataloader = data_loader
self._parameters = set()
self._num_eig = num_eig
self._frequency = frequency
def __init__(self,
kind,
message_bus=get_default_bus(),
tag='default',
subsample=1,
backend='tb'):
if not isinstance(kind, str):
raise ValueError('HistogramSubscriber only accepts 1 kind')
subscription = Subscription(self, [kind], tag=tag, subsample=subsample)
title = f'{kind}_histogram'
ylabel = 'Frequency'
super().__init__([subscription],
message_bus, {
'title': title,
'ylabel': ylabel
},
backend=backend)
def __init__(self, kind, message_bus=get_default_bus(), tag='default', subsample=1, ylims=None,
backend='tb'):
if not isinstance(kind, str):
raise ValueError('MessageMeanSubscriber only accepts 1 kind')
title = f'message_means'
ylabel = 'Mean'
xlabel = 'Train step'
subscription = Subscription(self, [kind, 'batch_finished'], tag=tag, subsample=subsample)
super().__init__([subscription], message_bus,
{'title': title,
'ylabel': ylabel,
'ylims': ylims,
'xlabel': xlabel},
backend=backend)
self._add_publication(f'{kind}_message_mean', type='META')
# kind -> list
self._buffer = defaultdict(list)
def __init__(self, message_bus=get_default_bus(), tag='default', subsample=1, ylims=None,
backend='tb'):
'''
For parameters see :class:`~ikkuna.export.subscriber.PlotSubscriber`
'''
subscription = SynchronizedSubscription(self, ['network_output', 'input_labels'], tag=tag,
subsample=subsample)
title = f'train_batch_accuracy'
xlabel = 'Step'
ylabel = 'Accuracy'
super().__init__([subscription],
message_bus,
{'title': title,
'xlabel': xlabel,
'ylims': ylims,
'ylabel': ylabel},
backend=backend)
self._add_publication(f'train_accuracy', type='META')
def __init__(self,
depth,
module_filter=None,
message_bus=get_default_bus()):
self._modules = {}
self._weight_cache = {}
self._bias_cache = {}
self._model = None
self._epoch = 0
# for gradient and activation to have the same step number, we need to increase it before
# propagation or after backpropagation. but we don't know when the backprop finishes, while
# we do know when the forward prop starts. So we step before and thus initialize the
# counters with -1 to effectively start at 0
self._train_step = -1
self._global_step = -1
self._is_training = True
self._depth = depth
self._frozen = set()
self._module_filter = module_filter
self._msg_bus = message_bus
self._current_publish_tag = 'default'
self._epoch_started_marker = False
def __init__(self,
kind,
message_bus=get_default_bus(),
tag='default',
subsample=1,
ylims=None,
backend='tb'):
'''
Parameters
----------
kind : str
Message kind to compute spectral norm on. Doesn't make sense with kinds of
non-matrix type.
For other parameters, see :class:`~ikkuna.export.subscriber.PlotSubscriber`
'''
if not isinstance(kind, str):
raise ValueError('SpectralNormSubscriber only accepts 1 kind')
subscription = Subscription(self, [kind], tag, subsample)
title = f'{kind}_spectral_norm'
xlabel = 'Step'
ylabel = 'Spectral norm'
super().__init__([subscription],
message_bus, {
'title': title,
'xlabel': xlabel,
'ylims': ylims,
'ylabel': ylabel
},
backend=backend)
self.u = dict()
self._add_publication(f'{kind}_spectral_norm', type='DATA')
def __init__(self, experiment, kinds):
self._experiment = experiment
subscription = Subscription(self, kinds)
super().__init__([subscription], get_default_bus())