def short_time_energy(signal_, window_size, stride, center=True):
"""Calculate short time energy
:param center: If `True`, the signal `y` is padded so that frame
`D[:, t]` is centered at `y[t * hop_length]`.
If `False`, then `D[:, t]` begins at `y[t * hop_length]`
"""
# Sanity check
assert isinstance(signal_, np.ndarray)
checker.check_positive_integer(window_size)
checker.check_positive_integer(stride)
if center:
signal_ = np.pad(signal_, window_size // 2, mode='reflect')
# Reshape signal
signal_ = np.reshape(signal_, newshape=(-1, ))
# Form causal matrix
frames = []
cursor = 0
while True:
frames.append(signal_[cursor:cursor + window_size])
# Move cursor forward
cursor += stride
if cursor + window_size > signal_.size: break
# Calculate energy
stack = np.stack(frames, axis=0)
energy = np.multiply(stack, stack)
energy = np.sum(energy, axis=1)
return energy
def initialize(self, input_size, batches_per_epoch):
checker.check_positive_integer(input_size)
checker.check_positive_integer(batches_per_epoch)
self.properties[self.INPUT_SIZE] = input_size
self.properties[self.BATCHES_PER_EPOCH] = batches_per_epoch
self.properties[self.READY] = True
def _get_dynamic_round_len(act_lens, num_steps, training):
"""
not train
x x x x x|x x x x x|x x x/x x:x x x x x x x
x x x x x|x x x x x|x x x/ :
x x x x x|x x x x x|x x x/x :
x x x x x|x x x x x|x x x/x x:x x x x x
train
"""
assert isinstance(act_lens, (np.ndarray, list)) and len(act_lens) > 0
checker.check_positive_integer(num_steps)
counter = 0
while len(act_lens) > 0:
# Find the shortest sequence
sl = min(act_lens)
assert sl > 0
# Calculate iterations (IMPORTANT). Note that during training act_len
# .. does not help to avoid inappropriate gradient flow thus sequences
# .. have to be truncated
n = int(np.ceil(sl / num_steps))
counter += n
# Update act_lens list
L = sl if training else n * num_steps
act_lens = [al for al in [al - L for al in act_lens] if al > 0]
return counter
def get_data_batches(self,
data_set,
batch_size,
num_steps=None,
shuffle=False):
""" Get batch generator.
:param data_set: an instance of DataSet or BigData from which data batches
will be extracted
:param batch_size: if is None, default value will be assigned according to
the input type of this model
:param num_steps: step number for RNN data batches
:param shuffle: whether to shuffle
:return: a generator or a list
"""
# Data set must be an instance of DataSet or BigData
assert isinstance(data_set, (DataSet, BigData))
if self.input_type is InputTypes.BATCH:
# If model's input type is normal batch, num_steps will be ignored
# If batch size is not specified and data is a DataSet, feed it all at
# once into model
if batch_size is None and isinstance(data_set, DataSet):
return [data_set.stack]
checker.check_positive_integer(batch_size)
data_batches = data_set.gen_batches(batch_size, shuffle=shuffle)
elif self.input_type is InputTypes.RNN_BATCH:
if batch_size is None: batch_size = 1
if num_steps is None: num_steps = -1
checker.check_positive_integer(batch_size)
checker.check_type(num_steps, int)
data_batches = data_set.gen_rnn_batches(batch_size, num_steps,
shuffle)
else:
raise ValueError('!! Can not resolve input type of this model')
return data_batches
def __init__(
self,
num_filter_list,
num_classes,
kernel_initializer='glorot_uniform',
left_repeats=2,
right_repeats=2,
activation='relu',
dropout_rate=0.5,
name='unet',
level=1,
**kwargs):
# Sanity check
assert isinstance(num_filter_list, (tuple, list)) and num_filter_list
# Call parent's constructor
# TODO: the level logic is not elegant
super().__init__(name, level=level, **kwargs)
# Specific attributes
self.num_filter_list = num_filter_list
self.num_classes = checker.check_positive_integer(num_classes)
# self.kernel_sizes = kernel_sizes
self.kernel_initializer = kernel_initializer
self.activation = activation
self.dropout_rate = checker.check_type(dropout_rate, float)
self.left_repeats = checker.check_positive_integer(left_repeats)
self.right_repeats = checker.check_positive_integer(right_repeats)
# Add layers
self._add_layers()
def _get_file_name(cls, size, N, T, var_x, add_noise, var_y):
checker.check_positive_integer(N)
checker.check_positive_integer(T)
file_name = '{}_{}_N{}T{}_vx{}_{}'.format(
'TSP', size, N, T, var_x, 'noisy' if add_noise else 'noise-free')
if add_noise: file_name += '_vy{}'.format(var_y)
return file_name
def fc_lstm(th):
assert isinstance(th, Config)
th.mark = 'fc_lstm_' + th.mark
# Initiate a model
model = Classifier(mark=th.mark, net_type=Recurrent)
# Add input layer
model.add(Input(sample_shape=th.input_shape))
# Add fc layers
for dim in th.fc_dims:
checker.check_positive_integer(dim)
model.add(Linear(output_dim=dim))
# model.add(BatchNorm())
model.add(Activation('relu'))
# Add lstm cells
for dim in th.rc_dims:
model.add(BasicLSTMCell(state_size=dim))
# Add output layer
# model.add(Linear(output_dim=th.output_dim))
# Build model
optimizer = tf.train.AdamOptimizer(th.learning_rate)
model.build(optimizer)
return model
def __init__(self,
num_neurons,
group_size,
head_size=-1,
activation=None,
use_bias=True,
weight_initializer='xavier_normal',
bias_initializer='zeros',
**kwargs):
"""
Softmax over groups applied to neurons.
Case 1: head_size < 0: does not use extra neurons
Case 2: head_size = 0: use extra neurons without a head
Case 3: head_size > 0: use extra neurons with a head
"""
# Call parent's constructor
super().__init__(activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
# Specific attributes
self._num_neurons = checker.check_positive_integer(num_neurons)
self._group_size = checker.check_positive_integer(group_size)
self._head_size = checker.check_type(head_size, int)
# Developer options
options = th.developer_options
def _gen_rnn_batches(self, x, y, batch_size, num_steps):
checker.check_positive_integer(batch_size, 'batch size')
checker.check_type(num_steps, int)
# Get batch partitions
data_x, L = self._get_batch_partition(x, batch_size)
if y is not None:
if len(x) == len(y):
data_y, Ly = self._get_batch_partition(y, batch_size)
assert L == Ly
else:
assert len(y) == 1
data_y = y
# Chop data further
if num_steps < 0: num_steps = L
round_len = int(np.ceil(L / num_steps))
for i in range(round_len):
batch_x = data_x[:, i * num_steps:min((i + 1) * num_steps, L)]
batch_y = None
if y is not None:
if len(x) == len(y):
batch_y = data_y[:,
i * num_steps:min((i + 1) * num_steps, L)]
else:
assert isinstance(y, np.ndarray)
batch_y = np.tile(y,
[batch_x.shape[0], batch_x.shape[1], 1])
batch = DataSet(batch_x, batch_y, in_rnn_format=True)
# State should be reset at the beginning of a sequence
if i == 0: batch.should_reset_state = True
batch.name = self.name + '_{}'.format(i + 1)
yield batch
def get_round_length(self, batch_size, num_steps=None):
"""Get round length for training
:param batch_size: Batch size. For irregular sequences, this value should
be set to 1.
:param num_steps: Step number. If provided, round length will be calculated
for RNN model
:return: Round length for training
"""
# Make sure features exist
self._check_feature()
checker.check_positive_integer(batch_size, 'batch_size')
if num_steps is None:
# :: For feed-forward models
return int(np.ceil(self.stack.size / batch_size))
else:
# :: For recurrent models
checker.check_type(num_steps, int)
if self.is_regular_array: arrays = [self.features]
elif self.parallel_on:
return self._get_pe_round_length(batch_size, num_steps)
else:
arrays = self.features
len_f = lambda x: x if self.len_f is None else self.len_f
if num_steps < 0: return len(arrays)
else:
return int(
sum([
np.ceil(len_f(len(array)) // batch_size / num_steps)
for array in arrays
]))
def __init__(self,
fast_size,
fast_layers,
slow_size,
hyper_kernel,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
input_dropout=0.0,
output_dropout=0.0,
forget_bias=0,
**kwargs):
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
self.kernel_key = checker.check_type(hyper_kernel, str)
# Specific attributes
self._fast_size = checker.check_positive_integer(fast_size)
self._fast_layers = checker.check_positive_integer(fast_layers)
self._slow_size = checker.check_positive_integer(slow_size)
self._hyper_kernel = self._get_hyper_kernel(hyper_kernel,
do=th.rec_dropout,
forget_bias=forget_bias)
self._input_do = checker.check_type(input_dropout, float)
self._output_do = checker.check_type(output_dropout, float)
def __init__(
self,
num_neurons,
group_size,
axis=0,
activation=None,
use_bias=True,
weight_initializer='xavier_normal',
bias_initializer='zeros',
**kwargs):
"""
axis: 0 or 1.
axis = 0: Partition input neurons.
Each output neuron uses only 1 input activation in each group.
axis = 1: Partition output neurons.
Each input neuron passes value to only 1 output neuron in each
group.
"""
# Call parent's constructor
super().__init__(activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
# Specific attributes
assert axis in (0, 1)
self._axis = axis
self._num_neurons = checker.check_positive_integer(num_neurons)
self._group_size = checker.check_positive_integer(group_size)
def __init__(
self,
state_size,
tape_length,
spatial_size=None,
spatial_heads=1,
temporal_heads=1,
spatial_dropout=0.0,
temporal_dropout=0.0,
activation='tanh',
weight_initializer='xavier_uniform',
use_bias=True,
bias_initializer='zeros',
rec_dropout=0.0,
**kwargs):
"""
"""
# Call parent's constructor
super().__init__(
length=tape_length, depth=state_size, num_tapes=1,
activation=activation, weight_initializer=weight_initializer,
use_bias=use_bias, bias_initializer=bias_initializer,
dropout_rate=rec_dropout, **kwargs)
# Specific attributes
self._state_size = state_size
self._spatial_size = spatial_size
self._spatial_heads = checker.check_positive_integer(spatial_heads)
self._temporal_heads = checker.check_positive_integer(temporal_heads)
self._spatial_dropout = spatial_dropout
self._temporal_dropout = temporal_dropout
def pop_subset(self, size, length_prone='random', name='subset'):
"""Pop sub data set containing 'size' samples for each class"""
# Check input parameters
checker.check_positive_integer(size)
assert length_prone in ('short', 'long', 'random')
if size > self.smallest_population:
raise ValueError(
'!! size should be less than the smallest population '
'of this data set: {}'.format(self.smallest_population))
# Check classes
assert len(self.groups) == self.NUM_CLASSES
# Fill subset
subset = self._copy_empty_container()
subset.name = name
for label, wavs in self.groups.items():
assert isinstance(wavs, list)
# Initialize file list for label class
subset.groups[label] = []
# Pop data from self
for _ in range(size):
if length_prone == 'short': i = -1
elif length_prone == 'long': i = 0
else: i = np.random.randint(len(wavs))
# Pop from self
file_name = wavs.pop(i)
subset.groups[label].append(file_name)
tfd_file_name = file_name + '.tfds'
subset.files[tfd_file_name] = self.files.pop(tfd_file_name)
# Check length
assert len(self.files) == sum(list(self.group_population.values()))
assert len(subset.files) == sum(list(subset.group_population.values()))
return subset
def _gen_parallel_batches(self, batch_size, num_steps, shuffle):
"""A beta method used only for RNN training"""
# Sanity check
features, targets = self.features, self.targets
assert isinstance(features, (tuple, list))
assert isinstance(targets, (tuple, list))
assert len(features) == len(targets)
checker.check_positive_integer(batch_size)
assert isinstance(num_steps, int)
assert isinstance(shuffle, bool)
# Initialize parallel engine
pe = ParallelEngine(batch_size)
cursor, num_sequences = 0, len(self.features)
round_len = self._get_pe_round_length(batch_size, num_steps)
# Start loop
global_reset = True
counter = 0
while True:
reset_indices = pe.inactive_indices
reset_values = []
# Load new sequence to engine if necessary
while not pe.is_ready:
if shuffle or cursor < num_sequences:
index = self._rand_indices() if shuffle else cursor
x, y = self.features[index], self.targets[index]
if self.init_f is not None: x, y = self.init_f(x, y)
cursor += 1
reset_values.append(0)
else:
x, y = None, None
reset_values.append(None)
pe.set_sequence(x, y)
if pe.flameout: break
# Get features and targets and wrap them into a DataSet
x, y = pe.emit(num_steps)
data_batch = DataSet(x, y)
if len(reset_indices) > 0:
if global_reset:
data_batch.should_reset_state = True
global_reset = False
assert len(reset_indices) == len(reset_values)
data_batch.reset_batch_indices = reset_indices
data_batch.reset_values = (reset_values if len(
[val
for val in reset_values if val is None]) > 0 else None)
# Yield batch
yield data_batch
counter += 1
if counter >= round_len: break
# Check round length
assert counter == round_len
def gen_batches(self, batch_size, **kwargs):
assert self.is_ready
checker.check_positive_integer(self.batches_per_epoch)
for i in range(self.batches_per_epoch):
matrix, labels = self._random_signal_matrix(
batch_size, self.input_size)
batch = DataSet(matrix, labels)
batch.name = 'gpat_{}of{}'.format(i + 1, self.batches_per_epoch)
yield batch
def causal_matrix(self, memory_depth, skip_head=False):
checker.check_positive_integer(memory_depth)
assert isinstance(self, np.ndarray)
if memory_depth == 1: return np.reshape(self, (-1, 1))
N, D = self.size, memory_depth
x = np.append(np.zeros(shape=(D - 1, )), self)
matrix = np.zeros(shape=(N, D))
for i in range(N):
matrix[i] = x[i:i + D]
return matrix[D - 1:] if skip_head else matrix
def gen_rnn_batches(self,
batch_size=1,
num_steps=-1,
shuffle=False,
is_training=False):
checker.check_positive_integer(batch_size)
while True:
data_set = self.engine(batch_size)
assert isinstance(data_set, DataSet)
for batch in data_set.gen_rnn_batches(batch_size, num_steps):
yield batch
def init_features_and_targets(self,
targets_key=None,
memory_depth=None,
skip_head=True):
"""Initialize features and targets using data in data_dict.
After initialization, data_dict will be cleared"""
# If target key is not provided, try to find one in data_dict
if targets_key is None:
targets_candidates = None
for key, val in self.data_dict.items():
if key != pedia.signals:
targets_candidates = val
break
else:
targets_candidates = self.data_dict.get(targets_key)
# If memory depth is None, init features as signals
if memory_depth is None:
self.features = self.signals
self.targets = targets_candidates
self._check_data()
return
# Initialize features (as causal matrix) and targets one by one
features = []
targets = []
checker.check_positive_integer(memory_depth)
start_at = memory_depth - 1 if skip_head else 0
for i, signal in enumerate(self.signals):
# Append signal to features
assert isinstance(signal, Signal)
features.append(signal.causal_matrix(memory_depth, skip_head))
# Append target
if targets_candidates is not None:
target = targets_candidates[i]
# For response target
if targets_key == pedia.responses:
assert isinstance(target, Signal)
target = target.causal_matrix(memory_depth=1)
target = target[start_at:]
elif targets_key == pedia.labels:
assert isinstance(target, np.ndarray)
label_len = len(target)
if label_len == len(signal):
target = target[start_at:]
else:
assert label_len == 1
# Append target to targets list
targets.append(target)
# Set features and targets
self.features = features
self.targets = None if targets_candidates is None else targets
# Abandon data_dict
if memory_depth > 1: self.data_dict = {}
# Check data
self._check_data()
def gen_rnn_batches(self, batch_size=1, num_steps=-1, *args, **kwargs):
assert self.is_ready
checker.check_positive_integer(self.batches_per_epoch)
for i in range(self.batches_per_epoch):
matrix, labels = self._random_signal_matrix(batch_size)
matrix, labels = self.init_f(matrix, labels)
for batch in self._gen_rnn_batches(matrix, labels, num_steps):
assert isinstance(batch, DataSet)
batch.name += '_in_{}of{}'.format(i + 1,
self.batches_per_epoch)
yield batch
def __init__(self,
num_channels,
channel_size,
read_head_size,
write_head_size=None):
self._num_channels = checker.check_positive_integer(num_channels)
self._channel_size = checker.check_positive_integer(channel_size)
self._read_head_size = checker.check_positive_integer(read_head_size)
if write_head_size is None: write_head_size = read_head_size
self._write_head_size = checker.check_positive_integer(write_head_size)
self._gam_tensor = None
def _get_state_dict(self, batch_size=None):
assert self.is_root
if batch_size is None:
# During training
state = self._state_array
assert state is not None
else:
# While is_training == False
checker.check_positive_integer(batch_size)
state = self._get_zero_state(batch_size)
return {self.init_state: state}
def _pad_sequences(sequences, max_steps):
"""Receive a list of irregular sequences and output a regular numpy array"""
assert isinstance(sequences, list)
checker.check_positive_integer(max_steps)
checker.check_type(sequences, np.ndarray)
if all([s.shape[0] == sequences[0].shape[0] for s in sequences]):
return np.stack(sequences, axis=0)
sample_shape = sequences[0].shape[1:]
assert len(sample_shape) > 0
stack = np.zeros(shape=(len(sequences), max_steps, *sample_shape))
for i, s in enumerate(sequences): stack[i, :len(s)] = s
return stack
def get_round_length(self, batch_size, num_steps=None, training=False):
assert isinstance(batch_size, int) and isinstance(training, bool)
if batch_size < 0: batch_size = self.size
if num_steps is None: round_len = np.ceil(self.size / batch_size)
else:
if self.is_rnn_input:
if num_steps < 0: num_steps = self.total_steps
round_len = np.ceil(self.total_steps / num_steps)
else:
if num_steps < 0: round_len = 1
elif training and hub.random_sample_length is not None:
# This branch is under testing
L = checker.check_positive_integer(
hub.random_sample_length)
round_len = int(np.ceil(L / num_steps))
else:
# e.g. PTB
M, N, p = self.size, batch_size, hub.overlap_pct if training else 0
assert 0 <= p < 1
L = int(M / ((N - 1) * (1 - p) + 1))
round_len = int(np.ceil(L / num_steps))
round_len = int(round_len)
if training: self._set_dynamic_round_len(round_len)
return round_len
def __init__(
self,
state_size,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
couple_fi=False,
cell_bias_initializer='zeros',
input_bias_initializer='zeros',
output_bias_initializer='zeros',
forget_bias_initializer='zeros',
use_output_activation=True,
**kwargs):
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer,
use_bias, cell_bias_initializer, **kwargs)
# Specific attributes
self._state_size = checker.check_positive_integer(state_size)
self._input_bias_initializer = initializers.get(input_bias_initializer)
self._output_bias_initializer = initializers.get(output_bias_initializer)
self._forget_bias_initializer = initializers.get(forget_bias_initializer)
self._couple_fi = checker.check_type(couple_fi, bool)
self._use_output_activation = checker.check_type(
use_output_activation, bool)
def get_tensor_to_export(trainer):
"""Used in trainer._take_notes_for_export"""
from tframe.trainers.trainer import Trainer
assert isinstance(trainer, Trainer)
tensors = OrderedDict()
num = checker.check_positive_integer(trainer.th.sample_num)
# .. fetch tensors
fetches_dict = context.tensors_to_export
if len(fetches_dict) == 0: return tensors
results = trainer.model.evaluate(list(fetches_dict.values()),
trainer.validation_set[:num])
exemplar_names = []
for i in range(num):
name = 'Exemplar {}'.format(i)
tensors[name] = OrderedDict()
exemplar_names.append(name)
# .. fill tensor_dict
for i, array_list in enumerate(results):
tensor_name = list(fetches_dict.keys())[i]
for j, array in enumerate(array_list):
if j < num: tensors[exemplar_names[j]][tensor_name] = array
return tensors
def __init__(
self,
config_string,
num_layers,
head_size,
activation='tanh',
use_bias=True,
weight_initializer='xavier_normal',
bias_initializer='zeros',
gutter=False,
gutter_bias=None,
**kwargs):
# Call parent's constructor
LayerWithNeurons.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
HardDriver.__init__(self, config_string, head_size, gutter=gutter,
gutter_bias=gutter_bias)
self._num_layers = checker.check_positive_integer(num_layers)
self._activation_string = activation
self.output_scale = [self.total_size]
self.write_heads = None
def run(self, times=1, save=False, mark=''):
if self._sys_runs is not None:
times = checker.check_positive_integer(self._sys_runs)
console.show_status('Run # set to {}'.format(times))
# Set the corresponding flags if save
if save:
self.common_parameters['save_model'] = True
# Show parameters
self._show_parameters()
# Begin iteration
counter = 0
for run_id in range(times):
history = []
for hyper_dict in self._hyper_parameter_dicts():
# Set counter here
counter += 1
# Grand self._add_script_suffix the highest priority
if self._add_script_suffix is not None:
save = self._add_script_suffix
if save:
self.common_parameters['script_suffix'] = '_{}{}'.format(
mark, counter)
params = self._get_all_configs(hyper_dict)
self._apply_constraints(params)
params_list = self._get_config_strings(params)
params_string = ' '.join(params_list)
if params_string in history: continue
history.append(params_string)
console.show_status('Loading task ...',
'[Run {}/{}]'.format(run_id + 1, times))
call([self._python_cmd, self.module_name] + params_list)
# call(self.command_head + params_list)
print()
def __init__(self,
state_size,
use_reset_gate=True,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
z_bias_initializer='zeros',
reset_who='s',
dropout=0.0,
zoneout=0.0,
**kwargs):
"""
:param reset_who: in ('x', 'y')
'x': a_h = W_h * (h_{t-1} \odot r_t)
'y': a_h = r_t \odot (W_h * h_{t-1})
\hat{h}_t = \varphi(Wx*x + a_h + b)
in which r_t is the reset gate at time step t,
\odot is the Hadamard product, W_h is the hidden-to-hidden matrix
"""
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
# Specific attributes
self._state_size = checker.check_positive_integer(state_size)
self._use_reset_gate = checker.check_type(use_reset_gate, bool)
self._z_bias_initializer = initializers.get(z_bias_initializer)
self._dropout_rate = checker.check_type(dropout, float)
self._zoneout_rate = checker.check_type(zoneout, float)
assert reset_who in ('s', 'a')
self._reset_who = reset_who
def __init__(self,
num_neurons,
activation=None,
use_bias=True,
weight_initializer='xavier_normal',
bias_initializer='zeros',
layer_normalization=False,
etch=None,
**kwargs):
"""
:param etch: if this argument is not None, it will be passed to the
neuron array and
(1) a masked weight matrix will be created
(2) the corresponding weight and gradient will be registered
to monitor
(3) corresponding etch method will be called during training
"""
# Call parent's constructor
super().__init__(activation, weight_initializer, use_bias,
bias_initializer, layer_normalization, **kwargs)
self.num_neurons = checker.check_positive_integer(num_neurons)
self.neuron_scale = [num_neurons]
self.etch = etch
