def short_time_energy(signal_, window_size, stride, center=True): """Calculate short time energy :param center: If `True`, the signal `y` is padded so that frame `D[:, t]` is centered at `y[t * hop_length]`. If `False`, then `D[:, t]` begins at `y[t * hop_length]` """ # Sanity check assert isinstance(signal_, np.ndarray) checker.check_positive_integer(window_size) checker.check_positive_integer(stride) if center: signal_ = np.pad(signal_, window_size // 2, mode='reflect') # Reshape signal signal_ = np.reshape(signal_, newshape=(-1, )) # Form causal matrix frames = [] cursor = 0 while True: frames.append(signal_[cursor:cursor + window_size]) # Move cursor forward cursor += stride if cursor + window_size > signal_.size: break # Calculate energy stack = np.stack(frames, axis=0) energy = np.multiply(stack, stack) energy = np.sum(energy, axis=1) return energy
def initialize(self, input_size, batches_per_epoch): checker.check_positive_integer(input_size) checker.check_positive_integer(batches_per_epoch) self.properties[self.INPUT_SIZE] = input_size self.properties[self.BATCHES_PER_EPOCH] = batches_per_epoch self.properties[self.READY] = True
def _get_dynamic_round_len(act_lens, num_steps, training): """ not train x x x x x|x x x x x|x x x/x x:x x x x x x x x x x x x|x x x x x|x x x/ : x x x x x|x x x x x|x x x/x : x x x x x|x x x x x|x x x/x x:x x x x x train """ assert isinstance(act_lens, (np.ndarray, list)) and len(act_lens) > 0 checker.check_positive_integer(num_steps) counter = 0 while len(act_lens) > 0: # Find the shortest sequence sl = min(act_lens) assert sl > 0 # Calculate iterations (IMPORTANT). Note that during training act_len # .. does not help to avoid inappropriate gradient flow thus sequences # .. have to be truncated n = int(np.ceil(sl / num_steps)) counter += n # Update act_lens list L = sl if training else n * num_steps act_lens = [al for al in [al - L for al in act_lens] if al > 0] return counter
def get_data_batches(self, data_set, batch_size, num_steps=None, shuffle=False): """ Get batch generator. :param data_set: an instance of DataSet or BigData from which data batches will be extracted :param batch_size: if is None, default value will be assigned according to the input type of this model :param num_steps: step number for RNN data batches :param shuffle: whether to shuffle :return: a generator or a list """ # Data set must be an instance of DataSet or BigData assert isinstance(data_set, (DataSet, BigData)) if self.input_type is InputTypes.BATCH: # If model's input type is normal batch, num_steps will be ignored # If batch size is not specified and data is a DataSet, feed it all at # once into model if batch_size is None and isinstance(data_set, DataSet): return [data_set.stack] checker.check_positive_integer(batch_size) data_batches = data_set.gen_batches(batch_size, shuffle=shuffle) elif self.input_type is InputTypes.RNN_BATCH: if batch_size is None: batch_size = 1 if num_steps is None: num_steps = -1 checker.check_positive_integer(batch_size) checker.check_type(num_steps, int) data_batches = data_set.gen_rnn_batches(batch_size, num_steps, shuffle) else: raise ValueError('!! Can not resolve input type of this model') return data_batches
def __init__( self, num_filter_list, num_classes, kernel_initializer='glorot_uniform', left_repeats=2, right_repeats=2, activation='relu', dropout_rate=0.5, name='unet', level=1, **kwargs): # Sanity check assert isinstance(num_filter_list, (tuple, list)) and num_filter_list # Call parent's constructor # TODO: the level logic is not elegant super().__init__(name, level=level, **kwargs) # Specific attributes self.num_filter_list = num_filter_list self.num_classes = checker.check_positive_integer(num_classes) # self.kernel_sizes = kernel_sizes self.kernel_initializer = kernel_initializer self.activation = activation self.dropout_rate = checker.check_type(dropout_rate, float) self.left_repeats = checker.check_positive_integer(left_repeats) self.right_repeats = checker.check_positive_integer(right_repeats) # Add layers self._add_layers()
def _get_file_name(cls, size, N, T, var_x, add_noise, var_y): checker.check_positive_integer(N) checker.check_positive_integer(T) file_name = '{}_{}_N{}T{}_vx{}_{}'.format( 'TSP', size, N, T, var_x, 'noisy' if add_noise else 'noise-free') if add_noise: file_name += '_vy{}'.format(var_y) return file_name
def fc_lstm(th): assert isinstance(th, Config) th.mark = 'fc_lstm_' + th.mark # Initiate a model model = Classifier(mark=th.mark, net_type=Recurrent) # Add input layer model.add(Input(sample_shape=th.input_shape)) # Add fc layers for dim in th.fc_dims: checker.check_positive_integer(dim) model.add(Linear(output_dim=dim)) # model.add(BatchNorm()) model.add(Activation('relu')) # Add lstm cells for dim in th.rc_dims: model.add(BasicLSTMCell(state_size=dim)) # Add output layer # model.add(Linear(output_dim=th.output_dim)) # Build model optimizer = tf.train.AdamOptimizer(th.learning_rate) model.build(optimizer) return model
def __init__(self, num_neurons, group_size, head_size=-1, activation=None, use_bias=True, weight_initializer='xavier_normal', bias_initializer='zeros', **kwargs): """ Softmax over groups applied to neurons. Case 1: head_size < 0: does not use extra neurons Case 2: head_size = 0: use extra neurons without a head Case 3: head_size > 0: use extra neurons with a head """ # Call parent's constructor super().__init__(activation, weight_initializer, use_bias, bias_initializer, **kwargs) # Specific attributes self._num_neurons = checker.check_positive_integer(num_neurons) self._group_size = checker.check_positive_integer(group_size) self._head_size = checker.check_type(head_size, int) # Developer options options = th.developer_options
def _gen_rnn_batches(self, x, y, batch_size, num_steps): checker.check_positive_integer(batch_size, 'batch size') checker.check_type(num_steps, int) # Get batch partitions data_x, L = self._get_batch_partition(x, batch_size) if y is not None: if len(x) == len(y): data_y, Ly = self._get_batch_partition(y, batch_size) assert L == Ly else: assert len(y) == 1 data_y = y # Chop data further if num_steps < 0: num_steps = L round_len = int(np.ceil(L / num_steps)) for i in range(round_len): batch_x = data_x[:, i * num_steps:min((i + 1) * num_steps, L)] batch_y = None if y is not None: if len(x) == len(y): batch_y = data_y[:, i * num_steps:min((i + 1) * num_steps, L)] else: assert isinstance(y, np.ndarray) batch_y = np.tile(y, [batch_x.shape[0], batch_x.shape[1], 1]) batch = DataSet(batch_x, batch_y, in_rnn_format=True) # State should be reset at the beginning of a sequence if i == 0: batch.should_reset_state = True batch.name = self.name + '_{}'.format(i + 1) yield batch
def get_round_length(self, batch_size, num_steps=None): """Get round length for training :param batch_size: Batch size. For irregular sequences, this value should be set to 1. :param num_steps: Step number. If provided, round length will be calculated for RNN model :return: Round length for training """ # Make sure features exist self._check_feature() checker.check_positive_integer(batch_size, 'batch_size') if num_steps is None: # :: For feed-forward models return int(np.ceil(self.stack.size / batch_size)) else: # :: For recurrent models checker.check_type(num_steps, int) if self.is_regular_array: arrays = [self.features] elif self.parallel_on: return self._get_pe_round_length(batch_size, num_steps) else: arrays = self.features len_f = lambda x: x if self.len_f is None else self.len_f if num_steps < 0: return len(arrays) else: return int( sum([ np.ceil(len_f(len(array)) // batch_size / num_steps) for array in arrays ]))
def __init__(self, fast_size, fast_layers, slow_size, hyper_kernel, activation='tanh', weight_initializer='xavier_normal', use_bias=True, bias_initializer='zeros', input_dropout=0.0, output_dropout=0.0, forget_bias=0, **kwargs): # Call parent's constructor CellBase.__init__(self, activation, weight_initializer, use_bias, bias_initializer, **kwargs) self.kernel_key = checker.check_type(hyper_kernel, str) # Specific attributes self._fast_size = checker.check_positive_integer(fast_size) self._fast_layers = checker.check_positive_integer(fast_layers) self._slow_size = checker.check_positive_integer(slow_size) self._hyper_kernel = self._get_hyper_kernel(hyper_kernel, do=th.rec_dropout, forget_bias=forget_bias) self._input_do = checker.check_type(input_dropout, float) self._output_do = checker.check_type(output_dropout, float)
def __init__( self, num_neurons, group_size, axis=0, activation=None, use_bias=True, weight_initializer='xavier_normal', bias_initializer='zeros', **kwargs): """ axis: 0 or 1. axis = 0: Partition input neurons. Each output neuron uses only 1 input activation in each group. axis = 1: Partition output neurons. Each input neuron passes value to only 1 output neuron in each group. """ # Call parent's constructor super().__init__(activation, weight_initializer, use_bias, bias_initializer, **kwargs) # Specific attributes assert axis in (0, 1) self._axis = axis self._num_neurons = checker.check_positive_integer(num_neurons) self._group_size = checker.check_positive_integer(group_size)
def __init__( self, state_size, tape_length, spatial_size=None, spatial_heads=1, temporal_heads=1, spatial_dropout=0.0, temporal_dropout=0.0, activation='tanh', weight_initializer='xavier_uniform', use_bias=True, bias_initializer='zeros', rec_dropout=0.0, **kwargs): """ """ # Call parent's constructor super().__init__( length=tape_length, depth=state_size, num_tapes=1, activation=activation, weight_initializer=weight_initializer, use_bias=use_bias, bias_initializer=bias_initializer, dropout_rate=rec_dropout, **kwargs) # Specific attributes self._state_size = state_size self._spatial_size = spatial_size self._spatial_heads = checker.check_positive_integer(spatial_heads) self._temporal_heads = checker.check_positive_integer(temporal_heads) self._spatial_dropout = spatial_dropout self._temporal_dropout = temporal_dropout
def pop_subset(self, size, length_prone='random', name='subset'): """Pop sub data set containing 'size' samples for each class""" # Check input parameters checker.check_positive_integer(size) assert length_prone in ('short', 'long', 'random') if size > self.smallest_population: raise ValueError( '!! size should be less than the smallest population ' 'of this data set: {}'.format(self.smallest_population)) # Check classes assert len(self.groups) == self.NUM_CLASSES # Fill subset subset = self._copy_empty_container() subset.name = name for label, wavs in self.groups.items(): assert isinstance(wavs, list) # Initialize file list for label class subset.groups[label] = [] # Pop data from self for _ in range(size): if length_prone == 'short': i = -1 elif length_prone == 'long': i = 0 else: i = np.random.randint(len(wavs)) # Pop from self file_name = wavs.pop(i) subset.groups[label].append(file_name) tfd_file_name = file_name + '.tfds' subset.files[tfd_file_name] = self.files.pop(tfd_file_name) # Check length assert len(self.files) == sum(list(self.group_population.values())) assert len(subset.files) == sum(list(subset.group_population.values())) return subset
def _gen_parallel_batches(self, batch_size, num_steps, shuffle): """A beta method used only for RNN training""" # Sanity check features, targets = self.features, self.targets assert isinstance(features, (tuple, list)) assert isinstance(targets, (tuple, list)) assert len(features) == len(targets) checker.check_positive_integer(batch_size) assert isinstance(num_steps, int) assert isinstance(shuffle, bool) # Initialize parallel engine pe = ParallelEngine(batch_size) cursor, num_sequences = 0, len(self.features) round_len = self._get_pe_round_length(batch_size, num_steps) # Start loop global_reset = True counter = 0 while True: reset_indices = pe.inactive_indices reset_values = [] # Load new sequence to engine if necessary while not pe.is_ready: if shuffle or cursor < num_sequences: index = self._rand_indices() if shuffle else cursor x, y = self.features[index], self.targets[index] if self.init_f is not None: x, y = self.init_f(x, y) cursor += 1 reset_values.append(0) else: x, y = None, None reset_values.append(None) pe.set_sequence(x, y) if pe.flameout: break # Get features and targets and wrap them into a DataSet x, y = pe.emit(num_steps) data_batch = DataSet(x, y) if len(reset_indices) > 0: if global_reset: data_batch.should_reset_state = True global_reset = False assert len(reset_indices) == len(reset_values) data_batch.reset_batch_indices = reset_indices data_batch.reset_values = (reset_values if len( [val for val in reset_values if val is None]) > 0 else None) # Yield batch yield data_batch counter += 1 if counter >= round_len: break # Check round length assert counter == round_len
def gen_batches(self, batch_size, **kwargs): assert self.is_ready checker.check_positive_integer(self.batches_per_epoch) for i in range(self.batches_per_epoch): matrix, labels = self._random_signal_matrix( batch_size, self.input_size) batch = DataSet(matrix, labels) batch.name = 'gpat_{}of{}'.format(i + 1, self.batches_per_epoch) yield batch
def causal_matrix(self, memory_depth, skip_head=False): checker.check_positive_integer(memory_depth) assert isinstance(self, np.ndarray) if memory_depth == 1: return np.reshape(self, (-1, 1)) N, D = self.size, memory_depth x = np.append(np.zeros(shape=(D - 1, )), self) matrix = np.zeros(shape=(N, D)) for i in range(N): matrix[i] = x[i:i + D] return matrix[D - 1:] if skip_head else matrix
def gen_rnn_batches(self, batch_size=1, num_steps=-1, shuffle=False, is_training=False): checker.check_positive_integer(batch_size) while True: data_set = self.engine(batch_size) assert isinstance(data_set, DataSet) for batch in data_set.gen_rnn_batches(batch_size, num_steps): yield batch
def init_features_and_targets(self, targets_key=None, memory_depth=None, skip_head=True): """Initialize features and targets using data in data_dict. After initialization, data_dict will be cleared""" # If target key is not provided, try to find one in data_dict if targets_key is None: targets_candidates = None for key, val in self.data_dict.items(): if key != pedia.signals: targets_candidates = val break else: targets_candidates = self.data_dict.get(targets_key) # If memory depth is None, init features as signals if memory_depth is None: self.features = self.signals self.targets = targets_candidates self._check_data() return # Initialize features (as causal matrix) and targets one by one features = [] targets = [] checker.check_positive_integer(memory_depth) start_at = memory_depth - 1 if skip_head else 0 for i, signal in enumerate(self.signals): # Append signal to features assert isinstance(signal, Signal) features.append(signal.causal_matrix(memory_depth, skip_head)) # Append target if targets_candidates is not None: target = targets_candidates[i] # For response target if targets_key == pedia.responses: assert isinstance(target, Signal) target = target.causal_matrix(memory_depth=1) target = target[start_at:] elif targets_key == pedia.labels: assert isinstance(target, np.ndarray) label_len = len(target) if label_len == len(signal): target = target[start_at:] else: assert label_len == 1 # Append target to targets list targets.append(target) # Set features and targets self.features = features self.targets = None if targets_candidates is None else targets # Abandon data_dict if memory_depth > 1: self.data_dict = {} # Check data self._check_data()
def gen_rnn_batches(self, batch_size=1, num_steps=-1, *args, **kwargs): assert self.is_ready checker.check_positive_integer(self.batches_per_epoch) for i in range(self.batches_per_epoch): matrix, labels = self._random_signal_matrix(batch_size) matrix, labels = self.init_f(matrix, labels) for batch in self._gen_rnn_batches(matrix, labels, num_steps): assert isinstance(batch, DataSet) batch.name += '_in_{}of{}'.format(i + 1, self.batches_per_epoch) yield batch
def __init__(self, num_channels, channel_size, read_head_size, write_head_size=None): self._num_channels = checker.check_positive_integer(num_channels) self._channel_size = checker.check_positive_integer(channel_size) self._read_head_size = checker.check_positive_integer(read_head_size) if write_head_size is None: write_head_size = read_head_size self._write_head_size = checker.check_positive_integer(write_head_size) self._gam_tensor = None
def _get_state_dict(self, batch_size=None): assert self.is_root if batch_size is None: # During training state = self._state_array assert state is not None else: # While is_training == False checker.check_positive_integer(batch_size) state = self._get_zero_state(batch_size) return {self.init_state: state}
def _pad_sequences(sequences, max_steps): """Receive a list of irregular sequences and output a regular numpy array""" assert isinstance(sequences, list) checker.check_positive_integer(max_steps) checker.check_type(sequences, np.ndarray) if all([s.shape[0] == sequences[0].shape[0] for s in sequences]): return np.stack(sequences, axis=0) sample_shape = sequences[0].shape[1:] assert len(sample_shape) > 0 stack = np.zeros(shape=(len(sequences), max_steps, *sample_shape)) for i, s in enumerate(sequences): stack[i, :len(s)] = s return stack
def get_round_length(self, batch_size, num_steps=None, training=False): assert isinstance(batch_size, int) and isinstance(training, bool) if batch_size < 0: batch_size = self.size if num_steps is None: round_len = np.ceil(self.size / batch_size) else: if self.is_rnn_input: if num_steps < 0: num_steps = self.total_steps round_len = np.ceil(self.total_steps / num_steps) else: if num_steps < 0: round_len = 1 elif training and hub.random_sample_length is not None: # This branch is under testing L = checker.check_positive_integer( hub.random_sample_length) round_len = int(np.ceil(L / num_steps)) else: # e.g. PTB M, N, p = self.size, batch_size, hub.overlap_pct if training else 0 assert 0 <= p < 1 L = int(M / ((N - 1) * (1 - p) + 1)) round_len = int(np.ceil(L / num_steps)) round_len = int(round_len) if training: self._set_dynamic_round_len(round_len) return round_len
def __init__( self, state_size, activation='tanh', weight_initializer='xavier_normal', use_bias=True, couple_fi=False, cell_bias_initializer='zeros', input_bias_initializer='zeros', output_bias_initializer='zeros', forget_bias_initializer='zeros', use_output_activation=True, **kwargs): # Call parent's constructor CellBase.__init__(self, activation, weight_initializer, use_bias, cell_bias_initializer, **kwargs) # Specific attributes self._state_size = checker.check_positive_integer(state_size) self._input_bias_initializer = initializers.get(input_bias_initializer) self._output_bias_initializer = initializers.get(output_bias_initializer) self._forget_bias_initializer = initializers.get(forget_bias_initializer) self._couple_fi = checker.check_type(couple_fi, bool) self._use_output_activation = checker.check_type( use_output_activation, bool)
def get_tensor_to_export(trainer): """Used in trainer._take_notes_for_export""" from tframe.trainers.trainer import Trainer assert isinstance(trainer, Trainer) tensors = OrderedDict() num = checker.check_positive_integer(trainer.th.sample_num) # .. fetch tensors fetches_dict = context.tensors_to_export if len(fetches_dict) == 0: return tensors results = trainer.model.evaluate(list(fetches_dict.values()), trainer.validation_set[:num]) exemplar_names = [] for i in range(num): name = 'Exemplar {}'.format(i) tensors[name] = OrderedDict() exemplar_names.append(name) # .. fill tensor_dict for i, array_list in enumerate(results): tensor_name = list(fetches_dict.keys())[i] for j, array in enumerate(array_list): if j < num: tensors[exemplar_names[j]][tensor_name] = array return tensors
def __init__( self, config_string, num_layers, head_size, activation='tanh', use_bias=True, weight_initializer='xavier_normal', bias_initializer='zeros', gutter=False, gutter_bias=None, **kwargs): # Call parent's constructor LayerWithNeurons.__init__(self, activation, weight_initializer, use_bias, bias_initializer, **kwargs) HardDriver.__init__(self, config_string, head_size, gutter=gutter, gutter_bias=gutter_bias) self._num_layers = checker.check_positive_integer(num_layers) self._activation_string = activation self.output_scale = [self.total_size] self.write_heads = None
def run(self, times=1, save=False, mark=''): if self._sys_runs is not None: times = checker.check_positive_integer(self._sys_runs) console.show_status('Run # set to {}'.format(times)) # Set the corresponding flags if save if save: self.common_parameters['save_model'] = True # Show parameters self._show_parameters() # Begin iteration counter = 0 for run_id in range(times): history = [] for hyper_dict in self._hyper_parameter_dicts(): # Set counter here counter += 1 # Grand self._add_script_suffix the highest priority if self._add_script_suffix is not None: save = self._add_script_suffix if save: self.common_parameters['script_suffix'] = '_{}{}'.format( mark, counter) params = self._get_all_configs(hyper_dict) self._apply_constraints(params) params_list = self._get_config_strings(params) params_string = ' '.join(params_list) if params_string in history: continue history.append(params_string) console.show_status('Loading task ...', '[Run {}/{}]'.format(run_id + 1, times)) call([self._python_cmd, self.module_name] + params_list) # call(self.command_head + params_list) print()
def __init__(self, state_size, use_reset_gate=True, activation='tanh', weight_initializer='xavier_normal', use_bias=True, bias_initializer='zeros', z_bias_initializer='zeros', reset_who='s', dropout=0.0, zoneout=0.0, **kwargs): """ :param reset_who: in ('x', 'y') 'x': a_h = W_h * (h_{t-1} \odot r_t) 'y': a_h = r_t \odot (W_h * h_{t-1}) \hat{h}_t = \varphi(Wx*x + a_h + b) in which r_t is the reset gate at time step t, \odot is the Hadamard product, W_h is the hidden-to-hidden matrix """ # Call parent's constructor CellBase.__init__(self, activation, weight_initializer, use_bias, bias_initializer, **kwargs) # Specific attributes self._state_size = checker.check_positive_integer(state_size) self._use_reset_gate = checker.check_type(use_reset_gate, bool) self._z_bias_initializer = initializers.get(z_bias_initializer) self._dropout_rate = checker.check_type(dropout, float) self._zoneout_rate = checker.check_type(zoneout, float) assert reset_who in ('s', 'a') self._reset_who = reset_who
def __init__(self, num_neurons, activation=None, use_bias=True, weight_initializer='xavier_normal', bias_initializer='zeros', layer_normalization=False, etch=None, **kwargs): """ :param etch: if this argument is not None, it will be passed to the neuron array and (1) a masked weight matrix will be created (2) the corresponding weight and gradient will be registered to monitor (3) corresponding etch method will be called during training """ # Call parent's constructor super().__init__(activation, weight_initializer, use_bias, bias_initializer, layer_normalization, **kwargs) self.num_neurons = checker.check_positive_integer(num_neurons) self.neuron_scale = [num_neurons] self.etch = etch