def __init__(self, network,
l1=0., l2=0., confusion_matrix=True,
tol=1e-4, patience=3, rollback=True,
batch_size=256, max_epoch=100, max_iter=None,
optimizer='adadelta', learning_rate=1.0, class_weights=None,
dtype='float32', seed=5218, verbose=False,
path=None, name=None):
super(NeuralNetworkClassifier, self).__init__()
if not isinstance(network, N.NNOp):
raise ValueError("`network` must be instance of odin.nnet.NNOp")
self._network = network
self._input_shape = None
self._output_shape = None
self._nb_classes = None
self._dtype = np.dtype(dtype)
self._class_weights = class_weights
# ====== flags ====== #
self.l1 = float(l1)
self.l2 = float(l2)
self.confusion_matrix = bool(confusion_matrix)
# ====== stop training ====== #
self.tol = float(tol)
self.patience = int(patience)
self.rollback = bool(rollback)
# ====== others ====== #
self._train_history = []
self._valid_history = []
self._rand_state = np.random.RandomState(seed=int(seed))
self.verbose = int(verbose)
# ====== others ====== #
if name is None:
name = self.__class__.__name__ + uuid()
self._name = str(name)
self._path = path
def __init__(self, sr):
verify_dependencies()
super(_openSMILEbase, self).__init__()
self._id = uuid(length=25)
self.sr = sr
self._first_config_generated = False
self._conf = _get_conf_file('%s.cfg' % self.__class__.__name__)
self._log_level = -1
def __init__(self, name=None, **kwargs):
super(NNOps, self).__init__()
self._arguments = {}
self.name = name
if name is None:
self.name = "%s_%s" % (self.__class__.__name__, uuid())
self._configuration = None
self._transpose_ops = None
def rnn(input_dim,
hidden_dim,
W_init=glorot_uniform,
b_init=constant(0.),
bidirectional=False,
one_vector=False,
return_variable=True,
name=None):
""" Fast initalize all Standard RNN weights
Parameters
----------
one_vector: bool
if True, all the weights are flatten and concatenated into 1 big vector
return_variable: bool
if False, only return the numpy array
bidirectional: bool
if True, return parameters for both forward and backward RNN
Return
------
[W_i, b_wi, R_h, b_wh]
"""
if name is None: name = uuid()
def init():
W_i = W_init((input_dim, hidden_dim))
b_wi = b_init((hidden_dim))
R_h = W_init((hidden_dim, hidden_dim))
b_wh = b_init((hidden_dim))
return [W_i, b_wi, R_h, b_wh]
params = init() + init() if bidirectional else init()
roles = [WEIGHT, BIAS]
if one_vector:
params = [np.concatenate([p.flatten() for p in params])]
roles = [PARAMETER]
# names
if one_vector:
names = [name + '_rnn']
else:
names = ["_W_i", "_b_wi", "_R_h", "_b_wh"]
if bidirectional:
names = [i + '_fw' for i in names] + [i + '_bw' for i in names]
names = [name + i for i in names]
# create variable or not
if return_variable:
params = [variable(p, name=n) for p, n in zip(params, names)]
for i, p in enumerate(params):
add_role(p, roles[i % 2])
return params if len(params) > 1 else params[0]
def __init__(self,
network,
l1=0.,
l2=0.,
confusion_matrix=True,
tol=1e-4,
patience=3,
rollback=True,
batch_size=256,
max_epoch=100,
max_iter=None,
optimizer='adadelta',
learning_rate=1.0,
class_weights=None,
dtype='float32',
seed=5218,
verbose=False,
path=None,
name=None):
super(NeuralNetworkClassifier, self).__init__()
if not isinstance(network, N.NNOp):
raise ValueError("`network` must be instance of odin.nnet.NNOp")
self._network = network
self._input_shape = None
self._output_shape = None
self._nb_classes = None
self._dtype = np.dtype(dtype)
self._class_weights = class_weights
# ====== flags ====== #
self.l1 = float(l1)
self.l2 = float(l2)
self.confusion_matrix = bool(confusion_matrix)
# ====== stop training ====== #
self.tol = float(tol)
self.patience = int(patience)
self.rollback = bool(rollback)
# ====== others ====== #
self._train_history = []
self._valid_history = []
self._rand_state = np.random.RandomState(seed=int(seed))
self.verbose = int(verbose)
# ====== others ====== #
if name is None:
name = self.__class__.__name__ + uuid()
self._name = str(name)
self._path = path
def lstm_batch_norm(num_units,
W_input_init=K.init.glorot_uniform,
W_hidden_init=K.init.orthogonal,
W_peephole_init=K.init.glorot_uniform,
activation=K.tanh, gate_activation=K.sigmoid,
tied_input=False, batch_norm=True, name=None):
if name is None:
name = 'lstm_batch_norm_%s' % utils.uuid()
# ====== create input_gates ====== #
ops_list = []
bias = None if batch_norm else K.init.constant(0)
if tied_input:
input_gates = Dense(num_units, W_init=W_input_init, b_init=bias,
activation=K.linear, name='%s_gates' % name)
else:
input_gates = Merge([
Dense(num_units, W_init=W_input_init, b_init=bias, activation=K.linear,
name='%s_ingate' % name), # input-gate
Dense(num_units, W_init=W_input_init, b_init=bias, activation=K.linear,
name='%s_forgetgate' % name), # forget-gate
Dense(num_units, W_init=W_input_init, b_init=bias, activation=K.linear,
name='%s_cellupdate' % name), # cell-update
Dense(num_units, W_init=W_input_init, b_init=bias, activation=K.linear,
name='%s_outgate' % name) # output-gate
], merge_function=K.concatenate)
ops_list.append(input_gates)
# ====== batch_norm ====== #
# normalize batch and time dimension
if batch_norm:
ops_list.append(BatchNorm(axes=(0, 1), name='%s_norm' % name))
# ====== add LSTM ====== #
ops_list.append(LSTM(num_units=num_units,
activation=activation,
gate_activation=gate_activation,
W_init=W_hidden_init,
W_peepholes=W_peephole_init,
name='%s_lstm' % name))
return Sequence(ops_list, name=name)
def __init__(self, lr, decay_steps=None, decay_rate=0.96, staircase=True,
clipnorm=None, clipvalue=None, clip_alg='total_norm',
name=None):
if name is None:
name = self.__class__.__name__ + '_' + str(uuid(length=4))
elif not isinstance(name, string_types):
name = str(name)
self._name = str(name)
self.staircase = bool(staircase)
with tf.variable_scope(self._name):
self._lr = _as_variable(lr, name='learning_rate', roles=LearningRate)
self._lr_decay = None
self._step = tf.Variable(0., dtype=floatX,
name="%s_step" % self.__class__.__name__)
self.decay_steps = decay_steps
self.decay_rate = decay_rate
if clipnorm is not None:
if (clipnorm if is_number(clipnorm) else get_value(clipnorm)) <= 0:
raise ValueError('`clipnorm` value must greater than 0.')
self.clipnorm = _as_variable(clipnorm, name="clip_norm",
roles=GraidentsClippingNorm)
if clipvalue is not None:
if (clipvalue if is_number(clipvalue) else get_value(clipvalue)) <= 0:
raise ValueError('`clipvalue` value must greater than 0.')
self.clipvalue = _as_variable(clipvalue, name="clip_value",
roles=GraidentsClippingValue)
# ====== internal states values ====== #
clip_alg = str(clip_alg).strip().lower()
if clip_alg not in ('total_norm', 'norm', 'avg_norm'):
raise ValueError("clip_arg must be one of the following: "
"'norm', 'total_norm', 'avg_norm'")
self._norm = 0.
self.clip_alg = clip_alg
self._algorithm = None
self._is_initialized = False
def rnn_dnn(X,
hidden_size,
rnn_mode,
num_layers=1,
parameters=None,
h0=None,
c0=None,
input_mode='linear',
direction_mode='unidirectional',
dropout=0.,
name=None):
"""CuDNN v5 RNN implementation.
Parameters
----------
X : input varialbe or placeholder
shape=(batch_size, timesteps, input_dims)
hidden_size : int
the number of units within the RNN model.
rnn_mode : {'rnn_relu', 'rnn_tanh', 'lstm', 'gru'}
See cudnn documentation for ``cudnnRNNMode_t``.
num_layers : int
the number of layers for the RNN model.
h0: tensor
h0 with shape [num_layers, batch_size, hidden_size]
c0: tensor
c0 (lstm) with shape [num_layers, batch_size, hidden_size]
parameters: vector
vector contain all flatten weights and bias
check `backend.init.lstm`, `backend.init.gru`, and `backend.init.rnn`
for more information
input_mode : {'linear', 'skip'}
linear: input will be multiplied by a biased matrix
skip: No operation is performed on the input. The size must
match the hidden size.
(CuDNN docs: cudnnRNNInputMode_t)
direction_mode : {'unidirectional', 'bidirectional'}
unidirectional: The network operates recurrently from the
first input to the last.
bidirectional: The network operates from first to last then from last
to first and concatenates the results at each layer.
dropout: float (0.0-1.0)
whether to enable dropout. With it is 0, dropout is disabled.
Returns
-------
[output, hidden_states, cell_states] for lstm
[output, hidden_states] for gru and rnn
output_shape: (batch_size, timesteps, hidden_size)
hidden_shape: (num_layers, batch_size, hidden_size)
cell_shape: (num_layers, batch_size, hidden_size)
Note
----
dropout is turn off if K.set_training(False) or K.is_training() == False
"""
if CONFIG['device'] == 'cpu':
raise Exception('This opt is not supported with CPU.')
if name is None: name = uuid()
# ====== Check arguments ====== #
if rnn_mode not in ('rnn_relu', 'rnn_tanh', 'lstm', 'gru'):
raise ValueError(
"rnn_mode=%s must be: 'rnn_relu', 'rnn_tanh', 'lstm', 'gru'" %
rnn_mode)
if input_mode not in ('linear', 'skip'):
raise ValueError("input_mode=%s must be: 'linear', 'skip'" %
input_mode)
input_mode = 'linear_input' if input_mode == 'linear' else 'skip_input'
if direction_mode not in ('unidirectional', 'bidirectional'):
raise ValueError(
"direction_mode=%s must be: 'unidirectional', 'bidirectional'" %
direction_mode)
is_bidirectional = direction_mode == 'bidirectional'
# ====== helper function ====== #
def check_init_states(s0, nb_layers, batch_size):
if s0 is None: return None
if s0.get_shape().ndims < 3:
s0 = expand_dims(s0, dim=0)
s0shape = get_shape(s0)
if s0shape[0] == 1 and s0shape[0] != nb_layers:
s0 = repeat(s0, n=nb_layers, axes=0)
if s0shape[1] == 1:
s0 = repeat(s0, n=batch_size, axes=1)
return s0
# ====== create RNNBlock ====== #
from tensorflow.contrib import cudnn_rnn
input_shape = get_shape(X)
if X.get_shape().ndims != 3:
raise ValueError('Input must be 3-D tensor, but X is %d-D tensor' %
X.ndim)
if input_shape[-1] != hidden_size and 'skip' in input_mode:
raise ValueError(
'In skip_input mode, input size must be equal to hidden size'
', but input_size=%d != hidden_size=%d' %
(input_shape[-1], hidden_size))
# IF we dimshuffle here, a lot of error concern GPUarray,
# and cudnn will happen
batch_size = get_shape(X, native=True)[0]
if rnn_mode == 'lstm':
rnn = cudnn_rnn.CudnnLSTM(num_layers=num_layers,
num_units=hidden_size,
input_size=input_shape[-1],
input_mode=input_mode,
direction=direction_mode,
dropout=dropout,
seed=0,
seed2=0)
else:
if rnn_mode == 'gru':
rnn_class = cudnn_rnn.CudnnGRU
elif rnn_mode == 'rnn_relu':
rnn_class = cudnn_rnn.CudnnRNNRelu
elif rnn_mode == 'rnn_tanh':
rnn_class = cudnn_rnn.CudnnRNNTanh
rnn = rnn_class(num_layers=num_layers,
num_units=hidden_size,
input_size=input_shape[-1],
input_mode=input_mode,
direction=direction_mode,
dropout=dropout,
seed=0,
seed2=0)
# layer info (note in case of bidirectional, output from previous
# layers are concatenated).
layer_info = [input_shape[-1], hidden_size] + \
[hidden_size * (2 if is_bidirectional else 1),
hidden_size] * (num_layers - 1)
with tf.device('/cpu:0'):
nb_params = rnn.params_size().eval(session=get_session())
# ====== create parameters ====== #
# check parameters
if parameters is None:
if rnn_mode == 'lstm':
from odin.backend.init import lstm as init_func
elif rnn_mode == 'gru':
from odin.backend.init import gru as init_func
else:
from odin.backend.init import rnn as init_func
parameters = np.concatenate([
init_func(layer_info[i * 2],
layer_info[i * 2 + 1],
one_vector=True,
return_variable=False,
bidirectional=True if is_bidirectional else False)
for i in range(num_layers)
]).astype(FLOATX)
parameters = variable(parameters, name=name)
assert nb_params == get_shape(parameters)[0], \
"Require %d parameters but only %d provided" % (nb_params, get_shape(parameters)[0])
# check initial states
num_layers = num_layers * 2 if is_bidirectional else num_layers
h0 = zeros((num_layers, batch_size, hidden_size)) if h0 is None else h0
h0 = check_init_states(h0, num_layers, batch_size)
c0 = (zeros((num_layers, batch_size,
hidden_size)) if rnn_mode == 'lstm' and c0 is None else c0)
c0 = check_init_states(c0, num_layers, batch_size)
# preprocess arguments
args = {'input_h': h0}
if rnn_mode == 'lstm':
args['input_c'] = c0
# ====== get output ====== #
output = rnn(input_data=tf.transpose(X, (1, 0, 2)),
params=parameters,
is_training=bool(is_training()),
**args)
output = [tf.transpose(output[0], (1, 0, 2))] + list(output[1:])
add_shape(output[0], (input_shape[0], input_shape[1], hidden_size *
(2 if is_bidirectional else 1)))
for o in output[1:]:
add_shape(o, (num_layers, input_shape[0], hidden_size))
return output
def lstm(input_dim,
hidden_dim,
W_init=glorot_uniform,
b_init=constant(0.),
bidirectional=False,
one_vector=False,
return_variable=True,
name=None):
""" Fast initalize all Standard LSTM weights (without peephole connection)
Parameters
----------
one_vector: bool
if True, all the weights are flatten and concatenated into 1 big vector
return_variable: bool
if False, only return the numpy array
bidirectional: bool
if True, return parameters for both forward and backward RNN
Return
------
[W_i, b_wi, W_f, b_wf, W_c, b_wc, W_o, b_wo,
R_i, b_ri, R_f, b_rf, R_c, b_rc, R_o, b_ro]
"""
if name is None: name = uuid()
def init():
# input to hidden
W_i = W_init((input_dim, hidden_dim))
b_wi = b_init((hidden_dim))
W_f = W_init((input_dim, hidden_dim))
b_wf = b_init((hidden_dim))
W_c = W_init((input_dim, hidden_dim))
b_wc = b_init((hidden_dim))
W_o = W_init((input_dim, hidden_dim))
b_wo = b_init((hidden_dim))
# hidden to hidden
R_i = W_init((hidden_dim, hidden_dim))
b_ri = b_init((hidden_dim))
R_f = W_init((hidden_dim, hidden_dim))
b_rf = b_init((hidden_dim))
R_c = W_init((hidden_dim, hidden_dim))
b_rc = b_init((hidden_dim))
R_o = W_init((hidden_dim, hidden_dim))
b_ro = b_init((hidden_dim))
return [
W_i, b_wi, W_f, b_wf, W_c, b_wc, W_o, b_wo, R_i, b_ri, R_f, b_rf,
R_c, b_rc, R_o, b_ro
]
params = init() + init() if bidirectional else init()
roles = [WEIGHT, BIAS]
if one_vector:
params = [np.concatenate([p.flatten() for p in params])]
roles = [PARAMETER]
# names
if one_vector:
names = [name + '_lstm']
else:
names = [
"_W_i", "_b_wi", "_W_f", "_b_wf", "_W_c", "_b_wc", "_W_o", "_b_wo",
"_R_i", "_b_ri", "_R_f", "_b_rf", "_R_c", "_b_rc", "_R_o", "_b_ro"
]
if bidirectional:
names = [i + '_fw' for i in names] + [i + '_bw' for i in names]
names = [name + i for i in names]
# create variable or not
if return_variable:
params = [variable(p, name=n) for p, n in zip(params, names)]
for i, p in enumerate(params):
add_role(p, roles[i % 2])
return params if len(params) > 1 else params[0]
def transform(self, X, indices=None, sad=None,
save_ivecs=False, keep_stats=False, name=None):
"""
Parameters
----------
X : ndarray
Training data [n_samples, n_features]
indices : {Mapping, tuple, list}
in case the data is given by a list of files, `indices`
act as file indicator mapping from
'file_name' -> (start_index_in_X, end_index_in_X)
This mapping can be provided by a dictionary, or list of
tuple.
sad : ndarray
inspired by the "Speech Activity Detection" (SAD) indexing,
this array is indicator of which samples will be taken into
training; the shape should be [n_samples,] or [n_samples, 1]
save_ivecs : bool
if True, save extracted i-vectors to disk at path `ivec_[name]`
if False, return directly the i-vectors without saving
keep_stats : bool
if True, keep the zero and first order statistics.
The first order statistics could consume huge amount
of disk space. Otherwise, they are deleted after training
name : {None, str}
identity of the i-vectors (for re-using in future).
If None, a random name is used
"""
if not self.is_fitted:
raise ValueError("Ivector has not been fitted, call Ivector.fit(...) first")
n_files = X.shape[0] if indices is None else len(indices)
if name is None:
name = uuid(length=8)
else:
name = str(name)
# ====== init ====== #
z_path = self.get_z_path(name)
f_path = self.get_f_path(name)
if save_ivecs:
i_path = self.get_i_path(name)
else:
i_path = None
name_path = self.get_name_path(name)
# ====== check exist i-vector file ====== #
if i_path is not None and os.path.exists(i_path):
ivec = MmapData(path=i_path, read_only=True)
assert ivec.shape[0] == n_files and ivec.shape[1] == self.tv_dim,\
"Need i-vectors for %d files, found exists data at path:'%s' with shape:%s" % \
(n_files, i_path, ivec.shape)
return ivec
# ====== extract Z and F ====== #
if os.path.exists(z_path) and os.path.exists(f_path):
pass
else:
if os.path.exists(z_path):
os.remove(z_path)
if os.path.exists(f_path):
os.remove(f_path)
if os.path.exists(name_path):
os.remove(name_path)
_extract_zero_and_first_stats(X=X, sad=sad, indices=indices, gmm=self.gmm,
z_path=z_path, f_path=f_path, name_path=name_path)
Z = MmapData(path=z_path, read_only=True)
F = MmapData(path=f_path, read_only=True)
# ====== extract I-vec ====== #
ivec = self.tmat.transform_to_disk(path=i_path, Z=Z, F=F, dtype='float32')
# ====== clean ====== #
Z.close()
F.close()
if not keep_stats:
if os.path.exists(z_path):
os.remove(z_path)
if os.path.exists(f_path):
os.remove(f_path)
else:
print("Zero-order stats saved at:", ctext(z_path, 'cyan'))
print("First-order stats saved at:", ctext(f_path, 'cyan'))
return ivec
def transform(self,
X,
indices=None,
sad=None,
save_ivecs=False,
keep_stats=False,
name=None):
"""
Parameters
----------
X : ndarray
Training data [n_samples, n_features]
indices : {Mapping, tuple, list}
in case the data is given by a list of files, `indices`
act as file indicator mapping from
'file_name' -> (start_index_in_X, end_index_in_X)
This mapping can be provided by a dictionary, or list of
tuple.
sad : ndarray
inspired by the "Speech Activity Detection" (SAD) indexing,
this array is indicator of which samples will be taken into
training; the shape should be [n_samples,] or [n_samples, 1]
save_ivecs : bool
if True, save extracted i-vectors to disk at path `ivec_[name]`
if False, return directly the i-vectors without saving
keep_stats : bool
if True, keep the zero and first order statistics.
The first order statistics could consume huge amount
of disk space. Otherwise, they are deleted after training
name : {None, str}
identity of the i-vectors (for re-using in future).
If None, a random name is used
"""
if not self.is_fitted:
raise ValueError(
"Ivector has not been fitted, call Ivector.fit(...) first")
n_files = X.shape[0] if indices is None else len(indices)
if name is None:
name = uuid(length=8)
else:
name = str(name)
# ====== init ====== #
z_path = self.get_z_path(name)
f_path = self.get_f_path(name)
if save_ivecs:
i_path = self.get_i_path(name)
else:
i_path = None
name_path = self.get_name_path(name)
# ====== check exist i-vector file ====== #
if i_path is not None and os.path.exists(i_path):
ivec = MmapArray(path=i_path)
assert ivec.shape[0] == n_files and ivec.shape[1] == self.tv_dim,\
"Need i-vectors for %d files, found exists data at path:'%s' with shape:%s" % \
(n_files, i_path, ivec.shape)
return ivec
# ====== extract Z and F ====== #
if os.path.exists(z_path) and os.path.exists(f_path):
pass
else:
if os.path.exists(z_path):
os.remove(z_path)
if os.path.exists(f_path):
os.remove(f_path)
if os.path.exists(name_path):
os.remove(name_path)
_extract_zero_and_first_stats(X=X,
sad=sad,
indices=indices,
gmm=self.gmm,
z_path=z_path,
f_path=f_path,
name_path=name_path)
Z = MmapArray(path=z_path)
F = MmapArray(path=f_path)
# ====== extract I-vec ====== #
ivec = self.tmat.transform_to_disk(path=i_path,
Z=Z,
F=F,
dtype='float32')
# ====== clean ====== #
Z.close()
F.close()
if not keep_stats:
if os.path.exists(z_path):
os.remove(z_path)
if os.path.exists(f_path):
os.remove(f_path)
else:
print("Zero-order stats saved at:", ctext(z_path, 'cyan'))
print("First-order stats saved at:", ctext(f_path, 'cyan'))
return ivec
def serialize(nnops, path=None, save_variables=True, variables=[],
binary_output=False, override=False):
""" Serialize NNOp or list of NNOp and all necessary variables
to a folder.
Parameters
----------
nnops: NNOp, Object, or list; tuple of NNOp and Object
path: str
path to a folder
save_variables: bool
if True, save all variables related to all given NNOps
variables: list of tensorflow Variables
additional list of variables to be saved with this model
binary_output: bool (default: False)
if `False` (by default), original way tensorflow serialize
all variables, save all variables and nnop info to separated
files within a folder `path`
if `True`, convert all files in the folder to binary
and save to a dictionary with its relative path, if
`path` is not None, use pickle to save all binary data
to a file
override: bool
if True, remove existed folder to override everything.
Return
------
path: str
path to the folder that store NNOps and variables
"""
# ====== check output_mode ====== #
if path is None and not binary_output:
raise ValueError('`path` cannot be None if `binary_output=False`')
is_path_given = False if path is None else True
if path is None:
path = '/tmp/tmp' # default path
path_folder = path + uuid(length=25) if binary_output else path
# ====== getting save data and variables ====== #
vars = []
if save_variables:
for op in as_tuple(nnops):
if hasattr(op, 'variables'):
for v in as_tuple(op.variables):
if K.is_variable(v):
vars.append(v)
vars = list(set(vars + as_list(variables)))
# ====== checking path ====== #
# It is important to remove the `path_folder` AFTER getting all
# the variables, since this can remove the path to restored
# variables required in `op.variables`
if os.path.exists(path_folder):
if os.path.isfile(path_folder):
raise ValueError("path: '%s' is NOT a folder." % path_folder)
elif override:
shutil.rmtree(path_folder)
os.mkdir(path_folder)
else:
os.mkdir(path_folder)
nnops_path = os.path.join(path_folder, 'nnops.ai')
vars_path = os.path.join(path_folder, 'variables')
# save NNOps
with open(nnops_path, 'wb') as f:
cPickle.dump(nnops, f, protocol=cPickle.HIGHEST_PROTOCOL)
# save Variables
if len(vars) > 0:
K.save_variables(vars, vars_path)
# ====== convert folder to file or binary ====== #
if binary_output:
data = folder2bin(path_folder)
# only return binary data
if not is_path_given:
shutil.rmtree(path_folder)
return data
# given path, save binary to path
# check if override
if os.path.exists(path):
if override:
if os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path)
else:
raise RuntimeError("File at path: %s exists, cannot override." % path)
# write file
with open(path, 'wb') as f:
cPickle.dump(data, f, protocol=cPickle.HIGHEST_PROTOCOL)
shutil.rmtree(path_folder)
return path
def deserialize(path_or_data, force_restore_vars=True):
"""
Parameters
----------
path_or_data : {string, dict}
if a path is given (i.e. string types), load dumped model from
given folder
if a dictionary is given, load binary data directly
force_restore_vars : bool (default=True)
if `False`, this is special tricks, the unpickled NNOp stay useless
until its variables are restored,
but if we restore the variables right away, it create a
session and prevent any possibility of running
tensorflow with multiprocessing
=> store the `_restore_vars_path` in NNOp for later,
and restore the variable when the NNOp is actually in used.
Note
----
if `force_restore_vars = False`, this create 1 flaw,
if the nested NNOp is called before the unpickled NNOp
restore its variables, the nested Ops cannot acquire its
variables.
"""
data = None
path_folder = '/tmp/tmp_%s' % uuid(12)
delete_after = True
# ====== check path ====== #
if is_string(path_or_data):
# path to a file
if os.path.isfile(path_or_data):
with open(path_or_data, 'rb') as f:
data = cPickle.load(f)
# path to a folder
elif os.path.isdir(path_or_data):
path_folder = path_or_data
delete_after = False
else: # pickle string
data = cPickle.loads(path_or_data)
# given data
elif isinstance(path_or_data, dict):
data = path_or_data
# ====== check data ====== #
if data is not None:
bin2folder(data, path=path_folder)
path = path_folder
# ====== read normally from folder ====== #
nnops_path = os.path.join(path, 'nnops.ai')
vars_path = os.path.join(path, 'variables')
# ====== load the NNOps ====== #
if not os.path.exists(nnops_path):
raise ValueError("Cannot file path to serialized NNOps at: %s" % nnops_path)
with open(nnops_path, 'rb') as f:
nnops = cPickle.load(f)
# ====== load the Variables ====== #
if os.path.exists(vars_path + '.index'):
if force_restore_vars:
K.restore_variables(vars_path)
# delete cached folder
if delete_after:
shutil.rmtree(path)
else:
nnops._set_restore_info(vars_path, delete_after)
return nnops
def __init__(self, nb_classes, l1=0., l2=0.,
fit_intercept=True, confusion_matrix=True,
tol=1e-4, patience=3, rollback=True,
batch_size=1024, max_epoch=100, max_iter=None,
optimizer='adadelta', learning_rate=1.0, class_weight=None,
dtype='float32', seed=5218,
verbose=False, path=None, name=None):
super(LogisticRegression, self).__init__()
# ====== basic dimensions ====== #
if isinstance(nb_classes, (tuple, list, np.ndarray)):
self._labels = tuple([str(i) for i in nb_classes])
self._nb_classes = len(nb_classes)
elif is_number(nb_classes):
self._labels = tuple([str(i) for i in range(nb_classes)])
self._nb_classes = int(nb_classes)
self._feat_dim = None
self._dtype = np.dtype(dtype)
# ====== preprocessing class weight ====== #
if class_weight is None:
class_weight = np.ones(shape=(self.nb_classes,),
dtype=self.dtype)
elif is_number(class_weight):
class_weight = np.zeros(shape=(self.nb_classes,),
dtype=self.dtype) + class_weight
self._class_weight = class_weight
# ====== flags ====== #
self.l1 = float(l1)
self.l2 = float(l2)
self.fit_intercept = bool(fit_intercept)
self.confusion_matrix = bool(confusion_matrix)
# ====== internal states ====== #
self._is_fitted = False
# ====== others ====== #
if name is None:
name = uuid(length=8)
self._name = 'LogisticRegression_%s' % name
else:
self._name = str(name)
self._path = path
# ====== training ====== #
self.batch_size = int(batch_size)
self.max_epoch = max_epoch
self.max_iter = max_iter
if not is_string(optimizer):
raise ValueError("`optimizer` must be one of the following")
optimizer = optimizer.lower()
if optimizer not in _optimizer_list:
raise ValueError("`optimizer` must be one of the following: %s" %
str(list(_optimizer_list.keys())))
self._optimizer = _optimizer_list[optimizer.lower()](lr=float(learning_rate))
self._optimizer_name = optimizer
self._optimizer_lr = learning_rate
# ====== stop training ====== #
self.tol = float(tol)
self.patience = int(patience)
self.rollback = bool(rollback)
# ====== others ====== #
self._train_history = []
self._valid_history = []
self._rand_state = np.random.RandomState(seed=int(seed))
self.verbose = int(verbose)
def deserialize(path_or_data, force_restore_vars=True):
"""
Parameters
----------
path_or_data : {string, dict}
if a path is given (i.e. string types), load dumped model from
given folder
if a dictionary is given, load binary data directly
force_restore_vars : bool (default=True)
if `False`, this is special tricks, the unpickled NNOp stay useless
until its variables are restored,
but if we restore the variables right away, it create a
session and prevent any possibility of running
tensorflow with multiprocessing
=> store the `_restore_vars_path` in NNOp for later,
and restore the variable when the NNOp is actually in used.
Note
----
if `force_restore_vars = False`, this create 1 flaw,
if the nested NNOp is called before the unpickled NNOp
restore its variables, the nested Ops cannot acquire its
variables.
"""
data = None
path_folder = '/tmp/tmp_%s' % uuid(12)
delete_after = True
# ====== check path ====== #
if is_string(path_or_data):
# path to a file
if os.path.isfile(path_or_data):
with open(path_or_data, 'rb') as f:
data = cPickle.load(f)
# path to a folder
elif os.path.isdir(path_or_data):
path_folder = path_or_data
delete_after = False
else: # pickle string
data = cPickle.loads(path_or_data)
# given data
elif isinstance(path_or_data, dict):
data = path_or_data
# ====== check data ====== #
if data is not None:
bin2folder(data, path=path_folder)
path = path_folder
# ====== read normally from folder ====== #
nnops_path = os.path.join(path, 'nnops.ai')
vars_path = os.path.join(path, 'variables')
# ====== load the NNOps ====== #
if not os.path.exists(nnops_path):
raise ValueError("Cannot file path to serialized NNOps at: %s" %
nnops_path)
with open(nnops_path, 'rb') as f:
nnops = cPickle.load(f)
# ====== load the Variables ====== #
if os.path.exists(vars_path + '.index'):
if force_restore_vars:
K.restore_variables(vars_path)
# delete cached folder
if delete_after:
shutil.rmtree(path)
else:
nnops._set_restore_info(vars_path, delete_after)
return nnops
def __init__(self,
nb_classes,
l1=0.,
l2=0.,
fit_intercept=True,
confusion_matrix=True,
tol=1e-4,
patience=3,
rollback=True,
batch_size=1024,
max_epoch=100,
max_iter=None,
optimizer='adadelta',
learning_rate=1.0,
class_weight=None,
dtype='float32',
seed=1234,
verbose=False,
path=None,
name=None):
super(LogisticRegression, self).__init__()
# ====== basic dimensions ====== #
if isinstance(nb_classes, (tuple, list, np.ndarray)):
self._labels = tuple([str(i) for i in nb_classes])
self._nb_classes = len(nb_classes)
elif is_number(nb_classes):
self._labels = tuple([str(i) for i in range(nb_classes)])
self._nb_classes = int(nb_classes)
self._feat_dim = None
self._dtype = np.dtype(dtype)
# ====== preprocessing class weight ====== #
if class_weight is None:
class_weight = np.ones(shape=(self.nb_classes, ), dtype=self.dtype)
elif is_number(class_weight):
class_weight = np.zeros(shape=(self.nb_classes, ),
dtype=self.dtype) + class_weight
self._class_weight = class_weight
# ====== flags ====== #
self.l1 = float(l1)
self.l2 = float(l2)
self.fit_intercept = bool(fit_intercept)
self.confusion_matrix = bool(confusion_matrix)
# ====== internal states ====== #
self._is_fitted = False
# ====== others ====== #
if name is None:
name = uuid(length=8)
self._name = 'LogisticRegression_%s' % name
else:
self._name = str(name)
self._path = path
# ====== training ====== #
self.batch_size = int(batch_size)
self.max_epoch = max_epoch
self.max_iter = max_iter
if not is_string(optimizer):
raise ValueError("`optimizer` must be one of the following")
optimizer = optimizer.lower()
if optimizer not in _optimizer_list:
raise ValueError("`optimizer` must be one of the following: %s" %
str(list(_optimizer_list.keys())))
self._optimizer = _optimizer_list[optimizer.lower()](
lr=float(learning_rate))
self._optimizer_name = optimizer
self._optimizer_lr = learning_rate
# ====== stop training ====== #
self.tol = float(tol)
self.patience = int(patience)
self.rollback = bool(rollback)
# ====== others ====== #
self._train_history = []
self._valid_history = []
self._rand_state = np.random.RandomState(seed=int(seed))
self.verbose = int(verbose)
def serialize(nnops,
path=None,
save_variables=True,
variables=[],
binary_output=False,
override=False):
""" Serialize NNOp or list of NNOp and all necessary variables
to a folder.
Parameters
----------
nnops: NNOp, Object, or list; tuple of NNOp and Object
path: str
path to a folder
save_variables: bool
if True, save all variables related to all given NNOps
variables: list of tensorflow Variables
additional list of variables to be saved with this model
binary_output: bool (default: False)
if `False` (by default), original way tensorflow serialize
all variables, save all variables and nnop info to separated
files within a folder `path`
if `True`, convert all files in the folder to binary
and save to a dictionary with its relative path, if
`path` is not None, use pickle to save all binary data
to a file
override: bool
if True, remove existed folder to override everything.
Return
------
path: str
path to the folder that store NNOps and variables
"""
# ====== check output_mode ====== #
if path is None and not binary_output:
raise ValueError('`path` cannot be None if `binary_output=False`')
is_path_given = False if path is None else True
if path is None:
path = '/tmp/tmp' # default path
path_folder = path + uuid(length=25) if binary_output else path
# ====== getting save data and variables ====== #
vars = []
if save_variables:
for op in as_tuple(nnops):
if hasattr(op, 'variables'):
for v in as_tuple(op.variables):
if K.is_variable(v):
vars.append(v)
vars = list(set(vars + as_list(variables)))
# ====== checking path ====== #
# It is important to remove the `path_folder` AFTER getting all
# the variables, since this can remove the path to restored
# variables required in `op.variables`
if os.path.exists(path_folder):
if os.path.isfile(path_folder):
raise ValueError("path: '%s' is NOT a folder." % path_folder)
elif override:
shutil.rmtree(path_folder)
os.mkdir(path_folder)
else:
os.mkdir(path_folder)
nnops_path = os.path.join(path_folder, 'nnops.ai')
vars_path = os.path.join(path_folder, 'variables')
# save NNOps
with open(nnops_path, 'wb') as f:
cPickle.dump(nnops, f, protocol=cPickle.HIGHEST_PROTOCOL)
# save Variables
if len(vars) > 0:
K.save_variables(vars, vars_path)
# ====== convert folder to file or binary ====== #
if binary_output:
data = folder2bin(path_folder)
# only return binary data
if not is_path_given:
shutil.rmtree(path_folder)
return data
# given path, save binary to path
# check if override
if os.path.exists(path):
if override:
if os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path)
else:
raise RuntimeError(
"File at path: %s exists, cannot override." % path)
# write file
with open(path, 'wb') as f:
cPickle.dump(data, f, protocol=cPickle.HIGHEST_PROTOCOL)
shutil.rmtree(path_folder)
return path
def test_cudnn_rnn(self):
if get_ngpu() == 0:
return
print()
batch_size = 2
time_steps = 5
input_dim = 12
hidden_dim = 8
X = K.variable(value=np.random.rand(batch_size, time_steps, input_dim),
dtype='float32',
name='X')
for rnn_mode in ('lstm', 'rnn_relu', 'gru'):
for num_layers in [1, 2]:
for W_init in [
init_ops.glorot_uniform_initializer(seed=1234),
init_ops.random_normal_initializer(seed=1234)
]:
for b_init in [0, 1]:
for bidirectional in (True, False):
for skip_input in (False, ):
print('RNNmode:%s' % rnn_mode,
"#Layers:%d" % num_layers,
'Bidirectional:%s' % bidirectional,
'SkipInput:%s' % skip_input)
weights, biases = K.init_rnn(
input_dim=input_dim,
hidden_dim=hidden_dim,
num_gates=rnn_mode,
num_layers=num_layers,
W_init=W_init,
b_init=b_init,
skip_input=skip_input,
cudnn_vector=False,
is_bidirectional=bidirectional,
name=None)
# ====== check number of params ====== #
params1 = K.params_to_cudnn(weights, biases)
n = params1.shape[0].value
nb_params = cudnn_rnn_ops.cudnn_rnn_opaque_params_size(
rnn_mode=rnn_mode,
num_layers=num_layers,
num_units=hidden_dim,
input_size=input_dim,
input_mode='skip_input'
if skip_input else 'linear_input',
direction='bidirectional'
if bidirectional else 'unidirectional')
nb_params = K.eval(nb_params)
assert n == nb_params
# ====== check cannonical shape match ====== #
kwargs = {
'num_layers':
num_layers,
'num_units':
hidden_dim,
'input_mode':
'skip_input'
if skip_input else 'linear_input',
'direction':
'bidirectional'
if bidirectional else 'unidirectional'
}
if rnn_mode == 'lstm':
rnn = cudnn_rnn.CudnnLSTM(**kwargs)
elif rnn_mode == 'gru':
rnn = cudnn_rnn.CudnnGRU(**kwargs)
if rnn_mode == 'rnn_relu':
rnn = cudnn_rnn.CudnnRNNRelu(**kwargs)
if rnn_mode == 'rnn_tanh':
rnn = cudnn_rnn.CudnnRNNTanh(**kwargs)
rnn.build(input_shape=(None, None, input_dim))
assert len(weights) == len(
rnn.canonical_weight_shapes)
assert len(biases) == len(
rnn.canonical_bias_shapes)
for w, s in zip(weights,
rnn.canonical_weight_shapes):
assert tuple(w.shape.as_list()) == s
# ====== check params conversion ====== #
K.initialize_all_variables()
params2 = cudnn_rnn_ops.cudnn_rnn_canonical_to_opaque_params(
rnn_mode=rnn_mode,
num_layers=num_layers,
num_units=hidden_dim,
input_size=input_dim,
input_mode='skip_input'
if skip_input else 'linear_input',
direction='bidirectional'
if bidirectional else 'unidirectional',
weights=weights,
biases=biases)
assert np.all(
K.eval(params1) == K.eval(params2))
# ====== odin cudnn implementation ====== #
name = 'TEST' + uuid(length=25)
outputs = K.cudnn_rnn(
X=X,
num_units=hidden_dim,
rnn_mode=rnn_mode,
num_layers=num_layers,
parameters=None,
skip_input=skip_input,
is_bidirectional=bidirectional,
dropout=0.1,
name=name)
K.initialize_all_variables()
s0 = K.eval(outputs[0]).sum()
s1 = K.eval(outputs[1]).sum()
all_variables = K.get_all_variables(scope=name)
new_weights = [
i for i in all_variables
if K.role.has_roles(i, roles=K.role.Weight)
]
new_biases = [
i for i in all_variables
if K.role.has_roles(i, roles=K.role.Bias)
]
new_weights, new_biases = K.sort_cudnn_params(
new_weights, new_biases, rnn_mode=rnn_mode)
assert len(weights) == len(weights)
assert len(biases) == len(biases)
for i, j in zip(weights + biases,
new_weights + new_biases):
assert i.name.split(
'/')[-1] == j.name.split('/')[-1]
# ====== CudnnRNN wrapper ====== #
rnn = N.CudnnRNN(
num_units=hidden_dim,
W_init=new_weights,
b_init=new_biases,
rnn_mode=rnn_mode,
num_layers=num_layers,
skip_input=skip_input,
is_bidirectional=bidirectional,
return_states=True,
dropout=0.)
outputs = rnn(X)
K.initialize_all_variables()
y0 = K.eval(outputs[0]).sum()
y1 = K.eval(outputs[1]).sum()
assert y0 == s0
assert y1 == s1