def get(self, scope=None, name=None, full_name=None,
roles=None, match_all=False, exact=False,
beginning_scope=False):
""" Return all variables and tensor with given roles
Parameters
----------
scope : {None, string}
name of variable scope, any of the scope that match given name
will be selected
name : {None, string}
the name of tensor without the output indexing ":0" and the scope
full_name : {None, string}
the full name includes both scope and tensor name without the
output indexing ":0"
roles : {None, odin.backend.role}
specific roles of the tensor
match_all : bool (default: False)
If ``True``, checks if the variable has all given roles.
If ``False``, any of the roles is sufficient.
exact : bool (default: False)
If ``True``, use ``==`` for comparison to get exactly same roles.
If ``False``, use `issubclass` for comparison, hence, also match the
descendant roles.
beginning_scope : bool (default: True)
if True, the provide scope must be the beginning scope,
otherwise, it could be in the middle of multiple scopes
"""
alltensors = self.tensors + self.variables
# ====== by role ====== #
if roles is not None:
alltensors = [t for t in alltensors
if has_roles(t, roles=roles,
match_all=match_all, exact=exact)]
# ====== from general to detail ====== #
if scope is not None:
scope = str(scope)
if len(scope) == 0:
alltensors = [t for t in alltensors
if '/' not in t.name]
else:
scope_name_pattern = _TF_SCOPE_PATTERN(scope, beginning_scope)
alltensors = [t for t in alltensors
if len(scope_name_pattern.findall(t.name))]
# ====== filter by name ====== #
if name is not None:
name = as_tuple(name, t=string_types)
alltensors = [t for t in alltensors
if any((n == t.name.split('/')[-1] or
n.split(':')[0] == t.name.split('/')[-1].split(':')[0])
for n in name)]
# ====== full name ====== #
if full_name is not None:
full_name = as_tuple(full_name, t=string_types)
alltensors = [t for t in alltensors
if any((n == t.name or
n.split(':')[0] == t.name.split(':')[0])
for n in full_name)]
return alltensors
def _initialize(self, X):
# ====== check inputs dimensions ====== #
if not hasattr(X, 'shape'):
raise ValueError("`X` must have `shape` attribute.")
feat_dim = np.prod(X.shape[1:])
if self._feat_dim is None:
self._feat_dim = feat_dim
# validate input dimension
if feat_dim != self._feat_dim:
raise RuntimeError("Feature dimension mismatch %d and %d" %
(feat_dim, self.feat_dim))
# check if tensorflow op initalized
if hasattr(self, '_f_train'):
return
# ====== binary or multi-classes ====== #
if self.nb_classes == 2:
out_shape = (None, )
fn_activation = tf.nn.sigmoid
fn_loss = tf.losses.sigmoid_cross_entropy
fn_acc = K.metrics.binary_accuracy
else:
out_shape = (None, self.nb_classes)
fn_activation = tf.nn.softmax
fn_loss = tf.losses.softmax_cross_entropy
fn_acc = K.metrics.categorical_accuracy
# ====== create model ====== #
with tf.name_scope(self.name, 'logistic_regression'):
# inputs
self._X = K.placeholder(shape=(None, self.feat_dim),
dtype=self.dtype,
name='%s_input' % self.name)
self._y = K.placeholder(shape=out_shape,
dtype=self.dtype,
name='%s_output' % self.name)
# check the bias
if is_number(self.fit_intercept):
b_init = float(self.fit_intercept)
elif self.fit_intercept is False or \
self.fit_intercept is None:
b_init = None
else:
b_init = self.fit_intercept
# create the model and initialize
with K.variable_dtype(dtype=self.dtype):
self._model = N.Dense(
num_units=self.nb_classes,
W_init=init_ops.glorot_uniform_initializer(
seed=self._rand_state.randint()),
b_init=b_init,
activation=K.linear)
y_logits = self._model(self._X)
y_prob = fn_activation(y_logits)
# applying class weights
class_weights = tf.constant(value=self._class_weight,
dtype=self.dtype,
name="class_weights")
weights = tf.gather(
class_weights,
tf.cast(self._y, 'int32')
if self.nb_classes == 2 else tf.argmax(self._y, axis=-1))
# optimizer
params = [
v for v in self._model.variables
if has_roles(v, Weight) or has_roles(v, Bias)
]
losses = fn_loss(self._y, y_logits, weights=weights)
l1_norm = tf.norm(self._model.get('W'),
ord=1) if self.l1 > 0. else 0
l2_norm = tf.norm(self._model.get('W'),
ord=2) if self.l2 > 0. else 0
losses = losses + self.l1 * l1_norm + self.l2 * l2_norm
acc = fn_acc(self._y, y_prob)
updates = self._optimizer.get_updates(losses, params)
# create function
if self.confusion_matrix:
cm = K.metrics.confusion_matrix(y_true=self._y,
y_pred=y_prob,
labels=self.nb_classes)
metrics = [losses, acc, cm
] if self.confusion_matrix else [losses, acc]
self._f_train = K.function(inputs=(self._X, self._y),
outputs=metrics,
updates=updates,
training=True)
self._f_score = K.function(inputs=(self._X, self._y),
outputs=metrics,
training=False)
self._f_pred_prob = K.function(inputs=self._X,
outputs=y_prob,
training=False)
self._f_pred_logit = K.function(inputs=self._X,
outputs=y_logits,
training=False)
return self
def auxiliary_tensors(self): return [t for t in self._tensors if has_roles(t, Auxiliary)]
def parameters(self): return [var for var in self._variables if has_roles(var, Parameter)]
def minimize(self, loss, var_list=None,
roles=[TrainableParameter], exclude_roles=[],
reuse_scope=None, verbose=False):
"""
Parameters
----------
loss : {Tensor, list of Tensor}
the loss for minimize
var_list : {None, list of tensorflow.Variable}
if None, automatically selecting the Variables, by getting
all Variables associated with `loss` and using given
`roles` and `exclude_roles`
roles : {None, list of odin.backend.role}
in case of `var_list=None`, all Variables with given role
in this list will be selected for calculating gradients.
exclude_roles : {None, list of odin.backend.role}
all Variables have role in this list will be ignored.
reuse_scope : {None, string}
if provided, searching for all `OptimizerVariable` within
given scope, and set the variables of this optimizer to
the desire values
verbose : bool (default: False)
if True, print out all found variables and their roles
"""
if exclude_roles is None:
exclude_roles = []
# ====== get all relevant variables ====== #
if var_list is not None:
all_variables = as_tuple(var_list, t=is_variable)
else:
all_variables = ComputationGraph(loss).variables
# ====== filtering by Roles ====== #
trainable = [v for v in all_variables
if has_roles(v, roles=roles)]
if isinstance(exclude_roles, Role) or len(exclude_roles) > 0:
trainable = [v for v in trainable
if not has_roles(v, roles=exclude_roles)]
# ====== filtering by dtype ====== #
# remove all boolean and string dtype
trainable = [v
for v in trainable
if 'int' in str(v.dtype) or 'float' in str(v.dtype)]
# ====== verbose ====== #
if bool(verbose):
# print loss first
print("Loss:", ctext(str(loss), 'yellow'))
# organize variable by role
role_vars = defaultdict(list)
for var in trainable:
# remove all tensorflow collection name string
roles = [r
for r in get_roles(var, return_string=False)
if not isinstance(r, string_types)]
# only select highest role in the hierarchy
high_roles = []
for r1 in roles:
found_ancester = False
for r2 in roles:
if r1 != r2 and issubclass(r2, r1):
found_ancester = True
if not found_ancester:
high_roles.append(r1)
# append to mapping role -> var_list
for r in roles:
role_vars[r.__name__].append(var)
# print debug info (this is ugly but look nice and fun)
if len(role_vars) > 0:
max_name_length = str(max(
max(len(i.name) for i in v)
for v in role_vars.values()))
max_shape_length = str(max(
max(len(str(i.shape.as_list())) for i in v)
for v in role_vars.values()))
for role, var_list in sorted(role_vars.items(),
key=lambda x: str(x[0])):
print('Role:', ctext(role, 'yellow'))
for var in sorted(var_list, key=lambda x: x.name):
print(' ',
('%-' + max_name_length + 's') % var.name.replace('/', ' '),
ctext(('%-' + max_shape_length + 's') % var.shape.as_list(), 'magenta'),
ctext(var.dtype.base_dtype.name, 'cyan'))
# ====== get the updates ====== #
updates = self.get_updates(loss_or_grads=loss, params=trainable)
# with tf.variable_scope(self.name):
# updates = self.algorithm.minimize(loss=loss, global_step=self._step,
# var_list=trainable)
# ====== re-use variable from previous scope ====== #
if isinstance(reuse_scope, string_types):
old_variables = [v for v in get_all_variables(scope=reuse_scope)
if has_roles(v, OptimizerVariable)]
old_variables = sorted(old_variables,
key=lambda x: x.name.replace(reuse_scope, ''))
old_varname = [v.name.replace(reuse_scope, '') for v in old_variables]
# the hyper-parameters may not be included in `old_variables`
# so they are needed to be excluded in new_variables
new_variables = [v for v in get_all_variables(scope=self.name)
if has_roles(v, OptimizerVariable)]
new_variables = sorted(new_variables,
key=lambda x: x.name.replace(self.name, ''))
new_variables = [v for v in new_variables
if v.name.replace(self.name, '') in old_varname]
assert len(old_variables) == len(new_variables), \
"Number of variables in scope:'%s' is %d mismatch scope:'%s' with %d variables"\
% (reuse_scope, len(old_variables), self.name, len(new_variables))
for v_old, v_new in zip(old_variables, new_variables):
assert v_new.shape == v_old.shape
set_value(v_new, get_value(v_old), return_ops=False)
return updates
def _initialize(self, X):
# ====== check inputs dimensions ====== #
if not hasattr(X, 'shape'):
raise ValueError("`X` must have `shape` attribute.")
feat_dim = np.prod(X.shape[1:])
if self._feat_dim is None:
self._feat_dim = feat_dim
# validate input dimension
if feat_dim != self._feat_dim:
raise RuntimeError("Feature dimension mismatch %d and %d" %
(feat_dim, self.feat_dim))
# check if tensorflow op initalized
if hasattr(self, '_f_train'):
return
# ====== binary or multi-classes ====== #
if self.nb_classes == 2:
out_shape = (None,)
fn_activation = tf.nn.sigmoid
fn_loss = tf.losses.sigmoid_cross_entropy
fn_acc = K.metrics.binary_accuracy
else:
out_shape = (None, self.nb_classes)
fn_activation = tf.nn.softmax
fn_loss = tf.losses.softmax_cross_entropy
fn_acc = K.metrics.categorical_accuracy
# ====== create model ====== #
with tf.name_scope(self.name, 'logistic_regression'):
# inputs
self._X = K.placeholder(shape=(None, self.feat_dim),
dtype=self.dtype,
name='%s_input' % self.name)
self._y = K.placeholder(shape=out_shape,
dtype=self.dtype,
name='%s_output' % self.name)
# check the bias
if is_number(self.fit_intercept):
b_init = float(self.fit_intercept)
elif self.fit_intercept is False or \
self.fit_intercept is None:
b_init = None
else:
b_init = self.fit_intercept
# create the model and initialize
with K.variable_dtype(dtype=self.dtype):
self._model = N.Dense(num_units=self.nb_classes,
W_init=init_ops.glorot_uniform_initializer(seed=self._rand_state.randint()),
b_init=b_init,
activation=K.linear)
y_logits = self._model(self._X)
y_prob = fn_activation(y_logits)
# applying class weights
class_weights = tf.constant(value=self._class_weight,
dtype=self.dtype,
name="class_weights")
weights = tf.gather(class_weights,
tf.cast(self._y, 'int32') if self.nb_classes == 2 else
tf.argmax(self._y, axis=-1))
# optimizer
params = [v for v in self._model.variables
if has_roles(v, Weight) or has_roles(v, Bias)]
losses = fn_loss(self._y, y_logits, weights=weights)
l1_norm = tf.norm(self._model.get('W'), ord=1) if self.l1 > 0. else 0
l2_norm = tf.norm(self._model.get('W'), ord=2) if self.l2 > 0. else 0
losses = losses + self.l1 * l1_norm + self.l2 * l2_norm
acc = fn_acc(self._y, y_prob)
updates = self._optimizer.get_updates(losses, params)
# create function
if self.confusion_matrix:
cm = K.metrics.confusion_matrix(y_true=self._y, y_pred=y_prob,
labels=self.nb_classes)
metrics = [losses, acc, cm] if self.confusion_matrix else [losses, acc]
self._f_train = K.function(inputs=(self._X, self._y),
outputs=metrics,
updates=updates,
training=True)
self._f_score = K.function(inputs=(self._X, self._y),
outputs=metrics,
training=False)
self._f_pred_prob = K.function(inputs=self._X,
outputs=y_prob,
training=False)
self._f_pred_logit = K.function(inputs=self._X,
outputs=y_logits,
training=False)
return self