def load(self, fullpath=None): """ Load model """ fullpath = fullpath if fullpath else self.filename logger.debug("Loading model: '%s'", fullpath) try: network = load_model(self.filename, custom_objects=get_custom_objects()) except ValueError as err: if str(err).lower().startswith( "cannot create group in read only mode"): self.convert_legacy_weights() return True logger.warning( "Failed loading existing training data. Generating new models") logger.debug("Exception: %s", str(err)) return False except OSError as err: # pylint: disable=broad-except logger.warning( "Failed loading existing training data. Generating new models") logger.debug("Exception: %s", str(err)) return False self.config = network.get_config() self.network = network # Update network with saved model self.network.name = self.name return True
def test_relation_layer(): backend.set_session(None) input_data = np.array( [[[3, 2, 4], [1, 5, 2]], [[30, 20, 40], [10, 50, 20]]], dtype=np.float32) weights = np.array([[1, 0], [5, 6], [7, 8]], dtype=np.float32) bias = np.array([4, 7], dtype=np.float32) expected_output = np.array([[[6926, 8642], [6845, 8822]], [[663440, 807500], [655340, 825500]]], dtype=np.float32) tf.reset_default_graph() get_custom_objects()['Relation'] = Relation kwargs = { 'relations': 2, 'kernel_initializer': tf.constant_initializer(weights), 'bias_initializer': tf.constant_initializer(bias) } a = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) output = testing_utils.layer_test(Relation, kwargs=kwargs, input_data=input_data, expected_output=expected_output) if not np.array_equal(output, expected_output): raise AssertionError( 'The output is not equal to our expected output')
self.scale_factor = scale_factor self.data_format = "channels_last" def build(self, input_shape): pass def call(self, x, mask=None): y = depth_to_space(x, self.scale_factor, self.data_format) return y def compute_output_shape(self, input_shape): if self.data_format == 'channels_first': b, k, r, c = input_shape return (b, k // (self.scale_factor**2), r * self.scale_factor, c * self.scale_factor) else: b, r, c, k = input_shape return (b, r * self.scale_factor, c * self.scale_factor, k // (self.scale_factor**2)) def get_config(self): config = { 'scale_factor': self.scale_factor, 'data_format': self.data_format } base_config = super(SubPixelUpscaling, self).get_config() return dict(list(base_config.items()) + list(config.items())) get_custom_objects().update({'SubPixelUpscaling': SubPixelUpscaling})
loc = os.path.join(self.path(), path) print("Loading weights", loc) self.probabilityNetwork.load_weights(loc) return self def save_model(self, path): self.probabilityNetwork.save(path) def load_model(self, path): # loc = os.path.join(self.path(), path) # print("Loading model", path) self.probabilityNetwork = load_model(path) return self def path(self): return os.path.dirname(os.path.realpath(__file__)) def contrastive_loss(y_true, y_pred): '''Contrastive loss from Hadsell-et-al.'06 http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf ''' margin = 1 return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0))) from tensorflow.python.keras.utils import get_custom_objects get_custom_objects().update({"contrastive_loss": contrastive_loss})
base_lr_params = [p for p in params if self._get_multiplier(p) is None] updates = [] base_lr = self._optimizer.lr for param, multiplier in mult_lr_params.items(): self._optimizer.lr = base_lr * multiplier updates.extend(self._optimizer.get_updates(loss, [param])) self._optimizer.lr = base_lr updates.extend(self._optimizer.get_updates(loss, base_lr_params)) return updates def get_config(self): config = {'optimizer': self._class, 'lr_multipliers': self._lr_multipliers} base_config = self._optimizer.get_config() return dict(list(base_config.items()) + list(config.items())) def __getattr__(self, name): return getattr(self._optimizer, name) def __setattr__(self, name, value): if name.startswith('_'): super(LearningRateMultiplier, self).__setattr__(name, value) else: self._optimizer.__setattr__(name, value) get_custom_objects().update({'LearningRateMultiplier': LearningRateMultiplier})
class DropConnect(KL.Layer): def __init__(self, drop_connect_rate=0., **kwargs): super().__init__(**kwargs) self.drop_connect_rate = drop_connect_rate def call(self, inputs, training=None): def drop_connect(): keep_prob = 1.0 - self.drop_connect_rate # Compute drop_connect tensor batch_size = tf.shape(inputs)[0] random_tensor = keep_prob random_tensor += tf.random.uniform([batch_size, 1, 1, 1], dtype=inputs.dtype) binary_tensor = tf.floor(random_tensor) output = tf.divide(inputs, keep_prob) * binary_tensor return output return K.in_train_phase(drop_connect, inputs, training=training) def get_config(self): config = super().get_config() config['drop_connect_rate'] = self.drop_connect_rate return config get_custom_objects().update({ 'DropConnect': DropConnect, 'Swish': Swish, })
from tensorflow.python.keras.engine.base_layer import AddMetric, AddLoss from tensorflow.python.keras.utils import get_custom_objects from models.layers import * from models.optimizer import * from models.losses import * # Custom Layer 구성하기 get_custom_objects().update({ "ConvFeatureExtractor": ConvFeatureExtractor, "ResidualConvFeatureExtractor": ResidualConvFeatureExtractor, "Map2Sequence": Map2Sequence, "BGRUEncoder": BGRUEncoder, "CTCDecoder": CTCDecoder, "DotAttention": DotAttention, "JamoCompose": JamoCompose, 'JamoDeCompose': JamoDeCompose, "JamoEmbedding": JamoEmbedding, "JamoClassifier": JamoClassifier, "TeacherForcing": TeacherForcing }) # Custom Optimizer 구성하기 get_custom_objects().update({'AdamW': AdamW, 'RectifiedAdam': RectifiedAdam}) # Custom Loss 구성하기 get_custom_objects().update({ 'ctc_loss': ctc_loss, "JamoCategoricalCrossEntropy": JamoCategoricalCrossEntropy })
return super().build(input_shape) def call(self, inputs, **kwargs): #if not K.is_tensor(inputs): # raise ValueError( # 'The layer can be called only with one tensor as an argument') _, seq_len, d_model = K.int_shape(inputs) # The first thing we need to do is to perform affine transformations # of the inputs to get the Queries, the Keys and the Values. qkv = K.dot(K.reshape(inputs, [-1, d_model]), self.qkv_weights) # splitting the keys, the values and the queries before further # processing pre_q, pre_k, pre_v = [ K.reshape( # K.slice(qkv, (0, i * d_model), (-1, d_model)), qkv[:, i * d_model:(i + 1) * d_model], (-1, seq_len, self.num_heads, d_model // self.num_heads)) for i in range(3)] attention_out = self.attention(pre_q, pre_v, pre_k, seq_len, d_model, training=kwargs.get('training')) return attention_out def compute_output_shape(self, input_shape): return input_shape get_custom_objects().update({ 'MultiHeadSelfAttention': MultiHeadSelfAttention, 'MultiHeadAttention': MultiHeadAttention, })
print('{} - {}'.format(type(sequence_length), type(d_model))) print('{}'.format(type(self.max_depth))) self.word_position_embeddings = self.add_weight( shape=(sequence_length, d_model), initializer='uniform', name='word_position_embeddings', trainable=True) self.depth_embeddings = self.add_weight( shape=(self.max_depth, d_model), initializer='uniform', name='depth_position_embeddings', trainable=True) super().build(input_shape) def call(self, inputs, **kwargs): depth = kwargs.get('step') if depth is None: raise ValueError("Please, provide current Transformer's step" "using 'step' keyword argument.") result = inputs + self.word_position_embeddings if depth is not None: result = result + self.depth_embeddings[depth] return result get_custom_objects().update({ 'TransformerCoordinateEmbedding': TransformerCoordinateEmbedding, 'AddCoordinateEncoding': AddCoordinateEncoding, 'AddPositionalEncoding': AddCoordinateEncoding, })
# some real computational time. if self.zeros_like_input is None: self.zeros_like_input = K.zeros_like( inputs, name='zeros_like_input') # just because K.any(step_is_active) doesn't work in PlaidML any_step_is_active = K.greater( K.sum(K.cast(step_is_active, 'int32')), 0) step_weighted_output = K.switch( any_step_is_active, K.expand_dims(halting_prob, -1) * inputs, self.zeros_like_input) if self.weighted_output is None: self.weighted_output = step_weighted_output else: self.weighted_output += step_weighted_output return [inputs, self.weighted_output] def compute_output_shape(self, input_shape): return [input_shape, input_shape] def finalize(self): self.add_loss(self.ponder_cost) get_custom_objects().update({ 'LayerNormalization': LayerNormalization, 'TransformerTransition': TransformerTransition, 'TransformerACT': TransformerACT, 'gelu': gelu, })
import pdb import netCDF4 as nc import xarray as xr import h5py from glob import glob import sys, os import seaborn as sns from tf_losses import * #from .models import PartialReLU, QLayer, ELayer from tf_models import PartialReLU, QLayer, ELayer #from tensorflow.keras.utils.generic_utils import get_custom_objects from tensorflow.python.keras.utils import get_custom_objects metrics_dict = dict([(f.__name__, f) for f in all_metrics]) get_custom_objects().update(metrics_dict) get_custom_objects().update({ 'PartialReLU': PartialReLU, 'QLayer': QLayer, 'ELayer': ELayer, }) from configargparse import ArgParser from ipykernel.kernelapp import IPKernelApp def in_notebook(): return IPKernelApp.initialized() if in_notebook():
if self.add_biases: projected = K.bias_add(projected, self.embedding_weights['biases'], data_format='channels_last') if 0 < self.projection_dropout < 1: projected = K.in_train_phase( lambda: K.dropout(projected, self.projection_dropout), projected, training=kwargs.get('training')) attention = K.dot(projected, K.transpose(embedding_matrix)) if self.scaled_attention: # scaled dot-product attention, described in # "Attention is all you need" (https://arxiv.org/abs/1706.03762) sqrt_d = K.constant(math.sqrt(emb_output_dim), dtype=K.floatx()) attention = attention / sqrt_d result = K.reshape( self.activation(attention), (input_shape_tensor[0], input_shape_tensor[1], emb_input_dim)) return result def compute_output_shape(self, input_shape): main_input_shape, embedding_matrix_shape = input_shape emb_input_dim, _ = embedding_matrix_shape return main_input_shape[0], main_input_shape[1], emb_input_dim get_custom_objects().update({ 'ReusableEmbedding': ReusableEmbedding, 'TiedOutputEmbedding': TiedOutputEmbedding, })
broadcast_shape = [1] * len(input_shape) if self.axis is not None: broadcast_shape[self.axis] = input_shape[self.axis] if self.scale: broadcast_gamma = K.reshape(self.gamma, broadcast_shape) normed = normed * broadcast_gamma if self.center: broadcast_beta = K.reshape(self.beta, broadcast_shape) normed = normed + broadcast_beta return normed def get_config(self): config = { 'axis': self.axis, 'epsilon': self.epsilon, 'center': self.center, 'scale': self.scale, 'beta_initializer': initializers.serialize(self.beta_initializer), 'gamma_initializer': initializers.serialize(self.gamma_initializer), 'beta_regularizer': regularizers.serialize(self.beta_regularizer), 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer), 'beta_constraint': constraints.serialize(self.beta_constraint), 'gamma_constraint': constraints.serialize(self.gamma_constraint) } base_config = super(InstanceNormalization, self).get_config() return dict(list(base_config.items()) + list(config.items())) get_custom_objects().update({'InstanceNormalization': InstanceNormalization})
else: # When softmax activation function is used for output operation, we # use logits from the softmax function directly to compute loss in order # to prevent collapsing zero when training. # See b/117284466 assert len(output.op.inputs) == 1 output = output.op.inputs[0] rank = len(output.shape) axis = axis % rank if axis != rank - 1: permutation = list(range(axis)) + list(range(axis + 1, rank)) + [axis] output = array_ops.transpose(output, perm=permutation) output_shape = output.shape targets = cast(flatten(target), 'int64') logits = array_ops.reshape(output, [-1, int(output_shape[-1])]) res = nn.sparse_softmax_cross_entropy_with_logits( labels=targets, logits=logits) if len(output_shape) >= 3: # If our output includes timesteps or spatial dimensions we need to reshape return array_ops.reshape(res, array_ops.shape(output)[:-1]) else: return res # Doing this allows you to then to refer to your custom object via string. get_custom_objects().update({'cast':cast, 'sparse_categorical_crossentropy': sparse_categorical_crossentropy})
def __init__(self, scale_factor=2, data_format=None, **kwargs): super(SubPixelUpscaling, self).__init__(**kwargs) self.scale_factor = scale_factor self.data_format = "channels_last" def build(self, input_shape): pass def call(self, x, mask=None): y = depth_to_space(x, self.scale_factor, self.data_format) return y def compute_output_shape(self, input_shape): if self.data_format == 'channels_first': b, k, r, c = input_shape return (b, k // (self.scale_factor ** 2), r * self.scale_factor, c * self.scale_factor) else: b, r, c, k = input_shape return (b, r * self.scale_factor, c * self.scale_factor, k // (self.scale_factor ** 2)) def get_config(self): config = {'scale_factor': self.scale_factor, 'data_format': self.data_format} base_config = super(SubPixelUpscaling, self).get_config() return dict(list(base_config.items()) + list(config.items())) get_custom_objects().update({'SubPixelUpscaling': SubPixelUpscaling})
'center': self.center, 'scale': self.scale, 'beta_initializer': initializers.serialize(self.beta_initializer), 'gamma_initializer': initializers.serialize(self.gamma_initializer), 'beta_regularizer': regularizers.serialize(self.beta_regularizer), 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer), 'beta_constraint': constraints.serialize(self.beta_constraint), 'gamma_constraint': constraints.serialize(self.gamma_constraint) } base_config = super(InstanceNormalization, self).get_config() return dict(list(base_config.items()) + list(config.items())) get_custom_objects().update({'InstanceNormalization': InstanceNormalization}) def _moments(x, axes, shift=None, keep_dims=False): ''' Wrapper over tensorflow backend call ''' if K.backend() == 'tensorflow': import tensorflow as tf return tf.nn.moments(x, axes, shift=shift, keep_dims=keep_dims) elif K.backend() == 'theano': import theano.tensor as T mean_batch = T.mean(x, axis=axes, keepdims=keep_dims) var_batch = T.var(x, axis=axes, keepdims=keep_dims) return mean_batch, var_batch else: raise RuntimeError("Currently does not support CNTK backend")
self.input_spec = InputSpec(ndim=4) def _compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() if self.data_format == 'channels_first': height = self.size[0] * input_shape[ 2] if input_shape[2] is not None else None width = self.size[1] * input_shape[ 3] if input_shape[3] is not None else None return tensor_shape.TensorShape( [input_shape[0], input_shape[1], height, width]) else: height = self.size[0] * input_shape[ 1] if input_shape[1] is not None else None width = self.size[1] * input_shape[ 2] if input_shape[2] is not None else None return tensor_shape.TensorShape( [input_shape[0], height, width, input_shape[3]]) def call(self, inputs): return resize_images_bilinear(inputs, self.size[0], self.size[1], self.data_format) def get_config(self): config = {'size': self.size, 'data_format': self.data_format} base_config = super(BilinearUpSampling2D, self).get_config() return dict(list(base_config.items()) + list(config.items())) # add this to custom objects for restoring model save files get_custom_objects().update({'SeparableConv2DKeras': SeparableConv2DKeras, 'BilinearUpSampling2D':BilinearUpSampling2D})