def test_separable_conv1d2d(): in_w = 32 in_h = 32 in_ch = 3 kernel = 32 ker_w = 3 ker_h = 3 model = Sequential( SeparableConv1D(kernel, (ker_w,), padding="same", input_shape=(in_w, in_ch)) ) flops = get_flops(model, batch_size=1) assert ( flops == 2 * ker_w * in_w * in_ch # depthwise conv with no bias + (2 * in_ch + 1) * in_w * kernel # pointwise conv ) model = Sequential( SeparableConv2D( kernel, (ker_w, ker_h), padding="same", input_shape=(in_w, in_h, in_ch) ) ) flops = get_flops(model, batch_size=1) assert ( flops == 2 * ker_w * ker_h * in_w * in_h * in_ch # depthwise conv with no bias + (2 * in_ch + 1) * in_w * in_h * kernel # pointwise conv )
def test_conv1d2d3d(): in_w = 32 in_h = 32 in_z = 32 in_ch = 3 kernel = 32 ker_w = 3 ker_h = 3 ker_z = 3 model = Sequential( Conv1D(kernel, (ker_w,), padding="same", input_shape=(in_w, in_ch)) ) flops = get_flops(model, batch_size=1) assert flops == ((2 * ker_w * in_ch) + 1) * in_w * kernel model = Sequential( Conv2D(kernel, (ker_w, ker_h), padding="same", input_shape=(in_w, in_h, in_ch)) ) flops = get_flops(model, batch_size=1) assert flops == ((2 * ker_w * ker_h * in_ch) + 1) * in_w * in_h * kernel model = Sequential( Conv3D( kernel, (ker_w, ker_h, ker_z), padding="same", input_shape=(in_w, in_h, in_z, in_ch), ) ) flops = get_flops(model, batch_size=1) assert ( flops == ((2 * ker_w * ker_h * ker_z * in_ch) + 1) * in_w * in_h * in_z * kernel )
def test_raise(): try: raised = False get_flops(None) except KeyError: raised = True assert raised
def test_layernormalization(): """ layer normalization is calculated as follows, fused_ 1. (2 ops * |var|) inv = rsqrt(var + eps) 2. (1 ops * |var|) inv *= gamma (scale) 3. (|x| + |mean| + |var| ops) x' = inv * x + beta (shift) - mean * inv , where |var| = |mean| = 1 in default Thus, 5 + input element size. Use nn.fused_batch_norm (gen_nn_ops.fused_batch_norm_v3) for layer normalization, above calculation. gen_nn_ops.fused_batch_norm_v3 support only 4D, so reshape data as 4D and input them. squeezed_shape (ndim ops), scale (|x| ops) and shift (not float ops) is calculated. NOTE: is_training = True, if make trainable attributes of tf.keras.Model instanse False. So, statistics will be incorrect. """ in_w = 32 in_h = 32 in_ch = 3 input_shape = (in_w, in_ch) model = Sequential( LayerNormalization( scale=False, center=False, input_shape=input_shape, )) flops = get_flops(model, batch_size=1) assert flops == len(input_shape) + 1 input_shape = (in_w, in_h, in_ch) model = Sequential( LayerNormalization( scale=False, center=False, input_shape=input_shape, )) flops = get_flops(model, batch_size=1) assert flops == len(input_shape) + 1 input_shape = (in_w, in_h, in_ch) model = Sequential( LayerNormalization( beta_initializer="ones", gamma_initializer="ones", input_shape=input_shape, )) flops = get_flops(model, batch_size=1) assert (flops == len(input_shape) + 1 + 5 + in_w * in_h * in_ch + 5 * in_ch + in_w * in_h * in_ch), "fused is True. check gen_nn_ops.fused_batch_norm_v3"
def evaluate_test_data(): test_dataset = Dataset(x_test_dir,y_test_dir,classes=CLASSES,augmentation=get_validation_augmentation(),preprocessing=get_preprocessing(preprocess_input),) test_dataloader = Dataloder(test_dataset, batch_size=1, shuffle=False) model = sm.Unet(BACKBONE, classes=1, activation='sigmoid', input_shape=(320,320,3)) # load best weights model.load_weights('./best_model.h5') optim = keras.optimizers.Adam(LR) # Segmentation models losses can be combined together by '+' and scaled by integer or float factor dice_loss = sm.losses.DiceLoss() focal_loss = sm.losses.BinaryFocalLoss() total_loss = dice_loss + (0.5 * focal_loss) metrics = [sm.metrics.IOUScore(threshold=0.5), sm.metrics.FScore(threshold=0.5)] # compile keras model with defined optimozer, loss and metrics model.compile(optim, total_loss, metrics) #compute number of flops for the trained model flops = get_flops(model, batch_size=1) print(f"FLOPS: {flops / 10 ** 9:.03} G") # compute the inference time start = timer() scores = model.evaluate_generator(test_dataloader) end = timer() print(str(end - start)) print("Loss: {:.5}".format(scores[0])) for metric, value in zip(metrics, scores[1:]): print("mean {}: {:.5}".format(metric.__name__, value))
def test_additive_attention(): """ Bahdanau-style attention. query (batch, Tq, dim), key (batch, Tv, dim) and value (batch, Tv, dim) are inputs. following computations is processed. 1. reshape query as shape [batch, Tq, 1, dim] and value as shape [batch, 1, Tv, dim] 2. broadcasting multiply between additive of above as output shape [batch, Tq, Tv, dim] 3. reduce_sum above with dim axis as output shape [batch, Tq, Tv] 4. softmax of above 5. MatMul between 4. and value as output shape [batch, Tq, dim] """ Tq = 10 Tv = 10 dim = 16 q_shape = (Tq, dim) k_shape = (Tv, dim) v_shape = (Tv, dim) q = Input(q_shape) k = Input(k_shape) v = Input(v_shape) x = AdditiveAttention()([q, k, v]) model = Model([q, k, v], x) flops = get_flops(model, batch_size=1) assert ( flops == Tq * Tv * dim # No.2 (multiply) + Tq * Tv * dim # No.3 (add) + Tq * Tv * (dim - 1) # No.3 (reduce_sum) + 5 * Tq * Tv # No.4 (softmax) + 2 * Tv * Tq * dim # No.5 (MatMul) )
def test_attention(): """ Luong-style attention. query (batch, Tq, dim), key (batch, Tv, dim) and value (batch, Tv, dim) are inputs. following computations is processed. 1. query-key dot-product as output shape [batch, Tq, Tv] 2. softmax of above 3. MatMul between 2. and value as output shape [batch, Tq, dim] """ Tq = 10 Tv = 10 dim = 16 q_shape = (Tq, dim) k_shape = (Tv, dim) v_shape = (Tv, dim) q = Input(q_shape) k = Input(k_shape) v = Input(v_shape) x = Attention()([q, k, v]) model = Model([q, k, v], x) flops = get_flops(model, batch_size=1) assert ( flops == 2 * Tq * Tv * dim # No.1 (dot-product (MatMul)) + 5 * Tq * Tv # No.2 (softmax) + 2 * Tv * Tq * dim # No.3 (MatMul) )
def test_maxpooling1d2d3d(): in_w = 32 in_h = 32 kernel = 32 pool_w = 2 pool_h = 2 model = Sequential(MaxPooling1D(pool_size=(pool_w,), input_shape=(in_w, kernel))) flops = get_flops(model, batch_size=1) assert flops == in_w * kernel model = Sequential( MaxPooling2D(pool_size=(pool_w, pool_h), input_shape=(in_w, in_h, kernel)) ) flops = get_flops(model, batch_size=1) assert flops == in_w * in_h * kernel
def test_global_averagepooling1d2d3d(): in_w = 32 in_h = 32 in_z = 32 kernel = 32 model = Sequential(GlobalAveragePooling1D(input_shape=(in_w, kernel))) flops = get_flops(model, batch_size=1) assert flops == in_w * kernel model = Sequential(GlobalAveragePooling2D(input_shape=(in_w, in_h, kernel))) flops = get_flops(model, batch_size=1) assert flops == in_w * in_h * kernel model = Sequential(GlobalAveragePooling3D(input_shape=(in_w, in_h, in_z, kernel))) flops = get_flops(model, batch_size=1) assert flops == in_w * in_h * in_z * kernel
def test_Embedding(): sentence = 32 onehot = 100 emb = 32 model = Sequential(Embedding(onehot, emb, input_length=sentence)) flops = get_flops(model, batch_size=1) assert flops > 0, "not supported"
def test_gru(): sentence = 32 emb = 32 rnn_unit = 3 model = Sequential( GRU(rnn_unit, use_bias=False, input_shape=(sentence, emb))) flops = get_flops(model, batch_size=1) assert (flops > 2 * emb * rnn_unit * sentence + 2 * rnn_unit * rnn_unit * sentence), "not supported"
def test_depthwise_conv2d(): in_w = 32 in_h = 32 in_ch = 3 ker_w = 3 ker_h = 3 model = Sequential( DepthwiseConv2D((ker_w, ker_h), padding="same", input_shape=(in_w, in_h, in_ch)) ) flops = get_flops(model, batch_size=1) assert flops == ((2 * ker_w * ker_h) + 1) * in_w * in_h * in_ch
def test_simpleRNN(): # NOTE: first input dense is only calculated. sentence = 32 emb = 32 rnn_unit = 3 model = Sequential( SimpleRNN(rnn_unit, use_bias=False, input_shape=(sentence, emb))) flops = get_flops(model, batch_size=1) assert (flops == 2 * emb * rnn_unit * sentence + 2 * rnn_unit * rnn_unit * sentence), "not supported. first input matmul is only calculated"
def test_dense(): in_dense = 8 out_dense = 3 def build_model1(): inp = Input((in_dense,)) out = Dense(out_dense)(inp) model = Model(inp, out) return model model = build_model1() flops = get_flops(model, batch_size=1) assert flops == 2 * in_dense * out_dense + out_dense def build_model2(): return Sequential(Dense(out_dense, use_bias=False, input_shape=(in_dense,))) model = build_model2() flops = get_flops(model, batch_size=1) assert flops == 2 * in_dense * out_dense
def test_global_maxpooling1d2d3d(): """ reduct rest (Ndim) of target axis. compare Ndim - 1 ops. """ in_w = 32 in_h = 32 in_z = 32 kernel = 3 model = Sequential(GlobalMaxPooling1D(input_shape=(in_w, kernel))) flops = get_flops(model, batch_size=1) assert flops == (in_w - 1) * kernel model = Sequential(GlobalMaxPooling2D(input_shape=(in_w, in_h, kernel))) flops = get_flops(model, batch_size=1) assert flops == (in_w * in_h - 1) * kernel model = Sequential(GlobalMaxPooling3D(input_shape=(in_w, in_h, in_z, kernel))) flops = get_flops(model, batch_size=1) assert flops == (in_w * in_h * in_z - 1) * kernel
def test_upsampling1d2d3d(): in_w = 32 in_h = 32 in_z = 32 kernel = 32 up_w = 2 up_h = 2 up_z = 2 model = Sequential(UpSampling1D(size=up_w, input_shape=(in_w, kernel))) flops = get_flops(model, batch_size=1) assert flops == in_w * kernel model = Sequential( UpSampling2D(size=(up_w, up_h), input_shape=(in_w, in_h, kernel))) flops = get_flops(model, batch_size=1) assert flops == in_w * in_h * kernel model = Sequential( UpSampling3D(size=(up_w, up_h, up_z), input_shape=(in_w, in_h, in_z, kernel))) flops = get_flops(model, batch_size=1) assert flops == in_w * in_h * in_z * kernel
def test_averagepooling1d2d3d(): in_w = 32 in_h = 32 in_z = 32 kernel = 32 pool_w = 2 pool_h = 2 pool_z = 2 model = Sequential( AveragePooling3D(pool_size=(pool_w, pool_h, pool_z), input_shape=(in_w, in_h, in_z, kernel))) flops = get_flops(model, batch_size=1) assert flops == in_w * in_h * in_z * kernel
def test_batchnormalization(): """ batch normalization is calculated as follows, 1. (3 ops * |var|) inv = rsqrt(var + eps) 2. (1 ops * |var|) inv *= gamma (scale) 3. (2 * |x| + |mean| + |var| ops) x' = inv * x + beta (shift) - mean * inv , where |var| = |mean| = channel size in default Thus, tot FLOPs = 6 * channel size + 2 * input element size. """ in_w = 32 in_h = 32 in_ch = 3 model = Sequential( BatchNormalization( beta_initializer="ones", gamma_initializer="ones", input_shape=(in_w, in_ch), ) ) flops = get_flops(model, batch_size=1) assert ( flops == 6 * in_ch + 2 * in_w * in_ch ), "fused is False. see nn_impl.batch_normalization" model = Sequential( BatchNormalization( beta_initializer="ones", gamma_initializer="ones", input_shape=(in_w, in_h, in_ch), ) ) flops = get_flops(model, batch_size=1) assert ( flops == 6 * in_ch + 2 * in_w * in_h * in_ch ), "fused is True, see gen_nn.fused_batch_norm_v3"
def test_conv2dtranspose(): in_w = 32 in_h = 32 in_ch = 3 kernel = 32 ker_w = 3 ker_h = 3 model = Sequential( Conv2DTranspose( kernel, (ker_w, ker_h), padding="same", input_shape=(in_w, in_h, in_ch) ) ) flops = get_flops(model, batch_size=1) assert flops >= ((2 * ker_w * ker_h * in_ch) + 1) * in_w * in_h * kernel
def test_subclass(): class SubClass(tf.keras.Model): def __init__(self): super().__init__() self.dense1 = Dense(10) self.dense2 = Dense(3) def call(self, x): x = self.dense1(x) return self.dense2(x) inp = Input((30,)) x = SubClass()(inp) model = Model(inp, x) flops = get_flops(model, 1) assert flops == (2 * 30 + 1) * 10 + (2 * 10 + 1) * 3
def test_conv1dtranspose(): ignore = True major, minor, _ = tf.version.VERSION.split(".") if int(major) >= 2 and int(minor) >= 3: ignore = False if ignore: return from tensorflow.keras.layers import Conv1DTranspose in_w = 32 in_ch = 3 kernel = 32 ker_w = 3 model = Sequential( Conv1DTranspose(kernel, (ker_w,), padding="same", input_shape=(in_w, in_ch)) ) flops = get_flops(model, batch_size=1) assert flops == ((2 * ker_w * in_ch) + 1) * in_w * kernel + 1
loaded_model_json = json_file.read() json_file.close() loaded_model = model_from_json(loaded_model_json) # load weights into new model loaded_model.load_weights("{}.h5".format(path)) # #sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) #loaded_model.compile(optimizer = sgd, loss= 'categorical_crossentropy', metrics = ['accuracy']) print_test_accuracy( test_model=loaded_model, test_batch_size=batchSize, sample_test=testData, labels_test=testTarget, classes=classNames, show_example_errors=False, show_confusion_matrix=True, ) #%%% from keras.models import load_model from keras_flops import get_flops ## Calculate FLOPS loaded_model.save(path) model = load_model(path) flops = get_flops(model, batch_size=32) print(f"FLOPS: {flops / 10 ** 9:.03} G")
def compute(model, input_shape, batch_size=1): inp = Input(input_shape) out = model(inp) _model = Model(inp, out) flops = get_flops(_model, batch_size=batch_size) print(f"FLOPS: {flops / 10 ** 9:.03} G")
loaded_model_json = json_file.read() json_file.close() loaded_model = model_from_json(loaded_model_json) # load weights into new model loaded_model.load_weights("{}.h5".format(path)) # #sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) #loaded_model.compile(optimizer = sgd, loss= 'categorical_crossentropy', metrics = ['accuracy']) print_test_accuracy( test_model=loaded_model, test_batch_size=batchSize, sample_test=testData, labels_test=testTarget, classes=classNames, show_example_errors=False, show_confusion_matrix=True, ) #%%% from keras.models import load_model from keras_flops import get_flops ## Calculate FLOPS loaded_model.save(path) model = load_model(path) flops = get_flops(model, batch_size=batchSize) print(f"FLOPS: {flops / 10 ** 9:.03} G")
def test_softmax(): kernel = 8 model = Sequential(Activation("softmax", input_shape=(kernel,))) flops = get_flops(model, batch_size=1) assert flops == 5 * kernel
def test_duplicated_calculation(): model = Sequential(Dense(5, input_shape=(3,))) flops1 = get_flops(model) flops2 = get_flops(model) flops3 = get_flops(model) assert flops1 == flops2 and flops2 == flops3
def test_multi_input(): inputs = [Input((5,)), Input(5,)] out = tf.keras.layers.Multiply()(inputs) model = Model(inputs, out) flops = get_flops(model, 1) assert flops == 5
def test_ignore(): model = Sequential( [Flatten(input_shape=(16, 16)), Activation("relu"), Dropout(0.25),] ) flops = get_flops(model, 1) assert flops == 0
def train_test_and_collect_data( model_constructor, model_kwargs, loss, optimizer, epochs, callbacks, train_ds, val_ds, test_ds, labels, false_ds=None, thresholds=[], num_timer_samples=0, timer_iters=100, load_weights='', model_save_path='', ): num_labels = len(labels) test_samples = [sample for sample in test_ds.take(8)] # draw_spectrogram_from_tensors(test_samples) for sample, _ in train_ds.take(1): # exclude batch dimension input_shape = sample.shape[1:] print('Number of classes', num_labels) print('Input shape', input_shape) model = model_constructor(input_shape, num_labels, **model_kwargs) trainable_count = np.sum( [tf.keras.backend.count_params(w) for w in model.trainable_weights]) non_trainable_count = np.sum([ tf.keras.backend.count_params(w) for w in model.non_trainable_weights ]) flops = get_flops(model, batch_size=1) print('trainable params: ', trainable_count) print('non-trainable params', non_trainable_count) print('total params', trainable_count + non_trainable_count) print('flops', flops) model.summary() if load_weights: try: with open(load_weights + '.json', 'r') as f: progress = json.load(f) epochs = epochs - int(progress['epoch']) - 1 except IOError: pass try: model.load_weights(load_weights) except tf.errors.NotFoundError: print('Weights not found.') model, history = train_model(model, train_ds, val_ds, loss=loss, optimizer=optimizer, epochs=epochs, callbacks=callbacks) test_loss, test_acc, test_times, confusion_matrix, thresholding_results = test_model( model, test_ds, false_ds, thresholds, num_timer_samples, timer_iters) print('Test accuracy: ', test_acc) if model_save_path: model.save(model_save_path) return { 'model': model, 'epoch': history.epoch, 'history': history.history, 'test_loss': test_loss, 'test_acc': test_acc, 'test_times': test_times, 'confusion_matrix': confusion_matrix.tolist(), 'thresholding_results': thresholding_results, 'total_params': int(trainable_count + non_trainable_count), 'trainable_params': int(trainable_count), 'non_trainable_params': int(non_trainable_count), 'flops': flops }