def __init__(self, network=None, log=False, dataset="mnist", model="Small", batch_size=128, nb_byz_worker=0): self.log = log self.network = network self.nb_byz_worker = nb_byz_worker self.batch_size = batch_size dsm = DatasetManager(network, dataset, self.batch_size) self.train_data, self.test_data = dsm.data_train, dsm.data_test devices = tf.config.list_physical_devices('gpu') if len(devices) == 1: self.strategy = tf.distribute.OneDeviceStrategy() elif len(devices) > 1: self.strategy = tf.distribute.MirroredStrategy() else: self.strategy = tf.distribute.get_strategy() with self.strategy.scope(): mdm = ModelManager(model=model, info=dsm.ds_info) self.model = mdm.model self.loss_fn = SparseCategoricalCrossentropy( from_logits=True, reduction=tf.keras.losses.Reduction.NONE) ps_hosts = network.get_all_ps() worker_hosts = network.get_all_other_worker() self.ps_connections = [tools.set_connection(host) for host in ps_hosts] self.worker_connections = [ tools.set_connection(host) for host in worker_hosts ] self.port = network.get_my_port() self.task_id = network.get_task_index() self.m = tf.keras.metrics.Accuracy() # Define grpc server self.service = grpc_message_exchange_servicer.MessageExchangeServicer( tools.flatten_weights(self.model.trainable_variables)) self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=30), options=[('grpc.max_send_message_length', 500 * 1024 * 1024), ('grpc.max_receive_message_length', 500 * 1024 * 1024)]) garfield_pb2_grpc.add_MessageExchangeServicer_to_server( self.service, self.server) self.server.add_insecure_port('[::]:' + str(self.port)) self.aggregated_weights = None self.loss_fn = SparseCategoricalCrossentropy(from_logits=True)
def __init__(self, in_channels, out_channels, hids=[16], acts=['relu'], dropout=0.5, weight_decay=5e-4, lr=0.01, use_bias=True): super().__init__() self.convs = [] inc = in_channels for hid, act in zip(hids, acts): layer = GraphConv(inc, hid, bias=use_bias, activation=activations.get(act)) self.convs.append(layer) inc = hid layer = GraphConv(inc, out_channels, bias=use_bias) self.convs.append(layer) self.dropout = layers.Dropout(dropout) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'])
def __init__(self, in_channels, out_channels, hiddens=[32], activations=['relu'], dropout=0.5, l2_norm=5e-4, lr=0.01, use_bias=True, aggregator='mean', output_normalize=False, n_samples=[15, 5]): Agg = _AGG.get(aggregator, None) if not Agg: raise ValueError( f"Invalid value of 'aggregator', allowed values {tuple(_AGG.keys())}, but got '{aggregator}'." ) _intx = intx() x = Input(batch_shape=[None, in_channels], dtype=floatx(), name='attr_matrix') nodes = Input(batch_shape=[None], dtype=_intx, name='nodes') neighbors = [ Input(batch_shape=[None], dtype=_intx, name=f'neighbors_{hop}') for hop, n_sample in enumerate(n_samples) ] aggregators = [] for hidden, activation in zip(hiddens, activations): # you can use `GCNAggregator` instead aggregators.append( Agg(hidden, concat=True, activation=activation, use_bias=use_bias, kernel_regularizer=regularizers.l2(l2_norm))) aggregators.append(Agg(out_channels, use_bias=use_bias)) h = [tf.nn.embedding_lookup(x, node) for node in [nodes, *neighbors]] for agg_i, aggregator in enumerate(aggregators): attribute_shape = h[0].shape[-1] for hop in range(len(n_samples) - agg_i): neighbor_shape = [-1, n_samples[hop], attribute_shape] h[hop] = aggregator( [h[hop], tf.reshape(h[hop + 1], neighbor_shape)]) if hop != len(n_samples) - 1: h[hop] = Dropout(rate=dropout)(h[hop]) h.pop() h = h[0] if output_normalize: h = tf.nn.l2_normalize(h, axis=1) super().__init__(inputs=[x, nodes, *neighbors], outputs=h) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'])
def main(): dataset = CIFAR10( binary=True, validation_split=0.0) # not using validation for anything model = mobilenet_v2_like(dataset.input_shape, dataset.num_classes) model.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=SGDW(lr=0.01, momentum=0.9, weight_decay=1e-5), metrics=['accuracy']) model.summary() batch_size = 128 train_data = dataset.train_dataset() \ .shuffle(8 * batch_size) \ .batch(batch_size) \ .prefetch(tf.data.experimental.AUTOTUNE) valid_data = dataset.test_dataset() \ .batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE) def lr_schedule(epoch): if 0 <= epoch < 35: return 0.01 if 35 <= epoch < 65: return 0.005 return 0.001 model.fit(train_data, validation_data=valid_data, epochs=80, callbacks=[LearningRateScheduler(lr_schedule)]) model.save("cnn-cifar10-binary.h5")
def generate_adversarial_perturbation(model, img_base, delta, base_pred, target_pred, eps, optimizer, steps): for step in range(steps): with tensorflow.GradientTape() as tape: tape.watch(delta) adversarial_example = img_base + delta prediction = model(adversarial_example, training=False) base_pred_labels = fen2labels(base_pred) target_pred_labels = fen2labels(target_pred) loss = SparseCategoricalCrossentropy(from_logits=True, reduction="auto") original_loss = -loss( tensorflow.convert_to_tensor(base_pred_labels), prediction) target_loss = loss( tensorflow.convert_to_tensor(target_pred_labels), prediction) total_loss = target_loss + original_loss if step % 10 == 0: print("step: {}, loss: {}...".format(step, total_loss.numpy())) gradients = tape.gradient(total_loss, delta) optimizer.apply_gradients([(gradients, delta)]) delta.assign_add(clip_eps(delta, eps=eps)) return delta
def build(self, hiddens=[16], activations=['relu'], dropout=0.5, l2_norm=5e-4, lr=0.01, use_bias=False): with tf.device(self.device): n_nodes = self.graph.n_nodes x = Input(batch_shape=[None, self.graph.n_attrs], dtype=self.floatx, name='attr_matrix') wavelet = Input(batch_shape=[n_nodes, n_nodes], dtype=self.floatx, sparse=True, name='wavelet_matrix') inverse_wavelet = Input(batch_shape=[n_nodes, n_nodes], dtype=self.floatx, sparse=True, name='inverse_wavelet_matrix') index = Input(batch_shape=[None], dtype=self.intx, name='node_index') h = x for hidden, activation in zip(hiddens, activations): h = WaveletConvolution(hidden, activation=activation, use_bias=use_bias, kernel_regularizer=regularizers.l2(l2_norm))([h, wavelet, inverse_wavelet]) h = Dropout(rate=dropout)(h) h = WaveletConvolution(self.graph.n_classes, use_bias=use_bias)( [h, wavelet, inverse_wavelet]) h = Gather()([h, index]) model = Model( inputs=[x, wavelet, inverse_wavelet, index], outputs=h) model.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy']) self.model = model
def process(self, train_nodes, unlabeled_nodes, self_training_labels, hids, use_relu, reset=True): self.ll_ratio = None with tf.device(self.device): self.train_nodes = gf.astensor(train_nodes, dtype=self.intx) self.unlabeled_nodes = gf.astensor(unlabeled_nodes, dtype=self.intx) self.labels_train = gf.astensor(self.graph.node_label[train_nodes], dtype=self.intx) self.self_training_labels = gf.astensor(self_training_labels, dtype=self.intx) self.adj_tensor = gf.astensor(self.graph.adj_matrix.A, dtype=self.floatx) self.x_tensor = gf.astensor(self.graph.node_attr, dtype=self.floatx) self.build(hids=hids) self.use_relu = use_relu self.loss_fn = SparseCategoricalCrossentropy(from_logits=True) self.adj_changes = tf.Variable(tf.zeros_like(self.adj_tensor)) self.x_changes = tf.Variable(tf.zeros_like(self.x_tensor)) if reset: self.reset() return self
def get_mlp_model(): model = Sequential([ Flatten(input_shape=X_train_grayscale[0].shape, name='flatten'), Dense(units=128, activation=relu, kernel_initializer=he_uniform(), bias_initializer=he_uniform(), name='dense_1'), Dense(units=64, activation=relu, kernel_initializer=he_uniform(), bias_initializer=he_uniform(), name='dense_2'), Dense(units=32, activation=relu, kernel_initializer=he_uniform(), bias_initializer=he_uniform(), name='dense_3'), Dense(units=10, activation=softmax, kernel_initializer=he_uniform(), bias_initializer=he_uniform(), name='dense_output') ]) model.compile( optimizer=Adam(), loss=SparseCategoricalCrossentropy(), metrics=['accuracy']) return model
def __init__(self, in_features, out_features, hids=[64], acts=['relu'], dropout=0.5, weight_decay=5e-3, lr=0.01, bias=False, K=10): x = Input(batch_shape=[None, in_features], dtype=floatx(), name='node_attr') adj = Input(batch_shape=[None, None], dtype=floatx(), sparse=True, name='adj_matrix') h = x for hid, act in zip(hids, acts): h = Dense(hid, use_bias=bias, activation=act, kernel_regularizer=regularizers.l2(weight_decay))(h) h = Dropout(dropout)(h) h = Dense(out_features, use_bias=bias, activation=acts[-1], kernel_regularizer=regularizers.l2(weight_decay))(h) h = Dropout(dropout)(h) h = PropConvolution(K, use_bias=bias, activation='sigmoid', kernel_regularizer=regularizers.l2(weight_decay))([h, adj]) super().__init__(inputs=[x, adj], outputs=h) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'])
def main(): dataset = SpeechCommands("/datasets/speech_commands_v0.02") model = get_model(dataset.input_shape, dataset.num_classes) model.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=AdamW(lr=0.0005, weight_decay=1e-5), metrics=['accuracy']) model.summary() batch_size = 100 train_data = dataset.train_dataset() \ .shuffle(8 * batch_size) \ .batch(batch_size) \ .prefetch(tf.data.experimental.AUTOTUNE) valid_data = dataset.test_dataset() \ .batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE) def lr_schedule(epoch): if 0 <= epoch < 20: return 0.0005 if 20 <= epoch < 40: return 0.0001 return 0.00002 model.fit(train_data, validation_data=valid_data, epochs=50, callbacks=[LearningRateScheduler(lr_schedule)]) model.save("cnn-speech-commands.h5")
def __init__(self, in_features, out_features, hids=[16], num_heads=[8], acts=['elu'], dropout=0.6, weight_decay=5e-4, lr=0.01, bias=True): x = Input(batch_shape=[None, in_features], dtype=floatx(), name='node_attr') adj = Input(batch_shape=[None, None], dtype=floatx(), sparse=True, name='adj_matrix') h = x for hid, num_head, act in zip(hids, num_heads, acts): h = GraphAttention(hid, attn_heads=num_head, reduction='concat', use_bias=bias, activation=act, kernel_regularizer=regularizers.l2(weight_decay), attn_kernel_regularizer=regularizers.l2( weight_decay), )([h, adj]) h = Dropout(rate=dropout)(h) h = GraphAttention(out_features, use_bias=bias, attn_heads=1, reduction='average')([h, adj]) super().__init__(inputs=[x, adj], outputs=h) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'])
def __init__(self, in_channels, out_channels, hiddens=[16], activations=['relu'], dropout=0.5, weight_decay=5e-4, lr=0.01, order=2, use_bias=False): x = Input(batch_shape=[None, in_channels], dtype=floatx(), name='node_attr') adj = [Input(batch_shape=[None, None], dtype=floatx(), sparse=True, name=f'adj_matrix_{i}') for i in range(order + 1)] index = Input(batch_shape=[None], dtype=intx(), name='node_index') h = x for hidden, activation in zip(hiddens, activations): h = ChebyConvolution(hidden, order=order, use_bias=use_bias, activation=activation, kernel_regularizer=regularizers.l2(weight_decay))([h, adj]) h = Dropout(rate=dropout)(h) h = ChebyConvolution(out_channels, order=order, use_bias=use_bias)([h, adj]) h = Gather()([h, index]) super().__init__(inputs=[x, *adj, index], outputs=h) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'])
def load_training_model(g, vocab_size): with select_strategy().scope(): if g['network-type'] == 'transformer': model = transformer(vocab_size, 128, 2048, 0.2, 8, 16, g['sequence-length'], True) opt = RMSprop(learning_rate = g['learning-rate']) elif g['network-type'] == 'lstm': model = lstm_model(vocab_size, g['embedding-size'], g['lstm1-units'], g['lstm2-units'], g['dropout'], g['recurrent-dropout'], False, g['batch-size']) opt = RMSprop(learning_rate = g['learning-rate']) elif g['network-type'] == 'gpt2': set_gpt2_params(g, vocab_size) model = gpt2.GPT2() opt = Adam(learning_rate = g['learning-rate'], epsilon=1e-08) else: assert False loss_fn = SparseCategoricalCrossentropy(from_logits = True) metrics = ['sparse_categorical_accuracy'] model.compile( optimizer = opt, loss = loss_fn, metrics = metrics) model(tf.constant([[0]])) return model
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, pe_input, pe_target, rate=0.1): super(Transformer, self).__init__() self.num_layers = num_layers self.num_heads = num_heads self.d_model = d_model self.train_loss = tf.keras.metrics.Mean(name='train_loss') self.train_accuracy = tf.keras.metrics.Mean(name='train_accuracy') self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, pe_input, rate) self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target) self.dense = Dense(target_vocab_size) self.loss_object = SparseCategoricalCrossentropy(from_logits=True, reduction='none')
def cnn2D(inputSize, opt='adam', loss=SparseCategoricalCrossentropy(from_logits=True), classes=10): model = models.Sequential() model.add( layers.Conv2D(32, (5, 5), activation='relu', padding="same", input_shape=inputSize + (1, ))) model.add(layers.BatchNormalization()) model.add(layers.Dropout(rate=0.3)) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), padding="same", activation='relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(rate=0.3)) model.add(layers.MaxPooling2D((2, 2))) # model.add(layers.Conv2D(64, (5, 5), padding="same", activation='relu')) model.add(layers.Flatten()) model.add(layers.Dense(128, activation='relu')) model.add(layers.Dropout(rate=0.5)) model.add(layers.Dense(128, activation='relu')) model.add(layers.Dropout(rate=0.5)) model.add(layers.Dense(classes)) model.compile(optimizer=opt, loss=loss, metrics=['accuracy']) return model
def main(): dataset = FashionMNIST() model = get_model(dataset.input_shape, dataset.num_classes) model.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=AdamW(lr=0.001, weight_decay=5e-5), metrics=['accuracy']) model.summary() batch_size = 128 train_data = dataset.train_dataset() \ .shuffle(8 * batch_size) \ .batch(batch_size) \ .prefetch(tf.data.experimental.AUTOTUNE) valid_data = dataset.test_dataset() \ .batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE) model.fit(train_data, validation_data=valid_data, epochs=40, callbacks=[ LearningRateScheduler(lambda e: 0.001 if e < 25 else 0.0001) ]) model.save("cnn-fashion-mnist.h5")
def __init__(self, num_classes: int, backbone, num_queries=100, **kwargs): super().__init__(**kwargs) self.num_classes = num_classes self.num_queries = num_queries self.hidden_dim = 256 self.backbone = backbone self.input_proj = tf.keras.layers.Conv2D(self.hidden_dim, 1) self.pos_embed = PositionEmbeddingSine(output_dim=self.hidden_dim) self.transformer_num_layers = 6 self.transformer = Transformer(num_layers=self.transformer_num_layers, d_model=self.hidden_dim, num_heads=8, dim_feedforward=2048) self.bbox_embed = tf.keras.models.Sequential([ tf.keras.layers.Dense(self.hidden_dim, activation='relu'), tf.keras.layers.Dense(self.hidden_dim, activation='relu'), tf.keras.layers.Dense(4, activation='sigmoid', dtype=tf.float32) # (x1, y1, x2, y2) ]) self.class_embed = tf.keras.layers.Dense(num_classes + 1, dtype=tf.float32) # Will create a learnable embedding matrix for all our queries # It is a matrix of [num_queries, self.hidden_dim] # The embedding layers self.query_embed = tf.keras.layers.Embedding(num_queries, self.hidden_dim) self.all_the_queries = tf.range(num_queries) # Loss computation self.weight_class, self.weight_l1, self.weight_giou = 1, 5, 2 similarity_func = DetrSimilarity(self.weight_class, self.weight_l1, self.weight_giou) self.target_assigner = TargetAssigner(similarity_func, hungarian_matching, lambda gt, pred: gt, negative_class_weight=1.0) # Relative classification weight applied to the no-object category # It down-weight the log-probability term of a no-object # by a factor 10 to account for class imbalance self.non_object_weight = tf.constant(0.1, dtype=self.compute_dtype) # Losses self.giou = GIoULoss(reduction=tf.keras.losses.Reduction.NONE) self.l1 = L1Loss(reduction=tf.keras.losses.Reduction.NONE) self.scc = SparseCategoricalCrossentropy( reduction=tf.keras.losses.Reduction.NONE, from_logits=True) # Metrics self.giou_metric = tf.keras.metrics.Mean(name="giou_last_layer") self.l1_metric = tf.keras.metrics.Mean(name="l1_last_layer") self.scc_metric = tf.keras.metrics.Mean(name="scc_last_layer") self.loss_metric = tf.keras.metrics.Mean(name="loss") self.precision_metric = tf.keras.metrics.SparseCategoricalAccuracy() # Object recall = foreground self.recall_metric = tf.keras.metrics.Mean(name="object_recall")
def __init__(self, in_features, out_features, hids=[16], acts=['relu'], dropout=0.5, weight_decay=5e-4, lr=0.01, bias=False): _intx = intx() _floatx = floatx() x = Input(batch_shape=[None, in_features], dtype=_floatx, name='node_attr') edge_index = Input(batch_shape=[None, 2], dtype=_intx, name='edge_index') edge_weight = Input(batch_shape=[None], dtype=_floatx, name='edge_weight') h = x for hid, act in zip(hids, acts): h = GraphEdgeConvolution(hid, use_bias=bias, activation=act, kernel_regularizer=regularizers.l2(weight_decay))([h, edge_index, edge_weight]) h = Dropout(rate=dropout)(h) h = GraphEdgeConvolution(out_features, use_bias=bias)( [h, edge_index, edge_weight]) super().__init__(inputs=[x, edge_index, edge_weight], outputs=h) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'])
def build(self, hiddens=[32], activations=['relu'], dropout=0.5, l2_norm=5e-4, lr=0.01, use_bias=False): with tf.device(self.device): x = Input(batch_shape=[None, self.graph.n_attrs], dtype=self.floatx, name='attr_matrix') adj = Input(batch_shape=[None, None], dtype=self.floatx, sparse=True, name='adj_matrix') h = x for hidden, activation in zip(hiddens, activations): h = Dense(hidden, use_bias=use_bias, activation=activation, kernel_regularizer=regularizers.l2(l2_norm))(h) h = Dropout(rate=dropout)(h) h = GraphConvolution(self.graph.n_classes, use_bias=use_bias)([h, adj]) model = Model(inputs=[x, adj], outputs=h) model.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy']) self.model = model
def __init__(self, anchor_ratios=(0.5, 1, 2), **kwargs): super().__init__( 2, SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE, from_logits=True), L1Loss(reduction=tf.keras.losses.Reduction.NONE), multiples=len(anchor_ratios), kernel_initializer_classification_head=initializers.RandomNormal(stddev=0.01), kernel_initializer_box_prediction_head=initializers.RandomNormal(stddev=0.01), **kwargs) #Force each ground_truths to match to at least one anchor matcher = Matcher([0.3, 0.7], [0, -1, 1], allow_low_quality_matches=True) self.target_assigner = TargetAssigner(IoUSimilarity(), matcher, encode_boxes_faster_rcnn, dtype=self._compute_dtype) anchor_strides = (4, 8, 16, 32, 64) anchor_zises = (32, 64, 128, 256, 512) self._anchor_ratios = anchor_ratios # Precompute a deterministic grid of anchors for each layer of the pyramid. # We will extract a subpart of the anchors according to self._anchors = [ Anchors(stride, size, self._anchor_ratios) for stride, size in zip(anchor_strides, anchor_zises) ]
def generate_model(output_col, input_columns, model_num, model_label): ### Import data x = np.loadtxt('./data/' + dataset, delimiter=',', usecols=input_columns) y = np.loadtxt('./data/' + dataset, delimiter=',', usecols=(output_col)) ### Build model model = Sequential() model.add(Dense(20, input_dim=len(input_columns), activation='relu')) model.add(Dense(60, activation='relu')) model.add(Dense(60, activation='relu')) model.add(Dense(58, activation='softmax')) model.compile( loss=SparseCategoricalCrossentropy( from_logits=False, reduction="auto", name="sparse_categorical_crossentropy" #For integer classes ), optimizer='Adam', metrics=['accuracy']) ### Train and evaluate model.fit(x, y, epochs=10) scores = model.evaluate(x, y) print(model.metrics_names[1], scores[1] * 100) name = 'P' + str(model_num) ### Save model model.save('./models/' + method + '/' + model_label + '/' + name)
def __init__(self, in_features, out_features, hids=[16], acts=['relu'], dropout=0.5, weight_decay=5e-4, lr=0.01, bias=False, experimental_run_tf_function=True): x = Input(batch_shape=[None, in_features], dtype=floatx(), name='node_attr') adj = Input(batch_shape=[None, None], dtype=floatx(), sparse=True, name='adj_matrix') h = x for hid, act in zip(hids, acts): h = GraphConvolution(hid, use_bias=bias, activation=act, kernel_regularizer=regularizers.l2(weight_decay))([h, adj]) h = Dropout(rate=dropout)(h) h = GraphConvolution(out_features, use_bias=bias)([h, adj]) super().__init__(inputs=[x, adj], outputs=h) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'], experimental_run_tf_function=experimental_run_tf_function)
def __init__(self, in_channels, out_channels, hiddens=[32], activations=['relu'], dropout=0.5, l2_norm=5e-4, lr=0.01, use_bias=False): x = Input(batch_shape=[None, in_channels], dtype=floatx(), name='attr_matrix') adj = Input(batch_shape=[None, None], dtype=floatx(), sparse=True, name='adj_matrix') h = x for hidden, activation in zip(hiddens, activations): h = Dense(hidden, use_bias=use_bias, activation=activation, kernel_regularizer=regularizers.l2(l2_norm))(h) h = Dropout(rate=dropout)(h) h = GraphConvolution(out_channels, use_bias=use_bias)([h, adj]) super().__init__(inputs=[x, adj], outputs=h) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'])
def construct_network2(): model = tf.keras.models.Sequential([ InputLayer(input_shape=(28, 28, 1)), Conv2D(20, 3, kernel_initializer=RandomNormal(0, 0.01), padding="same", activation="relu"), BatchNormalization(), MaxPool2D(pool_size=2, strides=2, padding="same"), Conv2D(30, 3, kernel_initializer=RandomNormal(0, 0.01), padding="same", activation="relu"), BatchNormalization(), MaxPool2D(pool_size=2, strides=2, padding="same"), Conv2D(50, 3, kernel_initializer=RandomNormal(0, 0.01), padding="same", activation="relu"), BatchNormalization(), MaxPool2D(pool_size=2, strides=2, padding="same"), Flatten(), Dense(10, kernel_initializer=RandomNormal(0, 0.01)), ]) model.compile( optimizer=SGD(learning_rate=0.01, momentum=0.9), loss=SparseCategoricalCrossentropy(from_logits=True), metrics=["accuracy"], ) model.summary() return model
def define_learning_model(): model = Sequential() model.add(Rescaling(1. / 255, input_shape=(800, 600, 3))) model.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu')) model.add(MaxPool2D(pool_size=(2, 2))) model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu')) model.add(MaxPool2D(pool_size=(2, 2))) model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu')) model.add(MaxPool2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(5, activation='softmax')) model.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=0.001), metrics=['accuracy']) model.summary() return model
def __init__(self, in_channels, out_channels, hids=[16], acts=['relu'], dropout=0.5, weight_decay=5e-4, lr=0.01, use_bias=False, experimental_run_tf_function=True): x = Input(batch_shape=[None, in_channels], dtype=floatx(), name='node_attr') h = x for hid, act in zip(hids, acts): h = Dense(hid, use_bias=use_bias, activation=act, kernel_regularizer=regularizers.l2(weight_decay))(h) h = Dropout(rate=dropout)(h) h = Dense(out_channels, use_bias=use_bias)(h) super().__init__(inputs=x, outputs=h) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy'], experimental_run_tf_function=experimental_run_tf_function)
def main(): BATCH_SIZE = 64 EPOCHS = 3 optimizer = Adam() train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy() test_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy() loss_fn = SparseCategoricalCrossentropy() # Load data ds = dataset.load("abhinavtuli/fashion-mnist") # transform into Tensorflow dataset ds = ds.to_tensorflow() # Splitting back into the original train and test sets train_dataset = ds.take(60000) test_dataset = ds.skip(60000) train_dataset = train_dataset.batch(BATCH_SIZE) test_dataset = test_dataset.batch(BATCH_SIZE) model = create_CNN() # model.summary() for epoch in range(EPOCHS): print("\nStarting Training Epoch {}".format(epoch)) train(model, train_dataset, optimizer, loss_fn, train_acc_metric) print("Training Epoch {} finished\n".format(epoch)) test(model, test_dataset, test_acc_metric)
def create_CNN_model(X_train, y_train, X_test, y_test, X_val, y_val): # Reshape data X_train_flattened = X_train.reshape(len(X_train), 28 * 28) X_test_flattened = X_test.reshape(len(X_test), 28 * 28) X_val_flattened = X_val.reshape(len(X_val), 28 * 28) cnn = Sequential() cnn.add(Dense(units=128, activation='relu')) cnn.add(Dense(units=64, activation='relu')) cnn.add(Dense(units=10, activation='softmax')) callback = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True) cnn.compile(optimizer=Adam(), loss=SparseCategoricalCrossentropy(), metrics=['accuracy']) history = cnn.fit(x=X_train_flattened, y=y_train, validation_data=(X_val_flattened, y_val), epochs=600, batch_size=32, verbose=False, callbacks=[callback]) test_loss, test_acc = cnn.evaluate(x=X_test_flattened, y=y_test) print(f"Test loss: {test_loss}") print(f"Test accuracy: {test_acc}") predictions = cnn.predict(X_test_flattened) cnn.save('trained_model.h5') return predictions
def __init__(self, model, lstm_size=64, lstm_num_layers=1, tanh_constant=1.5, cell_exit_extra_step=False, skip_target=0.4, temperature=None, branch_bias=0.25, entropy_reduction='sum'): super().__init__(model) self.tanh_constant = tanh_constant self.temperature = temperature self.cell_exit_extra_step = cell_exit_extra_step cells = [ LSTMCell(units=lstm_size, use_bias=False) for _ in range(lstm_num_layers) ] self.lstm = RNN(cells, stateful=True) self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1 self.skip_targets = tf.constant([1.0 - skip_target, skip_target]) self.max_layer_choice = 0 self.bias_dict = {} for mutable in self.mutables: if isinstance(mutable, LayerChoice): if self.max_layer_choice == 0: self.max_layer_choice = len(mutable) assert self.max_layer_choice == len(mutable), \ "ENAS mutator requires all layer choice have the same number of candidates." if 'reduce' in mutable.key: bias = [] for choice in mutable.choices: if 'conv' in str(type(choice)).lower(): bias.append(branch_bias) else: bias.append(-branch_bias) self.bias_dict[mutable.key] = tf.constant(bias) # exposed for trainer self.sample_log_prob = 0 self.sample_entropy = 0 self.sample_skip_penalty = 0 # internal nn layers self.embedding = Embedding(self.max_layer_choice + 1, lstm_size) self.soft = Dense(self.max_layer_choice, use_bias=False) self.attn_anchor = Dense(lstm_size, use_bias=False) self.attn_query = Dense(lstm_size, use_bias=False) self.v_attn = Dense(1, use_bias=False) assert entropy_reduction in [ 'sum', 'mean' ], 'Entropy reduction must be one of sum and mean.' self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean self.cross_entropy_loss = SparseCategoricalCrossentropy( from_logits=True, reduction=Reduction.NONE) self._first_sample = True
def __init__(self, adj, x, labels, idx_train, idx_unlabeled, hidden_layers, use_relu, self_training_labels=None, seed=None, name=None, device='CPU:0', **kwargs): super().__init__(adj=adj, x=x, labels=labels, seed=seed, name=name, device=device, **kwargs) adj, x, labels = self.adj, self.x, self.labels idx_train = asintarr(idx_train) idx_unlabeled = asintarr(idx_unlabeled) if self_training_labels is None: surrogate = DenseGCN(adj, x, labels, device='GPU', norm_x=None, seed=None) surrogate.build(16, activations='relu' if use_relu else None) his = surrogate.train(idx_train, verbose=0, epochs=200, save_best=False) self_training_labels = surrogate.predict(idx_unlabeled).argmax(1) self.ll_ratio = None # mettack can also conduct feature attack self.allow_feature_attack = True with tf.device(self.device): self.idx_train = astensor(idx_train, dtype=self.intx) self.idx_unlabeled = astensor(idx_unlabeled, dtype=self.intx) self.labels_train = astensor(self.labels[idx_train], dtype=self.floatx) self.self_training_labels = astensor(self_training_labels, dtype=self.floatx) self.tf_adj = astensor(adj.A, dtype=self.floatx) self.tf_x = astensor(x, dtype=self.floatx) self.build(hidden_layers=hidden_layers) self.use_relu = use_relu self.loss_fn = SparseCategoricalCrossentropy(from_logits=True) self.adj_changes = tf.Variable(tf.zeros_like(self.tf_adj)) self.x_changes = tf.Variable(tf.zeros_like(self.tf_x))