def autoencoder(encoder_model, decoder_model): decoder_output = decoder_model(encoder_model.output) model = Model(inputs=encoder_model.input, outputs=decoder_output) opt = optimizers.Adam() model.compile(loss='mse', optimizer=opt) return model
def discriminator(im_shape, relu_before_bn=True): conv2d = lambda x, filters: layers.Conv2D(filters, (5, 5), strides=(2, 2), padding='same', kernel_initializer=initializers. RandomNormal(stddev=0.02))(x) flatten = lambda x: layers.Flatten()(x) lrelu = lambda x: layers.LeakyReLU(alpha=0.2)(x) norm = lambda x: layers.BatchNormalization()(x) if relu_before_bn: block = lambda x, filters: norm(lrelu(conv2d(x, filters))) else: block = lambda x, filters: lrelu(norm(conv2d(x, filters))) inputs = Input(shape=im_shape) l1 = block(inputs, 128) l2 = block(l1, 256) l3 = block(l2, 512) l4 = flatten(block(l3, 1024)) d_out = layers.Dense(1, activation='sigmoid')(l4) d_model = Model(inputs=inputs, outputs=d_out) opt = optimizers.Adam(learning_rate=0.0002, beta_1=0.5) d_model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) return d_model
def test_pidon_operator_on_spherical_pde(): set_random_seed(0) diff_eq = DiffusionEquation(3) mesh = Mesh( [(1., 11.), (0., 2 * np.pi), (.25 * np.pi, .75 * np.pi)], [2., np.pi / 5., np.pi / 4], CoordinateSystem.SPHERICAL) bcs = [ (DirichletBoundaryCondition( lambda x, t: np.ones((len(x), 1)), is_static=True), DirichletBoundaryCondition( lambda x, t: np.full((len(x), 1), 1. / 11.), is_static=True)), (NeumannBoundaryCondition( lambda x, t: np.zeros((len(x), 1)), is_static=True), NeumannBoundaryCondition( lambda x, t: np.zeros((len(x), 1)), is_static=True)), (NeumannBoundaryCondition( lambda x, t: np.zeros((len(x), 1)), is_static=True), NeumannBoundaryCondition( lambda x, t: np.zeros((len(x), 1)), is_static=True)) ] cp = ConstrainedProblem(diff_eq, mesh, bcs) ic = ContinuousInitialCondition(cp, lambda x: 1. / x[:, :1]) t_interval = (0., .5) ivp = InitialValueProblem(cp, t_interval, ic) sampler = UniformRandomCollocationPointSampler() pidon = PIDONOperator(sampler, .001, True) training_loss_history, test_loss_history = pidon.train( cp, t_interval, training_data_args=DataArgs( y_0_functions=[ic.y_0], n_domain_points=20, n_boundary_points=10, n_batches=1 ), model_args=ModelArgs( latent_output_size=20, branch_hidden_layer_sizes=[30, 30], trunk_hidden_layer_sizes=[30, 30], ), optimization_args=OptimizationArgs( optimizer=optimizers.Adam(learning_rate=2e-5), epochs=3, verbose=False ) ) assert len(training_loss_history) == 3 for i in range(2): assert np.all( training_loss_history[i + 1].weighted_total_loss.numpy() < training_loss_history[i].weighted_total_loss.numpy()) solution = pidon.solve(ivp) assert solution.d_t == .001 assert solution.discrete_y().shape == (500, 6, 11, 3, 1)
def discriminator(n_cat): conv2d = lambda x, filters, strides: layers.Conv2D(filters, (4, 4), strides=strides, padding='same')(x) leakyrelu = lambda x: layers.LeakyReLU(alpha=0.2)(x) norm = lambda x: layers.BatchNormalization()(x) dense = lambda x, nodes: layers.Dense(nodes)(x) flatten = lambda x : layers.Flatten()(x) inputs = Input(shape=(64, 64, 3)) l1 = leakyrelu(conv2d(inputs, 64, (2, 2))) l2 = norm(leakyrelu(conv2d(l1, 128, (2, 2)))) l3 = norm(leakyrelu(conv2d(l2, 256, (2, 2)))) l4 = norm(leakyrelu(conv2d(l3, 256, (1, 1)))) l5 = norm(leakyrelu(conv2d(l4, 256, (1, 1)))) l6 = norm(leakyrelu(dense(flatten(l5), 1024))) d_out = layers.Dense(1, activation='sigmoid')(l6) # not sure if i should include the continuous variables. # for now, I will not implement continuous variables to keep things simple. aux1 = leakyrelu(norm(dense(l6, 128))) q_out = dense(aux1, n_cat) d_model = Model(inputs=inputs, outputs=d_out) opt = optimizers.Adam(learning_rate=0.0002, beta_1=0.5) d_model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) q_model = Model(inputs=inputs, outputs=q_out) return d_model, q_model
def gan(g_model, d_model): d_model.trainable = False d_output = d_model(g_model.output) gan_model = Model(inputs=g_model.input, outputs=d_output) opt = optimizers.Adam(learning_rate=0.0002, beta_1=0.5) gan_model.compile(loss='binary_crossentropy', optimizer=opt) return gan_model
def test_fdm_operator_on_pde_with_t_and_x_dependent_rhs(): class TestDiffEq(DifferentialEquation): def __init__(self): super(TestDiffEq, self).__init__(2, 1) @property def symbolic_equation_system(self) -> SymbolicEquationSystem: return SymbolicEquationSystem([ self.symbols.t / 100. * (self.symbols.x[0] + self.symbols.x[1]) ** 2 ]) diff_eq = TestDiffEq() mesh = Mesh([(-1., 1.), (0., 2.)], [2., 1.]) bcs = [ (NeumannBoundaryCondition(lambda x, t: np.zeros((len(x), 1))), NeumannBoundaryCondition(lambda x, t: np.zeros((len(x), 1)))) ] * 2 cp = ConstrainedProblem(diff_eq, mesh, bcs) ic = ContinuousInitialCondition(cp, lambda x: np.zeros((len(x), 1))) t_interval = (0., 1.) ivp = InitialValueProblem(cp, t_interval, ic) sampler = UniformRandomCollocationPointSampler() pidon = PIDONOperator(sampler, .05, True) training_loss_history, test_loss_history = pidon.train( cp, t_interval, training_data_args=DataArgs( y_0_functions=[ic.y_0], n_domain_points=20, n_boundary_points=10, n_batches=1 ), model_args=ModelArgs( latent_output_size=20, branch_hidden_layer_sizes=[30, 30], trunk_hidden_layer_sizes=[30, 30], ), optimization_args=OptimizationArgs( optimizer=optimizers.Adam(learning_rate=2e-5), epochs=3, verbose=False ) ) assert len(training_loss_history) == 3 for i in range(2): assert np.all( training_loss_history[i + 1].weighted_total_loss.numpy() < training_loss_history[i].weighted_total_loss.numpy()) solution = pidon.solve(ivp) assert solution.d_t == .05 assert solution.discrete_y().shape == (20, 2, 3, 1)
def test_pidon_operator_on_pde_system(): set_random_seed(0) diff_eq = NavierStokesEquation() mesh = Mesh([(-2.5, 2.5), (0., 4.)], [1., 1.]) ic_function = vectorize_ic_function(lambda x: [ 2. * x[0] - 4., 2. * x[0] ** 2 + 3. * x[1] - x[0] * x[1] ** 2, 4. * x[0] - x[1] ** 2, 2. * x[0] * x[1] - 3. ]) bcs = [ (DirichletBoundaryCondition( lambda x, t: ic_function(x), is_static=True), DirichletBoundaryCondition( lambda x, t: ic_function(x), is_static=True)) ] * 2 cp = ConstrainedProblem(diff_eq, mesh, bcs) ic = ContinuousInitialCondition(cp, ic_function) t_interval = (0., .5) ivp = InitialValueProblem(cp, t_interval, ic) sampler = UniformRandomCollocationPointSampler() pidon = PIDONOperator(sampler, .001, True) training_loss_history, test_loss_history = pidon.train( cp, t_interval, training_data_args=DataArgs( y_0_functions=[ic.y_0], n_domain_points=20, n_boundary_points=10, n_batches=1 ), model_args=ModelArgs( latent_output_size=20, branch_hidden_layer_sizes=[20, 20], trunk_hidden_layer_sizes=[20, 20], ), optimization_args=OptimizationArgs( optimizer=optimizers.Adam(learning_rate=1e-5), epochs=3, verbose=False ) ) assert len(training_loss_history) == 3 for i in range(2): assert np.all( training_loss_history[i + 1].weighted_total_loss.numpy() < training_loss_history[i].weighted_total_loss.numpy()) solution = pidon.solve(ivp) assert solution.d_t == .001 assert solution.discrete_y().shape == (500, 6, 5, 4)
def gan(g_model, d_model, q_model): d_model.trainable = False d_output = d_model(g_model.output) q_output = q_model(g_model.output) model = Model(inputs=g_model.input, outputs=[d_output, q_output]) opt = optimizers.Adam(learning_rate=0.0002, beta_1=0.5) model.compile(loss=['binary_crossentropy', 'categorical_crossentropy'], optimizer=opt) return model
def prepare_model(): model = create_unet() loss = losses.SparseCategoricalCrossentropy(from_logits=True) dice = metric_dice() optimizer = optimizers.Adam(learning_rate=2e-4) model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy', dice]) return model
def test_pidon_operator_in_ar_mode_on_pde(): set_random_seed(0) diff_eq = WaveEquation(1) mesh = Mesh([(0., 1.)], (.2,)) bcs = [ (NeumannBoundaryCondition( lambda x, t: np.zeros((len(x), 2)), is_static=True), NeumannBoundaryCondition( lambda x, t: np.zeros((len(x), 2)), is_static=True)), ] cp = ConstrainedProblem(diff_eq, mesh, bcs) t_interval = (0., 1.) ic = BetaInitialCondition(cp, [(3.5, 3.5), (3.5, 3.5)]) ivp = InitialValueProblem(cp, t_interval, ic) training_y_0_functions = [ BetaInitialCondition(cp, [(p, p), (p, p)]).y_0 for p in [2., 3., 4., 5.] ] sampler = UniformRandomCollocationPointSampler() pidon = PIDONOperator(sampler, .25, False, auto_regression_mode=True) assert pidon.auto_regression_mode pidon.train( cp, (0., .25), training_data_args=DataArgs( y_0_functions=training_y_0_functions, n_domain_points=50, n_boundary_points=20, n_batches=2 ), model_args=ModelArgs( latent_output_size=50, branch_hidden_layer_sizes=[50, 50], trunk_hidden_layer_sizes=[50, 50], ), optimization_args=OptimizationArgs( optimizer=optimizers.Adam(learning_rate=1e-4), epochs=2, ic_loss_weight=10., verbose=False ) ) sol = pidon.solve(ivp) assert np.allclose(sol.t_coordinates, [.25, .5, .75, 1.]) assert sol.discrete_y().shape == (4, 5, 2)
def build_model(input_shape, learning_rate, loss="sparse_categorical_crossentropy" ): #sparse_categorical_crossentropy # build network model = keras.Sequential() # conv1 model.add( keras.layers.Conv2D(64, (3, 3), activation='relu', input_shape=input_shape, kernel_regularizer=keras.regularizers.l2(0.001))) model.add(keras.layers.BatchNormalization()) model.add(keras.layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same')) # conv2 model.add( keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))) model.add(keras.layers.BatchNormalization()) model.add(keras.layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same')) # conv3 model.add( keras.layers.Conv2D(32, (2, 2), activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))) model.add(keras.layers.BatchNormalization()) model.add(keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same')) # flatten the output feed to output layer model.add(keras.layers.Flatten()) model.add(keras.layers.Dense(64, activation="relu")) model.add(keras.layers.Dropout(0.3)) # softmax classifier model.add(keras.layers.Dense(16, activation='softmax')) optimiser = optimizers.Adam(learning_rate=learning_rate) # compile model model.compile(optimizer=optimiser, loss=loss, metrics=["accuracy"]) model.summary() return model
def generate_3D_VGG(L2_constant, dropout, learning_rate, kernel_initializer='glorot_uniform', model_summary=True): # --- Define kwargs dictionary kwargs = { 'kernel_size': (3, 3, 3), 'padding': 'same', 'kernel_regularizer': keras.regularizers.l2(L2_constant), 'kernel_initializer': kernel_initializer } # --- Define block components conv = lambda x, filters, strides: layers.Conv3D( filters=filters, strides=strides, **kwargs)(x) relu = lambda x: layers.LeakyReLU()(x) norm = lambda x: layers.BatchNormalization()(x) # --- Define stride-1, stride-2 blocks conv1 = lambda filters, x: relu(norm(conv(x, filters, strides=1))) conv2 = lambda filters, x: relu(norm(conv(x, filters, strides=(2, 2, 2)))) # --- Architecture inputs = Input(shape=(64, 64, 64, 3)) l1 = conv2(16, conv1(16, inputs)) l2 = conv2(32, conv1(32, l1)) l3 = conv2(64, conv1(64, l2)) l4 = conv2(128, conv1(128, l3)) f0 = layers.Flatten()(l4) h1 = relu(layers.Dense(128)(f0)) h2 = relu(norm(h1)) h2 = layers.Dropout(dropout) logits = layers.Dense(2)(h1) model = Model(inputs=inputs, outputs=logits) if model_summary: model.summary() # --- Define a categorical cross-entropy loss loss = losses.SparseCategoricalCrossentropy(from_logits=True) optimizer = optimizers.Adam(learning_rate=learning_rate) # --- Compile model model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy']) return model
def build_cnn(): """Builds a convolutional neural net (cnn) using tensorflow Args: Returns: model: The model for the cnn """ drop_out = 0.25 model = keras.models.Sequential() model.add( tf.keras.layers.Conv2D(32, kernel_size=5, padding="valid", activation='relu')) model.add(tf.keras.layers.Dropout(drop_out)) model.add(tf.keras.layers.BatchNormalization(momentum=0.8)) model.add( tf.keras.layers.Conv2D(64, kernel_size=4, padding="valid", activation='relu')) model.add(tf.keras.layers.Dropout(drop_out)) model.add(tf.keras.layers.BatchNormalization(momentum=0.8)) model.add( tf.keras.layers.Conv2D(64, kernel_size=3, padding="valid", activation='relu')) model.add(tf.keras.layers.Dropout(drop_out)) model.add(tf.keras.layers.BatchNormalization(momentum=0.8)) model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Dense(256, activation='relu')) model.add(tf.keras.layers.Dropout(0.5)) model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax)) model.compile(optimizer=tf_optimizers.Adam(learning_rate=8e-4), loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model
def define_optimizer(opt_name, opt_kwargs): if opt_name == 'sgd': optimizer = optimizers.SGD(**opt_kwargs) elif opt_name == 'adam': optimizer = optimizers.Adam(**opt_kwargs) elif opt_name == 'adagrad': optimizer = optimizers.Adagrad(**opt_kwargs) elif opt_name == 'rmsprop': optimizer = optimizers.RMSprop(**opt_kwargs) elif opt_name == 'adadelta': optimizer = optimizers.Adadelta(**opt_kwargs) elif opt_name == 'nadam': optimizer = optimizers.Nadam(**opt_kwargs) elif opt_name == 'adamax': optimizer = optimizers.Adamax(**opt_kwargs) else: raise ValueError(f"Optimizer {opt_name} is wrong or not implemented.") return optimizer
def test_auto_regression_operator_on_pde(): set_random_seed(0) diff_eq = WaveEquation(2) mesh = Mesh([(-5., 5.), (-5., 5.)], [1., 1.]) bcs = [(DirichletBoundaryCondition(lambda x, t: np.zeros((len(x), 2)), is_static=True), DirichletBoundaryCondition(lambda x, t: np.zeros((len(x), 2)), is_static=True))] * 2 cp = ConstrainedProblem(diff_eq, mesh, bcs) ic = GaussianInitialCondition( cp, [(np.array([0., 2.5]), np.array([[.1, 0.], [0., .1]]))] * 2, [3., .0]) ivp = InitialValueProblem(cp, (0., 10.), ic) oracle = FDMOperator(RK4(), ThreePointCentralDifferenceMethod(), .1) ref_solution = oracle.solve(ivp) ml_op = AutoRegressionOperator(2.5, True) ml_op.train( ivp, oracle, SKLearnKerasRegressor( DeepONet([ np.prod(cp.y_shape(True)).item(), 100, 50, diff_eq.y_dimension * 10 ], [1 + diff_eq.x_dimension, 50, 50, diff_eq.y_dimension * 10], diff_eq.y_dimension), optimizer=optimizers.Adam( learning_rate=optimizers.schedules.ExponentialDecay( 1e-2, decay_steps=500, decay_rate=.95)), batch_size=968, epochs=500, ), 20, lambda t, y: y + np.random.normal(0., t / 75., size=y.shape)) ml_solution = ml_op.solve(ivp) assert ml_solution.vertex_oriented assert ml_solution.d_t == 2.5 assert ml_solution.discrete_y().shape == (4, 11, 11, 2) diff = ref_solution.diff([ml_solution]) assert np.all(diff.matching_time_points == np.linspace(2.5, 10., 4)) assert np.max(np.abs(diff.differences[0])) < .5
def retinanet_resnet50_3d(inputs, K, A, filter_ratio=1, n=2, include_fc_layer=False, shared_weights=False, tahn=False, lr=2e-4, feature_maps=('c3', 'c4', 'c5')): """Generates retinanet with resnet backbone. Can specify if classification and regression networks share weights""" r_model = resnet50_3d(inputs=inputs['dat'], filter_ratio=filter_ratio, n=n, include_fc_layer=include_fc_layer, kernal1=(1, 1, 1), kernal3=(1, 3, 3), kernal7=(1, 7, 7)) backbone_output = [ r_model.get_layer(layer_name).output for layer_name in ['c3-output', 'c4-output', 'c5-output'] ] fp_out = feature_pyramid_3d(inputs=backbone_output, filter_ratio=filter_ratio) logits = class_and_reg_subnets(feature_pyramid=fp_out, K=K, A=A, filter_ratio=filter_ratio, shared_weights=shared_weights, tahn=tahn, feature_maps=feature_maps) preds = LogisticEndpoint1()(logits, inputs) model = Model(inputs=inputs, outputs=preds) model.compile( optimizer=optimizers.Adam(learning_rate=lr), experimental_run_tf_function=False, ) return model
def __init__(self, num_layers, state_space, policy_controller, reg_param=0.001, discount_factor=0.99, exploration=0.8, ): self.num_layers = num_layers self.state_space = state_space self.policy_controller = policy_controller self.reg_strength = reg_param self.discount_factor = discount_factor self.exploration = exploration self.state_size = self.state_space.size self.reward_buffer = [] self.state_buffer = [] self.labels = [] self.global_step = tf.Variable(0) starter_learning_rate = 0.1 learning_rate = keras.optimizers.schedules.ExponentialDecay( starter_learning_rate, decay_steps=50, decay_rate=0.95, staircase=True) self.optimizer = optimizers.Adam(learning_rate=learning_rate)
train_generator = train_datagen.flow_from_directory( train_dir, # 将所有图像的大小调整为227*227 target_size=(227, 227), # 批量大小 batch_size=16, class_mode='binary') validation_generator = validation_datagen.flow_from_directory( validation_dir, target_size=(227, 227), batch_size=16, class_mode='binary') # 初始化模型 model = model() # 用于配置训练模型(优化器、目标函数、模型评估标准) model.compile(optimizer=optimizers.Adam(lr=1e-4), loss='sparse_categorical_crossentropy', metrics=['accuracy']) # 查看各个层的信息 model.summary() # 回调函数,在每个训练期之后保存模型 model_checkpoint = ModelCheckpoint( 'model_AlexNet_adam.hdf5', # 保存模型的路径 monitor='loss', # 被监测的数据 verbose=1, # 日志显示模式:0=>安静模式,1=>进度条,2=>每轮一行 save_best_only=True) # 若为True,最佳模型就不会被覆盖 # 用history接收返回值用于画loss/acc曲线 history = model.fit(train_generator, steps_per_epoch=313, epochs=200, callbacks=[model_checkpoint],
def unet(inputs, filter_ratio=1, logits_num=2, num_layers=6, class_num=1, _3d=False, compile=False, lr=2e-4): # --- Define kwargs dictionary if _3d: kwargs = {'kernel_size': (3, 3, 3), 'padding': 'same'} else: kwargs = {'kernel_size': (1, 3, 3), 'padding': 'same'} # --- Define lambda functions conv = lambda x, filters, strides: layers.Conv3D( filters=filters, strides=strides, **kwargs)(x) norm = lambda x: layers.BatchNormalization()(x) relu = lambda x: layers.LeakyReLU()(x) tran = lambda x, filters, strides: layers.Conv3DTranspose( filters=filters, strides=strides, **kwargs)(x) # --- Define stride-1, stride-2 blocks conv1 = lambda filters, x: relu(norm(conv(x, filters, strides=1))) conv2 = lambda filters, x: relu(norm(conv(x, filters, strides=(1, 2, 2)))) tran2 = lambda filters, x: relu(norm(tran(x, filters, strides=(1, 2, 2)))) # --- Define simple layers c_layer = lambda filters, x: conv1(filters, conv2(filters, x)) e_layer = lambda filters1, filters2, x: tran2(filters1, conv1(filters2, x)) contracting_layers = [] for i in range(num_layers): # 0,1,2,3,4,5 if i == 0: contracting_layers.append(conv1(int(4 * filter_ratio), inputs)) else: contracting_layers.append( c_layer(int(8 * filter_ratio) * i, contracting_layers[i - 1])) expanding_layers = [] for j in reversed(range(num_layers - 1)): # 4,3,2,1,0 if j == num_layers - 2: expanding_layers.append( tran2(int(8 * filter_ratio) * j, contracting_layers[j + 1])) else: expanding_layers.append( e_layer( int(8 * filter_ratio) * j if j != 0 else int(4 * filter_ratio), int(8 * filter_ratio) * (j + 1), expanding_layers[-1] + contracting_layers[j + 1])) last_layer = conv1( int(4 * filter_ratio), conv1(int(4 * filter_ratio), expanding_layers[-1] + contracting_layers[0])) # --- Create logits logits = {} for k in range(class_num): logits[f'zones{k}'] = layers.Conv3D(filters=logits_num, name=f'zones{k}', **kwargs)(last_layer) # --- Create model model = Model(inputs=inputs, outputs=logits) if compile: model.compile( optimizer=optimizers.Adam(learning_rate=lr), loss_weights={ i: keras.losses.SparseCategoricalCrossentropy(from_logits=True) for i in model.output_names }, metrics={i: custom.dsc(cls=1) for i in model.output_names}, # TODO: Check if leaving this parameter out affects model training. experimental_run_tf_function=False, ) return model
y_str = numpy.loadtxt("poco_1.prn", skiprows=11, usecols=8, dtype=numpy.str) label_encoder = preprocessing.LabelEncoder() label_encoder.fit(list(set(y_str))) y = label_encoder.transform(y_str) x_train, x_test, y_train, y_test = model_selection.train_test_split( X, y, test_size=0.33, random_state=42) #Building, saving initial values of kernel/bias, compiling and training model print("Prunable network:") nn = create_neural_network_prunable() nn.build(x_train.shape) for i in nn.layers: i.save_kernel() i.save_bias() nn.summary() nn.compile(optimizer=optimizers.Adam(learning_rate=0.0001), loss=losses.BinaryCrossentropy(), metrics=[metrics.BinaryAccuracy()]) print("Before pruning:") nn.fit(x_train, y_train, epochs=10, batch_size=64, validation_data=(x_train, y_train)) loss, accuracy = nn.evaluate(x_test, y_test, verbose=0) print("Loss:", loss) print("Accuracy:", accuracy) #Creating tensor of weights l1, l2 = [], [] for layer in nn.layers:
Test_Y = np_utils.to_categorical(Data_Test_Y) # normalize X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=30, shuffle=True) # create and fit the model model = Sequential() model.add(LSTM(200, input_shape=(X.shape[1], X.shape[2]))) model.add(Dropout(0.3)) model.add(Dense(y.shape[1], activation='softmax')) adam = optimizers.Adam(lr=0.0001) # use Adam optimizer model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) model.summary() # build the model history = model.fit(X_train, y_train, epochs=500, batch_size=200, validation_split=0.3, verbose=1, shuffle=True) # save model model.save(filepath="../checkpoints/LSTM_FB15K237", save_format='tf')
dnn_feature_columns = fixlen_feature_columns linear_feature_columns = fixlen_feature_columns feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns) # 3.generate input data for model train_model_input = {name: train[name] for name in feature_names} test_model_input = {name: test[name] for name in feature_names} # 4.Define Model,train,predict and evaluate model = DeepFM(linear_feature_columns, dnn_feature_columns, task='binary', dnn_dropout=dropout) optimizers.Adam(learning_rate=0.001) model.compile(optimizer, "binary_crossentropy", metrics=METRICS) log_dir = prefix_dir + 'dfm_' + data_type + '_' + str(epochs) if not os.path.exists(log_dir): os.makedirs(log_dir) logs = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) history = model.fit(train_model_input, train[target].values, batch_size=256, epochs=epochs, verbose=2, validation_split=0.2, callbacks=[logs])
import tensorflow as tf import tensorflow.keras.layers as layers import tensorflow.optimizers as optimizers import numpy as np # TEMPORARY STUFF UNRELATED TO MIDIS print(tf.__version__) mnist = tf.keras.datasets.mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train, x_test = x_train / 255.0, x_test / 255.0 x_train = np.expand_dims(x_train, 3) x_test = np.expand_dims(x_test, 3) print(x_train.shape) model = tf.keras.models.Sequential([ layers.Conv2D(16, (5, 5), input_shape=(28, 28, 1)), layers.Dropout(0.03), layers.Activation('relu'), layers.Flatten(), layers.Dense(10, activation=tf.nn.softmax), layers.Dropout(0.03), ]) model.compile(optimizer=optimizers.Adam(learning_rate=0.005), loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5, batch_size=128) model.evaluate(x_test, y_test)
model = keras.Sequential() model.add(layers.Input([28, 28])) model.add(layers.Reshape([28, 28, 1])) model.add(layers.Conv2D(filters=32, kernel_size=3, strides=2, padding='SAME')) model.add(layers.MaxPooling2D(pool_size=(2, 2), strides=2, padding='VALID')) model.add(layers.Conv2D(filters=64, kernel_size=3, strides=2, padding='SAME')) model.add(layers.MaxPooling2D(pool_size=(2, 2), strides=2, padding='VALID')) model.add(layers.Flatten()) model.add(layers.Dense(300, activation='relu')) model.add(layers.Dense(10, activation='softmax')) model.summary() loss_object = losses.SparseCategoricalCrossentropy() optimizer = optimizers.Adam() train_loss = tf.metrics.Mean(name='train_loss') train_acc = tf.metrics.SparseCategoricalAccuracy(name='train_acc') test_loss = tf.metrics.Mean(name='test_loss') test_acc = tf.metrics.SparseCategoricalAccuracy(name='test_acc') @tf.function def train_step(images, labels): with tf.GradientTape() as tape: predictions = model(images) loss = loss_object(labels, predictions) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) train_loss(loss)
#f0 = layers.Reshape((1, 1, 1, bf.shape[2] * bf.shape[3] * bf.shape[4]))(bf) # --- Create logits logits = {} #logits['lbl'] = layers.AveragePooling3D(pool_size=(1, bf.shape[2], bf.shape[3]), padding='same', name='lbl')(bf) logits['lbl'] = layers.Conv3D(filters=1, kernel_size=(1, 1, 1), activation='sigmoid', name='lbl')(f0) # --- Create model model = Model(inputs=inputs, outputs=logits) # --- Compile model model.compile( optimizer=optimizers.Adam(learning_rate=5e-5), #loss=losses.Huber(delta=0.042), loss=losses.MeanAbsoluteError(), metrics=['mse', 'mae', 'mape'], experimental_run_tf_function=False) # --- Load data into memory for faster training client.load_data_in_memory() # --- TensorBoard #tensor_board = TensorBoard(log_dir='./graph', histogram_freq=0, write_graph=True, write_images=True) # --- Learning rate scheduler lr_scheduler = callbacks.LearningRateScheduler(lambda epoch, lr: lr * 0.996) # --- csv Callback
BetaInitialCondition(cp, [(p, p)]).y_0 for p in np.arange(1.2, 6., .2) ] pidon.train(cp, t_interval, training_data_args=DataArgs(y_0_functions=training_y_0_functions, n_domain_points=500, n_boundary_points=100, n_batches=1), model_args=ModelArgs( latent_output_size=50, branch_hidden_layer_sizes=[50] * 7, trunk_hidden_layer_sizes=[50] * 7, ), optimization_args=OptimizationArgs( optimizer=optimizers.Adam( learning_rate=optimizers.schedules.ExponentialDecay( 2e-3, decay_steps=25, decay_rate=.98)), epochs=5000, ic_loss_weight=10., )) for p in [2., 3.5, 5.]: ic = BetaInitialCondition(cp, [(p, p)]) ivp = InitialValueProblem(cp, t_interval, ic) fdm_solution = fdm.solve(ivp) for i, plot in enumerate(fdm_solution.generate_plots()): plot.save('diff_1d_fdm_{:.1f}_{}'.format(p, i)).close() pidon_solution = pidon.solve(ivp) for i, plot in enumerate(pidon_solution.generate_plots()):
def build_origin(self, print_summary=False, num_classes=5, image_size=(352, 640, 3)): input_tensor = keras.layers.Input(image_size) conv_0 = self.build_conv2D_block(input_tensor, filters=24, kernel_size=1, strides=1) # conv stage 1 conv_1 = self.build_conv2D_block(conv_0, filters=64, kernel_size=3, strides=1) conv_1 = self.build_conv2D_block(conv_1, filters=64, kernel_size=3, strides=1) # pool stage 1 pool1 = MaxPooling2D()(conv_1) # conv stage 2 conv_2 = self.build_conv2D_block(pool1, filters=128, kernel_size=3, strides=1) conv_2 = self.build_conv2D_block(conv_2, filters=128, kernel_size=3, strides=1) # pool stage 2 pool2 = MaxPooling2D()(conv_2) # conv stage 3 conv_3 = self.build_conv2D_block(pool2, filters=256, kernel_size=3, strides=1) conv_3 = self.build_conv2D_block(conv_3, filters=256, kernel_size=3, strides=1) conv_3 = self.build_conv2D_block(conv_3, filters=256, kernel_size=3, strides=1) # pool stage 3 pool3 = MaxPooling2D()(conv_3) # conv stage 4 conv_4 = self.build_conv2D_block(pool3, filters=512, kernel_size=3, strides=1) conv_4 = self.build_conv2D_block(conv_4, filters=512, kernel_size=3, strides=1) conv_4 = self.build_conv2D_block(conv_4, filters=512, kernel_size=3, strides=1) # pool4 = MaxPooling2D()(conv_4) ### add dilated convolution ### # conv stage 5_1 conv_5 = self.build_conv2D_block(conv_4, filters=512, kernel_size=3, strides=1, dilation_rate=2) conv_5 = self.build_conv2D_block(conv_5, filters=512, kernel_size=3, strides=1, dilation_rate=2) conv_5 = self.build_conv2D_block(conv_5, filters=512, kernel_size=3, strides=1, dilation_rate=2) # added part of SCNN # conv_6_4 = self.build_conv2D_block(conv_5, filters=1024, kernel_size=3, strides=1, dilation_rate=4) conv_6_5 = self.build_conv2D_block(conv_6_4, filters=128, kernel_size=1, strides=1) # 8 x 36 x 100 x 128 # add message passing # # top to down # feature_list_new = self.space_cnn_part(conv_6_5) ####################### dropout_output = Dropout(0.9)(feature_list_new) conv_output = K.resize_images( dropout_output, height_factor=self.IMG_HEIGHT // dropout_output.shape[1], width_factor=self.IMG_WIDTH // dropout_output.shape[2], data_format="channels_last") ret_prob_output = Conv2D(filters=num_classes, kernel_size=1, activation='softmax', name='ctg_out_1')(conv_output) ### add lane existence prediction branch ### # spatial softmax # features = ret_prob_output # N x H x W x C softmax = Activation('softmax')(features) avg_pool = AvgPool2D(strides=2)(softmax) _, H, W, C = avg_pool.get_shape().as_list() reshape_output = tf.reshape(avg_pool, [-1, H * W * C]) fc_output = Dense(128)(reshape_output) relu_output = ReLU(max_value=6)(fc_output) existence_output = Dense(4, name='ctg_out_2')(relu_output) self.model = Model(inputs=input_tensor, outputs=[ret_prob_output, existence_output]) # print(self.model.summary()) adam = optimizers.Adam(lr=0.001) sgd = optimizers.SGD(lr=0.001) if num_classes == 1: self.model.compile(optimizer=sgd, loss="binary_crossentropy", metrics=['accuracy']) else: self.model.compile(optimizer=sgd, loss={ 'ctg_out_1': 'categorical_crossentropy', 'ctg_out_2': 'binary_crossentropy' }, loss_weights={ 'ctg_out_1': 1., 'ctg_out_2': 0.2, }, metrics=['accuracy', 'mse'])
def build(self, print_summary=False, num_classes=5, image_size=(352, 640, 3)): input_tensor = keras.layers.Input(image_size) conv_0 = self.build_conv2D_block(input_tensor, filters=24, kernel_size=1, strides=1) conv_0 = self.build_conv2D_block(conv_0, filters=24, kernel_size=3, strides=1) conv_0 = self.build_conv2D_block(conv_0, filters=24, kernel_size=3, strides=1) conv_0 = self.build_conv2D_block(conv_0, filters=24, kernel_size=3, strides=1) # first conv layer conv_1 = self.build_conv2D_block(conv_0, filters=48, kernel_size=3, strides=2) conv_1 = self.build_conv2D_block(conv_1, filters=48, kernel_size=3, strides=1) conv_1 = self.build_conv2D_block(conv_1, filters=48, kernel_size=3, strides=1) conv_1 = self.build_conv2D_block(conv_1, filters=48, kernel_size=3, strides=1) conv_1 = self.build_conv2D_block(conv_1, filters=48, kernel_size=3, strides=1) # second conv layer conv_2 = self.build_conv2D_block(conv_1, filters=64, kernel_size=3, strides=2) conv_2 = self.build_conv2D_block(conv_2, filters=64, kernel_size=3, strides=1) conv_2 = self.build_conv2D_block(conv_2, filters=64, kernel_size=3, strides=1) conv_2 = self.build_conv2D_block(conv_2, filters=64, kernel_size=3, strides=1) conv_2 = self.build_conv2D_block(conv_2, filters=64, kernel_size=3, strides=1) # third conv layer conv_3 = self.build_conv2D_block(conv_2, filters=96, kernel_size=3, strides=2) conv_3 = self.build_conv2D_block(conv_3, filters=96, kernel_size=3, strides=1) conv_3 = self.build_conv2D_block(conv_3, filters=96, kernel_size=3, strides=1) conv_3 = self.build_conv2D_block(conv_3, filters=96, kernel_size=3, strides=1) conv_3 = self.build_conv2D_block(conv_3, filters=96, kernel_size=3, strides=1) # fourth conv layer conv_4 = self.build_conv2D_block(conv_3, filters=128, kernel_size=3, strides=2) conv_4 = self.build_conv2D_block(conv_4, filters=128, kernel_size=3, strides=1) conv_4 = self.build_conv2D_block(conv_4, filters=128, kernel_size=3, strides=1) conv_4 = self.build_conv2D_block(conv_4, filters=128, kernel_size=3, strides=1) conv_4 = self.build_conv2D_block(conv_4, filters=128, kernel_size=3, strides=1) # fifth conv layer conv_5 = self.build_conv2D_block(conv_4, filters=256, kernel_size=3, strides=1, dilation_rate=1) conv_5 = self.build_conv2D_block(conv_5, filters=256, kernel_size=3, strides=1, dilation_rate=1) conv_5 = self.build_conv2D_block(conv_5, filters=256, kernel_size=3, strides=1, dilation_rate=1) conv_5 = self.build_conv2D_block(conv_5, filters=256, kernel_size=3, strides=1, dilation_rate=1) conv_5 = self.build_conv2D_block(conv_5, filters=256, kernel_size=3, strides=1, dilation_rate=1) # added part of SCNN # conv_6_4 = self.build_conv2D_block(conv_5, filters=256, kernel_size=3, strides=1, dilation_rate=1) conv_6_5 = self.build_conv2D_block(conv_6_4, filters=128, kernel_size=1, strides=1) # 8 x 36 x 100 x 128 scnn_part = self.space_cnn_part(conv_6_5) ####################### # conv2d_deconv5_1 = self.build_conv2D_block(conv_5,filters = 196,kernel_size=3,strides=1) # conv2d_deconv4 = self.build_conv2Dtranspose_block(conv2d_deconv5_1, filters=128, kernel_size=4, strides=2) Concat_concat4 = concatenate([scnn_part, conv_4], axis=-1) conv2d_deconv4_1 = self.build_conv2D_block(Concat_concat4, filters=96, kernel_size=3, strides=1) conv2d_deconv3 = self.build_conv2Dtranspose_block(conv2d_deconv4_1, filters=96, kernel_size=4, strides=2) Concat_concat3 = concatenate([conv2d_deconv3, conv2d_deconv3], axis=-1) conv2d_deconv3_1 = self.build_conv2D_block(Concat_concat3, filters=64, kernel_size=3, strides=1) conv2d_deconv2 = self.build_conv2Dtranspose_block(conv2d_deconv3_1, filters=64, kernel_size=4, strides=2) Concat_concat2 = concatenate([conv2d_deconv2, conv_2], axis=-1) conv2d_deconv2_1 = self.build_conv2D_block(Concat_concat2, filters=32, kernel_size=3, strides=1) conv2d_deconv1 = self.build_conv2Dtranspose_block(conv2d_deconv2_1, filters=32, kernel_size=4, strides=2) Concat_concat1 = concatenate([conv2d_deconv1, conv_1], axis=-1) conv2d_deconv1_1 = self.build_conv2D_block(Concat_concat1, filters=16, kernel_size=3, strides=1) conv2d_deconv0 = self.build_conv2Dtranspose_block(conv2d_deconv1_1, filters=128, kernel_size=4, strides=2) if num_classes == 1: ret_prob_output = Conv2DTranspose(filters=num_classes, kernel_size=1, strides=1, activation='sigmoid', padding='same', name='ctg_out_1')(conv2d_deconv0) else: ret_prob_output = Conv2DTranspose(filters=num_classes, kernel_size=1, strides=1, activation='softmax', padding='same', name='ctg_out_1')(conv2d_deconv0) ### add lane existence prediction branch ### # spatial softmax # # features = ret_prob_output # N x H x W x C # softmax = Activation('softmax')(features) # avg_pool = AvgPool2D(strides=2)(softmax) features = self.build_conv2D_block(ret_prob_output, filters=num_classes, kernel_size=1, strides=2) _, H, W, C = features.get_shape().as_list() reshape_output = tf.reshape(features, [-1, H * W * C]) fc_output = Dense(128)(reshape_output) relu_output = Activation('relu')(fc_output) existence_output = Dense(4)(relu_output) existence_output = Activation('softmax', name='ctg_out_2')(existence_output) self.model = Model(inputs=input_tensor, outputs=[ret_prob_output, existence_output]) # print(self.model.summary()) adam = optimizers.Adam(lr=0.001) sgd = optimizers.SGD(lr=0.001) if num_classes == 1: self.model.compile(optimizer=sgd, loss="binary_crossentropy", metrics=['accuracy']) else: self.model.compile(optimizer=sgd, loss=self.my_loss_error, metrics=['accuracy'])
if __name__ == '__main__': train_dataset = tf.data.Dataset.list_files(PATH + 'train/*.jpg') train_dataset = train_dataset.map( load_image_train, num_parallel_calls=tf.data.experimental.AUTOTUNE) train_dataset = train_dataset.shuffle(BUFFER_SIZE) train_dataset = train_dataset.batch(BATCH_SIZE) test_dataset = tf.data.Dataset.list_files(PATH + 'test/*.jpg') test_dataset = test_dataset.map(load_image_test) test_dataset = test_dataset.batch(BATCH_SIZE) generator = Generator(output_channels=OUTPUT_CHANNELS, name='generator') discriminator = Discriminator() generator_loss = GeneratorLoss discriminator_loss = DiscriminatorLoss() generator_optimizer = optimizers.Adam(learning_rate=1e-4, beta_1=0.5) discriminator_optimizer = optimizers.Adam(learning_rate=1e-4, beta_1=0.5) checkpoint_dir = './training_checkpoints' checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt") checkpoint = tf.train.Checkpoint( generator_optimizer=generator_optimizer, discriminator_optimizer=discriminator_optimizer, generator=generator, discriminator=discriminator) log_dir = "logs/" summary_writer = tf.summary.create_file_writer( log_dir + "fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")) @tf.function
n_ic_repeats=3 ), test_data_args=DataArgs( y_0_functions=test_y_0_functions, n_domain_points=50, n_batches=1 ), model_args=ModelArgs( latent_output_size=100, branch_hidden_layer_sizes=[100, 100, 100, 100, 100], trunk_hidden_layer_sizes=[100, 100, 100, 100, 100], ), optimization_args=OptimizationArgs( optimizer=optimizers.Adam( learning_rate=optimizers.schedules.ExponentialDecay( 1e-3, decay_steps=30, decay_rate=.97 ) ), epochs=2000, ), secondary_optimization_args=SecondaryOptimizationArgs( max_iterations=500 ) ) for y_0 in [.7, 1., 1.3]: ic = ContinuousInitialCondition(cp, lambda _: np.array([y_0])) ivp = InitialValueProblem(cp, t_interval, ic) fdm_solution = fdm.solve(ivp) for i, plot in enumerate(fdm_solution.generate_plots()):