def test_mask_loss_network(): model = Sequential() model.add(Dense(16*16, input_dim=16*16)) model.add(Reshape((1, 16, 16))) net_out = model.get_output() net_in = model.get_input() th_mask = T.tensor4() loss = mask_loss(th_mask, net_out)['loss'] updates = Adam().get_updates(model.params, model.constraints, loss) train_fn = theano.function([th_mask, net_in], [loss], updates=updates) nb_batches = 32 mask_idx = next(masks(64*nb_batches, scales=[0.25])) z = np.random.uniform(low=-1, high=1, size=mask_idx.shape).reshape((-1, 16*16)).astype(np.float32) first_loss = 0 epochs = 30 nb_batches = 10 for i, mask_idx in enumerate(itertools.islice(masks(64*nb_batches, scales=[0.25]), epochs)): z = np.random.uniform(low=-1, high=1, size=mask_idx.shape ).reshape((-1, 16*16)).astype(np.float32) loss = train_fn(mask_idx, z) # print(loss) if i == 0: first_loss = loss assert first_loss > loss
def __init__(self): left = Sequential() left.add(Dense(784, 50)) left.add(Activation('relu')) model = Sequential() model.add(Merge([left, left], mode='sum')) model.add(Dense(50, 10)) model.add(Activation('softmax')) pdb.set_trace() model = Sequential() left = Sequential() num_kernel = 32 l1_penalty = 0.0001 b_mode = 'full' left.add(Convolution2D(num_kernel, 3, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) left.add(Convolution2D(num_kernel, num_kernel, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) left.add(LeakyReLU(0.1)) #left.add(Activation('relu')) left.add(MaxPooling2D(poolsize=(2, 2))) #left.add(Convolution2D(num_kernel, 3, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) #left.add(Convolution2D(num_kernel, num_kernel, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) #left.add(LeakyReLU(0.1)) ##left.add(Activation('relu')) #left.add(MaxPooling2D(poolsize=(2, 2))) model.add(Merge([left, left], mode='sum')) pdb.set_trace() self.f = theano.function(model.get_input(), model.get_output())
def test_deconvolution2d(): model = Sequential() model.add(Deconvolution2D(10, 3, 3, subsample=(2, 2), input_shape=(1, 16, 16))) out = model.get_output() fn = theano.function([model.input], out) img = np.random.sample((64, 1, 16, 16)).astype(np.float32) deconv_out = fn(img) assert deconv_out.shape == (64, 10, 32, 32)
def test_deconvolution2d(): model = Sequential() model.add( Deconvolution2D(10, 3, 3, subsample=(2, 2), input_shape=(1, 16, 16))) out = model.get_output() fn = theano.function([model.input], out) img = np.random.sample((64, 1, 16, 16)).astype(np.float32) deconv_out = fn(img) assert deconv_out.shape == (64, 10, 32, 32)
class TestRecursive(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestRecursive, self).__init__(*args, **kwargs) self.input_dim = 2 self.state_dim = 2 self.model = Recursive(return_sequences=True) self.model.add_input('input', ndim=3) # Input is 3D tensor self.model.add_state('h', dim=self.state_dim) self.model.add_node(Dense(self.input_dim + self.state_dim, self.state_dim, init='one'), name='rec', inputs=['input', 'h'], return_state='h') self.model.add_node(Activation('linear'), name='out', input='rec', create_output=True) self.model2 = Sequential() self.model2.add( SimpleRNN(input_dim=self.input_dim, activation='linear', inner_init='one', output_dim=self.state_dim, init='one', return_sequences=True)) def test_step(self): XX = T.matrix() HH = T.matrix() A = self.model._step(XX, HH) F = function([XX, HH], A) x = np.ones((1, 2)) h = np.ones((1, 2)) y = F(x, h) r = np.asarray([[4., 4.]]) assert_allclose([r, r], y) def test_get_get_output(self): X = self.model.get_input() Y = self.model._get_output() F = function([X], Y, allow_input_downcast=True) x = np.ones((3, 5, self.input_dim)) y = F(x) print y X2 = self.model2.get_input() Y2 = self.model2.get_output() F2 = function([X2], Y2) y2 = F2(x) assert_allclose(y2, y[1])
class TestOrthoRNN(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestOrthoRNN, self).__init__(*args, **kwargs) self.input_dim = 2 self.state_dim = 2 self.model = Recursive(return_sequences=True) self.model.add_input('input', ndim=3) # Input is 3D tensor self.model.add_state('h', dim=self.state_dim) self.model.add_node(Dense(self.input_dim, self.state_dim, init='one'), name='i2h', inputs=['input', ]) self.model.add_node(Dense(self.state_dim, self.state_dim, init='orthogonal'), name='h2h', inputs=['h', ]) self.model.add_node(Lambda(lambda x: x), name='rec', inputs=['i2h', 'h2h'], merge_mode='sum', return_state='h', create_output=True) self.model2 = Sequential() self.model2.add(SimpleRNN(input_dim=self.input_dim, activation='linear', inner_init='one', output_dim=self.state_dim, init='one', return_sequences=True)) U = self.model.nodes['h2h'].W.get_value() self.model2.layers[0].U.set_value(U) def test_step(self): XX = T.matrix() HH = T.matrix() A = self.model._step(XX, HH) F = function([XX, HH], A, on_unused_input='warn') x = np.ones((1, 2)) h = np.ones((1, 2)) y = F(x, h) assert(y[-1].shape == (1, 2)) def test_get_get_output(self): X = self.model.get_input() Y = self.model._get_output() F = function([X], Y, allow_input_downcast=True) x = np.ones((3, 5, self.input_dim)) y = F(x) print y X2 = self.model2.get_input() Y2 = self.model2.get_output() F2 = function([X2], Y2) y2 = F2(x) assert_allclose(y2, y[-1])
class TestRecursive(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestRecursive, self).__init__(*args, **kwargs) self.input_dim = 2 self.state_dim = 2 self.model = Recursive(return_sequences=True) self.model.add_input('input', ndim=3) # Input is 3D tensor self.model.add_state('h', dim=self.state_dim) self.model.add_node(Dense(self.input_dim + self.state_dim, self.state_dim, init='one'), name='rec', inputs=['input', 'h'], return_state='h') self.model.add_node(Activation('linear'), name='out', input='rec', create_output=True) self.model2 = Sequential() self.model2.add(SimpleRNN(input_dim=self.input_dim, activation='linear', inner_init='one', output_dim=self.state_dim, init='one', return_sequences=True)) def test_step(self): XX = T.matrix() HH = T.matrix() A = self.model._step(XX, HH) F = function([XX, HH], A, allow_input_downcast=True) x = np.ones((1, 2)) h = np.ones((1, 2)) y = F(x, h) r = np.asarray([[4., 4.]]) assert_allclose([r, r], y) def test_get_get_output(self): X = self.model.get_input() Y = self.model._get_output() F = function([X], Y, allow_input_downcast=True) x = np.ones((3, 5, self.input_dim)).astype(floatX) y = F(x) print y X2 = self.model2.get_input() Y2 = self.model2.get_output() F2 = function([X2], Y2) y2 = F2(x) assert_allclose(y2, y[1])
def __init__(self): left = Sequential() left.add(Dense(784, 50)) left.add(Activation('relu')) model = Sequential() model.add(Merge([left, left], mode='sum')) model.add(Dense(50, 10)) model.add(Activation('softmax')) pdb.set_trace() model = Sequential() left = Sequential() num_kernel = 32 l1_penalty = 0.0001 b_mode = 'full' left.add( Convolution2D(num_kernel, 3, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) left.add( Convolution2D(num_kernel, num_kernel, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) left.add(LeakyReLU(0.1)) #left.add(Activation('relu')) left.add(MaxPooling2D(poolsize=(2, 2))) #left.add(Convolution2D(num_kernel, 3, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) #left.add(Convolution2D(num_kernel, num_kernel, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) #left.add(LeakyReLU(0.1)) ##left.add(Activation('relu')) #left.add(MaxPooling2D(poolsize=(2, 2))) model.add(Merge([left, left], mode='sum')) pdb.set_trace() self.f = theano.function(model.get_input(), model.get_output())
model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(Flatten()) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(1000, activation='softmax')) # load the weights of the VGG16 networks # (trained on ImageNet, won the ILSVRC competition in 2014) # note: when there is a complete match between your model definition # and your weight savefile, you can simply call model.load_weights(filename) assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).' model.load_weights(weights_path) output = model.get_output() f_class_output = K.function([input_image], output) x = preprocess_image(base_image_path) predicted_classes = f_class_output([x.reshape(1, 3, img_width, img_height)])[0] print('Class prediction {}'.format(np.argmax(predicted_classes)))
detector.add(Dropout(.3)) detector.add(Dense(1)) # 1: Yes, it belongs to S, 0: fake! detector.add(Activation('sigmoid')) # Fully Connected model sampler = Sequential() sampler.add(Dense(dim, input_dim=dim)) sampler.add(lrelu()) sampler.add(Dense(dim)) sampler.add(lrelu()) sampler.add(Dense(mnist_dim)) sampler.add(Activation('sigmoid')) # This is G itself!!! sample_fake = theano.function([sampler.get_input()], sampler.get_output()) # We add the detector G on top, but it won't be adapted with this cost function. # But here is a dirty hack: Theano shared variables on the GPU are the same for # `detector` and `detector_no_grad`, so, when we adapt `detector` the values of # `detector_no_grad` will be updated as well. But this only happens following the # correct gradients. # Don't you love pointers? Aliasing can be our friend sometimes. detector.trainable = False sampler.add(detector) opt_g = Adam(lr=.001) # I got better results when # detector's learning rate is faster sampler.compile(loss='binary_crossentropy', optimizer=opt_g) # debug
userModel.add(Activation('tanh')) userModel.add(Dropout(0.4)) userModel.add(Dense(700, 500)) userModel.add(Activation('tanh')) itemModel = Sequential() itemModel.add(TimeDistributedDense(itemfea, 1000)) itemModel.add(Activation('tanh')) itemModel.add(Dropout(0.4)) itemModel.add(TimeDistributedDense(1000, 500)) itemModel.add(Activation('tanh')) ##itemModel.add(Reshape(4)) ##itemModel.add(Dense(4, 2)) itm = itemModel.get_input(train=False) usr = userModel.get_input(train=False) itemrep = itemModel.get_output(train=False) userrep = userModel.get_output(train=False) model = Sequential() model.add(Cosine([userModel, itemModel])) #should output 2 values #model.add(TimeDistributedDense(300, 1)) ##model.add(Activation('normalization')) model.add(Reshape(2)) y_score = model.get_output(train=False) x_test = model.get_input(train=False) model.add(Activation('softmax')) print("Complie model...") model.compile(loss='categorical_crossentropy', optimizer='adam') print("Complie outs...") outv1 = theano.function([usr], userrep, allow_input_downcast=True, mode=None) outv2 = theano.function([itm], itemrep, allow_input_downcast=True, mode=None) print("load W...")
def main(MODEL_FILE): print "Loading hdf5's..." test_dict = io.load('./data/test_dict_IPConv_ntuple_'+ RUN_NAME +'.h5') train_dict = io.load('./data/train_dict_IPConv_ntuple_'+ RUN_NAME +'.h5') X_train = train_dict['X'] y_train = train_dict['y'] X_test = test_dict['X'] y_test = test_dict['y'] n_features = X_test.shape[2] # this is a df ip3d = test_dict['ip3d'] print 'Building model...' if (MODEL_FILE == 'CRNN'): graph = build_graph(n_features) model = Sequential() model.add(graph) # remove Maxout for tensorflow model.add(MaxoutDense(64, 5, input_shape=graph.nodes['dropout'].output_shape[1:])) model.add(Dense(64)) elif (MODEL_FILE == 'RNN'): model = Sequential() model.add(Masking(mask_value=-999, input_shape=(N_TRACKS, n_features))) model.add(GRU(25))#, input_shape=(N_TRACKS, n_features))) #GRU model.add(Dropout(0.2)) #0.2 # remove Maxout for tensorflow model.add(MaxoutDense(64, 5)) #, input_shape=graph.nodes['dropout'].output_shape[1:])) model.add(Dense(64)) model.add(Dropout(0.4)) model.add(Highway(activation = 'relu')) model.add(Dropout(0.3)) model.add(Dense(4)) model.add(Activation('softmax')) print 'Compiling model...' model.compile('adam', 'categorical_crossentropy') model.summary() print 'Training:' try: model.fit(X_train, y_train, batch_size=512, callbacks = [ EarlyStopping(verbose=True, patience=20, monitor='val_loss'), ModelCheckpoint(MODEL_FILE + RUN_NAME +'-progress', monitor='val_loss', verbose=True, save_best_only=True) ], nb_epoch=100, validation_split = 0.2, show_accuracy=True) except KeyboardInterrupt: print 'Training ended early.' # -- load in best network model.load_weights(MODEL_FILE + RUN_NAME +'-progress') if (SAVE_PROTOBUF): print 'Saving protobuf' # write out to a new directory called models # the actual graph file is graph.pb # the graph def is in the global session import tensorflow as tf import keras.backend.tensorflow_backend as tfbe sess = tfbe._SESSION saver = tf.train.Saver() tf.train.write_graph(sess.graph_def, 'models/', 'graph.pb', as_text=False) save_path = saver.save(sess, "./model-weights.ckpt") print "Model saved in file: %s" % save_path print saver.as_saver_def().filename_tensor_name print saver.as_saver_def().restore_op_name print model.get_output() print 'Saving weights...' model.save_weights('./weights/ip3d-replacement_' + MODEL_FILE + RUN_NAME +'.h5', overwrite=True) json_string = model.to_json() open(MODEL_FILE + RUN_NAME +'.json', 'w').write(json_string) print 'Testing...' yhat = model.predict(X_test, verbose = True, batch_size = 512) io.save('yhat'+ RUN_NAME +'.h5', yhat) print 'Plotting ROC...' fg = plot_ROC(y_test, yhat, ip3d, MODEL_FILE) #plt.show() fg.savefig('./plots/roc' + MODEL_FILE + RUN_NAME +'.pdf')
def build(self): inception_1 = Sequential() inception_2 = Sequential() model_1 = Sequential() model_3 = Sequential() model_5 = Sequential() model_1_ = Sequential() model_3_ = Sequential() model_5_ = Sequential() model_1.add( Convolution2D(self.nb_filter, 1, 1, dim_ordering='tf', activation='relu', border_mode='same', input_shape=(self.height, self.width, 6))) model_5.add( Convolution2D(self.nb_filter, 5, 5, dim_ordering='tf', activation='relu', border_mode='same', input_shape=(self.height, self.width, 6))) model_3.add( Convolution2D(self.nb_filter, 3, 3, dim_ordering='tf', activation='relu', border_mode='same', input_shape=(self.height, self.width, 6))) inception_1 = Merge([model_1, model_3, model_5], mode='concat') inception_2_input_shape = (inception_1.output_shape[1], inception_1.output_shape[2], inception_1.output_shape[3]) test1 = Convolution2D(self.nb_filter, 1, 1, dim_ordering='tf', activation='relu', border_mode='same', input_shape=inception_2_input_shape) test2 = Convolution2D(self.nb_filter, 3, 3, dim_ordering='tf', activation='relu', border_mode='same', input_shape=inception_2_input_shape) test3 = Convolution2D(self.nb_filter, 5, 5, dim_ordering='tf', activation='relu', border_mode='same', input_shape=inception_2_input_shape) test1.input = inception_1.get_output() test2.input = inception_1.get_output() test3.input = inception_1.get_output() inception_2 = Merge([test1, test2, test3], mode='concat') model = Sequential([inception_1]) model.add(inception_2) model.add(BatchNormalization(axis=-1, gamma_init='glorot_normal')) model.add( Convolution2D(128, 1, 1, dim_ordering='tf', activation='relu', border_mode='same')) model.add( Convolution2D(128, 1, 1, dim_ordering='tf', activation='relu', border_mode='same')) model.add( Convolution2D(256, 1, 1, dim_ordering='tf', activation='relu', border_mode='same')) model.add(BatchNormalization(axis=-1, gamma_init='glorot_normal')) model.add(Dropout(0.5)) model.add( Convolution2D(1, 3, 3, dim_ordering='tf', activation='linear', border_mode='same')) model.add( Convolution2D(1, 3, 3, dim_ordering='tf', activation='linear', border_mode='same')) model.add(Reshape(dims=(self.height, self.width))) self.model = model self.result_func = K.function(self.model.layers[0].input, self.model.layers[-1].get_output())
probs = model.predict_proba(X_test, batch_size=batch_size) predictions = collapse_results(probs) r, p, f1, cutoff = find_cutoff(y_reg_test, predictions) print("recall", r, "precision", p, "f1", f1, "cutoff", cutoff) return f1 while True: iterations += 1 accuracy = train(1) if accuracy < last_accuracy: best = prev_weights decreases +=1 else: decreases = 0 if decreases >= 10 and iterations > 30: print("Val accruacy decreased from %f to %f. Stopping" % (last_accuracy, accuracy)) break last_accuracy = accuracy prev_weights = model.get_output() print("at: " + str(datetime.datetime.now())) print("Best F1:", best) # Causer: recall 0.746835443038 precision 0.670454545455 f1 0.706586826347 - 32 embedding, lstm, sigmoid, adam
def main(MODEL_FILE): test_dict = io.load('./data/test_dict_IPConv.h5') train_dict = io.load('./data/train_dict_IPConv.h5') X_train = train_dict['X'] y_train = train_dict['y'] n_features = X_train.shape[2] X_test = test_dict['X'] y_test = test_dict['y'] # this is a df ip3d = test_dict['ip3d'] print 'Building model...' if (MODEL_FILE == 'CRNN'): graph = build_graph(n_features) model = Sequential() model.add(graph) # removing because of tensorflow #model.add(MaxoutDense(64, 5, input_shape=graph.nodes['dropout'].output_shape[1:])) model.add(Dense(64)) elif (MODEL_FILE == 'RNN'): model = Sequential() model.add(GRU(25, input_shape=(N_TRACKS, n_features))) #GRU model.add(Dropout(0.2)) # removing because of tensorflow #model.add(MaxoutDense(64, 5)) #, input_shape=graph.nodes['dropout'].output_shape[1:])) model.add(Dense(64)) model.add(Dropout(0.4)) model.add(Highway(activation = 'relu')) model.add(Dropout(0.3)) model.add(Dense(4)) model.add(Activation('softmax')) print 'Compiling model...' model.compile('adam', 'categorical_crossentropy') model.summary() print 'Training:' try: model.fit(X_train, y_train, batch_size=512, callbacks = [ EarlyStopping(verbose=True, patience=20, monitor='val_loss'), ModelCheckpoint(MODEL_FILE + '-progress', monitor='val_loss', verbose=True, save_best_only=True) ], nb_epoch=2, validation_split = 0.2, show_accuracy=True) except KeyboardInterrupt: print 'Training ended early.' # -- load in best network model.load_weights(MODEL_FILE + '-progress') print 'Saving protobuf' # write out to a new directory called models # the actual graph file is graph.pb # the graph def is in the global session import tensorflow as tf import keras.backend.tensorflow_backend as tfbe sess = tfbe._SESSION saver = tf.train.Saver() tf.train.write_graph(sess.graph_def, 'models/', 'graph.pb', as_text=False) save_path = saver.save(sess, "./model-weights.ckpt") print "Model saved in file: %s" % save_path print saver.as_saver_def().filename_tensor_name print saver.as_saver_def().restore_op_name print model.get_output() print 'Saving weights...' model.save_weights('./weights/ip3d-replacement_' + MODEL_FILE + '.h5', overwrite=True) json_string = model.to_json() open(MODEL_FILE + '.json', 'w').write(json_string) print 'Testing...' yhat = model.predict(X_test, verbose = True, batch_size = 512) print 'Plotting ROC...' fg = plot_ROC(y_test, yhat, ip3d, MODEL_FILE) #plt.show() fg.savefig('./plots/roc_' + MODEL_FILE + '.pdf')
userModel.add(Activation('tanh')) itemModel = Sequential() itemModel.add(TimeDistributedDense(943, 500)) itemModel.add(Activation('tanh')) itemModel.add(Dropout(0.4)) itemModel.add(TimeDistributedDense(500, 500)) itemModel.add(Activation('tanh')) ##itemModel.add(Reshape(4)) ##itemModel.add(Dense(4, 2)) model=Sequential() model.add(MaxDot([userModel,itemModel])) #should output 2 values #model.add(TimeDistributedDense(300, 1)) ##model.add(Activation('normalization')) model.add(Reshape(2)) y_score= model.get_output(train=False) x_test=model.get_input(train=False) model.add(Activation('softmax')) ##model.add(Merge([userModel, itemModel], mode='sum')) print('done model construction') model.compile(loss='categorical_crossentropy', optimizer='Adadelta') print('done complie') scoring= theano.function(x_test,y_score,allow_input_downcast=True, mode=None) history = model.fit([user ,Items] ,y_train, nb_epoch=100, batch_size=2048, verbose=2, show_accuracy=True) #history = model.train_on_batch([user ,Items] ,y_train,accuracy=True)# nb_epoch=10, batch_size=1024, verbose=2, show_accuracy=True) print('done training') user_test ,Items_test, y_test = load_dataset(r"C:\Users\t-alie\Downloads\movieLens_1M\movielens.userstest100k.centered",r"C:\Users\t-alie\Downloads\movieLens_1M\movielens.itemstest100k",r"C:\Users\t-alie\Downloads\movieLens_1M\movielens.itemstest100k.fakeneg",50781) y_p=model.custom_predict([user_test,Items_test],scoring)
class TestOrthoRNN(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestOrthoRNN, self).__init__(*args, **kwargs) self.input_dim = 2 self.state_dim = 2 self.model = Recursive(return_sequences=True) self.model.add_input('input', ndim=3) # Input is 3D tensor self.model.add_state('h', dim=self.state_dim) self.model.add_node(Dense(self.input_dim, self.state_dim, init='one'), name='i2h', inputs=[ 'input', ]) self.model.add_node(Dense(self.state_dim, self.state_dim, init='orthogonal'), name='h2h', inputs=[ 'h', ]) self.model.add_node(Lambda(lambda x: x), name='rec', inputs=['i2h', 'h2h'], merge_mode='sum', return_state='h', create_output=True) self.model2 = Sequential() self.model2.add( SimpleRNN(input_dim=self.input_dim, activation='linear', inner_init='one', output_dim=self.state_dim, init='one', return_sequences=True)) U = self.model.nodes['h2h'].W.get_value() self.model2.layers[0].U.set_value(U) def test_step(self): XX = T.matrix() HH = T.matrix() A = self.model._step(XX, HH) F = function([XX, HH], A, on_unused_input='warn') x = np.ones((1, 2)) h = np.ones((1, 2)) y = F(x, h) assert (y[-1].shape == (1, 2)) def test_get_get_output(self): X = self.model.get_input() Y = self.model._get_output() F = function([X], Y, allow_input_downcast=True) x = np.ones((3, 5, self.input_dim)) y = F(x) print y X2 = self.model2.get_input() Y2 = self.model2.get_output() F2 = function([X2], Y2) y2 = F2(x) assert_allclose(y2, y[-1])