def test_clone_sequential_model(): seq = Sequential() seq.add(Dense(8, input_shape=(3,))) seq.compile(optimizer='sgd', loss='mse') clone = clone_model(seq) clone.compile(optimizer='sgd', loss='mse') ins = np.random.random((4, 3)) y_pred_seq = seq.predict_on_batch(ins) y_pred_clone = clone.predict_on_batch(ins) assert y_pred_seq.shape == y_pred_clone.shape assert_allclose(y_pred_seq, y_pred_clone)
def opt_nn(): from keras.models import Sequential from keras.layers import Dense, Activation, Flatten, Dropout from keras.layers.convolutional import Convolution2D, MaxPooling2D from keras.layers import Embedding from keras.layers.recurrent import LSTM #Code for Convolutional Neural Network (it doesn't work) # First we have to initialize the neural network using Sequential() #cnn = Sequential() # process the data to fit in a keras CNN properly # input data needs to be (N, C, X) - shaped where # N - number of samples # C - number of channels per sample # X - sample size #cnn.add(Convolution1D(64, 4, # border_mode="same", # activation="relu", # input_shape=(1, 4))) #cnn.add(Convolution1D(64, 2, border_mode="same")) #cnn.add(Convolution1D(64, 1, border_mode="same")) #cnn.add(Flatten()) #cnn.add(Dense(256, activation="relu")) #cnn.add(Dropout(0.5)) #cnn.add(Dense(32, activation="relu")) #cnn.add(Dense(1, activation="softmax")) #cnn.compile(loss="mse", optimizer="rmsprop") #Code for LTSM RNN model = Sequential() model.add(LSTM(64,input_dim=4, return_sequences=False, activation='tanh')) model.add(Dense(128)) model.add(Dense(64, init='normal', activation='tanh')) model.add(Dense(4, init='normal', activation='tanh')) model.add(Dense(1, init='normal')) model.compile(loss='mse', optimizer='rmsprop') colors = np.vstack([quasar_table['PSFMAG_%d' % f]-quasar_table['PSFMAG_%d' % (f+1)] for f in range(0,4)]).T color_train = colors[::5] color_test = colors[::18] batch_size = len(z_train) model.fit(color_train.reshape(-1,1,4), z_train, batch_size=batch_size, nb_epoch=300, verbose=0, validation_split=0.5) predicted_output = model.predict_on_batch(color_test.reshape(-1,1,4)) rms_lstm = np.mean(np.sqrt((z_test-predicted_output)**2)) plt.scatter(z_test,predicted_output, color='k', s=0.1) plt.plot([-0.1, 6], [-0.1, 6], ':k') plt.text(0.04, 5, "rms = %.3f" % (rms_lstm)) plt.xlabel('$z_{true}$') plt.ylabel('$z_{fit}$')
class LSTM_RNN: def __init__(self, look_back, dropout_probability = 0.2, init ='he_uniform', loss='mse', optimizer='rmsprop'): self.rnn = Sequential() self.look_back = look_back self.rnn.add(LSTM(10, stateful = True, batch_input_shape=(1, 1, 1), init=init)) self.rnn.add(Dropout(dropout_probability)) self.rnn.add(Dense(1, init=init)) self.rnn.compile(loss=loss, optimizer=optimizer) def batch_train_test(self, trainX, trainY, testX, testY, nb_epoch=150): print('Training LSTM-RNN...') for epoch in range(nb_epoch): print('Epoch '+ str(epoch+1) +'/{}'.format(nb_epoch)) training_losses = [] testing_losses = [] for i in range(len(trainX)): y_actual = trainY[i] for j in range(self.look_back): training_loss = self.rnn.train_on_batch(np.expand_dims(np.expand_dims(trainX[i][j], axis=1), axis=1), np.array([y_actual])) training_losses.append(training_loss) self.rnn.reset_states() print('Mean training loss = {}'.format(np.mean(training_losses))) mean_testing_loss = [] for i in range(len(testX)): for j in range(self.look_back): testing_loss = self.rnn.test_on_batch(np.expand_dims(np.expand_dims(testX[i][j], axis=1), axis=1), np.array([testY[i]])) testing_losses.append(testing_loss) self.rnn.reset_states() for j in range(self.look_back): y_pred = self.rnn.predict_on_batch(np.expand_dims(np.expand_dims(testX[i][j], axis=1), axis=1)) self.rnn.reset_states() mean_testing_loss = np.mean(testing_losses) print('Mean testing loss = {}'.format(mean_testing_loss)) return mean_testing_loss
hist = forecaster.fit(xTrain, yTrain, epochs=100, shuffle=True) # epoch_loss_forecaster = hist.history['loss'] # epoch_accuracy_forecaster = [x*100 for x in hist.history['acc']] # epoch_num = [x for x in range(0,len(epoch_loss_forecaster))] # # plt.plot(epoch_num,epoch_loss_forecaster) # plt.plot(epoch_num, epoch_accuracy_forecaster) # plt.xlabel("Generation Number") # plt.ylabel("Loss (Mean Squared Error)") # plt.title("Forecaster Loss") # plt.show() l2Norms = [] for i in range(len(xTest)): l2NormRatio = LA.norm( forecaster.predict_on_batch(xTest[i:i + 1]) - yTest[i:i + 1]) / LA.norm(yTest[i:i + 1]) l2Norms.append(l2NormRatio) # # with open('timeLagTestResults.csv', 'a') as f: # writer = csv.writer(f) # writer.writerow(l2Norms) print np.sum(l2Norms) / len(l2Norms) with open('quickDel.csv', 'a') as f: writer = csv.writer(f) writer.writerow(l2Norms)
layer_a1 = LSTM(5, input_shape=(30, word_vector), return_sequences=False) layer_a2 = beShared_LSTM(layer_a1, input_shape=(30, word_vector)) model_1.add(layer_a1) model_2.add(layer_a2) model_1.compile(loss='categorical_crossentropy', optimizer='rmsprop') model_2.compile(loss='categorical_crossentropy', optimizer='rmsprop') ''' Starting testing ''' import cPickle as pickle print "Starting ..." ej_case = 'EJ_case.pkl' question_input, ans1_input, ans2_input, ans3_input, ans4_input, ans5_input, image_input, solution = pickle.load( open(ej_case, 'rb')) print ans1_input.shape print solution.shape for n in range(10): print "model_2", model_2.predict_on_batch(ans1_input) print "model_1", model_1.predict_on_batch(ans1_input) print model_1.train_on_batch(ans1_input, solution) print model_2.predict_on_batch(ans1_input) print model_1.predict_on_batch(ans1_input)
model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dropout(0.2)) model.add(Dense(1024, activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add(Dense(512, activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add(Dense(num_classes, activation='softmax')) # Compile model epochs = 1 lrate = 0.01 decay = lrate/epochs sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) print(model.summary()) tensorboard = TensorBoard(log_dir='logs/final', histogram_freq=0, write_graph=True, write_images=False) # Fit the model model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=epochs, batch_size=128,callbacks=[tensorboard]) # Final evaluation of the model scores = model.evaluate(X_test, y_test, verbose=0) print("Accuracy: %.2f%%" % (scores[1]*100)) ResultsList = model.predict_on_batch(X_test[0:4]) numpy.argmax(ResultsList[0]) print(ResultsList) print(y_test[0:4])
test_batches = list( BatchIterator(X_test, Y_train, batch_size, IMAGE_SIZE) ) # we only use X_test and Y_train, Y_train is a dummy arg progbar = generic_utils.Progbar(len(X_test)) # add progress bar since it takes a while preds = [] for X_batch, Y_batch in test_batches: # X_test:filenames, A_batch: annotation X_batch_image = [] for image_path in X_batch: # load pre-processed test images from filenames processed_img_arr = cv2.imread(DATA_DIR_PATH + "/" + image_path) X_batch_image.append(processed_img_arr.reshape(3, IMAGE_SIZE, IMAGE_SIZE)) # convert to ndarray X_batch_image = np.array(X_batch_image) X_batch_image = X_batch_image.astype("float32") X_batch_image /= 255 preds_batch = model.predict_on_batch(X_batch_image) progbar.add(batch_size, values=[]) preds += list(preds_batch) preds = np.array(preds) print("Saving prediction result...") with open("bin/head_%dx%d_noda_preds.bin" % (IMAGE_SIZE, IMAGE_SIZE), "w") as fid: pickle.dump(preds, fid) # create a submission file export_to_csv(preds, filenames, "data/head_%dx%d_noda.csv" % (IMAGE_SIZE, IMAGE_SIZE)) # ============================================ # allocate memory for all images first [OLD] # ============================================ else:
class CnnLstm(object): """docstring for cnn-lstm""" def __init__(self, conf): self.vs = conf["vocab_size"] self.ml = conf["maxlen"] self.bs = conf["batch_size"] self.ed = conf["embedding_dims"] self.nf = conf["nb_filter"] self.fl = conf["filter_length"] self.hs = conf["hidden_size"] self.ep = conf["nb_epoch"] self.pl = conf["pool_length"] self.sm = conf.get("save_model", "models/default.cnnlstm") self.lm = conf.get("load_model", "models/default.cnnlstm") self.do = conf.get("dropout",0.2) self.model = Sequential() def build_net(self): model_1 = Sequential() model_2 = Sequential() model_1.add(Embedding(self.vs, self.ed, input_length=self.ml, dropout=self.do)) model_1.add(Convolution1D(nb_filter=self.nf, filter_length=self.fl, border_mode='valid', activation='relu', subsample_length=1)) model_1.add(Lambda(max_1d, output_shape=(self.nf, ))) model_2.add(Embedding(self.vs, self.ed, input_length=self.ml, dropout=self.do)) model_2.add(Convolution1D(nb_filter=self.nf, filter_length=self.fl, border_mode='valid', activation='relu', subsample_length=1)) model_2.add(Lambda(max_1d, output_shape=(self.nf, ))) self.model.add(Merge([model_1, model_2],mode='concat')) print self.model.output_shape self.model.add(Reshape((2, self.nf), input_shape=(self.nf*2,))) self.model.add(LSTM(self.hs)) self.model.add(Dense(self.hs)) self.model.add(Dropout(self.do)) self.model.add(Dense(12)) self.model.add(Activation("softmax")) self.model.compile(loss='categorical_crossentropy', optimizer='adam') print "Network compile completed..." def save(self): f = open(self.sm, "w") f.write(self.model.to_json()) f.close() self.model.save_weights(self.sm+".weights", overwrite=True) def load(self): f = open(self.lm, "r") self.model = model_from_json(f.read()) self.model.load_weights(self.sm+".weights") self.model.compile(loss='categorical_crossentropy', optimizer='adam') print "Network compile completed..." def train(self, x, y, vx, vy, vvx=None, vvy=None): print "Begin to train ... training set: {0}, validation set: {1}".format(x[0].shape[0], vx[0].shape[0]) ep = 0 max_accuracy = 0 while ep < self.ep: loss = 0 cnt = 0 accuracy = 0.0 v_accuracy = 0.0 v_im_accuracy = 0.0 num_of_batch = int(len(y)/self.bs) idx_move = num_of_batch / 60 for i in xrange(0, len(y), self.bs): x_ = [x[0][i:i+self.bs], x[1][i:i+self.bs]] y_ = y[i:i+self.bs] loss_ = self.model.train_on_batch(x_, y_) pred_ = self.model.predict_on_batch(x_) acc_ = 0.0 for j in xrange(len(pred_)): max_p = np.argmax(pred_[j]) correct = 0 acc_ += (y_[j][max_p] > 0) acc_ /= len(pred_) accuracy += acc_ # print acc_ loss += loss_ cnt += 1 #sys.stdout.flush() if cnt % idx_move == 0: sys.stderr.write("=>\b") sys.stderr.flush() print ">" if vvx != None: vv_pred = self.model.predict_on_batch(vvx) for j in xrange(len(vv_pred)): max_vp = np.argmax(vv_pred[j]) v_im_accuracy += (vvy[j][max_vp] > 0) v_im_accuracy /= len(vv_pred) v_pred = self.model.predict_on_batch(vx) for j in xrange(len(v_pred)): max_p = np.argmax(v_pred[j]) v_accuracy += (vy[j][max_p] > 0) v_accuracy /= len(v_pred) if v_im_accuracy > max_accuracy: print "Model imporved on validation set, save model ..." self.save() max_accuracy = v_accuracy ep += 1 print "Epoch {0}, training loss {1}, train-accuracy {2}, valid-accuracy {3}, valid_im-accuracy {4}".format( ep, loss / cnt, accuracy / cnt, v_accuracy, v_im_accuracy) sys.stdout.flush()
# calculate heuristic metric for data metrics = np.zeros((0, 11)) for board in iter(data): metrics = np.vstack((metrics, get_metrics(board))) # maybe pass it to generative model too if np.random.random() > 0.75: gen_model.fit(metrics[:, 1:], metrics[:, 0], epochs=16, batch_size=32, verbose=0) # calculate probilistic (noisy) labels probabilistic = gen_model.predict_on_batch(metrics[:, 1:]) # calculate confidence score for each probabilistic label using error between probabilistic and weak label confidence = 1 / (1 + np.absolute(metrics[:, 0] - probabilistic[:, 0])) # fit labels to {-1, 1} probabilistic = np.sign(probabilistic) # concat board position data with heurstic metric and pass for training - removed # data = np.hstack((data, metrics[:, 1:])) disc_model.fit(data, probabilistic, epochs=32, batch_size=64, sample_weight=confidence, verbose=0)
def generate_lstm_gmm(seq, maxlen=1, bs=500, ep=2, output_iterations=10, num_mixture_components=3): # seq is a single sample, in the format (timesteps, features) ! # TODO: expand code to support multiple samples, fed into model together as a batch # Cut the timeseries data (variable name 'seq') into semi-redundant sequence chunks of maxlen X = [] y = [] for i in range(0, len(seq) - maxlen): X.append(seq[i:i+maxlen]) y.append(seq[i+maxlen]) dim = len((X[0][0])) print("sequence chunks:", len(X)) print("chunk width:", len(X[0])) print("vector dimension:", dim) print("number of mixture components:", num_mixture_components) print("batch size:", bs) X = np.array(X) y = np.array(y) # build the model: 2 stacked LSTM print('Build model...') model = Sequential() model.reset_states() model.add(LSTM((dim+2) * num_mixture_components, return_sequences=False, input_shape=(maxlen, dim))) model.add(Dense((dim+2) * num_mixture_components)) model.add(GMMActivation(num_mixture_components)) model.compile(loss=gmm_loss, optimizer=RMSprop(lr=0.001)) # Train the model model.fit(X, y, batch_size=bs, nb_epoch=ep) # Generate timeseries x_seed = X[len(X)-1] #choose final in-sample data point to initialize model x_array = [] x_array.append(x_seed) x = np.array(x_array) predicted = [] for i in range(output_iterations): pred_parameters = model.predict_on_batch(x)[0] means = pred_parameters[:num_mixture_components * dim] sds = pred_parameters[(num_mixture_components * dim):(num_mixture_components * (dim+1))] weights = pred_parameters[(num_mixture_components * (dim + 1)):] print(means) print(sds) print(weights) means = means.reshape(num_mixture_components, dim) sds = sds[:, np.newaxis] weights = weights[:, np.newaxis] pred = weights * np.random.normal(means, sds) pred = np.sum(pred, axis=0) predicted.append(pred) return predicted
gen_loss = [] for e in range(epochs + 1): for i in range(len(train_x) // batch_size): # Train discriminator model weights dis_model.trainable = True # real samples batch_x = train_x[i * batch_size:(i + 1) * batch_size] dis_loss_real = dis_model.train_on_batch(x=batch_x, y=real * (1 - smooth)) # fake Samples z = np.random.normal(loc=0, scale=1, size=(batch_size, dim)) fake_x = gen_model.predict_on_batch(z) dis_loss_fake = dis_model.train_on_batch(x=fake_x, y=fake) # discriminator_model loss dis_loss_batch = 0.5 * (dis_loss_real[0] + dis_loss_fake[0]) # Train generator_model weights dis_model.trainable = False gen_loss_batch = dis_gen.train_on_batch(x=z, y=real) print('epoch = {}/{}, batch = {}/{}, dis_loss={}, gen_loss={}'.format( e + 1, epochs, i, len(train_x) // batch_size, dis_loss_batch, gen_loss_batch[0]), 100 * ' ', end='\r')
model.add(layers.Activation('softmax')) #optimizer=keras.optimizers.Adam(lr=0.001) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() model.fit(X, Y, epochs=1, batch_size=64, validation_split=0.05, verbose=2) model.save(dir_path + '/log/qinggan.save.' + time.strftime("%Y%m%d", time.localtime())) l = [ '商品很好,样子很不错', '很满意,送的也很快', '物流太慢了', '垃圾,非常差', '鼠标我觉得眼睛\\\\ud83d\\\\udc40可以发个光,中间的白点有点我涂黑了', ] test_x = [] for j in l: index_list = tools.string2index(j)[0:input_data_X_size] #给末尾不足30位的补0,让输入x都是30 size x = np.pad(np.array(index_list), (0, (input_data_X_size - len(index_list))), 'constant') test_x.append(x) test_x = np.array(test_x) res = model.predict_on_batch(test_x) print(res)
# create a GAN, a generator and a discriminator gan, generator, discriminator = make_simple_GAN(sample_size, g_hidden_size, d_hidden_size, leaky_alpha, g_learning_rate, d_learning_rate) losses = [] for e in range(epochs): for i in range(len(X_train_real) // batch_size): # real MNIST digit images X_batch_real = X_train_real[i * batch_size:(i + 1) * batch_size] # latent samples and the generated digit images latent_samples = make_latent_samples(batch_size, sample_size) X_batch_fake = generator.predict_on_batch(latent_samples) # train the discriminator to detect real and fake images make_trainable(discriminator, True) discriminator.train_on_batch(X_batch_real, y_train_real * (1 - smooth)) discriminator.train_on_batch(X_batch_fake, y_train_fake) # train the generator via GAN make_trainable(discriminator, False) gan.train_on_batch(latent_samples, y_train_real) # evaluate X_eval_real = X_test_real[np.random.choice(len(X_test_real), eval_size, replace=False)]
def stuff(alpha, nums_hidden, seed, dim, nums_epochs): # Config model s = {'dataset':'atis.pkl', 'lr':alpha, 'nhidden': nums_hidden, # number of hidden units 'batch': seed, 'emb_dimension': dim, # dimension of word embedding 'nepochs': nums_epochs} # Make result folder, contain: model file, result.json, output file for validation and train, also save config.json again folder = os.path.basename("Result").split('.')[0] if not os.path.exists(folder): os.mkdir(folder) write_JSON(s,folder + '/config.json') # load the dataset train_set, test_set, dic = load_atis(s['dataset']) # Convert for index # Vocabulary of meaningful words and labels are covered in dic data of Atis datset idx2label = dict((k,v) for v,k in dic['labels2idx'].items()) idx2word = dict((k,v) for v,k in dic['words2idx'].items()) # Seperate data for hold-out: train set, validation set #words2idx, tables2idx and labels2idx (use only vocabs and labels. Tables will list all meaning of words_which not neccessary) train_words, train_tables, train_labels = train_set valid_words = train_words[0:499] valid_tables = train_tables[0:499] valid_labels = train_labels[0:499] train_words = train_words[500:len(train_words)] train_tables = train_tables[500:len(train_tables)] train_labels = train_labels[500:len(train_labels)] # Print some info #print('Train set:',str(len(train_words)), 'sentences') #print('Validation set:',str(len(valid_words)),'sentences') # Some para use in 'for loop' vocsize = len(dic['words2idx']) nclasses = len(dic['labels2idx']) nsentences = len(train_words) #print('Nums of vocabulary get from words of each sentence: ', vocsize) #print('Nums of slot: ', nclasses) #print('Nums of sentence use for training: ', nsentences) # instanciate the model (for randomize duel to bath size_ optimization convergence) np.random.seed(s['batch']) random.seed(s['batch']) #Making model model = Sequential() #Init model.add(Embedding(vocsize, s['emb_dimension'])) # Word Embedding model.add(SimpleRNN(s['nhidden'], activation='sigmoid', return_sequences=True)) # Recurrent use Sigmoid Activation model.add(TimeDistributed(Dense(output_dim=nclasses))) # For making Dense Layer (Context Layer) keep updating model.add(Activation("softmax")) # Softmax activation for classification adam = Adam(lr=s['lr'], beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) # Adam optimizer (some hyperparameter will be locked) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) # Lost funct: Cross entropy # Train for e in range(s['nepochs']): # shuffle shuffle([train_words, train_tables, train_labels], s['batch']) s['ce'] = e for i in range(nsentences): X = np.asarray([train_words[i]]) Y = to_categorical(np.asarray(train_labels[i])[:, np.newaxis],nclasses)[np.newaxis, :, :] if X.shape[1] == 1: continue # bug with X, Y of len 1 model.train_on_batch(X, Y) # Evaluation // back into meaning word for output prediction : idx -> words # Train predictions_train = [map(lambda x: idx2label[x], model.predict_on_batch( np.asarray([x])).argmax(2)[0]) for x in train_words] groundtruth_train = [ map(lambda x: idx2label[x], y) for y in train_labels ] words_train = [ map(lambda x: idx2word[x], w) for w in train_words] # Validatae predictions_valid = [map(lambda x: idx2label[x], model.predict_on_batch( np.asarray([x])).argmax(2)[0]) for x in valid_words] groundtruth_valid = [ map(lambda x: idx2label[x], y) for y in valid_labels ] words_valid = [ map(lambda x: idx2word[x], w) for w in valid_words] # Evaluation valid_error = acc_measurement(predictions_valid, groundtruth_valid, words_valid, folder + '/current.valid.txt') train_error = acc_measurement(predictions_train, groundtruth_train, words_train, folder + '/current.train.txt') # Save weight in file 'model.h5' as HDF5 file (default), can be load by: model.load_weights('model.h5', by_name=False) model.save_weights(folder +'/model_weight.h5', overwrite=True) #print ('MODEL built at epoch = ', e, ', error in validation set = ', valid_error) s['current_valid_error'] = valid_error s['current_train_error'] = train_error # Make output file if os.path.exists(folder + '/valid.txt'): os.remove(folder + '/valid.txt') if os.path.exists(folder + '/train.txt'): os.remove(folder + '/train.txt') os.rename(folder + '/current.valid.txt',folder + '/valid.txt') os.rename(folder + '/current.train.txt',folder + '/train.txt') result = read_JSON('result.json') result['validation error'] = float(s['current_valid_error']) result['train error'] = float(s['current_train_error']) write_JSON(result,folder + '/result.json') #Print final result model print ('RESULT MODEL built at epoch = ', e, ',error in validation set = ', s['current_valid_error'], ',error in train set = ', s['current_train_error']) print('\n')
#%% plt.figure() plt.plot(Time[:-step], testy[:, 0], label="y1-test", color="c") plt.plot(Time[:-step], predtest[:, 0], label="y1-pred") plt.plot(Time[:-step], testy[:, 1], label="y2-test", color="m") plt.plot(Time[:-step], predtest[:, 1], label="y2-pred") plt.legend() #%% val = trainx with tf.device('/GPU:0'): U = val[:1, 0:1, :] predtest2 = np.ndarray(shape=(len(val), 2)) for k in range(0, len(val) - 1): predtest2[k, :] = model.predict_on_batch(U) U = val[k + 1:k + 2, 0:1, :] U[0, 0, 0] = predtest2[k, 0] U[0, 0, 1] = predtest2[k, 1] #%% plt.figure() plt.subplot(211) plt.title("NARX model Training Simulation Result") plt.plot(predtest2[:-2, 0]) plt.plot(trainy[:, 0], label="y1-test", color="c") plt.subplot(212) plt.plot(predtest2[:-2, 1]) plt.plot(trainy[:, 1], label="y2-test", color="m") #%%
class BasicTemplateAlgorithm(QCAlgorithm): '''Basic template algorithm simply initializes the date range and cash''' def Initialize(self): '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.''' self.session = K.get_session() self.graph = tf.get_default_graph() self.SetStartDate(2018, 8, 1) #Set Start Date self.SetEndDate(2018, 11, 21) #Set End Date self.SetCash(100000) #Set Strategy Cash ## start the Keras/ Tensorflow session self.session = K.get_session() self.graph = tf.get_default_graph() ## set the currency pair that we are trading, and the correlated currency pair self.currency = "AUDUSD" self.AddForex(self.currency, Resolution.Daily) self.correl_currency = "USDCHF" self.AddForex(self.correl_currency, Resolution.Daily) ## define a long list, short list and portfolio self.long_list, self.short_list = [], [] # Initialise indicators self.rsi = RelativeStrengthIndex(9) self.bb = BollingerBands(14, 2, 2) self.macd = MovingAverageConvergenceDivergence(12, 26, 9) self.stochastic = Stochastic(14, 3, 3) self.ema = ExponentialMovingAverage(9) ## Arrays to store the past indicators prev_rsi, prev_bb, prev_macd, lower_bb, upper_bb, sd_bb, prev_stochastic, prev_ema = [],[],[],[],[],[],[],[] ## Make history calls for both currency pairs self.currency_data = self.History( [self.currency], 150, Resolution.Daily) # Drop the first 20 for indicators to warm up self.correl_data = self.History([self.correl_currency], 150, Resolution.Daily) ## save the most recent open and close ytd_open = self.currency_data["open"][-1] ytd_close = self.currency_data["close"][-1] ## remove yesterday's data. We will query this onData self.currency_data = self.currency_data[:-1] self.correl_data = self.correl_data[:-1] ## iterate over past data to update the indicators for tup in self.currency_data.loc[self.currency].itertuples(): # making Ibasedatabar for stochastic bar = QuoteBar( tup.Index, self.currency, Bar(tup.bidclose, tup.bidhigh, tup.bidlow, tup.bidopen), 0, Bar(tup.askclose, tup.askhigh, tup.asklow, tup.askopen), 0, timedelta(days=1)) self.stochastic.Update(bar) prev_stochastic.append(float(self.stochastic.ToString())) self.rsi.Update(tup.Index, tup.close) prev_rsi.append(float(self.rsi.ToString())) self.bb.Update(tup.Index, tup.close) prev_bb.append(float(self.bb.ToString())) lower_bb.append(float(self.bb.LowerBand.ToString())) upper_bb.append(float(self.bb.UpperBand.ToString())) sd_bb.append(float(self.bb.StandardDeviation.ToString())) self.macd.Update(tup.Index, tup.close) prev_macd.append(float(self.macd.ToString())) self.ema.Update(tup.Index, tup.close) prev_ema.append(float(self.ema.ToString())) ## Forming the Indicators df ## This is common to the Price Prediction rsi_df = pd.DataFrame(prev_rsi, columns=["rsi"]) macd_df = pd.DataFrame(prev_macd, columns=["macd"]) upper_bb_df = pd.DataFrame(upper_bb, columns=["upper_bb"]) lower_bb_df = pd.DataFrame(lower_bb, columns=["lower_bb"]) sd_bb_df = pd.DataFrame(sd_bb, columns=["sd_bb"]) stochastic_df = pd.DataFrame(prev_stochastic, columns=["stochastic"]) ema_df = pd.DataFrame(prev_ema, columns=["ema"]) self.indicators_df = pd.concat([ rsi_df, macd_df, upper_bb_df, lower_bb_df, sd_bb_df, stochastic_df, ema_df ], axis=1) self.indicators_df = self.indicators_df.iloc[20:] self.indicators_df.reset_index(inplace=True, drop=True) ## Currency Data Price self._currency_data = deepcopy(self.currency_data) self._currency_data = self._currency_data.reset_index(level=[0, 1], drop=True) self._currency_data.drop(columns=[ "askopen", "askhigh", "asklow", "askclose", "bidopen", "bidhigh", "bidhigh", "bidlow", "bidclose" ], inplace=True) self._currency_data = self._currency_data.iloc[20:] self._currency_data.reset_index(inplace=True, drop=True) ## saving the previous 6 days OHLC for the price prediction model _close_prev_prices = self._previous_prices( "close", self._currency_data["close"], 6) _open_prev_prices = self._previous_prices("open", self._currency_data["open"], 6) _high_prev_prices = self._previous_prices("high", self._currency_data["high"], 6) _low_prev_prices = self._previous_prices("low", self._currency_data["low"], 6) _all_prev_prices = pd.concat([ _close_prev_prices, _open_prev_prices, _high_prev_prices, _low_prev_prices ], axis=1) _final_table = self._currency_data.join(_all_prev_prices, how="outer") _final_table = _final_table.join(self.indicators_df, how="outer") # Drop NaN from feature table self._features = _final_table.dropna() self._features.reset_index(inplace=True, drop=True) # Make labels for LSTM model self._labels = self._features["close"] self._labels = pd.DataFrame(self._labels) self._labels.index -= 1 self._labels = self._labels[1:] _new_row = pd.DataFrame({"close": [ytd_close]}) self._labels = self._labels.append(_new_row) self._labels.reset_index(inplace=True, drop=True) # Currency Data Direction self.currency_data_direction = self.currency_data.reset_index( level=[0, 1], drop=True) self.currency_data_direction.drop(columns=[ "askopen", "askhigh", "asklow", "askclose", "bidopen", "bidhigh", "bidhigh", "bidlow", "bidclose", "open", "high", "low" ], inplace=True) self.currency_data_direction = self.currency_data_direction.iloc[20:] self.currency_data_direction.reset_index(inplace=True, drop=True) # Correlation Currency Data self.correl_data = self.correl_data.reset_index(level=[0, 1], drop=True) self.correl_data.drop(columns=[ "askopen", "askhigh", "asklow", "askclose", "bidopen", "bidhigh", "bidhigh", "bidlow", "bidclose", "open", "high", "low" ], inplace=True) self.correl_data = self.correl_data.iloc[20:] self.correl_data.reset_index(inplace=True, drop=True) self.correl_data.rename(index=str, columns={"close": "correl_close"}, inplace=True) # Close Price Direction Change self.close_dir_change = self.direction_change( "close", self.currency_data_direction["close"], 11) # Correlation Currency Direction Change self.correl_dir_change = self.direction_change( "correl_close", self.correl_data["correl_close"], 11) # Join the tables joined_table_direction = self.currency_data_direction.join( self.close_dir_change, how="outer") joined_table_direction = joined_table_direction.join( self.correl_dir_change, how="outer") joined_table_direction = joined_table_direction.join( self.indicators_df, how="outer") # Features Direction self.features_direction = joined_table_direction.dropna() self.features_direction.reset_index(inplace=True, drop=True) ## lowerBB and upperBB should change to the difference self.features_direction["lower_bb_diff"] = self.features_direction[ "close"] - self.features_direction["lower_bb"] self.features_direction["upper_bb_diff"] = self.features_direction[ "upper_bb"] - self.features_direction["close"] self.features_direction["ema_diff"] = self.features_direction[ "ema"] - self.features_direction["close"] self.features_direction.drop(columns=["upper_bb", "lower_bb", "ema"], inplace=True) # Make raw df for labels self.labels = self.features_direction["close"] self.labels = pd.DataFrame(self.labels) self.labels.index -= 1 self.labels = self.labels[1:] new_row = pd.DataFrame({"close": [ytd_close]}) self.labels = self.labels.append(new_row) self.labels.reset_index(inplace=True, drop=True) ## Form the binary labels: 1 for up and 0 for down self.labels_direction_new = pd.DataFrame(columns=["direction"]) for row in self.labels.iterrows(): new_close, old_close = row[1], self.features_direction["close"][ row[0]] change = (new_close - old_close)[0] percent_change = 100 * change / old_close if percent_change >= 0: this_df = pd.DataFrame({"direction": [1]}) elif percent_change < 0: this_df = pd.DataFrame({"direction": [0]}) self.labels_direction_new = self.labels_direction_new.append( this_df) self.labels_direction_new.reset_index(inplace=True, drop=True) ## Test out different features self.features_direction.drop( columns=["rsi", "stochastic", "close", "sd_bb"], inplace=True) self.scaler_X = MinMaxScaler() self.scaler_X.fit(self.features_direction) scaled_features_direction = self.scaler_X.transform( self.features_direction) # Hyperparameters Funetuning max_depth = [10, 15, 20, 30] n_estimators = [100, 200, 300, 500] criterion = ["gini", "entropy"] tscv = TimeSeriesSplit(n_splits=4) params_df = pd.DataFrame( columns=["depth", "n_est", "criterion", "acc_score"]) for depth in max_depth: for n_est in n_estimators: for crn in criterion: acc_scores = [] for train_index, test_index in tscv.split( scaled_features_direction): X_train, X_test = scaled_features_direction[ train_index], scaled_features_direction[test_index] #Y_train, Y_test = labels_direction.loc[train_index], labels_direction.loc[test_index] Y_train, Y_test = self.labels_direction_new[ "direction"][ train_index], self.labels_direction_new[ "direction"][test_index] Y_train, Y_test = Y_train.astype('int'), Y_test.astype( 'int') RF = RandomForestClassifier(criterion=crn, n_estimators=n_est, max_depth=depth, random_state=12345) RF_model = RF.fit(X_train, Y_train) y_pred = RF_model.predict(X_test) acc_score = accuracy_score(Y_test, y_pred) acc_scores.append(acc_score) average_acc = np.mean(acc_scores) # self.Debug("ACC") # self.Debug(average_acc) ## make this df for cells, epoch and mse and append to params_df this_df = pd.DataFrame({ "depth": [depth], "n_est": [n_est], "criterion": [crn], "acc_score": [average_acc] }) params_df = params_df.append(this_df) opt_values = params_df[params_df['acc_score'] == params_df['acc_score'].max()] opt_depth, opt_n_est, opt_crn = opt_values["depth"][0], opt_values[ "n_est"][0], opt_values["criterion"][0] self.RF = RandomForestClassifier(criterion="gini", n_estimators=300, max_depth=10, random_state=123) self.RF_model = self.RF.fit( scaled_features_direction, self.labels_direction_new["direction"].astype('int')) ## Define scaler for this class self._scaler_X = MinMaxScaler() self._scaler_X.fit(self._features) self._scaled_features = self._scaler_X.transform(self._features) self._scaler_Y = MinMaxScaler() self._scaler_Y.fit(self._labels) self._scaled_labels = self._scaler_Y.transform(self._labels) ## fine tune the model to determine hyperparameters ## only done once (upon inititialize) _tscv = TimeSeriesSplit(n_splits=2) _cells = [100, 200] _epochs = [50, 100] ## create dataframee to store optimal hyperparams _params_df = pd.DataFrame(columns=["cells", "epoch", "mse"]) # ## loop thru all combinations of cells and epochs for i in _cells: for j in _epochs: print("CELL", i, "EPOCH", j) # list to store the mean square errors cvscores = [] for train_index, test_index in _tscv.split( self._scaled_features): #print(train_index, test_index) X_train, X_test = self._scaled_features[ train_index], self._scaled_features[test_index] Y_train, Y_test = self._scaled_labels[ train_index], self._scaled_labels[test_index] X_train = np.reshape( X_train, (X_train.shape[0], 1, X_train.shape[1])) X_test = np.reshape(X_test, (X_test.shape[0], 1, X_test.shape[1])) model = Sequential() model.add( LSTM(i, input_shape=(1, X_train.shape[2]), return_sequences=True)) model.add(Dropout(0.10)) model.add(LSTM(i, return_sequences=True)) model.add(LSTM(i)) model.add(Dropout(0.10)) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='rmsprop', metrics=['mean_squared_error']) model.fit(X_train, Y_train, epochs=j, verbose=0) scores = model.evaluate(X_test, Y_test) cvscores.append(scores[1]) ## get average value of mean sq error MSE = np.mean(cvscores) ## make this df for cells, epoch and mse and append to params_df this_df = pd.DataFrame({ "cells": [i], "epoch": [j], "mse": [MSE] }) # self.Debug(this_df) # params_df = params_df.append(this_df) _params_df = _params_df.append(this_df) self.Debug(_params_df) # # Check the optimised values (O_values) obtained from cross validation # # This code gives the row which has minimum mse and store the values to O_values # _O_values = _params_df[_params_df['mse'] == _params_df['mse'].min()] # # Extract the optimised values of cells and epochcs from abbove row (having min mse) self._opt_cells = 200 self._opt_epochs = 100 # self._opt_cells = _O_values["cells"][0] # self._opt_epochs = _O_values["epoch"][0] _X_train = np.reshape(self._scaled_features, (self._scaled_features.shape[0], 1, self._scaled_features.shape[1])) _y_train = self._scaled_labels self._session = K.get_session() self._graph = tf.get_default_graph() # Intialise the model with optimised parameters self._model = Sequential() self._model.add( LSTM(self._opt_cells, input_shape=(1, _X_train.shape[2]), return_sequences=True)) self._model.add(Dropout(0.20)) self._model.add(LSTM(self._opt_cells, return_sequences=True)) self._model.add(Dropout(0.20)) self._model.add(LSTM(self._opt_cells, return_sequences=True)) self._model.add(LSTM(self._opt_cells)) self._model.add(Dropout(0.20)) self._model.add(Dense(1)) # self.model.add(Activation("softmax")) self._model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mean_squared_error']) def OnData(self, data): '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. Arguments: data: Slice object keyed by symbol containing the stock data ''' # Make a historical call for yesterday's prices ytd_data = self.History([ self.currency, ], 1, Resolution.Daily) if ytd_data.empty: ytd_data = self.History([ self.currency, ], 2, Resolution.Daily) ytd_data.dropna(inplace=True) ytd_data_correl = self.History([ self.correl_currency, ], 1, Resolution.Daily) if ytd_data_correl.empty: ytd_data_correl = self.History([ self.correl_currency, ], 2, Resolution.Daily) ytd_data_correl.dropna(inplace=True) # Features for price prediction _features_t_minus_1 = self._features[-6:] ## generate prev 6 datapoints (as features for price prediction) _close_prev_prices = self._previous_prices( "close", _features_t_minus_1["close"], 6) _open_prev_prices = self._previous_prices("open", _features_t_minus_1["open"], 6) _high_prev_prices = self._previous_prices("high", _features_t_minus_1["high"], 6) _low_prev_prices = self._previous_prices("low", _features_t_minus_1["low"], 6) ## join all OPHL prices to form the price prediction features _all_prev_prices = pd.concat([ _close_prev_prices, _open_prev_prices, _high_prev_prices, _low_prev_prices ], axis=1) _all_prev_prices.reset_index(drop=True, inplace=True) ## Update indicators ## get the indicators prev_stochastic, prev_rsi, prev_bb, lower_bb, upper_bb, prev_macd, sd_bb, prev_ema = [],[],[],[],[],[],[],[] for tup in ytd_data.loc[self.currency].itertuples(): # making Ibasedatabar for stochastic bar = QuoteBar( tup.Index, self.currency, Bar(tup.bidclose, tup.bidhigh, tup.bidlow, tup.bidopen), 0, Bar(tup.askclose, tup.askhigh, tup.asklow, tup.askopen), 0, timedelta(days=1)) self.stochastic.Update(bar) prev_stochastic.append(float(self.stochastic.ToString())) self.rsi.Update(tup.Index, tup.close) prev_rsi.append(float(self.rsi.ToString())) self.bb.Update(tup.Index, tup.close) prev_bb.append(float(self.bb.ToString())) lower_bb.append(float(self.bb.LowerBand.ToString())) upper_bb.append(float(self.bb.UpperBand.ToString())) sd_bb.append(float(self.bb.StandardDeviation.ToString())) self.macd.Update(tup.Index, tup.close) prev_macd.append(float(self.macd.ToString())) self.ema.Update(tup.Index, tup.close) prev_ema.append(float(self.ema.ToString())) # Dataframes to store all the indicators rsi_df = pd.DataFrame(prev_rsi, columns=["rsi"]) macd_df = pd.DataFrame(prev_macd, columns=["macd"]) upper_bb_df = pd.DataFrame(upper_bb, columns=["upper_bb"]) lower_bb_df = pd.DataFrame(lower_bb, columns=["lower_bb"]) sd_bb_df = pd.DataFrame(sd_bb, columns=["sd_bb"]) stochastic_df = pd.DataFrame(prev_stochastic, columns=["stochastic"]) ema_df = pd.DataFrame(prev_ema, columns=["ema"]) indicators_df = pd.concat([ rsi_df, macd_df, upper_bb_df, lower_bb_df, sd_bb_df, stochastic_df, ema_df ], axis=1) indicators_df.reset_index(inplace=True, drop=True) # Price Prediction Model's Yesterday Data _ytd_data = deepcopy(ytd_data) _ytd_data = _ytd_data.reset_index(drop=True) _ytd_data.drop(columns=[ "askopen", "askhigh", "asklow", "askclose", "bidopen", "bidhigh", "bidhigh", "bidlow", "bidclose" ], inplace=True) _ytd_data.reset_index(drop=True, inplace=True) _ytd_data = _ytd_data.join(_all_prev_prices, how="outer") _ytd_data = _ytd_data.join(indicators_df, how="outer") ## Direction Prediction Model's Yesterday Data (Drop everything from ytd data so only close price remains) ytd_data.drop(columns=[ "askopen", "askhigh", "asklow", "askclose", "bidopen", "bidhigh", "bidhigh", "bidlow", "bidclose", "open", "high", "low" ], inplace=True) ytd_data.reset_index(drop=True, inplace=True) ytd_data_correl.drop(columns=[ "askopen", "askhigh", "asklow", "askclose", "bidopen", "bidhigh", "bidhigh", "bidlow", "bidclose", "open", "high", "low" ], inplace=True) ytd_data_correl.reset_index(drop=True, inplace=True) ytd_data_correl.rename(index=str, columns={"close": "correl_close"}, inplace=True) self.currency_data_direction = self.currency_data_direction.append( ytd_data) self.correl_data = self.correl_data.append(ytd_data_correl) curr_price = ytd_data["close"][0] # Prediction for Direction new_dir_change = self.direction_change( "close", self.currency_data_direction[-11:]["close"], 11) new_correl_dir_change = self.direction_change( "correl_close", self.correl_data[-11:]["correl_close"], 11) X_pred = new_dir_change.join(new_correl_dir_change, how="outer") X_pred.reset_index(inplace=True, drop=True) X_pred = X_pred.join(indicators_df, how="outer") ## lowerBB and upperBB should change to the difference X_pred["lower_bb_diff"] = ytd_data["close"] - X_pred["lower_bb"] X_pred["upper_bb_diff"] = X_pred["upper_bb"] - ytd_data["close"] X_pred["ema_diff"] = X_pred["ema"] - ytd_data["close"] curr_price - float(self.bb.LowerBand.ToString()) ## Define the buy signal and sell signal for direction buy_sig = (55 < float(self.rsi.ToString()) < 70 and 0 <= curr_price - float(self.bb.LowerBand.ToString()) <= 0.02) or float(self.macd.ToString()) > 0.001 sell_sig = (20 < float(self.rsi.ToString()) < 65 and 0 <= float(self.bb.UpperBand.ToString()) - curr_price <= 0.025) or float(self.macd.ToString()) < -0.001 X_pred.drop(columns=[ "rsi", "stochastic", "sd_bb", "ema", "lower_bb", "upper_bb" ], inplace=True) X_pred.reset_index(inplace=True, drop=True) X_pred = self.scaler_X.transform(X_pred) result = self.RF_model.predict(X_pred) direction_pred = result[0] #self.Debug(direction_pred) # LSTM model for price prediction with self._session.as_default(): with self._graph.as_default(): _scaled_ytd_data = self._scaler_X.transform(_ytd_data) _X_predict = np.reshape( _scaled_ytd_data, (_scaled_ytd_data.shape[0], 1, _scaled_ytd_data.shape[1])) _close_price = self._model.predict_on_batch(_X_predict) _close_price_prediction = self._scaler_Y.inverse_transform( _close_price) _close_price_prediction = _close_price_prediction[0][0] # Buy Sell Strategy if direction_pred == 0 and ( sell_sig or _close_price_prediction <= 0.995 * curr_price ) and self.currency not in self.short_list and self.currency not in self.long_list: self.SetHoldings(self.currency, -0.9) self.short_list.append(self.currency) self.Debug("short") if self.currency in self.short_list: cost_basis = self.Portfolio[self.currency].AveragePrice #self.Debug("cost basis is " +str(cost_basis)) if ((curr_price <= float(0.97) * float(cost_basis)) or (curr_price >= float(1.02) * float(cost_basis))): self.Debug("SL-TP reached") #self.Debug("price is" + str(price)) #If true then sell self.SetHoldings(self.currency, 0) self.short_list.remove(self.currency) self.Debug("squared") if direction_pred == 1 and ( buy_sig or _close_price_prediction > 1.015 * curr_price ) and self.currency not in self.short_list and self.currency not in self.long_list and float( self.rsi.ToString()) < 70: self.Debug("output is greater") # Buy the currency with X% of holding in this case 90% self.SetHoldings(self.currency, 0.5) self.long_list.append(self.currency) self.Debug("long") if self.currency in self.long_list: cost_basis = self.Portfolio[self.currency].AveragePrice #self.Debug("cost basis is " +str(cost_basis)) if ((curr_price <= float(0.98) * float(cost_basis)) or (curr_price >= float(1.03) * float(cost_basis))): self.Debug("SL-TP reached") #self.Debug("price is" + str(price)) #If true then sell self.SetHoldings(self.currency, 0) self.long_list.remove(self.currency) self.Debug("squared") def direction_change(self, raw_type, data, num_lookback): ''' num_lookback is the number of previous prices raw_type is a string: open, high, low or close Data is a series Returns a dataframe of previous prices ''' prices = [] length = len(data) for i in range(num_lookback, length + 1): this_data = np.array(data[i - num_lookback:i]) # input is change in priice prices.append(np.diff(this_data.copy())) prices_df = pd.DataFrame(prices) columns = {} for index in prices_df.columns: columns[index] = "{0}_shifted_by_{1}".format( raw_type, num_lookback - index - 1) prices_df.rename(columns=columns, inplace=True) prices_df.index += num_lookback - 1 return prices_df def _previous_prices(self, raw_type, data, num_lookback): ''' num_lookback is the number of previous prices Data is open, high, low or close Data is a series Returns a dataframe of previous prices ''' prices = [] length = len(data) for i in range(num_lookback, length + 1): this_data = np.array(data[i - num_lookback:i]) prices.append(this_data) prices_df = pd.DataFrame(prices) columns = {} for index in prices_df.columns: columns[index] = "{0}_shifted_by_{1}".format( raw_type, num_lookback - index) prices_df.rename(columns=columns, inplace=True) prices_df.index += num_lookback return prices_df
gps.subscribe(stater.updategps) clock.subscribe(stater.updateclock) irL.subscribe(stater.updateirL) irR.subscribe(stater.updateirR) irC.subscribe(stater.updateirC) simu.sleep(1.1) init_time = stater.currenttime prev_clock_tick = init_time currenttime = 0 state = stater.state() prewards = [] pactions = [] pstates = np.zeros((0,state.size)) pstates = np.concatenate((pstates, state)) guess = model.predict_on_batch(pstates.reshape(1, pstates.shape[0], pstates.shape[1])) action, value = policy(guess[0][0, -1, :] * 300)######### pactions.append(action) truedonut = 0 num_runs = 0 niet = 0 now = simu.time() while not award.doquit: niet += 1 lel = False if currenttime >= 300: award.failed = True
checkpoint = ModelCheckpoint("./saved/weights."+str(i)+"{epoch:02d}-{val_loss:.2f}.hdf5", verbose = 2, monitor='loss', save_best_only=False, mode='auto') model.fit(x_train, y_train, batch_size=batch_size, nb_epoch=n_epoch, validation_split=0.2, verbose=2, shuffle='batch', callbacks=[checkpoint]) # Generation mode else: model.load_weights(savingFileName) matrices = getStateMatrices(getKeepActivated) x_comp, y_comp = getNextBatch(matrices, 1, n_timesteps) # x_comp is 1 x n_timesteps x n_input for i in x_comp[0]: print i print composition = np.copy(x_comp[0]) # Composition is n_timesteps x n_input for i in range(composition_size): pred = model.predict_on_batch(x_comp) # Predict takes a None x n_timesteps x n_input and returns a n_input pred = tresholdActivation(pred) composition = np.vstack((composition, pred)) # Composition becomes len(composition)+1 x n_input del x_comp x_comp = np.asarray([composition[-n_timesteps:]]) # Keep only the n_timesteps last composition = unflattenStateMatrix(composition[n_timesteps:], getKeepActivated) noteStateMatrixToMidi(composition, output_file) # Reconstitute midi file
save_best_only=True, mode='max') # Let's train the model using RMSprop model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) print('Not using data augmentation.') model.fit(x_train_processed, y_train_processed, batch_size=batch_size, nb_epoch=nb_epoch, validation_data=(x_valid_processed, y_valid_processed), callbacks=[tfBoard, checkpoint], shuffle=True) ## test accc print("predict test data") y_res = tf.placeholder(tf.float32, [None, n_classes], name='y-input') pred = model.predict_on_batch(x_test_processed) acc = tf.equal(tf.argmax(pred, 1), tf.argmax(y_res, 1)) acc = tf.reduce_mean(tf.cast(acc, tf.float32)) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) p = acc.eval({y_res: y_test_processed}) print(p)
class MLPClassifier: def __init__(self, input_shape=None): self._num_classes = 3 self._model = Sequential() self._model.add(Dense(32, input_dim=input_shape, activation='relu')) self._model.add(Dropout(0.3)) self._model.add((Dense(32, activation='relu'))) self._model.add(Dropout(0.3)) self._model.add(Dense(self._num_classes, activation='softmax')) self._model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) self._encoder = LabelEncoder() def train(self, train_x, train_y): assert len(train_y) == train_x.shape[0] self._encoder.fit(train_y) train_y = self._encoder.transform(train_y) train_y = to_categorical(train_y) # train_y = np.reshape(train_y, (len(train_y), 1)) self._model.fit(train_x, train_y, epochs=20, batch_size=128) def accuracy_test(self, test_x, test_y): assert (len(test_y) == test_x.shape[0]) test_y = to_categorical(self._encoder.transform(test_y)) # test_y = np.reshape(test_y, (len(test_y), 1)) accuracy = self._model.evaluate(test_x, test_y, batch_size=128)[1] return accuracy def _save_model(self): relative_path_project_dir = os.path.dirname( os.path.dirname(os.path.dirname(__file__))) if not os.path.exists( os.path.join(relative_path_project_dir, "saved_model_data/mlp")): os.makedirs(os.path.join(relative_path_project_dir, "output_data")) # self._model.save() def get_average_f1_score(self, x_test, y_test): labels = [1, 0, -1] label_category = to_categorical(self._encoder.transform(labels)) y_pred = self._model.predict_on_batch(x_test) y_pred_labels = [] for i in range(y_pred.shape[0]): y_pred_i = y_pred[i, :] max_val = np.max(y_pred_i) y_pred_i = (y_pred_i >= max_val).astype(int) if np.array_equal(y_pred_i, label_category[0, :]): y_pred_labels.append('1') elif np.array_equal(y_pred_i, label_category[1, :]): y_pred_labels.append('0') else: y_pred_labels.append('-1') # Save predicted labels project_relative_path = os.path.dirname( os.path.dirname(os.path.dirname(__file__))) print(project_relative_path) output_file_sentiment_label = open( os.path.join(project_relative_path, 'saved_model_data/mlp_labels.txt'), 'a') for label in y_pred_labels: output_file_sentiment_label.write(str(label)) output_file_sentiment_label.write('\n') return f1_score(y_test, y_pred_labels, average='weighted', labels=labels)
model.add(Dense(8, activation='relu')) #model.add(Dropout(0.5)) model.add(Dense(nb_classes, activation='softmax')) batch_size = 32 #model.compile(loss='binary_crossentropy', model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) model.fit(X_train, Y_train, nb_epoch=100, batch_size=batch_size, shuffle=True) score = model.evaluate(X_test, Y_test, batch_size=batch_size) print "score= ", score y = model.predict_on_batch(X_test) #y = y.reshape(np.prod(y.shape)) print y.shape, Y_test.shape print "y[0], Y_test[0]= ", y[0], Y_test[0] y1 = np.where(y < 0.5, 0, 1) print "y1[0]= ", y1[0] for i in xrange(len(y)): print y1[i], Y_test[i] print "y-Y_test= ", (y1 - Y_test) print "nb errors: ", (np.abs(y - Y_test)).sum()
y = t_X[['athome']] X_train, X_test, Y_train, Y_test = sklearn.model_selection.train_test_split(X, y, test_size = 0.33, random_state = 4) model = Sequential() # Dense(64) is a fully-connected layer with 64 hidden units. # in the first layer, you must specify the expected input data shape: # here, 2-dimensional vectors. model.add(Dense(64, activation='relu', input_dim=2)) model.add(Dropout(0.5)) model.add(Dense(64, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(3, activation='softmax')) sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) model.fit(X_train, Y_train, epochs=50, batch_size=None) #playing around prediction = model.predict(X_train, batch_size=None, verbose=0, steps=None) model.predict_on_batch(X_train) model.evaluate(X_test, Y_test, batch_size=32) score = model.evaluate(X_test, Y_test, batch_size=32) print('HELLOO IM THE SCORE: ' , score)
def main(): srcImg = cv2.imread(image_training,cv2.IMREAD_GRAYSCALE) tgtImg = cv2.imread(image_expecting,cv2.IMREAD_GRAYSCALE) valImg = cv2.imread(image_validating,cv2.IMREAD_GRAYSCALE) rows = int(srcImg.shape[0] / img_size) columns = int(srcImg.shape[1] / img_size) losses = [] metric = [] accuracies = [] num_of_epochs = [] setTrain = None setTarget = None # Preparing training data.... print ("Preparing training data....") for i in range(0, train_samples): r = random.randint(0, rows - 1) c = random.randint(0, columns - 1) y = r * img_size x = c * img_size h = img_size w = img_size srcTile = srcImg[y:y+h, x:x+w] tgtTile = tgtImg[y:y+h, x:x+w] trainIn = img_to_array(srcTile) trainIn = trainIn.reshape(1,numNeurons) trainIn = np.apply_along_axis(prepareInput, 1, trainIn) trainOut = img_to_array(tgtTile) trainOut = trainOut.reshape(1,numNeurons) trainOut = np.apply_along_axis(prepareInput, 1, trainOut) if setTrain is None: setTrain = trainIn else: setTrain = np.vstack((setTrain, trainIn)) if setTarget is None: setTarget = trainOut else: setTarget = np.vstack((setTarget, trainOut)) # setting up the dnn model (fully connected feed forward dnn) model = Sequential() model.add(Dense(numNeurons, activation=activationFunction, input_shape=(numNeurons,), use_bias=True, bias_initializer='zeros', kernel_initializer=initializers.RandomUniform(minval=-0.5, maxval=0.5, seed=42))) model.add(Dense(numNeurons, activation=activationFunction, input_shape=(int(numNeurons),), use_bias=True, bias_initializer='zeros', kernel_initializer=initializers.RandomUniform(minval=-0.5, maxval=0.5, seed=42))) model.add(Dense(numNeurons, activation=activationFunction, input_shape=(int(numNeurons),), use_bias=True, bias_initializer='zeros', kernel_initializer=initializers.RandomUniform(minval=-0.5, maxval=0.5, seed=42))) model.add(Dense(numNeurons, activation=activationFunction, input_shape=(numNeurons,), use_bias=True, bias_initializer='zeros', kernel_initializer=initializers.RandomUniform(minval=-0.5, maxval=0.5, seed=42))) model.summary() sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy', metrics.binary_accuracy]) # initialization magic for the ui plot plt.ion() ls = DynamicPlot() ls() #let's train the model cnt = 0 for i in range(0, num_iterations): history = model.fit(setTrain, setTarget, batch_size=batch_size, epochs=epochs, verbose=0, validation_data=(setTrain, setTarget)) score = model.evaluate(setTrain, setTarget, verbose=0) cnt = cnt + epochs customScore = 0 p = model.predict_on_batch(setTrain) a = setTrain.flatten() b = p.flatten() for j in range(0, a.size): customScore = customScore + (1- abs(a[j] - b[j])) customAccuracy = float(customScore) / a.size num_of_epochs.append(cnt) losses.append(score[0]) metric.append(score[2]) accuracies.append(customAccuracy) ls.drawPlot(np.asarray(num_of_epochs), np.asarray(losses), np.asarray(metric), np.asarray(accuracies)) print('Loss:', score[0]) print('Metrics:', score[2]) print ('Accuracy', customAccuracy) print('evaluating next iteration: ', i) #let's run a final prediction on another image for validation purposes # Preparing input data for validation prediction.... print ("Preparing input data for validation prediction....") setResult = None rows = int(valImg.shape[0] / img_size) columns = int(valImg.shape[1] / img_size) print(rows, columns) for r in range(0, rows) : for c in range(0, columns): y = r * img_size x = c * img_size h = img_size w = img_size srcTile = valImg[y:y+h, x:x+w] srcIn = img_to_array(srcTile) srcIn = srcIn.reshape(1,numNeurons) srcIn = np.apply_along_axis(prepareInput, 1, srcIn) if setResult is None: setResult = srcIn else: setResult = np.vstack((setResult, srcIn)) print('Predicting....') result = model.predict_on_batch(setResult) s = np.shape(result) print(s) # preparing image for display print ('Preparing image for display') i = 0 for r in range(0, rows): print('proccesing row: ', r) for c in range(0, columns): resMat = np.asmatrix(result[i]) resMat = resMat.reshape(img_size,img_size) for x in range(0, img_size): for y in range(0, img_size): valImg[x + r * img_size,y + c * img_size] = int(255 * resMat[x,y]) i = i + 1 print('Calculations complete! Result image might not be visible, see taskbar. Hit enter in image to terminate run.') cv2.imshow('Result',valImg) cv2.waitKey(0) & 0xFF # see https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_gui/py_image_display/py_image_display.html st = datetime.datetime.now().strftime('%Y%m%d%H%M%S') directory = output_path + st # store the parameters of the trained network for later purposes if not os.path.exists(directory): os.makedirs(directory) # save the validation image resImage = directory + '\\result.png' cv2.imwrite(resImage, valImg) cv2.destroyAllWindows() modelFile = directory + '\\model.json' modelJson = model.to_json() f = open(modelFile, 'w') f.write(modelJson) f.close() modelH5 = directory + '\\model.h5' model.save(modelH5)
for i in range(val_in.shape[0]): te_loss = model.test_on_batch( np.expand_dims(np.expand_dims(val_in[i, :], axis=0), axis=0), np.atleast_2d(val_out[i, :])) mean_te_loss.append(te_loss) model.reset_states() print('loss validation = {}'.format(np.mean(mean_te_loss))) print('___________________________________') print("Test..") y_pred = [] for j in range(test_in.shape[0]): batch = test_in[j, :] y_pred.append(model.predict_on_batch(batch[np.newaxis, np.newaxis, :])) y_pred = np.vstack(y_pred) model.reset_states() mean_te_loss = [] for i in range(test_in.shape[0]): batch = test_in[i, :] batch_y = test_out[i, :] te_loss = model.test_on_batch(batch[np.newaxis, np.newaxis, :], batch_y[np.newaxis, :]) mean_te_loss.append(te_loss) model.reset_states() print('loss test = {}'.format(np.mean(mean_te_loss))) print('___________________________________')
class AC(): def __init__(self, action_min, action_max, states_n, actor_learning_rate=0.001, critic_learning_rate=0.01, gamma=0.9): self.action_min = action_min self.action_max = action_max self.states_n = states_n self.actor_lr = actor_learning_rate self.critic_lr = critic_learning_rate self.gamma = gamma self.__init_net() def __init_net(self): weight_init = initializers.RandomNormal(0, 0.1) bias_init = initializers.Constant(0.1) input_layer = Input(shape=(self.states_n, )) hidden_layer = Dense(50, input_dim=self.states_n, activation='relu', kernel_initializer=weight_init, bias_initializer=bias_init)(input_layer) hidden_layer_mu = Dense(1, activation='tanh', kernel_initializer=weight_init, bias_initializer=bias_init)(hidden_layer) hidden_layer_sigma = Dense(1, activation='softplus', kernel_initializer=weight_init, bias_initializer=bias_init)(hidden_layer) output_mu = Lambda(lambda x: x * 2)(hidden_layer_mu) output_sigma = Lambda(lambda x: x + 0.1)(hidden_layer_sigma) output_layer = Concatenate()([output_mu, output_sigma]) self.actor = Model(inputs=input_layer, outputs=output_layer) self.actor.compile(optimizer=optimizers.Adam(self.actor_lr), loss=actor_loss) self.critic = Sequential() self.critic.add( Dense(50, input_dim=self.states_n, activation='relu', kernel_initializer=weight_init, bias_initializer=bias_init)) self.critic.add( Dense(1, kernel_initializer=weight_init, bias_initializer=bias_init)) self.critic.compile(optimizer=optimizers.Adam(self.critic_lr), loss='mse') def learn(self, state, action, reward, next_state): state = state.reshape(-1, self.states_n) next_state = next_state.reshape(-1, self.states_n) td_error, critic_loss = self.__critic_learn(state, reward, next_state) actor_loss = self.actor.train_on_batch(state, action, sample_weight=td_error.reshape( -1, )) return critic_loss, actor_loss def __critic_learn(self, state, reward, next_state): V_now = self.critic.predict_on_batch(state) V_next = self.critic.predict_on_batch(next_state) y = reward + self.gamma * V_next td_error = y - V_now loss = self.critic.train_on_batch(state, y) return td_error, loss def choose_action(self, state): mu, sigma = self.actor.predict_on_batch(state.reshape(1, -1))[0] action = np.random.normal(mu, sigma) return np.clip(action, self.action_min, self.action_max), sigma
print("Training =>") train_pred_label = [] avgLoss = 0 bar = progressbar.ProgressBar(maxval=len(train_x)) for n_batch, sent in bar(enumerate(train_x)): label = train_label[n_batch] label = np.eye(n_classes)[label][np.newaxis, :] sent = sent[np.newaxis, :] if sent.shape[1] > 1: #some bug in keras loss = model.train_on_batch(sent, label) avgLoss += loss pred = model.predict_on_batch(sent) pred = np.argmax(pred, -1)[0] train_pred_label.append(pred) avgLoss = avgLoss / n_batch predword_train = [ list(map(lambda x: idx2la[x], y)) for y in train_pred_label ] con_dict = conlleval(predword_train, groundtruth_train, words_train, 'r.txt') train_f_scores.append(con_dict['f1']) print('Loss = {}, Precision = {}, Recall = {}, F1 = {}'.format( avgLoss, con_dict['r'], con_dict['p'], con_dict['f1'])) print("Validating =>")
# Loading model weight start = time.time() classifier.load_weights('cat_dog_rasnet50.h5') # Part 3 Prediction Image from video import numpy as np from keras.preprocessing import image as image_utils import time import cv2 cap = cv2.VideoCapture(0) while (cap.isOpened()): start = time.time() ret, frame = cap.read() if ret == True: cv2.imshow('frame', frame) cv2.imwrite('temp.jpg', frame) test_image = image_utils.load_img('temp.jpg', target_size=(64, 64)) test_image = image_utils.img_to_array(test_image) test_image = np.expand_dims(test_image, axis=0) result = classifier.predict_on_batch(test_image) print('Dog ', round(result[0][1] * 100.0), '%') if cv2.waitKey(1) & 0xFF == ord('q'): break else: break cap.release() cv2.destroyAllWindows
for epoch in range(10): for batch in os.listdir('Dataset'): if int(batch[:-4]) >=154: break arr = np.load('Dataset/' + batch) np.random.shuffle(arr) while True: if len(arr) > 100: cur_batch = arr[0:100] arr = arr[100:] print epoch, batch, model.train_on_batch(np.expand_dims(cur_batch, axis=2), cur_batch) else: if overFlow is None or len(overFlow) == 0: overFlow = arr else: overFlow = np.concatenate([overFlow, arr]) break while len(overFlow) >= 100: cur_batch = overFlow[0:100] overFlow = overFlow[100:] print model.train_on_batch(np.expand_dims(cur_batch, axis=2), cur_batch) model.save('firstModel') X_test = np.matrix([decode('fma_small/024/024420.mp3')[:220000]]) X_test = np.expand_dims(X_test, axis=2) preds = model.predict_on_batch(X_test) write('pred.wav', AudioSegment.from_file('fma_small/024/024420.mp3').frame_rate,preds[0,:])
''' Starting testing ''' import cPickle as pickle print "Starting ..." ej_case = 'EJ_case.pkl' question_input, ans1_input, ans2_input, ans3_input, ans4_input, ans5_input, image_input, solution = pickle.load(open(ej_case,'rb')) image_v = np.array([ image_input[0][0] ]) print solution.shape temp = np.hstack(( question_input[0][0], ans1_input[0][0] , ans2_input[0][0] ,ans3_input[0][0] ,ans4_input[0][0] ,ans5_input[0][0] )) temp = np.array([[temp]]) print temp[0].shape print image_v.shape input_data = np.hstack(( image_v , temp[0] )) print input_data.shape for n in range(100): #print "model_1", model.predict_on_batch( [ ans1_input , image_input] , solution ) print model.train_on_batch( [ input_data , image_v] , solution ) #print model_1.train_on_batch(ans1_input , solution) print model.predict_on_batch( [ input_data , image_v ]) #print model_1.predict_on_batch(ans1_input)
"supported file extensions are '.bmp', '.jpeg', '.jpg', '.png', '.tif', '.tiff'" ) continue d = my_root + '/temporary/' + os.path.splitext(f)[0] + '/' make_new_dir(d) shutil.copyfile(files_path + f, d + f) f_count += 1 _filenames.append(f) _filenames = sorted(_filenames) for batch, lbls in datagen.flow_from_directory(my_root + '/temporary/', target_size=(target_size, target_size), batch_size=1): prediction = model.predict_on_batch(batch) print('file {0}: {1}'.format(_filenames[lbls.argmax()], class_names[prediction.argmax()])) f_count -= 1 if f_count == 0: break shutil.rmtree(my_root + '/temporary/') else: print("plotting model...") plot_model(model, show_shapes=True, to_file=os.path.dirname(weights_file) + '/model.png') generator = datagen.flow_from_directory(images_dir, target_size=(target_size,
def generate(seq, maxlen=1, bs=3, ep=5, output_iterations=10, num_mixture_components=10): # seq is a single sample, in the format (timesteps, features) ! # TODO: expand code to support multiple samples, fed into model together as a batch # Cut the timeseries data (variable name 'seq') into semi-redundant sequence chunks of maxlen X = [] y = [] for i in range(0, len(seq) - maxlen): X.append(seq[i:i + maxlen]) y.append(seq[i + maxlen]) dim = len((X[0][0])) print("sequence chunks:", len(X)) print("chunk width:", len(X[0])) print("vector dimension:", dim) print("number of mixture components:", num_mixture_components) X = np.array(X) y = np.array(y) # build the model: 2 stacked LSTM print('Build model...') model = Sequential() model.reset_states() model.add( LSTM((dim + 2) * num_mixture_components, return_sequences=False, input_shape=(maxlen, dim))) model.add(Dense((dim + 2) * num_mixture_components)) model.add(GMMActivation(num_mixture_components)) model.compile(loss=gmm_loss, optimizer=RMSprop(lr=0.001)) # Train the model model.fit(X, y, batch_size=bs, nb_epoch=ep) # Generate timeseries x_seed = X[len(X) - 1] #choose final in-sample data point to initialize model x_array = [] x_array.append(x_seed) x = np.array(x_array) predicted = [] for i in range(output_iterations): pred_parameters = model.predict_on_batch(x)[0] means = pred_parameters[:num_mixture_components * dim] sds = pred_parameters[(num_mixture_components * dim):(num_mixture_components * (dim + 1))] weights = pred_parameters[(num_mixture_components * (dim + 1)):] print(means) print(sds) print(weights) means = means.reshape(num_mixture_components, dim) sds = sds[:, np.newaxis] weights = weights[:, np.newaxis] pred = weights * np.random.normal(means, sds) pred = np.sum(pred, axis=0) predicted.append(pred) return predicted
kernel_regularizer=regularizers.l2(0.1))) gen_model.add(Dense(1, activation='linear')) gen_model.compile(optimizer='nadam', loss='mean_squared_error') labels = np.zeros(5478, dtype='b') # intialize weak labels, using from generated heuristics: delta(pieces) + 2*delta(lines) + 10*winner labels += (data[:, 9] - data[:, 10]) + 2 * (data[:, 11] - data[:, 12]) + 10 * data[:, 13] # take game state data and weak labels to train model gen_model.fit(data[:, 9:], labels, epochs=64, batch_size=256, verbose=0) print('gen total loss: ', gen_model.test_on_batch(data[:, 9:], labels) * 5478) # calculate probabilistic (still noisy) labels using trained generative model probabilistic = gen_model.predict_on_batch(data[:, 9:]) # calculate confidence score for each label from the error between the original weak label and the new probabilistic label confidence = 1 / (1 + np.absolute(labels - probabilistic[:, 0])) # discriminative model that learns from just the board values and probabilistic labels disc_model = Sequential() # use regularizers for noisy labels disc_model.add( Dense(16, activation='linear', input_dim=9, kernel_regularizer=regularizers.l2(0.01))) disc_model.add( Dense(8, activation='relu', kernel_regularizer=regularizers.l2(0.01)))
##print(metrics.binary_accuracy(y_test,round(model.predict(X_test)))) #import pprint #pprint.pprint() logging.info("Test Accuracy: %.3f%%" % (scores[1] * 100)) logging.info( "**END****Epoch {2} of {3}, Minibatch {0} of {1} in {4} ********". format(minibatch + 1, no_mini_batches, e + 1, epochs, str((datetime.datetime.now() - minibatch_start_time)))) logging.info('') # Final evaluation of the model logging.info("For {0} test samples".format(len(y_test_all))) logging.info("True Values") logging.info(y_test_all) logging.info("Predicted Values") result = model.predict_on_batch(X_test_all) logging.info(model.predict_on_batch(X_test_all)) logging.info("Comparison on all:") result_both = [(float(result[i]), round(float(result[i])), y_test_all[i]) for i in range(len(result))] y_score = [round(float(y)) for y in result] #Calculate Precision scores = model.evaluate(X_test_all, y_test_all, verbose=0) accuracy.append((scores[1] * 100, scores[0])) #class epoch: e=e #import sklearn #sklearn.metrics.metrics epoch.data['accuracy'] = scores[1] epoch.data['loss'] = scores[0]
class RNNOneHotK(RNNBase): """RNNOneHot are recurrent neural networks that do not depend on the factorization: they are based on one-hot encoding. The parameters specific to the RNNOneHot are: diversity_bias: a float in [0, inf) that tunes how the cost function of the network is biased towards less seen movies. In practice, the classification error given by the categorical cross-entropy is divided by exp(diversity_bias * popularity (on a scale from 1 to 10)). This will reduce the error associated to movies with a lot of views, putting therefore more importance on the ability of the network to correctly predict the rare movies. A diversity_bias of 0 produces the normal behavior, with no bias. """ def __init__(self, updater=None, recurrent_layer=None, backend='tensorflow', mem_frac=None, diversity_bias=0.0, regularization=0.0, **kwargs): super(RNNOneHotK, self).__init__(**kwargs) # self.diversity_bias = np.cast[theano.config.floatX](diversity_bias) self.regularization = regularization self.backend = backend self.updater = updater self.recurrent_layer = recurrent_layer self.tf_mem_frac = mem_frac self.name = "RNN with categorical cross entropy" self.set_keras_backend(self.backend) if self.backend == 'tensorflow': self.framework = 'ktf' else: self.framework = 'kth' def set_keras_backend(self, backend): if be.backend() != backend: environ['KERAS_BACKEND'] = backend reload(be) assert be.backend() == backend def _get_model_filename(self, epochs): """Return the name of the file to save the current model """ # filename = "rnn_cce_db"+str(self.diversity_bias)+"_r"+str(self.regularization)+"_"+self._common_filename(epochs) filename = "rnn_cce_" + self._common_filename(epochs) + "." + self.framework return filename def prepare_networks(self, n_items): self.n_items = n_items if be.backend() == 'tensorflow': import tensorflow as tf from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = self.tf_mem_frac set_session(tf.Session(config=config)) self.model = Sequential() if self.recurrent_layer.embedding_size > 0: self.model.add(Embedding(self.n_items, self.recurrent_layer.embedding_size, input_length=self.max_length)) self.model.add(Masking(mask_value=0.0)) else: self.model.add(Masking(mask_value=0.0, input_shape=(self.max_length, self.n_items))) rnn = self.get_rnn_type(self.recurrent_layer.layer_type, self.recurrent_layer.bidirectional) for i, h in enumerate(self.recurrent_layer.layers): if i != len(self.recurrent_layer.layers) - 1: self.model.add(rnn(h, return_sequences=True, activation=self.active_f)) else: #last rnn return only last output self.model.add(rnn(h, return_sequences=False, activation=self.active_f)) self.model.add(Dense(self.n_items)) self.model.add(Activation('softmax')) optimizer = self.updater() self.model.compile(loss='categorical_crossentropy', optimizer=optimizer) def get_rnn_type(self, rnn_type, bidirectional): if rnn_type == 'GRU': rnn= GRU elif rnn_type=='LSTM': rnn= LSTM else: rnn= RNN if bidirectional: return Bidirectional(rnn) else: return rnn def _prepare_input(self, sequences): """ Sequences is a list of [user_id, input_sequence, targets] """ # print("_prepare_input()") batch_size = len(sequences) # Shape of return variables if self.recurrent_layer.embedding_size > 0: X = np.zeros((batch_size, self.max_length), dtype=self._input_type) # keras embedding requires movie-id sequence, not one-hot else: X = np.zeros((batch_size, self.max_length, self.n_items), dtype=self._input_type) # input of the RNN Y = np.zeros((batch_size, self.n_items), dtype='float32') # output target for i, sequence in enumerate(sequences): user_id, in_seq, target = sequence if self.recurrent_layer.embedding_size > 0: X[i,:len(in_seq)] = np.array([item[0] for item in in_seq]) else: seq_features = np.array(list(map(lambda x: self._get_features(x), in_seq))) X[i, :len(in_seq), :] = seq_features # Copy sequences into X Y[i][target[0][0]] = 1. return X, Y def _compute_validation_metrics(self, metrics): """ add value to lists in metrics dictionary """ ev = evaluation.Evaluator(self.dataset, k=10) if not self.iter: for batch_input, goal in self._gen_mini_batch(self.dataset.validation_set(epochs=1), test=True): output = self.model.predict_on_batch(batch_input[0]) predictions = np.argpartition(-output, list(range(10)), axis=-1)[0, :10] # print("predictions") # print(predictions) ev.add_instance(goal, predictions) else: for sequence, user in self.dataset.validation_set(epochs=1): seq_lengths = list(range(1, len(sequence))) # 1, 2, 3, ... len(sequence)-1 for length in seq_lengths: X = np.zeros((1, self.max_length, self._input_size()), dtype=self._input_type) # input shape of the RNN seq_by_max_length = sequence[max(length - self.max_length, 0):length] # last max length or all X[0, :len(seq_by_max_length), :] = np.array(map(lambda x: self._get_features(x), seq_by_max_length)) output = self.model.predict_on_batch(X) predictions = np.argpartition(-output, list(range(10)), axis=-1)[0, :10] # print("predictions") # print(predictions) goal = sequence[length:][0] ev.add_instance(goal, predictions) metrics['recall'].append(ev.average_recall()) metrics['sps'].append(ev.sps()) metrics['precision'].append(ev.average_precision()) metrics['ndcg'].append(ev.average_ndcg()) metrics['user_coverage'].append(ev.user_coverage()) metrics['item_coverage'].append(ev.item_coverage()) metrics['blockbuster_share'].append(ev.blockbuster_share()) # del ev ev.instances = [] return metrics def _save(self, filename): super(RNNOneHotK, self)._save(filename) self.model.save(filename) def _load(self, filename): '''Load parameters values from a file ''' self.model = load_model(filename)
deep_NN.add( Activation('relu') ) # output layer deep_NN.add( Dense(num_labels, W_regularizer=l2(0.01), init='glorot_uniform') ) deep_NN.add( Activation('linear') ) # initialize stochastic gradient descent #sgd = SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False) #rms_prop = RMSprop(lr=0.0001, rho=0.9, epsilon=1e-06) adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08) deep_NN.compile(loss='mse', optimizer= adam, metrics=['accuracy']) # train hist = deep_NN.fit(xTrain, yTrain, batch_size=1000, nb_epoch=10, validation_data=(xTest, yTest), verbose=1, shuffle=True) sys.stdout.write("xTrain type: " + str(type(xTrain)) + "\n") y_pred = deep_NN.predict_on_batch(xTrain[1:1000]) ''' # saving history sys.stdout.write('history: saving to file') with open('../training/results:(2hl,200u,200ep).csv', 'wb') as f: t_steps = len(hist.history['acc']) writer = csv.writer(f, delimiter=',' ) writer.writerow( ('loss', 'acc', 'val_loss', 'val_acc') ) for i in range(t_steps): loss = hist.history['loss'][i] acc = hist.history['acc'][i] val_loss = hist.history['val_loss'][i] val_acc = hist.history['val_acc'][i] writer.writerow( (loss, acc, val_loss, val_acc) ) f.close()
ax.set_xticklabels(classes) ax.set_yticklabels(classes) thresh = cm.max() / 2. for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): ax.text(j, i, cm[i, j], horizontalalignment="center", color="white" if cm[i, j] > thresh else "black") ax.set_ylabel('True label') ax.set_xlabel('Predicted label') # ________________________________________________________ PLOT CM ___________________________ y_train_pred = model.predict_on_batch(np.array(X_Train))[:, 0] y_test_pred = model.predict_on_batch(np.array(X_Test))[:, 0] fig, ax = plt.subplots(1, 3) fig.set_size_inches(15, 5) plot_cm(ax[0], Y_Train, y_train_pred, [0, 1], 'Confusion matrix (TRAIN)') plot_cm(ax[1], Y_Test, y_test_pred, [0, 1], 'Confusion matrix (TEST)') #plot_auc(ax[2], y_train, y_train_pred, y_test, y_test_pred) plt.tight_layout() plt.show() # __________________________________________ END PLOT _________________________________________________________
model_1.add(layer_a1) model_2.add(layer_a2) model_1.compile(loss='categorical_crossentropy', optimizer='rmsprop') model_2.compile(loss='categorical_crossentropy', optimizer='rmsprop') ''' Starting testing ''' import cPickle as pickle print "Starting ..." ej_case = 'EJ_case.pkl' question_input, ans1_input, ans2_input, ans3_input, ans4_input, ans5_input, image_input, solution = pickle.load(open(ej_case,'rb')) print ans1_input.shape print solution.shape for n in range(10): print "model_2", model_2.predict_on_batch(ans1_input) print "model_1", model_1.predict_on_batch(ans1_input) print model_1.train_on_batch(ans1_input , solution) print model_2.predict_on_batch(ans1_input) print model_1.predict_on_batch(ans1_input)
compare_model.compile(optimizer=optimizers.SGD(0.01), loss='categorical_crossentropy', metrics=['accuracy']) x1 = np.linspace(-1, 1, 10).reshape(-1, 1) x2 = np.linspace(-1, 1, 10).reshape(-1, 1) x1, x2 = np.meshgrid(x1, x2) x = np.hstack((x1.reshape(-1, 1), x2.reshape(-1, 1))) labels = np.sign(x[:, 1]).reshape(-1, 1) OHE = OneHotEncoder() OHE.fit(labels) y = OHE.transform(labels).toarray() vt = np.arange(y.shape[0], dtype='float32').reshape(-1, 1) y_predict = model.predict_on_batch(x) crossentropy = -vt * y * np.log(y_predict) print(np.mean(crossentropy, axis=0), np.sum(np.mean(crossentropy, axis=0))) for i in range(1): loss = np.array(model.train_on_batch(x, y, sample_weight=vt.reshape(-1, ))) #compare_loss = np.array(compare_model.train_on_batch(x, y, sample_weight=vt.reshape(-1, ))) #loss = np.array(model.train_on_batch(x, y*vt)) compare_loss = np.array(compare_model.train_on_batch(x, y * vt)) #print(loss) print(model.get_weights()) print(compare_model.get_weights()) y_predict = model.predict_on_batch(x) loss_func = tf.keras.losses.CategoricalCrossentropy() print(loss_func(y, y_predict))
model.add(Activation('sigmoid')) model.add(Dense(output_dim=1)) #编译模型 adagrad = Adagrad(lr=0.3, epsilon=1e-06) model.compile(loss='mean_squared_error', optimizer=adagrad) #准备数据 xvalue=numpy.asarray([numpy.arange(0,2,0.0002)]).T yvalue=numpy.sin(xvalue*2*numpy.pi)*0.8 num=len(xvalue) trainset= (numpy.array([xvalue[i] for i in range(0,num,3)]),numpy.array([yvalue[i] for i in range(0,num,3)])) validset= (numpy.array([xvalue[i] for i in range(1,num,9)]),numpy.array([yvalue[i] for i in range(1,num,9)])) #训练模型 info=model.fit(trainset[0], trainset[1], nb_epoch=400, batch_size=16, validation_data=validset, callbacks=[EarlyStopping(patience=10,verbose=1)], show_accuracy=True) #绘制图表 r_val=model.predict_on_batch(xvalue) fig,(ax1)=plt.subplots(1,1,True,True) ax1.plot(xvalue,r_val) ax1.plot(xvalue.reshape((xvalue.size,)),yvalue.reshape((yvalue.size,))) plt.show()
plt.subplot(2, 2, i+1) im = np.reshape(images[i], (1, -1)) im = (np.reshape(im, [28, 28]) + 1) * 255 im = np.clip(im, 0, 255) im = np.uint8(im) plt.imshow(im, cmap='gray') plt.show() (xtrain, _), (xtest, _) = mnist.load_data() xtrain = np.reshape(xtrain, [-1, 28, 28, 1]) xtest = np.reshape(xtest, [-1, 28, 28, 1]) xtrain = (xtrain.astype(np.float32) - 127.5) / 127.5 epochs = 1000 batch_size = 100 for e in range(epochs): for i in tqdm(range(int(xtrain.shape[0]/batch_size))): xreal = xtrain[(i) * batch_size:(i + 1) * batch_size] noise = generateRandomData(batch_size, input_size) xfake = generator.predict_on_batch(noise) discriminator.trainable = True discriminator.train_on_batch(xreal, np.array([[0.9]] * batch_size)) discriminator.train_on_batch(xfake, np.array([[0.]] * batch_size)) discriminator.trainable = False gan.train_on_batch(noise, np.array([[1.]] * batch_size)) print(e,")") showResults(generator)
# Формируем модель сети model = Sequential() model.add(Embedding(2*datadim, 256, input_length=None)) model.add(LSTM(output_dim=128, activation='tanh', inner_activation='sigmoid')) model.add(Dropout(0.5)) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='rmsprop') model.summary() model.fit(x_train_all, y_train_all, batch_size=128, nb_epoch=10) file_name_weights = 'C:\\Python34\\Puzzle15\\utils\\Solutions\\weights' file_name_model = 'C:\\Python34\\Puzzle15\\utils\\Solutions\\model' model.save_weights(file_name_weights, overwrite=True) json_string = model.to_json() with open(file_name_model, 'w') as outfile: json.dump(json_string, outfile) outfile.close() # Печать на экран предсказаний только что созданной модели сети for i in range(20): X = x_val_all[i] X.shape = (-1, 2*datadim) print('X - ', X) prediction = model.predict_on_batch(X) print('Prediction', prediction) print('Y val - ', y_val_all[i])
ax.plot(fpr_test, tpr_test) ax.plot([0, 1], [0, 1], 'k--') ax.set_xlim([0.0, 1.0]) ax.set_ylim([0.0, 1.05]) ax.set_xlabel('False Positive Rate') ax.set_ylabel('True Positive Rate') ax.set_title('ROC curve') train_text = 'train acc = {:.3f}, auc = {:.2f}'.format( acc_train, roc_auc_train) test_text = 'test acc = {:.3f}, auc = {:.2f}'.format( acc_test, roc_auc_test) ax.legend([train_text, test_text]) ### 5th part y_train_pred = model.predict_on_batch(np.array(x_train))[:, 0] y_test_pred = model.predict_on_batch(np.array(x_test))[:, 0] fig, ax = plt.subplots(1, 3) fig.set_size_inches(15, 5) plot_cm(ax[0], y_train, y_train_pred, [0, 1], 'Confusion matrix (TRAIN)') plot_cm(ax[1], y_test, y_test_pred, [0, 1], 'Confusion matrix (TEST)') plot_auc(ax[2], y_train, y_train_pred, y_test, y_test_pred) plt.tight_layout() plt.show()
print("Training =>") train_pred_label = [] avgLoss = 0 bar = progressbar.ProgressBar(max_value=len(train_x)) for n_batch, sent in bar(enumerate(train_x)): label = train_label[n_batch] label = np.eye(n_classes)[label][np.newaxis,:] sent = sent[np.newaxis,:] if sent.shape[1] > 1: #some bug in keras loss = model.train_on_batch(sent, label) avgLoss += loss pred = model.predict_on_batch(sent) pred = np.argmax(pred,-1)[0] train_pred_label.append(pred) avgLoss = avgLoss/n_batch predword_train = [ list(map(lambda x: idx2la[x], y)) for y in train_pred_label] con_dict = conlleval(predword_train, groundtruth_train, words_train, 'r.txt') train_f_scores.append(con_dict['f1']) print('Loss = {}, Precision = {}, Recall = {}, F1 = {}'.format(avgLoss, con_dict['r'], con_dict['p'], con_dict['f1'])) print("Validating =>") val_pred_label = [] avgLoss = 0
for fra in range(0, fra_num - num_steps, skp) + [fra_num - num_steps]: x_fra = np.zeros((fre_num, num_steps, flen)) x_fra[:, :, 0] = np.abs( np.concatenate((data_x[0, fra:fra + num_steps].reshape( 1, num_steps), data_x[:-1, fra:fra + num_steps]), axis=0)) x_fra[:, :, 1] = np.abs(data_x[:, fra:fra + num_steps]) x_fra[:, :, 2] = np.abs( np.concatenate( (data_x[1:, fra:fra + num_steps], data_x[-1, fra:fra + num_steps].reshape(1, num_steps)), axis=0)) norml = np.mean(x_fra[:, :, 1], axis=1).reshape(fre_num, 1, 1) xn_fra = x_fra / (norml) pred_fra = model.predict_on_batch(xn_fra) pred_fra = (np.exp(pred_fra) * np.square(norml)).reshape( fre_num, num_steps) if fra == 0: pred_y[:, :num_steps] = pred_fra elif fra == fra_num - num_steps: pred_y[:, fra + stp:] = pred_fra[:, stp:] else: pred_y[:, fra + stp:fra + enp] = pred_fra[:, stp:enp] ################################################### LogErr[noiseindx, snrindx] = np.abs(10 * np.log10(pred_y) - 10 * np.log10(NPSD)).mean() print LogErr[noiseindx, ]
class speechLSTM: # Initializing the LSTM Model def __init__(self): self.prevData = 30 self.batchsize=200 self.model = Sequential() def build_nnet(self): self.model.add(LSTM(300,return_sequences=True, stateful=True, batch_input_shape=(self.batchsize, self.prevData, 2))) self.model.add(Activation("linear")) self.model.add(Dropout(0.5)) # self.model.add(LSTM(400,return_sequences=True,stateful=True)) # self.model.add(Activation("linear")) # self.model.add(Dropout(0.5)) # self.model.add(LSTM(500, return_sequences=True, stateful=True)) # self.model.add(Activation("linear")) # self.model.add(Dropout(0.5)) self.model.add(LSTM(400, return_sequences=True, stateful=True)) self.model.add(Activation("relu")) # self.model.add(Dropout(0.5)) # self.model.add(LSTM(700, return_sequences=True, stateful=True)) # self.model.add(Activation("linear")) self.model.add(LSTM(500, return_sequences=False, stateful=True)) self.model.add(Activation("linear")) self.model.add(Dropout(0.5)) self.model.add(Dense(1, activation='sigmoid')) self.model.compile(loss='binary_crossentropy', optimizer='adadelta') def load_data_file(self): outputdata = [] for f in gb.glob("/media/vyassu/OS/Users/vyas/Documents/Assigments/BigData/AudioData/KL/*.wav"): frate, inputdata = sc.read(f) pitch=lp.getPitch(f,frate) emotion = "" loudness = abs(an.loudness(inputdata)) filename = f.split("/")[-1].split(".")[0] if filename[0] == "s": emotion = filename[0:2] emotion = float(int(hashlib.md5(emotion).hexdigest(), 16)) else: emotion = filename[0] emotion = float(int(hashlib.md5(emotion).hexdigest(), 16)) outputdata.append(list([loudness,pitch, emotion])) return outputdata def get_train_test_data(self,data,percent_split): ftestList,ltestlist,fvalidList,lvalidList,ftrainList,ltrainList=[],[],[],[],[],[] noOfTrainSamples = len(data)*(1-percent_split) noOfTestSamples = len(data)-noOfTrainSamples self.batchsize = int(noOfTestSamples) noOfTrainSamples = int((noOfTrainSamples - self.prevData)/noOfTestSamples) for i in range(int(noOfTrainSamples)*self.batchsize): #ltrainList.append(data.iloc[i:i+self.prevData, 2:].as_matrix()) ftrainList.append(data.iloc[i:i+self.prevData, 0:2].as_matrix()) ltrainList = data.iloc[0:int(noOfTrainSamples)*self.batchsize, 2:].values for i in range(self.batchsize): fvalidList.append(data.iloc[i:i + self.prevData, 0:2].as_matrix()) lvalidList = data.iloc[0:self.batchsize, 2:].values randNum = random.randint(0,noOfTrainSamples) for i in range(randNum,randNum+self.batchsize): ftestList.append(data.iloc[i:i + self.prevData, 0:2].as_matrix()) ltestlist = data.iloc[randNum: randNum+self.batchsize, 2:].values return np.array(ftestList),np.array(ltestlist),np.array(ftrainList),np.array(ltrainList),np.array(fvalidList),np.array(lvalidList) def trainNNet(self,data_,label_,valid_data,valid_label): data = data_/data_.max(axis=0) label = label_/label_.max(axis=0) valid_data = valid_data/valid_data.max(axis=0) valid_label = valid_label/valid_label.max(axis=0) self.model.fit(data, label, batch_size=self.batchsize, nb_epoch=5,validation_data=(valid_data,valid_label),show_accuracy=True,shuffle=False) def predict(self,ftest_,ltest_): ltest=ltest_/ltest_.max(axis=0) ftest=ftest_/ftest_.max(axis=0) count=0 predcited_data= self.model.predict_on_batch(ftest) print ("Score:",self.model.evaluate(ftest,ltest, show_accuracy=True)) for i in range(len(predcited_data)): if predcited_data[0][i]==ltest[0][i]: count+=1 print ("No of element Matching:",count) print ("No of dissimilar elements:",len(predcited_data[0])-count) print predcited_data print ltest def saveModel(self): self.model.save_weights("/media/vyassu/OS/Users/vyas/Documents/Assigments/BigData/", overwrite=False) def getIntermediateLayer(self): get_3rd_layer_output = K.function([self.model.layers[0].input], [self.model.layers[3].get_output(train=False)]) #layer_output = get_3rd_layer_output[0] print K.get_value(get_3rd_layer_output)
class KerasModel: def __init__(self, data_directory, verbose=False): #Get io file dim_file = "" if data_directory[len(data_directory) - 1] != "/": data_directory = data_directory + "/" for filename in os.listdir(data_directory): if "io_" in filename: dim_file = data_directory + filename if not dim_file: sys.stderr.write("There must be a file with the dimensions " + "of the data, file name must contain 'io'\n") sys.exit(1) self.model = Sequential() self.dim_file = dim_file self.data_dir = data_directory self.v = verbose def train(self, find_feats, nb_layers=None, activation_functs=None, nodes=None, merge_nb_layers=None, merge_activation_functs=None, merge_nodes=None, nb_epochs=None, early_stop=False): #Train model on data #Get dimensions of data dimensions_dictionary = mf.get_dimensions(self.dim_file) array_location = self.data_dir + "numpy_arrays" #set defaults if nb_layers == None: nb_layers = 2 if activation_functs == None: activation_functs = ['relu'] if nodes == None: nodes = [50] if merge_nb_layers == None: merge_nb_layers = 0 if merge_activation_functs == None: merge_activation_functs = ['relu'] if merge_nodes == None: merge_nodes = [50] if nb_epochs == None: nb_epochs = 50 #Get train data if find_feats: train_data, Y_train, feats = mf.getTrainData( array_location, self.v, find_feats) else: train_data, Y_train, feats = mf.getTrainData( array_location, self.v) self.feats = feats self.model = mf.createModel(dimensions_dictionary, feats, self.v, nb_layers, activation_functs, nodes, merge_nb_layers, merge_activation_functs, merge_nodes) early_stopping = EarlyStopping(monitor='val_loss', verbose=1, patience=2) if self.v: sys.stderr.write("fitting model...\n") if early_stop: self.model.fit(train_data, Y_train, callbacks=[early_stopping], nb_epoch=nb_epochs, verbose=self.v, batch_size=1000, validation_split=0.1) else: self.model.fit(train_data, Y_train, nb_epoch=nb_epochs, verbose=self.v, batch_size=1000, validation_split=0.1) #Need to update predict def predict(self, fann_file, fm_file): if self.v: sys.stderr.write("getting prediction from data...\n") #Get prediction data prediction_data, feats = mf.getPredictionData(fann_file, fm_file, self.dim_file, self.v) #need to arrange prediction_data to fit train data form prediction_data = mf.arrangeData(prediction_data, self.feats, feats) prediction = self.model.predict_on_batch(prediction_data) #Transform prediction from one-hot to int max_pos = -1 max_value = 0.0 #Location with largest probability is 1 in one-hot encoding for i in xrange(len(prediction[0])): if max_value < float(prediction[0][i]): max_value = float(prediction[0][i]) max_pos = i mvt = max_pos if mvt >= 0: return mvt else: return "ERROR" def save(self, filename): if self.v: sys.stderr.write("saving model...\n") saved_directory = "saved_models/" if not filename: filename = "trained_model" saved_file = saved_directory + filename model_json = self.model.to_json() with open(saved_file + ".json", "w") as json_file: json_file.write(model_json) self.model.save_weights(saved_file + "_weights.h5", overwrite=True) mf.saveFeats(self.feats, saved_file + "_feats.txt") def load(self, filename): #Load model from saved arch, weights, and features used if self.v: sys.stderr.write("loading model...\n") saved_directory = "./" if not filename: filename = "trained_model" saved_file = saved_directory + filename model_arch = saved_file + ".json" model_weights = saved_file + "_weights.h5" self.feats = mf.loadFeats(saved_file + "_feats.txt") self.model = model_from_json(open(model_arch).read()) self.model.load_weights(model_weights) self.model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) def evaluate(self): #Evaluate test data on model if self.v: sys.stderr.write("evaluating model on test data...\n") #Get test data array_location = self.data_dir + "numpy_arrays" test_data, Y_test, feats = mf.getTestData(array_location, self.v) #need to arrange test_data to fit train data form test_data = mf.arrangeData(test_data, self.feats, feats) scores = self.model.evaluate(test_data, Y_test, verbose=self.v) print("%s: %.2f%%" % (self.model.metrics_names[1], scores[1] * 100)) def graph(self, filename): if not filename: filename = "graph_model.png" if self.v: sys.stderr.write("graphing model...\n") plot(self.model, filename, show_shapes=True)
class SCNN(object): """docstring for CNN""" def __init__(self, conf): self.vs = conf["vocab_size"] self.ml = conf["maxlen"] self.bs = conf["batch_size"] self.ed = conf["embedding_dims"] self.nf = conf["nb_filter"] self.fl = conf["filter_length"] self.hs = conf["hidden_size"] self.ep = conf["nb_epoch"] self.sm = conf.get("save_model", "models/default.scnn") self.lm = conf.get("load_model", "models/default.scnn") self.do = conf.get("dropout",0.2) self.model = Sequential() def build_net(self): word_model_1 = Sequential() word_model_2 = Sequential() pos_model_1 = Sequential() pos_model_2 = Sequential() model_1 = Sequential() model_2 = Sequential() embedding_1 = Embedding(self.vs, self.ed, input_length=self.ml, dropout=self.do) embedding_p1 = Embedding(44, 10, input_length=self.ml, dropout=self.do) embedding_2 = copy.deepcopy(embedding_1) embedding_p2 = copy.deepcopy(embedding_p1) word_model_1.add(embedding_1) # print model_1.output_shape word_model_1.add(Lambda(ctn, output_shape=(self.ed * 3, ))) pos_model_1.add(embedding_p1) pos_model_1.add(Lambda(ctn, output_shape=(10 * 3, ))) model_1.add(Merge([word_model_1, pos_model_1], mode='concat')) # print model_1.output_shape word_model_2.add(embedding_2) word_model_2.add(Lambda(ctn, output_shape=(self.ed * 3, ))) pos_model_2.add(embedding_p2) pos_model_2.add(Lambda(ctn, output_shape=(10 * 3, ))) model_2.add(Merge([word_model_2, pos_model_2], mode='concat')) self.model.add(Merge([model_1, model_2], mode='concat')) # print self.model.output_shape # print self.model.output_shape self.model.add(Dropout(self.do)) self.model.add(Dense(12)) self.model.add(Activation("softmax")) self.model.compile(loss='categorical_crossentropy', optimizer=Adam( lr=0.1)) print "Network compile completed..." def save(self): f = open(self.sm, "w") f.write(self.model.to_json()) f.close() self.model.save_weights(self.sm+".weights", overwrite=True) def load(self): f = open(self.lm, "r") self.model = model_from_json(f.read()) self.model.load_weights(self.sm+".weights") self.model.compile(loss='categorical_crossentropy', optimizer='adam') print "Network compile completed..." def test(self, vx, vy, vvx=None, vvy=None, v_id=None, v_im_id=None): v_accuracy = 0.0 v_im_accuracy = 0.0 if vvx != None: vv_pred = self.model.predict_on_batch(vvx) for j in xrange(len(vv_pred)): max_vp = np.argmax(vv_pred[j]) v_im_accuracy += (vvy[j][max_vp] > 0) v_im_accuracy /= len(vv_pred) v_pred = self.model.predict_on_batch(vx) for j in xrange(len(v_pred)): max_p = np.argmax(v_pred[j]) if vy[j][max_p] > 0: v_accuracy += 1 else: #print v_id[j], max_p, vy[j] pass v_accuracy /= len(v_pred) print "{0} valid-accuracy {1}, valid_im-accuracy {2}".format("Testing", v_accuracy, v_im_accuracy) def train(self, x, y, vx, vy, vvx=None, vvy=None, v_id=None, v_im_id=None): try: print "Begin to train ... training set: {0}, validation set: {1}".format(x[0].shape[0], vx[0].shape[0]) except: print "Begin to train ... training set: {0}, validation set: {1}".format(x[0][0].shape[0], vx[0][0].shape[0]) if vvx != None: try: print "Impilicit validation set: {0}".format(vvx[0].shape[0]) except: print "Impilicit validation set: {0}".format(vvx[0][0].shape[0]) ep = 0 max_accuracy = 0 while ep < self.ep: loss = 0 cnt = 0 accuracy = 0.0 v_accuracy = 0.0 v_im_accuracy = 0.0 num_of_batch = int(len(y)/self.bs) idx_move = num_of_batch / 60 for i in xrange(0, len(y), self.bs): x_ = [x[0][i:i+self.bs], x[1][i:i+self.bs], x[2][i:i+self.bs], x[3][i:i+self.bs]] y_ = y[i:i+self.bs] loss_ = self.model.train_on_batch(x_, y_) pred_ = self.model.predict_on_batch(x_) acc_ = 0.0 for j in xrange(len(pred_)): max_p = np.argmax(pred_[j]) correct = 0 acc_ += (y_[j][max_p] > 0) acc_ /= len(pred_) accuracy += acc_ # print acc_ loss += loss_ cnt += 1 if cnt % idx_move == 0: sys.stderr.write("=>\b") sys.stderr.flush() print ">" if vvx != None: vv_pred = self.model.predict_on_batch(vvx) for j in xrange(len(vv_pred)): max_vp = np.argmax(vv_pred[j]) v_im_accuracy += (vvy[j][max_vp] > 0) v_im_accuracy /= len(vv_pred) v_pred = self.model.predict_on_batch(vx) for j in xrange(len(v_pred)): max_p = np.argmax(v_pred[j]) if vy[j][max_p] > 0: v_accuracy += 1 else: #print v_id[j], v_pred[j], vy[j] pass v_accuracy /= len(v_pred) ep += 1 print "Epoch {0}, training loss {1}, train-accuracy {2}, valid-accuracy {3}, valid_im-accuracy {4}".format( ep, loss / cnt, accuracy / cnt, v_accuracy, v_im_accuracy) if v_im_accuracy > max_accuracy: print "Model imporved on validation set, save model ..." #self.save() max_accuracy = v_accuracy
# print(Y.shape) if X.shape[1] == 1: continue # bug with X, Y of len 1 model.train_on_batch(X, Y) if s['verbose']: print( '[learning] epoch %i >> %2.2f%%' % (e, (i + 1) * 100. / nsentences), 'completed in %.2f (sec) <<\r' % (time.time() - tic)) sys.stdout.flush() # evaluation // back into the real world : idx -> words predictions_test = [ map(lambda x: idx2label[x], model.predict_on_batch(np.asarray([x])).argmax(2)[0]) for x in test_lex ] groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y ] # result is word,not idx #(function, list) words_test = [map(lambda x: idx2word[x], w) for w in test_lex] predictions_valid = [ map(lambda x: idx2label[x], model.predict_on_batch(np.asarray([x])).argmax(2)[0]) for x in valid_lex ] groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y] words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex] # evaluation // compute the accuracy using conlleval.pl
exit(0) outfile = sys.argv[4] if mode == "synth": print("loading audio input...") data, sample_rate = sf.read(audiofile) data = data[:,0] sf.write("test.wav", data, sample_rate) outbuf = [] print("synthesizing audio...") inp = data[:chunk_size * chunk_history] for i in range(10000): outp = model.predict_on_batch(np.array([inp])) outp = outp[0][0] outbuf += list(outp) #inp = data[i * chunk_size :\ # (i * chunk_size) + (chunk_size * chunk_history)] for j in range(chunk_size * (chunk_history - 1)): inp[j] = inp[j + chunk_size] for j in range(chunk_size): inp[j + (chunk_size * (chunk_history - 1))] = outp[j] print("writing audio output...") sf.write(outfile, np.array(outbuf), sample_rate) elif mode == "test": print("not yet implemented") else:
# recs = [r[-maxlen:] for r in recipes] #print(maxlen, maxWord,len(mh.char_indices),len(mh.word_indices)) Xchar,Xword,Xcon,dummy = helper.getCharAndWordNoState(recipes,contextVec,maxlen,maxWord,mh.char_indices,mh.word_indices,step=1,predict=True) newLength = (Xchar.shape[0])/4 inds = [(newLength*(divind+1))-1 for divind in range(0,batchSize)] #helper.checkExampleWords(Xword[inds[1]],mh.vocab) Xchar = Xchar[inds] Xword = Xword[inds] Xcon = Xcon[inds] preds = model.predict_on_batch([Xchar,Xword,Xcon])[0] for d,pred in enumerate(preds): #print(d,pred) next_index = helper.sample(pred, div[d]) next_char = mh.indices_char[next_index] if recipes[d][-1] != "$": recipes[d] += next_char dropIt = [(r[-1] == '$')*1. for r in recipes] if int(np.sum(dropIt)) == batchSize: break with open("../somerecipesfriend.txt",'a') as f: for d,rec in enumerate(recipes): print("with diversity:",div[d],"\n\n",rec,'\n\n')