def test__LPPLLayer(self): """given""" model = Sequential([LPPLLayer()]) model.compile(loss='mse', optimizer=SGD(0.2, 0.01)) #model.compile(loss='mse', optimizer='adam') x = np.log(df["Close"].values) x = ReScaler((x.min(), x.max()), (1, 2))(x) x = x.reshape(1, -1) x2 = np.vstack([x, x]) """when""" model.fit(x2, x2, epochs=5000, verbose=0, callbacks=[EarlyStopping('loss')]) """then""" print(model.predict_on_batch(x)) res = pd.DataFrame({ "close": x[0], "lppl": model.predict_on_batch(x) }, index=df.index) res.to_csv('/tmp/lppl.csv') print(res.head()) print(model.layers[0].get_weights())
def glove(self, embedding_index, vtr_dim): # return GloVe embedding texts embedding_matrix = np.zeros((self.vocab_size + 1, vtr_dim)) for word, i in self.tokenizer.word_index.iteritems(): embedding_vector = embedding_index.get(word) if embedding_vector is not None: # words not found in embedding index will be all-zeros. embedding_matrix[i] = embedding_vector model = Sequential() model.add( Embedding(self.vocab_size + 1, output_dim=vtr_dim, weights=[embedding_matrix], trainable=False)) model.compile('rmsprop', 'mse') try: encoded_tweets = np.asarray( pad_sequences(self.tokenizer.texts_to_sequences( self.processed_texts), maxlen=140, padding='post')) except: encoded_tweets = np.asarray( pad_sequences(self.tokenizer.texts_to_sequences(self.df.tweet), maxlen=140, padding='post')) embedding_texts = model.predict_on_batch(encoded_tweets) return embedding_texts
def __test__LinearRegressionLayer(self): """given""" df = DF_TEST.copy() model = Sequential() model.add(LinearRegressionLayer()) model.compile(loss='mse', optimizer='nadam') x = df["Close"].values.reshape(1, -1) """when""" model.fit(x, x, epochs=500, verbose=0) "then" res = pd.DataFrame({"close": df["Close"], "reg": model.predict_on_batch(x)}, index=df.index) print(res.head())
class Discriminator: def __init__(self, expert_data: np.ndarray): self.expert_data = expert_data self.num_samples = expert_data.shape[0] val_std = np.std(self.expert_data) val_avg = np.average(self.expert_data) self.normalize = lambda trajectories: (trajectories - val_avg ) / val_std self.discriminator = Sequential() self.discriminator.add( layers.Flatten(input_shape=expert_data[0].shape)) self.discriminator.add(layers.Dense(32)) self.discriminator.add(layers.LeakyReLU(alpha=0.4)) self.discriminator.add(layers.Dense(32)) self.discriminator.add(layers.LeakyReLU(alpha=0.2)) self.discriminator.add(layers.Dense(1, activation='sigmoid')) self.discriminator.summary() # self.discriminator.compile( # optimizer=tf.train.AdamOptimizer(), # loss='sparse_categorical_crossentropy', # metrics=['accuracy'], # ) optimizer = Adam(0.0002, 0.5) self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) def pretrain_discriminator(self): pass # the discriminator should at least be able to distinguish random noise batch_size = 32 for i in range(4): expert_trajectories = self.sample_expert_data(batch_size) noise = np.random.normal(0.0, 1.0, size=expert_trajectories.size).reshape( expert_trajectories.shape) x_train = np.concatenate((noise, expert_trajectories)) y_train = np.concatenate( (np.zeros(batch_size), np.ones(batch_size))) loss = self.discriminator.train_on_batch(x_train, y_train) if i % 8 == 0: print("pretrain loss: {}".format(loss)) def predict(self, trajectory: np.ndarray): inputs = self.normalize(np.array([trajectory])) return self.discriminator.predict_on_batch(inputs)[0][0] def sample_expert_data(self, n: int): ids = np.random.randint(0, self.num_samples, size=n) return self.normalize(self.expert_data[ids]) def train(self, trajectories: np.ndarray, expert_trajectories: tp.Optional[np.ndarray] = None): trajectories = self.normalize(trajectories) n_trajectories = len(trajectories) if expert_trajectories is None: expert_trajectories = self.sample_expert_data(n_trajectories) else: assert n_trajectories == len(expert_trajectories) x_train = np.concatenate((trajectories, expert_trajectories)) y_train = np.concatenate( (np.zeros(n_trajectories), np.ones(n_trajectories))) return self.discriminator.train_on_batch(x_train, y_train)
import numpy as np from keras import Sequential from keras.layers import BatchNormalization bn = BatchNormalization(input_shape=(28, 28, 3), epsilon=0) model = Sequential([bn]) model.compile(optimizer='rmsprop', loss='mse') n_filters = 3 new_weights = [ np.ones(n_filters, dtype=np.float32), np.zeros(n_filters, dtype=np.float32), np.zeros(n_filters, dtype=np.float32), np.ones(n_filters, dtype=np.float32) ] bn.set_weights(new_weights) x_train = np.random.rand(2, 28, 28, 3) output = model.predict_on_batch(x_train) print(x_train.shape) print(output.shape) print(np.sum(np.abs(x_train - output)))
class GAN: def __init__(self): self.batch_size = 32 self.log_step = 50 self.scaler = MinMaxScaler((-1, 1)) self.data = self.get_data_banknotes() self.init_model() # Logging loss self.logs_loss = pd.DataFrame(columns=['d_train_r', # real data from discriminator training 'd_train_f', # fake data from discriminator training 'd_test_r', # real data from discriminator testing 'd_test_f', # fake data from discriminator testing 'a' # data from GAN(adversarial) training ]) # Logging accuracy self.logs_acc = pd.DataFrame(columns=['d_train_r', 'd_train_f', 'd_test_r', 'd_test_f', 'a']) # Logging generated rows self.results = pd.DataFrame(columns=['iteration','variance', 'skewness', 'curtosis', 'entropy', 'prediction']) def get_data_banknotes(self): """ Get data from file :return: """ names = ['variance', 'skewness', 'curtosis', 'entropy', 'class'] dataset = pd.read_csv('data/data_banknotes.csv', names=names) dataset = dataset.loc[dataset['class'] == 0].values # only real banknotes, because fake ones will be generated X = dataset[:, :4] # omitting last column, we already know it will be 0 data = self.structure_data(X) return data def scale(self, X): return self.scaler.fit_transform(X) def descale(self, X): return self.scaler.inverse_transform(X) def structure_data(self, X): """ Structure data :param X: :return: """ data_subsets = {'normal': X, 'scaled': self.scale(X)} for subset, data in data_subsets.items(): # splitting each subset on train and test splited_data = train_test_split(data, test_size=0.3, shuffle=True) data_subsets.update({ subset: { 'train': splited_data[0], 'test': splited_data[1]} }) return data_subsets def init_discriminator(self): """ Init trainable discriminator model. Will be used for training and testing itself outside connected GAN model. LeakyReLU activation function, Adam optimizer and Dropout are recommended in GAN papers """ self.D = Sequential() self.D.add(Dense(16, input_dim=4)) self.D.add(LeakyReLU()) self.D.add(Dropout(0.3)) self.D.add(Dense(16)) self.D.add(LeakyReLU()) self.D.add(Dense(16)) self.D.add(LeakyReLU()) self.D.add(Dense(1, activation='sigmoid')) self.D.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) def init_discriminator_G(self): """ Init non-trainable discriminator model. Will be used for training generator inside connected GAN model. LeakyReLU activation function, Adam optimizer and Dropout are recommended in GAN papers """ self.Dg = Sequential() self.Dg.add(Dense(16, input_dim=4)) # activation function: ganhacks self.Dg.add(LeakyReLU()) self.Dg.add(Dropout(0.3)) self.Dg.add(Dense(16)) self.Dg.add(LeakyReLU()) self.Dg.add(Dense(16)) self.Dg.add(LeakyReLU()) # activation function: ganhacks self.Dg.add(Dense(1, activation='sigmoid')) self.Dg.trainable = False self.Dg.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) def init_generator(self): """ LeakyReLU activation function, Adam optimizer and Dropout are recommended in GAN papers for BOTH D and G """ self.G = Sequential() self.G.add(Dense(16, input_dim=64)) self.G.add(LeakyReLU()) self.G.add(Dropout(0.3)) self.G.add(Dense(16)) self.G.add(LeakyReLU()) self.G.add(GaussianNoise(0.1)) self.G.add(Dense(16)) self.G.add(LeakyReLU()) self.G.add(Dense(4, activation='tanh')) self.G.compile(loss='binary_crossentropy', optimizer='adam') def init_model(self): """ Connecting non trainable model with Generator. Initializing D. :return: """ self.init_discriminator() self.init_discriminator_G() self.init_generator() self.GAN = Sequential() self.GAN.add(self.G) self.GAN.add(self.Dg) self.GAN.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) def get_adversarial_data(self, mode='train'): """ Get data for adversarial training. """ data = self.data['scaled'][mode].copy() np.random.shuffle(data) features_real = data[:int(self.batch_size / 2)] # random rows with real data noise = np.random.uniform(-1.0, 1.0, size=[int(self.batch_size / 2), 64]) # random noise for generator features_fake = self.G.predict(noise) # fake data y_real = np.zeros([int(self.batch_size / 2), 1]) # array of zeros for real rows labels y_fake = np.ones([int(self.batch_size / 2), 1]) # array of ones for fake rows labels return features_real, y_real, features_fake, y_fake def train(self, train_steps): try: for i in range(train_steps): # Training D xr, yr, xf, yf = self.get_adversarial_data() # train D separately from G d_loss_r = self.D.train_on_batch(xr, yr) # separating real and fake data is recommended d_loss_f = self.D.train_on_batch(xf, yf) # Training G # flipping the label before prediction will # not influence D prediction as here D is not trainable and is getting weights from trainable D y = np.zeros([int(self.batch_size / 2), 1]) # flipping labels is recommended self.Dg.set_weights(self.D.get_weights()) # Copying weights from trainable D noise = np.random.uniform(-1.0, 1.0, size=[int(self.batch_size / 2), 64]) # getting input noise for G a_loss = self.GAN.train_on_batch(noise, y) # Testing xr_t, yr_t, xf_t, yf_t = self.get_adversarial_data(mode='test') d_pred_r = self.D.predict_on_batch(xr_t) # getting example predictions d_pred_f = self.D.predict_on_batch(xf_t) d_loss_r_t = self.D.test_on_batch(xr_t, yr_t) # getting loss and acc d_loss_f_t = self.D.test_on_batch(xf_t, yf_t) # Logging important data self.log(locals()) finally: """ Plot and save data when finished. """ self.plot() self.results.to_csv('results/results.csv', index=False) def plot(self): """ Preparing for plotting, plotting and saving plots. """ import matplotlib.pyplot as plt ax_loss = self.logs_loss.plot(linewidth=0.75, figsize=(20, 10)) ax_loss.set_xlabel('iteration') ax_loss.set_ylabel('loss') fig = plt.gcf() fig.set_dpi(200) plt.legend(loc='upper right', framealpha=0, prop={'size': 'large'}) fig.savefig('results/loss.png', dpi=200) ax_acc = self.logs_acc.plot(linewidth=0.75, figsize=(20, 10)) ax_acc.set_xlabel('iteration') ax_acc.set_ylabel('accuracy') fig = plt.gcf() fig.set_dpi(200) plt.legend(loc='upper right', framealpha=0, prop={'size': 'large'}) fig.savefig('results/acc.png', dpi=200) plt.show() def log(self, variables): """ Logging and printing all the necessary data """ r_rows = pd.DataFrame(self.descale(variables['xr_t']), columns=['variance', 'skewness', 'curtosis', 'entropy']) r_rows['prediction'] = variables['d_pred_r'] f_rows = pd.DataFrame(self.descale(variables['xf_t']), columns=['variance', 'skewness', 'curtosis', 'entropy']) f_rows['prediction'] = variables['d_pred_f'] f_rows['iteration'] = variables['i'] self.logs_loss = self.logs_loss.append(pd.Series( # logging loss [variables['d_loss_r'][0], variables['d_loss_f'][0], variables['d_loss_r_t'][0], variables['d_loss_f_t'][0], variables['a_loss'][0]], index=self.logs_loss.columns), ignore_index=True) self.logs_acc = self.logs_acc.append(pd.Series( # logging acc [variables['d_loss_r'][1], variables['d_loss_f'][1], variables['d_loss_r_t'][1], variables['d_loss_f_t'][1], variables['a_loss'][1]], index=self.logs_loss.columns), ignore_index=True) self.results = self.results.append(f_rows, ignore_index=True, sort=False) # logging generated data if self.log_step and variables['i'] % self.log_step == 0: # print metrics every 'log_step' iteration # preparing strings for printing log_msg = f""" Batch {variables['i']}: D(training): loss: real : {variables['d_loss_r'][0]:.4f} fake : {variables['d_loss_f'][0]:.4f} acc: real: {variables['d_loss_r'][1]:.4f} fake: {variables['d_loss_f'][1]:.4f} D(testing): loss: real : {variables['d_loss_r_t'][0]:.4f} fake : {variables['d_loss_f_t'][0]:.4f} acc: real: {variables['d_loss_r_t'][1]:.4f} fake: {variables['d_loss_f_t'][1]:.4f} GAN: loss: {variables['a_loss'][0]:.4f} acc: {variables['a_loss'][1]:.4f} """ print(log_msg) np.set_printoptions(precision=5, linewidth=140, suppress=True) # set how np.array will be printed predictions = f""" Example results: Real rows: {r_rows} Fake rows: {f_rows} """ print(predictions)
def create_model(_rows, data_dict, max_flow): def rmse(y_true, y_pred, axis=0): return np.sqrt(((y_pred - y_true)**2).mean(axis=axis)) def create_dataset_from_dict(ddata, lookback=1, steps=1): dataX = [] dataY = [] for dt, data in ddata.items(): timestep = [] yval = ddata.get(dt + timedelta(minutes=5 * steps)) # check the future value exists and is not an error if yval is not None: for j in range(lookback): offset = dt - timedelta(minutes=5 * steps * j) # make sure we have all previous values in the lookback if ddata.get(offset) is not None: timestep.append(ddata[offset]) if len(timestep) == lookback: dataX.append(timestep) dataY.append(yval[0]) fields = len(dataX[0][0]) return np.array(dataX, dtype=np.double), np.array(dataY, dtype=np.double), fields def fit_to_batch(arr, b_size): lim = len(arr) - (len(arr) % b_size) return arr[:lim] class TerminateOnNaN(Callback): """Callback that terminates training when a NaN loss is encountered. """ def __init__(self): super(TerminateOnNaN, self).__init__() self.terminated = False def on_batch_end(self, batch, logs=None): logs = logs or {} loss = logs.get('loss') if loss is not None: if np.isnan(loss) or np.isinf(loss): print('Batch %d: Invalid loss, terminating training' % (batch)) self.model.stop_training = True self.terminated = True # input fields are: """ Input is: [ flow dayOfWeek MinuteOfDay month week isWeekend ] for `lookback` records """ lookback = int({{quniform(1, 40, 1)}}) scaler = MinMaxScaler((0, 1)) # rows = scaler.fit_transform(_rows) # dataX, dataY, fields = create_dataset(rows, lookback) scaled = scaler.fit_transform(list(data_dict.values())) scaled_data_dict = dict(zip(data_dict.keys(), scaled)) dataX, dataY, fields = create_dataset_from_dict(scaled_data_dict, lookback) test_train_split = 0.60 ## 60% training 40% test split_idx = int(len(dataX) * test_train_split) train_x = dataX[:split_idx] train_y = dataY[:split_idx] test_x = dataX[split_idx:] test_y = dataY[split_idx:] batch_size = int({{quniform(1, 5, 1)}}) train_x = fit_to_batch(train_x, batch_size) train_y = fit_to_batch(train_y, batch_size) test_x = fit_to_batch(test_x, batch_size) test_y = fit_to_batch(test_y, batch_size) nb_epoch = 1 lstm_size_1 = {{quniform(96, 300, 4)}} lstm_size_2 = {{quniform(96, 300, 4)}} lstm_size_3 = {{quniform(69, 300, 4)}} optimizer = {{choice(['adam', 'rmsprop'])} } # 'nadam', 'adamax', 'adadelta', 'adagrad'])}} l1_dropout = {{uniform(0.001, 0.7)}} l2_dropout = {{uniform(0.001, 0.7)}} l3_dropout = {{uniform(0.001, 0.7)}} output_activation = {{choice(['relu', 'tanh', 'linear'])}} # reset_interval = int({{quniform(1, 100, 1)}}) # layer_count = {{choice([1, 2, 3])}} l1_reg = {{uniform(0.0001, 0.1)}} l2_reg = {{uniform(0.0001, 0.1)}} params = { 'batch_size': batch_size, 'lookback': lookback, 'lstm_size_1': lstm_size_1, 'lstm_size_2': lstm_size_2, 'lstm_size_3': lstm_size_3, 'l1_dropout': l1_dropout, 'l2_dropout': l2_dropout, 'l3_dropout': l3_dropout, 'l1_reg': l1_reg, 'l2_reg': l2_reg, 'optimizer': optimizer, 'output_activation': output_activation, # 'state_reset': reset_interval, # 'layer_count': layer_count, # 'use_embedding': use_embedding } print("PARAMS=", json.dumps(params, indent=4)) def krmse(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) def geh(y_true, y_pred): return K.sqrt(2 * K.pow(y_pred - y_true, 2) / (y_pred + y_true)).mean(axis=-1) reg = L1L2(l1_reg, l2_reg) start = datetime.now() model = Sequential() # if conditional(use_embedding): # model.add(Embedding()) model.add( LSTM(int(lstm_size_1), batch_input_shape=(batch_size, lookback, fields), return_sequences=True, stateful=True, activity_regularizer=reg, bias_initializer='ones')) model.add(Dropout(l1_dropout)) model.add(Activation('relu')) model.add( LSTM(int(lstm_size_2), return_sequences=True, bias_initializer='ones', stateful=True, activity_regularizer=reg)) model.add(Dropout(l2_dropout)) model.add(Activation('relu')) model.add( LSTM(int(lstm_size_3), bias_initializer='ones', stateful=True, activity_regularizer=reg)) model.add(Dropout(l3_dropout)) model.add(Activation('relu')) model.add(Dense(1, activation='relu')) terminate_cb = TerminateOnNaN() model.compile(loss='mse', optimizer=optimizer) try: model.fit( train_x, train_y, epochs=1, verbose=1, batch_size=batch_size, shuffle=False, callbacks=[terminate_cb], ) except Exception as e: print(e) return {'status': STATUS_FAIL, 'msg': e} if terminate_cb.terminated: return {'status': STATUS_FAIL, 'msg': "Invalid loss"} # have it continue learning during this phase # split the test_x,test_y preds = [] def group(iterable, n): it = iter(iterable) while True: chunk = tuple(itertools.islice(it, n)) if not chunk: return yield chunk test_y_it = iter(group(test_y, batch_size)) test_batch_idx = 0 prog = tqdm(range(len(test_y / batch_size)), desc='Train ') for batch in group(test_x, batch_size): batch = np.array(batch) test_y_batch = np.array(next(test_y_it)) model.train_on_batch(batch, test_y_batch) batch_preds = model.predict_on_batch(batch)[:, 0] preds.extend(batch_preds) test_batch_idx += 1 prog.update() # if test_batch_idx % reset_interval == 0: # model.reset_states() preds = np.array(preds) finish = datetime.now() preds_pad = np.zeros((preds.shape[0], fields)) preds_pad[:, 0] = preds.flatten() test_y_pad = np.zeros((preds.shape[0], fields)) test_y_pad[:, 0] = test_y.flatten() unscaled_pred = scaler.inverse_transform(preds_pad) unscaled_test_y = scaler.inverse_transform(test_y_pad) rmse_result = rmse(unscaled_pred, unscaled_test_y)[0] plot_x = np.arange(test_x.shape[0]) dpi = 80 width = 1920 / dpi height = 1080 / dpi plt.figure(figsize=(width, height), dpi=dpi) plt.plot(plot_x, unscaled_test_y[:, 0], color='b', label='Actual') plt.plot(plot_x, unscaled_pred[:, 0], color='r', label='Predictions') plt.legend() plt.title("LSTM Discrete Predictions at 115, SI 2\nRMSE:{}".format( round(rmse_result, 3))) plt.xlabel('Time') plt.ylabel('Flow') fig_name = 'model_{}.png'.format(time()) plt.savefig(fig_name) plt.show() with open(fig_name, 'rb') as img_file: fig_b64 = base64.b64encode(img_file.read()).decode('ascii') return { 'loss': rmse_result, 'status': STATUS_OK, 'model': model._updated_config(), 'metrics': { 'rmse': rmse_result, # 'geh': geh(unscaled_pred, unscaled_test_y)[0], 'duration': (finish - start).total_seconds() }, 'figure': fig_b64, 'params': params }
for i in range(32): plt.subplot(4, 8, i + 1) im = np.reshape(images[i], (1, -1)) im = (np.reshape(im, [28, 28]) + 1) * 255 im = np.clip(im, 0, 255) im = np.uint8(im) plt.imshow(im, cmap='gray') plt.show() epochs = 100 batch_size = 100 xtrain = np.reshape(xtrain, [xtrain.shape[0], -1]) xtest = np.reshape(xtest, [xtest.shape[0], -1]) xtrain = (xtrain.astype(np.float32) - 127.5) / 127.5 plt.imshow(np.reshape(xtrain[0], [28, 28]), cmap="gray") plt.show() for e in range(epochs): for i in tqdm(range(int(xtrain.shape[0] / batch_size))): xreal = xtrain[(i) * batch_size:(i + 1) * batch_size] noise = generateRandomData(batch_size, input_size) xfake = generator.predict_on_batch(noise) discriminator.trainable = True discriminator.train_on_batch(xreal, np.array([[0.9]] * batch_size)) discriminator.train_on_batch(xfake, np.array([[0.]] * batch_size)) discriminator.trainable = False gan.train_on_batch(noise, np.array([[1.]] * batch_size)) showResults(generator)