def debug(im, conf): mode = im.mode width = int(conf['window_width']) half_width = width >> 1 full_half_width = width * width >> 1 fil = conf['filter'] if mode not in ['L']: show_error( 'Simulations for this module just supports Gray-scale images, check your images !' ) if fil not in ['mean', 'mid']: show_error( '"filter" just supports "mean" and "mid"m check your conf !') data_src = im.getdata() data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue if fil == 'mean': data_res += '1' if mean_filter( win) >= win[half_width][half_width] else '0' else: data_res += '1' if rank_filter( win, full_half_width) >= win[half_width][half_width] else '0' return data_res
def create_dat(im, conf): mode = im.mode width = int(conf['window_width']) rank = int(conf['rank']) if mode not in ['L']: show_error( 'Simulations for this module just supports Gray-scale images, check your images !' ) if width not in [3, 5]: show_error( 'Simulations for this module just supports "window_width" 3 and 5, check your conf !' ) if rank < 0 or rank > width * width - 1: show_error( '"rank" must greater than 0 and less than window * window - 1, check your conf !' ) xsize, ysize = im.size data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue win.reverse() for row in win: row = list(row) row.reverse() for p in row: data_res += color_format(mode, p) data_res += '\n' return data_res[:-1]
def transform(im, conf): mode = im.mode template = conf['template'] if mode not in ['1']: show_error('Simulations for this module just supports binary images, check your images !') for row in template: if len(template) != len(row): show_error('every row in "template" must equal to length of template, check your conf !') if len(template) not in [3, 5]: show_error('size of "template" must equal to e or 5, check your conf !') for p in row: if p not in [0, 1]: show_error('Elements in "template" must equal to 0 or 1, check your conf !') template = eval(str(template).replace('1', '255')) width = len(template) data_res = [] rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res.append(1 if win == template else 0) im_res = Image.new('1', im.size) im_res.putdata(data_res) return im_res
def debug(im, conf): mode = im.mode template = conf['template'] ed_mode = conf['mode'] if mode not in ['L']: show_error('Simulations for this module just supports binary images, check your images !') if ed_mode not in ['Erosion', 'Dilation']: show_error('"mode" just supports "Erosion" and "Dilation", check your conf !') for row in template: if len(template) != len(row): show_error('every row in "template" must equal to length of template, check your conf !') if len(template) not in [3, 5]: show_error('size of "template" must equal to e or 5, check your conf !') for p in row: if p not in [0, 1]: show_error('Elements in "template" must equal to 0 or 1, check your conf !') ed_mode = 0 if ed_mode == 'Erosion' else 1 width = len(template) data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue pix = 1 for wy in xrange(width): for wx in xrange(width): p_w = w[wy][wx] ^ ed_mode p_w = p_w | ~mask[wy][wx] pix = pix & p_w pix = pix ^ ed_mode data_res += '%d\n' % pix return data_res
def create_dat(im, conf): mode = im.mode width = int(conf['window_width']) half_width = width >> 1 full_half_width = width * width >> 1 fil = conf['filter'] if mode not in ['L']: show_error( 'Simulations for this module just supports Gray-scale images, check your images !' ) if fil not in ['mean', 'mid']: show_error( '"filter" just supports "mean" and "mid"m check your conf !') data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res += '%s\n' % color_format(mode, win[half_width][half_width]) if fil == 'mean': data_res += '%s\n' % color_format(mode, mean_filter(win)) else: data_res += '%s\n' % color_format( mode, rank_filter(win, full_half_width)) return data_res[:-1]
def create_dat(im, conf): mode = im.mode template = conf['template'] if mode not in ['1']: show_error('Simulations for this module just supports binary images, check your images !') for row in template: if len(template) != len(row): show_error('every row in "template" must equal to length of template, check your conf !') if len(template) not in [3, 5]: show_error('size of "template" must equal to e or 5, check your conf !') for p in row: if p not in [0, 1]: show_error('Elements in "template" must equal to 0 or 1, check your conf !') width = len(template) data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue win.reverse() for row in win: row = list(row) row.reverse() for p in row: data_res += color_format(mode, p) data_res += '\n' return data_res[:-1]
def debug(im, conf): mode = im.mode width = int(conf['window_width']) rank = int(conf['rank']) if mode not in ['L']: show_error( 'Simulations for this module just supports Gray-scale images, check your images !' ) if width not in [3, 5]: show_error( 'Simulations for this module just supports "window_width" 3 and 5, check your conf !' ) if rank < 0 or rank > width * width - 1: show_error( '"rank" must greater than 0 and less than window * window - 1, check your conf !' ) data_src = im.getdata() data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res += '%d\n' % rank_filter(win, rank) return data_res
def transform(im, conf): mode = im.mode width = int(conf['window_width']) rank = int(conf['rank']) if mode not in ['L']: show_error( 'Simulations for this module just supports Gray-scale images, check your images !' ) if width not in [3, 5]: show_error( 'Simulations for this module just supports "window_width" 3 and 5, check your conf !' ) if rank < 0 or rank > width * width - 1: show_error( '"rank" must greater than 0 and less than window * window - 1, check your conf !' ) data_res = [] rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res.append(rank_filter(win, rank)) im_res = Image.new('L', im.size) im_res.putdata(data_res) return im_res
def create_dat(im, conf): mode = im.mode width = int(conf['window_width']) rank = int(conf['rank']) if mode not in ['L']: show_error('Simulations for this module just supports Gray-scale images, check your images !') if width not in [3, 5]: show_error('Simulations for this module just supports "window_width" 3 and 5, check your conf !') if rank < 0 or rank > width * width - 1: show_error('"rank" must greater than 0 and less than window * window - 1, check your conf !') xsize, ysize = im.size data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue win.reverse() for row in win: row = list(row) row.reverse() for p in row: data_res += color_format(mode, p) data_res += '\n' return data_res[:-1]
def debug(im, conf): mode = im.mode template = conf['template'] if mode not in ['1']: show_error( 'Simulations for this module just supports binary images, check your images !' ) for row in template: if len(template) != len(row): show_error( 'every row in "template" must equal to length of template, check your conf !' ) if len(template) not in [3, 5]: show_error( 'size of "template" must equal to e or 5, check your conf !') for p in row: if p not in [0, 1]: show_error( 'Elements in "template" must equal to 0 or 1, check your conf !' ) width = len(template) data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res += '%d\n' % 1 if win == template else 0 return data_res
def transform(im, conf): mode = im.mode template = conf['template'] if mode not in ['1']: show_error( 'Simulations for this module just supports binary images, check your images !' ) for row in template: if len(template) != len(row): show_error( 'every row in "template" must equal to length of template, check your conf !' ) if len(template) not in [3, 5]: show_error( 'size of "template" must equal to e or 5, check your conf !') for p in row: if p not in [0, 1]: show_error( 'Elements in "template" must equal to 0 or 1, check your conf !' ) template = eval(str(template).replace('1', '255')) width = len(template) data_res = [] rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res.append(1 if win == template else 0) im_res = Image.new('1', im.size) im_res.putdata(data_res) return im_res
def transform(im, conf): mode = im.mode width = int(conf['window_width']) half_width = width >> 1 full_half_width = width * width >> 1 fil = conf['filter'] if mode not in ['L']: show_error( 'Simulations for this module just supports Gray-scale images, check your images !' ) if fil not in ['mean', 'mid']: show_error( '"filter" just supports "mean" and "mid"m check your conf !') data_res = [] rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue if fil == 'mean': data_res.append( 1 if win[half_width][half_width] >= mean_filter(win) else 0) else: data_res.append(1 if win[half_width][half_width] >= rank_filter( win, full_half_width) else 0) im_res = Image.new('1', im.size) im_res.putdata(data_res) return im_res
def debug(im, conf): mode = im.mode width = int(conf['window_width']) if mode not in ['L']: show_error('Simulations for this module just supports Gray-scale images, check your images !') if width not in [3, 5]: show_error('Simulations for this module just supports "window_width" 3 and 5, check your conf !') data_src = im.getdata() data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res += '%d\n' % mean_fitter(win) return data_res
def transform(im, conf): mode = im.mode width = int(conf['window_width']) if mode not in ['L']: show_error('Simulations for this module just supports Gray-scale images, check your images !') if width not in [3, 5]: show_error('Simulations for this module just supports "window_width" 3 and 5, check your conf !') data_res = [] rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res.append(mean_filter(win)) im_res = Image.new('L', im.size) im_res.putdata(data_res) return im_res
def debug(im, conf): mode = im.mode width = int(conf['window_width']) rank = int(conf['rank']) if mode not in ['L']: show_error('Simulations for this module just supports Gray-scale images, check your images !') if width not in [3, 5]: show_error('Simulations for this module just supports "window_width" 3 and 5, check your conf !') if rank < 0 or rank > width * width - 1: show_error('"rank" must greater than 0 and less than window * window - 1, check your conf !') data_src = im.getdata() data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res += '%d\n' % rank_filter(win, rank) return data_res
def debug(im, conf): mode = im.mode width = int(conf['window_width']) if mode not in ['L']: show_error( 'Simulations for this module just supports Gray-scale images, check your images !' ) if width not in [3, 5]: show_error( 'Simulations for this module just supports "window_width" 3 and 5, check your conf !' ) data_src = im.getdata() data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res += '%d\n' % mean_fitter(win) return data_res
def transform(im, conf): mode = im.mode width = int(conf['window_width']) if mode not in ['L']: show_error( 'Simulations for this module just supports Gray-scale images, check your images !' ) if width not in [3, 5]: show_error( 'Simulations for this module just supports "window_width" 3 and 5, check your images !' ) data_res = [] rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res.append(mean_filter(win)) im_res = Image.new('L', im.size) im_res.putdata(data_res) return im_res
def debug(im, conf): mode = im.mode xsize = im.size[0] if mode not in ['L', '1']: show_error('Simulations for this module just supports Gray-scale and binary images, check your images !') if xsize != 512: show_error('Simulations for this module just supports 512xN images, check your images !') if conf['width'] not in [3, 5]: show_error('''Simulations for this module just supports conf "width" 3 and 5, check your images !''') rows = RG(im, conf['width']) window = WG(conf['width']) data_res = '' while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue for row in win: row = str(row) if mode == '1': row = row.replace('255', '1') data_res += '%s\n' % row.replace('[', '').replace(']', '').replace(',', '') data_res += '\n' return data_res[:-1]
def debug(im, conf): mode = im.mode template = conf['template'] if mode not in ['1']: show_error('Simulations for this module just supports binary images, check your images !') for row in template: if len(template) != len(row): show_error('every row in "template" must equal to length of template, check your conf !') if len(template) not in [3, 5]: show_error('size of "template" must equal to e or 5, check your conf !') for p in row: if p not in [0, 1]: show_error('Elements in "template" must equal to 0 or 1, check your conf !') width = len(template) data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res += '%d\n' % 1 if win == template else 0 return data_res
def create_dat(im, conf): mode = im.mode width = int(conf['window_width']) half_width = width >> 1 full_half_width = width * width >> 1 fil = conf['filter'] if mode not in ['L']: show_error('Simulations for this module just supports Gray-scale images, check your images !') if fil not in ['mean', 'mid']: show_error('"filter" just supports "mean" and "mid"m check your conf !') data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue data_res += '%s\n' % color_format(mode, win[half_width][half_width]) if fil == 'mean': data_res += '%s\n' % color_format(mode, mean_filter(win)) else: data_res += '%s\n' % color_format(mode, rank_filter(win, full_half_width)) return data_res[:-1]
def debug(im, conf): mode = im.mode width = int(conf['window_width']) half_width = width >> 1 full_half_width = width * width >> 1 fil = conf['filter'] if mode not in ['L']: show_error('Simulations for this module just supports Gray-scale images, check your images !') if fil not in ['mean', 'mid']: show_error('"filter" just supports "mean" and "mid"m check your conf !') data_src = im.getdata() data_res = '' rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue if fil == 'mean': data_res += '1' if mean_filter(win) >= win[half_width][half_width] else '0' else: data_res += '1' if rank_filter(win, full_half_width) >= win[half_width][half_width] else '0' return data_res
def transform(im, conf): mode = im.mode template = conf['template'] ed_mode = conf['mode'] if mode not in ['1']: show_error('Simulations for this module just supports binary images, check your images !') if ed_mode not in ['Erosion', 'Dilation']: show_error('"mode" just supports "Erosion" and "Dilation", check your conf !') for row in template: if len(template) != len(row): show_error('every row in "template" must equal to length of template, check your conf !') if len(template) not in [3, 5]: show_error('size of "template" must equal to e or 5, check your conf !') for p in row: if p not in [0, 1]: show_error('Elements in "template" must equal to 0 or 1, check your conf !') ed_mode = 0 if ed_mode == 'Erosion' else 1 width = len(template) data_res = [] rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue pix = 1 for wy in xrange(width): for wx in xrange(width): w = 0 if win[wy][wx] == 0 else 1 p_w = w ^ ed_mode p_w = p_w | ~template[wy][wx] pix = pix & p_w pix = pix ^ ed_mode data_res.append(pix) im_res = Image.new('1', im.size) im_res.putdata(data_res) return im_res
def transform(im, conf): mode = im.mode width = int(conf['window_width']) half_width = width >> 1 full_half_width = width * width >> 1 fil = conf['filter'] if mode not in ['L']: show_error('Simulations for this module just supports Gray-scale images, check your images !') if fil not in ['mean', 'mid']: show_error('"filter" just supports "mean" and "mid"m check your conf !') data_res = [] rows = RG(im, width) window = WG(width) while not rows.frame_empty(): win = window.update(rows.update()) if not window.is_enable(): continue if fil == 'mean': data_res.append(1 if win[half_width][half_width] >= mean_filter(win) else 0) else: data_res.append(1 if win[half_width][half_width] >= rank_filter(win, full_half_width) else 0) im_res = Image.new('1', im.size) im_res.putdata(data_res) return im_res
def predict(self): request_data_df = pd.DataFrame(self.request_data['seriesData'], columns=['Column1']) print('\n\nrequest_data_df', request_data_df) ########## SPLIT THE DATA ########## # We'll use a (70%, 20%, 10%) split for the training, validation, and test sets. # Note the data is not being randomly shuffled before splitting. column_indices = {name: i for i, name in enumerate(request_data_df.columns)} print('\n\ncolumn_indices', column_indices) n = len(request_data_df) train_df = request_data_df[0:int(n * self.train_percentage)] val_df = request_data_df[int(n * self.train_percentage):int(n * (self.train_percentage + self.val_percentage))] test_df = request_data_df[int(n * (self.train_percentage + self.val_percentage)):] num_features = request_data_df.shape[1] ########## NORMALIZE TEH DATA ########## train_mean = train_df.mean() train_std = train_df.std() train_df = (train_df - train_mean) / train_std val_df = (val_df - train_mean) / train_std test_df = (test_df - train_mean) / train_std print('\n\nlen(train_df)') print(len(train_df)) print('\nlen(val_df)') print(len(val_df)) print('\nlen(test_df)') print(len(test_df)) # Create a WindowGenerator that will produce batches of the 3h of inputs and, 1h of labels: # Note that the Window's shift parameter is relative to the end of the two windows. CONV_WIDTH = 3 conv_window = WindowGenerator( input_width=CONV_WIDTH, label_width=1, shift=1, label_columns=['Column1'], train_df=train_df, val_df=val_df, test_df=test_df) print('\n\nConfigure a WindowGenerator object to produce these single-step (input, label) pairs: conv_window') print(conv_window) print('\n\n') conv_model = tf.keras.Sequential([ tf.keras.layers.Conv1D(filters=32, kernel_size=(CONV_WIDTH,), activation='relu'), tf.keras.layers.Dense(units=32, activation='relu'), tf.keras.layers.Dense(units=1), ]) print("Conv model on `conv_window`") print('Input shape:', conv_window.example[0].shape) print('Output shape:', conv_model(conv_window.example[0]).shape) print('\n\n') history = self.compile_and_fit(conv_model, conv_window) result = {'training_data_predictions': [], 'validation_data_predictions': [], 'test_data_predictions': []} predictions = conv_model.predict(conv_window.train) for train in predictions: prediction = (train[0][0]*train_std[0])+train_mean[0] result['training_data_predictions'].append(prediction) predictions = conv_model.predict(conv_window.val) for val in predictions: prediction = (val[0][0]*train_std[0])+train_mean[0] result['validation_data_predictions'].append(prediction) predictions = conv_model.predict(conv_window.test) for test in predictions: prediction = (test[0][0]*train_std[0])+train_mean[0] result['test_data_predictions'].append(prediction) print('\n\nResult: ', result) return result
num_features = df.shape[1] train_mean = train_df.mean() train_std = train_df.std() train_df = (train_df - train_mean) / train_std val_df = (val_df - train_mean) / train_std test_df = (test_df - train_mean) / train_std df_std = (df - train_mean) / train_std df_std = df_std.melt(var_name='Column', value_name='Normalized') w1 = WindowGenerator(input_width=24, label_width=1, shift=24, train_df=train_df, val_df=val_df, test_df=test_df, label_columns=['T (degC)']) w2 = WindowGenerator(input_width=6, label_width=1, shift=1, train_df=train_df, val_df=val_df, test_df=test_df, label_columns=['T (degC)']) # Stack three slices, the length of the total window: example_window = tf.stack([ np.array(train_df[:w2.total_window_size]),
# df_std = (df - train_mean) / train_std # df_std = df_std.melt(var_name='Column', value_name='Normalized') def norm(x): return (x-df.min()) / (df.max()-df.min()) train_df = norm(train_df) val_df = norm(val_df) test_df = norm(test_df) from WindowGenerator import WindowGenerator # test of window splitting w1 = WindowGenerator(input_width=24,label_width=1,shift=24,train_df=train_df, val_df=val_df, test_df=test_df,label_columns=["qps"]) example_window = tf.stack([np.array(train_df[:w1.total_window_size]), np.array(train_df[100:100+w1.total_window_size]), np.array(train_df[200:200+w1.total_window_size])]) example_inputs, example_labels = w1.split_window(example_window) # test of single step models from models import * linear = tf.keras.Sequential([ tf.keras.layers.Dense(units=1) ]) # creating different windows here single_step_window = WindowGenerator( input_width=1, label_width=1, shift=1,train_df=train_df, val_df=val_df, test_df=test_df,
def predict(self): request_data_df = pd.DataFrame(self.request_data['seriesData'], columns=['Column1']) print('\n\nrequest_data_df', request_data_df) ########## SPLIT THE DATA ########## # We'll use a (70%, 20%, 10%) split for the training, validation, and test sets. # Note the data is not being randomly shuffled before splitting. column_indices = {name: i for i, name in enumerate(request_data_df.columns)} print('\n\ncolumn_indices', column_indices) n = len(request_data_df) train_df = request_data_df[0:int(n * self.train_percentage)] val_df = request_data_df[int(n * self.train_percentage):int(n * (self.train_percentage + self.val_percentage))] test_df = request_data_df[int(n * (self.train_percentage + self.val_percentage)):] num_features = request_data_df.shape[1] ########## NORMALIZE TEH DATA ########## train_mean = train_df.mean() train_std = train_df.std() train_df = (train_df - train_mean) / train_std val_df = (val_df - train_mean) / train_std test_df = (test_df - train_mean) / train_std print('\n\nlen(train_df)') print(len(train_df)) print('\nlen(val_df)') print(len(val_df)) print('\nlen(test_df)') print(len(test_df)) # Create a WindowGenerator that will produce batches of the 3h of inputs and, 1h of labels: # Note that the Window's shift parameter is relative to the end of the two windows. CONV_WIDTH = self.input_steps conv_window = WindowGenerator( input_width=CONV_WIDTH, label_width=1, shift=1, label_columns=['Column1'], train_df=train_df, val_df=val_df, test_df=test_df) print('\n\nConfigure a WindowGenerator object to produce these single-step (input, label) pairs: conv_window') print(conv_window) print('\n\n') # You could train a dense model on a multiple-input-step window by adding a layers.Flatten as the first layer of the model: #multi_step_dense = tf.keras.Sequential([ # # Shape: (time, features) => (time*features) # tf.keras.layers.Flatten(), # tf.keras.layers.Dense(units=32, activation='relu'), # tf.keras.layers.Dense(units=32, activation='relu'), # tf.keras.layers.Dense(units=1), # # Add back the time dimension. # # Shape: (outputs) => (1, outputs) # tf.keras.layers.Reshape([1, -1]), #]) multi_step_dense = tf.keras.Sequential() multi_step_dense.add(tf.keras.layers.Flatten()) for layer in self.hidden_layers: multi_step_dense.add(tf.keras.layers.Dense(units=layer, activation='relu')) multi_step_dense.add(tf.keras.layers.Dense(units=1)) multi_step_dense.add(tf.keras.layers.Reshape([1, -1])) # Train the model and evaluate its performance: print('\n\nTrain the model and evaluate its performance:') history = self.compile_and_fit(multi_step_dense, conv_window) print('Input shape:', conv_window.example[0].shape) print('Output shape:', multi_step_dense(conv_window.example[0]).shape) print('\n\nsummary: ') multi_step_dense.summary() result = {'training_data_predictions': [], 'validation_data_predictions': [], 'test_data_predictions': []} predictions = multi_step_dense.predict(conv_window.train) for train in predictions: prediction = (train[0][0]*train_std[0])+train_mean[0] result['training_data_predictions'].append(prediction) predictions = multi_step_dense.predict(conv_window.val) for val in predictions: prediction = (val[0][0]*train_std[0])+train_mean[0] result['validation_data_predictions'].append(prediction) predictions = multi_step_dense.predict(conv_window.test) for test in predictions: prediction = (test[0][0]*train_std[0])+train_mean[0] result['test_data_predictions'].append(prediction) print('\n\nResult: ', result) return result
def predict(self): request_data_df = pd.DataFrame(self.request_data['seriesData'], columns=['Column1']) print('\n\nrequest_data_df', request_data_df) ########## SPLIT THE DATA ########## # We'll use a (70%, 20%, 10%) split for the training, validation, and test sets. # Note the data is not being randomly shuffled before splitting. column_indices = { name: i for i, name in enumerate(request_data_df.columns) } print('\n\ncolumn_indices', column_indices) n = len(request_data_df) train_df = request_data_df[0:int(n * self.train_percentage)] val_df = request_data_df[int(n * self.train_percentage):int(n * ( self.train_percentage + self.val_percentage))] test_df = request_data_df[int(n * (self.train_percentage + self.val_percentage)):] num_features = request_data_df.shape[1] ########## NORMALIZE TEH DATA ########## train_mean = train_df.mean() train_std = train_df.std() train_df = (train_df - train_mean) / train_std val_df = (val_df - train_mean) / train_std test_df = (test_df - train_mean) / train_std print('\n\nlen(train_df)') print(len(train_df)) print('\nlen(val_df)') print(len(val_df)) print('\nlen(test_df)') print(len(test_df)) ################## SINGLE STEPS MODELS ################## #### Configure a WindowGenerator object to produce these single-step (input, label) pairs: single_step_window = WindowGenerator(input_width=1, label_width=1, shift=1, label_columns=['Column1'], train_df=train_df, val_df=val_df, test_df=test_df) print( '\n\nConfigure a WindowGenerator object to produce these single-step (input, label) pairs: single_step_window' ) print(single_step_window) print('\n\n') ########## BASELINE MODEL ########## ########## BASELINE MODEL ########## ########## BASELINE MODEL ########## ########## BASELINE MODEL ########## baseline = Baseline(label_index=column_indices['Column1']) baseline.compile(loss=tf.losses.MeanSquaredError(), metrics=[tf.metrics.MeanAbsoluteError()]) result = { 'training_data_predictions': [], 'validation_data_predictions': [], 'test_data_predictions': [] } predictions = baseline.predict(single_step_window.train) for train in predictions: prediction = (train[0][0] * train_std[0]) + train_mean[0] result['training_data_predictions'].append(prediction) predictions = baseline.predict(single_step_window.val) for val in predictions: prediction = (val[0][0] * train_std[0]) + train_mean[0] result['validation_data_predictions'].append(prediction) predictions = baseline.predict(single_step_window.test) print('\n\n\npredictions - test') ii = 0 for test in predictions: prediction = (test[0][0] * train_std[0]) + train_mean[0] result['test_data_predictions'].append(prediction) print('\n\nResult: ', result) return result
def predict(self): request_data_df = pd.DataFrame(self.request_data['seriesData'], columns=['Column1']) print('\n\nrequest_data_df', request_data_df) ########## SPLIT THE DATA ########## # We'll use a (70%, 20%, 10%) split for the training, validation, and test sets. # Note the data is not being randomly shuffled before splitting. column_indices = {name: i for i, name in enumerate(request_data_df.columns)} print('\n\ncolumn_indices', column_indices) n = len(request_data_df) train_df = request_data_df[0:int(n * self.train_percentage)] val_df = request_data_df[int(n * self.train_percentage):int(n * (self.train_percentage + self.val_percentage))] test_df = request_data_df[int(n * (self.train_percentage + self.val_percentage)):] num_features = request_data_df.shape[1] ########## NORMALIZE TEH DATA ########## train_mean = train_df.mean() train_std = train_df.std() train_df = (train_df - train_mean) / train_std val_df = (val_df - train_mean) / train_std test_df = (test_df - train_mean) / train_std print('\n\nlen(train_df)') print(len(train_df)) print('\nlen(val_df)') print(len(val_df)) print('\nlen(test_df)') print(len(test_df)) inputWidth = self.window_size wide_window_RNN = WindowGenerator( input_width=inputWidth, label_width=inputWidth, shift=1, label_columns=['Column1'], train_df=train_df, val_df=val_df, test_df=test_df) print('\n\nwide_window_RNN') print(wide_window_RNN) print('\n\n') lstm_model = tf.keras.models.Sequential([ # Shape [batch, time, features] => [batch, time, lstm_units] tf.keras.layers.LSTM(32, return_sequences=True), # Shape => [batch, time, featuremv s] tf.keras.layers.Dense(units=1) ]) history = self.compile_and_fit(lstm_model, wide_window_RNN) print('\nInput shape:', wide_window_RNN.example[0].shape) print('Output shape:', lstm_model(wide_window_RNN.example[0]).shape) print('\n\nsummary: ') lstm_model.summary() result = {'training_data_predictions': [], 'validation_data_predictions': [], 'test_data_predictions': []} predictions = lstm_model.predict(wide_window_RNN.train) for train in predictions: prediction = (train[0][0]*train_std[0])+train_mean[0] result['training_data_predictions'].append(prediction) predictions = lstm_model.predict(wide_window_RNN.val) for val in predictions: prediction = (val[0][0]*train_std[0])+train_mean[0] result['validation_data_predictions'].append(prediction) predictions = lstm_model.predict(wide_window_RNN.test) for test in predictions: prediction = (test[0][0]*train_std[0])+train_mean[0] result['test_data_predictions'].append(prediction) print('\n\nResult: ', result) return result
def predict(self): request_data_df = pd.DataFrame(self.request_data['seriesData'], columns=['Column1']) print('\n\nrequest_data_df', request_data_df) ########## SPLIT THE DATA ########## # We'll use a (70%, 20%, 10%) split for the training, validation, and test sets. # Note the data is not being randomly shuffled before splitting. column_indices = { name: i for i, name in enumerate(request_data_df.columns) } print('\n\ncolumn_indices', column_indices) n = len(request_data_df) train_df = request_data_df[0:int(n * self.train_percentage)] val_df = request_data_df[int(n * self.train_percentage):int(n * ( self.train_percentage + self.val_percentage))] test_df = request_data_df[int(n * (self.train_percentage + self.val_percentage)):] num_features = request_data_df.shape[1] ########## NORMALIZE TEH DATA ########## train_mean = train_df.mean() train_std = train_df.std() train_df = (train_df - train_mean) / train_std val_df = (val_df - train_mean) / train_std test_df = (test_df - train_mean) / train_std print('\n\nlen(train_df)') print(len(train_df)) print('\nlen(val_df)') print(len(val_df)) print('\nlen(test_df)') print(len(test_df)) #### Configure a WindowGenerator object to produce these single-step (input, label) pairs: single_step_window = WindowGenerator(input_width=1, label_width=1, shift=1, label_columns=['Column1'], train_df=train_df, val_df=val_df, test_df=test_df) print( '\n\nConfigure a WindowGenerator object to produce these single-step (input, label) pairs: single_step_window' ) print(single_step_window) print('\n\n') linear = tf.keras.Sequential( [tf.keras.layers.Dense(units=1, use_bias=True)]) print('Input shape:', single_step_window.example[0].shape) print('Output shape:', linear(single_step_window.example[0]).shape) # Train the model and evaluate its performance: print('\n\nTrain the model and evaluate its performance:') history = self.compile_and_fit(linear, single_step_window) print('\n\nlinear.layers[0].kernel[:,0].numpy(): ', linear.layers[0].kernel[:, 0].numpy()) result = { 'training_data_predictions': [], 'validation_data_predictions': [], 'test_data_predictions': [] } predictions = linear.predict(single_step_window.train) for train in predictions: prediction = (train[0][0] * train_std[0]) + train_mean[0] result['training_data_predictions'].append(prediction) predictions = linear.predict(single_step_window.val) for val in predictions: prediction = (val[0][0] * train_std[0]) + train_mean[0] result['validation_data_predictions'].append(prediction) predictions = linear.predict(single_step_window.test) for test in predictions: prediction = (test[0][0] * train_std[0]) + train_mean[0] result['test_data_predictions'].append(prediction) print('\n\nResult: ', result) return result
########## DATA WINDOWING ########## # This section focuses on implementing the data windowing so that it can be reused for all of those models. # The rest of this section defines a WindowGenerator class. This class can: # 1 - Handle the indexes and offsets as shown in the diagrams above. # 2 - Split windows of features into a (features, labels) pairs. # 3 - Plot the content of the resulting windows. # 4 - Efficiently generate batches of these windows from the training, evaluation, and test data, using tf.data.Datasets. # Here is code to create the 2 windows shown in the diagrams at the start of this section: w1 = WindowGenerator(input_width=24, label_width=1, shift=24, label_columns=['ST_'], train_df=train_df, val_df=val_df, test_df=test_df) print( '\n\nCreate the 2 windows shown in the diagrams at the start of this section:' ) print(w1) print('\n') w2 = WindowGenerator(input_width=6, label_width=1, shift=1, label_columns=['ST_'], train_df=train_df,