def debug(im, conf):
mode = im.mode
width = int(conf['window_width'])
half_width = width >> 1
full_half_width = width * width >> 1
fil = conf['filter']
if mode not in ['L']:
show_error(
'Simulations for this module just supports Gray-scale images, check your images !'
)
if fil not in ['mean', 'mid']:
show_error(
'"filter" just supports "mean" and "mid"m check your conf !')
data_src = im.getdata()
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
if fil == 'mean':
data_res += '1' if mean_filter(
win) >= win[half_width][half_width] else '0'
else:
data_res += '1' if rank_filter(
win, full_half_width) >= win[half_width][half_width] else '0'
return data_res
def create_dat(im, conf):
mode = im.mode
width = int(conf['window_width'])
rank = int(conf['rank'])
if mode not in ['L']:
show_error(
'Simulations for this module just supports Gray-scale images, check your images !'
)
if width not in [3, 5]:
show_error(
'Simulations for this module just supports "window_width" 3 and 5, check your conf !'
)
if rank < 0 or rank > width * width - 1:
show_error(
'"rank" must greater than 0 and less than window * window - 1, check your conf !'
)
xsize, ysize = im.size
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
win.reverse()
for row in win:
row = list(row)
row.reverse()
for p in row:
data_res += color_format(mode, p)
data_res += '\n'
return data_res[:-1]
def transform(im, conf):
mode = im.mode
template = conf['template']
if mode not in ['1']:
show_error('Simulations for this module just supports binary images, check your images !')
for row in template:
if len(template) != len(row):
show_error('every row in "template" must equal to length of template, check your conf !')
if len(template) not in [3, 5]:
show_error('size of "template" must equal to e or 5, check your conf !')
for p in row:
if p not in [0, 1]:
show_error('Elements in "template" must equal to 0 or 1, check your conf !')
template = eval(str(template).replace('1', '255'))
width = len(template)
data_res = []
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res.append(1 if win == template else 0)
im_res = Image.new('1', im.size)
im_res.putdata(data_res)
return im_res
def debug(im, conf):
mode = im.mode
template = conf['template']
ed_mode = conf['mode']
if mode not in ['L']:
show_error('Simulations for this module just supports binary images, check your images !')
if ed_mode not in ['Erosion', 'Dilation']:
show_error('"mode" just supports "Erosion" and "Dilation", check your conf !')
for row in template:
if len(template) != len(row):
show_error('every row in "template" must equal to length of template, check your conf !')
if len(template) not in [3, 5]:
show_error('size of "template" must equal to e or 5, check your conf !')
for p in row:
if p not in [0, 1]:
show_error('Elements in "template" must equal to 0 or 1, check your conf !')
ed_mode = 0 if ed_mode == 'Erosion' else 1
width = len(template)
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
pix = 1
for wy in xrange(width):
for wx in xrange(width):
p_w = w[wy][wx] ^ ed_mode
p_w = p_w | ~mask[wy][wx]
pix = pix & p_w
pix = pix ^ ed_mode
data_res += '%d\n' % pix
return data_res
def create_dat(im, conf):
mode = im.mode
width = int(conf['window_width'])
half_width = width >> 1
full_half_width = width * width >> 1
fil = conf['filter']
if mode not in ['L']:
show_error(
'Simulations for this module just supports Gray-scale images, check your images !'
)
if fil not in ['mean', 'mid']:
show_error(
'"filter" just supports "mean" and "mid"m check your conf !')
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res += '%s\n' % color_format(mode, win[half_width][half_width])
if fil == 'mean':
data_res += '%s\n' % color_format(mode, mean_filter(win))
else:
data_res += '%s\n' % color_format(
mode, rank_filter(win, full_half_width))
return data_res[:-1]
def create_dat(im, conf):
mode = im.mode
template = conf['template']
if mode not in ['1']:
show_error('Simulations for this module just supports binary images, check your images !')
for row in template:
if len(template) != len(row):
show_error('every row in "template" must equal to length of template, check your conf !')
if len(template) not in [3, 5]:
show_error('size of "template" must equal to e or 5, check your conf !')
for p in row:
if p not in [0, 1]:
show_error('Elements in "template" must equal to 0 or 1, check your conf !')
width = len(template)
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
win.reverse()
for row in win:
row = list(row)
row.reverse()
for p in row:
data_res += color_format(mode, p)
data_res += '\n'
return data_res[:-1]
def debug(im, conf):
mode = im.mode
width = int(conf['window_width'])
rank = int(conf['rank'])
if mode not in ['L']:
show_error(
'Simulations for this module just supports Gray-scale images, check your images !'
)
if width not in [3, 5]:
show_error(
'Simulations for this module just supports "window_width" 3 and 5, check your conf !'
)
if rank < 0 or rank > width * width - 1:
show_error(
'"rank" must greater than 0 and less than window * window - 1, check your conf !'
)
data_src = im.getdata()
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res += '%d\n' % rank_filter(win, rank)
return data_res
def transform(im, conf):
mode = im.mode
width = int(conf['window_width'])
rank = int(conf['rank'])
if mode not in ['L']:
show_error(
'Simulations for this module just supports Gray-scale images, check your images !'
)
if width not in [3, 5]:
show_error(
'Simulations for this module just supports "window_width" 3 and 5, check your conf !'
)
if rank < 0 or rank > width * width - 1:
show_error(
'"rank" must greater than 0 and less than window * window - 1, check your conf !'
)
data_res = []
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res.append(rank_filter(win, rank))
im_res = Image.new('L', im.size)
im_res.putdata(data_res)
return im_res
def create_dat(im, conf):
mode = im.mode
width = int(conf['window_width'])
rank = int(conf['rank'])
if mode not in ['L']:
show_error('Simulations for this module just supports Gray-scale images, check your images !')
if width not in [3, 5]:
show_error('Simulations for this module just supports "window_width" 3 and 5, check your conf !')
if rank < 0 or rank > width * width - 1:
show_error('"rank" must greater than 0 and less than window * window - 1, check your conf !')
xsize, ysize = im.size
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
win.reverse()
for row in win:
row = list(row)
row.reverse()
for p in row:
data_res += color_format(mode, p)
data_res += '\n'
return data_res[:-1]
def debug(im, conf):
mode = im.mode
template = conf['template']
if mode not in ['1']:
show_error(
'Simulations for this module just supports binary images, check your images !'
)
for row in template:
if len(template) != len(row):
show_error(
'every row in "template" must equal to length of template, check your conf !'
)
if len(template) not in [3, 5]:
show_error(
'size of "template" must equal to e or 5, check your conf !')
for p in row:
if p not in [0, 1]:
show_error(
'Elements in "template" must equal to 0 or 1, check your conf !'
)
width = len(template)
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res += '%d\n' % 1 if win == template else 0
return data_res
def transform(im, conf):
mode = im.mode
template = conf['template']
if mode not in ['1']:
show_error(
'Simulations for this module just supports binary images, check your images !'
)
for row in template:
if len(template) != len(row):
show_error(
'every row in "template" must equal to length of template, check your conf !'
)
if len(template) not in [3, 5]:
show_error(
'size of "template" must equal to e or 5, check your conf !')
for p in row:
if p not in [0, 1]:
show_error(
'Elements in "template" must equal to 0 or 1, check your conf !'
)
template = eval(str(template).replace('1', '255'))
width = len(template)
data_res = []
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res.append(1 if win == template else 0)
im_res = Image.new('1', im.size)
im_res.putdata(data_res)
return im_res
def transform(im, conf):
mode = im.mode
width = int(conf['window_width'])
half_width = width >> 1
full_half_width = width * width >> 1
fil = conf['filter']
if mode not in ['L']:
show_error(
'Simulations for this module just supports Gray-scale images, check your images !'
)
if fil not in ['mean', 'mid']:
show_error(
'"filter" just supports "mean" and "mid"m check your conf !')
data_res = []
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
if fil == 'mean':
data_res.append(
1 if win[half_width][half_width] >= mean_filter(win) else 0)
else:
data_res.append(1 if win[half_width][half_width] >= rank_filter(
win, full_half_width) else 0)
im_res = Image.new('1', im.size)
im_res.putdata(data_res)
return im_res
def debug(im, conf):
mode = im.mode
width = int(conf['window_width'])
if mode not in ['L']:
show_error('Simulations for this module just supports Gray-scale images, check your images !')
if width not in [3, 5]:
show_error('Simulations for this module just supports "window_width" 3 and 5, check your conf !')
data_src = im.getdata()
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res += '%d\n' % mean_fitter(win)
return data_res
def transform(im, conf):
mode = im.mode
width = int(conf['window_width'])
if mode not in ['L']:
show_error('Simulations for this module just supports Gray-scale images, check your images !')
if width not in [3, 5]:
show_error('Simulations for this module just supports "window_width" 3 and 5, check your conf !')
data_res = []
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res.append(mean_filter(win))
im_res = Image.new('L', im.size)
im_res.putdata(data_res)
return im_res
def debug(im, conf):
mode = im.mode
width = int(conf['window_width'])
rank = int(conf['rank'])
if mode not in ['L']:
show_error('Simulations for this module just supports Gray-scale images, check your images !')
if width not in [3, 5]:
show_error('Simulations for this module just supports "window_width" 3 and 5, check your conf !')
if rank < 0 or rank > width * width - 1:
show_error('"rank" must greater than 0 and less than window * window - 1, check your conf !')
data_src = im.getdata()
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res += '%d\n' % rank_filter(win, rank)
return data_res
def debug(im, conf):
mode = im.mode
width = int(conf['window_width'])
if mode not in ['L']:
show_error(
'Simulations for this module just supports Gray-scale images, check your images !'
)
if width not in [3, 5]:
show_error(
'Simulations for this module just supports "window_width" 3 and 5, check your conf !'
)
data_src = im.getdata()
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res += '%d\n' % mean_fitter(win)
return data_res
def transform(im, conf):
mode = im.mode
width = int(conf['window_width'])
if mode not in ['L']:
show_error(
'Simulations for this module just supports Gray-scale images, check your images !'
)
if width not in [3, 5]:
show_error(
'Simulations for this module just supports "window_width" 3 and 5, check your images !'
)
data_res = []
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res.append(mean_filter(win))
im_res = Image.new('L', im.size)
im_res.putdata(data_res)
return im_res
def debug(im, conf):
mode = im.mode
xsize = im.size[0]
if mode not in ['L', '1']:
show_error('Simulations for this module just supports Gray-scale and binary images, check your images !')
if xsize != 512:
show_error('Simulations for this module just supports 512xN images, check your images !')
if conf['width'] not in [3, 5]:
show_error('''Simulations for this module just supports conf "width" 3 and 5, check your images !''')
rows = RG(im, conf['width'])
window = WG(conf['width'])
data_res = ''
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
for row in win:
row = str(row)
if mode == '1':
row = row.replace('255', '1')
data_res += '%s\n' % row.replace('[', '').replace(']', '').replace(',', '')
data_res += '\n'
return data_res[:-1]
def debug(im, conf):
mode = im.mode
template = conf['template']
if mode not in ['1']:
show_error('Simulations for this module just supports binary images, check your images !')
for row in template:
if len(template) != len(row):
show_error('every row in "template" must equal to length of template, check your conf !')
if len(template) not in [3, 5]:
show_error('size of "template" must equal to e or 5, check your conf !')
for p in row:
if p not in [0, 1]:
show_error('Elements in "template" must equal to 0 or 1, check your conf !')
width = len(template)
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res += '%d\n' % 1 if win == template else 0
return data_res
def create_dat(im, conf):
mode = im.mode
width = int(conf['window_width'])
half_width = width >> 1
full_half_width = width * width >> 1
fil = conf['filter']
if mode not in ['L']:
show_error('Simulations for this module just supports Gray-scale images, check your images !')
if fil not in ['mean', 'mid']:
show_error('"filter" just supports "mean" and "mid"m check your conf !')
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
data_res += '%s\n' % color_format(mode, win[half_width][half_width])
if fil == 'mean':
data_res += '%s\n' % color_format(mode, mean_filter(win))
else:
data_res += '%s\n' % color_format(mode, rank_filter(win, full_half_width))
return data_res[:-1]
def debug(im, conf):
mode = im.mode
width = int(conf['window_width'])
half_width = width >> 1
full_half_width = width * width >> 1
fil = conf['filter']
if mode not in ['L']:
show_error('Simulations for this module just supports Gray-scale images, check your images !')
if fil not in ['mean', 'mid']:
show_error('"filter" just supports "mean" and "mid"m check your conf !')
data_src = im.getdata()
data_res = ''
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
if fil == 'mean':
data_res += '1' if mean_filter(win) >= win[half_width][half_width] else '0'
else:
data_res += '1' if rank_filter(win, full_half_width) >= win[half_width][half_width] else '0'
return data_res
def transform(im, conf):
mode = im.mode
template = conf['template']
ed_mode = conf['mode']
if mode not in ['1']:
show_error('Simulations for this module just supports binary images, check your images !')
if ed_mode not in ['Erosion', 'Dilation']:
show_error('"mode" just supports "Erosion" and "Dilation", check your conf !')
for row in template:
if len(template) != len(row):
show_error('every row in "template" must equal to length of template, check your conf !')
if len(template) not in [3, 5]:
show_error('size of "template" must equal to e or 5, check your conf !')
for p in row:
if p not in [0, 1]:
show_error('Elements in "template" must equal to 0 or 1, check your conf !')
ed_mode = 0 if ed_mode == 'Erosion' else 1
width = len(template)
data_res = []
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
pix = 1
for wy in xrange(width):
for wx in xrange(width):
w = 0 if win[wy][wx] == 0 else 1
p_w = w ^ ed_mode
p_w = p_w | ~template[wy][wx]
pix = pix & p_w
pix = pix ^ ed_mode
data_res.append(pix)
im_res = Image.new('1', im.size)
im_res.putdata(data_res)
return im_res
def transform(im, conf):
mode = im.mode
width = int(conf['window_width'])
half_width = width >> 1
full_half_width = width * width >> 1
fil = conf['filter']
if mode not in ['L']:
show_error('Simulations for this module just supports Gray-scale images, check your images !')
if fil not in ['mean', 'mid']:
show_error('"filter" just supports "mean" and "mid"m check your conf !')
data_res = []
rows = RG(im, width)
window = WG(width)
while not rows.frame_empty():
win = window.update(rows.update())
if not window.is_enable():
continue
if fil == 'mean':
data_res.append(1 if win[half_width][half_width] >= mean_filter(win) else 0)
else:
data_res.append(1 if win[half_width][half_width] >= rank_filter(win, full_half_width) else 0)
im_res = Image.new('1', im.size)
im_res.putdata(data_res)
return im_res
def predict(self):
request_data_df = pd.DataFrame(self.request_data['seriesData'], columns=['Column1'])
print('\n\nrequest_data_df', request_data_df)
########## SPLIT THE DATA ##########
# We'll use a (70%, 20%, 10%) split for the training, validation, and test sets.
# Note the data is not being randomly shuffled before splitting.
column_indices = {name: i for i, name in enumerate(request_data_df.columns)}
print('\n\ncolumn_indices', column_indices)
n = len(request_data_df)
train_df = request_data_df[0:int(n * self.train_percentage)]
val_df = request_data_df[int(n * self.train_percentage):int(n * (self.train_percentage + self.val_percentage))]
test_df = request_data_df[int(n * (self.train_percentage + self.val_percentage)):]
num_features = request_data_df.shape[1]
########## NORMALIZE TEH DATA ##########
train_mean = train_df.mean()
train_std = train_df.std()
train_df = (train_df - train_mean) / train_std
val_df = (val_df - train_mean) / train_std
test_df = (test_df - train_mean) / train_std
print('\n\nlen(train_df)')
print(len(train_df))
print('\nlen(val_df)')
print(len(val_df))
print('\nlen(test_df)')
print(len(test_df))
# Create a WindowGenerator that will produce batches of the 3h of inputs and, 1h of labels:
# Note that the Window's shift parameter is relative to the end of the two windows.
CONV_WIDTH = 3
conv_window = WindowGenerator(
input_width=CONV_WIDTH,
label_width=1,
shift=1,
label_columns=['Column1'], train_df=train_df, val_df=val_df, test_df=test_df)
print('\n\nConfigure a WindowGenerator object to produce these single-step (input, label) pairs: conv_window')
print(conv_window)
print('\n\n')
conv_model = tf.keras.Sequential([
tf.keras.layers.Conv1D(filters=32,
kernel_size=(CONV_WIDTH,),
activation='relu'),
tf.keras.layers.Dense(units=32, activation='relu'),
tf.keras.layers.Dense(units=1),
])
print("Conv model on `conv_window`")
print('Input shape:', conv_window.example[0].shape)
print('Output shape:', conv_model(conv_window.example[0]).shape)
print('\n\n')
history = self.compile_and_fit(conv_model, conv_window)
result = {'training_data_predictions': [], 'validation_data_predictions': [], 'test_data_predictions': []}
predictions = conv_model.predict(conv_window.train)
for train in predictions:
prediction = (train[0][0]*train_std[0])+train_mean[0]
result['training_data_predictions'].append(prediction)
predictions = conv_model.predict(conv_window.val)
for val in predictions:
prediction = (val[0][0]*train_std[0])+train_mean[0]
result['validation_data_predictions'].append(prediction)
predictions = conv_model.predict(conv_window.test)
for test in predictions:
prediction = (test[0][0]*train_std[0])+train_mean[0]
result['test_data_predictions'].append(prediction)
print('\n\nResult: ', result)
return result
num_features = df.shape[1]
train_mean = train_df.mean()
train_std = train_df.std()
train_df = (train_df - train_mean) / train_std
val_df = (val_df - train_mean) / train_std
test_df = (test_df - train_mean) / train_std
df_std = (df - train_mean) / train_std
df_std = df_std.melt(var_name='Column', value_name='Normalized')
w1 = WindowGenerator(input_width=24,
label_width=1,
shift=24,
train_df=train_df,
val_df=val_df,
test_df=test_df,
label_columns=['T (degC)'])
w2 = WindowGenerator(input_width=6,
label_width=1,
shift=1,
train_df=train_df,
val_df=val_df,
test_df=test_df,
label_columns=['T (degC)'])
# Stack three slices, the length of the total window:
example_window = tf.stack([
np.array(train_df[:w2.total_window_size]),
# df_std = (df - train_mean) / train_std
# df_std = df_std.melt(var_name='Column', value_name='Normalized')
def norm(x):
return (x-df.min()) / (df.max()-df.min())
train_df = norm(train_df)
val_df = norm(val_df)
test_df = norm(test_df)
from WindowGenerator import WindowGenerator
# test of window splitting
w1 = WindowGenerator(input_width=24,label_width=1,shift=24,train_df=train_df, val_df=val_df, test_df=test_df,label_columns=["qps"])
example_window = tf.stack([np.array(train_df[:w1.total_window_size]),
np.array(train_df[100:100+w1.total_window_size]),
np.array(train_df[200:200+w1.total_window_size])])
example_inputs, example_labels = w1.split_window(example_window)
# test of single step models
from models import *
linear = tf.keras.Sequential([
tf.keras.layers.Dense(units=1)
])
# creating different windows here
single_step_window = WindowGenerator(
input_width=1, label_width=1, shift=1,train_df=train_df, val_df=val_df, test_df=test_df,
def predict(self):
request_data_df = pd.DataFrame(self.request_data['seriesData'], columns=['Column1'])
print('\n\nrequest_data_df', request_data_df)
########## SPLIT THE DATA ##########
# We'll use a (70%, 20%, 10%) split for the training, validation, and test sets.
# Note the data is not being randomly shuffled before splitting.
column_indices = {name: i for i, name in enumerate(request_data_df.columns)}
print('\n\ncolumn_indices', column_indices)
n = len(request_data_df)
train_df = request_data_df[0:int(n * self.train_percentage)]
val_df = request_data_df[int(n * self.train_percentage):int(n * (self.train_percentage + self.val_percentage))]
test_df = request_data_df[int(n * (self.train_percentage + self.val_percentage)):]
num_features = request_data_df.shape[1]
########## NORMALIZE TEH DATA ##########
train_mean = train_df.mean()
train_std = train_df.std()
train_df = (train_df - train_mean) / train_std
val_df = (val_df - train_mean) / train_std
test_df = (test_df - train_mean) / train_std
print('\n\nlen(train_df)')
print(len(train_df))
print('\nlen(val_df)')
print(len(val_df))
print('\nlen(test_df)')
print(len(test_df))
# Create a WindowGenerator that will produce batches of the 3h of inputs and, 1h of labels:
# Note that the Window's shift parameter is relative to the end of the two windows.
CONV_WIDTH = self.input_steps
conv_window = WindowGenerator(
input_width=CONV_WIDTH,
label_width=1,
shift=1,
label_columns=['Column1'], train_df=train_df, val_df=val_df, test_df=test_df)
print('\n\nConfigure a WindowGenerator object to produce these single-step (input, label) pairs: conv_window')
print(conv_window)
print('\n\n')
# You could train a dense model on a multiple-input-step window by adding a layers.Flatten as the first layer of the model:
#multi_step_dense = tf.keras.Sequential([
# # Shape: (time, features) => (time*features)
# tf.keras.layers.Flatten(),
# tf.keras.layers.Dense(units=32, activation='relu'),
# tf.keras.layers.Dense(units=32, activation='relu'),
# tf.keras.layers.Dense(units=1),
# # Add back the time dimension.
# # Shape: (outputs) => (1, outputs)
# tf.keras.layers.Reshape([1, -1]),
#])
multi_step_dense = tf.keras.Sequential()
multi_step_dense.add(tf.keras.layers.Flatten())
for layer in self.hidden_layers:
multi_step_dense.add(tf.keras.layers.Dense(units=layer, activation='relu'))
multi_step_dense.add(tf.keras.layers.Dense(units=1))
multi_step_dense.add(tf.keras.layers.Reshape([1, -1]))
# Train the model and evaluate its performance:
print('\n\nTrain the model and evaluate its performance:')
history = self.compile_and_fit(multi_step_dense, conv_window)
print('Input shape:', conv_window.example[0].shape)
print('Output shape:', multi_step_dense(conv_window.example[0]).shape)
print('\n\nsummary: ')
multi_step_dense.summary()
result = {'training_data_predictions': [], 'validation_data_predictions': [], 'test_data_predictions': []}
predictions = multi_step_dense.predict(conv_window.train)
for train in predictions:
prediction = (train[0][0]*train_std[0])+train_mean[0]
result['training_data_predictions'].append(prediction)
predictions = multi_step_dense.predict(conv_window.val)
for val in predictions:
prediction = (val[0][0]*train_std[0])+train_mean[0]
result['validation_data_predictions'].append(prediction)
predictions = multi_step_dense.predict(conv_window.test)
for test in predictions:
prediction = (test[0][0]*train_std[0])+train_mean[0]
result['test_data_predictions'].append(prediction)
print('\n\nResult: ', result)
return result
def predict(self):
request_data_df = pd.DataFrame(self.request_data['seriesData'],
columns=['Column1'])
print('\n\nrequest_data_df', request_data_df)
########## SPLIT THE DATA ##########
# We'll use a (70%, 20%, 10%) split for the training, validation, and test sets.
# Note the data is not being randomly shuffled before splitting.
column_indices = {
name: i
for i, name in enumerate(request_data_df.columns)
}
print('\n\ncolumn_indices', column_indices)
n = len(request_data_df)
train_df = request_data_df[0:int(n * self.train_percentage)]
val_df = request_data_df[int(n * self.train_percentage):int(n * (
self.train_percentage + self.val_percentage))]
test_df = request_data_df[int(n * (self.train_percentage +
self.val_percentage)):]
num_features = request_data_df.shape[1]
########## NORMALIZE TEH DATA ##########
train_mean = train_df.mean()
train_std = train_df.std()
train_df = (train_df - train_mean) / train_std
val_df = (val_df - train_mean) / train_std
test_df = (test_df - train_mean) / train_std
print('\n\nlen(train_df)')
print(len(train_df))
print('\nlen(val_df)')
print(len(val_df))
print('\nlen(test_df)')
print(len(test_df))
################## SINGLE STEPS MODELS ##################
#### Configure a WindowGenerator object to produce these single-step (input, label) pairs:
single_step_window = WindowGenerator(input_width=1,
label_width=1,
shift=1,
label_columns=['Column1'],
train_df=train_df,
val_df=val_df,
test_df=test_df)
print(
'\n\nConfigure a WindowGenerator object to produce these single-step (input, label) pairs: single_step_window'
)
print(single_step_window)
print('\n\n')
########## BASELINE MODEL ##########
########## BASELINE MODEL ##########
########## BASELINE MODEL ##########
########## BASELINE MODEL ##########
baseline = Baseline(label_index=column_indices['Column1'])
baseline.compile(loss=tf.losses.MeanSquaredError(),
metrics=[tf.metrics.MeanAbsoluteError()])
result = {
'training_data_predictions': [],
'validation_data_predictions': [],
'test_data_predictions': []
}
predictions = baseline.predict(single_step_window.train)
for train in predictions:
prediction = (train[0][0] * train_std[0]) + train_mean[0]
result['training_data_predictions'].append(prediction)
predictions = baseline.predict(single_step_window.val)
for val in predictions:
prediction = (val[0][0] * train_std[0]) + train_mean[0]
result['validation_data_predictions'].append(prediction)
predictions = baseline.predict(single_step_window.test)
print('\n\n\npredictions - test')
ii = 0
for test in predictions:
prediction = (test[0][0] * train_std[0]) + train_mean[0]
result['test_data_predictions'].append(prediction)
print('\n\nResult: ', result)
return result
def predict(self):
request_data_df = pd.DataFrame(self.request_data['seriesData'], columns=['Column1'])
print('\n\nrequest_data_df', request_data_df)
########## SPLIT THE DATA ##########
# We'll use a (70%, 20%, 10%) split for the training, validation, and test sets.
# Note the data is not being randomly shuffled before splitting.
column_indices = {name: i for i, name in enumerate(request_data_df.columns)}
print('\n\ncolumn_indices', column_indices)
n = len(request_data_df)
train_df = request_data_df[0:int(n * self.train_percentage)]
val_df = request_data_df[int(n * self.train_percentage):int(n * (self.train_percentage + self.val_percentage))]
test_df = request_data_df[int(n * (self.train_percentage + self.val_percentage)):]
num_features = request_data_df.shape[1]
########## NORMALIZE TEH DATA ##########
train_mean = train_df.mean()
train_std = train_df.std()
train_df = (train_df - train_mean) / train_std
val_df = (val_df - train_mean) / train_std
test_df = (test_df - train_mean) / train_std
print('\n\nlen(train_df)')
print(len(train_df))
print('\nlen(val_df)')
print(len(val_df))
print('\nlen(test_df)')
print(len(test_df))
inputWidth = self.window_size
wide_window_RNN = WindowGenerator(
input_width=inputWidth, label_width=inputWidth, shift=1,
label_columns=['Column1'], train_df=train_df, val_df=val_df, test_df=test_df)
print('\n\nwide_window_RNN')
print(wide_window_RNN)
print('\n\n')
lstm_model = tf.keras.models.Sequential([
# Shape [batch, time, features] => [batch, time, lstm_units]
tf.keras.layers.LSTM(32, return_sequences=True),
# Shape => [batch, time, featuremv s]
tf.keras.layers.Dense(units=1)
])
history = self.compile_and_fit(lstm_model, wide_window_RNN)
print('\nInput shape:', wide_window_RNN.example[0].shape)
print('Output shape:', lstm_model(wide_window_RNN.example[0]).shape)
print('\n\nsummary: ')
lstm_model.summary()
result = {'training_data_predictions': [], 'validation_data_predictions': [], 'test_data_predictions': []}
predictions = lstm_model.predict(wide_window_RNN.train)
for train in predictions:
prediction = (train[0][0]*train_std[0])+train_mean[0]
result['training_data_predictions'].append(prediction)
predictions = lstm_model.predict(wide_window_RNN.val)
for val in predictions:
prediction = (val[0][0]*train_std[0])+train_mean[0]
result['validation_data_predictions'].append(prediction)
predictions = lstm_model.predict(wide_window_RNN.test)
for test in predictions:
prediction = (test[0][0]*train_std[0])+train_mean[0]
result['test_data_predictions'].append(prediction)
print('\n\nResult: ', result)
return result
def predict(self):
request_data_df = pd.DataFrame(self.request_data['seriesData'],
columns=['Column1'])
print('\n\nrequest_data_df', request_data_df)
########## SPLIT THE DATA ##########
# We'll use a (70%, 20%, 10%) split for the training, validation, and test sets.
# Note the data is not being randomly shuffled before splitting.
column_indices = {
name: i
for i, name in enumerate(request_data_df.columns)
}
print('\n\ncolumn_indices', column_indices)
n = len(request_data_df)
train_df = request_data_df[0:int(n * self.train_percentage)]
val_df = request_data_df[int(n * self.train_percentage):int(n * (
self.train_percentage + self.val_percentage))]
test_df = request_data_df[int(n * (self.train_percentage +
self.val_percentage)):]
num_features = request_data_df.shape[1]
########## NORMALIZE TEH DATA ##########
train_mean = train_df.mean()
train_std = train_df.std()
train_df = (train_df - train_mean) / train_std
val_df = (val_df - train_mean) / train_std
test_df = (test_df - train_mean) / train_std
print('\n\nlen(train_df)')
print(len(train_df))
print('\nlen(val_df)')
print(len(val_df))
print('\nlen(test_df)')
print(len(test_df))
#### Configure a WindowGenerator object to produce these single-step (input, label) pairs:
single_step_window = WindowGenerator(input_width=1,
label_width=1,
shift=1,
label_columns=['Column1'],
train_df=train_df,
val_df=val_df,
test_df=test_df)
print(
'\n\nConfigure a WindowGenerator object to produce these single-step (input, label) pairs: single_step_window'
)
print(single_step_window)
print('\n\n')
linear = tf.keras.Sequential(
[tf.keras.layers.Dense(units=1, use_bias=True)])
print('Input shape:', single_step_window.example[0].shape)
print('Output shape:', linear(single_step_window.example[0]).shape)
# Train the model and evaluate its performance:
print('\n\nTrain the model and evaluate its performance:')
history = self.compile_and_fit(linear, single_step_window)
print('\n\nlinear.layers[0].kernel[:,0].numpy(): ',
linear.layers[0].kernel[:, 0].numpy())
result = {
'training_data_predictions': [],
'validation_data_predictions': [],
'test_data_predictions': []
}
predictions = linear.predict(single_step_window.train)
for train in predictions:
prediction = (train[0][0] * train_std[0]) + train_mean[0]
result['training_data_predictions'].append(prediction)
predictions = linear.predict(single_step_window.val)
for val in predictions:
prediction = (val[0][0] * train_std[0]) + train_mean[0]
result['validation_data_predictions'].append(prediction)
predictions = linear.predict(single_step_window.test)
for test in predictions:
prediction = (test[0][0] * train_std[0]) + train_mean[0]
result['test_data_predictions'].append(prediction)
print('\n\nResult: ', result)
return result
########## DATA WINDOWING ##########
# This section focuses on implementing the data windowing so that it can be reused for all of those models.
# The rest of this section defines a WindowGenerator class. This class can:
# 1 - Handle the indexes and offsets as shown in the diagrams above.
# 2 - Split windows of features into a (features, labels) pairs.
# 3 - Plot the content of the resulting windows.
# 4 - Efficiently generate batches of these windows from the training, evaluation, and test data, using tf.data.Datasets.
# Here is code to create the 2 windows shown in the diagrams at the start of this section:
w1 = WindowGenerator(input_width=24,
label_width=1,
shift=24,
label_columns=['ST_'],
train_df=train_df,
val_df=val_df,
test_df=test_df)
print(
'\n\nCreate the 2 windows shown in the diagrams at the start of this section:'
)
print(w1)
print('\n')
w2 = WindowGenerator(input_width=6,
label_width=1,
shift=1,
label_columns=['ST_'],
train_df=train_df,
