def main():
path = 'D:\\ydata-labeled-time-series-anomalies-v1_0\\A4Benchmark\\A4Benchmark-TS3.csv'
time_steps = 256
epochs = 200
batch_size = 1
ts = np.genfromtxt(path, delimiter=',', skip_header=1)[:, 1]
data = processing.TimeSeries(ts,
window=time_steps,
window_stride=time_steps,
return_sequences=True,
use_time_diff=False,
scaler=utils.StandardScaler)
model = Model(batch_size=batch_size, time_steps=time_steps, features=1)
model.describe()
for epoch in range(epochs):
losses = []
for t, x, y in data.train_samples_generator(batch_size=batch_size):
loss = model.train(x, y)
losses.append(loss)
print('Epoch {0}/{1} Loss: {2:.3f}'.format(epoch, epochs,
np.mean(losses)))
model.reset_rnn_state()
y_true = []
y_pred = []
t_all = []
for t, x_batch, y_batch in data.all_samples_generator(
batch_size=batch_size):
predicted_part = model.predict(x_batch)
y_pred += list(predicted_part)
y_true += list(y_batch)
t_all += list(t)
y_pred = data.inverse_transform_predictions(np.array(t_all),
np.array(y_pred))
y_true = data.inverse_transform_predictions(np.array(t_all),
np.array(y_true))
import matplotlib.pyplot as plt
plt.plot(y_pred, label='Predicted')
plt.plot(y_true, label='True')
plt.legend()
plt.grid()
plt.show()
def run_for_single_file(path, args, config):
logging.info('Opening: {0}'.format(path))
# Read the time-series from file
raw_ts = read_file(path, args)
if raw_ts.ndim == 1:
raw_ts = np.expand_dims(raw_ts, axis=-1)
epochs = config['epochs']
batch_size = config['batch_size']
scaler = utils.instantiate(config['scaler'])
data = processing.TimeSeries(
raw_ts,
window=config['window_size'],
window_stride=config['window_stride'],
return_sequences=config['return_sequences'],
train_test_split=config['train_test_split'],
num_derivatives=config['num_derivatives'],
deepcast_derivatives=config['deepcast_derivatives'],
scaler=scaler,
use_time_diff=False)
model = utils.instantiate(config['model'], data.input_shape,
**config['model_args']) # type: keras.Model
model.summary()
all_len = batch_size * (len(data.x) // batch_size)
train_len = batch_size * (len(data.x_train) // batch_size)
test_len = batch_size * (len(data.x_test) // batch_size)
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=20,
verbose=0,
mode='auto')
history = model.fit(data.x_train[:train_len],
data.y_train[:train_len],
batch_size=batch_size,
epochs=epochs,
shuffle=True,
verbose=2,
callbacks=[early_stopping],
validation_data=(data.x_test[:test_len],
data.y_test[:test_len]))
if args.weights_out:
model.save_weights(args.weights_out)
# Evaluation
y_true = data.y_test[:test_len]
y_pred = model.predict(data.x_test[:test_len], batch_size=batch_size)
y_true = data.inverse_transform_predictions(data.t_test[:test_len],
y_true)[:, 0]
y_pred = data.inverse_transform_predictions(data.t_test[:test_len],
y_pred)[:, 0]
scores = metrics.evaluate(y_true, y_pred, metrics=config['metrics'])
logging.info('Scores on test data: {0}'.format(scores))
if not args.noplot:
output_dir = os.path.dirname(args.output)
filename = '{0}_{1}.csv'.format(
args.run,
os.path.splitext(os.path.basename(path))[0])
plot_path = os.path.join(output_dir, filename)
# Prediction
y_true = data.y[:all_len]
y_pred = model.predict(data.x[:all_len], batch_size=batch_size)
y_true = data.inverse_transform_predictions(data.t[:all_len],
y_true)[:, 0]
y_pred = data.inverse_transform_predictions(data.t[:all_len],
y_pred)[:, 0]
data = np.vstack((y_true, y_pred)).T
np.savetxt(plot_path,
data,
delimiter=',',
header='True, Predicted',
comments='')
else:
plot_path = ''
if args.output:
results = config.copy()
results.update({
'file': path,
'run': args.run,
'plot': plot_path,
'finished': datetime.datetime.now(),
'epochs_trained': history.epoch[-1],
})
results.update(scores)
utils.write_row_to_csv(args.output, **results)
keras.backend.clear_session()
def main():
path = 'D:\\ydata-labeled-time-series-anomalies-v1_0\\A4Benchmark\\A4Benchmark-TS3.csv'
#path = 'D:\\ydata-labeled-time-series-anomalies-v1_0\\A1Benchmark\\real_38.csv'
#path = 'D:\\NAB-master\\data\\realKnownCause\\machine_temperature_system_failure.csv'
epochs = 10
batch_size = 16
return_sequences = False
train_test_split = 0.6
window_size = 64
window_stride = 1
# Raw time-series
raw_ts = np.genfromtxt(path, delimiter=',', skip_header=1)[:, 1]
#raw_ts = np.sin(np.linspace(0, 100, 1000))
data = processing.TimeSeries(raw_ts,
window=window_size,
window_stride=window_stride,
return_sequences=return_sequences,
train_test_split=0.6,
scaler=None,
use_time_diff=False)
#model = wavenet(data.input_shape, return_sequences=return_sequences)
model = ddcc(data.input_shape,
batch_size=batch_size,
return_sequences=return_sequences)
describe_ddcc_layer(model, data.x_train[:batch_size])
all_len = batch_size * (len(data.x) // batch_size)
train_len = batch_size * (len(data.x_train) // batch_size)
test_len = batch_size * (len(data.x_test) // batch_size)
model.fit(data.x_train[:train_len],
data.y_train[:train_len],
batch_size=batch_size,
epochs=epochs,
shuffle=True,
verbose=1,
callbacks=[
keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=20,
verbose=0,
mode='auto')
],
validation_data=(data.x_test[:test_len], data.y_test[:test_len]))
describe_ddcc_layer(model, data.x_train[:batch_size])
# Evaluation
y_true = data.y_test[:test_len]
y_pred = model.predict(data.x_test[:test_len], batch_size=batch_size)
y_true = data.inverse_transform_predictions(data.t_test[:test_len],
y_true)[:, 0]
y_pred = data.inverse_transform_predictions(data.t_test[:test_len],
y_pred)[:, 0]
from pprint import pprint
scores = metrics.evaluate(y_true, y_pred)
pprint(scores)
# Prediction
y_true = data.y[:all_len]
y_pred = model.predict(data.x[:all_len], batch_size=batch_size)
y_true = data.inverse_transform_predictions(data.t[:all_len], y_true)[:, 0]
y_pred = data.inverse_transform_predictions(data.t[:all_len], y_pred)[:, 0]
import matplotlib.pyplot as plt
plt.plot(y_pred, label='Predicted')
plt.plot(y_true, label='True')
plt.legend()
plt.grid()
plt.show()
"""
y_free_will = []
x = data.x[-1]
for i in range(300):
prediction = model.predict_on_batch(np.expand_dims(x, axis=0))[0, -1]
y_free_will.append(prediction[0])
#x = utils.shift(x, offset=-1, fill_val=np.expand_dims(prediction, axis=0))
x = np.concatenate((x[:-1], np.expand_dims(prediction, axis=0)))
y_free_will = data.inverse_target_scale(np.array(y_free_will))
"""
# Compute metrics
_, test_y = utils.split(y_true, ratio=train_test_split)
_, test_y_pred = utils.split(y_pred, ratio=train_test_split)
import pprint
metric_results = metrics.evaluate_all(test_y, test_y_pred)
pprint.pprint(metric_results)
def main():
path = 'D:\\ydata-labeled-time-series-anomalies-v1_0\\A4Benchmark\\A4Benchmark-TS3.csv'
window = 64
epochs = 100
stop_loss = 0.001
batch_size = 64
is_stateful = True
return_sequences = False
window_stride = 1
use_time_diff = False
train_test_split = 0.6
# Raw time-series
raw_ts = np.genfromtxt(path, delimiter=',', skip_header=1)[:, 1]
data = processing.TimeSeries(raw_ts,
window=window,
window_stride=window_stride,
return_sequences=return_sequences,
train_test_split=train_test_split,
use_time_diff=use_time_diff)
# Create model
model = create_model(input_shape=data.input_shape,
batch_size=batch_size,
stateful=is_stateful,
return_sequences=return_sequences)
model.summary()
describe_deform_layer(model, data.x_train[:batch_size])
model.reset_states()
for epoch in range(epochs):
losses = []
for t, x_batch, y_batch in data.train_samples_generator(batch_size=batch_size, shuffle=True):
loss = model.train_on_batch(x_batch, y_batch)
losses.append(loss)
epoch_loss = np.mean(losses)
print('Epoch {0}/{1} Loss: {2:.4f}'.format(epoch, epochs, epoch_loss))
model.reset_states()
if epoch_loss < stop_loss:
break
describe_deform_layer(model, data.x_train[:batch_size])
y_true = []
y_pred = []
t_all = []
for t, x_batch, y_batch in data.all_samples_generator(batch_size=batch_size):
predicted_part = model.predict_on_batch(x_batch)
y_pred += list(predicted_part)
y_true += list(y_batch)
t_all += list(t)
y_pred = data.inverse_transform_predictions(np.array(t_all), np.array(y_pred))
y_true = data.inverse_transform_predictions(np.array(t_all), np.array(y_true))
plt.plot(y_pred, label='Predicted')
plt.plot(y_true, label='True')
plt.legend()
plt.grid()
plt.show()
# Compute metrics
_, test_y = utils.split(y_true, ratio=train_test_split)
_, test_y_pred = utils.split(y_pred, ratio=train_test_split)
import pprint
metric_results = metrics.evaluate_all(test_y, test_y_pred)
pprint.pprint(metric_results)
def test_time_series_inputs(self):
raw = np.arange(10)
ts = processing.TimeSeries(raw,
window=2,
predict_steps=1,
predict_gap=1,
return_sequences=False,
train_test_split=0.6,
window_stride=1)
desired_x = [
[0, 1],
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
[6, 7],
[7, 8],
]
desired_y = [2, 3, 4, 5, 6, 7, 8, 9]
assert_arrays_equal(ts.x, desired_x)
assert_arrays_equal(ts.y, desired_y)
assert_arrays_equal(ts.t, desired_y)
ts = processing.TimeSeries(raw, window=2, predict_steps=1,
return_sequences=True, train_test_split=0.6, window_stride=1)
desired_x = [
[0, 1],
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
[6, 7],
[7, 8],
]
desired_y = [
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
[6, 7],
[7, 8],
[8, 9]
]
assert_arrays_equal(ts.x, desired_x)
assert_arrays_equal(ts.y, desired_y)
assert_arrays_equal(ts.t, desired_y)
ts = processing.TimeSeries(raw,
window=2,
predict_steps=3,
return_sequences=False,
train_test_split=0.6,
window_stride=1)
desired_x = [
[0, 1],
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
]
desired_y = [
[2, 3, 4],
[3, 4, 5],
[4, 5, 6],
[5, 6, 7],
[6, 7, 8],
[7, 8, 9]
]
assert_arrays_equal(ts.x, desired_x)
assert_arrays_equal(ts.y, desired_y)
assert_arrays_equal(ts.t, desired_y)
ts = processing.TimeSeries(raw,
window=2,
predict_steps=3,
return_sequences=True,
train_test_split=0.6,
window_stride=1)
desired_x = [
[0, 1],
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
]
desired_y = [
[[1, 2, 3], [2, 3, 4]],
[[2, 3, 4], [3, 4, 5]],
[[3, 4, 5], [4, 5, 6]],
[[4, 5, 6], [5, 6, 7]],
[[5, 6, 7], [6, 7, 8]],
[[6, 7, 8], [7, 8, 9]]
]
assert_arrays_equal(ts.x, desired_x)
assert_arrays_equal(ts.y, desired_y)
assert_arrays_equal(ts.t, desired_y)
ts = processing.TimeSeries(raw,
window=2,
predict_steps=1,
predict_gap=2,
return_sequences=False,
train_test_split=0.6,
window_stride=1)
desired_x = [
[0, 1],
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
[6, 7],
]
desired_y = [3, 4, 5, 6, 7, 8, 9]
assert_arrays_equal(ts.x, desired_x)
assert_arrays_equal(ts.y, desired_y)
assert_arrays_equal(ts.t, desired_y)
ts = processing.TimeSeries(raw,
window=2,
predict_steps=2,
predict_gap=2,
return_sequences=False,
train_test_split=0.6,
window_stride=1)
desired_x = [
[0, 1],
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
]
desired_y = [
[3, 4],
[4, 5],
[5, 6],
[6, 7],
[7, 8],
[8, 9]
]
assert_arrays_equal(ts.x, desired_x)
assert_arrays_equal(ts.y, desired_y)
assert_arrays_equal(ts.t, desired_y)
ts = processing.TimeSeries(raw,
window=2,
predict_steps=2,
predict_gap=2,
return_sequences=True,
train_test_split=0.6,
window_stride=1)
desired_x = [
[0, 1],
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
]
desired_y = [
[[2, 3], [3, 4]],
[[3, 4], [4, 5]],
[[4, 5], [5, 6]],
[[5, 6], [6, 7]],
[[6, 7], [7, 8]],
[[7, 8], [8, 9]]
]
assert_arrays_equal(ts.x, desired_x)
assert_arrays_equal(ts.y, desired_y)
assert_arrays_equal(ts.t, desired_y)