def train_predict(self, datax, datay, predict_len):
data = {
tf.contrib.timeseries.TrainEvalFeatures.TIMES: datax,
tf.contrib.timeseries.TrainEvalFeatures.VALUES: datay,
}
reader = NumpyReader(data)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=128, window_size=8)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units=28),
optimizer=tf.train.AdamOptimizer(0.001))
estimator.train(input_fn=train_input_fn, steps=2000)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=predict_len)))
#观察(times表示时间序列)
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
#评估
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
#预测
predicted_times = predictions['times']
predicted = predictions["mean"]
#转列表
predicts = (predictions['mean'].T.tolist())
return predicts
def model_h(data):
data_group = data.reset_index(drop=True)
data_ = {
tf.contrib.timeseries.TrainEvalFeatures.TIMES:
np.array(data_group.index),
tf.contrib.timeseries.TrainEvalFeatures.VALUES:
np.array(data_group.values)
}
reader = NumpyReader(data_)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=data_group.shape[0], window_size=10)
LSTM = ts_estimators.TimeSeriesRegressor(
model=TS._LSTMModel(num_features=1, num_units=128),
optimizer=tf.train.AdamOptimizer(0.001))
LSTM.train(input_fn=train_input_fn, steps=500)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = LSTM.evaluate(input_fn=evaluation_input_fn, steps=3)
(predictions, ) = tuple(
LSTM.predict(input_fn=tf.contrib.timeseries.
predict_continuation_input_fn(evaluation, steps=5)))
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
predicted = predictions["mean"]
return observed, evaluated, predicted
def train_and_predict(
csv_file_name=_DATA_FILE, training_steps=200, estimator_config=None):
"""Train and predict using a custom time series model."""
# Construct an Estimator from our LSTM model.
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=5, num_units=128),
optimizer=tf.train.AdamOptimizer(0.001), config=estimator_config)
reader = tf.contrib.timeseries.CSVReader(
csv_file_name,
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES,)
+ (tf.contrib.timeseries.TrainEvalFeatures.VALUES,) * 5))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=4, window_size=32)
estimator.train(input_fn=train_input_fn, steps=training_steps)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
(predictions,) = tuple(estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=100)))
times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
predicted_mean = numpy.squeeze(numpy.concatenate(
[evaluation["mean"][0], predictions["mean"]], axis=0))
all_times = numpy.concatenate([times, predictions["times"]], axis=0)
return times, observed, all_times, predicted_mean
def main():
x = np.array(range(1000))
noise = np.random.uniform(-0.2, 0.2, 1000)
y = np.sin(np.pi * x / 50) + np.cos(np.pi * x / 50) +\
np.sin(np.pi * x / 25) + np.exp(0.001 * x) + noise
plt.plot(x, y)
plt.show()
data = {
tf.contrib.timeseries.TrainEvalFeatures.TIMES: x,
tf.contrib.timeseries.TrainEvalFeatures.VALUES: y,
}
reader = NumpyReader(data)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=5,
window_size=100)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units=128),
optimizer=tf.train.AdamOptimizer(0.01))
estimator.train(input_fn=train_input_fn, steps=2000)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=200)))
observed_times = evaluation['times'][0]
observed = evaluation['observed'][0, :, :]
evaluated_times = evaluation['times'][0]
evaluated = evaluation['mean'][0]
predicted_times = predictions['times']
predicted = predictions['mean']
plt.figure(figsize=(15, 5))
plt.axvline(999, linestyle="dotted", linewidth=4, color='r')
observed_line = plt.plot(observed_times,
observed,
label='observation',
color='k')
evaluated_line = plt.plot(evaluated_times,
evaluated,
label='evaluation',
color='g')
predicted_line = plt.plot(predicted_times,
predicted,
label='prediction',
color='r')
plt.legend(
handles=[observed_line[0], evaluated_line[0], predicted_line[0]],
loc='upper left')
plt.savefig('lstm_single_var.jpg')
plt.show()
def train(queue_name, csv_file, pre_file, model_dir, train_step, predict_step):
tf.logging.set_verbosity(tf.logging.INFO)
csv_file_name = path.join(csv_file)
pre_file_name = path.join(pre_file)
reader = tf.contrib.timeseries.CSVReader(
csv_file_name,
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES, ) +
(tf.contrib.timeseries.TrainEvalFeatures.VALUES, ) * 1))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=2,
window_size=2)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units=512),
optimizer=tf.train.AdamOptimizer(0.001),
model_dir=model_dir)
estimator.train(input_fn=train_input_fn, steps=train_step)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(
input_fn=evaluation_input_fn,
steps=1,
)
# Predict starting after the evaluation
# (predictions,) = tuple(estimator.predict(
# input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
# evaluation, steps=FLAGS.predict_step)))
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=predict_step)))
# save model for serving
# export_dir_base = "./serving_save_model"
# serving_input_receiver_fn = estimator.build_raw_serving_input_receiver_fn()
# estimator.export_savedmodel(
# "../model",
# serving_input_receiver_fn
# )
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
predicted = predictions["mean"]
df = pd.DataFrame(predicted)
df.insert(0, "times", predicted_times)
df.insert(2, "queue", queue_name)
df.columns = ['times', 'pre', 'queue_name']
df[['pre']].astype(float)
df.loc[df['pre'] < 0, 'pre'] = 0.1
df.to_csv(pre_file_name, header=None, mode="a", index=False)
def read_csv():
''' 使用tf读入待训练数据
:return:
'''
paths = get_paths()
for path in paths:
reader = tf.contrib.timeseries.CSVReader(path)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=4, window_size=100)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units=7),
optimizer=tf.train.AdamOptimizer(0.001))
estimator.train(input_fn=train_input_fn, steps=2000)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=20)))
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
predicted = predictions["mean"]
image_path = './pre_image/{date}/'.format(
date=time.strftime("%Y-%m-%d", time.localtime()))
OSUtil.mkdir(image_path)
plt.figure(figsize=(15, 5))
plt.axvline(999, linestyle="dotted", linewidth=4, color='r')
observed_lines = plt.plot(observed_times,
observed,
label="observation",
color="k")
evaluated_lines = plt.plot(evaluated_times,
evaluated,
label="evaluation",
color="g")
predicted_lines = plt.plot(predicted_times,
predicted,
label="prediction",
color="r")
plt.legend(handles=[
observed_lines[0], evaluated_lines[0], predicted_lines[0]
],
loc="upper left")
plt.savefig(image_path + path.spilt('/')[3].split('.')[0] + '.png')
def _custom_time_series_regressor(
model_dir, head_type, exogenous_feature_columns):
return ts_estimators.TimeSeriesRegressor(
model=lstm_example._LSTMModel(
num_features=5, num_units=128,
exogenous_feature_columns=exogenous_feature_columns),
optimizer=adam.AdamOptimizer(0.001),
config=estimator_lib.RunConfig(tf_random_seed=4),
state_manager=state_management.ChainingStateManager(),
head_type=head_type,
model_dir=model_dir)
def _estimator_fn(model_dir, exogenous_feature_columns):
return estimators.TimeSeriesRegressor(model=ar_model.ARModel(
periodicities=10,
input_window_size=10,
output_window_size=6,
num_features=1,
exogenous_feature_columns=exogenous_feature_columns,
prediction_model_factory=functools.partial(
ar_model.LSTMPredictionModel, num_units=10)),
config=_SeedRunConfig(),
model_dir=model_dir)
def _ar_lstm_regressor(model_dir, head_type, exogenous_feature_columns):
return ts_estimators.TimeSeriesRegressor(model=ar_model.ARModel(
periodicities=10,
input_window_size=10,
output_window_size=6,
num_features=5,
exogenous_feature_columns=exogenous_feature_columns,
prediction_model_factory=functools.partial(
ar_model.LSTMPredictionModel, num_units=10)),
head_type=head_type,
model_dir=model_dir)
def main():
csv_file_name = 'multivariate_level.csv'
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=5, num_units=128),
optimizer=tf.train.AdamOptimizer(0.001))
reader = tf.contrib.timeseries.CSVReader(
csv_file_name,
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES, ) +
(tf.contrib.timeseries.TrainEvalFeatures.VALUES, ) * 5))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=10,
window_size=32)
estimator.train(input_fn=train_input_fn, steps=2000)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=100)))
observed_times = evaluation['times'][0]
observed = evaluation['observed'][0, :, :]
evaluated_times = evaluation['times'][0]
evaluated = evaluation['mean'][0]
predicted_times = predictions['times']
predicted = predictions['mean']
plt.figure(figsize=(15, 5))
plt.axvline(100, linestyle="dotted", linewidth=4, color='r')
observed_line = plt.plot(observed_times,
observed,
label='observation',
color='k')
evaluated_line = plt.plot(evaluated_times,
evaluated,
label='evaluation',
color='g')
predicted_line = plt.plot(predicted_times,
predicted,
label='prediction',
color='r')
plt.legend(
handles=[observed_line[0], evaluated_line[0], predicted_line[0]],
loc='upper left')
plt.savefig('lstm_multiple_var.jpg')
plt.show()
def train(self, ori_data, x, y):
data = self._get_timeseries(y)
reader = NumpyReader(data)
# 历史数据必须超过三个预测时间
if len(y) > 3*self.predict_time:
# tf.contrib.timeseries.RandomWindowInputFn会在reader的所有数据中，随机选取窗口长度为window_size的序列，
# 并包装成batch_size大小的batch数据。换句话说，一个batch内共有batch_size个序列，每个序列的长度为window_size。
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=4, window_size=3*self.predict_time)
# 定义lstm模型，num_features = 1表示单变量时间序列，
# num_units=n表示使用隐层(记忆和储存过去状态的节点个数)为n大小的LSTM模型。
estimator = ts_estimators.TimeSeriesRegressor(
model=LSTMModel(self.predict_time, num_features=1, num_units=3*self.predict_time),
optimizer=tf.train.AdamOptimizer(0.001))
# 在原有len(y)基础上预测
estimator.train(input_fn=train_input_fn, steps=len(y))
return estimator
else:
print('few data')
def test_custom_metrics(self):
"""Tests that the custom metrics can be applied to the estimator."""
model_dir = self.get_temp_dir()
estimator = ts_estimators.TimeSeriesRegressor(
model=lstm_example._LSTMModel(num_features=1, num_units=4),
optimizer=adam.AdamOptimizer(0.001),
config=estimator_lib.RunConfig(tf_random_seed=4),
model_dir=model_dir)
def input_fn():
return {
feature_keys.TrainEvalFeatures.TIMES: [[1, 2, 3], [7, 8, 9]],
feature_keys.TrainEvalFeatures.VALUES:
numpy.array([[[0.], [1.], [0.]], [[2.], [3.], [2.]]])
}
def metrics_fn(predictions, features):
# checking that the inputs are properly passed.
predict = predictions["mean"]
target = features[feature_keys.TrainEvalFeatures.VALUES][:, -1, 0]
return {
"plain_boring_metric386":
(math_ops.reduce_mean(math_ops.abs(predict - target)),
control_flow_ops.no_op()),
"fun_metric101": (math_ops.reduce_sum(predict + target),
control_flow_ops.no_op()),
}
# Evaluation without training is enough for testing custom metrics.
estimator = extenders.add_metrics(estimator, metrics_fn)
evaluation = estimator.evaluate(input_fn, steps=1)
self.assertIn("plain_boring_metric386", evaluation)
self.assertIn("fun_metric101", evaluation)
self.assertIn("average_loss", evaluation)
# The values are deterministic because of fixed tf_random_seed.
# However if they become flaky, remove such exacts comparisons.
self.assertAllClose(evaluation["plain_boring_metric386"], 1.130380)
self.assertAllClose(evaluation["fun_metric101"], 10.435442)
def predictor_LSTM(self, data, batch_size, window_size, num_features,
num_units, train_steps, predict_steps, learning_rate):
reader = NumpyReader(data)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=batch_size, window_size=window_size)
estimator = ts_estimators.TimeSeriesRegressor(
model=LSTMModel(num_features=num_features, num_units=num_units),
optimizer=tf.train.AdamOptimizer(learning_rate))
estimator.train(input_fn=train_input_fn, steps=train_steps)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=predict_steps)))
print(self.Description)
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
predicted = predictions["mean"]
result = {}
result["observed_times"] = observed_times
result["observed"] = observed
result["evaluated_times"] = evaluated_times
result["evaluated"] = evaluated
result["predicted_times"] = predicted_times
result["predicted"] = predicted
result["average_loss"] = evaluation['average_loss']
result["loss"] = evaluation["loss"]
return result
def predict(frame, columns, model_dir="Models"):
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=len(columns), num_units=128),
optimizer=tf.train.AdamOptimizer(0.001),
model_dir=model_dir)
#compute the prediction for the next k steps
x = np.arange(len(frame))
y = np.zeros((len(frame), len(columns)))
for i in range(len(columns)):
y[:, i] = frame[columns[i]].values
data = {
tf.contrib.timeseries.TrainEvalFeatures.TIMES: x,
tf.contrib.timeseries.TrainEvalFeatures.VALUES: y,
}
reader = tf.contrib.timeseries.NumpyReader(data) #entire data
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(
reader) # evaluate all the data
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
return evaluation["observed"][0], evaluation["mean"][0]
tf.contrib.timeseries.TrainEvalFeatures.TIMES: x,
tf.contrib.timeseries.TrainEvalFeatures.VALUES: y,
}
# 3. 生成reader对象
reader = NumpyReader(data)
# 4. 从reader对象中读取batch数据
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=4,
window_size=100)
# 5. 定义一个 lstm 模型
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(
num_features=1, # 单变量时间序列，也可以设为多变量时间序列
num_units=128 # 隐层为128的lstm model
),
optimizer=tf.train.AdamOptimizer(0.001))
# 6. train
estimator.train(input_fn=train_input_fn, steps=2000)
# 7. evaluation
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# 8. prediction
# Predict starting after the evaluation
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
def _train_on_generated_data(
generate_fn, generative_model, train_iterations, seed,
learning_rate=0.1, ignore_params_fn=lambda _: (),
derived_param_test_fn=lambda _: (),
train_input_fn_type=input_pipeline.WholeDatasetInputFn,
train_state_manager=state_management.PassthroughStateManager()):
"""The training portion of parameter recovery tests."""
random_seed.set_random_seed(seed)
generate_graph = ops.Graph()
with generate_graph.as_default():
with session.Session(graph=generate_graph):
generative_model.initialize_graph()
time_series_reader, true_parameters = generate_fn(generative_model)
true_parameters = {
tensor.name: value for tensor, value in true_parameters.items()}
eval_input_fn = input_pipeline.WholeDatasetInputFn(time_series_reader)
eval_state_manager = state_management.PassthroughStateManager()
true_parameter_eval_graph = ops.Graph()
with true_parameter_eval_graph.as_default():
generative_model.initialize_graph()
ignore_params = ignore_params_fn(generative_model)
feature_dict, _ = eval_input_fn()
eval_state_manager.initialize_graph(generative_model)
feature_dict[TrainEvalFeatures.VALUES] = math_ops.cast(
feature_dict[TrainEvalFeatures.VALUES], generative_model.dtype)
model_outputs = eval_state_manager.define_loss(
model=generative_model,
features=feature_dict,
mode=estimator_lib.ModeKeys.EVAL)
with session.Session(graph=true_parameter_eval_graph) as sess:
variables.global_variables_initializer().run()
coordinator = coordinator_lib.Coordinator()
queue_runner_impl.start_queue_runners(sess, coord=coordinator)
true_param_loss = model_outputs.loss.eval(feed_dict=true_parameters)
true_transformed_params = {
param: param.eval(feed_dict=true_parameters)
for param in derived_param_test_fn(generative_model)}
coordinator.request_stop()
coordinator.join()
saving_hook = _SavingTensorHook(
tensors=true_parameters.keys(),
every_n_iter=train_iterations - 1)
class _RunConfig(estimator_lib.RunConfig):
@property
def tf_random_seed(self):
return seed
estimator = estimators.TimeSeriesRegressor(
model=generative_model,
config=_RunConfig(),
state_manager=train_state_manager,
optimizer=adam.AdamOptimizer(learning_rate))
train_input_fn = train_input_fn_type(time_series_reader=time_series_reader)
trained_loss = (estimator.train(
input_fn=train_input_fn,
max_steps=train_iterations,
hooks=[saving_hook]).evaluate(
input_fn=eval_input_fn, steps=1))["loss"]
logging.info("Final trained loss: %f", trained_loss)
logging.info("True parameter loss: %f", true_param_loss)
return (ignore_params, true_parameters, true_transformed_params,
trained_loss, true_param_loss, saving_hook,
true_parameter_eval_graph)
if __name__ == '__main__':
iteration = 1
n_hidden = 252
while iteration>0:
tf.logging.set_verbosity(tf.logging.INFO)
csv_file_name = 'period_trend_A.csv'
reader = tf.contrib.timeseries.CSVReader(csv_file_name)
original_file = 'period_trend_B.csv'
original_data = pd.read_csv(original_file)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=4, window_size=100)#batch_size
#window_size
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units= n_hidden),#单变量，使用隐层为n_hidden大小的LSTM模型。
optimizer=tf.train.AdamOptimizer(0.001))
estimator.train(input_fn=train_input_fn, steps=100)#####################number of steps to train,can be changed to 3000 or more?
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
time_start = time.time()
(predictions,) = tuple(estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=500)))#########################################num of predicted points
time_end = time.time()
time_consume = time_end - time_start
print ("time_consume = " + str(time_consume))
def train(queue_name, csv_file, pre_file, model_dir, train_step, predict_step):
"""
:param queue_name: the queue_name of the hadoop
:param csv_file_name: the input file to trainning model
:param pre_file_name: the output file of predict
:param model_dir: the dir to save model
"""
tf.logging.set_verbosity(tf.logging.INFO)
csv_file_name = path.join(csv_file)
pre_file_name = path.join(pre_file)
reader = tf.contrib.timeseries.CSVReader(
csv_file_name,
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES, ) +
(tf.contrib.timeseries.TrainEvalFeatures.VALUES, ) * 2))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=8,
window_size=32)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=2, num_units=256),
optimizer=tf.train.AdamOptimizer(0.001),
model_dir=model_dir)
estimator.train(input_fn=train_input_fn, steps=train_step)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(
input_fn=evaluation_input_fn,
steps=1,
)
# Predict starting after the evaluation
# (predictions,) = tuple(estimator.predict(
# input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
# evaluation, steps=FLAGS.predict_step)))
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=predict_step)))
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
predicted = predictions["mean"]
df = pd.DataFrame(predicted)
df.insert(0, "times", predicted_times)
df.insert(3, "queue", queue_name)
df.to_csv(pre_file_name, header=None, mode="a", index=False)
plt.figure(figsize=(15, 2))
plt.axvline(99, linestyle="dotted", linewidth=4, color='r')
observed_lines = plt.plot(observed_times,
observed,
label="observation",
color="k")
evaluated_lines = plt.plot(evaluated_times,
evaluated,
label="evaluation",
color="g")
predicted_lines = plt.plot(predicted_times,
predicted,
label="prediction",
color="r")
plt.legend(
handles=[observed_lines[0], evaluated_lines[0], predicted_lines[0]],
loc="upper left")
plt.savefig('predict_result.png')
x = np.array(range(1000))
noise = np.random.uniform(-0.2, 0.2, 1000)
y = np.sin(np.pi * x / 50 ) + np.cos(np.pi * x / 50) + np.sin(np.pi * x / 25) + noise
data = {
tf.contrib.timeseries.TrainEvalFeatures.TIMES: x,
tf.contrib.timeseries.TrainEvalFeatures.VALUES: y,
}
reader = NumpyReader(data)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=4, window_size=100)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units=128),
optimizer=tf.train.AdamOptimizer(0.001))
estimator.train(input_fn=train_input_fn, steps=2000)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
(predictions,) = tuple(estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=200)))
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
def train_and_predict_timeseries_lstm(data,
window_size=100,
num_features=1,
num_units=128,
train_steps=1000,
batch_size=4):
'''
function created by txy based on the __main__ function in this file.
data is a dict with keys "time" and "values"
'''
tf.logging.set_verbosity(tf.logging.INFO)
reader = NumpyReader(data)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=batch_size, window_size=window_size
) # window_size 是时间序列段的总长度，= input_length + output_length
# model
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=num_features, num_units=num_units),
optimizer=tf.train.AdamOptimizer(0.001),
model_dir="./output_model")
# training
# 这个模型使用的 window_size 中的 input_length 和 output_length 分别是多少？
estimator.train(input_fn=train_input_fn, steps=train_steps)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
# 以 以上数据的结尾处为起点，对后续 steps 步骤的值进行预测
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=200)))
# 绘制结果
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
predicted = predictions["mean"]
plt.figure(figsize=(15, 5))
plt.axvline(999, linestyle="dotted", linewidth=4, color='r')
observed_lines = plt.plot(observed_times,
observed,
label="observation",
color="k")
evaluated_lines = plt.plot(evaluated_times,
evaluated,
label="evaluation",
color="g")
predicted_lines = plt.plot(predicted_times,
predicted,
label="prediction",
color="r")
plt.legend(
handles=[observed_lines[0], evaluated_lines[0], predicted_lines[0]],
loc="upper left")
plt.savefig('predict_result.png')
def test_one_shot_prediction_head_export(self):
model_dir = self.get_temp_dir()
categorical_column = feature_column.categorical_column_with_hash_bucket(
key="categorical_exogenous_feature", hash_bucket_size=16)
exogenous_feature_columns = [
feature_column.numeric_column("2d_exogenous_feature", shape=(2, )),
feature_column.embedding_column(
categorical_column=categorical_column, dimension=10)
]
estimator = ts_estimators.TimeSeriesRegressor(
model=lstm_example._LSTMModel(
num_features=5,
num_units=128,
exogenous_feature_columns=exogenous_feature_columns),
optimizer=adam.AdamOptimizer(0.001),
config=estimator_lib.RunConfig(tf_random_seed=4),
state_manager=state_management.ChainingStateManager(),
head_type=ts_head_lib.OneShotPredictionHead,
model_dir=model_dir)
train_features = {
feature_keys.TrainEvalFeatures.TIMES:
numpy.arange(20, dtype=numpy.int64),
feature_keys.TrainEvalFeatures.VALUES:
numpy.tile(numpy.arange(20, dtype=numpy.float32)[:, None], [1, 5]),
"2d_exogenous_feature":
numpy.ones([20, 2]),
"categorical_exogenous_feature":
numpy.array(["strkey"] * 20)[:, None]
}
train_input_fn = input_pipeline.RandomWindowInputFn(
input_pipeline.NumpyReader(train_features),
shuffle_seed=2,
num_threads=1,
batch_size=16,
window_size=16)
estimator.train(input_fn=train_input_fn, steps=5)
input_receiver_fn = estimator.build_raw_serving_input_receiver_fn()
export_location = estimator.export_savedmodel(self.get_temp_dir(),
input_receiver_fn)
graph = ops.Graph()
with graph.as_default():
with session_lib.Session() as session:
signatures = loader.load(session, [tag_constants.SERVING],
export_location)
self.assertEqual([feature_keys.SavedModelLabels.PREDICT],
list(signatures.signature_def.keys()))
predict_signature = signatures.signature_def[
feature_keys.SavedModelLabels.PREDICT]
six.assertCountEqual(self, [
feature_keys.FilteringFeatures.TIMES,
feature_keys.FilteringFeatures.VALUES,
"2d_exogenous_feature", "categorical_exogenous_feature"
], predict_signature.inputs.keys())
features = {
feature_keys.TrainEvalFeatures.TIMES:
numpy.tile(
numpy.arange(35, dtype=numpy.int64)[None, :], [2, 1]),
feature_keys.TrainEvalFeatures.VALUES:
numpy.tile(
numpy.arange(20, dtype=numpy.float32)[None, :, None],
[2, 1, 5]),
"2d_exogenous_feature":
numpy.ones([2, 35, 2]),
"categorical_exogenous_feature":
numpy.tile(
numpy.array(["strkey"] * 35)[None, :, None], [2, 1, 1])
}
feeds = {
graph.as_graph_element(input_value.name):
features[input_key]
for input_key, input_value in
predict_signature.inputs.items()
}
fetches = {
output_key: graph.as_graph_element(output_value.name)
for output_key, output_value in
predict_signature.outputs.items()
}
output = session.run(fetches, feed_dict=feeds)
self.assertAllEqual((2, 15, 5), output["mean"].shape)
reader = tf.contrib.timeseries.CSVReader(
"./data/multivariate_periods.csv",
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES, ) +
(tf.contrib.timeseries.TrainEvalFeatures.VALUES, ) * 5))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader,
batch_size=4,
window_size=32 # much smaller window_size produces flat lines.
)
# See: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/estimator/estimator.py#L70
# See: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/timeseries/python/timeseries/estimators.py#L41
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=5, num_units=128),
optimizer=tf.train.AdamOptimizer(
0.001
) # Not much different than default: train.AdamOptimizer(0.02)
)
# See: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/estimator/estimator.py#L288
estimator.train(input_fn=train_input_fn, steps=200)
# See: https://www.tensorflow.org/api_docs/python/tf/contrib/timeseries/WholeDatasetInputFn
# Q: Why are we evaluating using the same data as the training data? Can we pass data from different time ranges?
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
# See: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/estimator/estimator.py#L366
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict after the evaluation
# See: https://www.tensorflow.org/api_docs/python/tf/contrib/timeseries/predict_continuation_input_fn
def lstm_predict(csv_file_name,
model_save_dir,
city_name,
training_steps=300,
predicted_steps=3,
batch_size=1,
window_size=132):
# 显示训练信息
tf.logging.set_verbosity(tf.logging.INFO)
# 读取数据文件
reader = tf.contrib.timeseries.CSVReader(csv_file_name)
# 训练参数设置
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=batch_size, window_size=window_size)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units=128),
optimizer=tf.train.AdamOptimizer(0.001),
model_dir=model_save_dir)
# 训练开始，训练training_steps次
estimator.train(input_fn=train_input_fn, steps=training_steps)
# 模型拟合
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# 预测开始，向后预测predicted_steps个数据
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=predicted_steps)))
# 原数据
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
# 模型拟合数据
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
# 向后预测数据
predicted_times = predictions['times']
predicted = predictions["mean"]
# 画图观察
plt.figure(figsize=(15, 5))
observed_lines = plt.plot(observed_times,
observed,
label="observed",
color="k") # 原数据曲线
evaluated_lines = plt.plot(evaluated_times,
evaluated,
label="evaluated",
color="g") # 模型拟合曲线
predicted_lines = plt.plot(predicted_times,
predicted,
label="predicted",
color="r") # 预测结果曲线
plt.legend(
handles=[observed_lines[0], evaluated_lines[0], predicted_lines[0]],
loc="upper left")
plt.savefig(model_save_dir + 'predict_result.png') # 图保存在对应模型目录下
# plt.show()
# 预测数据存储在predicted中，predicted_times代表时间点，predicted中包含时间点对应的预测数据
# return predicted_times, predicted
pre_data = predicted
i = 0
pre_dir = 'Dromatters/pre_data/' + city_name + '.txt'
while i < 3:
if pre_data[i] < 0:
pre_data[i] += 30
with open(pre_dir, 'a') as f:
f.write(str(round(pre_data[i][0], 1)) + '\n')
i += 1
def train_and_predict(csv_file_name=_DATA_FILE,
training_steps=200,
estimator_config=None,
export_directory=None):
"""Train and predict using a custom time series model."""
# Construct an Estimator from our LSTM model.
categorical_column = tf.feature_column.categorical_column_with_hash_bucket(
key="categorical_exogenous_feature", hash_bucket_size=16)
exogenous_feature_columns = [
# Exogenous features are not part of the loss, but can inform
# predictions. In this example the features have no extra information, but
# are included as an API example.
tf.feature_column.numeric_column("2d_exogenous_feature", shape=(2, )),
tf.feature_column.embedding_column(
categorical_column=categorical_column, dimension=10)
]
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=5,
num_units=128,
exogenous_feature_columns=exogenous_feature_columns),
optimizer=tf.train.AdamOptimizer(0.001),
config=estimator_config,
# Set state to be saved across windows.
state_manager=state_management.ChainingStateManager())
reader = tf.contrib.timeseries.CSVReader(
csv_file_name,
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES, ) +
(tf.contrib.timeseries.TrainEvalFeatures.VALUES, ) * 5 +
("2d_exogenous_feature", ) * 2 +
("categorical_exogenous_feature", )),
# Data types other than for `times` need to be specified if they aren't
# float32. In this case one of our exogenous features has string dtype.
column_dtypes=((tf.int64, ) + (tf.float32, ) * 7 + (tf.string, )))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=4,
window_size=32)
estimator.train(input_fn=train_input_fn, steps=training_steps)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
predict_exogenous_features = {
"2d_exogenous_feature":
numpy.concatenate([numpy.ones([1, 100, 1]),
numpy.zeros([1, 100, 1])],
axis=-1),
"categorical_exogenous_feature":
numpy.array(["strkey"] * 100)[None, :, None]
}
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation,
steps=100,
exogenous_features=predict_exogenous_features)))
times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
predicted_mean = numpy.squeeze(
numpy.concatenate([evaluation["mean"][0], predictions["mean"]],
axis=0))
all_times = numpy.concatenate([times, predictions["times"]], axis=0)
# Export the model in SavedModel format. We include a bit of extra boilerplate
# for "cold starting" as if we didn't have any state from the Estimator, which
# is the case when serving from a SavedModel. If Estimator output is
# available, the result of "Estimator.evaluate" can be passed directly to
# `tf.contrib.timeseries.saved_model_utils.predict_continuation` as the
# `continue_from` argument.
with tf.Graph().as_default():
filter_feature_tensors, _ = evaluation_input_fn()
with tf.train.MonitoredSession() as session:
# Fetch the series to "warm up" our state, which will allow us to make
# predictions for its future values. This is just a dictionary of times,
# values, and exogenous features mapping to numpy arrays. The use of an
# input_fn is just a convenience for the example; they can also be
# specified manually.
filter_features = session.run(filter_feature_tensors)
if export_directory is None:
export_directory = tempfile.mkdtemp()
input_receiver_fn = estimator.build_raw_serving_input_receiver_fn()
export_location = estimator.export_savedmodel(export_directory,
input_receiver_fn)
# Warm up and predict using the SavedModel
with tf.Graph().as_default():
with tf.Session() as session:
signatures = tf.saved_model.loader.load(
session, [tf.saved_model.tag_constants.SERVING],
export_location)
state = tf.contrib.timeseries.saved_model_utils.cold_start_filter(
signatures=signatures,
session=session,
features=filter_features)
saved_model_output = (
tf.contrib.timeseries.saved_model_utils.predict_continuation(
continue_from=state,
signatures=signatures,
session=session,
steps=100,
exogenous_features=predict_exogenous_features))
# The exported model gives the same results as the Estimator.predict()
# call above.
numpy.testing.assert_allclose(
predictions["mean"],
numpy.squeeze(saved_model_output["mean"], axis=0))
return times, observed, all_times, predicted_mean
for i in range(len(y_train)):
X_time[i] = x_time
x_time = x_time + 1
data = {
tf.contrib.timeseries.TrainEvalFeatures.TIMES: X_time,
tf.contrib.timeseries.TrainEvalFeatures.VALUES: y_train,
}
reader = NumpyReader(data)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=16, window_size=50)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units=32),
optimizer=tf.train.AdamOptimizer(0.001),
model_dir=model_name_lstm)
estimator.train(input_fn=train_input_fn, steps=100)
index_front = index_back
index_back += 1
x_time_test = 1
X_time_test = np.zeros(len(y_test[0:90]))
for i in range(len(y_test[0:90])):
X_time_test[i] = x_time_test
x_time_test = x_time_test + 1
data2 = {
def wanzhengxing(da_input):
date_haha = date_list(da_input)
dates_list = list(set(list(date_haha)))
days = len(dates_list)
date_numbs = 24*days
net_num_list = list(set(list(da_input['net_num'])))
net_num_list_input = list(da_input['net_num'])
date_lost_list = list()
for net_item in net_num_list:
net_num_counter = net_num_list_input.count(net_item)
if net_num_counter < date_numbs:
date_lost_list.append(net_item)
for item in date_lost_list:
return date_lost_list
class _LSTMModel(ts_model.SequentialTimeSeriesModel):
"""A time series model-building example using an RNNCell."""
def __init__(self, num_units, num_features, dtype=tf.float32):
"""Initialize/configure the model object.
Note that we do not start graph building here. Rather, this object is a
configurable factory for TensorFlow graphs which are run by an Estimator.
Args:
num_units: The number of units in the model's LSTMCell.
num_features: The dimensionality of the time series (features per
timestep).
dtype: The floating point data type to use.
"""
super(_LSTMModel, self).__init__(
# Pre-register the metrics we'll be outputting (just a mean here).
train_output_names=["mean"],
predict_output_names=["mean"],
num_features=num_features,
dtype=dtype)
self._num_units = num_units
# Filled in by initialize_graph()
self._lstm_cell = None
self._lstm_cell_run = None
self._predict_from_lstm_output = None
def initialize_graph(self, input_statistics):
"""Save templates for components, which can then be used repeatedly.
This method is called every time a new graph is created. It's safe to start
adding ops to the current default graph here, but the graph should be
constructed from scratch.
Args:
input_statistics: A math_utils.InputStatistics object.
"""
super(_LSTMModel, self).initialize_graph(input_statistics=input_statistics)
self._lstm_cell = tf.nn.rnn_cell.LSTMCell(num_units=self._num_units)
# Create templates so we don't have to worry about variable reuse.
self._lstm_cell_run = tf.make_template(
name_="lstm_cell",
func_=self._lstm_cell,
create_scope_now_=True)
# Transforms LSTM output into mean predictions.
self._predict_from_lstm_output = tf.make_template(
name_="predict_from_lstm_output",
func_=lambda inputs: tf.layers.dense(inputs=inputs, units=self.num_features),
create_scope_now_=True)
def get_start_state(self):
"""Return initial state for the time series model."""
return (
# Keeps track of the time associated with this state for error checking.
tf.zeros([], dtype=tf.int64),
# The previous observation or prediction.
tf.zeros([self.num_features], dtype=self.dtype),
# The state of the RNNCell (batch dimension removed since this parent
# class will broadcast).
[tf.squeeze(state_element, axis=0)
for state_element
in self._lstm_cell.zero_state(batch_size=1, dtype=self.dtype)])
def _transform(self, data):
"""Normalize data based on input statistics to encourage stable training."""
mean, variance = self._input_statistics.overall_feature_moments
return (data - mean) / variance
def _de_transform(self, data):
"""Transform data back to the input scale."""
mean, variance = self._input_statistics.overall_feature_moments
return data * variance + mean
def _filtering_step(self, current_times, current_values, state, predictions):
"""Update model state based on observations.
Note that we don't do much here aside from computing a loss. In this case
it's easier to update the RNN state in _prediction_step, since that covers
running the RNN both on observations (from this method) and our own
predictions. This distinction can be important for probabilistic models,
where repeatedly predicting without filtering should lead to low-confidence
predictions.
Args:
current_times: A [batch size] integer Tensor.
current_values: A [batch size, self.num_features] floating point Tensor
with new observations.
state: The model's state tuple.
predictions: The output of the previous `_prediction_step`.
Returns:
A tuple of new state and a predictions dictionary updated to include a
loss (note that we could also return other measures of goodness of fit,
although only "loss" will be optimized).
"""
state_from_time, prediction, lstm_state = state
with tf.control_dependencies(
[tf.assert_equal(current_times, state_from_time)]):
transformed_values = self._transform(current_values)
# Use mean squared error across features for the loss.
predictions["loss"] = tf.reduce_mean(
(prediction - transformed_values) ** 2, axis=-1)
# Keep track of the new observation in model state. It won't be run
# through the LSTM until the next _imputation_step.
new_state_tuple = (current_times, transformed_values, lstm_state)
return (new_state_tuple, predictions)
def _prediction_step(self, current_times, state):
"""Advance the RNN state using a previous observation or prediction."""
_, previous_observation_or_prediction, lstm_state = state
lstm_output, new_lstm_state = self._lstm_cell_run(
inputs=previous_observation_or_prediction, state=lstm_state)
next_prediction = self._predict_from_lstm_output(lstm_output)
new_state_tuple = (current_times, next_prediction, new_lstm_state)
return new_state_tuple, {"mean": self._de_transform(next_prediction)}
def _imputation_step(self, current_times, state):
"""Advance model state across a gap."""
# Does not do anything special if we're jumping across a gap. More advanced
# models, especially probabilistic ones, would want a special case that
# depends on the gap size.
return state
def _exogenous_input_step(
self, current_times, current_exogenous_regressors, state):
"""Update model state based on exogenous regressors."""
raise NotImplementedError(
"Exogenous inputs are not implemented for this example.")
def ratio_mark(data_input, standard):
div_mod = divmod(data_input, standard)
ratio_mark_out = abs(div_mod[0]-1)*div_mod[1]/standard
ratio_mark_out = ratio_mark_out.replace(0, 1)
return ratio_mark_out
def not_ok(data_input, lastday_input, data_input_ratio, net_item, flow_standard, alert_standard):
ratio_need = data_input_ratio[['UE', 'erab', 'handover', 'rrc']]
ratio_need_columns = ratio_need.columns
ratio_out = DataFrame()
for columns_need in ratio_need_columns:
ratio_item = ratio_need[columns_need]
ratio_isok = list(map(lambda y:0 if y>alert_standard else 1, ratio_item))
ratio_out[columns_need] = ratio_isok
# rows_input = ratio_need.iloc[:,0].size
# columns_input = ratio_need.columns.size
# for i in range(rows_input):
# for j in range(columns_input):
# value_need = ratio_need.iloc[[i]].values[0][j]
# if value_need > 0.3 :
# isok = 0
# else:
# isok = 1
# ratio_need.iloc[[i]].values[0][j] = isok
flow_ratio = data_input_ratio['flow']
flow_ratio.index = range(len(flow_ratio))
flow_data_pred = data_input['flow']
flow_data_pred.index = range(len(flow_data_pred))
last_flow = lastday_input['flow']
flow_mark = ratio_mark(flow_data_pred, flow_standard)# 超过3G则不考虑比例转换
flow_ratio_out = flow_ratio*flow_mark
flow_isok = list(map(lambda y:0 if y>alert_standard else 1, flow_ratio_out))
ratio_out['flow'] = flow_isok
#画图
notok_hours = 24 - ratio_out.sum(axis=0)
if notok_hours['flow'] > 3:
fig = plt.figure(figsize=(6, 3))
ax = fig.add_subplot(111)
ax.plot(range(24), flow_data_pred, label="flow_pred", color="g")
ax.plot(range(24), last_flow, label="flow_lastday", color="r")
ax.set_ylabel('GB')
ax.set_xlabel('Hour')
plt.legend(loc="upper left")
plt.title('net_num: ' + net_item + ' flow lastday')
#plt.show()
plt.savefig('F:/work/tianhe4location/output/picture/' + net_item +'.jpg')
return ratio_out
if __name__ == '__main__':
tf.logging.set_verbosity(tf.logging.INFO)
#计算当天指标是否有异常
predicted_zhibiao = pd.read_csv('F:/work/tianhe4location/output/predicted_last.csv')
predicted_zhibiao.columns = ['hour', 'UE', 'erab', 'flow', 'handover', 'rrc', 'net_num']
zhibiao_lastday = pd.read_csv("F:/work/tianhe4location/20180521.csv")
zhibiao_last_day = zhibiao_lastday[['hour', 'UE' , 'erab', 'flow', 'handover', 'rrc', 'net_num']]
net_num_list = list(set(list(predicted_zhibiao['net_num'])))
bijiao_result = DataFrame()
for net_item in net_num_list:
pred_zhibiao = predicted_zhibiao[predicted_zhibiao.net_num == net_item][
['hour', 'UE' , 'erab', 'flow', 'handover', 'rrc']]
last_zhibiao = zhibiao_last_day[zhibiao_last_day.net_num == net_item][[
'hour', 'UE' , 'erab', 'flow', 'handover', 'rrc']]
#bijiao = DataFrame(np.array(pred_zhibiao) - np.array(last_zhibiao),
#columns=['hour', 'UE' , 'erab', 'flow', 'handover', 'rrc'])
bijiao = np.array(pred_zhibiao) - np.array(last_zhibiao)
bijiao_ratio = np.array(bijiao)/np.array(pred_zhibiao)
bijiao_da = DataFrame(bijiao, columns=['hour', 'UE' , 'erab',
'flow', 'handover', 'rrc'])
bijiao_ratio_da = DataFrame(bijiao_ratio, columns=['hour', 'UE' , 'erab',
'flow', 'handover', 'rrc'])
bijiao_notok = not_ok(pred_zhibiao, last_zhibiao, bijiao_ratio_da, net_item, 5, 0.5) # 3表示比例转换的流量基准， 0.5表示告警门限
bijiao_notok['net_num'] = net_item
bijiao_result = bijiao_result.append(bijiao_notok)
#bijiao_result.to_csv("F:/work/tianhe4location/output/isok.csv")
#计算是否为公休日
workday_calender = pd.read_csv("F:/work/tianhe4location/basic_data/2018workday.csv")
workday_2018 = str2time(list(workday_calender['date']))
zhibiao_oneday = pd.read_csv("F:/work/tianhe4location/20180521.csv")
data = date_list(zhibiao_oneday)
is_workday = isworkday(data)
if is_workday:
zhibiao = pd.read_csv("F:/work/tianhe4location/workday.csv")
zhibiao1 = zhibiao.append(zhibiao_oneday)
my_file = "F:/work/tianhe4location/workday.csv"
os.remove(my_file)
new_index = range(len(zhibiao1))
zhibiao1.index = new_index
zhibiao1.to_csv("F:/work/tianhe4location/workday.csv")
else:
zhibiao = pd.read_csv("F:/work/tianhe4location/holiday.csv")
zhibiao1 = zhibiao.append(zhibiao_oneday)
my_file = "F:/work/tianhe4location/holiday.csv"
os.remove(my_file)
new_index = range(len(zhibiao1))
zhibiao1.index = new_index
zhibiao1.to_csv("F:/work/tianhe4location/holiday.csv")
#date_input =
date_index = date_list(zhibiao)
zhibiao.index = date_index
#zhibiao1.to_csv("F:/work/tianhe4location/test/wori.csv")
date_index_list = sorted(list(set(list(date_index))), reverse=True)
date_len = len(date_index_list)
if date_len <= 30:
zhibiao_need_index = date_index_list
else:
zhibiao_need_index = date_index_list[:30]
zhibiao_need = zhibiao.ix[zhibiao_need_index]
zhibiao_need = zhibiao_need.sort_index()
net_name = set(list(zhibiao_need.net_num))
prediction_cell = list()
predicted_last = pd.DataFrame(columns=['UE', 'erab', 'flow', 'handover', 'rrc', 'net_num'])
for cell in net_name:
#reader_numpy = numpy.array(zhibiao[zhibiao.net_num =='GK477'][['UE', 'erab',
#'flow', 'handover', 'rrc']])
y = numpy.array(zhibiao_need[zhibiao_need.net_num ==cell][['UE', 'erab',
'flow', 'handover', 'rrc']])
x = np.array(range(len(y)))
data = {tf.contrib.timeseries.TrainEvalFeatures.TIMES: x,
tf.contrib.timeseries.TrainEvalFeatures.VALUES: y,}
reader = NumpyReader(data)
#reader = tf.contrib.timeseries.CSVReader(
#csv_file_name,
#column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES,)
#+ (tf.contrib.timeseries.TrainEvalFeatures.VALUES,) * 1))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=20, window_size=168)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=5, num_units=96),
optimizer=tf.train.AdamOptimizer(0.001))
estimator.train(input_fn=train_input_fn, steps=200)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
(predictions,) = tuple(estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=24)))
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
predicted = predictions["mean"]
predicted_out = DataFrame(predicted)
predicted_out.columns = ['UE', 'erab', 'flow', 'handover', 'rrc']
predicted_out['net_num'] = cell
predicted_last = predicted_last.append(predicted_out)
my_pred_file = 'F:/work/tianhe4location/output/predicted_last.csv'
os.remove(my_pred_file)
predicted_last.to_csv('F:/work/tianhe4location/output/predicted_last.csv')
"Exogenous inputs are not implemented for this example.")
if __name__ == '__main__':
tf.logging.set_verbosity(tf.logging.INFO)
csv_file_name = path.join("./data/multivariate_periods.csv") #读取数据
reader = tf.contrib.timeseries.CSVReader(
csv_file_name,
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES, ) +
(tf.contrib.timeseries.TrainEvalFeatures.VALUES, ) * 1))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=10,
window_size=32)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=1, num_units=32), #est的超参数
optimizer=tf.train.AdamOptimizer(0.001)) #adam优化器
estimator.train(input_fn=train_input_fn, steps=1000)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# 拟合结束，开始预测 start to predict
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=2500)))
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
data = {
tf.contrib.timeseries.TrainEvalFeatures.TIMES: data[:, 0],
tf.contrib.timeseries.TrainEvalFeatures.VALUES: data[:, 1]
}
reader = NumpyReader(data)
#超参数
rnn_unit = 64
output_size = 1
learning_rate = 0.0006
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=60,
window_size=20)
estimator = tft_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=output_size, num_units=rnn_unit),
optimizer=tf.train.AdamOptimizer(learning_rate))
estimator.train(input_fn=train_input_fn, steps=2000)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
#预测
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=500)))
#可视化
visualization.show_result(observed_times=range(0, 180000, 100),
observed_data=evaluation['observed'][0, :, :],
evaluated_times=range(0, 180000, 100),
def train_and_predict(csv_file_name=_DATA_FILE,
training_steps=200,
estimator_config=None,
export_directory=None):
"""Train and predict using a custom time series model."""
# Construct an Estimator from our LSTM model.
exogenous_feature_columns = [
# Exogenous features are not part of the loss, but can inform
# predictions. In this example the features have no extra information, but
# are included as an API example.
tf.contrib.layers.real_valued_column("2d_exogenous_feature",
dimension=2)
]
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=5,
num_units=128,
exogenous_feature_columns=exogenous_feature_columns),
optimizer=tf.train.AdamOptimizer(0.001),
config=estimator_config,
# Set state to be saved across windows.
state_manager=state_management.ChainingStateManager())
reader = tf.contrib.timeseries.CSVReader(
csv_file_name,
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES, ) +
(tf.contrib.timeseries.TrainEvalFeatures.VALUES, ) * 5 +
("2d_exogenous_feature", ) * 2))
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
batch_size=4,
window_size=32)
estimator.train(input_fn=train_input_fn, steps=training_steps)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
predict_exogenous_features = {
"2d_exogenous_feature":
numpy.concatenate([numpy.ones([1, 100, 1]),
numpy.zeros([1, 100, 1])],
axis=-1)
}
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation,
steps=100,
exogenous_features=predict_exogenous_features)))
times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
predicted_mean = numpy.squeeze(
numpy.concatenate([evaluation["mean"][0], predictions["mean"]],
axis=0))
all_times = numpy.concatenate([times, predictions["times"]], axis=0)
# Export the model in SavedModel format.
if export_directory is None:
export_directory = tempfile.mkdtemp()
input_receiver_fn = estimator.build_raw_serving_input_receiver_fn()
export_location = estimator.export_savedmodel(export_directory,
input_receiver_fn)
# Predict using the SavedModel
with tf.Graph().as_default():
with tf.Session() as session:
signatures = tf.saved_model.loader.load(
session, [tf.saved_model.tag_constants.SERVING],
export_location)
saved_model_output = (
tf.contrib.timeseries.saved_model_utils.predict_continuation(
continue_from=evaluation,
signatures=signatures,
session=session,
steps=100,
exogenous_features=predict_exogenous_features))
# The exported model gives the same results as the Estimator.predict()
# call above.
numpy.testing.assert_allclose(
predictions["mean"],
numpy.squeeze(saved_model_output["mean"], axis=0))
return times, observed, all_times, predicted_mean
def train_and_predict(frame_interval, columns, parameters):
number_of_points = parameters["number_of_points_used_for_training"]
number_of_steps_to_train = parameters["number_of_steps_to_train"]
model_dir = "Models" if "model_dir" not in parameters else parameters[
"model_dir"]
num_units = 128 if "num_units" in parameters else parameters["num_units"]
windows_size = 64 if "windows_size" in parameters else parameters[
"windows_size"]
batch_size = 4 if "batch_size" in parameters else parameters["batch_size"]
learning_rate = 0.001 if "learning_rate" in parameters else parameters[
"learning_rate"]
#number_of_points = from the frame_interval the subset starting from 0 to number_of_points will be used for training
#number_of_steps_to_train = how many steps (forward-backward) will be used for training
print(columns)
values = []
num_features = len(columns)
for i in range(len(frame_interval)):
item = []
for j in range(num_features):
item.append(frame_interval[columns[j]].values[i])
values.append(item)
_data_frame = pd.DataFrame(values)
csv_file_name = "sensorData_frame.csv"
_data_frame.to_csv(csv_file_name, sep=',', encoding='utf-8',
header=False) # save the data to a csv file
tf.logging.set_verbosity(tf.logging.INFO)
if number_of_points is None:
reader = tf.contrib.timeseries.CSVReader(
csv_file_name,
column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES, ) +
(tf.contrib.timeseries.TrainEvalFeatures.VALUES, ) *
num_features))
else:
x = np.arange(len(frame_interval))
y = np.zeros((len(frame_interval), len(columns)))
for i in range(len(columns)):
y[:, i] = frame_interval[columns[i]].values
data = {
tf.contrib.timeseries.TrainEvalFeatures.TIMES:
x[0:number_of_points],
tf.contrib.timeseries.TrainEvalFeatures.VALUES:
y[0:number_of_points],
}
reader = tf.contrib.timeseries.NumpyReader(data)
train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(
reader, batch_size=batch_size, window_size=windows_size)
estimator = ts_estimators.TimeSeriesRegressor(
model=_LSTMModel(num_features=num_features, num_units=num_units),
optimizer=tf.train.AdamOptimizer(learning_rate),
model_dir=model_dir)
estimator.train(input_fn=train_input_fn, steps=number_of_steps_to_train)
evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)
# Predict starting after the evaluation
(predictions, ) = tuple(
estimator.predict(
input_fn=tf.contrib.timeseries.predict_continuation_input_fn(
evaluation, steps=400)))
observed_times = evaluation["times"][0]
observed = evaluation["observed"][0, :, :]
evaluated_times = evaluation["times"][0]
evaluated = evaluation["mean"][0]
predicted_times = predictions['times']
predicted = predictions["mean"]
plt.figure(figsize=(15, num_features))
plt.axvline(99, linestyle="dotted", linewidth=4, color='r')
observed_lines = plt.plot(observed_times,
observed,
label="observation",
color="k")
evaluated_lines = plt.plot(evaluated_times,
evaluated,
label="evaluation",
color="g")
predicted_lines = plt.plot(predicted_times,
predicted,
label="prediction",
color="r")
plt.legend(
handles=[observed_lines[0], evaluated_lines[0], predicted_lines[0]],
loc="upper left")
plt.savefig('predict_result.jpg')
return estimator
