def create_model(scope, index, prefix, seed):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
# split = splitter.splits[index]
pipe = InputPipe(inp=None, features='./data/train_normed.csv', n_pages=train_size,
mode=ModelMode.TRAIN, batch_size=batch_size, n_epoch=None, verbose=verbose,
train_completeness_threshold=train_completeness_threshold,
predict_completeness_threshold=train_completeness_threshold,
train_window=train_window, predict_window=predict_window,
rand_seed=seed, train_skip_first=hparams.train_skip_first,
back_offset=predict_window if forward_split else 0)
inp_scope.reuse_variables()
# asgd_decay = 0.99 if avg_sgd else None
train_model = Model(pipe, hparams, is_train=True, graph_prefix=prefix, asgd_decay=asgd_decay, seed=seed)
scope.reuse_variables()
eval_stages = []
if write_summaries:
summ_path = f"{logdir}/{name}_{index}"
if os.path.exists(summ_path):
shutil.rmtree(summ_path)
summ_writer = tf.summary.FileWriter(summ_path) # , graph=tf.get_default_graph()
else:
summ_writer = None
stop_metric = None
return ModelTrainerV2(train_model, eval_stages, index, patience=patience,
stop_metric=stop_metric, summary_writer=summ_writer)
def predict(checkpoints, hparams, return_x=False, verbose=False, predict_window=6, back_offset=0, n_models=1,
target_model=0, asgd=False, seed=1, batch_size=1024):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
pipe = InputPipe(inp, page_features(inp), inp.n_pages, mode=ModelMode.PREDICT, batch_size=batch_size,
n_epoch=1, verbose=verbose,
train_completeness_threshold=0.01,
predict_window=predict_window,
predict_completeness_threshold=0.0, train_window=hparams.train_window,
back_offset=back_offset)
asgd_decay = 0.99 if asgd else None
if n_models == 1:
model = Model(pipe, hparams, is_train=False, seed=seed, asgd_decay=asgd_decay)
else:
models = []
for i in range(n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(Model(pipe, hparams, is_train=False, seed=seed, asgd_decay=asgd_decay, graph_prefix=prefix))
model = models[target_model]
if asgd:
var_list = model.ema.variables_to_restore()
print("$$$$$$$$$=",var_list)
prefix = f"m_{target_model}"
for var in list(var_list.keys()):
if var.endswith('ExponentialMovingAverage') and not var.startswith(prefix):
del var_list[var]
else:
var_list = None
saver = tf.train.Saver(name='eval_saver', var_list=var_list)
x_buffer = []
predictions = None
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) as sess:
pipe.load_vars(sess)
for checkpoint in checkpoints:
pred_buffer = []
pipe.init_iterator(sess)
saver.restore(sess, checkpoint)
cnt = 0
while True:
try:
if return_x:
pred, x, pname = sess.run([model.predictions, model.inp.true_x, model.inp.page_ix])
else:
pred, pname = sess.run([model.predictions, model.inp.page_ix])
utf_names = [str(name, 'utf-8') for name in pname]
pred_df = pd.DataFrame(index=utf_names, data=np.expm1(pred))
pred_buffer.append(pred_df)
if return_x:
# noinspection PyUnboundLocalVariable
x_values = pd.DataFrame(index=utf_names, data=np.round(np.expm1(x)).astype(np.int64))
x_buffer.append(x_values)
newline = cnt % 80 == 0
if cnt > 0:
print('.', end='\n' if newline else '', flush=True)
if newline:
print(cnt, end='')
cnt += 1
except tf.errors.OutOfRangeError:
print('🎉')
break
cp_predictions = pd.concat(pred_buffer)
if predictions is None:
predictions = cp_predictions
else:
predictions += cp_predictions
predictions /= len(checkpoints)
offset = pd.Timedelta(back_offset, 'D')
start_prediction = inp.data_end + pd.Timedelta('1D') - offset
end_prediction = start_prediction + pd.Timedelta(predict_window - 1, 'D')
predictions.columns = pd.date_range(start_prediction, end_prediction)
if return_x:
x = pd.concat(x_buffer)
start_data = inp.data_end - pd.Timedelta(hparams.train_window - 1, 'D') - back_offset
end_data = inp.data_end - back_offset
x.columns = pd.date_range(start_data, end_data)
return predictions, x
else:
return predictions
def create_model(scope, index, prefix, seed):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
split = splitter.splits[index]
pipe = InputPipe(inp, features=split.train_set, n_pages=split.train_size,
mode=ModelMode.TRAIN, batch_size=batch_size, n_epoch=None, verbose=verbose,
train_completeness_threshold=train_completeness_threshold,
predict_completeness_threshold=train_completeness_threshold, train_window=train_window,
predict_window=predict_window,
rand_seed=seed, train_skip_first=hparams.train_skip_first,
back_offset=predict_window if forward_split else 0)
inp_scope.reuse_variables()
if side_split:
side_eval_pipe = InputPipe(inp, features=split.test_set, n_pages=split.test_size,
mode=ModelMode.EVAL, batch_size=eval_batch_size, n_epoch=None,
verbose=verbose, predict_window=predict_window,
train_completeness_threshold=0.01, predict_completeness_threshold=0,
train_window=train_window, rand_seed=seed, runs_in_burst=eval_batches,
back_offset=predict_window * (2 if forward_split else 1))
else:
side_eval_pipe = None
if forward_split:
forward_eval_pipe = InputPipe(inp, features=split.test_set, n_pages=split.test_size,
mode=ModelMode.EVAL, batch_size=eval_batch_size, n_epoch=None,
verbose=verbose, predict_window=predict_window,
train_completeness_threshold=0.01, predict_completeness_threshold=0,
train_window=train_window, rand_seed=seed, runs_in_burst=eval_batches,
back_offset=predict_window)
else:
forward_eval_pipe = None
avg_sgd = asgd_decay is not None
#asgd_decay = 0.99 if avg_sgd else None
train_model = Model(pipe, hparams, is_train=True, graph_prefix=prefix, asgd_decay=asgd_decay, seed=seed)
scope.reuse_variables()
eval_stages = []
if side_split:
side_eval_model = Model(side_eval_pipe, hparams, is_train=False,
#loss_mask=np.concatenate([np.zeros(50, dtype=np.float32), np.ones(10, dtype=np.float32)]),
seed=seed)
eval_stages.append((Stage.EVAL_SIDE, side_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_SIDE_EMA, side_eval_model))
if forward_split:
forward_eval_model = Model(forward_eval_pipe, hparams, is_train=False, seed=seed)
eval_stages.append((Stage.EVAL_FRWD, forward_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_FRWD_EMA, forward_eval_model))
if write_summaries:
summ_path = f"{logdir}/{name}_{index}"
if os.path.exists(summ_path):
shutil.rmtree(summ_path)
summ_writer = tf.summary.FileWriter(summ_path) # , graph=tf.get_default_graph()
else:
summ_writer = None
if do_eval and forward_split:
stop_metric = lambda metrics: metrics[Stage.EVAL_FRWD]['SMAPE'].avg_epoch
else:
stop_metric = None
return ModelTrainerV2(train_model, eval_stages, index, patience=patience,
stop_metric=stop_metric,
summary_writer=summ_writer)
def main(_):
if len(sys.argv) < 3:
print(
'Usage: ucdoc_saved_model.py [--model_version=y] --data_dir=xxx --ckpt_dir=xxx --saved_dir=xxx'
)
sys.exit(-1)
if FLAGS.training_iteration <= 0:
print('Please specify a positive value for training iteration.')
sys.exit(-1)
if FLAGS.model_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
# create deploy model first
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
#inp = VarFeeder.read_vars("data/vars")
inp = VarFeeder.read_vars(FLAGS.data_dir)
pipe = InputPipe(inp,
ucdoc_features(inp),
inp.hits.shape[0],
mode=ModelMode.PREDICT,
batch_size=FLAGS.batch_size,
n_epoch=1,
verbose=False,
train_completeness_threshold=0.01,
predict_window=FLAGS.predict_window,
predict_completeness_threshold=0.0,
train_window=FLAGS.train_window,
back_offset=FLAGS.predict_window + 1)
asgd_decay = 0.99 if FLAGS.asgd else None
if FLAGS.n_models == 1:
model = Model(pipe,
build_from_set(FLAGS.hparam_set),
is_train=False,
seed=1,
asgd_decay=asgd_decay)
else:
models = []
for i in range(FLAGS.n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(
Model(pipe,
build_from_set(FLAGS.hparam_set),
is_train=False,
seed=1,
asgd_decay=asgd_decay,
graph_prefix=prefix))
model = models[FLAGS.target_model]
# load checkpoint model from training
#ckpt_path = FLAGS.ckpt_dir
print('loading checkpoint model...')
ckpt_file = tf.train.latest_checkpoint(FLAGS.ckpt_dir)
#graph = tf.Graph()
graph = model.predictions.graph
saver = tf.train.Saver(name='deploy_saver', var_list=None)
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
pipe.load_vars(sess)
pipe.init_iterator(sess)
saver.restore(sess, ckpt_file)
print('Done loading checkpoint model')
export_path_base = FLAGS.saved_dir
export_path = os.path.join(
tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(FLAGS.model_version)))
print('Exporting trained model to', export_path)
if os.path.isdir(export_path):
shutil.rmtree(export_path)
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
true_x = tf.saved_model.utils.build_tensor_info(model.inp.true_x)
time_x = tf.saved_model.utils.build_tensor_info(model.inp.time_x)
norm_x = tf.saved_model.utils.build_tensor_info(model.inp.norm_x)
lagged_x = tf.saved_model.utils.build_tensor_info(model.inp.lagged_x)
true_y = tf.saved_model.utils.build_tensor_info(model.inp.true_y)
time_y = tf.saved_model.utils.build_tensor_info(model.inp.time_y)
norm_y = tf.saved_model.utils.build_tensor_info(model.inp.norm_y)
norm_mean = tf.saved_model.utils.build_tensor_info(model.inp.norm_mean)
norm_std = tf.saved_model.utils.build_tensor_info(model.inp.norm_std)
pg_features = tf.saved_model.utils.build_tensor_info(
model.inp.ucdoc_features)
page_ix = tf.saved_model.utils.build_tensor_info(model.inp.page_ix)
pred = tf.saved_model.utils.build_tensor_info(model.predictions)
labeling_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
"truex": true_x,
"timex": time_x,
"normx": norm_x,
"laggedx": lagged_x,
"truey": true_y,
"timey": time_y,
"normy": norm_y,
"normmean": norm_mean,
"normstd": norm_std,
"page_features": pg_features,
"pageix": page_ix,
},
outputs={"pred": pred},
method_name="tensorflow/serving/predict"))
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
labeling_signature
},
main_op=tf.tables_initializer(),
strip_default_attrs=True)
builder.save()
print("Build Done")
def predict(checkpoints,
hparams,
datadir="data",
verbose=False,
n_models=1,
target_model=0,
asgd=False,
seed=1,
batch_size=50):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars(os.path.join(datadir, "vars"))
pipe = InputPipe(datadir,
inp,
infer_features(inp),
mode=ModelMode.PREDICT,
batch_size=batch_size,
n_epoch=1,
verbose=verbose,
train_completeness_threshold=0.01,
train_window=hparams.train_window)
asgd_decay = 0.99 if asgd else None
if n_models == 1:
model = Model(pipe,
hparams,
is_train=False,
seed=seed,
asgd_decay=asgd_decay)
else:
models = []
for i in range(n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(
Model(pipe,
hparams,
is_train=False,
seed=seed,
asgd_decay=asgd_decay,
graph_prefix=prefix))
model = models[target_model]
if asgd:
var_list = model.ema.variables_to_restore()
prefix = f"m_{target_model}"
for var in list(var_list.keys()):
if var.endswith(
'ExponentialMovingAverage') and not var.startswith(prefix):
del var_list[var]
else:
var_list = None
saver = tf.train.Saver(name='eval_saver', var_list=var_list)
x_buffer = []
predictions = None
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
pipe.load_vars(sess)
for checkpoint in checkpoints:
pred_buffer = []
pipe.init_iterator(sess)
saver.restore(sess, checkpoint)
cnt = 0
while True:
try:
pred, pname = sess.run([model.prediction, model.inp.vm_ix])
# utf_names = [str(name, 'utf-8') for name in pname]
utf_names = pname
pred_df = pd.DataFrame(index=utf_names,
data=np.expm1(pred) - 1)
pred_buffer.append(pred_df)
newline = cnt % 80 == 0
if cnt > 0:
print('.', end='\n' if newline else '', flush=True)
if newline:
print(cnt, end='')
cnt += 1
except tf.errors.OutOfRangeError:
print('Done!')
break
cp_predictions = pd.concat(pred_buffer)
if predictions is None:
predictions = cp_predictions
else:
predictions += cp_predictions
predictions /= len(checkpoints)
return predictions.iloc[:, -1]
def main(_):
if not FLAGS.server:
print('please specify server host:port')
return
channel = grpc.insecure_channel(FLAGS.server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = "ucdoc"
request.model_spec.signature_name = "serving_default"
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
pipe = InputPipe(inp,
ucdoc_features(inp),
inp.n_pages,
mode=ModelMode.PREDICT,
batch_size=FLAGS.batch_size,
n_epoch=1,
verbose=FLAGS.verbose,
train_completeness_threshold=0.01,
predict_window=FLAGS.predict_window,
predict_completeness_threshold=0.0,
train_window=FLAGS.train_window,
back_offset=FLAGS.predict_window + 1)
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
pipe.load_vars(sess)
pipe.init_iterator(sess)
while True:
try:
truex, timex, normx, laggedx, truey, timey, normy, normmean, normstd, pgfeatures, pageix = \
sess.run([pipe.true_x, pipe.time_x, pipe.norm_x, pipe.lagged_x, pipe.true_y, pipe.time_y,
pipe.norm_y, pipe.norm_mean, pipe.norm_std, pipe.ucdoc_features, pipe.page_ix])
request.inputs["truex"].CopyFrom(tf.make_tensor_proto(truex))
request.inputs["timex"].CopyFrom(tf.make_tensor_proto(timex))
request.inputs["normx"].CopyFrom(tf.make_tensor_proto(normx))
request.inputs["laggedx"].CopyFrom(
tf.make_tensor_proto(laggedx))
request.inputs["truey"].CopyFrom(tf.make_tensor_proto(truey))
request.inputs["timey"].CopyFrom(tf.make_tensor_proto(timey))
request.inputs["normy"].CopyFrom(tf.make_tensor_proto(normy))
request.inputs["normmean"].CopyFrom(
tf.make_tensor_proto(normmean))
request.inputs["normstd"].CopyFrom(
tf.make_tensor_proto(normstd))
request.inputs["page_features"].CopyFrom(
tf.make_tensor_proto(pgfeatures))
request.inputs["pageix"].CopyFrom(tf.make_tensor_proto(pageix))
response = stub.Predict(request, 10)
tensor_proto = response.outputs['pred']
if not 'pred_result' in locals():
pred_result = tf.contrib.util.make_ndarray(tensor_proto)
else:
pred_result = np.concatenate([
pred_result,
tf.contrib.util.make_ndarray(tensor_proto)
])
except tf.errors.OutOfRangeError:
print('done with prediction')
break
pred_result = np.expm1(pred_result) + 0.5
pred_result = pred_result.astype(int)
if not os.path.exists(FLAGS.result_dir):
os.mkdir(FLAGS.result_dir)
result_file = os.path.join(FLAGS.result_dir, "predict.pkl")
pickle.dump(pred_result, open(result_file, "wb"))
print('finished prediction')
def main(_):
if len(sys.argv) < 3:
print(
'Usage: saved_model.py [--model_version=y] --data_dir=xxx --ckpt_dir=xxx --saved_dir=xxx'
)
sys.exit(-1)
if FLAGS.training_iteration <= 0:
print('Please specify a positive value for training iteration.')
sys.exit(-1)
if FLAGS.model_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
with open(FLAGS.config_file, 'r') as ymlfile:
cfg = yaml.load(ymlfile)
holiday_list = cfg['pipeline']['normalization']['holidays']
if FLAGS.back_offset < FLAGS.predict_window:
extend_inp(FLAGS.data_dir, FLAGS.predict_window, holiday_list)
# create deploy model first
back_offset_ = FLAGS.back_offset
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
if FLAGS.back_offset < FLAGS.predict_window:
inp = VarFeeder.read_vars(
os.path.join(FLAGS.data_dir, 'predict_future'))
back_offset_ += FLAGS.predict_window
else:
inp = VarFeeder.read_vars(FLAGS.data_dir)
pipe = InputPipe(inp,
ucdoc_features(inp),
inp.hits.shape[0],
mode=ModelMode.PREDICT,
batch_size=FLAGS.batch_size,
n_epoch=1,
verbose=False,
train_completeness_threshold=0.01,
predict_window=FLAGS.predict_window,
predict_completeness_threshold=0.0,
train_window=FLAGS.train_window,
back_offset=back_offset_)
asgd_decay = 0.99 if FLAGS.asgd else None
if FLAGS.n_models == 1:
model = Model(pipe,
build_from_set(FLAGS.hparam_set),
is_train=False,
seed=1,
asgd_decay=asgd_decay)
else:
models = []
for i in range(FLAGS.n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(
Model(pipe,
build_from_set(FLAGS.hparam_set),
is_train=False,
seed=1,
asgd_decay=asgd_decay,
graph_prefix=prefix))
model = models[FLAGS.target_model]
if FLAGS.asgd:
var_list = model.ema.variables_to_restore()
if FLAGS.n_models > 1:
prefix = f"m_{target_model}"
for var in list(var_list.keys()):
if var.endswith('ExponentialMovingAverage'
) and not var.startswith(prefix):
del var_list[var]
else:
var_list = None
# load checkpoint model from training
#ckpt_path = FLAGS.ckpt_dir
print('loading checkpoint model...')
ckpt_file = tf.train.latest_checkpoint(FLAGS.ckpt_dir)
#graph = tf.Graph()
graph = model.predictions.graph
init = tf.global_variables_initializer()
saver = tf.train.Saver(name='deploy_saver', var_list=var_list)
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
sess.run(init)
pipe.load_vars(sess)
pipe.init_iterator(sess)
saver.restore(sess, ckpt_file)
print('Done loading checkpoint model')
export_path_base = FLAGS.saved_dir
export_path = os.path.join(
tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(FLAGS.model_version)))
print('Exporting trained model to', export_path)
if os.path.isdir(export_path):
shutil.rmtree(export_path)
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
true_x = tf.saved_model.utils.build_tensor_info(
model.inp.true_x) # pipe.true_x
time_x = tf.saved_model.utils.build_tensor_info(
model.inp.time_x) # pipe.time_x
norm_x = tf.saved_model.utils.build_tensor_info(
model.inp.norm_x) # pipe.norm_x
lagged_x = tf.saved_model.utils.build_tensor_info(
model.inp.lagged_x) # pipe.lagged_x
true_y = tf.saved_model.utils.build_tensor_info(
model.inp.true_y) # pipe.true_y
time_y = tf.saved_model.utils.build_tensor_info(
model.inp.time_y) # pipe.time_y
norm_y = tf.saved_model.utils.build_tensor_info(
model.inp.norm_y) # pipe.norm_y
norm_mean = tf.saved_model.utils.build_tensor_info(
model.inp.norm_mean) # pipe.norm_mean
norm_std = tf.saved_model.utils.build_tensor_info(
model.inp.norm_std) # pipe.norm_std
pg_features = tf.saved_model.utils.build_tensor_info(
model.inp.ucdoc_features) # pipe.ucdoc_features
page_ix = tf.saved_model.utils.build_tensor_info(
model.inp.page_ix) # pipe.page_ix
#pred = tf.saved_model.utils.build_tensor_info(graph.get_operation_by_name('m_0/add').outputs[0])
pred = tf.saved_model.utils.build_tensor_info(model.predictions)
labeling_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
"truex": true_x,
"timex": time_x,
"normx": norm_x,
"laggedx": lagged_x,
"truey": true_y,
"timey": time_y,
"normy": norm_y,
"normmean": norm_mean,
"normstd": norm_std,
"page_features": pg_features,
"pageix": page_ix,
},
outputs={"predictions": pred},
method_name="tensorflow/serving/predict"))
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
labeling_signature
},
main_op=tf.tables_initializer(),
strip_default_attrs=True)
builder.save()
print("Build Done")
def create_model(scope, index, prefix, seed):
# todo 主要是创建了模型，以及返回一些None的东西。
# 数据在构建模型的时候使用了，模型中只使用了数据的预测窗口的长度--不对，应该是创建模型的时候直接喂入数据了。
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
split = splitter.splits[index]
pipe = InputPipe(
inp,
features=split.train_set,
n_pages=split.train_size,
mode=ModelMode.TRAIN,
batch_size=batch_size,
n_epoch=None,
verbose=verbose,
train_completeness_threshold=train_completeness_threshold,
predict_completeness_threshold=train_completeness_threshold,
train_window=train_window,
predict_window=predict_window,
rand_seed=seed,
train_skip_first=hparams.train_skip_first,
back_offset=predict_window if forward_split else 0)
inp_scope.reuse_variables()
# todo side_split: False; forward_split:False; eval_stages: [];
if side_split:
side_eval_pipe = InputPipe(
inp,
features=split.test_set,
n_pages=split.test_size,
mode=ModelMode.EVAL,
batch_size=eval_batch_size,
n_epoch=None,
verbose=verbose,
predict_window=predict_window,
train_completeness_threshold=0.01,
predict_completeness_threshold=0,
train_window=train_window,
rand_seed=seed,
runs_in_burst=eval_batches,
back_offset=predict_window *
(2 if forward_split else 1))
else:
side_eval_pipe = None
if forward_split:
forward_eval_pipe = InputPipe(
inp,
features=split.test_set,
n_pages=split.test_size,
mode=ModelMode.EVAL,
batch_size=eval_batch_size,
n_epoch=None,
verbose=verbose,
predict_window=predict_window,
train_completeness_threshold=0.01,
predict_completeness_threshold=0,
train_window=train_window,
rand_seed=seed,
runs_in_burst=eval_batches,
back_offset=predict_window)
else:
forward_eval_pipe = None
avg_sgd = asgd_decay is not None
#asgd_decay = 0.99 if avg_sgd else None
train_model = Model(pipe,
hparams,
is_train=True,
graph_prefix=prefix,
asgd_decay=asgd_decay,
seed=seed)
scope.reuse_variables()
eval_stages = []
if side_split:
# print('2 side_split side_eval_model')
side_eval_model = Model(
side_eval_pipe,
hparams,
is_train=False,
#loss_mask=np.concatenate([np.zeros(50, dtype=np.float32), np.ones(10, dtype=np.float32)]),
seed=seed)
# print("2 side_eval_model -- 2")
# todo TRAIN = 0; EVAL_SIDE = 1; EVAL_FRWD = 2; EVAL_SIDE_EMA = 3; EVAL_FRWD_EMA = 4
eval_stages.append((Stage.EVAL_SIDE, side_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_SIDE_EMA, side_eval_model))
if forward_split:
# print("3 forward_split forward_eval_model")
# tf.reset_default_graph()
forward_eval_model = Model(forward_eval_pipe,
hparams,
is_train=False,
seed=seed)
# print("3 forward_split forward_eval_model -- 2")
eval_stages.append((Stage.EVAL_FRWD, forward_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_FRWD_EMA, forward_eval_model))
if write_summaries:
summ_path = f"{logdir}/{name}_{index}"
# print("write_summaries summ_path",summ_path)
if os.path.exists(summ_path):
shutil.rmtree(summ_path)
summ_writer = tf.summary.FileWriter(
summ_path) # , graph=tf.get_default_graph()
else:
summ_writer = None
if do_eval and forward_split:
stop_metric = lambda metrics: metrics[Stage.EVAL_FRWD]['SMAPE'
].avg_epoch
else:
stop_metric = None
# todo side_split: False; forward_split:False;
# summ_writer=<tensorflow.python.summary.writer.writer.FileWriter object at 0x7ff5dc176710>；
# eval_stages: []; stop_metric=None; patience=2; index=0
# print(f"side_split: {side_split}; forward_split:{forward_split}; summ_writer={summ_writer}；"
# f"eval_stages: {eval_stages}； stop_metric={stop_metric}; patience={patience}; index={index}")
return ModelTrainerV2(train_model,
eval_stages,
index,
patience=patience,
stop_metric=stop_metric,
summary_writer=summ_writer)