def run():
parser = argparse.ArgumentParser(description='Prepare data')
parser.add_argument(
'--data_dir',
default='/home/oniculaescu/train/features',
help=
'The location where we are going to save the features for constructing the NN'
)
parser.add_argument(
'--threshold',
default=0.0,
type=float,
help='Series minimal length threshold/points of data length')
parser.add_argument(
'--add_days',
default=64,
type=int,
help='Number of days to be added in the future for prediction')
parser.add_argument('--start', help='Effective start date')
parser.add_argument('--end', help="Effective end date")
parser.add_argument('--attr',
default='download',
help="tell what pkl file to use for feature creation")
parser.add_argument('--corr_backoffset',
default=0,
type=int,
help="Offset for correlation computation")
args = parser.parse_args()
# get the data
df, nans, starts, ends = prepare_data(args.attr, args.start, args.end,
args.threshold)
# find the working date range
data_start, data_end = df.columns[0], df.columns[-1]
# project date-dependent features (like day of week) to the future dates for prediction
#features_end = data_end + " " + str(pd.Timedelta(args.add_days, unit='D'))
print data_end
data_end_1 = datetime.datetime.strptime(data_end, '%Y-%m-%d')
#features_end = data_end_1 + pd.TimeDelta(args.add_days, unit='D')
features_end = data_end_1 + datetime.timedelta(days=args.add_days)
print("start: " + data_start + ", end: " + data_end + ", features_end: " +
str(features_end))
# Group unique ases by continent
assert df.index.is_monotonic_increasing
continent_map = uniq_continent_map(df.index.values)
# Group unique ases by country
country_map = uniq_country_map(df.index.values)
# yearly autocorrelation
raw_year_autocorr = batch_autocorrelation(df.values, 365, starts, ends,
1.5, args.corr_backoffset)
year_unknown_pct = np.sum(np.isnan(raw_year_autocorr)) / len(
raw_year_autocorr) # type: float
# quarterly autocorrelation
raw_quarter_autocorr = batch_autocorrelation(df.values,
int(round(365.25 / 4)),
starts, ends, 2,
args.corr_backoffset)
quarter_unknown_pct = np.sum(np.isnan(raw_quarter_autocorr)) / len(
raw_quarter_autocorr) # type: float
print("Percent of undefined autocorr = yearly:%.3f, quarterly:%.3f" %
(year_unknown_pct, quarter_unknown_pct))
# Normalise all the things
year_autocorr = normalize(np.nan_to_num(raw_year_autocorr))
quarter_autocorr = normalize(np.nan_to_num(raw_quarter_autocorr))
# Make time-dependent features
features_days = pd.date_range(data_start, features_end)
#dow = normalize(features_days.dayofweek.values)
week_period = 7 / (2 * np.pi)
dow_norm = features_days.dayofweek / week_period
print dow_norm
dow = np.stack([np.cos(dow_norm), np.sin(dow_norm)], axis=-1)
# Assemble indices for quarterly lagged data
lagged_ix = np.stack(lag_indexes(data_start, features_end), axis=-1)
#page_popularity = df.median(axis=1)
#page_popularity = (page_popularity - page_popularity.mean()) / page_popularity.std()
# Put NaNs back
df[nans] = np.NaN
# Assemble final output
tensors = dict(
hits=df,
lagged_ix=lagged_ix,
continent_map=continent_map,
#country_map=country_map,
as_ix=df.index.values,
year_autocorr=year_autocorr,
quarter_autocorr=quarter_autocorr,
dow=dow,
)
plain = dict(features_days=len(features_days),
data_days=len(df.columns),
n_ases=len(df),
data_start=data_start,
data_end=data_end,
features_end=features_end)
print args.data_dir
#print tensors
# Store data to the disk
VarFeeder(args.data_dir, tensors, plain)
def run(args):
# Get the data
df, nans, starts, ends = prepare_data(args['start'], args['end'], args['valid_threshold'])
# Our working date range
data_start, data_end = df.columns[0], df.columns[-1]
# We have to project some date-dependent features (day of week, etc) to the future dates for prediction
features_end = data_end + pd.Timedelta(args['add_days'], unit='D')
print(f"start: {data_start}, end:{data_end}, features_end:{features_end}")
# Group unique pages by agents
assert df.index.is_monotonic_increasing # 判断index是不是单调增
page_map = uniq_page_map(df.index.values)
# Yearly(annual) autocorrelation
raw_year_autocorr = batch_autocorr(df.values, 365, starts, ends, 1.5, args['corr_backoffset'])
year_unknown_pct = np.sum(np.isnan(raw_year_autocorr))/len(raw_year_autocorr) # type: float
# Quarterly autocorrelation
raw_quarter_autocorr = batch_autocorr(df.values, int(round(365.25/4)), starts, ends, 2, args['corr_backoffset'])
quarter_unknown_pct = np.sum(np.isnan(raw_quarter_autocorr)) / len(raw_quarter_autocorr) # type: float
print("Percent of undefined autocorr = yearly:%.3f, quarterly:%.3f" % (year_unknown_pct, quarter_unknown_pct))
# Normalize all the things
year_autocorr = normalize(np.nan_to_num(raw_year_autocorr)) # replace NaN with 0 and infinity with large finite numbers
quarter_autocorr = normalize(np.nan_to_num(raw_quarter_autocorr))
# Calculate and encode page features (site, contry, etc)
page_features = make_page_features(df.index.values)
encoded_page_features = encode_page_features(page_features)
# Make time-dependent features
features_days = pd.date_range(data_start, features_end)
#dow = normalize(features_days.dayofweek.values)
week_period = 7 / (2 * np.pi)
dow_norm = features_days.dayofweek.values / week_period
dow = np.stack([np.cos(dow_norm), np.sin(dow_norm)], axis=-1) # 纵向连接，两列
# Assemble 聚集 indices 指数 for quarterly lagged data
lagged_ix = np.stack(lag_indexes(data_start, features_end), axis=-1)
page_popularity = df.median(axis=1)
page_popularity = (page_popularity - page_popularity.mean()) / page_popularity.std() # 标准化
# Put NaNs back
df[nans] = np.NaN
# Assemble final output
tensors = dict(
hits=df,
lagged_ix=lagged_ix,
page_map=page_map,
page_ix=df.index.values,
pf_agent=encoded_page_features['agent'],
pf_country=encoded_page_features['country'],
pf_site=encoded_page_features['site'],
page_popularity=page_popularity,
year_autocorr=year_autocorr,
quarter_autocorr=quarter_autocorr,
dow=dow,
)
plain = dict(
features_days=len(features_days),
data_days=len(df.columns),
n_pages=len(df),
data_start=data_start,
data_end=data_end,
features_end=features_end
)
# Store data to the disk
VarFeeder(args['data_dir'], tensors, plain)
def main(_):
if len(sys.argv) < 3:
print(
'Usage: ucdoc_saved_model.py [--model_version=y] --data_dir=xxx --ckpt_dir=xxx --saved_dir=xxx'
)
sys.exit(-1)
if FLAGS.training_iteration <= 0:
print('Please specify a positive value for training iteration.')
sys.exit(-1)
if FLAGS.model_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
# create deploy model first
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
#inp = VarFeeder.read_vars("data/vars")
inp = VarFeeder.read_vars(FLAGS.data_dir)
pipe = InputPipe(inp,
ucdoc_features(inp),
inp.hits.shape[0],
mode=ModelMode.PREDICT,
batch_size=FLAGS.batch_size,
n_epoch=1,
verbose=False,
train_completeness_threshold=0.01,
predict_window=FLAGS.predict_window,
predict_completeness_threshold=0.0,
train_window=FLAGS.train_window,
back_offset=FLAGS.predict_window + 1)
asgd_decay = 0.99 if FLAGS.asgd else None
if FLAGS.n_models == 1:
model = Model(pipe,
build_from_set(FLAGS.hparam_set),
is_train=False,
seed=1,
asgd_decay=asgd_decay)
else:
models = []
for i in range(FLAGS.n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(
Model(pipe,
build_from_set(FLAGS.hparam_set),
is_train=False,
seed=1,
asgd_decay=asgd_decay,
graph_prefix=prefix))
model = models[FLAGS.target_model]
# load checkpoint model from training
#ckpt_path = FLAGS.ckpt_dir
print('loading checkpoint model...')
ckpt_file = tf.train.latest_checkpoint(FLAGS.ckpt_dir)
#graph = tf.Graph()
graph = model.predictions.graph
saver = tf.train.Saver(name='deploy_saver', var_list=None)
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
pipe.load_vars(sess)
pipe.init_iterator(sess)
saver.restore(sess, ckpt_file)
print('Done loading checkpoint model')
export_path_base = FLAGS.saved_dir
export_path = os.path.join(
tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(FLAGS.model_version)))
print('Exporting trained model to', export_path)
if os.path.isdir(export_path):
shutil.rmtree(export_path)
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
true_x = tf.saved_model.utils.build_tensor_info(model.inp.true_x)
time_x = tf.saved_model.utils.build_tensor_info(model.inp.time_x)
norm_x = tf.saved_model.utils.build_tensor_info(model.inp.norm_x)
lagged_x = tf.saved_model.utils.build_tensor_info(model.inp.lagged_x)
true_y = tf.saved_model.utils.build_tensor_info(model.inp.true_y)
time_y = tf.saved_model.utils.build_tensor_info(model.inp.time_y)
norm_y = tf.saved_model.utils.build_tensor_info(model.inp.norm_y)
norm_mean = tf.saved_model.utils.build_tensor_info(model.inp.norm_mean)
norm_std = tf.saved_model.utils.build_tensor_info(model.inp.norm_std)
pg_features = tf.saved_model.utils.build_tensor_info(
model.inp.ucdoc_features)
page_ix = tf.saved_model.utils.build_tensor_info(model.inp.page_ix)
pred = tf.saved_model.utils.build_tensor_info(model.predictions)
labeling_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
"truex": true_x,
"timex": time_x,
"normx": norm_x,
"laggedx": lagged_x,
"truey": true_y,
"timey": time_y,
"normy": norm_y,
"normmean": norm_mean,
"normstd": norm_std,
"page_features": pg_features,
"pageix": page_ix,
},
outputs={"pred": pred},
method_name="tensorflow/serving/predict"))
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
labeling_signature
},
main_op=tf.tables_initializer(),
strip_default_attrs=True)
builder.save()
print("Build Done")
def predict(checkpoints, hparams, return_x=False, verbose=False, predict_window=6, back_offset=0, n_models=1,
target_model=0, asgd=False, seed=1, batch_size=1024):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
pipe = InputPipe(inp, page_features(inp), inp.n_pages, mode=ModelMode.PREDICT, batch_size=batch_size,
n_epoch=1, verbose=verbose,
train_completeness_threshold=0.01,
predict_window=predict_window,
predict_completeness_threshold=0.0, train_window=hparams.train_window,
back_offset=back_offset)
asgd_decay = 0.99 if asgd else None
if n_models == 1:
model = Model(pipe, hparams, is_train=False, seed=seed, asgd_decay=asgd_decay)
else:
models = []
for i in range(n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(Model(pipe, hparams, is_train=False, seed=seed, asgd_decay=asgd_decay, graph_prefix=prefix))
model = models[target_model]
if asgd:
var_list = model.ema.variables_to_restore()
prefix = f"m_{target_model}"
for var in list(var_list.keys()):
if var.endswith('ExponentialMovingAverage') and not var.startswith(prefix):
del var_list[var]
else:
var_list = None
saver = tf.train.Saver(name='eval_saver', var_list=var_list)
x_buffer = []
predictions = None
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) as sess:
pipe.load_vars(sess)
for checkpoint in checkpoints:
pred_buffer = []
pipe.init_iterator(sess)
saver.restore(sess, checkpoint)
cnt = 0
while True:
try:
if return_x:
pred, x, pname = sess.run([model.predictions, model.inp.true_x, model.inp.page_ix])
else:
pred, pname = sess.run([model.predictions, model.inp.page_ix])
utf_names = [str(name, 'utf-8') for name in pname]
pred_df = pd.DataFrame(index=utf_names, data=np.expm1(pred))
pred_buffer.append(pred_df)
if return_x:
# noinspection PyUnboundLocalVariable
x_values = pd.DataFrame(index=utf_names, data=np.round(np.expm1(x)).astype(np.int64))
x_buffer.append(x_values)
newline = cnt % 80 == 0
if cnt > 0:
print('.', end='\n' if newline else '', flush=True)
if newline:
print(cnt, end='')
cnt += 1
except tf.errors.OutOfRangeError:
print('🎉')
break
cp_predictions = pd.concat(pred_buffer)
if predictions is None:
predictions = cp_predictions
else:
predictions += cp_predictions
predictions /= len(checkpoints)
offset = pd.Timedelta(back_offset, 'D')
start_prediction = inp.data_end + pd.Timedelta('1D') - offset
end_prediction = start_prediction + pd.Timedelta(predict_window - 1, 'D')
predictions.columns = pd.date_range(start_prediction, end_prediction)
if return_x:
x = pd.concat(x_buffer)
start_data = inp.data_end - pd.Timedelta(hparams.train_window - 1, 'D') - back_offset
end_data = inp.data_end - back_offset
x.columns = pd.date_range(start_data, end_data)
return predictions, x
else:
return predictions
def run():
parser = argparse.ArgumentParser(description='Prepare data')
parser.add_argument('data_dir')
parser.add_argument(
'--valid_threshold',
default=0.04,
type=float,
help="Series minimal length threshold (pct of data length)")
parser.add_argument('--add_timestamp',
default=288,
type=int,
help="Add N timestamp in a future for prediction")
parser.add_argument(
'--start',
default=0,
type=int,
help="Effective start date. Data before the start is dropped")
parser.add_argument(
'--end',
default=-288,
type=int,
help="Effective end date. Data past the end is dropped")
parser.add_argument('--corr_backoffset',
default=0,
type=int,
help='Offset for correlation calculation')
parser.add_argument('--split_df',
default=0,
type=int,
help="Whether to split vms w.r.t. abnormal behaviour")
args = parser.parse_args()
# Get the data
df, starts, ends = prepare_data(args.split_df, args.start, args.end,
args.valid_threshold)
# Our working date range
data_start, data_end = df.columns[0], df.columns[-1]
# We have to project some date-dependent features (day of week, etc) to the future dates for prediction
features_end = data_end + args.add_timestamp
print(f"start: {data_start}, end:{data_end}, features_end:{features_end}")
features_time = features_end - data_start
assert df.index.is_monotonic_increasing
# daily autocorrelation
day_autocorr = batch_autocorr(df.values, 288, starts, ends, 1.5,
args.corr_backoffset)
# weekly autocorrelation
week_autocorr = batch_autocorr(df.values, 288 * 7, starts, ends, 2,
args.corr_backoffset)
# Normalise all the things
day_autocorr = normalize(np.nan_to_num(day_autocorr))
week_autocorr = normalize(np.nan_to_num(week_autocorr))
# Make time-dependent features
feature_time = np.arange(data_start, features_end + 1) % 288
day_period = 288 / (2 * np.pi)
dow_norm = feature_time / day_period
dow = np.stack([np.cos(dow_norm), np.sin(dow_norm)], axis=-1)
# Assemble indices for quarterly lagged data
lagged_ix = np.stack(lag_indexes(data_start, features_end), axis=-1)
# Assemble final output
tensors = dict(
usage=df,
lagged_ix=lagged_ix,
vm_ix=df.index.values,
day_autocorr=day_autocorr,
week_autocorr=week_autocorr,
starts=starts,
ends=ends,
dow=dow,
)
plain = dict(features_time=features_time,
data_time=len(df.columns),
n_vm=len(df),
data_start=data_start,
data_end=data_end,
features_end=features_end)
# Store data to the disk
VarFeeder(args.data_dir, tensors, plain)
def predict(checkpoints, hparams, return_x=False, verbose=False, predict_window=6, back_offset=0, n_models=1,
target_model=0, asgd=False, seed=1, batch_size=1024):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
pipe = InputPipe(inp, page_features(inp), inp.n_pages, mode=ModelMode.PREDICT, batch_size=batch_size,
n_epoch=1, verbose=verbose,
train_completeness_threshold=0.01,
predict_window=predict_window,
predict_completeness_threshold=0.0, train_window=hparams.train_window,
back_offset=back_offset)
asgd_decay = 0.99 if asgd else None
if n_models == 1:
model = Model(pipe, hparams, is_train=False, seed=seed, asgd_decay=asgd_decay)
else:
models = []
for i in range(n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(Model(pipe, hparams, is_train=False, seed=seed, asgd_decay=asgd_decay, graph_prefix=prefix))
model = models[target_model]
if asgd:
var_list = model.ema.variables_to_restore()
print("$$$$$$$$$=",var_list)
prefix = f"m_{target_model}"
for var in list(var_list.keys()):
if var.endswith('ExponentialMovingAverage') and not var.startswith(prefix):
del var_list[var]
else:
var_list = None
saver = tf.train.Saver(name='eval_saver', var_list=var_list)
x_buffer = []
predictions = None
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) as sess:
pipe.load_vars(sess)
for checkpoint in checkpoints:
pred_buffer = []
pipe.init_iterator(sess)
saver.restore(sess, checkpoint)
cnt = 0
while True:
try:
if return_x:
pred, x, pname = sess.run([model.predictions, model.inp.true_x, model.inp.page_ix])
else:
pred, pname = sess.run([model.predictions, model.inp.page_ix])
utf_names = [str(name, 'utf-8') for name in pname]
pred_df = pd.DataFrame(index=utf_names, data=np.expm1(pred))
pred_buffer.append(pred_df)
if return_x:
# noinspection PyUnboundLocalVariable
x_values = pd.DataFrame(index=utf_names, data=np.round(np.expm1(x)).astype(np.int64))
x_buffer.append(x_values)
newline = cnt % 80 == 0
if cnt > 0:
print('.', end='\n' if newline else '', flush=True)
if newline:
print(cnt, end='')
cnt += 1
except tf.errors.OutOfRangeError:
print('🎉')
break
cp_predictions = pd.concat(pred_buffer)
if predictions is None:
predictions = cp_predictions
else:
predictions += cp_predictions
predictions /= len(checkpoints)
offset = pd.Timedelta(back_offset, 'D')
start_prediction = inp.data_end + pd.Timedelta('1D') - offset
end_prediction = start_prediction + pd.Timedelta(predict_window - 1, 'D')
predictions.columns = pd.date_range(start_prediction, end_prediction)
if return_x:
x = pd.concat(x_buffer)
start_data = inp.data_end - pd.Timedelta(hparams.train_window - 1, 'D') - back_offset
end_data = inp.data_end - back_offset
x.columns = pd.date_range(start_data, end_data)
return predictions, x
else:
return predictions
def run():
parser = argparse.ArgumentParser(description='Prepare data')
parser.add_argument('data_dir')
parser.add_argument(
'--valid_threshold',
default=0.0,
type=float,
help="Series minimal length threshold (pct of data length)")
parser.add_argument('--add_days',
default=63,
type=int,
help="Add N days in a future for prediction")
parser.add_argument(
'--start',
help="Effective start date. Data before the start is dropped")
parser.add_argument(
'--end', help="Effective end date. Data past the end is dropped")
parser.add_argument('--corr_backoffset',
default=0,
type=int,
help='Offset for correlation calculation')
args = parser.parse_args()
# print("args",args.start)
# todo python make_features.py data/vars --add_days=63
# Get the data
df, nans, starts, ends = prepare_data(args.start, args.end,
args.valid_threshold)
# print(f"starts={starts},ends={ends}; df.head()={df.head()}")
# Our working date range
# todo 页面最早最晚访问时间
# print("df", df.head(), df.shape)
data_start, data_end = df.columns[0], df.columns[-1]
# We have to project some date-dependent features (day of week, etc) to the future dates for prediction
features_end = data_end + pd.Timedelta(args.add_days, unit='D')
# todo start: 2015-07-01, end:2017-09-11 00:00:00, features_end:2017-11-13 00:00:00
# print(f"start: {data_start}, end:{data_end}, features_end:{features_end}")
# Group unique pages by agents
# print("df.index.is_monotonic_increasing",df.index.is_monotonic_increasing)
assert df.index.is_monotonic_increasing
page_map = uniq_page_map(df.index.values)
# print(f"page_map={page_map}")
# Yearly(annual) autocorrelation
raw_year_autocorr = batch_autocorr(df.values, 365, starts, ends, 1.5,
args.corr_backoffset)
# todo 相关系数为空的数量
year_unknown_pct = np.sum(np.isnan(raw_year_autocorr)) / len(
raw_year_autocorr) # type: float
# Quarterly autocorrelation
raw_quarter_autocorr = batch_autocorr(df.values, int(round(365.25 / 4)),
starts, ends, 2,
args.corr_backoffset)
quarter_unknown_pct = np.sum(np.isnan(raw_quarter_autocorr)) / len(
raw_quarter_autocorr) # type: float
print("Percent of undefined autocorr = yearly:%.3f, quarterly:%.3f" %
(year_unknown_pct, quarter_unknown_pct))
# Normalise all the things使用0代替数组x中的nan元素，使用有限的数字代替inf元素
# todo 对年度相关系数和季度相关系数做均值方差标准化
year_autocorr = normalize(np.nan_to_num(raw_year_autocorr))
quarter_autocorr = normalize(np.nan_to_num(raw_quarter_autocorr))
# Calculate and encode page features
page_features = make_page_features(df.index.values)
encoded_page_features = encode_page_features(page_features)
# todo encoded_page_features 对网页的中介和国家做了 onehot 编码
# print("encoded_page_features",type(encoded_page_features))
# Make time-dependent features
features_days = pd.date_range(data_start, features_end)
#dow = normalize(features_days.dayofweek.values)
week_period = 7 / (2 * np.pi)
dow_norm = features_days.dayofweek.values / week_period
dow = np.stack([np.cos(dow_norm), np.sin(dow_norm)], axis=-1)
# Assemble indices for quarterly lagged data
# todo 分别滑窗3，6，9，12个月的时间index
lagged_ix = np.stack(lag_indexes(data_start, features_end), axis=-1)
# todo data_start=2015-07-01;
# lagged_ix=[[ -1 -1 -1 -1]
# [ -1 -1 -1 -1]
# ...
# [773 681 592 500]
# [774 682 593 501]]
# print(f"data_start={data_start}; lagged_ix={lagged_ix}")
page_popularity = df.median(axis=1)
# todo page_popularity (145036,)
# todo 这里用每个页面的访问量的中位数表示该页面的流行程度
# print("page_popularity",page_popularity.shape,page_popularity)
page_popularity = (page_popularity -
page_popularity.mean()) / page_popularity.std()
# Put NaNs back
df[nans] = np.NaN
# Assemble final output
# todo
# hits : 过滤掉了访问量很多为0的网页，已经按照网页名称排序，并做log变换
# lagged_ix : 日期index的滑窗，小于最早日期的-1做填充
# page_map : 背景：每个页面最多4种中间商。数据：每一行代表一个页面名称，每一列代表一个中间商
# page_ix : 每个网页全名（包括页面名称，中间商，国家等）
# pf_agent : 网页的中间商的onehot编码
# pf_country : 网页国家的onehot编码
# pf_site : 网页地址sire编码
# page_popularity : 用页面访问量的中位数代表流行度
# year_autocorr : 访问量滑窗一年的相关系数
# quarter_autocorr : 访问量滑窗一个季度的相关系数
# dow : 周几转换为sin,cos
tensors = dict(
hits=df,
lagged_ix=lagged_ix,
page_map=page_map,
page_ix=df.index.values,
pf_agent=encoded_page_features['agent'],
pf_country=encoded_page_features['country'],
pf_site=encoded_page_features['site'],
page_popularity=page_popularity,
year_autocorr=year_autocorr,
quarter_autocorr=quarter_autocorr,
dow=dow,
)
# todo
# data_start :访问量的起始日期
# data_end:访问量的终止日期
# features_end:特征的终止日期，比访问量的终止日期晚63天
# features_days:特征横跨多少天
# data_days:总共有多少天的数据
# n_pages :多少个页面
plain = dict(features_days=len(features_days),
data_days=len(df.columns),
n_pages=len(df),
data_start=data_start,
data_end=data_end,
features_end=features_end)
# todo len(features_days)=867,len(df.columns)=805,len(df)=145036,data_start=2015-07-01,
# data_end=2017-09-11 00:00:00,features_end=2017-11-13 00:00:00
# print(f"len(features_days)={len(features_days)},len(df.columns)={len(df.columns)},len(df)={len(df)},data_start={data_start},data_end={data_end},features_end={features_end}")
# Store data to the disk
VarFeeder(args.data_dir, tensors, plain)
def run():
parser = argparse.ArgumentParser(description='Prepare data')
parser.add_argument('data_dir')
parser.add_argument(
'--valid_threshold',
default=0.0,
type=float,
help="Series minimal length threshold (pct of data length)")
parser.add_argument('--add_days',
default=64,
type=int,
help="Add N days in a future for prediction")
parser.add_argument(
'--start',
help="Effective start date. Data before the start is dropped")
parser.add_argument(
'--end', help="Effective end date. Data past the end is dropped")
parser.add_argument('--corr_backoffset',
default=0,
type=int,
help='Offset for correlation calculation')
args = parser.parse_args()
# Get the data
df, nans, starts, ends = prepare_data(args.start, args.end,
args.valid_threshold)
# Our working date range
data_start, data_end = df.columns[0], df.columns[-1]
# We have to project some date-dependent features (day of week, etc) to the future dates for prediction
features_end = data_end + pd.Timedelta(args.add_days, unit='D')
print(f"start: {data_start}, end:{data_end}, features_end:{features_end}")
# Group unique pages by agents
assert df.index.is_monotonic_increasing
page_map = uniq_page_map(df.index.values)
# Yearly(annual) autocorrelation of each page. The return is a list of auto_correl of each page.
raw_year_autocorr = batch_autocorr(df.values, 365, starts, ends, 1.5,
args.corr_backoffset)
year_unknown_pct = np.sum(np.isnan(raw_year_autocorr)) / len(
raw_year_autocorr) # type: float
# Quarterly autocorrelation of each page. The return is a list of auto_correl of each page.
raw_quarter_autocorr = batch_autocorr(df.values, int(round(365.25 / 4)),
starts, ends, 2,
args.corr_backoffset)
quarter_unknown_pct = np.sum(np.isnan(raw_quarter_autocorr)) / len(
raw_quarter_autocorr) # type: float
print("Percent of undefined autocorr = yearly:%.3f, quarterly:%.3f" %
(year_unknown_pct, quarter_unknown_pct))
# Normalise all the things
year_autocorr = normalize(np.nan_to_num(raw_year_autocorr))
quarter_autocorr = normalize(np.nan_to_num(raw_quarter_autocorr))
# Calculate and encode page features. To create the following page features, only pages are passed to the function. Date columns and hits are not passed.
# Features are extracted from the page urls.
page_features = make_page_features(df.index.values)
encoded_page_features = encode_page_features(page_features)
# Make time-dependent features
# The following gives an array of all the days in the given data range. Eg: ['2015-07-01', '2015-07-02', .... '2017-10-01']
features_days = pd.date_range(data_start, features_end)
#dow = normalize(features_days.dayofweek.values)
week_period = 7 / (2 * np.pi)
# features_days.dayofweek.values below gives which day of the week does the corresponding date belongs to.
# Eg: '2015-07-01' belongs to 2nd day of week i.e Tue. So value will be 2.
dow_norm = features_days.dayofweek.values / week_period
dow = np.stack([np.cos(dow_norm), np.sin(dow_norm)], axis=-1)
# Assemble indices for quarterly lagged data
lagged_ix = np.stack(lag_indexes(data_start, features_end), axis=-1)
# This is median of each page hits. So the output is a Df with just one column that has median of all hits of page.
page_popularity = df.median(axis=1)
page_popularity = (page_popularity -
page_popularity.mean()) / page_popularity.std()
median_7 = traffic_median(df, 7)
median_30 = traffic_median(df, 30)
median_90 = traffic_median(df, 90)
median_180 = traffic_median(df, 180)
# Put NaNs back
df[nans] = np.NaN
# Assemble final output
tensors = dict(
hits=df,
lagged_ix=lagged_ix,
page_map=page_map,
page_ix=df.index.values,
pf_agent=encoded_page_features['agent'],
pf_country=encoded_page_features['country'],
pf_site=encoded_page_features['site'],
page_popularity=page_popularity,
median_7=median_7,
median_30=median_30,
median_90=median_90,
median_180=median_180,
year_autocorr=year_autocorr,
quarter_autocorr=quarter_autocorr,
dow=dow,
)
plain = dict(features_days=len(features_days),
data_days=len(df.columns),
n_pages=len(df),
data_start=data_start,
data_end=data_end,
features_end=features_end)
# Store data to the disk
VarFeeder(args.data_dir, tensors, plain)
eval_pct = 0.1
batch_size = batch_size
train_window = train_window
train_completeness_threshold = 1.0
predict_window = 63
verbose = False
train_skip_first = 0
train_skip_first = 0
tf.reset_default_graph()
forward_split = False
train_sampling = 1.0
if seed:
tf.set_random_seed(seed)
sess = tf.Session()
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
# 恢复变量
inp.restore(sess)
# splitter = FakeSplitter(page_features(inp), 3, seed=seed, test_sampling=eval_sampling)
splitter = Splitter(page_features(inp),
inp.page_map,
3,
train_sampling=train_sampling,
test_sampling=eval_sampling,
seed=seed)
real_train_pages = splitter.splits[0].train_size
real_eval_pages = splitter.splits[0].test_size
# 14503.6
items_per_eval = real_eval_pages * eval_pct
# 30
def run(city_path='/nfs/isolation_project/intern/project/lihaocheng/city_forcast/city_day_features_to_yesterday.gbk.csv',
weafor_path='/nfs/isolation_project/intern/project/lihaocheng/city_forcast/weather_forecast.csv',
datadir='data',
city_list=None,
**args):
start_time = time.time()
# Get the data
[train_x, train_embed_weekday, train_embed_month,
train_embed_city, train_real_city, train_y_origin],\
[val_x, val_embed_weekday, val_embed_month,
val_embed_city, val_real_city, val_y_origin],\
[infer_x, infer_embed_weekday, infer_embed_month,
infer_embed_city, infer_city_map, infer_y_origin],\
city_map, city_max, city_min, train_mean, train_std = read_all(city_list, city_path, weafor_path)
log.debug(
"complete generating df_cpu_max and df_cpu_num, time elapse = %S",
time.time() - start_time)
train_total = pd.concat([train_x, train_y_origin], axis=1)
train_features = [pd.DataFrame() for city in city_list]
train_y = [pd.DataFrame() for city in city_list]
y_mean = list()
y_std = list()
month_autocorr = dict()
week_autocorr = dict()
# Make train features
attrs = [
'online_time', 'total_finish_order_cnt', 'total_gmv',
'strive_order_cnt', 'total_no_call_order_cnt'
]
dfs = [pd.DataFrame() for i in range(5)]
for i, city in enumerate(city_list):
for idx, attr in enumerate(attrs):
per_city = train_total[train_total['city_id'] == city]
train_features[i] = per_city[train_x.columns].values
train_y[i] = per_city[train_y_origin.columns].apply(
lambda x: np.log(x + 1))
series = train_y[i].loc[:, [attr]]
series.columns = [city]
series = series.reset_index(drop=True)
dfs[idx] = pd.concat([dfs[idx], series], axis=1)
for idx, attr in enumerate(attrs):
df = dfs[idx].T
# monthly autocorrelation
month = batch_autocorr(df.values, 30)
# weekly autocorrelation
week = batch_autocorr(df.values, 7)
# Normalise all the things
month_autocorr[attr] = normalize(np.nan_to_num(month))
week_autocorr[attr] = normalize(np.nan_to_num(week))
# Find train_y mean & std
for i, per_city in enumerate(train_y):
y_mean.append(per_city.mean())
y_std.append(per_city.std())
# Make val features
val_total = pd.concat([val_x, val_y_origin], axis=1)
val_features = list()
val_y = list()
for city in city_list:
per_city = val_total[val_total['city_id'] == city]
val_features.append(per_city[val_x.columns].values)
val_y.append(
per_city[val_y_origin.columns].apply(lambda x: np.log(x + 1)))
# Make infer features
infer_x = infer_x.drop(['city_id'], axis=1)
infer_total = pd.concat([infer_x, infer_y_origin], axis=1)
infer_features = list()
infer_y = list()
for city in city_list:
per_city = infer_total[infer_total['city_id'] == city]
infer_features.append(per_city[infer_x.columns].values)
infer_y.append(
per_city[infer_y_origin.columns].apply(lambda x: np.log(x + 1)))
# Make time-dependent features
time_period = 7 / (2 * np.pi)
train_dow_norm = train_embed_weekday / time_period
val_dow_norm = val_embed_weekday / time_period
infer_dow_norm = infer_embed_weekday / time_period
time_period = 12 / (2 * np.pi)
train_dom_norm = train_embed_month / time_period
val_dom_norm = val_embed_month / time_period
infer_dom_norm = infer_embed_month / time_period
train_dow = np.stack([
np.cos(train_dow_norm),
np.sin(train_dow_norm),
np.cos(train_dom_norm),
np.sin(train_dom_norm)
],
axis=-1)
val_dow = np.stack([
np.cos(val_dow_norm),
np.sin(val_dow_norm),
np.cos(val_dom_norm),
np.sin(val_dom_norm)
],
axis=-1)
infer_dow = np.stack([
np.cos(infer_dow_norm),
np.sin(infer_dow_norm),
np.cos(infer_dom_norm),
np.sin(infer_dom_norm)
],
axis=-1)
# Assemble final output
tensors = dict(train_x=train_features,
val_x=val_features,
infer_x=infer_features,
train_dow=train_dow,
val_dow=val_dow,
infer_dow=infer_dow,
train_y=[df['total_no_call_order_cnt'] for df in train_y],
val_y=[df['total_no_call_order_cnt'] for df in val_y],
infer_y=[df['total_no_call_order_cnt'] for df in infer_y],
train_time=train_features[0].shape[0],
val_time=val_features[0].shape[0],
infer_time=infer_features[0].shape[0],
month_autocorr=month_autocorr['total_no_call_order_cnt'],
week_autocorr=week_autocorr['total_no_call_order_cnt'],
cities=np.array([city_map[city] for city in city_list]),
mean=[per['total_no_call_order_cnt'] for per in y_mean],
std=[per['total_no_call_order_cnt'] for per in y_std])
plain = dict()
# Store data to the disk
VarFeeder(os.path.join(datadir, 'vars'), tensors, plain)
with open(os.path.join(datadir, 'city_map.pickle'), 'wb') as handle:
pkl.dump(city_map, handle, protocol=pkl.HIGHEST_PROTOCOL)
infer_y_origin.to_pickle(os.path.join(datadir, 'infer_y.pickle'))
def main(_):
if not FLAGS.server:
print('please specify server host:port')
return
channel = grpc.insecure_channel(FLAGS.server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = "ucdoc"
request.model_spec.signature_name = "serving_default"
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
pipe = InputPipe(inp,
ucdoc_features(inp),
inp.n_pages,
mode=ModelMode.PREDICT,
batch_size=FLAGS.batch_size,
n_epoch=1,
verbose=FLAGS.verbose,
train_completeness_threshold=0.01,
predict_window=FLAGS.predict_window,
predict_completeness_threshold=0.0,
train_window=FLAGS.train_window,
back_offset=FLAGS.predict_window + 1)
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
pipe.load_vars(sess)
pipe.init_iterator(sess)
while True:
try:
truex, timex, normx, laggedx, truey, timey, normy, normmean, normstd, pgfeatures, pageix = \
sess.run([pipe.true_x, pipe.time_x, pipe.norm_x, pipe.lagged_x, pipe.true_y, pipe.time_y,
pipe.norm_y, pipe.norm_mean, pipe.norm_std, pipe.ucdoc_features, pipe.page_ix])
request.inputs["truex"].CopyFrom(tf.make_tensor_proto(truex))
request.inputs["timex"].CopyFrom(tf.make_tensor_proto(timex))
request.inputs["normx"].CopyFrom(tf.make_tensor_proto(normx))
request.inputs["laggedx"].CopyFrom(
tf.make_tensor_proto(laggedx))
request.inputs["truey"].CopyFrom(tf.make_tensor_proto(truey))
request.inputs["timey"].CopyFrom(tf.make_tensor_proto(timey))
request.inputs["normy"].CopyFrom(tf.make_tensor_proto(normy))
request.inputs["normmean"].CopyFrom(
tf.make_tensor_proto(normmean))
request.inputs["normstd"].CopyFrom(
tf.make_tensor_proto(normstd))
request.inputs["page_features"].CopyFrom(
tf.make_tensor_proto(pgfeatures))
request.inputs["pageix"].CopyFrom(tf.make_tensor_proto(pageix))
response = stub.Predict(request, 10)
tensor_proto = response.outputs['pred']
if not 'pred_result' in locals():
pred_result = tf.contrib.util.make_ndarray(tensor_proto)
else:
pred_result = np.concatenate([
pred_result,
tf.contrib.util.make_ndarray(tensor_proto)
])
except tf.errors.OutOfRangeError:
print('done with prediction')
break
pred_result = np.expm1(pred_result) + 0.5
pred_result = pred_result.astype(int)
if not os.path.exists(FLAGS.result_dir):
os.mkdir(FLAGS.result_dir)
result_file = os.path.join(FLAGS.result_dir, "predict.pkl")
pickle.dump(pred_result, open(result_file, "wb"))
print('finished prediction')
def run_local():
data_dir = 'data/vars'
add_days = 63
valid_threshold = 0.0
corr_backoffset = 0
start = None
end = None
# Get the data df:log1p后结果 浮点数 145036--过滤掉27条 长度不够的
df, nans, starts, ends = prepare_data(start, end, valid_threshold)
# Our working date range
data_start, data_end = df.columns[0], df.columns[-1]
# We have to project some date-dependent features (day of week, etc) to the future dates for prediction
features_end = data_end + pd.Timedelta(add_days, unit='D')
print(f"start: {data_start}, end:{data_end}, features_end:{features_end}")
# Group unique pages by agents 每个唯一页面对应最多四个agent,0 1 2 3 在相应位置填上 页面原来的行号
assert df.index.is_monotonic_increasing
page_map = uniq_page_map(df.index.values)
# Yearly(annual) autocorrelation 求365天的相关系数值,不符合条件的置np.nan
raw_year_autocorr = batch_autocorr(df.values, 365, starts, ends, 1.5,
corr_backoffset)
year_unknown_pct = np.sum(np.isnan(raw_year_autocorr)) / len(
raw_year_autocorr) # type: float
# Quarterly autocorrelation
raw_quarter_autocorr = batch_autocorr(df.values, int(round(365.25 / 4)),
starts, ends, 2, corr_backoffset)
quarter_unknown_pct = np.sum(np.isnan(raw_quarter_autocorr)) / len(
raw_quarter_autocorr) # type: float
print("Percent of undefined autocorr = yearly:%.3f, quarterly:%.3f" %
(year_unknown_pct, quarter_unknown_pct))
# Normalise all the things
year_autocorr = normalize(np.nan_to_num(raw_year_autocorr)) # -3,+3
quarter_autocorr = normalize(
np.nan_to_num(raw_quarter_autocorr)) # -4.5 ,+4.5
# Calculate and encode page features 页面url没有变化，提取了三个特征 agent country site
page_features = make_page_features(df.index.values)
encoded_page_features = encode_page_features(
page_features) # 10555 + 127181 + 7300 =145036
# Make time-dependent features 867
features_days = pd.date_range(data_start, features_end)
# dow = normalize(features_days.dayofweek.values)
week_period = 7 / (2 * np.pi)
dow_norm = features_days.dayofweek.values / week_period
dow = np.stack([np.cos(dow_norm), np.sin(dow_norm)], axis=-1)
# Assemble indices for quarterly lagged data
lagged_ix = np.stack(lag_indexes(data_start, features_end), axis=-1)
# 一列
page_popularity = df.median(axis=1)
page_popularity = (page_popularity -
page_popularity.mean()) / page_popularity.std()
# Put NaNs back 缺失重置np.NaN
df[nans] = np.NaN
# Assemble final output
tensors = dict(
hits=df,
lagged_ix=lagged_ix,
page_map=page_map,
page_ix=df.index.values,
pf_agent=encoded_page_features['agent'],
pf_country=encoded_page_features['country'],
pf_site=encoded_page_features['site'],
page_popularity=page_popularity,
year_autocorr=year_autocorr,
quarter_autocorr=quarter_autocorr,
dow=dow,
)
plain = dict(features_days=len(features_days),
data_days=len(df.columns),
n_pages=len(df),
data_start=data_start,
data_end=data_end,
features_end=features_end)
# Store data to the disk
VarFeeder(data_dir, tensors, plain)
def extend_inp(data_path, predict_window, holiday_list):
with tf.variable_scope('input', reuse=tf.AUTO_REUSE) as inp_scope:
with tf.device("/cpu:0"):
# inp = VarFeeder.read_vars("data/vars")
inp = VarFeeder.read_vars(data_path)
yesterday = pd.to_datetime(inp.data_end) + pd.Timedelta(
predict_window, 'D')
yesterday = yesterday.date().strftime('%Y-%m-%d')
day = datetime.strptime(yesterday, '%Y-%m-%d')
day_list = []
for _ in range(0, inp.features_days + predict_window):
day_list.append(datetime.strftime(day, '%Y-%m-%d'))
day = day + timedelta(days=-1)
day_list.sort()
# computing lagged_ix
date_range = pd.date_range(day_list[0], day_list[-1])
base_index = pd.Series(np.arange(0, len(date_range)),
index=date_range)
def lag(offset):
dates = date_range - offset
return pd.Series(data=base_index[dates].fillna(-1).astype(
np.int16).values,
index=date_range)
lagged_ix = np.stack(
[lag(pd.DateOffset(months=m)) for m in (1, 2)], axis=-1)
with tf.Session() as sess:
inp.restore(sess)
# ts_log, page_ix_, pf_age_, pf_si_, pf_network_, pf_price_model_, \
ts_log, page_ix_, pf_age_, pf_si_, pf_network_, pf_price_cat_, \
pf_gender_, page_popularity_, quarter_autocorr_ = \
sess.run([inp.hits, inp.page_ix, inp.pf_age, inp.pf_si,
# inp.pf_network, inp.pf_price_model, inp.pf_gender,
inp.pf_network, inp.pf_price_cat, inp.pf_gender,
inp.page_popularity, inp.quarter_autocorr])
print(
f'start: {inp.data_start}\tend: {inp.data_end}\tlength: {inp.features_days}'
)
df_ts = pd.DataFrame(np.append(ts_log,
np.zeros((len(page_ix_),
predict_window)),
axis=1),
index=list(page_ix_),
columns=day_list)
df_age = pd.DataFrame(pf_age_, index=list(page_ix_))
df_si = pd.DataFrame(pf_si_, index=list(page_ix_))
df_network = pd.DataFrame(pf_network_, index=list(page_ix_))
# df_price_model = pd.DataFrame(pf_price_model_, index=list(page_ix_))
df_price_cat = pd.DataFrame(pf_price_cat_, index=list(page_ix_))
df_gender = pd.DataFrame(pf_gender_, index=list(page_ix_))
def get_dow(day_list):
dow_list = []
for day in day_list:
dow = datetime.strptime(day, '%Y-%m-%d').weekday()
dow_list.append(dow)
week_period = 7.0 / (2 * math.pi)
sin_list = [math.sin(x / week_period) for x in dow_list]
cos_list = [math.cos(x / week_period) for x in dow_list]
return (sin_list, cos_list)
dow_ = get_dow(day_list)
# holiday_list = cfg['pipeline']['normalization']['holidays']
holidays = [1 if _ in holiday_list else 0 for _ in day_list]
a_list = []
b_list = []
for _ in holidays:
a, b = math.sin(_), math.cos(_)
a_list.append(a)
b_list.append(b)
holiday = (a_list, b_list)
tensors = dict(
hits=df_ts,
lagged_ix=lagged_ix,
page_ix=list(page_ix_),
pf_age=df_age,
pf_si=df_si,
pf_network=df_network,
# pf_price_model=df_price_model,
pf_price_cat=df_price_cat,
pf_gender=df_gender,
page_popularity=page_popularity_,
quarter_autocorr=quarter_autocorr_,
dow=pd.DataFrame(dow_).T,
holiday=pd.DataFrame(holiday).T)
batch_size = len(page_ix_)
data_len = tensors['hits'].shape[1]
plain = dict(data_days=data_len - 0,
features_days=data_len,
data_start=day_list[0],
data_end=day_list[-1],
n_pages=batch_size)
# dump_path = 'data/vars/predict_future'
dump_path = os.path.join(data_path, 'predict_future')
if not os.path.exists(dump_path):
os.mkdir(dump_path)
VarFeeder(dump_path, tensors, plain)
tf.reset_default_graph()
def main(_):
if len(sys.argv) < 3:
print(
'Usage: saved_model.py [--model_version=y] --data_dir=xxx --ckpt_dir=xxx --saved_dir=xxx'
)
sys.exit(-1)
if FLAGS.training_iteration <= 0:
print('Please specify a positive value for training iteration.')
sys.exit(-1)
if FLAGS.model_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
with open(FLAGS.config_file, 'r') as ymlfile:
cfg = yaml.load(ymlfile)
holiday_list = cfg['pipeline']['normalization']['holidays']
if FLAGS.back_offset < FLAGS.predict_window:
extend_inp(FLAGS.data_dir, FLAGS.predict_window, holiday_list)
# create deploy model first
back_offset_ = FLAGS.back_offset
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
if FLAGS.back_offset < FLAGS.predict_window:
inp = VarFeeder.read_vars(
os.path.join(FLAGS.data_dir, 'predict_future'))
back_offset_ += FLAGS.predict_window
else:
inp = VarFeeder.read_vars(FLAGS.data_dir)
pipe = InputPipe(inp,
ucdoc_features(inp),
inp.hits.shape[0],
mode=ModelMode.PREDICT,
batch_size=FLAGS.batch_size,
n_epoch=1,
verbose=False,
train_completeness_threshold=0.01,
predict_window=FLAGS.predict_window,
predict_completeness_threshold=0.0,
train_window=FLAGS.train_window,
back_offset=back_offset_)
asgd_decay = 0.99 if FLAGS.asgd else None
if FLAGS.n_models == 1:
model = Model(pipe,
build_from_set(FLAGS.hparam_set),
is_train=False,
seed=1,
asgd_decay=asgd_decay)
else:
models = []
for i in range(FLAGS.n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(
Model(pipe,
build_from_set(FLAGS.hparam_set),
is_train=False,
seed=1,
asgd_decay=asgd_decay,
graph_prefix=prefix))
model = models[FLAGS.target_model]
if FLAGS.asgd:
var_list = model.ema.variables_to_restore()
if FLAGS.n_models > 1:
prefix = f"m_{target_model}"
for var in list(var_list.keys()):
if var.endswith('ExponentialMovingAverage'
) and not var.startswith(prefix):
del var_list[var]
else:
var_list = None
# load checkpoint model from training
#ckpt_path = FLAGS.ckpt_dir
print('loading checkpoint model...')
ckpt_file = tf.train.latest_checkpoint(FLAGS.ckpt_dir)
#graph = tf.Graph()
graph = model.predictions.graph
init = tf.global_variables_initializer()
saver = tf.train.Saver(name='deploy_saver', var_list=var_list)
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
sess.run(init)
pipe.load_vars(sess)
pipe.init_iterator(sess)
saver.restore(sess, ckpt_file)
print('Done loading checkpoint model')
export_path_base = FLAGS.saved_dir
export_path = os.path.join(
tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(FLAGS.model_version)))
print('Exporting trained model to', export_path)
if os.path.isdir(export_path):
shutil.rmtree(export_path)
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
true_x = tf.saved_model.utils.build_tensor_info(
model.inp.true_x) # pipe.true_x
time_x = tf.saved_model.utils.build_tensor_info(
model.inp.time_x) # pipe.time_x
norm_x = tf.saved_model.utils.build_tensor_info(
model.inp.norm_x) # pipe.norm_x
lagged_x = tf.saved_model.utils.build_tensor_info(
model.inp.lagged_x) # pipe.lagged_x
true_y = tf.saved_model.utils.build_tensor_info(
model.inp.true_y) # pipe.true_y
time_y = tf.saved_model.utils.build_tensor_info(
model.inp.time_y) # pipe.time_y
norm_y = tf.saved_model.utils.build_tensor_info(
model.inp.norm_y) # pipe.norm_y
norm_mean = tf.saved_model.utils.build_tensor_info(
model.inp.norm_mean) # pipe.norm_mean
norm_std = tf.saved_model.utils.build_tensor_info(
model.inp.norm_std) # pipe.norm_std
pg_features = tf.saved_model.utils.build_tensor_info(
model.inp.ucdoc_features) # pipe.ucdoc_features
page_ix = tf.saved_model.utils.build_tensor_info(
model.inp.page_ix) # pipe.page_ix
#pred = tf.saved_model.utils.build_tensor_info(graph.get_operation_by_name('m_0/add').outputs[0])
pred = tf.saved_model.utils.build_tensor_info(model.predictions)
labeling_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
"truex": true_x,
"timex": time_x,
"normx": norm_x,
"laggedx": lagged_x,
"truey": true_y,
"timey": time_y,
"normy": norm_y,
"normmean": norm_mean,
"normstd": norm_std,
"page_features": pg_features,
"pageix": page_ix,
},
outputs={"predictions": pred},
method_name="tensorflow/serving/predict"))
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
labeling_signature
},
main_op=tf.tables_initializer(),
strip_default_attrs=True)
builder.save()
print("Build Done")
def run():
parser = argparse.ArgumentParser(description='Prepare data')
parser.add_argument('data_dir')
parser.add_argument(
'--valid_threshold',
default=0.0,
type=float,
help="Series minimal length threshold (pct of data length)")
parser.add_argument('--add_days',
default=64,
type=int,
help="Add N days in a future for prediction")
parser.add_argument(
'--start',
help="Effective start date. Data before the start is dropped")
parser.add_argument(
'--end', help="Effective end date. Data past the end is dropped")
parser.add_argument('--corr_backoffset',
default=0,
type=int,
help='Offset for correlation calculation')
args = parser.parse_args()
# Get the data
df, nans, starts, ends = prepare_data(args.start, args.end,
args.valid_threshold)
# Our working date range
data_start, data_end = df.columns[0], df.columns[-1]
# We have to project some date-dependent features (day of week, etc) to the future dates for prediction
features_end = data_end + pd.Timedelta(args.add_days, unit='D')
print(f"start: {data_start}, end:{data_end}, features_end:{features_end}")
# Group unique pages by agents
assert df.index.is_monotonic_increasing
page_map = uniq_page_map(df.index.values)
# Yearly(annual) autocorrelation
raw_year_autocorr = batch_autocorr(df.values, 365, starts, ends, 1.5,
args.corr_backoffset)
year_unknown_pct = np.sum(np.isnan(raw_year_autocorr)) / len(
raw_year_autocorr) # type: float
# Quarterly autocorrelation
raw_quarter_autocorr = batch_autocorr(df.values, int(round(365.25 / 4)),
starts, ends, 2,
args.corr_backoffset)
quarter_unknown_pct = np.sum(np.isnan(raw_quarter_autocorr)) / len(
raw_quarter_autocorr) # type: float
print("Percent of undefined autocorr = yearly:%.3f, quarterly:%.3f" %
(year_unknown_pct, quarter_unknown_pct))
# Normalise all the things
year_autocorr = normalize(np.nan_to_num(raw_year_autocorr))
quarter_autocorr = normalize(np.nan_to_num(raw_quarter_autocorr))
# Calculate and encode page features
page_features = make_page_features(df.index.values)
encoded_page_features = encode_page_features(page_features)
# Make time-dependent features
features_days = pd.date_range(data_start, features_end)
# dow = normalize(features_days.dayofweek.values)
week_period = 7 / (2 * np.pi)
dow_norm = features_days.dayofweek.values / week_period
dow = np.stack([np.cos(dow_norm), np.sin(dow_norm)], axis=-1)
# Assemble indices for quarterly lagged data
lagged_ix = np.stack(lag_indexes(data_start, features_end), axis=-1)
page_popularity = df.median(axis=1)
page_popularity = (page_popularity -
page_popularity.mean()) / page_popularity.std()
# Put NaNs back
df[nans] = np.NaN
# Assemble final output
tensors = dict(
lagged_ix=lagged_ix,
page_map=page_map,
dow=dow,
hits=df,
pf_agent=encoded_page_features['agent'],
pf_country=encoded_page_features['country'],
pf_site=encoded_page_features['site'],
page_ix=df.index.values,
page_popularity=page_popularity,
year_autocorr=year_autocorr,
quarter_autocorr=quarter_autocorr,
)
plain = dict(features_days=len(features_days),
data_days=len(df.columns),
n_pages=len(df),
data_start=data_start,
data_end=data_end,
features_end=features_end)
# Store data to the disk
VarFeeder(args.data_dir, tensors, plain)
def predict(checkpoints,
hparams,
datadir="data",
verbose=False,
n_models=1,
target_model=0,
asgd=False,
seed=1,
batch_size=50):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars(os.path.join(datadir, "vars"))
pipe = InputPipe(datadir,
inp,
infer_features(inp),
mode=ModelMode.PREDICT,
batch_size=batch_size,
n_epoch=1,
verbose=verbose,
train_completeness_threshold=0.01,
train_window=hparams.train_window)
asgd_decay = 0.99 if asgd else None
if n_models == 1:
model = Model(pipe,
hparams,
is_train=False,
seed=seed,
asgd_decay=asgd_decay)
else:
models = []
for i in range(n_models):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
models.append(
Model(pipe,
hparams,
is_train=False,
seed=seed,
asgd_decay=asgd_decay,
graph_prefix=prefix))
model = models[target_model]
if asgd:
var_list = model.ema.variables_to_restore()
prefix = f"m_{target_model}"
for var in list(var_list.keys()):
if var.endswith(
'ExponentialMovingAverage') and not var.startswith(prefix):
del var_list[var]
else:
var_list = None
saver = tf.train.Saver(name='eval_saver', var_list=var_list)
x_buffer = []
predictions = None
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
pipe.load_vars(sess)
for checkpoint in checkpoints:
pred_buffer = []
pipe.init_iterator(sess)
saver.restore(sess, checkpoint)
cnt = 0
while True:
try:
pred, pname = sess.run([model.prediction, model.inp.vm_ix])
# utf_names = [str(name, 'utf-8') for name in pname]
utf_names = pname
pred_df = pd.DataFrame(index=utf_names,
data=np.expm1(pred) - 1)
pred_buffer.append(pred_df)
newline = cnt % 80 == 0
if cnt > 0:
print('.', end='\n' if newline else '', flush=True)
if newline:
print(cnt, end='')
cnt += 1
except tf.errors.OutOfRangeError:
print('Done!')
break
cp_predictions = pd.concat(pred_buffer)
if predictions is None:
predictions = cp_predictions
else:
predictions += cp_predictions
predictions /= len(checkpoints)
return predictions.iloc[:, -1]
def train(name, hparams, multi_gpu=False, n_models=1, train_completeness_threshold=0.01,
seed=None, logdir='data/logs', max_epoch=100, patience=2, train_sampling=1.0,
eval_sampling=1.0, eval_memsize=5, gpu=0, gpu_allow_growth=False, save_best_model=False,
forward_split=False, write_summaries=False, verbose=False, asgd_decay=None, tqdm=True,
side_split=True, max_steps=None, save_from_step=None, do_eval=True, predict_window=63):
eval_k = int(round(26214 * eval_memsize / n_models))
eval_batch_size = int(
eval_k / (hparams.rnn_depth * hparams.encoder_rnn_layers)) # 128 -> 1024, 256->512, 512->256
eval_pct = 0.1
batch_size = hparams.batch_size
train_window = hparams.train_window
tf.reset_default_graph()
if seed:
tf.set_random_seed(seed)
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
if side_split:
splitter = Splitter(page_features(inp), inp.page_map, 3, train_sampling=train_sampling,
test_sampling=eval_sampling, seed=seed)
else:
splitter = FakeSplitter(page_features(inp), 3, seed=seed, test_sampling=eval_sampling)
real_train_pages = splitter.splits[0].train_size
real_eval_pages = splitter.splits[0].test_size
items_per_eval = real_eval_pages * eval_pct
eval_batches = int(np.ceil(items_per_eval / eval_batch_size))
steps_per_epoch = real_train_pages // batch_size
eval_every_step = int(round(steps_per_epoch * eval_pct))
# eval_every_step = int(round(items_per_eval * train_sampling / batch_size))
global_step = tf.train.get_or_create_global_step()
inc_step = tf.assign_add(global_step, 1)
all_models: List[ModelTrainerV2] = []
def create_model(scope, index, prefix, seed):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
split = splitter.splits[index]
pipe = InputPipe(inp, features=split.train_set, n_pages=split.train_size,
mode=ModelMode.TRAIN, batch_size=batch_size, n_epoch=None, verbose=verbose,
train_completeness_threshold=train_completeness_threshold,
predict_completeness_threshold=train_completeness_threshold, train_window=train_window,
predict_window=predict_window,
rand_seed=seed, train_skip_first=hparams.train_skip_first,
back_offset=predict_window if forward_split else 0)
inp_scope.reuse_variables()
if side_split:
side_eval_pipe = InputPipe(inp, features=split.test_set, n_pages=split.test_size,
mode=ModelMode.EVAL, batch_size=eval_batch_size, n_epoch=None,
verbose=verbose, predict_window=predict_window,
train_completeness_threshold=0.01, predict_completeness_threshold=0,
train_window=train_window, rand_seed=seed, runs_in_burst=eval_batches,
back_offset=predict_window * (2 if forward_split else 1))
else:
side_eval_pipe = None
if forward_split:
forward_eval_pipe = InputPipe(inp, features=split.test_set, n_pages=split.test_size,
mode=ModelMode.EVAL, batch_size=eval_batch_size, n_epoch=None,
verbose=verbose, predict_window=predict_window,
train_completeness_threshold=0.01, predict_completeness_threshold=0,
train_window=train_window, rand_seed=seed, runs_in_burst=eval_batches,
back_offset=predict_window)
else:
forward_eval_pipe = None
avg_sgd = asgd_decay is not None
#asgd_decay = 0.99 if avg_sgd else None
train_model = Model(pipe, hparams, is_train=True, graph_prefix=prefix, asgd_decay=asgd_decay, seed=seed)
scope.reuse_variables()
eval_stages = []
if side_split:
side_eval_model = Model(side_eval_pipe, hparams, is_train=False,
#loss_mask=np.concatenate([np.zeros(50, dtype=np.float32), np.ones(10, dtype=np.float32)]),
seed=seed)
eval_stages.append((Stage.EVAL_SIDE, side_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_SIDE_EMA, side_eval_model))
if forward_split:
forward_eval_model = Model(forward_eval_pipe, hparams, is_train=False, seed=seed)
eval_stages.append((Stage.EVAL_FRWD, forward_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_FRWD_EMA, forward_eval_model))
if write_summaries:
summ_path = f"{logdir}/{name}_{index}"
if os.path.exists(summ_path):
shutil.rmtree(summ_path)
summ_writer = tf.summary.FileWriter(summ_path) # , graph=tf.get_default_graph()
else:
summ_writer = None
if do_eval and forward_split:
stop_metric = lambda metrics: metrics[Stage.EVAL_FRWD]['SMAPE'].avg_epoch
else:
stop_metric = None
return ModelTrainerV2(train_model, eval_stages, index, patience=patience,
stop_metric=stop_metric,
summary_writer=summ_writer)
if n_models == 1:
with tf.device(f"/gpu:{gpu}"):
scope = tf.get_variable_scope()
all_models = [create_model(scope, 0, None, seed=seed)]
else:
for i in range(n_models):
device = f"/gpu:{i}" if multi_gpu else f"/gpu:{gpu}"
with tf.device(device):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
all_models.append(create_model(scope, i, prefix=prefix, seed=seed + i))
trainer = MultiModelTrainer(all_models, inc_step)
if save_best_model or save_from_step:
saver_path = f'data/cpt/{name}'
if os.path.exists(saver_path):
shutil.rmtree(saver_path)
os.makedirs(saver_path)
saver = tf.train.Saver(max_to_keep=10, name='train_saver')
else:
saver = None
avg_sgd = asgd_decay is not None
if avg_sgd:
from itertools import chain
def ema_vars(model):
ema = model.train_model.ema
return {ema.average_name(v):v for v in model.train_model.ema._averages}
ema_names = dict(chain(*[ema_vars(model).items() for model in all_models]))
#ema_names = all_models[0].train_model.ema.variables_to_restore()
ema_loader = tf.train.Saver(var_list=ema_names, max_to_keep=1, name='ema_loader')
ema_saver = tf.train.Saver(max_to_keep=1, name='ema_saver')
else:
ema_loader = None
init = tf.global_variables_initializer()
if forward_split and do_eval:
eval_smape = trainer.metric(Stage.EVAL_FRWD, 'SMAPE')
eval_mae = trainer.metric(Stage.EVAL_FRWD, 'MAE')
else:
eval_smape = DummyMetric()
eval_mae = DummyMetric()
if side_split and do_eval:
eval_mae_side = trainer.metric(Stage.EVAL_SIDE, 'MAE')
eval_smape_side = trainer.metric(Stage.EVAL_SIDE, 'SMAPE')
else:
eval_mae_side = DummyMetric()
eval_smape_side = DummyMetric()
train_smape = trainer.metric(Stage.TRAIN, 'SMAPE')
train_mae = trainer.metric(Stage.TRAIN, 'MAE')
grad_norm = trainer.metric(Stage.TRAIN, 'GrNorm')
eval_stages = []
ema_eval_stages = []
if forward_split and do_eval:
eval_stages.append(Stage.EVAL_FRWD)
ema_eval_stages.append(Stage.EVAL_FRWD_EMA)
if side_split and do_eval:
eval_stages.append(Stage.EVAL_SIDE)
ema_eval_stages.append(Stage.EVAL_SIDE_EMA)
# gpu_options=tf.GPUOptions(allow_growth=False),
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=tf.GPUOptions(allow_growth=gpu_allow_growth))) as sess:
sess.run(init)
# pipe.load_vars(sess)
inp.restore(sess)
for model in all_models:
model.init(sess)
# if beholder:
# visualizer = Beholder(session=sess, logdir=summ_path)
step = 0
prev_top = np.inf
best_smape = np.inf
# Contains best value (first item) and subsequent values
best_epoch_smape = []
for epoch in range(max_epoch):
# n_steps = pusher.n_pages // batch_size
if tqdm:
tqr = trange(steps_per_epoch, desc="%2d" % (epoch + 1), leave=False)
else:
tqr = range(steps_per_epoch)
for _ in tqr:
try:
step = trainer.train_step(sess, epoch)
except tf.errors.OutOfRangeError:
break
# if beholder:
# if step % 5 == 0:
# noinspection PyUnboundLocalVariable
# visualizer.update()
if step % eval_every_step == 0:
if eval_stages:
trainer.eval_step(sess, epoch, step, eval_batches, stages=eval_stages)
if save_best_model and epoch > 0 and eval_smape.last < best_smape:
best_smape = eval_smape.last
saver.save(sess, f'data/cpt/{name}/cpt', global_step=step)
if save_from_step and step >= save_from_step:
saver.save(sess, f'data/cpt/{name}/cpt', global_step=step)
if avg_sgd and ema_eval_stages:
ema_saver.save(sess, 'data/cpt_tmp/ema', write_meta_graph=False)
# restore ema-backed vars
ema_loader.restore(sess, 'data/cpt_tmp/ema')
trainer.eval_step(sess, epoch, step, eval_batches, stages=ema_eval_stages)
# restore normal vars
ema_saver.restore(sess, 'data/cpt_tmp/ema')
MAE = "%.3f/%.3f/%.3f" % (eval_mae.last, eval_mae_side.last, train_mae.last)
improvement = '↑' if eval_smape.improved else ' '
SMAPE = "%s%.3f/%.3f/%.3f" % (improvement, eval_smape.last, eval_smape_side.last, train_smape.last)
if tqdm:
tqr.set_postfix(gr=grad_norm.last, MAE=MAE, SMAPE=SMAPE)
if not trainer.has_active() or (max_steps and step > max_steps):
break
if tqdm:
tqr.close()
trainer.end_epoch()
if not best_epoch_smape or eval_smape.avg_epoch < best_epoch_smape[0]:
best_epoch_smape = [eval_smape.avg_epoch]
else:
best_epoch_smape.append(eval_smape.avg_epoch)
current_top = eval_smape.top
if prev_top > current_top:
prev_top = current_top
has_best_indicator = '↑'
else:
has_best_indicator = ' '
status = "%2d: Best top SMAPE=%.3f%s (%s)" % (
epoch + 1, current_top, has_best_indicator,
",".join(["%.3f" % m.top for m in eval_smape.metrics]))
if trainer.has_active():
status += ", frwd/side best MAE=%.3f/%.3f, SMAPE=%.3f/%.3f; avg MAE=%.3f/%.3f, SMAPE=%.3f/%.3f, %d am" % \
(eval_mae.best_epoch, eval_mae_side.best_epoch, eval_smape.best_epoch, eval_smape_side.best_epoch,
eval_mae.avg_epoch, eval_mae_side.avg_epoch, eval_smape.avg_epoch, eval_smape_side.avg_epoch,
trainer.has_active())
print(status, file=sys.stderr)
else:
print(status, file=sys.stderr)
print("Early stopping!", file=sys.stderr)
break
if max_steps and step > max_steps:
print("Max steps calculated", file=sys.stderr)
break
sys.stderr.flush()
# noinspection PyUnboundLocalVariable
return np.mean(best_epoch_smape, dtype=np.float64)
def train(name, hparams, multi_gpu=False, n_models=1, train_completeness_threshold=0.01,
seed=None, logdir='data/logs', max_epoch=100, patience=2, train_sampling=1.0,
eval_sampling=1.0, eval_memsize=5, gpu=0, gpu_allow_growth=False, save_best_model=False,
forward_split=False, write_summaries=False, verbose=False, asgd_decay=None, tqdm=True,
side_split=True, max_steps=None, save_from_step=None, do_eval=True, predict_window=63):
eval_k = int(round(26214 * eval_memsize / n_models))
eval_batch_size = int(
eval_k / (hparams.rnn_depth * hparams.encoder_rnn_layers)) # 128 -> 1024, 256->512, 512->256
eval_pct = 0.1
batch_size = hparams.batch_size
train_window = hparams.train_window
tf.reset_default_graph()
if seed:
tf.set_random_seed(seed)
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
if side_split:
splitter = Splitter(page_features(inp), inp.page_map, 3, train_sampling=train_sampling,
test_sampling=eval_sampling, seed=seed)
else:
splitter = FakeSplitter(page_features(inp), 3, seed=seed, test_sampling=eval_sampling)
real_train_pages = splitter.splits[0].train_size
real_eval_pages = splitter.splits[0].test_size
items_per_eval = real_eval_pages * eval_pct
eval_batches = int(np.ceil(items_per_eval / eval_batch_size))
steps_per_epoch = real_train_pages // batch_size
eval_every_step = int(round(steps_per_epoch * eval_pct))
# eval_every_step = int(round(items_per_eval * train_sampling / batch_size))
global_step = tf.train.get_or_create_global_step()
inc_step = tf.assign_add(global_step, 1)
all_models: List[ModelTrainerV2] = []
def create_model(scope, index, prefix, seed):
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
split = splitter.splits[index]
pipe = InputPipe(inp, features=split.train_set, n_pages=split.train_size,
mode=ModelMode.TRAIN, batch_size=batch_size, n_epoch=None, verbose=verbose,
train_completeness_threshold=train_completeness_threshold,
predict_completeness_threshold=train_completeness_threshold, train_window=train_window,
predict_window=predict_window,
rand_seed=seed, train_skip_first=hparams.train_skip_first,
back_offset=predict_window if forward_split else 0)
inp_scope.reuse_variables()
if side_split:
side_eval_pipe = InputPipe(inp, features=split.test_set, n_pages=split.test_size,
mode=ModelMode.EVAL, batch_size=eval_batch_size, n_epoch=None,
verbose=verbose, predict_window=predict_window,
train_completeness_threshold=0.01, predict_completeness_threshold=0,
train_window=train_window, rand_seed=seed, runs_in_burst=eval_batches,
back_offset=predict_window * (2 if forward_split else 1))
else:
side_eval_pipe = None
if forward_split:
forward_eval_pipe = InputPipe(inp, features=split.test_set, n_pages=split.test_size,
mode=ModelMode.EVAL, batch_size=eval_batch_size, n_epoch=None,
verbose=verbose, predict_window=predict_window,
train_completeness_threshold=0.01, predict_completeness_threshold=0,
train_window=train_window, rand_seed=seed, runs_in_burst=eval_batches,
back_offset=predict_window)
else:
forward_eval_pipe = None
avg_sgd = asgd_decay is not None
#asgd_decay = 0.99 if avg_sgd else None
train_model = Model(pipe, hparams, is_train=True, graph_prefix=prefix, asgd_decay=asgd_decay, seed=seed)
scope.reuse_variables()
eval_stages = []
if side_split:
side_eval_model = Model(side_eval_pipe, hparams, is_train=False,
#loss_mask=np.concatenate([np.zeros(50, dtype=np.float32), np.ones(10, dtype=np.float32)]),
seed=seed)
eval_stages.append((Stage.EVAL_SIDE, side_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_SIDE_EMA, side_eval_model))
if forward_split:
forward_eval_model = Model(forward_eval_pipe, hparams, is_train=False, seed=seed)
eval_stages.append((Stage.EVAL_FRWD, forward_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_FRWD_EMA, forward_eval_model))
if write_summaries:
summ_path = f"{logdir}/{name}_{index}"
if os.path.exists(summ_path):
shutil.rmtree(summ_path)
summ_writer = tf.summary.FileWriter(summ_path) # , graph=tf.get_default_graph()
else:
summ_writer = None
if do_eval and forward_split:
stop_metric = lambda metrics: metrics[Stage.EVAL_FRWD]['SMAPE'].avg_epoch
else:
stop_metric = None
return ModelTrainerV2(train_model, eval_stages, index, patience=patience,
stop_metric=stop_metric,
summary_writer=summ_writer)
if n_models == 1:
with tf.device(f"/gpu:{gpu}"):
scope = tf.get_variable_scope()
all_models = [create_model(scope, 0, None, seed=seed)]
else:
for i in range(n_models):
device = f"/gpu:{i}" if multi_gpu else f"/gpu:{gpu}"
with tf.device(device):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
all_models.append(create_model(scope, i, prefix=prefix, seed=seed + i))
trainer = MultiModelTrainer(all_models, inc_step)
if save_best_model or save_from_step:
saver_path = f'data/cpt/{name}'
if os.path.exists(saver_path):
shutil.rmtree(saver_path)
os.makedirs(saver_path)
saver = tf.train.Saver(max_to_keep=10, name='train_saver')
else:
saver = None
avg_sgd = asgd_decay is not None
if avg_sgd:
from itertools import chain
def ema_vars(model):
ema = model.train_model.ema
return {ema.average_name(v):v for v in model.train_model.ema._averages}
ema_names = dict(chain(*[ema_vars(model).items() for model in all_models]))
#ema_names = all_models[0].train_model.ema.variables_to_restore()
ema_loader = tf.train.Saver(var_list=ema_names, max_to_keep=1, name='ema_loader')
ema_saver = tf.train.Saver(max_to_keep=1, name='ema_saver')
else:
ema_loader = None
init = tf.global_variables_initializer()
if forward_split and do_eval:
eval_smape = trainer.metric(Stage.EVAL_FRWD, 'SMAPE')
eval_mae = trainer.metric(Stage.EVAL_FRWD, 'MAE')
else:
eval_smape = DummyMetric()
eval_mae = DummyMetric()
if side_split and do_eval:
eval_mae_side = trainer.metric(Stage.EVAL_SIDE, 'MAE')
eval_smape_side = trainer.metric(Stage.EVAL_SIDE, 'SMAPE')
else:
eval_mae_side = DummyMetric()
eval_smape_side = DummyMetric()
train_smape = trainer.metric(Stage.TRAIN, 'SMAPE')
train_mae = trainer.metric(Stage.TRAIN, 'MAE')
grad_norm = trainer.metric(Stage.TRAIN, 'GrNorm')
eval_stages = []
ema_eval_stages = []
if forward_split and do_eval:
eval_stages.append(Stage.EVAL_FRWD)
ema_eval_stages.append(Stage.EVAL_FRWD_EMA)
if side_split and do_eval:
eval_stages.append(Stage.EVAL_SIDE)
ema_eval_stages.append(Stage.EVAL_SIDE_EMA)
# gpu_options=tf.GPUOptions(allow_growth=False),
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=tf.GPUOptions(allow_growth=gpu_allow_growth))) as sess:
sess.run(init)
# pipe.load_vars(sess)
inp.restore(sess)
for model in all_models:
model.init(sess)
# if beholder:
# visualizer = Beholder(session=sess, logdir=summ_path)
step = 0
prev_top = np.inf
best_smape = np.inf
# Contains best value (first item) and subsequent values
best_epoch_smape = []
for epoch in range(max_epoch):
# n_steps = pusher.n_pages // batch_size
if tqdm:
tqr = trange(steps_per_epoch, desc="%2d" % (epoch + 1), leave=False)
else:
tqr = range(steps_per_epoch)
for _ in tqr:
try:
step = trainer.train_step(sess, epoch)
except tf.errors.OutOfRangeError:
break
# if beholder:
# if step % 5 == 0:
# noinspection PyUnboundLocalVariable
# visualizer.update()
if step % eval_every_step == 0:
if eval_stages:
trainer.eval_step(sess, epoch, step, eval_batches, stages=eval_stages)
if save_best_model and epoch > 0 and eval_smape.last < best_smape:
best_smape = eval_smape.last
saver.save(sess, f'data/cpt/{name}/cpt', global_step=step)
if save_from_step and step >= save_from_step:
saver.save(sess, f'data/cpt/{name}/cpt', global_step=step)
if avg_sgd and ema_eval_stages:
ema_saver.save(sess, 'data/cpt_tmp/ema', write_meta_graph=False)
# restore ema-backed vars
ema_loader.restore(sess, 'data/cpt_tmp/ema')
trainer.eval_step(sess, epoch, step, eval_batches, stages=ema_eval_stages)
# restore normal vars
ema_saver.restore(sess, 'data/cpt_tmp/ema')
MAE = "%.3f/%.3f/%.3f" % (eval_mae.last, eval_mae_side.last, train_mae.last)
improvement = '↑' if eval_smape.improved else ' '
SMAPE = "%s%.3f/%.3f/%.3f" % (improvement, eval_smape.last, eval_smape_side.last, train_smape.last)
if tqdm:
tqr.set_postfix(gr=grad_norm.last, MAE=MAE, SMAPE=SMAPE)
if not trainer.has_active() or (max_steps and step > max_steps):
break
if tqdm:
tqr.close()
trainer.end_epoch()
if not best_epoch_smape or eval_smape.avg_epoch < best_epoch_smape[0]:
best_epoch_smape = [eval_smape.avg_epoch]
else:
best_epoch_smape.append(eval_smape.avg_epoch)
current_top = eval_smape.top
if prev_top > current_top:
prev_top = current_top
has_best_indicator = '↑'
else:
has_best_indicator = ' '
status = "%2d: Best top SMAPE=%.3f%s (%s)" % (
epoch + 1, current_top, has_best_indicator,
",".join(["%.3f" % m.top for m in eval_smape.metrics]))
if trainer.has_active():
status += ", frwd/side best MAE=%.3f/%.3f, SMAPE=%.3f/%.3f; avg MAE=%.3f/%.3f, SMAPE=%.3f/%.3f, %d am" % \
(eval_mae.best_epoch, eval_mae_side.best_epoch, eval_smape.best_epoch, eval_smape_side.best_epoch,
eval_mae.avg_epoch, eval_mae_side.avg_epoch, eval_smape.avg_epoch, eval_smape_side.avg_epoch,
trainer.has_active())
print(status, file=sys.stderr)
else:
print(status, file=sys.stderr)
print("Early stopping!", file=sys.stderr)
break
if max_steps and step > max_steps:
print("Max steps calculated", file=sys.stderr)
break
sys.stderr.flush()
# noinspection PyUnboundLocalVariable
return np.mean(best_epoch_smape, dtype=np.float64)
def run(cfg):
with open(cfg['tf_statistics_path'], 'rb') as f:
tf_stat = pickle.load(f)
names = []
tfrecord_location = cfg['tfrecords_local_path']
for file in os.listdir(tfrecord_location):
if file.startswith("part"):
names.append(file)
file_paths = [os.path.join(tfrecord_location, name) for name in names]
# read and make the dataset from tfrecord
dataset = tf.data.TFRecordDataset(file_paths)
dataset = dataset.map(__data_parser)
batch_size = cfg['batch_size']
duration = cfg['duration']
dataset = dataset.batch(batch_size).shuffle(SHUFFLE_BUFFER)
iterator = dataset.make_one_shot_iterator()
next_el = iterator.get_next()
# lagged_ix = numpy.ones((duration, 4), dtype=float)
# lagged_ix = np.where(lagged_ix == 1, -1, lagged_ix)
lagged_ix = np.stack(lag_indexes(tf_stat), axis=-1)
# quarter_autocorr = numpy.ones((batch_size,), dtype=float)
date_list = tf_stat['days']
dow = get_dow(date_list)
holiday_list = cfg['holidays']
holidays = [1 if _ in holiday_list else 0 for _ in date_list]
a_list = []
b_list = []
for _ in holidays:
a, b = holiday_norm(_)
a_list.append(a)
b_list.append(b)
holiday = (a_list, b_list)
with tf.Session() as sess:
x = sess.run(next_el)
quarter_autocorr = numpy.ones((x[0].size, ), dtype=float)
page_indx = list(x[0])
fill_isolated_zeros(x[21])
tensors = dict(
hits=pd.DataFrame(x[21], index=page_indx, columns=date_list),
lagged_ix=lagged_ix,
page_ix=page_indx,
pf_age=pd.DataFrame(x[8:15],
columns=page_indx,
index=(1, 2, 3, 4, 5, 6, 7)).T,
pf_si=pd.DataFrame(x[20], index=page_indx),
pf_network=pd.DataFrame(x[15:20],
columns=page_indx,
index=('2G', '3G', '4G', 'UNKNOWN',
'WIFI')).T,
pf_price_cat=pd.DataFrame(x[1:4],
columns=page_indx,
index=('pc1', 'pc2', 'pc3')).T,
pf_gender=pd.DataFrame(x[4:8],
columns=page_indx,
index=('none', 'f', 'm', 'x')).T,
page_popularity=x[22],
# page_popularity = quarter_autocorr,
quarter_autocorr=quarter_autocorr,
dow=pd.DataFrame(dow).T,
holiday=pd.DataFrame(holiday).T)
data_len = tensors['hits'].shape[1]
plain = dict(data_days=data_len - cfg['add_days'],
features_days=data_len,
data_start=date_list[0],
data_end=date_list[-1],
features_end=date_list[-1],
n_pages=batch_size)
VarFeeder(cfg['data_dir'], tensors, plain)
def train(name,
hparams,
multi_gpu=False,
n_models=1,
train_completeness_threshold=0.01,
seed=None,
logdir='data/logs',
max_epoch=100,
patience=2,
train_sampling=1.0,
eval_sampling=1.0,
eval_memsize=5,
gpu=0,
gpu_allow_growth=False,
save_best_model=False,
forward_split=False,
write_summaries=False,
verbose=False,
asgd_decay=None,
tqdm=True,
side_split=True,
max_steps=None,
save_from_step=None,
do_eval=True,
predict_window=63):
eval_k = int(round(26214 * eval_memsize / n_models))
eval_batch_size = int(
eval_k /
(hparams.rnn_depth *
hparams.encoder_rnn_layers)) # 128 -> 1024, 256->512, 512->256
eval_pct = 0.1
batch_size = hparams.batch_size
train_window = hparams.train_window
# todo eval_k = 43690,eval_batch_size = 163,eval_pct = 0,batch_size = 128,train_window = 283
# print("eval_k = %d,eval_batch_size = %d,eval_pct = %d,batch_size = %d,train_window = %d" %(eval_k,eval_batch_size,eval_pct,batch_size,train_window))
tf.reset_default_graph()
if seed:
tf.set_random_seed(seed)
with tf.device("/cpu:0"):
inp = VarFeeder.read_vars("data/vars")
# print("side_split = %d,train_sampling= %d,eval_sampling= %d,seed= %d" % (
# side_split,train_sampling,eval_sampling,seed),f"inp={inp}, side_split={side_split}; type(inp)={type(inp)}")
# todo side_split = 0,train_sampling= 1,eval_sampling= 1,seed= 5
# inp={'hits': <tf.Variable 'hits:0' shape=(145036, 805) dtype=float32_ref>,
# 'lagged_ix': <tf.Variable 'lagged_ix:0' shape=(867, 4) dtype=int16_ref>,
# 'page_map': <tf.Variable 'page_map:0' shape=(52752, 4) dtype=int32_ref>,
# 'page_ix': <tf.Variable 'page_ix:0' shape=(145036,) dtype=string_ref>,
# 'pf_agent': <tf.Variable 'pf_agent:0' shape=(145036, 4) dtype=float32_ref>,
# 'pf_country': <tf.Variable 'pf_country:0' shape=(145036, 7) dtype=float32_ref>,
# 'pf_site': <tf.Variable 'pf_site:0' shape=(145036, 3) dtype=float32_ref>,
# 'page_popularity': <tf.Variable 'page_popularity:0' shape=(145036,) dtype=float32_ref>,
# 'year_autocorr': <tf.Variable 'year_autocorr:0' shape=(145036,) dtype=float32_ref>,
# 'quarter_autocorr': <tf.Variable 'quarter_autocorr:0' shape=(145036,) dtype=float32_ref>,
# 'dow': <tf.Variable 'dow:0' shape=(867, 2) dtype=float32_ref>,'features_days': 867,
# 'data_days': 805, 'n_pages': 145036, 'data_start': '2015-07-01',
# 'data_end': Timestamp('2017-09-11 00:00:00'), 'features_end': Timestamp('2017-11-13 00:00:00')}
# side_split=False;
# type(inp)=<class 'feeder.FeederVars'>;
# if True:
if side_split:
splitter = Splitter(page_features(inp),
inp.page_map,
3,
train_sampling=train_sampling,
test_sampling=eval_sampling,
seed=seed)
else:
splitter = FakeSplitter(page_features(inp),
3,
seed=seed,
test_sampling=eval_sampling)
real_train_pages = splitter.splits[0].train_size
real_eval_pages = splitter.splits[0].test_size
items_per_eval = real_eval_pages * eval_pct
eval_batches = int(np.ceil(items_per_eval / eval_batch_size))
steps_per_epoch = real_train_pages // batch_size
eval_every_step = int(round(steps_per_epoch * eval_pct))
# todo real_train_pages = 145036,real_eval_pages= 145036,items_per_eval= 14503,eval_batches= 89,
# steps_per_epoch= 1133,eval_every_step= 113 -- 每个epoch有1133个step，每113个step打印一下当前模型的效果
# print("real_train_pages = %d,real_eval_pages= %d,items_per_eval= %d,eval_batches= %d,steps_per_epoch= %d,eval_every_step= %d; eval_pct" % (
# real_train_pages, real_eval_pages, items_per_eval, eval_batches, steps_per_epoch, eval_every_step,eval_pct
# ))
# return
# eval_every_step = int(round(items_per_eval * train_sampling / batch_size))
# todo get_or_create_global_step 这个函数主要用于返回或者创建（如果有必要的话）一个全局步数的tensor变量。
global_step = tf.train.get_or_create_global_step()
# todo tf.assign_add(ref,value,use_locking=None,name=None)；通过增加value，更新ref的值，即：ref = ref + value；
# inc increase_step
inc_step = tf.assign_add(global_step, 1)
all_models: List[ModelTrainerV2] = []
def create_model(scope, index, prefix, seed):
# todo 主要是创建了模型，以及返回一些None的东西。
# 数据在构建模型的时候使用了，模型中只使用了数据的预测窗口的长度--不对，应该是创建模型的时候直接喂入数据了。
with tf.variable_scope('input') as inp_scope:
with tf.device("/cpu:0"):
split = splitter.splits[index]
pipe = InputPipe(
inp,
features=split.train_set,
n_pages=split.train_size,
mode=ModelMode.TRAIN,
batch_size=batch_size,
n_epoch=None,
verbose=verbose,
train_completeness_threshold=train_completeness_threshold,
predict_completeness_threshold=train_completeness_threshold,
train_window=train_window,
predict_window=predict_window,
rand_seed=seed,
train_skip_first=hparams.train_skip_first,
back_offset=predict_window if forward_split else 0)
inp_scope.reuse_variables()
# todo side_split: False; forward_split:False; eval_stages: [];
if side_split:
side_eval_pipe = InputPipe(
inp,
features=split.test_set,
n_pages=split.test_size,
mode=ModelMode.EVAL,
batch_size=eval_batch_size,
n_epoch=None,
verbose=verbose,
predict_window=predict_window,
train_completeness_threshold=0.01,
predict_completeness_threshold=0,
train_window=train_window,
rand_seed=seed,
runs_in_burst=eval_batches,
back_offset=predict_window *
(2 if forward_split else 1))
else:
side_eval_pipe = None
if forward_split:
forward_eval_pipe = InputPipe(
inp,
features=split.test_set,
n_pages=split.test_size,
mode=ModelMode.EVAL,
batch_size=eval_batch_size,
n_epoch=None,
verbose=verbose,
predict_window=predict_window,
train_completeness_threshold=0.01,
predict_completeness_threshold=0,
train_window=train_window,
rand_seed=seed,
runs_in_burst=eval_batches,
back_offset=predict_window)
else:
forward_eval_pipe = None
avg_sgd = asgd_decay is not None
#asgd_decay = 0.99 if avg_sgd else None
train_model = Model(pipe,
hparams,
is_train=True,
graph_prefix=prefix,
asgd_decay=asgd_decay,
seed=seed)
scope.reuse_variables()
eval_stages = []
if side_split:
# print('2 side_split side_eval_model')
side_eval_model = Model(
side_eval_pipe,
hparams,
is_train=False,
#loss_mask=np.concatenate([np.zeros(50, dtype=np.float32), np.ones(10, dtype=np.float32)]),
seed=seed)
# print("2 side_eval_model -- 2")
# todo TRAIN = 0; EVAL_SIDE = 1; EVAL_FRWD = 2; EVAL_SIDE_EMA = 3; EVAL_FRWD_EMA = 4
eval_stages.append((Stage.EVAL_SIDE, side_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_SIDE_EMA, side_eval_model))
if forward_split:
# print("3 forward_split forward_eval_model")
# tf.reset_default_graph()
forward_eval_model = Model(forward_eval_pipe,
hparams,
is_train=False,
seed=seed)
# print("3 forward_split forward_eval_model -- 2")
eval_stages.append((Stage.EVAL_FRWD, forward_eval_model))
if avg_sgd:
eval_stages.append((Stage.EVAL_FRWD_EMA, forward_eval_model))
if write_summaries:
summ_path = f"{logdir}/{name}_{index}"
# print("write_summaries summ_path",summ_path)
if os.path.exists(summ_path):
shutil.rmtree(summ_path)
summ_writer = tf.summary.FileWriter(
summ_path) # , graph=tf.get_default_graph()
else:
summ_writer = None
if do_eval and forward_split:
stop_metric = lambda metrics: metrics[Stage.EVAL_FRWD]['SMAPE'
].avg_epoch
else:
stop_metric = None
# todo side_split: False; forward_split:False;
# summ_writer=<tensorflow.python.summary.writer.writer.FileWriter object at 0x7ff5dc176710>；
# eval_stages: []; stop_metric=None; patience=2; index=0
# print(f"side_split: {side_split}; forward_split:{forward_split}; summ_writer={summ_writer}；"
# f"eval_stages: {eval_stages}； stop_metric={stop_metric}; patience={patience}; index={index}")
return ModelTrainerV2(train_model,
eval_stages,
index,
patience=patience,
stop_metric=stop_metric,
summary_writer=summ_writer)
# todo n_models == 3
if n_models == 1:
with tf.device(f"/gpu:{gpu}"):
scope = tf.get_variable_scope()
all_models = [create_model(scope, 0, None, seed=seed)]
else:
for i in range(n_models):
device = f"/gpu:{i}" if multi_gpu else f"/gpu:{gpu}"
with tf.device(device):
prefix = f"m_{i}"
with tf.variable_scope(prefix) as scope:
all_models.append(
create_model(scope, i, prefix=prefix, seed=seed + i))
# todo inc_step = tf.assign_add(global_step, 1)
trainer = MultiModelTrainer(all_models, inc_step)
# return
# todo save_best_model or save_from_step: False 10500
# print("save_best_model or save_from_step: ", save_best_model, save_from_step)
if save_best_model or save_from_step:
saver_path = f'data/cpt/{name}'
# todo saver_path: data/cpt/s32
# print("saver_path: ",saver_path)
if os.path.exists(saver_path):
shutil.rmtree(saver_path)
os.makedirs(saver_path)
# todo max_to_keep 参数，这个是用来设置保存模型的个数，默认为5，即 max_to_keep=5，保存最近的5个模型
saver = tf.train.Saver(max_to_keep=10, name='train_saver')
else:
saver = None
# todo EMA decay for averaged SGD. Not use ASGD if not set
avg_sgd = asgd_decay is not None
# todo asgd_decay=0.99; avg_sgd=True
# print(f"asgd_decay={asgd_decay}; avg_sgd={avg_sgd}")
if avg_sgd:
from itertools import chain
def ema_vars(model):
ema = model.train_model.ema
# todo: average_name() methods give access to the shadow variables and their names
return {
ema.average_name(v): v
for v in model.train_model.ema._averages
}
ema_names = dict(
chain(*[ema_vars(model).items() for model in all_models]))
# todo ema_names=
# {'m_0/m_0/cudnn_gru/opaque_kernel/ExponentialMovingAverage': <tf.Variable 'm_0/cudnn_gru/opaque_kernel:0' shape=<unknown> dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d/kernel:0' shape=(7, 5, 16) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d/bias/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d/bias:0' shape=(16,) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_1/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_1/kernel:0' shape=(3, 16, 16) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_1/bias/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_1/bias:0' shape=(16,) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_2/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_2/kernel:0' shape=(3, 16, 32) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_2/bias/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_2/bias:0' shape=(32,) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_3/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_3/kernel:0' shape=(3, 32, 32) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_3/bias/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_3/bias:0' shape=(32,) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_4/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_4/kernel:0' shape=(3, 32, 64) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_4/bias/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_4/bias:0' shape=(64,) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_5/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_5/kernel:0' shape=(3, 64, 64) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/convnet/conv1d_5/bias/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/convnet/conv1d_5/bias:0' shape=(64,) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/fc_convnet/fc_encoder/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/fc_convnet/fc_encoder/kernel:0' shape=(2304, 512) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/fc_convnet/fc_encoder/bias/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/fc_convnet/fc_encoder/bias:0' shape=(512,) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/fc_convnet/out_encoder/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/fc_convnet/out_encoder/kernel:0' shape=(512, 16) dtype=float32_ref>,
# 'm_0/m_0/fingerpint/fc_convnet/out_encoder/bias/ExponentialMovingAverage': <tf.Variable 'm_0/fingerpint/fc_convnet/out_encoder/bias:0' shape=(16,) dtype=float32_ref>,
# 'm_0/m_0/attn_focus/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/attn_focus/kernel:0' shape=(16, 221) dtype=float32_ref>,
# 'm_0/m_0/attn_focus/bias/ExponentialMovingAverage': <tf.Variable 'm_0/attn_focus/bias:0' shape=(221,) dtype=float32_ref>,
# 'm_0/m_0/gru_cell/w_ru/ExponentialMovingAverage': <tf.Variable 'm_0/gru_cell/w_ru:0' shape=(291, 534) dtype=float32_ref>,
# 'm_0/m_0/gru_cell/b_ru/ExponentialMovingAverage': <tf.Variable 'm_0/gru_cell/b_ru:0' shape=(534,) dtype=float32_ref>,
# 'm_0/m_0/gru_cell/w_c/ExponentialMovingAverage': <tf.Variable 'm_0/gru_cell/w_c:0' shape=(291, 267) dtype=float32_ref>,
# 'm_0/m_0/gru_cell/b_c/ExponentialMovingAverage': <tf.Variable 'm_0/gru_cell/b_c:0' shape=(267,) dtype=float32_ref>,
# 'm_0/m_0/decoder_output_proj/kernel/ExponentialMovingAverage': <tf.Variable 'm_0/decoder_output_proj/kernel:0' shape=(267, 1) dtype=float32_ref>,
# 'm_0/m_0/decoder_output_proj/bias/ExponentialMovingAverage': <tf.Variable 'm_0/decoder_output_proj/bias:0' shape=(1,) dtype=float32_ref>,
# print(f"ema_names={ema_names}")
# todo chain=<itertools.chain object at 0x7fe6587cbf98>,
# [] = [dict_items([
# ('m_0/m_0/cudnn_gru/opaque_kernel/ExponentialMovingAverage', <tf.Variable 'm_0/cudnn_gru/opaque_kernel:0' shape=<unknown> dtype=float32_ref>),
# ...
# ('m_2/m_2/decoder_output_proj/bias/ExponentialMovingAverage', <tf.Variable 'm_2/decoder_output_proj/bias:0' shape=(1,) dtype=float32_ref>)
# ])]
# print(f"chain={chain(*[ema_vars(model).items() for model in all_models])},\n[] = {[ema_vars(model).items() for model in all_models]}")
#ema_names = all_models[0].train_model.ema.variables_to_restore()
ema_loader = tf.train.Saver(var_list=ema_names,
max_to_keep=1,
name='ema_loader')
ema_saver = tf.train.Saver(max_to_keep=1, name='ema_saver')
else:
ema_loader = None
init = tf.global_variables_initializer()
# print(f"forward_split={forward_split}; do_eval={do_eval}; side_split={side_split}")
if forward_split and do_eval:
eval_smape = trainer.metric(Stage.EVAL_FRWD, 'SMAPE')
eval_mae = trainer.metric(Stage.EVAL_FRWD, 'MAE')
else:
eval_smape = DummyMetric()
eval_mae = DummyMetric()
if side_split and do_eval:
eval_mae_side = trainer.metric(Stage.EVAL_SIDE, 'MAE')
eval_smape_side = trainer.metric(Stage.EVAL_SIDE, 'SMAPE')
else:
eval_mae_side = DummyMetric()
eval_smape_side = DummyMetric()
train_smape = trainer.metric(Stage.TRAIN, 'SMAPE')
train_mae = trainer.metric(Stage.TRAIN, 'MAE')
grad_norm = trainer.metric(Stage.TRAIN, 'GrNorm')
eval_stages = []
ema_eval_stages = []
if forward_split and do_eval:
eval_stages.append(Stage.EVAL_FRWD)
ema_eval_stages.append(Stage.EVAL_FRWD_EMA)
if side_split and do_eval:
eval_stages.append(Stage.EVAL_SIDE)
ema_eval_stages.append(Stage.EVAL_SIDE_EMA)
# todo eval_stages=[]; ema_eval_stages=[]
# print(f"eval_stages={eval_stages}; ema_eval_stages={ema_eval_stages}")
# gpu_options=tf.GPUOptions(allow_growth=False),
with tf.Session(
config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=tf.GPUOptions(
allow_growth=gpu_allow_growth))) as sess:
sess.run(init)
# pipe.load_vars(sess)
# todo 之前inp是加载了这个数据对象，restore是把数据tensor加载到sess中吧？
# 这里加载了数据在哪里用到了呢？
inp.restore(sess)
for model in all_models:
# todo 这里是什么意思呢？这样的到什么呢？运行了一下init_iterator？
# 上面建好模型结构之后，在哪里喂入数据呢？
model.init(sess)
# if beholder:
# visualizer = Beholder(session=sess, logdir=summ_path)
step = 0
prev_top = np.inf
best_smape = np.inf
# Contains best value (first item) and subsequent values
best_epoch_smape = []
for epoch in range(max_epoch):
# n_steps = pusher.n_pages // batch_size
if tqdm:
# todo Tqdm 是一个快速，可扩展的Python进度条，可以在 Python 长循环中添加一个进度提示信息，
# 用户只需要封装任意的迭代器 tqdm(iterator)。trange(i) 是 tqdm(range(i)) 的简单写法
# desc=进度条前面的描述；leave：保留进度条存在的痕迹，简单来说就是会把进度条的最终形态保留下来，默认为True
tqr = trange(steps_per_epoch,
desc="%2d" % (epoch + 1),
leave=False)
else:
tqr = range(steps_per_epoch)
for _ in tqr:
try:
# print("PRINT step = trainer.train_step")
# todo 训练模型只有这一行对吧
step = trainer.train_step(sess, epoch)
# if epoch == 0:
# print(f"step={step}, _={_}, epoch = {epoch}")
except tf.errors.OutOfRangeError:
break
# if beholder:
# if step % 5 == 0:
# noinspection PyUnboundLocalVariable
# visualizer.update()
# todo 应该是每训练一个epoch，会对其中的100（eval_pct）个batch的结果做一个评估；eval_every_step= 113
if step % eval_every_step == 0:
# todo eval_stages=[]；save_best_model=False; save_from_step=10500; avg_sgd=True; ema_eval_stages=[]
# print(f"eval_stages={eval_stages}；save_best_model={save_best_model}; save_from_step={save_from_step}; avg_sgd={avg_sgd}; ema_eval_stages={ema_eval_stages}")
if eval_stages:
trainer.eval_step(sess,
epoch,
step,
eval_batches,
stages=eval_stages)
if save_best_model and epoch > 0 and eval_smape.last < best_smape:
best_smape = eval_smape.last
saver.save(sess,
f'data/cpt/{name}/cpt',
global_step=step)
if save_from_step and step >= save_from_step:
saver.save(sess,
f'data/cpt/{name}/cpt',
global_step=step)
if avg_sgd and ema_eval_stages:
ema_saver.save(sess,
'data/cpt_tmp/ema',
write_meta_graph=False)
# restore ema-backed vars
ema_loader.restore(sess, 'data/cpt_tmp/ema')
trainer.eval_step(sess,
epoch,
step,
eval_batches,
stages=ema_eval_stages)
# restore normal vars
ema_saver.restore(sess, 'data/cpt_tmp/ema')
MAE = "%.3f/%.3f/%.3f" % (eval_mae.last, eval_mae_side.last,
train_mae.last)
improvement = '↑' if eval_smape.improved else ' '
SMAPE = "%s%.3f/%.3f/%.3f" % (improvement, eval_smape.last,
eval_smape_side.last,
train_smape.last)
if tqdm:
# todo .set_description("GEN %i"%i) #设置进度条左边显示的信息
# .set_postfix(loss=random(),gen=randint(1,999),str="h",lst=[1,2]) #设置进度条右边显示的信息
tqr.set_postfix(gr=grad_norm.last, MAE=MAE, SMAPE=SMAPE)
if not trainer.has_active() or (max_steps
and step > max_steps):
break
if tqdm:
tqr.close()
trainer.end_epoch()
if not best_epoch_smape or eval_smape.avg_epoch < best_epoch_smape[
0]:
best_epoch_smape = [eval_smape.avg_epoch]
else:
best_epoch_smape.append(eval_smape.avg_epoch)
current_top = eval_smape.top
if prev_top > current_top:
prev_top = current_top
has_best_indicator = '↑'
else:
has_best_indicator = ' '
status = "%2d: Best top SMAPE=%.3f%s (%s)" % (
epoch + 1, current_top, has_best_indicator, ",".join(
["%.3f" % m.top for m in eval_smape.metrics]))
if trainer.has_active():
status += ", frwd/side best MAE=%.3f/%.3f, SMAPE=%.3f/%.3f; avg MAE=%.3f/%.3f, SMAPE=%.3f/%.3f, %d am" % \
(eval_mae.best_epoch, eval_mae_side.best_epoch, eval_smape.best_epoch, eval_smape_side.best_epoch,
eval_mae.avg_epoch, eval_mae_side.avg_epoch, eval_smape.avg_epoch, eval_smape_side.avg_epoch,
trainer.has_active())
else:
print("Early stopping!", file=sys.stderr)
break
if max_steps and step > max_steps:
print("Max steps calculated", file=sys.stderr)
break
sys.stderr.flush()
# todo best_epoch_smape=[nan]; eval_smape.avg_epoch=nan; trainer.has_active()=3; prev_top=inf; current_top=nan
# print(f"best_epoch_smape={best_epoch_smape}; eval_smape.avg_epoch={eval_smape.avg_epoch}; "
# f"trainer.has_active()={trainer.has_active()}; prev_top={prev_top}; current_top={current_top}")
# noinspection PyUnboundLocalVariable
return np.mean(best_epoch_smape, dtype=np.float64)
