def get_inputs(self):
self.log_x_encode_mean = sequence_mean(tf.log(self.y_encode + 1), self.encode_series_len-self.decode_series_len)
self.log_x_encode = transform(self.y_encode, self.log_x_encode_mean)
self.log_volume_encode_mean = sequence_mean(tf.log(self.volume_encode + 1), self.encode_series_len-self.decode_series_len)
self.log_volume_encode = transform(self.volume_encode, self.log_volume_encode_mean)
self.x = tf.expand_dims(self.log_x_encode, 2)
self.encode_features = tf.concat([
tf.expand_dims(self.log_volume_encode, 2),
tf.expand_dims(tf.cast(self.is_today, tf.float32), 2),
tf.tile(tf.reshape(self.log_volume_encode_mean, (-1, 1, 1)), (1, tf.shape(self.y_encode)[1], 1)),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)), (1, tf.shape(self.y_encode)[1], 1)),
], axis=2)
decode_idx = tf.tile(tf.expand_dims(tf.range(self.decode_series_len), 0), (tf.shape(self.y_decode)[0], 1))
self.decode_features = tf.concat([
tf.one_hot(decode_idx, self.decode_series_len),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)), (1, self.decode_series_len, 1))
], axis=2)
def get_input_sequences(self):
self.x_encode = tf.placeholder(tf.float32, [None, None])
self.encode_len = tf.placeholder(tf.int32, [None])
self.y_decode = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.decode_len = tf.placeholder(tf.int32, [None])
self.is_nan_encode = tf.placeholder(tf.float32, [None, None])
self.is_nan_decode = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.page_id = tf.placeholder(tf.int32, [None])
self.project = tf.placeholder(tf.int32, [None])
self.access = tf.placeholder(tf.int32, [None])
self.agent = tf.placeholder(tf.int32, [None])
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
self.log_x_encode_mean = sequence_mean(tf.log(self.x_encode + 1),
self.encode_len)
self.log_x_encode = self.transform(self.x_encode)
self.x = tf.expand_dims(self.log_x_encode, 2)
self.encode_features = tf.concat([
tf.expand_dims(self.is_nan_encode, 2),
tf.expand_dims(tf.cast(tf.equal(self.x_encode, 0.0), tf.float32),
2),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1),
(1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1),
(1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1),
(1, tf.shape(self.x_encode)[1], 1)),
],
axis=2)
decode_idx = tf.tile(
tf.expand_dims(tf.range(self.num_decode_steps), 0),
(tf.shape(self.y_decode)[0], 1))
self.decode_features = tf.concat([
tf.one_hot(decode_idx, self.num_decode_steps),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)),
(1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1),
(1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1),
(1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1),
(1, self.num_decode_steps, 1)),
],
axis=2)
return self.x
def get_input_sequences(self):
"""
返回log_x_encode
:return:
"""
# batch,encode_steps
self.x_encode = tf.placeholder(tf.float32, [None, None])
# batch 每条数据实际长度
self.encode_len = tf.placeholder(tf.int32, [None])
# batch,decode_steps
self.y_decode = tf.placeholder(tf.float32, [None, self.num_decode_steps])
# batch
self.decode_len = tf.placeholder(tf.int32, [None])
# batch ,decode_steps
self.is_nan_encode = tf.placeholder(tf.float32, [None, None])
# batch,decode_steps
self.is_nan_decode = tf.placeholder(tf.float32, [None, self.num_decode_steps])
# 其他特征
self.page_id = tf.placeholder(tf.int32, [None])
self.project = tf.placeholder(tf.int32, [None])
self.access = tf.placeholder(tf.int32, [None])
self.agent = tf.placeholder(tf.int32, [None])
# dropout
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
# 每行求对数- 再求平均值
self.log_x_encode_mean = sequence_mean(tf.log(self.x_encode + 1), self.encode_len)
self.log_x_encode = self.transform(self.x_encode)
# 销量的对数
self.x = tf.expand_dims(self.log_x_encode, 2)
# batch ts feature(1+1+1+9+3+2=17）
self.encode_features = tf.concat([
# nan销量0 1 掩码
tf.expand_dims(self.is_nan_encode, 2),
# 0销量 0 1 掩码
tf.expand_dims(tf.cast(tf.equal(self.x_encode, 0.0), tf.float32), 2),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)), (1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1), (1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1), (1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1), (1, tf.shape(self.x_encode)[1], 1)),
], axis=2)
# (batch ,64) 位置信息(0,1,2,...,num_decode_steps)
decode_idx = tf.tile(tf.expand_dims(tf.range(self.num_decode_steps), 0), (tf.shape(self.y_decode)[0], 1))
# (batch,64,64),features(64,1+9+3+2) 没有is_nan_encode,x_encode
self.decode_features = tf.concat([
# 把每一步独热编码
tf.one_hot(decode_idx, self.num_decode_steps),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)), (1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1), (1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1), (1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1), (1, self.num_decode_steps, 1)),
], axis=2)
return self.x
def get_inputs(self, opens, highs, lows, closes, volumes, positions,
order_prices, current_prices, time_since, todays):
log_x_mean = sequence_mean(tf.log((highs + lows) / 2. + 1),
self.series_length)
log_x = transform((highs + lows) / 2., log_x_mean)
log_opens_mean = sequence_mean(tf.log(opens + 1), self.series_length)
log_opens = transform(opens, log_opens_mean)
log_highs_mean = sequence_mean(tf.log(highs + 1), self.series_length)
log_highs = transform(highs, log_highs_mean)
log_lows_mean = sequence_mean(tf.log(lows + 1), self.series_length)
log_lows = transform(lows, log_lows_mean)
log_closes_mean = sequence_mean(tf.log(closes + 1), self.series_length)
log_closes = transform(closes, log_closes_mean)
log_volumes_mean = sequence_mean(tf.log(volumes + 1),
self.series_length)
log_volumes = transform(volumes, log_volumes_mean)
log_order_pricess = tf.log(order_prices + 1) - log_x_mean
log_current_prices = tf.log(current_prices + 1) - log_x_mean
x = tf.expand_dims(log_x, 2)
features = tf.concat([
tf.expand_dims(log_opens, 2),
tf.expand_dims(log_highs, 2),
tf.expand_dims(log_lows, 2),
tf.expand_dims(log_closes, 2),
tf.expand_dims(log_volumes, 2),
tf.tile(tf.expand_dims(tf.one_hot(positions + 1, 3), 1),
(1, tf.shape(opens)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(time_since, 60), 1),
(1, tf.shape(opens)[1], 1)),
tf.expand_dims(tf.cast(todays, tf.float32), 2),
tf.tile(tf.reshape(log_opens_mean, (-1, 1, 1)),
(1, tf.shape(opens)[1], 1)),
tf.tile(tf.reshape(log_highs_mean, (-1, 1, 1)),
(1, tf.shape(opens)[1], 1)),
tf.tile(tf.reshape(log_lows_mean, (-1, 1, 1)),
(1, tf.shape(opens)[1], 1)),
tf.tile(tf.reshape(log_closes_mean, (-1, 1, 1)),
(1, tf.shape(opens)[1], 1)),
tf.tile(tf.reshape(log_volumes_mean, (-1, 1, 1)),
(1, tf.shape(opens)[1], 1)),
tf.tile(tf.reshape(log_x_mean, (-1, 1, 1)),
(1, tf.shape(opens)[1], 1)),
tf.tile(tf.reshape(log_order_pricess, (-1, 1, 1)),
(1, tf.shape(opens)[1], 1)),
tf.tile(tf.reshape(log_current_prices, (-1, 1, 1)),
(1, tf.shape(opens)[1], 1)),
],
axis=2)
return tf.concat([x, features], axis=2)
def getData(self):
self.xEncode = tf.placeholder(tf.float32, [None, None])
self.lengthOfencode = tf.placeholder(tf.int32, [None])
self.ydecode = tf.placeholder(tf.float32, [None, self.decodeCount])
self.lengthOfDecode = tf.placeholder(tf.int32, [None])
self.isNANEncode = tf.placeholder(tf.float32, [None, None])
self.isNANdecode = tf.placeholder(tf.float32, [None, self.decodeCount])
self.page_id = tf.placeholder(tf.int32, [None])
self.project = tf.placeholder(tf.int32, [None])
self.access = tf.placeholder(tf.int32, [None])
self.agent = tf.placeholder(tf.int32, [None])
self.keepProbability = tf.placeholder(tf.float32)
#to know if we are training on data.
self.isTraining = tf.placeholder(tf.bool)
#log_xEncode_mean is log(mean of encode sequence)
self.log_xEncode_mean = sequence_mean(tf.log(self.xEncode + 1), self.lengthOfencode)
#computing transformFunction on each encode input value
self.log_xEncode = self.transformFunction(self.xEncode)
#expanding dimension by 2
self.x = tf.expand_dims(self.log_xEncode, 2)
#concat is to concatenate all these vectors around the axis. here axis is 2 which implies that this axis is 2 dimensional
#expand_dims adds an additional dimension 1 to the shape -- returns the given input but with an extra dimension.
self.encodeTensorFeatures = tf.concat([
tf.expand_dims(self.isNANEncode, 2),
#tf.cast to cast a tensor to required type.
tf.expand_dims(tf.cast(tf.equal(self.xEncode, 0.0), tf.float32), 2),
#-1 is to flatten.
tf.tile(tf.reshape(self.log_xEncode_mean, (-1, 1, 1)), (1, tf.shape(self.xEncode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1), (1, tf.shape(self.xEncode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1), (1, tf.shape(self.xEncode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1), (1, tf.shape(self.xEncode)[1], 1)),
], axis=2)
decode_idx = tf.tile(tf.expand_dims(tf.range(self.decodeCount), 0), (tf.shape(self.ydecode)[0], 1))
self.decodeTensorFeatures = tf.concat([
tf.one_hot(decode_idx, self.decodeCount),
tf.tile(tf.reshape(self.log_xEncode_mean, (-1, 1, 1)), (1, self.decodeCount, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1), (1, self.decodeCount, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1), (1, self.decodeCount, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1), (1, self.decodeCount, 1)),
], axis=2)
return self.x
def get_input_sequences(self):
"""
Placeholder for input sequences
:return:
"""
self.x_encode = tf.placeholder(tf.float32, [None, None])
self.encode_len = tf.placeholder(tf.int32, [None])
self.y_decode = tf.placeholder(tf.float32, [None, self.num_decode_steps])
self.decode_len = tf.placeholder(tf.int32, [None])
self.is_nan_encode = tf.placeholder(tf.float32, [None, None])
self.is_nan_decode = tf.placeholder(tf.float32, [None, self.num_decode_steps])
self.page_id = tf.placeholder(tf.int32, [None])
self.project = tf.placeholder(tf.int32, [None])
self.access = tf.placeholder(tf.int32, [None])
self.agent = tf.placeholder(tf.int32, [None])
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
self.log_x_encode_mean = sequence_mean(tf.log(self.x_encode + 1), self.encode_len)
self.log_x_encode = self.transform(self.x_encode)
self.x = tf.expand_dims(self.log_x_encode, 2)
# extra features in addition to time series values
# [batch size, max sequence length, output_units]
self.encode_features = tf.concat([
tf.expand_dims(self.is_nan_encode, 2),
tf.expand_dims(tf.cast(tf.equal(self.x_encode, 0.0), tf.float32), 2),
# each time step has the same web specific features and statistics
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)), (1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1), (1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1), (1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1), (1, tf.shape(self.x_encode)[1], 1)),
], axis=2)
# use the idx of decoded series as one feature, intuitively each prediction for different idx has a unique
# fixed effect
decode_idx = tf.tile(tf.expand_dims(tf.range(self.num_decode_steps), 0), (tf.shape(self.y_decode)[0], 1))
self.decode_features = tf.concat([
tf.one_hot(decode_idx, self.num_decode_steps),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)), (1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1), (1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1), (1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1), (1, self.num_decode_steps, 1)),
], axis=2)
return self.x
def get_input_sequences(self):
self.open_encode = tf.placeholder(tf.float32, [None, None])
self.high_encode = tf.placeholder(tf.float32, [None, None])
self.low_encode = tf.placeholder(tf.float32, [None, None])
self.close_encode = tf.placeholder(tf.float32, [None, None])
self.volume_encode = tf.placeholder(tf.float32, [None, None])
self.is_today = tf.placeholder(tf.int32, [None, None])
self.encode_len = tf.placeholder(tf.int32, [None])
#self.symbol = tf.placeholder(tf.int32, [None])
#self.day = tf.placeholder(tf.int32, [None])
self.y_decode = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.decode_len = tf.placeholder(tf.int32, [None])
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
self.log_x_encode_mean = sequence_mean(
tf.log((self.high_encode + self.low_encode) / 2. + 1),
self.encode_len)
self.log_x_encode = self.transform(
(self.high_encode + self.low_encode) / 2., self.log_x_encode_mean)
self.log_x_target_mean = sequence_mean(
tf.log((self.high_encode[-self.forecast_window:] +
self.low_encode[-self.forecast_window:]) / 2. + 1),
self.decode_len)
self.log_open_encode_mean = sequence_mean(tf.log(self.open_encode + 1),
self.encode_len)
self.log_open_encode = self.transform(self.open_encode,
self.log_open_encode_mean)
self.log_high_encode_mean = sequence_mean(tf.log(self.high_encode + 1),
self.encode_len)
self.log_high_encode = self.transform(self.high_encode,
self.log_high_encode_mean)
self.log_low_encode_mean = sequence_mean(tf.log(self.low_encode + 1),
self.encode_len)
self.log_low_encode = self.transform(self.low_encode,
self.log_low_encode_mean)
self.log_close_encode_mean = sequence_mean(
tf.log(self.close_encode + 1), self.encode_len)
self.log_close_encode = self.transform(self.close_encode,
self.log_close_encode_mean)
self.log_volume_encode_mean = sequence_mean(
tf.log(self.volume_encode + 1), self.encode_len)
self.log_volume_encode = self.transform(self.volume_encode,
self.log_volume_encode_mean)
self.x = tf.expand_dims(self.log_x_encode, 2)
self.encode_features = tf.concat([
tf.expand_dims(self.log_open_encode, 2),
tf.expand_dims(self.log_high_encode, 2),
tf.expand_dims(self.log_low_encode, 2),
tf.expand_dims(self.log_close_encode, 2),
tf.expand_dims(self.log_volume_encode, 2),
tf.expand_dims(tf.cast(self.is_today, tf.float32), 2),
tf.tile(tf.reshape(self.log_open_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_high_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_low_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_close_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_volume_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
],
axis=2)
decode_idx = tf.tile(
tf.expand_dims(tf.range(self.num_decode_steps), 0),
(tf.shape(self.y_decode)[0], 1))
self.decode_features = tf.concat([
tf.one_hot(decode_idx, self.num_decode_steps),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)),
(1, self.num_decode_steps, 1))
],
axis=2)
return self.x
def get_input_sequences(self):
self.x = tf.placeholder(tf.float32, [None, None])
self.y = tf.placeholder(tf.float32, [None, self.num_decode_steps])
self.x_raw = tf.placeholder(tf.float32, [None, None])
self.x_len = tf.placeholder(tf.int32, [None])
self.y_len = tf.placeholder(tf.int32, [None])
self.x_holi = tf.placeholder(tf.int32, [None, None])
self.y_holi = tf.placeholder(tf.int32, [None, self.num_decode_steps])
self.x_id = tf.placeholder(tf.int32, [None, None])
self.y_id = tf.placeholder(tf.int32, [None, self.num_decode_steps])
self.x_idx = tf.placeholder(tf.int32, [None, None])
self.y_idx = tf.placeholder(tf.int32, [None, self.num_decode_steps])
self.x_dow = tf.placeholder(tf.int32, [None, None])
self.y_dow = tf.placeholder(tf.int32, [None, self.num_decode_steps])
self.x_month = tf.placeholder(tf.int32, [None, None])
self.y_month = tf.placeholder(tf.int32, [None, self.num_decode_steps])
self.x_week = tf.placeholder(tf.int32, [None, None])
self.y_week = tf.placeholder(tf.int32, [None, self.num_decode_steps])
self.x_lags = tf.placeholder(tf.float32, [None, None, 10])
self.y_lags = tf.placeholder(tf.float32,
[None, self.num_decode_steps, 7])
self.air_area_name = tf.placeholder(tf.int32, [None])
self.air_genre_name = tf.placeholder(tf.int32, [None])
self.air_store_id2 = tf.placeholder(tf.int32, [None])
self.weights = tf.placeholder(tf.float32, [None])
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
# item_class_embeddings = tf.get_variable(
# name='item_class_embeddings',
# shape=[self.reader.num_item_class, 20],
# dtype=tf.float32
# )
#
# item_class = tf.nn.embedding_lookup(item_class_embeddings, self.item_class)
#
# item_nbr_embeddings = tf.get_variable(
# name='item_nbr_embeddings',
# shape=[self.reader.num_item_nbr, 50],
# dtype=tf.float32
# )
#
# item_nbr = tf.nn.embedding_lookup(item_nbr_embeddings, self.item_nbr)
self.x_mean = tf.expand_dims(sequence_mean(self.x, self.x_len), 1)
self.x_centered = self.x - self.x_mean
self.y_centered = self.y - self.x_mean
self.x_lags_centered = self.x_lags - tf.expand_dims(self.x_mean, 2)
self.y_lags_centered = self.y_lags - tf.expand_dims(self.x_mean, 2)
self.x_is_zero = tf.cast(
tf.equal(self.x_raw, tf.zeros_like(self.x_raw)), tf.float32)
# self.x_is_negative = tf.cast(tf.less(self.x_raw, tf.zeros_like(self.x_raw)), tf.float32)
self.encode_features = tf.concat(
[
self.x_lags_centered,
tf.one_hot(self.x_holi, 2),
tf.one_hot(self.x_idx, 9),
# tf.expand_dims(self.x_is_zero, 2),
# tf.expand_dims(self.x_is_negative, 2),
tf.tile(tf.expand_dims(self.x_mean, 2),
(1, tf.shape(self.x)[1], 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.air_area_name,
self.reader.num_area_name), 1),
(1, tf.shape(self.x)[1], 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.air_genre_name,
self.reader.num_genre_name), 1),
(1, tf.shape(self.x)[1], 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.air_store_id2,
self.reader.num_store_id), 1),
(1, tf.shape(self.x)[1], 1)),
],
axis=2)
decode_idx = tf.tile(
tf.expand_dims(tf.range(self.num_decode_steps), 0),
(tf.shape(self.y)[0], 1))
decode_features = tf.concat(
[
self.y_lags_centered,
tf.one_hot(decode_idx, self.num_decode_steps),
tf.one_hot(self.y_holi, 2),
tf.one_hot(self.y_idx, 9),
tf.tile(tf.expand_dims(self.x_mean, 2),
(1, self.num_decode_steps, 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.air_area_name,
self.reader.num_area_name), 1),
(1, self.num_decode_steps, 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.air_genre_name,
self.reader.num_genre_name), 1),
(1, self.num_decode_steps, 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.air_store_id2,
self.reader.num_store_id), 1),
(1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.cluster, self.reader.num_cluster), 1), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.family, self.reader.num_family), 1), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(item_class, 1), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.perishable, self.reader.num_perishable), 1), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(tf.expand_dims(self.is_discrete, 1), 2), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.store_nbr, self.reader.num_store_nbr), 1), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(item_nbr, 1), (1, self.num_decode_steps, 1)),
],
axis=2)
lstm_decode_features = bidirectional_lstm_layer(
decode_features, self.y_len, 100)
self.decode_features = tf.concat(
[decode_features, lstm_decode_features], axis=2)
return tf.expand_dims(self.x_centered, 2)
def get_input_sequences(self):
self.x_encode = tf.placeholder(tf.float32, [None, None])
self.encode_len = tf.placeholder(tf.int32, [None])
self.y_decode = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.decode_len = tf.placeholder(tf.int32, [None])
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
self.log_x_encode_mean = sequence_mean(tf.log(self.x_encode + 1),
self.encode_len)
# print(" self.log_x_encode_mean " ,self.log_x_encode_mean)
self.log_x_encode = self.transform(self.x_encode)
self.x = tf.expand_dims(self.log_x_encode, 2)
self.encode_features = tf.placeholder(
tf.float32,
[None, None, len(self.reader.data_cols) - 2])
feature_encode_len = tf.tile(tf.expand_dims(self.encode_len, axis=1),
(1, self.encode_features.shape[2]))
self.log_encode_features_mean = sequence_mean(
tf.log(self.encode_features + 1), feature_encode_len)
self.encode_features = self.transform(self.encode_features,
is_features=True)
##### Add code to transform features ######
# self.encode_features = #### Add code here
##### Delete this chunk once self.encode_features are transformed
# self.encode_features = tf.concat([
# tf.expand_dims(tf.cast(tf.equal(self.x_encode, 0.0), tf.float32), 2),
# tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)), (1, tf.shape(self.x_encode)[1], 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1), (1, tf.shape(self.x_encode)[1], 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1), (1, tf.shape(self.x_encode)[1], 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1), (1, tf.shape(self.x_encode)[1], 1)),
# ], axis=2)
# self.encode_features = tf.placeholder(tf.float32, [None, None])
# self.encode_features = tf.expand_dims(self.encode_features, 2)
# self.encode_features = tf.concat([self.x,self.x,], axis=2)
###### Walk through this code to see if needed anymore?
# decode_idx = tf.tile(tf.expand_dims(tf.range(self.num_decode_steps), 0), (tf.shape(self.y_decode)[0], 1))
# self.decode_features = tf.concat([
# tf.one_hot(decode_idx, self.num_decode_steps),
# tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1), (1, self.num_decode_steps, 1)),
# tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1), (1, self.num_decode_steps, 1)),
# ], axis=2)
self.decode_features = tf.placeholder(
tf.float32,
[None, self.num_decode_steps,
len(self.reader.data_cols) - 2])
self.decode_features = self.transform(self.decode_features,
is_features=True)
# decode_idx = tf.tile(tf.expand_dims(tf.range(self.num_decode_steps), 0), (tf.shape(self.y_decode)[0], 1))
# self.decode_features = tf.concat([
# tf.one_hot(decode_idx, self.num_decode_steps),
# self.x,
# ], axis=2)
return self.x
def get_input_sequences(self):
self.open_encode = tf.placeholder(tf.float32, [None, None])
self.high_encode = tf.placeholder(tf.float32, [None, None])
self.low_encode = tf.placeholder(tf.float32, [None, None])
self.close_encode = tf.placeholder(tf.float32, [None, None])
self.volume_encode = tf.placeholder(tf.float32, [None, None])
self.is_today = tf.placeholder(tf.int32, [None, None])
self.encode_len = tf.placeholder(tf.int32, [None])
self.position = tf.placeholder(
tf.int32, [None]) # -1 short position, 0 neutral, +1 long position
self.order_price = tf.placeholder(tf.float32, [None])
self.est_current_price = tf.placeholder(tf.float32, [None])
self.time_since_open = tf.placeholder(tf.int32, [None])
#we can not have a position open overnight, so it can not be more than 390 minutes in theory.
# But in reality, we won't be holding a position too long, so maybe let's say one hour max, 60 max
#self.symbol = tf.placeholder(tf.int32, [None])
#self.day = tf.placeholder(tf.int32, [None])
self.y_decode = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.decode_len = tf.placeholder(tf.int32, [None])
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
self.log_x_encode_mean = sequence_mean(
tf.log((self.high_encode + self.low_encode) / 2. + 1),
self.encode_len)
self.log_x_encode = self.transform(
(self.high_encode + self.low_encode) / 2., self.log_x_encode_mean)
self.log_x_target_mean = sequence_mean(
tf.log((self.high_encode[-self.forecast_window:] +
self.low_encode[-self.forecast_window:]) / 2. + 1),
self.decode_len)
self.log_open_encode_mean = sequence_mean(tf.log(self.open_encode + 1),
self.encode_len)
self.log_open_encode = self.transform(self.open_encode,
self.log_open_encode_mean)
self.log_high_encode_mean = sequence_mean(tf.log(self.high_encode + 1),
self.encode_len)
self.log_high_encode = self.transform(self.high_encode,
self.log_high_encode_mean)
self.log_low_encode_mean = sequence_mean(tf.log(self.low_encode + 1),
self.encode_len)
self.log_low_encode = self.transform(self.low_encode,
self.log_low_encode_mean)
self.log_close_encode_mean = sequence_mean(
tf.log(self.close_encode + 1), self.encode_len)
self.log_close_encode = self.transform(self.close_encode,
self.log_close_encode_mean)
self.log_volume_encode_mean = sequence_mean(
tf.log(self.volume_encode + 1), self.encode_len)
self.log_volume_encode = self.transform(self.volume_encode,
self.log_volume_encode_mean)
self.position = tf.placeholder(tf.int32, [None])
self.log_order_price = tf.log(self.order_price +
1) - self.log_x_encode_mean
self.log_est_current_price = tf.log(self.est_current_price +
1) - self.log_x_encode_mean
self.x = tf.expand_dims(self.log_x_encode, 2)
self.encode_features = tf.concat([
tf.expand_dims(self.log_open_encode, 2),
tf.expand_dims(self.log_high_encode, 2),
tf.expand_dims(self.log_low_encode, 2),
tf.expand_dims(self.log_close_encode, 2),
tf.expand_dims(self.log_volume_encode, 2),
tf.tile(tf.expand_dims(tf.one_hot(self.position + 1, 3), 1),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.time_since_open, 60), 1),
(1, tf.shape(self.open_encode)[1], 1)),
tf.expand_dims(tf.cast(self.is_today, tf.float32), 2),
tf.tile(tf.reshape(self.log_open_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_high_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_low_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_close_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_volume_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_order_price, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_est_current_price, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
],
axis=2)
return self.x
def get_input_sequences(self):
self.x_encode = tf.placeholder(tf.float32, [None, None])
self.encode_len = tf.placeholder(tf.int32, [None])
self.y_decode = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.decode_len = tf.placeholder(tf.int32, [None])
self.is_nan_encode = tf.placeholder(tf.float32, [None, None])
self.is_nan_decode = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.page_id = tf.placeholder(tf.int32, [None])
self.project = tf.placeholder(tf.int32, [None])
self.access = tf.placeholder(tf.int32, [None])
self.agent = tf.placeholder(tf.int32, [None])
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
# The mean is the average hits for each web sit in all the given dates
self.log_x_encode_mean = sequence_mean(tf.log(self.x_encode + 1),
self.encode_len)
self.log_x_encode = self.transform(self.x_encode)
self.x = tf.expand_dims(
self.log_x_encode,
2) # minus mean and expand to (batch_size, encode_len, 1)
# Encoded features, its shape is (batch_size, seq_len, 17)
self.encode_features = tf.concat([
tf.expand_dims(self.is_nan_encode, 2),
tf.expand_dims(tf.cast(tf.equal(self.x_encode, 0.0), tf.float32),
2),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1),
(1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1),
(1, tf.shape(self.x_encode)[1], 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1),
(1, tf.shape(self.x_encode)[1], 1)),
],
axis=2)
# shape (batch_size, num_decode_steps)
decode_idx = tf.tile(
tf.expand_dims(tf.range(self.num_decode_steps), 0),
(tf.shape(self.y_decode)[0], 1))
# shape (batch_size, num_decode_steps, 64 + 1 + 9 + 3 + 2 = 79)
self.decode_features = tf.concat([
tf.one_hot(decode_idx, self.num_decode_steps),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)),
(1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.project, 9), 1),
(1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.access, 3), 1),
(1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(tf.one_hot(self.agent, 2), 1),
(1, self.num_decode_steps, 1)),
],
axis=2)
# x is the hits minus mean and transformed by log(1+x) to make it easy to predict
# shape (batch_size, seq_len, 1)
return self.x
def get_input_sequences(self):
# self.x_scales = tf.placeholder(tf.float32, [None, None])
self.x = tf.placeholder(tf.float32, [None, None])
self.y = tf.placeholder(tf.float32, [None, self.num_decode_steps])
self.x_len = tf.placeholder(tf.int32, [None])
self.y_len = tf.placeholder(tf.int32, [None])
self.all_id = tf.placeholder(tf.int32, [None])
self.x_for_mean = tf.placeholder(tf.float32, [None, None])
self.x_len_for_mean = tf.placeholder(tf.int32, [None])
self.x_ts = tf.placeholder(tf.float32, [None, None, 10])
self.x_lags = tf.placeholder(tf.float32, [None, None, 13])
self.y_lags = tf.placeholder(tf.float32,
[None, self.num_decode_steps, 8])
self.x_day = tf.placeholder(tf.float32, [None, None])
self.y_day = tf.placeholder(tf.float32, [None, self.num_decode_steps])
self.x_snap = tf.placeholder(tf.float32, [None, None])
self.y_snap = tf.placeholder(tf.float32, [None, self.num_decode_steps])
self.x_event_name_1 = tf.placeholder(tf.float32, [None, None])
self.y_event_name_1 = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.x_event_type_1 = tf.placeholder(tf.float32, [None, None])
self.y_event_type_1 = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.x_sell_price = tf.placeholder(tf.float32, [None, None])
self.y_sell_price = tf.placeholder(tf.float32,
[None, self.num_decode_steps])
self.x_sell_price_last_digit = tf.placeholder(tf.int32, [None, None])
self.y_sell_price_last_digit = tf.placeholder(
tf.int32, [None, self.num_decode_steps])
self.x_wday = tf.placeholder(tf.int32, [None, None])
self.y_wday = tf.placeholder(tf.int32, [None, self.num_decode_steps])
self.store_id = tf.placeholder(tf.int32, [None])
self.state_id = tf.placeholder(tf.int32, [None])
self.dept_id = tf.placeholder(tf.int32, [None])
self.cat_id = tf.placeholder(tf.int32, [None])
self.item_id = tf.placeholder(tf.int32, [None])
self.hierarchy_data = tf.placeholder(tf.float32, [None, 11])
self.weights = tf.placeholder(tf.float32, [None])
self.keep_prob = tf.placeholder(tf.float32)
self.is_training = tf.placeholder(tf.bool)
item_id_embeddings = tf.get_variable(name='item_id_embeddings',
shape=[self.reader.num_item, 20],
dtype=tf.float32)
item_id = tf.nn.embedding_lookup(item_id_embeddings, self.item_id)
self.x_mean = tf.expand_dims(
sequence_mean(self.x_for_mean, self.x_len_for_mean), 1)
self.x_centered = self.x - self.x_mean
self.y_centered = self.y - self.x_mean
self.x_ts_centered = self.x_ts - tf.expand_dims(self.x_mean, 2)
self.x_lags_centered = self.x_lags - tf.expand_dims(self.x_mean, 2)
self.y_lags_centered = self.y_lags - tf.expand_dims(self.x_mean, 2)
self.x_is_zero = tf.cast(tf.equal(self.x, tf.zeros_like(self.x)),
tf.float32)
self.encode_features = tf.concat(
[
self.x_ts_centered,
self.x_lags_centered,
# tf.expand_dims(self.x_day, 2),
tf.one_hot(self.x_sell_price_last_digit, 9),
# tf.one_hot(self.x_wday, 6),
tf.expand_dims(self.x_snap, 2),
tf.expand_dims(self.x_is_zero, 2),
tf.expand_dims(self.x_event_name_1, 2),
tf.expand_dims(self.x_event_type_1, 2),
tf.expand_dims(self.x_sell_price, 2),
tf.tile(tf.expand_dims(self.x_mean, 2),
(1, tf.shape(self.x)[1], 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.store_id, self.reader.num_store), 1),
(1, tf.shape(self.x)[1], 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.state_id, self.reader.num_state), 1),
(1, tf.shape(self.x)[1], 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.dept_id, self.reader.num_dept), 1),
(1, tf.shape(self.x)[1], 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.cat_id, self.reader.num_cat), 1),
(1, tf.shape(self.x)[1], 1)),
tf.tile(tf.expand_dims(item_id, 1),
(1, tf.shape(self.x)[1], 1)),
],
axis=2)
decode_idx = tf.tile(
tf.expand_dims(tf.range(self.num_decode_steps), 0),
(tf.shape(self.y)[0], 1))
self.decode_features = tf.concat(
[
self.y_lags_centered,
# tf.expand_dims(self.y_day, 2),
tf.one_hot(self.y_sell_price_last_digit, 9),
# tf.one_hot(self.y_wday, 6),
tf.expand_dims(self.y_snap, 2),
tf.expand_dims(self.y_event_name_1, 2),
tf.expand_dims(self.y_event_type_1, 2),
tf.expand_dims(self.y_sell_price, 2),
tf.one_hot(decode_idx, self.num_decode_steps),
tf.tile(tf.expand_dims(self.x_mean, 2),
(1, self.num_decode_steps, 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.store_id, self.reader.num_store), 1),
(1, self.num_decode_steps, 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.state_id, self.reader.num_state), 1),
(1, self.num_decode_steps, 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.dept_id, self.reader.num_dept), 1),
(1, self.num_decode_steps, 1)),
tf.tile(
tf.expand_dims(
tf.one_hot(self.cat_id, self.reader.num_cat), 1),
(1, self.num_decode_steps, 1)),
tf.tile(tf.expand_dims(item_id, 1),
(1, self.num_decode_steps, 1)),
],
axis=2)
return tf.expand_dims(self.x_centered, 2)
def get_inputs(self):
self.log_x_encode_mean = sequence_mean(
tf.log((self.high_encode + self.low_encode) / 2. + 1),
self.series_length)
self.log_x_encode = transform(
(self.high_encode + self.low_encode) / 2., self.log_x_encode_mean)
self.log_open_encode_mean = sequence_mean(tf.log(self.open_encode + 1),
self.series_length)
self.log_open_encode = transform(self.open_encode,
self.log_open_encode_mean)
self.log_high_encode_mean = sequence_mean(tf.log(self.high_encode + 1),
self.series_length)
self.log_high_encode = transform(self.high_encode,
self.log_high_encode_mean)
self.log_low_encode_mean = sequence_mean(tf.log(self.low_encode + 1),
self.series_length)
self.log_low_encode = transform(self.low_encode,
self.log_low_encode_mean)
self.log_close_encode_mean = sequence_mean(
tf.log(self.close_encode + 1), self.series_length)
self.log_close_encode = transform(self.close_encode,
self.log_close_encode_mean)
self.log_volume_encode_mean = sequence_mean(
tf.log(self.volume_encode + 1), self.series_length)
self.log_volume_encode = transform(self.volume_encode,
self.log_volume_encode_mean)
self.position = tf.placeholder(tf.int32, [None])
self.log_order_price = tf.log(self.order_price +
1) - self.log_x_encode_mean
self.log_est_current_price = tf.log(self.est_current_price +
1) - self.log_x_encode_mean
self.x = tf.expand_dims(self.log_x_encode, 2)
self.features = tf.concat(
[
tf.expand_dims(self.log_open_encode, 2),
tf.expand_dims(self.log_high_encode, 2),
tf.expand_dims(self.log_low_encode, 2),
tf.expand_dims(self.log_close_encode, 2),
tf.expand_dims(self.log_volume_encode, 2),
tf.tile(tf.expand_dims(tf.one_hot(self.position + 1, 3), 1),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(
tf.expand_dims(tf.one_hot(self.time_since_open, 60), 1),
(1, tf.shape(self.open_encode)[1], 1)),
#tf.expand_dims(tf.cast(self.is_today, tf.float32), 2),
tf.tile(tf.reshape(self.log_open_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_high_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_low_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_close_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_volume_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_x_encode_mean, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_order_price, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
tf.tile(tf.reshape(self.log_est_current_price, (-1, 1, 1)),
(1, tf.shape(self.open_encode)[1], 1)),
],
axis=2)
self.x = tf.concat([self.x, self.features], axis=2)
