def main():
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu
# preprocessing class
pre_process = MinMaxNormalization01()
print('load train, validate, test data...')
split = [43824, 8760, 8760]
data, train_data, val_data, test_data = load_data(filename=['data/taxi/p_map.mat', 'data/taxi/d_map.mat'],
split=split)
# data: [num, row, col, channel]
print('preprocess train data...')
pre_process.fit(train_data)
if 'ResNet' in FLAGS.model:
pre_index = max(FLAGS.closeness * 1, FLAGS.period * 7, FLAGS.trend * 7 * 24)
all_timestamps = gen_timestamps(['2009', '2010', '2011', '2012', '2013', '2014', '2015'])
data = pre_process.transform(data)
# train_data = train_data
train_data = data[:split[0]]
val_data = data[split[0] - pre_index:split[0] + split[1]]
test_data = data[split[0] + split[1] - pre_index:split[0] + split[1] + split[2]]
del data
# get train, validate, test timestamps
train_timestamps = all_timestamps[:split[0]]
val_timestamps = all_timestamps[split[0] - pre_index:split[0] + split[1]]
test_timestamps = all_timestamps[split[0] + split[1] - pre_index:split[0] + split[1] + split[2]]
# get x, y
train_x, train_y = batch_data_cpt_ext(train_data, train_timestamps,
batch_size=FLAGS.batch_size, close=FLAGS.closeness, period=FLAGS.period,
trend=FLAGS.trend)
val_x, val_y = batch_data_cpt_ext(val_data, val_timestamps,
batch_size=FLAGS.batch_size, close=FLAGS.closeness, period=FLAGS.period,
trend=FLAGS.trend)
test_x, test_y = batch_data_cpt_ext(test_data, test_timestamps,
batch_size=FLAGS.batch_size, close=FLAGS.closeness, period=FLAGS.period,
trend=FLAGS.trend)
train = {'x': train_x, 'y': train_y}
val = {'x': val_x, 'y': val_y}
test = {'x': test_x, 'y': test_y}
nb_flow = train_data.shape[-1]
row = train_data.shape[1]
col = train_data.shape[2]
if FLAGS.model == 'AttResNet':
print('k-means to cluster...')
model_path = 'taxi-results/model_save/AttResNet/'
log_path = 'taxi-results/log/AttResNet/'
if FLAGS.pre_saved_cluster:
cluster_centroid = np.load(model_path + 'cluster_centroid.npy')
else:
vector_data = np.reshape(train_data, (train_data.shape[0], -1))
kmeans = KMeans(n_clusters=FLAGS.cluster_num, init='random', n_init=FLAGS.kmeans_run_num,
tol=0.00000001).fit(vector_data)
cluster_centroid = kmeans.cluster_centers_
cluster_centroid = np.reshape(cluster_centroid,
(-1, train_data.shape[1], train_data.shape[2], train_data.shape[3]))
if not os.path.exists(model_path):
os.makedirs(model_path)
if not os.path.exists(log_path):
os.makedirs(log_path)
np.save(model_path + 'cluster_centroid.npy', cluster_centroid)
print('build AttResNet model...')
model = AttResNet(input_conf=[[FLAGS.closeness, nb_flow, row, col], [FLAGS.period, nb_flow, row, col],
[FLAGS.trend, nb_flow, row, col], [8]],
att_inputs=cluster_centroid, att_nodes=FLAGS.att_nodes,
att_layer=['conv', 'conv'],
att_layer_param=[[[3, 3], [1, 1, 1, 1], 8], [[3, 3], [1, 1, 1, 1], 2]],
batch_size=FLAGS.batch_size,
layer=['conv', 'res_net', 'conv'],
layer_param=[[[3, 3], [1, 1, 1, 1], 64],
[3, [[[3, 3], [1, 1, 1, 1], 64], [[3, 3], [1, 1, 1, 1], 64]]],
[[3, 3], [1, 1, 1, 1], 2]]
)
else:
print('build ResNet model...')
model_path = 'taxi-results/model_save/ResNet/'
log_path = 'taxi-results/log/ResNet/'
model = ResNet(input_conf=[[FLAGS.closeness, nb_flow, row, col], [FLAGS.period, nb_flow, row, col],
[FLAGS.trend, nb_flow, row, col], [8]], batch_size=FLAGS.batch_size,
layer=['conv', 'res_net', 'conv'],
layer_param=[[[3, 3], [1, 1, 1, 1], 64],
[3, [[[3, 3], [1, 1, 1, 1], 64], [[3, 3], [1, 1, 1, 1], 64]]],
[[3, 3], [1, 1, 1, 1], 2]]
)
print('model solver...')
solver = ModelSolver(model, train, val, preprocessing=pre_process,
n_epochs=FLAGS.n_epochs,
batch_size=FLAGS.batch_size,
update_rule=FLAGS.update_rule,
learning_rate=FLAGS.lr, save_every=FLAGS.save_every,
pretrained_model=FLAGS.pretrained_model, model_path=model_path,
test_model='taxi-results/model_save/ResNet/model-' + str(FLAGS.n_epochs),
log_path=log_path,
cross_val=False, cpt_ext=True)
if FLAGS.train:
print('begin training...')
test_n = {'data': test_data, 'timestamps': test_timestamps}
_, test_prediction = solver.train(test, test_n, output_steps=FLAGS.output_steps)
# get test_target and test_prediction
i = pre_index
test_target = []
while i < len(test_data) - FLAGS.output_steps:
test_target.append(test_data[i:i + FLAGS.output_steps])
i += 1
test_target = np.asarray(test_target)
if FLAGS.test:
print('begin testing for predicting next 1 step')
solver.test(test)
print('begin testing for predicting next' + str(FLAGS.output_steps) + 'steps')
test_n = {'data': test_data, 'timestamps': test_timestamps}
solver.test_1_to_n(test_n)
else:
train_data = pre_process.transform(train_data)
train_x, train_y = batch_data(data=train_data, batch_size=FLAGS.batch_size,
input_steps=FLAGS.input_steps, output_steps=FLAGS.output_steps)
val_data = pre_process.transform(val_data)
val_x, val_y = batch_data(data=val_data, batch_size=FLAGS.batch_size,
input_steps=FLAGS.input_steps, output_steps=FLAGS.output_steps)
test_data = pre_process.transform(test_data)
test_x, test_y = batch_data(data=test_data, batch_size=FLAGS.batch_size,
input_steps=FLAGS.input_steps, output_steps=FLAGS.output_steps)
train = {'x': train_x, 'y': train_y}
val = {'x': val_x, 'y': val_y}
test = {'x': test_x, 'y': test_y}
input_dim = [train_data.shape[1], train_data.shape[2], train_data.shape[3]]
if FLAGS.model == 'ConvLSTM':
print('build ConvLSTM model...')
model = ConvLSTM(input_dim=input_dim, batch_size=FLAGS.batch_size,
layer={'encoder': ['conv', 'conv', 'conv_lstm', 'conv_lstm'],
'decoder': ['conv_lstm', 'conv_lstm', 'conv', 'conv']},
layer_param={'encoder': [[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 16],
[[16, 16], [3, 3], 64],
[[16, 16], [3, 3], 64]],
'decoder': [[[16, 16], [3, 3], 64],
[[16, 16], [3, 3], 64],
[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 2]]},
input_steps=10, output_steps=10)
print('model solver...')
solver = ModelSolver(model, train, val, preprocessing=pre_process,
n_epochs=FLAGS.n_epochs,
batch_size=FLAGS.batch_size,
update_rule=FLAGS.update_rule,
learning_rate=FLAGS.lr, save_every=FLAGS.save_every,
pretrained_model=FLAGS.pretrained_model,
model_path='taxi-results/model_save/ConvLSTM/',
test_model='taxi-results/model_save/ConvLSTM/model-' + str(FLAGS.n_epochs),
log_path='taxi-results/log/ConvLSTM/')
elif 'AttConvLSTM' in FLAGS.model:
# train_data: [num, row, col, channel]
if FLAGS.use_ae:
# auto-encoder to cluster train_data
print('auto-encoder to cluster...')
model_path = 'taxi-results/model_save/AEAttConvLSTM/'
log_path = 'taxi-results/log/AEAttConvLSTM/'
if FLAGS.pre_saved_cluster:
cluster_centroid = np.load(model_path + 'cluster_centroid.npy')
else:
ae = AutoEncoder(input_dim=input_dim, z_dim=[16, 16, 16],
layer={'encoder': ['conv', 'conv'],
'decoder': ['conv', 'conv']},
layer_param={'encoder': [[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 16]],
'decoder': [[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 2]]},
model_save_path=model_path,
batch_size=FLAGS.batch_size)
if FLAGS.ae_train:
ae.train(train_data, batch_size=FLAGS.batch_size, learning_rate=FLAGS.lr, n_epochs=20,
pretrained_model=FLAGS.ae_pretrained_model)
train_z_data = ae.get_z(train_data, pretrained_model=FLAGS.ae_pretrained_model)
print(train_z_data.shape)
# k-means to cluster train_z_data
vector_data = np.reshape(train_z_data, (train_z_data.shape[0], -1))
kmeans = KMeans(n_clusters=FLAGS.cluster_num, init='random', n_init=FLAGS.kmeans_run_num,
tol=0.00000001).fit(vector_data)
cluster_centroid = kmeans.cluster_centers_
print(np.array(cluster_centroid).shape)
# reshape to [cluster_num, row, col, channel]
cluster_centroid = np.reshape(cluster_centroid,
(-1, train_z_data.shape[1], train_z_data.shape[2],
train_z_data.shape[3]))
# decoder to original space
cluster_centroid = ae.get_y(cluster_centroid, pretrained_model=FLAGS.ae_pretrained_model)
print(cluster_centroid.shape)
np.save(model_path + 'cluster_centroid.npy', cluster_centroid)
else:
# k-means to cluster train_data
print('k-means to cluster...')
model_path = 'taxi-results/model_save/' + FLAGS.model + '/'
log_path = 'taxi-results/log/' + FLAGS.model + '/'
if not os.path.exists(model_path):
os.makedirs(model_path)
if not os.path.exists(log_path):
os.makedirs(log_path)
if FLAGS.pre_saved_cluster:
cluster_centroid = np.load(model_path + 'cluster_centroid.npy')
else:
vector_data = np.reshape(train_data, (train_data.shape[0], -1))
# init_vectors = vector_data[:FLAGS.cluster_num, :]
# cluster_centroid = init_vectors
kmeans = KMeans(n_clusters=FLAGS.cluster_num, init='random', n_init=FLAGS.kmeans_run_num,
tol=0.00000001).fit(vector_data)
cluster_centroid = kmeans.cluster_centers_
# reshape to [cluster_num, row, col, channel]
cluster_centroid = np.reshape(cluster_centroid,
(-1, train_data.shape[1], train_data.shape[2], train_data.shape[3]))
np.save(model_path + 'cluster_centroid.npy', cluster_centroid)
# build model
print('build ' + FLAGS.model + ' model...')
if FLAGS.model == 'AttConvLSTM':
model = AttConvLSTM(input_dim=input_dim,
att_inputs=cluster_centroid, att_nodes=FLAGS.att_nodes,
batch_size=FLAGS.batch_size,
layer={'encoder': ['conv', 'conv', 'conv_lstm', 'conv_lstm'],
'decoder': ['conv_lstm', 'conv_lstm', 'conv', 'conv'],
'attention': ['conv', 'conv']},
layer_param={'encoder': [[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 16],
[[16, 16], [3, 3], 64],
[[16, 16], [3, 3], 64]],
'decoder': [[[16, 16], [3, 3], 64],
[[16, 16], [3, 3], 64],
[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 2]],
'attention': [[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 16]]},
input_steps=10, output_steps=10)
elif FLAGS.model == 'MultiAttConvLSTM':
model = MultiAttConvLSTM(input_dim=input_dim,
att_inputs=cluster_centroid, att_nodes=FLAGS.att_nodes,
batch_size=FLAGS.batch_size,
layer={'encoder': ['conv', 'conv', 'conv_lstm', 'conv_lstm'],
'decoder': ['conv_lstm', 'conv_lstm', 'conv', 'conv'],
'attention': ['conv', 'conv']},
layer_param={'encoder': [[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 16],
[[16, 16], [3, 3], 64],
[[16, 16], [3, 3], 64]],
'decoder': [[[16, 16], [3, 3], 64],
[[16, 16], [3, 3], 64],
[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 2]],
'attention': [[[3, 3], [1, 2, 2, 1], 8],
[[3, 3], [1, 2, 2, 1], 16]]},
input_steps=10, output_steps=10)
print('model solver...')
solver = ModelSolver(model, train, val, preprocessing=pre_process,
n_epochs=FLAGS.n_epochs,
batch_size=FLAGS.batch_size,
update_rule=FLAGS.update_rule,
learning_rate=FLAGS.lr, save_every=FLAGS.save_every,
pretrained_model=FLAGS.pretrained_model, model_path=model_path,
test_model=model_path + 'model-' + str(FLAGS.n_epochs), log_path=log_path)
if FLAGS.train:
print('begin training...')
test_prediction, _ = solver.train(test)
test_target = np.asarray(test_y)
if FLAGS.test:
print('test trained model...')
solver.test_model = solver.model_path + FLAGS.pretrained_model
test_prediction = solver.test(test)
test_target = np.asarray(test_y)
np.save('taxi-results/results/'+FLAGS.model+'/test_target.npy', test_target)
np.save('taxi-results/results/'+FLAGS.model+'/test_prediction.npy', test_prediction)
print(test_prediction.shape)
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu
# preprocessing class
pre_process = MinMaxNormalization01()
print('load train, validate, test data...')
split = [17520, 4416, 4368]
data, train_data, val_data, test_data = load_npy_data(
filename=['data/citybike/p_map.npy', 'data/citybike/d_map.npy'],
split=split)
# data: [num, row, col, channel]
print('preprocess train data...')
pre_process.fit(train_data)
train_data = pre_process.transform(train_data)
train_x, train_y = batch_data(data=train_data,
batch_size=FLAGS.batch_size,
input_steps=FLAGS.input_steps,
output_steps=FLAGS.output_steps)
val_data = pre_process.transform(val_data)
val_x, val_y = batch_data(data=val_data,
batch_size=FLAGS.batch_size,
input_steps=FLAGS.input_steps,
output_steps=FLAGS.output_steps)
test_data = pre_process.transform(test_data)
test_x, test_y = batch_data(data=test_data,
batch_size=FLAGS.batch_size,
input_steps=FLAGS.input_steps,
output_steps=FLAGS.output_steps)
train = {'x': train_x, 'y': train_y}
val = {'x': val_x, 'y': val_y}
test = {'x': test_x, 'y': test_y}
input_dim = [train_data.shape[1], train_data.shape[2], train_data.shape[3]]
if 'AttConvLSTM' in FLAGS.model:
# train_data: [num, row, col, channel]
model_path = 'citybike-results/model_save/' + FLAGS.model + '/'
log_path = 'citybike-results/log/' + FLAGS.model + '/'
if not os.path.exists(model_path):
os.makedirs(model_path)
if not os.path.exists(log_path):
os.makedirs(log_path)
if FLAGS.pre_saved_cluster:
cluster_centroid = np.load(model_path + 'cluster_centroid.npy')
else:
vector_data = np.reshape(train_data, (train_data.shape[0], -1))
cluster_centroid_1 = None
cluster_centroid_2 = None
cluster_centroid = None
if FLAGS.kmeans_cluster:
print('k-means to cluster...')
kmeans = KMeans(n_clusters=FLAGS.cluster_num,
init='random',
n_init=FLAGS.kmeans_run_num,
tol=0.00000001).fit(vector_data)
cluster_centroid_1 = kmeans.cluster_centers_
if FLAGS.average_cluster:
print('average cluster...')
if FLAGS.average_cluster == 24:
cluster_centroid_2 = average_cluster_24(vector_data)
elif FLAGS.average_cluster == 48:
cluster_centroid_2 = average_cluster_48(vector_data)
if cluster_centroid_1 is not None:
cluster_centroid = cluster_centroid_1
if cluster_centroid_2 is not None:
if cluster_centroid is not None:
cluster_centroid = np.concatenate(
(cluster_centroid_1, cluster_centroid_2), axis=0)
else:
cluster_centroid = cluster_centroid_2
# reshape to [cluster_num, row, col, channel]
cluster_centroid = np.reshape(
cluster_centroid, (-1, train_data.shape[1],
train_data.shape[2], train_data.shape[3]))
np.save(model_path + 'cluster_centroid.npy', cluster_centroid)
# build model
print 'build ' + FLAGS.model + ' model...'
if FLAGS.model == 'AttConvLSTM':
model = AttConvLSTM(input_dim=input_dim,
att_inputs=cluster_centroid,
att_nodes=FLAGS.att_nodes,
batch_size=FLAGS.batch_size,
layer={
'encoder':
['conv', 'conv', 'conv_lstm', 'conv_lstm'],
'decoder':
['conv_lstm', 'conv_lstm', 'conv', 'conv'],
'attention': ['conv', 'conv']
},
layer_param={
'encoder': [[[3, 3], [1, 1, 1, 1], 8],
[[3, 3], [1, 1, 1, 1], 16],
[[16, 16], [3, 3], 64],
[[16, 16], [3, 3], 64]],
'decoder': [[[16, 16], [3, 3], 64],
[[16, 16], [3, 3], 64],
[[3, 3], [1, 1, 1, 1], 8],
[[3, 3], [1, 1, 1, 1], 2]],
'attention': [[[3, 3], [1, 1, 1, 1], 8],
[[3, 3], [1, 1, 1, 1], 16]]
},
input_steps=10,
output_steps=10)
elif FLAGS.model == 'MultiAttConvLSTM':
model = MultiAttConvLSTM(
input_dim=input_dim,
att_inputs=cluster_centroid,
att_nodes=FLAGS.att_nodes,
batch_size=FLAGS.batch_size,
layer={
'encoder': ['conv', 'conv', 'conv_lstm', 'conv_lstm'],
'decoder': ['conv_lstm', 'conv_lstm', 'conv', 'conv'],
'attention': ['conv', 'conv']
},
layer_param={
'encoder': [[[3, 3], [1, 1, 1, 1], 8],
[[3, 3], [1, 1, 1, 1], 16],
[[16, 16], [3, 3], 64], [[16, 16], [3, 3],
64]],
'decoder': [[[16, 16], [3, 3], 64], [[16, 16], [3, 3], 64],
[[3, 3], [1, 1, 1, 1], 8],
[[3, 3], [1, 1, 1, 1], 2]],
'attention': [[[3, 3], [1, 1, 1, 1], 8],
[[3, 3], [1, 1, 1, 1], 16]]
},
input_steps=10,
output_steps=10)
print('model solver...')
solver = ModelSolver(model,
train,
val,
preprocessing=pre_process,
n_epochs=FLAGS.n_epochs,
batch_size=FLAGS.batch_size,
update_rule=FLAGS.update_rule,
learning_rate=FLAGS.lr,
save_every=FLAGS.save_every,
pretrained_model=FLAGS.pretrained_model,
model_path=model_path,
test_model=model_path + 'model-' +
str(FLAGS.n_epochs),
log_path=log_path)
if FLAGS.train:
print('begin training...')
test_prediction, _ = solver.train(test)
test_target = np.asarray(test_y)
if FLAGS.test:
print('test trained model...')
solver.test_model = solver.model_path + FLAGS.pretrained_model
test_prediction = solver.test(test)
test_target = np.asarray(test_y)
np.save('citybike-results/results/' + FLAGS.model + '/test_target.npy',
test_target)
np.save('citybike-results/results/' + FLAGS.model + '/test_prediction.npy',
test_prediction)