args.update(nni.get_next_parameter())
model_dir = os.path.join(model_dir_path, args['Group'])
code_version = 'CPT_STMeta_{}_K{}L{}_{}'.format(
''.join([e[0] for e in args['Graph'].split('-')]), args['K'], args['L'],
args['CodeVersion'] + nni.get_sequence_id())
# Config data loader
data_loader = NodeTrafficLoader(
dataset=args['Dataset'],
city=args['City'],
data_range=args['DataRange'],
train_data_length=args['TrainDays'],
test_ratio=0.1,
C_T=int(args['CT']),
P_T=int(args['PT']),
T_T=int(args['TT']),
TI=args['TI'],
TD=args['TD'],
TC=args['TC'],
normalize=True if args['Normalize'] == 'True' else False,
graph=args['Graph'],
with_lm=True)
de_normalizer = None if args[
'Normalize'] == 'False' else data_loader.normalizer.min_max_denormal
CPT_STMeta_Obj = STMeta_V1(num_node=data_loader.station_number,
num_graph=data_loader.LM.shape[0],
external_dim=data_loader.external_dim,
C_T=int(args['CT']),
parser.add_argument('--city', default="Shanghai", type=str)
parser.add_argument('--MergeIndex', default=1)
parser.add_argument('--DataRange', default="all")
parser.add_argument('--TrainDays', default="all")
parser.add_argument('--MergeWay', default="sum")
parser.add_argument('--test_ratio', default=0.1, type=float)
# note that the args is different from param
args = vars(parser.parse_args())
data_loader = NodeTrafficLoader(dataset=args["dataset"],
city=args['city'],
closeness_len=int(params['CT']),
period_len=int(params['PT']),
trend_len=int(params['TT']),
data_range=args['DataRange'],
train_data_length=args['TrainDays'],
test_ratio=args['test_ratio'],
with_lm=False,
normalize=False,
MergeIndex=args['MergeIndex'],
MergeWay=args['MergeWay'])
train_closeness, val_closeness = SplitData.split_data(
data_loader.train_closeness, [0.9, 0.1])
train_period, val_period = SplitData.split_data(data_loader.train_period,
[0.9, 0.1])
train_trend, val_trend = SplitData.split_data(data_loader.train_trend,
[0.9, 0.1])
train_y, val_y = SplitData.split_data(data_loader.train_y, [0.9, 0.1])
from UCTB.dataset import NodeTrafficLoader #from UCTB.utils import st_map from dateutil.parser import parse # Config data loader data_loader = NodeTrafficLoader(dataset='Bike', city='NYC', with_lm=False) data_loader.st_map()
import numpy as np
from UCTB.dataset import NodeTrafficLoader
from sklearn.ensemble import GradientBoostingRegressor
from UCTB.evaluation import metric
dataset = 'ChargeStation'
city = 'Beijing'
data_loader = NodeTrafficLoader(dataset=dataset, city=city, with_lm=False,
closeness_len=5, period_len=3, trend_len=4, test_ratio=0.1, normalize=False)
prediction = []
for i in range(data_loader.station_number):
print('*************************************************************')
print('Station', i)
model = GradientBoostingRegressor(n_estimators=540, max_depth=3)
train_x = np.concatenate([data_loader.train_closeness[:, 0, i, :],
data_loader.train_period[:, 0, i, :],
data_loader.train_trend[:, 0, i, :]], axis=-1)
test_x = np.concatenate([data_loader.test_closeness[:, 0, i, :],
data_loader.test_period[:, 0, i, :],
data_loader.test_trend[:, 0, i, :]], axis=-1)
model.fit(train_x, data_loader.train_y[:, i])
p = model.predict(test_x).reshape([-1, 1, 1])
args['trend_len'], ''.join([e[0] for e in args['graph'].split('-')]),
args['gcn_k'], args['gcn_layers'],
int(args["MergeIndex"]) * 5, args['mark'])
model_dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'model_dir')
model_dir_path = os.path.join(model_dir_path, args['group'])
#####################################################################
data_loader = NodeTrafficLoader(
dataset=args['dataset'],
city=args['city'],
data_range=args['data_range'],
train_data_length=args['train_data_length'],
test_ratio=float(args['test_ratio']),
closeness_len=args['closeness_len'],
period_len=args['period_len'],
trend_len=args['trend_len'],
normalize=args['normalize'],
with_tpe=True if args['st_method'] == 'gal_gcn' else False,
workday_parser=is_work_day_america
if args['dataset'] == 'Bike' else is_work_day_china,
MergeIndex=args['MergeIndex'],
MergeWay=args["MergeWay"])
# split data
train_closeness, val_closeness = SplitData.split_data(
data_loader.train_closeness, [0.9, 0.1])
train_period, val_period = SplitData.split_data(data_loader.train_period,
[0.9, 0.1])
train_trend, val_trend = SplitData.split_data(data_loader.train_trend,
[0.9, 0.1])
if len(deviceIDs) == 0:
current_device = '-1'
else:
if nni_params:
current_device = str(deviceIDs[int(nni_sid) % len(deviceIDs)])
else:
current_device = str(deviceIDs[0])
# Config data loader
data_loader = NodeTrafficLoader(
dataset=args['Dataset'],
city=args['City'],
data_range=args['DataRange'],
train_data_length=args['TrainDays'],
closeness_len=int(args['CT']),
period_len=int(args['PT']),
trend_len=int(args['TT']),
normalize=True,
MergeIndex=args['MergeIndex'],
MergeWay="max" if args["Dataset"] == "ChargeStation" else "sum")
# build graphs
graph_obj = GraphGenerator(graph=args['Graph'],
data_loader=data_loader,
threshold_distance=args['TD'],
threshold_correlation=args['TC'],
threshold_interaction=args['TI'])
ST_MGCN_Obj = ST_MGCN(T=int(args['CT']) + int(args['PT']) + int(args['TT']),
input_dim=1,
parser = gacn_param_parser()
args = parser.parse_args()
model_dir = os.path.join(model_dir_path, args.Group)
code_version = 'GACN_{}_K{}L{}_{}'.format(
''.join([e[0] for e in args.Graph.split('-')]), args.K, args.L,
args.CodeVersion)
# Config data loader
data_loader = NodeTrafficLoader(dataset=args.Dataset,
city=args.City,
data_range=args.DataRange,
train_data_length=args.TrainDays,
test_ratio=0.1,
normalize=True,
T=args.T,
TI=args.TI,
TD=args.TD,
TC=args.TC,
graph=args.Graph,
with_lm=True)
de_normalizer = data_loader.normalizer.min_max_denormal
GACN_Obj = GACN(num_node=data_loader.station_number,
input_dim=1,
time_embedding_dim=data_loader.tpe_position_index.shape[-1],
external_feature_dim=data_loader.external_dim,
T=int(args.T),
gcl_k=int(args.K),
gcl_layers=int(args.L),
import numpy as np
from UCTB.dataset import NodeTrafficLoader
from UCTB.model import HMM
from UCTB.evaluation import metric
data_loader = NodeTrafficLoader(dataset='Bike',
city='Chicago',
closeness_len=12,
period_len=0,
trend_len=0,
with_lm=False,
normalize=False)
prediction = []
for station_index in range(data_loader.station_number):
# train the hmm model
try:
hmm = HMM(num_components=8, n_iter=100)
hmm.fit(data_loader.train_closeness[:, station_index:station_index + 1,
-1, 0])
# predict
p = []
for time_index in range(data_loader.test_closeness.shape[0]):
p.append(
hmm.predict(data_loader.test_closeness[time_index,
station_index, :, :],
length=1))
except Exception as e:
print('Failed at station', station_index, 'with error', e)
# using zero as prediction
import nni
from UCTB.dataset import NodeTrafficLoader
from UCTB.model import HM
from UCTB.evaluation import metric
params = nni.get_next_parameter()
data_loader = NodeTrafficLoader(dataset=params['Dataset'],
city=params['City'],
with_lm=False,
normalize=False,
test_ratio=0.1)
test_start_index = data_loader.traffic_data.shape[
0] - data_loader.test_data.shape[0]
val_start_index = data_loader.traffic_data.shape[
0] - data_loader.test_data.shape[0] * 2
hm_obj = HM(c=int(params['CT']), p=int(params['PT']), t=int(params['TT']))
val_prediction = hm_obj.predict(val_start_index,
data_loader.traffic_data[:test_start_index],
time_fitness=data_loader.dataset.time_fitness)
test_prediction = hm_obj.predict(test_start_index,
data_loader.traffic_data,
time_fitness=data_loader.dataset.time_fitness)
val_rmse = metric.rmse(
import numpy as np
from UCTB.dataset import NodeTrafficLoader
from UCTB.model import XGBoost
from UCTB.evaluation import metric
data_loader = NodeTrafficLoader(dataset='Bike',
city='DC',
closeness_len=6,
period_len=7,
trend_len=4,
with_lm=False,
normalize=False)
prediction_test = []
for i in range(data_loader.station_number):
print('*************************************************************')
print('Station', i)
model = XGBoost(n_estimators=100,
max_depth=3,
objective='reg:squarederror')
model.fit(
np.concatenate((
data_loader.train_closeness[:, i, :, 0],
data_loader.train_period[:, i, :, 0],
data_loader.train_trend[:, i, :, 0],
),
parser.add_argument('--sd', default='0', type=int)
parser.add_argument('--sma', default='0', type=int)
parser.add_argument('--sp', default='0', type=int)
parser.add_argument('--DataRange', default="all")
parser.add_argument('--TrainDays', default="all")
args = vars(parser.parse_args())
data_loader = NodeTrafficLoader(
dataset=args['dataset'],
city=args['city'],
data_range=args['DataRange'],
train_data_length=args['TrainDays'],
test_ratio=0.1,
closeness_len=int(args['CT']),
period_len=0,
trend_len=0,
with_lm=False,
with_tpe=False,
normalize=False,
MergeIndex=args['MergeIndex'],
MergeWay="max" if args["dataset"] == "ChargeStation" else "sum")
train_closeness, val_closeness = SplitData.split_data(
data_loader.train_closeness, [0.9, 0.1])
train_y, val_y = SplitData.split_data(data_loader.train_y, [0.9, 0.1])
val_prediction_collector = []
test_prediction_collector = []
print('*************************************************************')
model_dir_path = os.path.join(model_dir_path, args['group'])
#####################################################################
# Config data loader
data_loader = NodeTrafficLoader(
dataset=args['dataset'],
city=args['city'],
data_range=args['data_range'],
train_data_length=args['train_data_length'],
test_ratio=0.1,
closeness_len=args['closeness_len'],
period_len=args['period_len'],
trend_len=args['trend_len'],
external_method=args['external_method'],
external_lstm_len=args['external_lstm_len'],
threshold_distance=args['threshold_distance'],
threshold_correlation=args['threshold_correlation'],
threshold_interaction=args['threshold_interaction'],
normalize=args['normalize'],
graph=args['graph'],
with_lm=True,
with_tpe=True if args['st_method'] == 'gal_gcn' else False,
workday_parser=is_work_day_america
if args['dataset'] == 'Bike' else is_work_day_china,
external_use=args['external_use'],
MergeIndex=args['MergeIndex'],
MergeWay="max" if args["dataset"] == "ChargeStation" else "sum")
# split data
train_closeness, val_closeness = SplitData.split_data(
data_loader.train_closeness, [0.9, 0.1])
from UCTB.dataset import NodeTrafficLoader
from UCTB.model import STMeta
from UCTB.evaluation import metric
from UCTB.preprocess.GraphGenerator import GraphGenerator
# Config data loader
data_loader = NodeTrafficLoader(dataset='Bike',
city='NYC',
graph='Correlation',
closeness_len=6,
period_len=7,
trend_len=4,
normalize=True)
# Build Graph
graph_obj = GraphGenerator(graph='Correlation', data_loader=data_loader)
# Init model object
STMeta_Obj = STMeta(closeness_len=data_loader.closeness_len,
period_len=data_loader.period_len,
trend_len=data_loader.trend_len,
num_node=data_loader.station_number,
num_graph=graph_obj.LM.shape[0],
external_dim=data_loader.external_dim)
# Build tf-graph
STMeta_Obj.build()
# Training
STMeta_Obj.fit(closeness_feature=data_loader.train_closeness,
period_feature=data_loader.train_period,
trend_feature=data_loader.train_trend,
laplace_matrix=graph_obj.LM,
parser.add_argument('--ar', default='6', type=int)
parser.add_argument('--d', default='0', type=int)
parser.add_argument('--ma', default='1', type=int)
parser.add_argument('--sar', default='0', type=int)
parser.add_argument('--sd', default='0', type=int)
parser.add_argument('--sma', default='0', type=int)
parser.add_argument('--sp', default='0', type=int)
parser.add_argument('--DataRange', default='All')
parser.add_argument('--TrainDays', default='365')
args = vars(parser.parse_args())
data_loader = NodeTrafficLoader(dataset=args['Dataset'], city=args['City'],
closeness_len=int(args['CT']), period_len=0, trend_len=0,
data_range=args['DataRange'], train_data_length=args['TrainDays'],
with_lm=False, with_tpe=False, normalize=False)
def task(share_queue, locker, data, parameters):
print('Child process %s with pid %s' % (parameters[0], os.getpid()))
val_collector = {}
test_collector = {}
for i in data:
print('Child process %s' % (parameters[0]),
args['Dataset'], args['City'], 'Station', i, 'total', data_loader.station_number)
import numpy as np
from sklearn.ensemble import GradientBoostingRegressor
from UCTB.dataset import NodeTrafficLoader
from UCTB.evaluation import metric
closeness_len = 6
period_len = 7
trend_len = 4
data_loader = NodeTrafficLoader(dataset='Bike',
city='NYC',
closeness_len=closeness_len,
period_len=period_len,
trend_len=trend_len,
with_lm=False,
normalize=False)
prediction = []
for i in range(data_loader.station_number):
print('*************************************************************')
print('Station', i)
model = GradientBoostingRegressor(n_estimators=100, max_depth=3)
X_Train = []
X_Test = []
if closeness_len > 0:
parser.add_argument('--num_components', type=int, default=8)
parser.add_argument('--n_iter', type=int, default=365)
args = vars(parser.parse_args())
nni_params = nni.get_next_parameter()
nni_sid = nni.get_sequence_id()
if nni_params:
args.update(nni_params)
args['CodeVersion'] += str(nni_sid)
data_loader = NodeTrafficLoader(dataset=args['Dataset'],
city=args['City'],
closeness_len=args['CT'],
period_len=args['PT'],
trend_len=args['TT'],
with_lm=False,
with_tpe=False,
normalize=False)
model = HMM(num_components=args['num_components'], n_iter=args['n_iter'])
train_closeness, val_closeness = SplitData.split_data(
data_loader.train_closeness, [0.9, 0.1])
train_period, val_period = SplitData.split_data(data_loader.train_period,
[0.9, 0.1])
train_trend, val_trend = SplitData.split_data(data_loader.train_trend,
[0.9, 0.1])
train_label, val_label = SplitData.split_data(data_loader.train_y, [0.9, 0.1])
if len(deviceIDs) == 0:
current_device = '-1'
else:
if nni_params:
current_device = str(deviceIDs[int(nni_sid) % len(deviceIDs)])
else:
current_device = str(deviceIDs[0])
# Config data loader
data_loader = NodeTrafficLoader(dataset=args['Dataset'],
city=args['City'],
test_ratio=float(args['test_ratio']),
data_range=args['DataRange'],
train_data_length=args['TrainDays'],
closeness_len=int(args['CT']),
period_len=int(args['PT']),
trend_len=int(args['TT']),
normalize=True,
MergeIndex=args['MergeIndex'],
MergeWay=args["MergeWay"])
# build graphs
graph_obj = GraphGenerator(graph=args['Graph'],
data_loader=data_loader,
threshold_distance=args['TD'],
threshold_correlation=args['TC'],
threshold_interaction=args['TI'])
ST_MGCN_Obj = ST_MGCN(T=int(args['CT']) + int(args['PT']) + int(args['TT']),
input_dim=1,
import numpy as np
from UCTB.model import ARIMA
from UCTB.dataset import NodeTrafficLoader
from UCTB.evaluation import metric
data_loader = NodeTrafficLoader(dataset='ChargeStation', city='Beijing')
prediction = []
for i in range(data_loader.station_number):
print('*************************************************************')
print('Station', i)
try:
model_obj = ARIMA(data_loader.train_data[:, i], [30, 0, 2])
p = model_obj.predict(data_loader.test_x[:, :, i, 0])
except Exception as e:
print('Converge failed with error', e)
print('Using zero as prediction')
p = np.zeros([data_loader.test_x[:, :, i, 0].shape[0], 1, 1])
prediction.append(p)
print(np.concatenate(prediction, axis=-1).shape)
prediction = np.concatenate(prediction, axis=-1)
print('RMSE', metric.rmse(prediction, data_loader.test_y, threshold=0))