def test_cities(cities, repeat=1):
beijing_only = True
for city in cities:
for i in range(repeat):
train_data, valid_data = load_data3(stations=city2stations[city],
starttime=starttime,
endtime=endtime,
filter=(not beijing_only),
normalize_target=False)
X_train, y_train, X_valid, y_valid = build_lstm_dataset(
train_data, valid_data, pred_range=pred_range, hist_len=3)
print 'X_train[0].shape =', X_train[0].shape
name = city
rlstm = build_rlstm(X_train[0].shape[-1],
h0_dim=20,
h1_dim=20,
rec_layer_init='zero',
fix_b_f=True,
base_name=name,
add_input_noise=beijing_only,
add_target_noise=False)
rlstm.name = name + str(i)
rlstm.data = [train_data, valid_data]
rlstm.X_mask = np.ones((X_train[0].shape[-1], ), dtype='int')
rlstm.X_mask[-1:] = not beijing_only # pm25 mean
print '\ntraining', rlstm.name
X_train[0], X_valid[0] = normalize(X_train[0], X_valid[0], rlstm)
rlstm.save_normalization_info(name + '_norm_info.pkl')
batch_size = (1 + (not beijing_only)) * 64
# patience = (1 + int(beijing_only)) * 10
patience = 10
train(X_train,
y_train,
X_valid,
y_valid,
rlstm,
batch_size=batch_size,
patience=patience,
nb_epoch=300)
for city in cities:
for i in range(repeat):
name = city
rlstm = model_from_yaml(
open(model_savedir + name + '.yaml').read())
rlstm.name = name + str(i)
rlstm.load_normalization_info(model_savedir + name +
'_norm_info.pkl')
rlstm.load_weights(model_savedir + rlstm.name + '_weights.hdf5')
train_data, valid_data = load_data3(stations=city2stations[city],
starttime=starttime,
endtime=endtime)
rlstm.data = [train_data, valid_data]
print '\n' + rlstm.name
test_model(rlstm, dataset='train', show_details=False)
test_model(rlstm, dataset='valid', show_details=True)
def train_model(name, is_city=False, latest=True):
if is_city:
if name == 'beijing':
train_data, valid_data, train_data2 = load_data3(stations=city2stations[name],
starttime='20150901', endtime='20160123',
train_stop=630, valid_start=680,
latest=True,
filter=False)
else:
train_data, valid_data, train_data2 = load_data3(stations=city2stations[name],
latest=latest,
filter=False)
else:
train_data, valid_data, train_data2 = load_data3(lon_range=area2lonlat[name][0], lat_range=area2lonlat[name][1],
latest=latest,
filter=True)
X_train, y_train, X_valid, y_valid = build_lstm_dataset(train_data, valid_data, pred_range=pred_range, hist_len=3)
print 'X_train[0].shape =', X_train[0].shape
rlstm = build_rlstm(X_train[0].shape[-1], h0_dim=20, h1_dim=20,
rec_layer_init='zero', fix_b_f=is_city, base_name=name,
add_input_noise=is_city, add_target_noise=False)
rlstm.name = name + '_valid'
rlstm.data = [train_data, valid_data]
rlstm.X_mask = np.ones((X_train[0].shape[-1],), dtype='int')
rlstm.X_mask[-1:] = not is_city # pm25 mean
print '\ntraining', rlstm.name
X_train[0], X_valid[0] = normalize(X_train[0], X_valid[0], rlstm)
rlstm.save_normalization_info(model_savedir + rlstm.name + '_norm_info.pkl')
batch_size = (1 + int(not is_city)) * 64
patience = 10 + int(is_city) * 10
train(X_train, y_train, X_valid, y_valid, rlstm, batch_size=batch_size, patience=patience, nb_epoch=300)
result_str = get_train_result(rlstm)
print 'result_str =', result_str
with open(model_savedir + name + '.log', 'a') as f:
f.write(result_str + '\n')
epoch = estimate_early_stop_epoch(name)
X_train, y_train, X_valid, y_valid = build_lstm_dataset(train_data2, valid_data, pred_range=pred_range, hist_len=3)
print 'X_train[0].shape =', X_train[0].shape
rlstm = build_rlstm(X_train[0].shape[-1], h0_dim=20, h1_dim=20,
rec_layer_init='zero', fix_b_f=is_city, base_name=name,
add_input_noise=is_city, add_target_noise=False)
rlstm.name = name
rlstm.data = [train_data, valid_data]
rlstm.X_mask = np.ones((X_train[0].shape[-1],), dtype='int')
rlstm.X_mask[-1:] = not is_city # pm25 mean
print '\ntraining', rlstm.name
X_train[0], X_valid[0] = normalize(X_train[0], X_valid[0], rlstm)
rlstm.save_normalization_info(model_savedir + rlstm.name + '_norm_info.pkl')
batch_size = (1 + int(not is_city)) * 64
patience = 10 + int(is_city) * 10
train(X_train, y_train, X_valid, y_valid, rlstm, batch_size=batch_size, patience=patience, nb_epoch=epoch)
def test_areas(areas, repeat=1):
beijing_only = False
for area in areas:
for i in range(repeat):
train_data, valid_data = load_data3(
lon_range=area2lonlat[area][0],
lat_range=area2lonlat[area][1],
starttime=starttime,
endtime=endtime,
filter=(not beijing_only),
)
X_train, y_train, X_valid, y_valid = build_lstm_dataset(
train_data, valid_data, pred_range=pred_range, hist_len=3
)
print "X_train[0].shape =", X_train[0].shape
name = area
rlstm = build_rlstm(
X_train[0].shape[-1],
h0_dim=20,
h1_dim=20,
rec_layer_init="zero",
fix_b_f=False,
base_name=name,
add_input_noise=beijing_only,
add_target_noise=False,
)
rlstm.name = name + str(i)
rlstm.data = [train_data, valid_data]
rlstm.X_mask = np.ones((X_train[0].shape[-1],), dtype="int")
rlstm.X_mask[-1:] = not beijing_only # pm25 mean
print "\ntraining", rlstm.name
X_train[0], X_valid[0] = normalize(X_train[0], X_valid[0], rlstm)
rlstm.save_normalization_info(name + "_norm_info.pkl")
batch_size = (1 + (not beijing_only)) * 64
# patience = (1 + int(beijing_only)) * 10
patience = 10
train(X_train, y_train, X_valid, y_valid, rlstm, batch_size=batch_size, patience=patience, nb_epoch=300)
for area in areas:
for i in range(repeat):
name = area
rlstm = model_from_yaml(open(name + ".yaml").read())
rlstm.name = name + str(i)
rlstm.load_normalization_info(name + "_norm_info.pkl")
rlstm.load_weights(rlstm.name + "_weights.hdf5")
train_data, valid_data = load_data3(
lon_range=area2lonlat[area][0], lat_range=area2lonlat[area][1], starttime=starttime, endtime=endtime
)
rlstm.data = [train_data, valid_data]
print "\n" + rlstm.name
test_model(rlstm, dataset="train", show_details=False)
test_model(rlstm, dataset="valid", show_details=True)
def test_cities(cities, repeat=1):
beijing_only = True
for city in cities:
for i in range(repeat):
train_data, valid_data = load_data3(stations=city2stations[city],
starttime=starttime, endtime=endtime,
filter=(not beijing_only), normalize_target=False)
X_train, y_train, X_valid, y_valid = build_lstm_dataset(train_data, valid_data, pred_range=pred_range, hist_len=3)
print 'X_train[0].shape =', X_train[0].shape
name = city
rlstm = build_rlstm(X_train[0].shape[-1], h0_dim=20, h1_dim=20,
rec_layer_init='zero', fix_b_f=True, base_name=name,
add_input_noise=beijing_only, add_target_noise=False)
rlstm.name = name + str(i)
rlstm.data = [train_data, valid_data]
rlstm.X_mask = np.ones((X_train[0].shape[-1],), dtype='int')
rlstm.X_mask[-1:] = not beijing_only # pm25 mean
print '\ntraining', rlstm.name
X_train[0], X_valid[0] = normalize(X_train[0], X_valid[0], rlstm)
rlstm.save_normalization_info(name + '_norm_info.pkl')
batch_size = (1 + (not beijing_only)) * 64
# patience = (1 + int(beijing_only)) * 10
patience = 10
train(X_train, y_train, X_valid, y_valid, rlstm, batch_size=batch_size, patience=patience, nb_epoch=300)
for city in cities:
for i in range(repeat):
name = city
rlstm = model_from_yaml(open(model_savedir + name + '.yaml').read())
rlstm.name = name + str(i)
rlstm.load_normalization_info(model_savedir + name + '_norm_info.pkl')
rlstm.load_weights(model_savedir + rlstm.name + '_weights.hdf5')
train_data, valid_data = load_data3(stations=city2stations[city],
starttime=starttime, endtime=endtime)
rlstm.data = [train_data, valid_data]
print '\n' + rlstm.name
test_model(rlstm, dataset='train', show_details=False)
test_model(rlstm, dataset='valid', show_details=True)
def test_area_on_cities(area, i, cities):
name = area
rlstm = model_from_yaml(open(name + ".yaml").read())
rlstm.name = name + str(i)
rlstm.load_normalization_info(name + "_norm_info.pkl")
rlstm.load_weights(rlstm.name + "_weights.hdf5")
print "\n In" + area
for city in cities:
train_data, valid_data = load_data3(stations=city2stations[city], starttime=starttime, endtime=endtime)
rlstm.data = [train_data, valid_data]
print "\n" + city
test_model(rlstm, dataset="train", show_details=False)
test_model(rlstm, dataset="valid", show_details=True)
def test_area_on_cities(area, i, cities):
name = area
rlstm = model_from_yaml(open(model_savedir + name + '.yaml').read())
rlstm.name = name + str(i)
rlstm.load_normalization_info(model_savedir + name + '_norm_info.pkl')
rlstm.load_weights(model_savedir + rlstm.name + '_weights.hdf5')
print '\n In' + area
for city in cities:
train_data, valid_data = load_data3(stations=city2stations[city],
starttime=starttime, endtime=endtime,
)
rlstm.data = [train_data, valid_data]
print '\n' + city
test_model(rlstm, dataset='train', show_details=False)
test_model(rlstm, dataset='valid', show_details=True)
def test_area_on_cities(area, i, cities):
name = area
rlstm = model_from_yaml(open(model_savedir + name + '.yaml').read())
rlstm.name = name + str(i)
rlstm.load_normalization_info(model_savedir + name + '_norm_info.pkl')
rlstm.load_weights(model_savedir + rlstm.name + '_weights.hdf5')
print '\n In' + area
for city in cities:
train_data, valid_data = load_data3(
stations=city2stations[city],
starttime=starttime,
endtime=endtime,
)
rlstm.data = [train_data, valid_data]
print '\n' + city
test_model(rlstm, dataset='train', show_details=False)
test_model(rlstm, dataset='valid', show_details=True)
def train_model(name, is_city=False, latest=True):
if is_city:
if name == 'beijing':
train_data, valid_data, train_data2 = load_data3(
stations=city2stations[name],
starttime='20150901',
endtime='20160123',
train_stop=630,
valid_start=680,
latest=True,
filter=False)
else:
train_data, valid_data, train_data2 = load_data3(
stations=city2stations[name], latest=latest, filter=False)
else:
train_data, valid_data, train_data2 = load_data3(
lon_range=area2lonlat[name][0],
lat_range=area2lonlat[name][1],
latest=latest,
filter=True)
X_train, y_train, X_valid, y_valid = build_lstm_dataset(
train_data, valid_data, pred_range=pred_range, hist_len=3)
print 'X_train[0].shape =', X_train[0].shape
rlstm = build_rlstm(X_train[0].shape[-1],
h0_dim=20,
h1_dim=20,
rec_layer_init='zero',
fix_b_f=is_city,
base_name=name,
add_input_noise=is_city,
add_target_noise=False)
rlstm.name = name + '_valid'
rlstm.data = [train_data, valid_data]
rlstm.X_mask = np.ones((X_train[0].shape[-1], ), dtype='int')
rlstm.X_mask[-1:] = not is_city # pm25 mean
print '\ntraining', rlstm.name
X_train[0], X_valid[0] = normalize(X_train[0], X_valid[0], rlstm)
rlstm.save_normalization_info(model_savedir + rlstm.name +
'_norm_info.pkl')
batch_size = (1 + int(not is_city)) * 64
patience = 10 + int(is_city) * 10
train(X_train,
y_train,
X_valid,
y_valid,
rlstm,
batch_size=batch_size,
patience=patience,
nb_epoch=300)
result_str = get_train_result(rlstm)
print 'result_str =', result_str
with open(model_savedir + name + '.log', 'a') as f:
f.write(result_str + '\n')
epoch = estimate_early_stop_epoch(name)
X_train, y_train, X_valid, y_valid = build_lstm_dataset(
train_data2, valid_data, pred_range=pred_range, hist_len=3)
print 'X_train[0].shape =', X_train[0].shape
rlstm = build_rlstm(X_train[0].shape[-1],
h0_dim=20,
h1_dim=20,
rec_layer_init='zero',
fix_b_f=is_city,
base_name=name,
add_input_noise=is_city,
add_target_noise=False)
rlstm.name = name
rlstm.data = [train_data, valid_data]
rlstm.X_mask = np.ones((X_train[0].shape[-1], ), dtype='int')
rlstm.X_mask[-1:] = not is_city # pm25 mean
print '\ntraining', rlstm.name
X_train[0], X_valid[0] = normalize(X_train[0], X_valid[0], rlstm)
rlstm.save_normalization_info(model_savedir + rlstm.name +
'_norm_info.pkl')
batch_size = (1 + int(not is_city)) * 64
patience = 10 + int(is_city) * 10
train(X_train,
y_train,
X_valid,
y_valid,
rlstm,
batch_size=batch_size,
patience=patience,
nb_epoch=epoch)
