def get_prediction_real_time(sparkEngine, model=None, url_weight="", dim=15, prediction_weight="", encoder_length=24, decoder_length=24, attention_length=24, is_close_cuda=True):
# continuously crawl aws and aqi & weather
end = utils.get_datetime_now()
end = end - timedelta(hours=1)
# end = datetime.strptime("2018-06-19 11:01:00", p.fm)
# e_ = end.strftime(p.fm)
start = end - timedelta(hours=23)
start = start.replace(minute=0, second=0, microsecond=0)
# s_ = start.strftime(p.fm)
# 2. process normalize data
vectors, w_pred, china_vectors, timestamp = sparkEngine.process_vectors(start, end, dim)
v_l = len(vectors)
if v_l:
sp_vectors = psv.convert_data_to_grid_exe(vectors)
if v_l < encoder_length:
sp_vectors = np.pad(sp_vectors, ((encoder_length - v_l,0), (0,0), (0,0), (0, 0)), 'constant', constant_values=0)
# repeat for 25 districts
if w_pred:
w_pred = np.repeat(np.expand_dims(w_pred, 1), p.grid_size, 1)
de_vectors = psv.convert_data_to_grid_exe(w_pred)
# pad to fill top elements of decoder vectors
de_vectors = np.pad(de_vectors, ((0, 0), (0, 0), (0, 0), (6, 0)), 'constant', constant_values=0)
else:
# know nothing about future weather forecast
de_vectors = np.zeros((decoder_length, p.grid_size, p.grid_size, dim))
sp_vectors = np.concatenate((sp_vectors, de_vectors), axis=0)
c_l = len(china_vectors)
if c_l < attention_length:
# print(attention_length - c_l)
china_vectors = np.pad(china_vectors, ((attention_length - c_l, 0), (0, 0)), 'constant', constant_values=0)
# 4. Feed to model
if model is None:
# model = BaselineModel(encoder_length=encoder_length, encode_vector_size=12, batch_size=1, decoder_length=decoder_length, rnn_layers=1,
# dtype='grid', grid_size=25, use_cnn=True)
# model.set_data(sp_vectors, [0], None)
# model = MaskGan(encoder_length=encoder_length, encode_vector_size=15, batch_size=1, decode_vector_size=9, grid_size=25, use_cnn=True)
model = APGan(encoder_length=24, decoder_length=24, encode_vector_size=15, batch_size=1, decode_vector_size=9, grid_size=25, forecast_factor=0)
# model = APNet(encoder_length=24, decoder_length=24, encode_vector_size=15, batch_size=1, decode_vector_size=9, grid_size=25, forecast_factor=0)
model.set_data(sp_vectors, [0], None, china_vectors)
with tf.device('/%s' % p.device):
model.init_ops(is_train=False)
saver = tf.train.Saver()
tconfig = get_gpu_options(False)
with tf.Session(config=tconfig) as session:
model.assign_datasets(session)
preds_pm25 = realtime_execute(model, session, saver, decoder_length, p.prediction_weight_pm25)
model.forecast_factor = 1
preds_pm10 = realtime_execute(model, session, saver, decoder_length, p.prediction_weight_pm10)
china_vectors = np.array(china_vectors)
# print("china", china_vectors.shape)
# tf.reset_default_graph()
# session.close()
if is_close_cuda:
cuda.select_device(0)
cuda.close()
return (preds_pm25, preds_pm10), timestamp, np.transpose(china_vectors[:,:2] * 500)
else:
return ([],[]), [], []
def get_gan_model(model_name,
encoder_length,
embed_size,
batch_size,
decoder_size,
decoder_length,
grid_size,
is_test,
forecast_factor,
use_attention=True):
if model_name == "APGAN":
model = APGan(encoder_length=encoder_length,
encode_vector_size=embed_size,
batch_size=batch_size,
decode_vector_size=decoder_size,
decoder_length=decoder_length,
grid_size=grid_size,
forecast_factor=forecast_factor,
use_attention=use_attention)
elif model_name == "MASKGAN":
model = MaskGan(encoder_length=encoder_length,
encode_vector_size=embed_size,
batch_size=batch_size,
decode_vector_size=decoder_size,
grid_size=grid_size,
use_cnn=1,
forecast_factor=forecast_factor,
use_attention=use_attention)
elif model_name == "APGAN_LSTM":
model = APGAN_LSTM(encoder_length=encoder_length,
encode_vector_size=embed_size,
batch_size=batch_size,
decode_vector_size=decoder_size,
decoder_length=decoder_length,
grid_size=grid_size,
forecast_factor=forecast_factor,
use_attention=use_attention)
elif model_name == "CAPGAN":
model = CAPGan(encoder_length=encoder_length,
encode_vector_size=embed_size,
batch_size=batch_size,
decode_vector_size=decoder_size,
grid_size=grid_size,
forecast_factor=forecast_factor,
use_attention=use_attention)
elif model_name == "TGAN":
model = TGAN(encoder_length=8, decoder_length=8, grid_size=32)
else:
model = TGANLSTM(encoder_length=8, decoder_length=8, grid_size=32)
with tf.device('/%s' % p.device):
model.init_ops(not is_test)
return model
def train_gan(url_feature="",
attention_url="",
url_weight="sp",
batch_size=128,
encoder_length=24,
embed_size=None,
decoder_length=24,
decoder_size=4,
grid_size=25,
is_folder=False,
is_test=False,
restore=False,
model_name="APGAN",
forecast_factor=0):
if model_name == "APGAN":
model = APGan(encoder_length=encoder_length,
encode_vector_size=embed_size,
batch_size=batch_size,
decode_vector_size=decoder_size,
decoder_length=decoder_length,
grid_size=grid_size,
forecast_factor=forecast_factor,
use_attention=bool(attention_url))
elif model_name == "MASKGAN":
model = MaskGan(encoder_length=encoder_length,
encode_vector_size=embed_size,
batch_size=batch_size,
decode_vector_size=decoder_size,
grid_size=grid_size,
use_cnn=1)
elif model_name == "APGAN_LSTM":
model = APGAN_LSTM(encoder_length=encoder_length,
encode_vector_size=embed_size,
batch_size=batch_size,
decode_vector_size=decoder_size,
decoder_length=decoder_length,
grid_size=grid_size)
elif model_name == "CAPGAN":
model = CAPGan(encoder_length=encoder_length,
encode_vector_size=embed_size,
batch_size=batch_size,
decode_vector_size=decoder_size,
grid_size=grid_size)
elif model_name == "TGAN":
model = TGAN(encoder_length=8, decoder_length=8, grid_size=32)
else:
model = TGANLSTM(encoder_length=8, decoder_length=8, grid_size=32)
tconfig = get_gpu_options()
utils.assert_url(url_feature)
if not utils.check_file('summaries'):
os.makedirs('summaries')
print('==> initializing models')
with tf.device('/%s' % p.device):
model.init_ops(not is_test)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
train_writer = None
with tf.Session(config=tconfig) as session:
if not restore:
session.run(init)
else:
print("==> Reload pre-trained weights")
saver.restore(session, url_weight)
csn = int(time.time())
if not is_test:
url_weight = url_weight.split("/")[-1]
url_weight = url_weight.rstrip(".weights")
train_writer = tf.summary.FileWriter(
"summaries/%s_%i" % (url_weight, csn), session.graph)
folders = None
if is_folder:
folders = os.listdir(url_feature)
folders = sorted(folders)
if attention_url:
a_folders = sorted(os.listdir(attention_url))
folders = zip(folders, a_folders)
for i, files in enumerate(folders):
if attention_url:
x, y = files
att_url = os.path.join(attention_url, y)
print("==> Training set (%i, %s, %s)" % (i + 1, x, y))
else:
att_url = None
x = files
print("==> Training set (%i, %s)" % (i + 1, x))
execute_gan(os.path.join(url_feature, x), att_url, url_weight,
model, session, saver, batch_size, encoder_length,
decoder_length, is_test, train_writer,
i * p.total_iteration)
else:
execute_gan(url_feature, attention_url, url_weight, model, session,
saver, batch_size, encoder_length, decoder_length,
is_test, train_writer)