Beispiel #1
0
def get_prediction_real_time(sparkEngine, model=None, url_weight="", dim=15, prediction_weight="", encoder_length=24, decoder_length=24, attention_length=24, is_close_cuda=True):
    # continuously crawl aws and aqi & weather
    end = utils.get_datetime_now()
    end = end - timedelta(hours=1)
    # end = datetime.strptime("2018-06-19 11:01:00", p.fm)
    # e_ = end.strftime(p.fm)
    start = end - timedelta(hours=23)
    start = start.replace(minute=0, second=0, microsecond=0)
    # s_ = start.strftime(p.fm)
    # 2. process normalize data
    vectors, w_pred, china_vectors, timestamp = sparkEngine.process_vectors(start, end, dim)
    v_l = len(vectors)
    if v_l:
        sp_vectors = psv.convert_data_to_grid_exe(vectors)
        if v_l < encoder_length:
            sp_vectors = np.pad(sp_vectors, ((encoder_length - v_l,0), (0,0), (0,0), (0, 0)), 'constant', constant_values=0)
        # repeat for 25 districts
        if w_pred:
            w_pred = np.repeat(np.expand_dims(w_pred, 1), p.grid_size, 1)
            de_vectors = psv.convert_data_to_grid_exe(w_pred)
            # pad to fill top elements of decoder vectors
            de_vectors = np.pad(de_vectors, ((0, 0), (0, 0), (0, 0), (6, 0)), 'constant', constant_values=0)
        else:
            # know nothing about future weather forecast
            de_vectors = np.zeros((decoder_length, p.grid_size, p.grid_size, dim))
        sp_vectors = np.concatenate((sp_vectors, de_vectors), axis=0)

        c_l = len(china_vectors)
        if c_l < attention_length:
            # print(attention_length - c_l)
            china_vectors = np.pad(china_vectors, ((attention_length - c_l, 0), (0, 0)), 'constant', constant_values=0)

        # 4. Feed to model
        if model is None:
            # model = BaselineModel(encoder_length=encoder_length, encode_vector_size=12, batch_size=1, decoder_length=decoder_length, rnn_layers=1,
            #                 dtype='grid', grid_size=25, use_cnn=True)
            # model.set_data(sp_vectors, [0], None)
            # model = MaskGan(encoder_length=encoder_length, encode_vector_size=15, batch_size=1, decode_vector_size=9, grid_size=25, use_cnn=True)
            model = APGan(encoder_length=24, decoder_length=24, encode_vector_size=15, batch_size=1, decode_vector_size=9, grid_size=25, forecast_factor=0)
            # model = APNet(encoder_length=24, decoder_length=24, encode_vector_size=15, batch_size=1, decode_vector_size=9, grid_size=25, forecast_factor=0)
        model.set_data(sp_vectors, [0], None, china_vectors)
        with tf.device('/%s' % p.device):
            model.init_ops(is_train=False)
            saver = tf.train.Saver()
        tconfig = get_gpu_options(False)        
        with tf.Session(config=tconfig) as session:
            model.assign_datasets(session)    
            preds_pm25 = realtime_execute(model, session, saver, decoder_length, p.prediction_weight_pm25)
            model.forecast_factor = 1
            preds_pm10 = realtime_execute(model, session, saver, decoder_length, p.prediction_weight_pm10)
            china_vectors = np.array(china_vectors)
            # print("china", china_vectors.shape)
            # tf.reset_default_graph()
            # session.close()
            if is_close_cuda:
                cuda.select_device(0)
                cuda.close()
        return (preds_pm25, preds_pm10), timestamp, np.transpose(china_vectors[:,:2] * 500)
    else:
        return ([],[]), [], []
Beispiel #2
0
def get_gan_model(model_name,
                  encoder_length,
                  embed_size,
                  batch_size,
                  decoder_size,
                  decoder_length,
                  grid_size,
                  is_test,
                  forecast_factor,
                  use_attention=True):
    if model_name == "APGAN":
        model = APGan(encoder_length=encoder_length,
                      encode_vector_size=embed_size,
                      batch_size=batch_size,
                      decode_vector_size=decoder_size,
                      decoder_length=decoder_length,
                      grid_size=grid_size,
                      forecast_factor=forecast_factor,
                      use_attention=use_attention)
    elif model_name == "MASKGAN":
        model = MaskGan(encoder_length=encoder_length,
                        encode_vector_size=embed_size,
                        batch_size=batch_size,
                        decode_vector_size=decoder_size,
                        grid_size=grid_size,
                        use_cnn=1,
                        forecast_factor=forecast_factor,
                        use_attention=use_attention)
    elif model_name == "APGAN_LSTM":
        model = APGAN_LSTM(encoder_length=encoder_length,
                           encode_vector_size=embed_size,
                           batch_size=batch_size,
                           decode_vector_size=decoder_size,
                           decoder_length=decoder_length,
                           grid_size=grid_size,
                           forecast_factor=forecast_factor,
                           use_attention=use_attention)
    elif model_name == "CAPGAN":
        model = CAPGan(encoder_length=encoder_length,
                       encode_vector_size=embed_size,
                       batch_size=batch_size,
                       decode_vector_size=decoder_size,
                       grid_size=grid_size,
                       forecast_factor=forecast_factor,
                       use_attention=use_attention)
    elif model_name == "TGAN":
        model = TGAN(encoder_length=8, decoder_length=8, grid_size=32)
    else:
        model = TGANLSTM(encoder_length=8, decoder_length=8, grid_size=32)
    with tf.device('/%s' % p.device):
        model.init_ops(not is_test)
    return model
Beispiel #3
0
def train_gan(url_feature="",
              attention_url="",
              url_weight="sp",
              batch_size=128,
              encoder_length=24,
              embed_size=None,
              decoder_length=24,
              decoder_size=4,
              grid_size=25,
              is_folder=False,
              is_test=False,
              restore=False,
              model_name="APGAN",
              forecast_factor=0):
    if model_name == "APGAN":
        model = APGan(encoder_length=encoder_length,
                      encode_vector_size=embed_size,
                      batch_size=batch_size,
                      decode_vector_size=decoder_size,
                      decoder_length=decoder_length,
                      grid_size=grid_size,
                      forecast_factor=forecast_factor,
                      use_attention=bool(attention_url))
    elif model_name == "MASKGAN":
        model = MaskGan(encoder_length=encoder_length,
                        encode_vector_size=embed_size,
                        batch_size=batch_size,
                        decode_vector_size=decoder_size,
                        grid_size=grid_size,
                        use_cnn=1)
    elif model_name == "APGAN_LSTM":
        model = APGAN_LSTM(encoder_length=encoder_length,
                           encode_vector_size=embed_size,
                           batch_size=batch_size,
                           decode_vector_size=decoder_size,
                           decoder_length=decoder_length,
                           grid_size=grid_size)
    elif model_name == "CAPGAN":
        model = CAPGan(encoder_length=encoder_length,
                       encode_vector_size=embed_size,
                       batch_size=batch_size,
                       decode_vector_size=decoder_size,
                       grid_size=grid_size)
    elif model_name == "TGAN":
        model = TGAN(encoder_length=8, decoder_length=8, grid_size=32)
    else:
        model = TGANLSTM(encoder_length=8, decoder_length=8, grid_size=32)
    tconfig = get_gpu_options()
    utils.assert_url(url_feature)
    if not utils.check_file('summaries'):
        os.makedirs('summaries')
    print('==> initializing models')
    with tf.device('/%s' % p.device):
        model.init_ops(not is_test)
        init = tf.global_variables_initializer()
        saver = tf.train.Saver()
    train_writer = None
    with tf.Session(config=tconfig) as session:
        if not restore:
            session.run(init)
        else:
            print("==> Reload pre-trained weights")
            saver.restore(session, url_weight)
        csn = int(time.time())
        if not is_test:
            url_weight = url_weight.split("/")[-1]
            url_weight = url_weight.rstrip(".weights")
            train_writer = tf.summary.FileWriter(
                "summaries/%s_%i" % (url_weight, csn), session.graph)
        folders = None
        if is_folder:
            folders = os.listdir(url_feature)
            folders = sorted(folders)
            if attention_url:
                a_folders = sorted(os.listdir(attention_url))
                folders = zip(folders, a_folders)
            for i, files in enumerate(folders):
                if attention_url:
                    x, y = files
                    att_url = os.path.join(attention_url, y)
                    print("==> Training set (%i, %s, %s)" % (i + 1, x, y))
                else:
                    att_url = None
                    x = files
                    print("==> Training set (%i, %s)" % (i + 1, x))
                execute_gan(os.path.join(url_feature, x), att_url, url_weight,
                            model, session, saver, batch_size, encoder_length,
                            decoder_length, is_test, train_writer,
                            i * p.total_iteration)
        else:
            execute_gan(url_feature, attention_url, url_weight, model, session,
                        saver, batch_size, encoder_length, decoder_length,
                        is_test, train_writer)