Exemple #1
0
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep,
                        start_time, end_time):
    """
    Update rainfall observations for flo2d models
    :param flo2d_model: flo2d model
    :param method: value interpolation method
    :param grid_interpolation: grid interpolation method
    :param timestep: output timeseries timestep
    :return:
    """

    obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')

    try:

        # Connect to the database
        curw_obs_pool = get_Pool(host=con_params.CURW_OBS_HOST,
                                 user=con_params.CURW_OBS_USERNAME,
                                 password=con_params.CURW_OBS_PASSWORD,
                                 port=con_params.CURW_OBS_PORT,
                                 db=con_params.CURW_OBS_DATABASE)

        curw_obs_connection = curw_obs_pool.connection()

        curw_sim_pool = get_Pool(host=con_params.CURW_SIM_HOST,
                                 user=con_params.CURW_SIM_USERNAME,
                                 password=con_params.CURW_SIM_PASSWORD,
                                 port=con_params.CURW_SIM_PORT,
                                 db=con_params.CURW_SIM_DATABASE)

        TS = Sim_Timeseries(pool=curw_sim_pool)

        # [hash_id, station_id, station_name, latitude, longitude]
        # active_obs_stations = read_csv(os.path.join(ROOT_DIR,'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'))
        active_obs_stations = extract_active_curw_obs_rainfall_stations(
            start_time=start_time, end_time=end_time)[1:]
        flo2d_grids = read_csv(
            os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format(
                flo2d_model)))  # [Grid_ ID, X(longitude), Y(latitude)]

        stations_dict_for_obs = {}  # keys: obs station id , value: hash id

        for obs_index in range(len(active_obs_stations)):
            stations_dict_for_obs[active_obs_stations[obs_index]
                                  [1]] = active_obs_stations[obs_index][0]

        # flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model)
        flo2d_obs_mapping = find_nearest_obs_stations_for_flo2d_stations(
            flo2d_stations_csv=os.path.join(
                ROOT_DIR, 'grids/flo2d/{}m.csv'.format(flo2d_model)),
            obs_stations=active_obs_stations,
            flo2d_model=flo2d_model)

        for flo2d_index in range(len(flo2d_grids)):
            lat = flo2d_grids[flo2d_index][2]
            lon = flo2d_grids[flo2d_index][1]
            cell_id = flo2d_grids[flo2d_index][0]
            meta_data = {
                'latitude':
                float('%.6f' % float(lat)),
                'longitude':
                float('%.6f' % float(lon)),
                'model':
                flo2d_model,
                'method':
                method,
                'grid_id':
                '{}_{}_{}'.format(flo2d_model, grid_interpolation,
                                  (str(cell_id)).zfill(10))
            }

            tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
                                          method=meta_data.get('method'))

            if tms_id is None:
                tms_id = TS.generate_timeseries_id(meta_data=meta_data)
                meta_data['id'] = tms_id
                TS.insert_run(meta_data=meta_data)

            print("grid_id:", cell_id)
            print("grid map:", flo2d_obs_mapping.get(cell_id))
            obs1_station_id = flo2d_obs_mapping.get(cell_id)[0]
            obs2_station_id = flo2d_obs_mapping.get(cell_id)[1]
            obs3_station_id = flo2d_obs_mapping.get(cell_id)[2]

            obs_timeseries = []

            if timestep == 5:
                if obs1_station_id != str(-1):
                    obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)

                    ts = extract_obs_rain_5_min_ts(
                        connection=curw_obs_connection,
                        start_time=obs_start,
                        id=obs1_hash_id,
                        end_time=end_time)

                    if ts is not None and len(ts) > 1:
                        obs_timeseries.extend(
                            process_5_min_ts(newly_extracted_timeseries=ts,
                                             expected_start=obs_start)[1:])
                        # obs_start = ts[-1][0]

                    if obs2_station_id != str(-1):
                        obs2_hash_id = stations_dict_for_obs.get(
                            obs2_station_id)

                        ts2 = extract_obs_rain_5_min_ts(
                            connection=curw_obs_connection,
                            start_time=obs_start,
                            id=obs2_hash_id,
                            end_time=end_time)
                        if ts2 is not None and len(ts2) > 1:
                            obs_timeseries = fill_missing_values(
                                newly_extracted_timeseries=ts2,
                                OBS_TS=obs_timeseries)
                            if obs_timeseries is not None and len(
                                    obs_timeseries) > 0:
                                expected_start = obs_timeseries[-1][0]
                            else:
                                expected_start = obs_start
                            obs_timeseries.extend(
                                process_5_min_ts(
                                    newly_extracted_timeseries=ts2,
                                    expected_start=expected_start)[1:])
                            # obs_start = ts2[-1][0]

                        if obs3_station_id != str(-1):
                            obs3_hash_id = stations_dict_for_obs.get(
                                obs3_station_id)

                            ts3 = extract_obs_rain_5_min_ts(
                                connection=curw_obs_connection,
                                start_time=obs_start,
                                id=obs3_hash_id,
                                end_time=end_time)
                            if ts3 is not None and len(ts3) > 1 and len(
                                    obs_timeseries) > 0:
                                obs_timeseries = fill_missing_values(
                                    newly_extracted_timeseries=ts3,
                                    OBS_TS=obs_timeseries)
                                if obs_timeseries is not None:
                                    expected_start = obs_timeseries[-1][0]
                                else:
                                    expected_start = obs_start
                                obs_timeseries.extend(
                                    process_5_min_ts(
                                        newly_extracted_timeseries=ts3,
                                        expected_start=expected_start)[1:])
            elif timestep == 15:
                if obs1_station_id != str(-1):
                    obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)

                    ts = extract_obs_rain_15_min_ts(
                        connection=curw_obs_connection,
                        start_time=obs_start,
                        id=obs1_hash_id,
                        end_time=end_time)

                    if ts is not None and len(ts) > 1:
                        obs_timeseries.extend(
                            process_15_min_ts(newly_extracted_timeseries=ts,
                                              expected_start=obs_start)[1:])
                        # obs_start = ts[-1][0]

                    if obs2_station_id != str(-1):
                        obs2_hash_id = stations_dict_for_obs.get(
                            obs2_station_id)

                        ts2 = extract_obs_rain_15_min_ts(
                            connection=curw_obs_connection,
                            start_time=obs_start,
                            id=obs2_hash_id,
                            end_time=end_time)
                        if ts2 is not None and len(ts2) > 1:
                            obs_timeseries = fill_missing_values(
                                newly_extracted_timeseries=ts2,
                                OBS_TS=obs_timeseries)
                            if obs_timeseries is not None and len(
                                    obs_timeseries) > 0:
                                expected_start = obs_timeseries[-1][0]
                            else:
                                expected_start = obs_start
                            obs_timeseries.extend(
                                process_15_min_ts(
                                    newly_extracted_timeseries=ts2,
                                    expected_start=expected_start)[1:])
                            # obs_start = ts2[-1][0]

                        if obs3_station_id != str(-1):
                            obs3_hash_id = stations_dict_for_obs.get(
                                obs3_station_id)

                            ts3 = extract_obs_rain_15_min_ts(
                                connection=curw_obs_connection,
                                start_time=obs_start,
                                id=obs3_hash_id,
                                end_time=end_time)
                            if ts3 is not None and len(ts3) > 1 and len(
                                    obs_timeseries) > 0:
                                obs_timeseries = fill_missing_values(
                                    newly_extracted_timeseries=ts3,
                                    OBS_TS=obs_timeseries)
                                if obs_timeseries is not None:
                                    expected_start = obs_timeseries[-1][0]
                                else:
                                    expected_start = obs_start
                                obs_timeseries.extend(
                                    process_15_min_ts(
                                        newly_extracted_timeseries=ts3,
                                        expected_start=expected_start)[1:])

            for i in range(len(obs_timeseries)):
                if obs_timeseries[i][1] == -99999:
                    obs_timeseries[i][1] = 0

            print("### obs timeseries length ###", len(obs_timeseries))
            if obs_timeseries is not None and len(
                    obs_timeseries) > 0 and obs_timeseries[-1][0] != end_time:
                obs_timeseries.append(
                    [datetime.strptime(end_time, DATE_TIME_FORMAT), 0])

            final_ts = process_continuous_ts(original_ts=obs_timeseries,
                                             expected_start=datetime.strptime(
                                                 start_time, DATE_TIME_FORMAT),
                                             filling_value=0,
                                             timestep=timestep)

            if final_ts is not None and len(final_ts) > 0:
                TS.insert_data(timeseries=final_ts, tms_id=tms_id, upsert=True)
                TS.update_latest_obs(id_=tms_id, obs_end=(final_ts[-1][1]))

    except Exception as e:
        traceback.print_exc()
        logger.error(
            "Exception occurred while updating obs rainfalls in curw_sim.")
    finally:
        curw_obs_connection.close()
        destroy_Pool(pool=curw_sim_pool)
        destroy_Pool(pool=curw_obs_pool)
        logger.info("Process finished")
def update_rainfall_obs(target_model, method, timestep, start_time, end_time):
    """
    Update rainfall observations for flo2d models
    :param model: target model
    :param method: value interpolation method
    :param grid_interpolation: grid interpolation method
    :param timestep:
    :return:
    """
    obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
    try:

        # Connect to the database
        curw_obs_pool = get_Pool(host=con_params.CURW_OBS_HOST,
                                 user=con_params.CURW_OBS_USERNAME,
                                 password=con_params.CURW_OBS_PASSWORD,
                                 port=con_params.CURW_OBS_PORT,
                                 db=con_params.CURW_OBS_DATABASE)

        curw_obs_connection = curw_obs_pool.connection()

        curw_sim_pool = get_Pool(host=con_params.CURW_SIM_HOST,
                                 user=con_params.CURW_SIM_USERNAME,
                                 password=con_params.CURW_SIM_PASSWORD,
                                 port=con_params.CURW_SIM_PORT,
                                 db=con_params.CURW_SIM_DATABASE)

        TS = Timeseries(pool=curw_sim_pool)

        # [hash_id, station_id, station_name, latitude, longitude]
        active_obs_stations = extract_active_curw_obs_rainfall_stations(
            start_time=start_time, end_time=end_time)[1:]
        obs_stations_dict = {
        }  # keys: obs station id , value: [hash id, name, latitude, longitude]

        for obs_index in range(len(active_obs_stations)):
            obs_stations_dict[active_obs_stations[obs_index][1]] = [
                active_obs_stations[obs_index][0],
                active_obs_stations[obs_index][2],
                active_obs_stations[obs_index][3],
                active_obs_stations[obs_index][4]
            ]

        for obs_id in obs_stations_dict.keys():
            meta_data = {
                'latitude':
                float('%.6f' % float(obs_stations_dict.get(obs_id)[2])),
                'longitude':
                float('%.6f' % float(obs_stations_dict.get(obs_id)[3])),
                'model':
                target_model,
                'method':
                method,
                'grid_id':
                'rainfall_{}_{}'.format(obs_id,
                                        obs_stations_dict.get(obs_id)[1])
            }

            tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data)

            if tms_id is None:
                tms_id = TS.generate_timeseries_id(meta_data=meta_data)
                meta_data['id'] = tms_id
                TS.insert_run(meta_data=meta_data)

            TS.update_grid_id(id_=tms_id, grid_id=meta_data['grid_id'])

            obs_hash_id = obs_stations_dict.get(obs_id)[0]

            obs_timeseries = []

            if timestep == 5:
                ts = extract_obs_rain_5_min_ts(connection=curw_obs_connection,
                                               start_time=obs_start,
                                               end_time=end_time,
                                               id=obs_hash_id)
                if ts is not None and len(ts) > 1:
                    obs_timeseries.extend(
                        process_5_min_ts(newly_extracted_timeseries=ts,
                                         expected_start=obs_start)[1:])
                    # obs_start = ts[-1][0]
            elif timestep == 15:
                ts = extract_obs_rain_15_min_ts(connection=curw_obs_connection,
                                                start_time=obs_start,
                                                end_time=end_time,
                                                id=obs_hash_id)
                if ts is not None and len(ts) > 1:
                    obs_timeseries.extend(
                        process_15_min_ts(newly_extracted_timeseries=ts,
                                          expected_start=obs_start)[1:])
                    # obs_start = ts[-1][0]

            # for i in range(len(obs_timeseries)):
            #     if obs_timeseries[i][1] == -99999:
            #         obs_timeseries[i][1] = 0

            if obs_timeseries is not None and len(obs_timeseries) > 0:
                TS.replace_data(timeseries=obs_timeseries, tms_id=tms_id)

    except Exception as e:
        traceback.print_exc()
        logger.error(
            "Exception occurred while updating obs rainfalls in curw_sim.")
    finally:
        curw_obs_connection.close()
        destroy_Pool(pool=curw_sim_pool)
        destroy_Pool(pool=curw_obs_pool)
Exemple #3
0
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep):
    """
    Update rainfall observations for flo2d models
    :param flo2d_model: flo2d model
    :param method: value interpolation method
    :param grid_interpolation: grid interpolation method
    :param timestep: output timeseries timestep
    :return:
    """

    now = datetime.now()
    OBS_START_STRING = (now - timedelta(days=10)).strftime('%Y-%m-%d %H:00:00')
    OBS_START = datetime.strptime(OBS_START_STRING, '%Y-%m-%d %H:%M:%S')

    try:

        # Connect to the database
        curw_obs_pool = get_Pool(host=CURW_OBS_HOST,
                                 user=CURW_OBS_USERNAME,
                                 password=CURW_OBS_PASSWORD,
                                 port=CURW_OBS_PORT,
                                 db=CURW_OBS_DATABASE)

        curw_obs_connection = curw_obs_pool.connection()

        curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
                                 user=CURW_SIM_USERNAME,
                                 password=CURW_SIM_PASSWORD,
                                 port=CURW_SIM_PORT,
                                 db=CURW_SIM_DATABASE)

        # test ######
        # pool = get_Pool(host=HOST, user=USERNAME, password=PASSWORD, port=PORT, db=DATABASE)

        TS = Sim_Timeseries(pool=curw_sim_pool)

        # [hash_id, station_id, station_name, latitude, longitude]
        active_obs_stations = read_csv(
            'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'
        )
        flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format(
            flo2d_model))  # [Grid_ ID, X(longitude), Y(latitude)]

        stations_dict_for_obs = {}  # keys: obs station id , value: hash id

        for obs_index in range(len(active_obs_stations)):
            stations_dict_for_obs[active_obs_stations[obs_index]
                                  [1]] = active_obs_stations[obs_index][0]

        flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(
            pool=curw_sim_pool,
            grid_interpolation=grid_interpolation,
            flo2d_model=flo2d_model)

        for flo2d_index in range(len(flo2d_grids)):
            obs_start = OBS_START
            lat = flo2d_grids[flo2d_index][2]
            lon = flo2d_grids[flo2d_index][1]
            cell_id = flo2d_grids[flo2d_index][0]
            meta_data = {
                'latitude':
                float('%.6f' % float(lat)),
                'longitude':
                float('%.6f' % float(lon)),
                'model':
                flo2d_model,
                'method':
                method,
                'grid_id':
                '{}_{}_{}'.format(flo2d_model, grid_interpolation,
                                  (str(cell_id)).zfill(10))
            }

            tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
                                          method=meta_data.get('method'))

            if tms_id is None:
                tms_id = TS.generate_timeseries_id(meta_data=meta_data)
                meta_data['id'] = tms_id
                TS.insert_run(meta_data=meta_data)

            obs_end = TS.get_obs_end(id_=tms_id)

            if obs_end is not None:
                obs_start = obs_end - timedelta(hours=1)

            obs1_station_id = str(
                flo2d_obs_mapping.get(meta_data['grid_id'])[0])
            obs2_station_id = str(
                flo2d_obs_mapping.get(meta_data['grid_id'])[1])
            obs3_station_id = str(
                flo2d_obs_mapping.get(meta_data['grid_id'])[2])

            obs_timeseries = []

            if timestep == 5:
                if obs1_station_id != str(-1):
                    obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)

                    ts = extract_obs_rain_5_min_ts(
                        connection=curw_obs_connection,
                        start_time=obs_start,
                        id=obs1_hash_id)
                    if ts is not None and len(ts) > 1:
                        obs_timeseries.extend(
                            process_5_min_ts(newly_extracted_timeseries=ts,
                                             expected_start=obs_start)[1:])
                        # obs_start = ts[-1][0]

                    if obs2_station_id != str(-1):
                        obs2_hash_id = stations_dict_for_obs.get(
                            obs2_station_id)

                        ts2 = extract_obs_rain_5_min_ts(
                            connection=curw_obs_connection,
                            start_time=obs_start,
                            id=obs2_hash_id)
                        if ts2 is not None and len(ts2) > 1:
                            obs_timeseries = fill_missing_values(
                                newly_extracted_timeseries=ts2,
                                OBS_TS=obs_timeseries)
                            if obs_timeseries is not None and len(
                                    obs_timeseries) > 0:
                                expected_start = obs_timeseries[-1][0]
                            else:
                                expected_start = obs_start
                            obs_timeseries.extend(
                                process_5_min_ts(
                                    newly_extracted_timeseries=ts2,
                                    expected_start=expected_start)[1:])
                            # obs_start = ts2[-1][0]

                        if obs3_station_id != str(-1):
                            obs3_hash_id = stations_dict_for_obs.get(
                                obs3_station_id)

                            ts3 = extract_obs_rain_5_min_ts(
                                connection=curw_obs_connection,
                                start_time=obs_start,
                                id=obs3_hash_id)
                            if ts3 is not None and len(ts3) > 1 and len(
                                    obs_timeseries) > 0:
                                obs_timeseries = fill_missing_values(
                                    newly_extracted_timeseries=ts3,
                                    OBS_TS=obs_timeseries)
                                if obs_timeseries is not None:
                                    expected_start = obs_timeseries[-1][0]
                                else:
                                    expected_start = obs_start
                                obs_timeseries.extend(
                                    process_5_min_ts(
                                        newly_extracted_timeseries=ts3,
                                        expected_start=expected_start)[1:])
            elif timestep == 15:
                if obs1_station_id != str(-1):
                    obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)

                    ts = extract_obs_rain_15_min_ts(
                        connection=curw_obs_connection,
                        start_time=obs_start,
                        id=obs1_hash_id)
                    if ts is not None and len(ts) > 1:
                        obs_timeseries.extend(
                            process_15_min_ts(newly_extracted_timeseries=ts,
                                              expected_start=obs_start)[1:])
                        # obs_start = ts[-1][0]

                    if obs2_station_id != str(-1):
                        obs2_hash_id = stations_dict_for_obs.get(
                            obs2_station_id)

                        ts2 = extract_obs_rain_15_min_ts(
                            connection=curw_obs_connection,
                            start_time=obs_start,
                            id=obs2_hash_id)
                        if ts2 is not None and len(ts2) > 1:
                            obs_timeseries = fill_missing_values(
                                newly_extracted_timeseries=ts2,
                                OBS_TS=obs_timeseries)
                            if obs_timeseries is not None and len(
                                    obs_timeseries) > 0:
                                expected_start = obs_timeseries[-1][0]
                            else:
                                expected_start = obs_start
                            obs_timeseries.extend(
                                process_15_min_ts(
                                    newly_extracted_timeseries=ts2,
                                    expected_start=expected_start)[1:])
                            # obs_start = ts2[-1][0]

                        if obs3_station_id != str(-1):
                            obs3_hash_id = stations_dict_for_obs.get(
                                obs3_station_id)

                            ts3 = extract_obs_rain_15_min_ts(
                                connection=curw_obs_connection,
                                start_time=obs_start,
                                id=obs3_hash_id)
                            if ts3 is not None and len(ts3) > 1 and len(
                                    obs_timeseries) > 0:
                                obs_timeseries = fill_missing_values(
                                    newly_extracted_timeseries=ts3,
                                    OBS_TS=obs_timeseries)
                                if obs_timeseries is not None:
                                    expected_start = obs_timeseries[-1][0]
                                else:
                                    expected_start = obs_start
                                obs_timeseries.extend(
                                    process_15_min_ts(
                                        newly_extracted_timeseries=ts3,
                                        expected_start=expected_start)[1:])

            for i in range(len(obs_timeseries)):
                if obs_timeseries[i][1] == -99999:
                    obs_timeseries[i][1] = 0

            if obs_timeseries is not None and len(obs_timeseries) > 0:
                TS.insert_data(timeseries=obs_timeseries,
                               tms_id=tms_id,
                               upsert=True)
                TS.update_latest_obs(id_=tms_id,
                                     obs_end=(obs_timeseries[-1][1]))

    except Exception as e:
        traceback.print_exc()
        logger.error(
            "Exception occurred while updating obs rainfalls in curw_sim.")
    finally:
        curw_obs_connection.close()
        destroy_Pool(pool=curw_sim_pool)
        destroy_Pool(pool=curw_obs_pool)
        logger.info("Process finished")