Exemple #1
0
def main_data(start, end, **kwargs):
    span = kwargs["span"]
    region = kwargs["region"]
    get_columns = kwargs["get_columns"]
    accumulate = kwargs["accumulate"]

    date_ax, date_ax_str = get_date_ax(start, end)
    N = len(date_ax_str)
    skipping_date_str = []
    accumulate_data = []
    data = []
    for i, day in enumerate(date_ax_str):
        print("{}/{}: {}".format(i + 1, N, day))
        print("start: {}, end: {}".format(start, end))
        year = day[2:4]
        month = day[4:6]

        #ファイル名の生成
        wind_file_name = "../data/csv_w/ecm" + day[2:] + ".csv"
        ice_file_name = "../data/csv_iw/" + day[2:] + ".csv"
        ic0_145_file_name = "../data/csv_ic0/IC0_" + day + ".csv"
        sit_145_file_name = "../data/csv_sit/SIT_" + day + ".csv"
        coeff_file_name = "../data/csv_A_30/ssc_amsr_ads" + str(year) + str(
            month) + "_" + str(span) + "_fin.csv"
        hermert_file_name = "../data/csv_Helmert_30/Helmert_30_" + str(
            day)[:6] + ".csv"
        # wind10m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
        # t2m_file_name = "../data/netcdf4/" + day[2:] + ".csv"

        skipping_boolean = ("coeff" not in get_columns) and (not all([
            os.path.isfile(wind_file_name),
            os.path.isfile(ice_file_name),
            os.path.isfile(coeff_file_name)
        ]))
        if ("ic0_145" in get_columns):
            skipping_boolean = ("coeff" not in get_columns) and (not all([
                os.path.isfile(wind_file_name),
                os.path.isfile(ice_file_name),
                os.path.isfile(coeff_file_name),
                os.path.isfile(ic0_145_file_name)
            ]))
        if ("sit_145" in get_columns):
            skipping_boolean = ("coeff" not in get_columns) and (not all([
                os.path.isfile(wind_file_name),
                os.path.isfile(ice_file_name),
                os.path.isfile(coeff_file_name),
                os.path.isfile(sit_145_file_name)
            ]))

        if skipping_boolean == True:
            print("\tSkipping " + day + " file...")
            date_ax_str.remove(day)
            bb = date(int(day[:4]), int(day[4:6]), int(day[6:]))
            date_ax.remove(bb)
            skipping_date_str.append(day)
            continue

        data = pd.DataFrame({"data_idx": np.array(ocean_grid_145).ravel()})
        if "ex_1" in get_columns:
            print("\t{}\n\t{}\n\t{}\n\t{}".format(wind_file_name,
                                                  ice_file_name,
                                                  coeff_file_name))
            tmp = calc_data.get_w_regression_data(wind_file_name,
                                                  ice_file_name,
                                                  coeff_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "ex_2" in get_columns:
            print("\t{}\n\t{}\n\t{}\n\t{}".format(wind_file_name,
                                                  ice_file_name,
                                                  hermert_file_name))
            tmp = calc_data.get_w_hermert_data(wind_file_name, ice_file_name,
                                               hermert_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "w" in get_columns:
            print("\t{}".format(wind_file_name))
            tmp = calc_data.get_1day_w_data(wind_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "iw" in get_columns:
            print("\t{}".format(ice_file_name))
            tmp = calc_data.get_1day_iw_data(ice_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "ic0_145" in get_columns:
            print("\t{}".format(ic0_145_file_name))
            tmp = calc_data.get_1day_ic0_data(ic0_145_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "sit_145" in get_columns:
            print("\t{}".format(sit_145_file_name))
            tmp = calc_data.get_1day_sit_data(sit_145_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "coeff" in get_columns:
            print("\t{}".format(coeff_file_name))
            tmp = calc_data.get_1month_coeff_data(coeff_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "hermert" in get_columns:
            print("\t{}".format(hermert_file_name))
            tmp = calc_data.get_1month_hermert_data(hermert_file_name)
            data = pd.concat([data, tmp], axis=1)
        """
		if "w10m" in get_columns:
			tmp = calc_data.get_1day_w10m_data(wind10m_file_name)
			data = pd.concat([data, tmp], axis=1)
		if "t2m" in get_columns:
			tmp = calc_data.get_1day_t2m_data(t2m_file_name)
			data = pd.concat([data, tmp], axis=1)
		"""

        data = calc_data.get_masked_region_data(data, region)

        if ("coeff" in get_columns):
            print("\tSelected only coeff data. Getting out of the loop...")
            continue

        if accumulate == True:
            data_1 = data.drop("data_idx", axis=1)
            print("\t{}".format(data_1.columns))
            accumulate_data.append(np.array(data_1))

    if accumulate == True:
        print("accumulate: True\tdata type: array")
        return date_ax, date_ax_str, skipping_date_str, accumulate_data
    else:
        print("accumulate: False\tdata type: DataFrame")
        return date_ax, date_ax_str, skipping_date_str, data
Exemple #2
0
def main_data(start, end, **kwargs):
    span = kwargs["span"]
    region = kwargs["region"]
    get_columns = kwargs["get_columns"]
    accumulate = kwargs["accumulate"]

    date_ax, date_ax_str = get_date_ax(start, end)
    N = len(date_ax_str)
    skipping_date_str = []
    accumulate_data = []
    for i, day in enumerate(date_ax_str):
        print("{}/{}: {}".format(i + 1, N, day))
        year = day[2:4]
        month = day[4:6]

        #ファイル名の生成
        wind_file_name = "../data/wind_data/ecm" + day[2:] + ".csv"
        ice_file_name = "../data/ice_wind_data/" + day[2:] + ".csv"
        ic0_145_file_name = "../data/IC0_csv/2" + day + "A.csv"
        ic0_900_file_name = "../data/IC0_csv/2" + day + "A.csv"
        coeff_file_name = "../data/A_csv/ssc_amsr_ads" + str(year) + str(
            month) + "_" + str(span) + "_fin.csv"
        wind10m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
        t2m_file_name = "../data/netcdf4/" + day[2:] + ".csv"

        if not all([
                os.path.isfile(wind_file_name),
                os.path.isfile(ice_file_name),
                os.path.isfile(ic0_145_file_name),
                os.path.isfile(coeff_file_name)
        ]):
            print("\tSkipping " + day + " file...")
            date_ax_str.remove(day)
            date_ax.remove(datetime(day[:4] + "-" + day[4:6] + "-" + day[6:]))
            skipping_date_str.append(day)
            continue

        data = pd.DataFrame({"data_idx": np.zeros(145 * 145)})
        iw_idx_t, ic0_idx_t = np.array([-1]), np.array([-1])
        if "ex_1" in get_columns:
            tmp = calc_data.get_w_regression_data(wind_file_name,
                                                  ice_file_name,
                                                  coeff_file_name)
            iw_idx_t = np.array(tmp["data_idx"])
            data = pd.concat([data, tmp.loc[:, ["A_by_day", "theta_by_day"]]],
                             axis=1)
        if "w" in get_columns:
            tmp = calc_data.get_1day_w_data(wind_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "iw" in get_columns:
            tmp = calc_data.get_1day_ice_data(ice_file_name)
            iw_idx_t = np.array(tmp["iw_idx_t"])
            data = pd.concat([data, tmp.loc[:, ["iw_u", "iw_v", "iw_speed"]]],
                             axis=1)
        if "ic0_145" in get_columns:
            tmp = calc_data.get_1day_ic0_data(ic0_file_name,
                                              grid900to145_file_name)
            ic0_idx_t = np.array(tmp["ic0_idx_t"])
            data = pd.concat([data, tmp.loc[:, ["ic0_145"]]], axis=1)
        if "coeff" in get_columns:
            tmp = calc_data.get_1month_coeff_data(coeff_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "w10m" in get_columns:
            tmp = calc_data.get_1day_w10m_data(wind10m_file_name)
            data = pd.concat([data, tmp], axis=1)
        if "t2m" in get_columns:
            tmp = calc_data.get_1day_t2m_data(t2m_file_name)
            data = pd.concat([data, tmp], axis=1)

        mat = iw_idx_t.ravel().tolist() + ic0_idx_t.ravel().tolist()
        data_t_idx = calc_data.get_non_nan_idx(mat, ocean_idx, strict=True)
        data.loc[data.data_idx, data_t_idx] = 1
        data = calc_data.get_masked_region_data(data, region)

        if ("coeff" in get_columns) and (len(get_columns) == 1):
            print("\tSelected only coeff data. Getting out of the loop...")
            continue

        if accumulate == True:
            data = data.loc[:, get_columns]
            print("\t{}".format(data.columns))
            accumulate_data.append(np.array(data))

    if accumulate == True:
        print("accumulate: True\tdata type: array")
        return date_ax, date_ax_str, skipping_date_str, accumulate_data
    else:
        print("accumulate: False\tdata type: DataFrame")
        return date_ax, date_ax_str, skipping_date_str, data