def get_non_sensor_index():
ic0_file_name = "../data/csv_ic0/IC0_20030101.csv"
data = calc_data.get_1day_ic0_data(ic0_file_name)
data = pd.concat([latlon_ex, data], axis=1)
lonlat_idx = np.array(data.loc[(data.Lat>=80)&(data.ic0_145.isnull()), ["Lon", "Lat"]].index)
rst = np.zeros(145**2)
rst[lonlat_idx] = 1
np.savetxt("../data/non_sensor_index.csv", rst, delimiter=",")
"""
def main_data(start, end, **kwargs):
span = kwargs["span"]
region = kwargs["region"]
get_columns = kwargs["get_columns"]
accumulate = kwargs["accumulate"]
date_ax, date_ax_str = get_date_ax(start, end)
N = len(date_ax_str)
skipping_date_str = []
accumulate_data = []
data = []
for i, day in enumerate(date_ax_str):
print("{}/{}: {}".format(i + 1, N, day))
print("start: {}, end: {}".format(start, end))
year = day[2:4]
month = day[4:6]
#ファイル名の生成
wind_file_name = "../data/csv_w/ecm" + day[2:] + ".csv"
ice_file_name = "../data/csv_iw/" + day[2:] + ".csv"
ic0_145_file_name = "../data/csv_ic0/IC0_" + day + ".csv"
sit_145_file_name = "../data/csv_sit/SIT_" + day + ".csv"
coeff_file_name = "../data/csv_A_30/ssc_amsr_ads" + str(year) + str(
month) + "_" + str(span) + "_fin.csv"
hermert_file_name = "../data/csv_Helmert_30/Helmert_30_" + str(
day)[:6] + ".csv"
# wind10m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
# t2m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
skipping_boolean = ("coeff" not in get_columns) and (not all([
os.path.isfile(wind_file_name),
os.path.isfile(ice_file_name),
os.path.isfile(coeff_file_name)
]))
if ("ic0_145" in get_columns):
skipping_boolean = ("coeff" not in get_columns) and (not all([
os.path.isfile(wind_file_name),
os.path.isfile(ice_file_name),
os.path.isfile(coeff_file_name),
os.path.isfile(ic0_145_file_name)
]))
if ("sit_145" in get_columns):
skipping_boolean = ("coeff" not in get_columns) and (not all([
os.path.isfile(wind_file_name),
os.path.isfile(ice_file_name),
os.path.isfile(coeff_file_name),
os.path.isfile(sit_145_file_name)
]))
if skipping_boolean == True:
print("\tSkipping " + day + " file...")
date_ax_str.remove(day)
bb = date(int(day[:4]), int(day[4:6]), int(day[6:]))
date_ax.remove(bb)
skipping_date_str.append(day)
continue
data = pd.DataFrame({"data_idx": np.array(ocean_grid_145).ravel()})
if "ex_1" in get_columns:
print("\t{}\n\t{}\n\t{}\n\t{}".format(wind_file_name,
ice_file_name,
coeff_file_name))
tmp = calc_data.get_w_regression_data(wind_file_name,
ice_file_name,
coeff_file_name)
data = pd.concat([data, tmp], axis=1)
if "ex_2" in get_columns:
print("\t{}\n\t{}\n\t{}\n\t{}".format(wind_file_name,
ice_file_name,
hermert_file_name))
tmp = calc_data.get_w_hermert_data(wind_file_name, ice_file_name,
hermert_file_name)
data = pd.concat([data, tmp], axis=1)
if "w" in get_columns:
print("\t{}".format(wind_file_name))
tmp = calc_data.get_1day_w_data(wind_file_name)
data = pd.concat([data, tmp], axis=1)
if "iw" in get_columns:
print("\t{}".format(ice_file_name))
tmp = calc_data.get_1day_iw_data(ice_file_name)
data = pd.concat([data, tmp], axis=1)
if "ic0_145" in get_columns:
print("\t{}".format(ic0_145_file_name))
tmp = calc_data.get_1day_ic0_data(ic0_145_file_name)
data = pd.concat([data, tmp], axis=1)
if "sit_145" in get_columns:
print("\t{}".format(sit_145_file_name))
tmp = calc_data.get_1day_sit_data(sit_145_file_name)
data = pd.concat([data, tmp], axis=1)
if "coeff" in get_columns:
print("\t{}".format(coeff_file_name))
tmp = calc_data.get_1month_coeff_data(coeff_file_name)
data = pd.concat([data, tmp], axis=1)
if "hermert" in get_columns:
print("\t{}".format(hermert_file_name))
tmp = calc_data.get_1month_hermert_data(hermert_file_name)
data = pd.concat([data, tmp], axis=1)
"""
if "w10m" in get_columns:
tmp = calc_data.get_1day_w10m_data(wind10m_file_name)
data = pd.concat([data, tmp], axis=1)
if "t2m" in get_columns:
tmp = calc_data.get_1day_t2m_data(t2m_file_name)
data = pd.concat([data, tmp], axis=1)
"""
data = calc_data.get_masked_region_data(data, region)
if ("coeff" in get_columns):
print("\tSelected only coeff data. Getting out of the loop...")
continue
if accumulate == True:
data_1 = data.drop("data_idx", axis=1)
print("\t{}".format(data_1.columns))
accumulate_data.append(np.array(data_1))
if accumulate == True:
print("accumulate: True\tdata type: array")
return date_ax, date_ax_str, skipping_date_str, accumulate_data
else:
print("accumulate: False\tdata type: DataFrame")
return date_ax, date_ax_str, skipping_date_str, data
def main_data(start, end, **kwargs):
span = kwargs["span"]
region = kwargs["region"]
get_columns = kwargs["get_columns"]
accumulate = kwargs["accumulate"]
date_ax, date_ax_str = get_date_ax(start, end)
N = len(date_ax_str)
skipping_date_str = []
accumulate_data = []
for i, day in enumerate(date_ax_str):
print("{}/{}: {}".format(i + 1, N, day))
year = day[2:4]
month = day[4:6]
#ファイル名の生成
wind_file_name = "../data/wind_data/ecm" + day[2:] + ".csv"
ice_file_name = "../data/ice_wind_data/" + day[2:] + ".csv"
ic0_145_file_name = "../data/IC0_csv/2" + day + "A.csv"
ic0_900_file_name = "../data/IC0_csv/2" + day + "A.csv"
coeff_file_name = "../data/A_csv/ssc_amsr_ads" + str(year) + str(
month) + "_" + str(span) + "_fin.csv"
wind10m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
t2m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
if not all([
os.path.isfile(wind_file_name),
os.path.isfile(ice_file_name),
os.path.isfile(ic0_145_file_name),
os.path.isfile(coeff_file_name)
]):
print("\tSkipping " + day + " file...")
date_ax_str.remove(day)
date_ax.remove(datetime(day[:4] + "-" + day[4:6] + "-" + day[6:]))
skipping_date_str.append(day)
continue
data = pd.DataFrame({"data_idx": np.zeros(145 * 145)})
iw_idx_t, ic0_idx_t = np.array([-1]), np.array([-1])
if "ex_1" in get_columns:
tmp = calc_data.get_w_regression_data(wind_file_name,
ice_file_name,
coeff_file_name)
iw_idx_t = np.array(tmp["data_idx"])
data = pd.concat([data, tmp.loc[:, ["A_by_day", "theta_by_day"]]],
axis=1)
if "w" in get_columns:
tmp = calc_data.get_1day_w_data(wind_file_name)
data = pd.concat([data, tmp], axis=1)
if "iw" in get_columns:
tmp = calc_data.get_1day_ice_data(ice_file_name)
iw_idx_t = np.array(tmp["iw_idx_t"])
data = pd.concat([data, tmp.loc[:, ["iw_u", "iw_v", "iw_speed"]]],
axis=1)
if "ic0_145" in get_columns:
tmp = calc_data.get_1day_ic0_data(ic0_file_name,
grid900to145_file_name)
ic0_idx_t = np.array(tmp["ic0_idx_t"])
data = pd.concat([data, tmp.loc[:, ["ic0_145"]]], axis=1)
if "coeff" in get_columns:
tmp = calc_data.get_1month_coeff_data(coeff_file_name)
data = pd.concat([data, tmp], axis=1)
if "w10m" in get_columns:
tmp = calc_data.get_1day_w10m_data(wind10m_file_name)
data = pd.concat([data, tmp], axis=1)
if "t2m" in get_columns:
tmp = calc_data.get_1day_t2m_data(t2m_file_name)
data = pd.concat([data, tmp], axis=1)
mat = iw_idx_t.ravel().tolist() + ic0_idx_t.ravel().tolist()
data_t_idx = calc_data.get_non_nan_idx(mat, ocean_idx, strict=True)
data.loc[data.data_idx, data_t_idx] = 1
data = calc_data.get_masked_region_data(data, region)
if ("coeff" in get_columns) and (len(get_columns) == 1):
print("\tSelected only coeff data. Getting out of the loop...")
continue
if accumulate == True:
data = data.loc[:, get_columns]
print("\t{}".format(data.columns))
accumulate_data.append(np.array(data))
if accumulate == True:
print("accumulate: True\tdata type: array")
return date_ax, date_ax_str, skipping_date_str, accumulate_data
else:
print("accumulate: False\tdata type: DataFrame")
return date_ax, date_ax_str, skipping_date_str, data