def main_data(start, end, **kwargs):
span = kwargs["span"]
region = kwargs["region"]
get_columns = kwargs["get_columns"]
accumulate = kwargs["accumulate"]
date_ax, date_ax_str = get_date_ax(start, end)
N = len(date_ax_str)
skipping_date_str = []
accumulate_data = []
data = []
for i, day in enumerate(date_ax_str):
print("{}/{}: {}".format(i + 1, N, day))
print("start: {}, end: {}".format(start, end))
year = day[2:4]
month = day[4:6]
#ファイル名の生成
wind_file_name = "../data/csv_w/ecm" + day[2:] + ".csv"
ice_file_name = "../data/csv_iw/" + day[2:] + ".csv"
ic0_145_file_name = "../data/csv_ic0/IC0_" + day + ".csv"
sit_145_file_name = "../data/csv_sit/SIT_" + day + ".csv"
coeff_file_name = "../data/csv_A_30/ssc_amsr_ads" + str(year) + str(
month) + "_" + str(span) + "_fin.csv"
hermert_file_name = "../data/csv_Helmert_30/Helmert_30_" + str(
day)[:6] + ".csv"
# wind10m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
# t2m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
skipping_boolean = ("coeff" not in get_columns) and (not all([
os.path.isfile(wind_file_name),
os.path.isfile(ice_file_name),
os.path.isfile(coeff_file_name)
]))
if ("ic0_145" in get_columns):
skipping_boolean = ("coeff" not in get_columns) and (not all([
os.path.isfile(wind_file_name),
os.path.isfile(ice_file_name),
os.path.isfile(coeff_file_name),
os.path.isfile(ic0_145_file_name)
]))
if ("sit_145" in get_columns):
skipping_boolean = ("coeff" not in get_columns) and (not all([
os.path.isfile(wind_file_name),
os.path.isfile(ice_file_name),
os.path.isfile(coeff_file_name),
os.path.isfile(sit_145_file_name)
]))
if skipping_boolean == True:
print("\tSkipping " + day + " file...")
date_ax_str.remove(day)
bb = date(int(day[:4]), int(day[4:6]), int(day[6:]))
date_ax.remove(bb)
skipping_date_str.append(day)
continue
data = pd.DataFrame({"data_idx": np.array(ocean_grid_145).ravel()})
if "ex_1" in get_columns:
print("\t{}\n\t{}\n\t{}\n\t{}".format(wind_file_name,
ice_file_name,
coeff_file_name))
tmp = calc_data.get_w_regression_data(wind_file_name,
ice_file_name,
coeff_file_name)
data = pd.concat([data, tmp], axis=1)
if "ex_2" in get_columns:
print("\t{}\n\t{}\n\t{}\n\t{}".format(wind_file_name,
ice_file_name,
hermert_file_name))
tmp = calc_data.get_w_hermert_data(wind_file_name, ice_file_name,
hermert_file_name)
data = pd.concat([data, tmp], axis=1)
if "w" in get_columns:
print("\t{}".format(wind_file_name))
tmp = calc_data.get_1day_w_data(wind_file_name)
data = pd.concat([data, tmp], axis=1)
if "iw" in get_columns:
print("\t{}".format(ice_file_name))
tmp = calc_data.get_1day_iw_data(ice_file_name)
data = pd.concat([data, tmp], axis=1)
if "ic0_145" in get_columns:
print("\t{}".format(ic0_145_file_name))
tmp = calc_data.get_1day_ic0_data(ic0_145_file_name)
data = pd.concat([data, tmp], axis=1)
if "sit_145" in get_columns:
print("\t{}".format(sit_145_file_name))
tmp = calc_data.get_1day_sit_data(sit_145_file_name)
data = pd.concat([data, tmp], axis=1)
if "coeff" in get_columns:
print("\t{}".format(coeff_file_name))
tmp = calc_data.get_1month_coeff_data(coeff_file_name)
data = pd.concat([data, tmp], axis=1)
if "hermert" in get_columns:
print("\t{}".format(hermert_file_name))
tmp = calc_data.get_1month_hermert_data(hermert_file_name)
data = pd.concat([data, tmp], axis=1)
"""
if "w10m" in get_columns:
tmp = calc_data.get_1day_w10m_data(wind10m_file_name)
data = pd.concat([data, tmp], axis=1)
if "t2m" in get_columns:
tmp = calc_data.get_1day_t2m_data(t2m_file_name)
data = pd.concat([data, tmp], axis=1)
"""
data = calc_data.get_masked_region_data(data, region)
if ("coeff" in get_columns):
print("\tSelected only coeff data. Getting out of the loop...")
continue
if accumulate == True:
data_1 = data.drop("data_idx", axis=1)
print("\t{}".format(data_1.columns))
accumulate_data.append(np.array(data_1))
if accumulate == True:
print("accumulate: True\tdata type: array")
return date_ax, date_ax_str, skipping_date_str, accumulate_data
else:
print("accumulate: False\tdata type: DataFrame")
return date_ax, date_ax_str, skipping_date_str, data
def visual_2winds(wind_file_name,
ice_file_name,
latlon145_file_name,
points,
v_ratio=False,
v_difftheta=False,
threshold=0.05,
show=True):
"""
風と氷の速度の相関の可視化
"""
from matplotlib.colors import LinearSegmentedColormap
def generate_cmap(colors):
#自分で定義したカラーマップを返す
values = range(len(colors))
vmax = np.ceil(np.max(values))
color_list = []
for v, c in zip(values, colors):
color_list.append((v / vmax, c))
return LinearSegmentedColormap.from_list('custom_cmap', color_list)
cm = generate_cmap(['#87CEEB', '#2E8B57', '#F4A460'])
w_u1, w_v1, w_speed1 = calc_data.get_1day_w_data(wind_file_name)
u_true, v_true, speed_true = calc_data.get_1day_ice_data(ice_file_name)
lon, lat = calc_data.get_lonlat(latlon145_file_name)
def get_theta(u, v):
#u,v: Matrix(145*145)
return np.arctan2(v, u)
def get_wpc(ice_v, wind_v):
#A,B: Matrix(145*145)
return (ice_v / wind_v)
def get_diff_theta(i_t, w_t):
#t1,t2: theta Matrix(145*145)
return i_t - w_t
#角度
wind_theta = get_theta(w_v1, w_u1)
ice_theta = get_theta(v_true, u_true)
diff_theta = get_diff_theta(ice_theta, wind_theta)
#風力比
wpc = get_wpc(speed_true, w_speed1) / 100
wpc[wpc > threshold] = np.nan
wpc_high = get_wpc(speed_true, w_speed1) / 100
wpc_high[wpc_high <= threshold] = np.nan
m = Basemap(lon_0=180,
boundinglat=50,
resolution='l',
projection='npstere')
fig = plt.figure(figsize=(6, 6))
x, y = m(lon, lat)
x1 = np.reshape(x, (145, 145), order='F')
y1 = np.reshape(y, (145, 145), order='F')
m.drawcoastlines(color='0.15')
if v_ratio == True:
wpc = np.reshape(wpc, (145, 145), order='F')
wpc = np.ma.masked_invalid(wpc)
wpc_high = np.reshape(wpc_high, (145, 145), order='F')
wpc_high = np.ma.masked_invalid(wpc_high)
#m.pcolormesh(x1[points], y1[points], wpc_high[points], cmap='tab10')
#m.plot(x1[points], y1[points], 'bo', markersize=2)
m.pcolormesh(x1[points], y1[points], wpc[points], cmap=plt.cm.jet)
m.colorbar(location='bottom')
elif v_difftheta == True:
diff_theta = np.reshape(diff_theta, (145, 145), order='F')
#m.pcolormesh(x1[points], y1[points], np.absolute(diff_theta)[points], cmap=plt.cm.jet)
m.pcolormesh(x1[points],
y1[points],
diff_theta[points],
cmap=plt.cm.jet)
m.colorbar(location='bottom')
#hexbinで可視化
#seismic = 'seismic'
#m.hexbin(x, y, C=diff_theta.reshape((145*145,1), order='F'), reduce_C_function = max, gridsize=100, cmap="seismic")
#m.colorbar(location='bottom', format='%.1f')
#コンター図
#m.contour(x1[points], y1[points], diff_theta)
#風の可視化(quiver)
#m.quiver(x1[points], y1[points], w_u1[points], w_v1[points], w_speed1[points])
#m.quiver(x1[points], y1[points], u_true[points], v_true[points], speed_true[points])
if show == True:
plt.show()
def main_data(start, end, **kwargs):
span = kwargs["span"]
region = kwargs["region"]
get_columns = kwargs["get_columns"]
accumulate = kwargs["accumulate"]
date_ax, date_ax_str = get_date_ax(start, end)
N = len(date_ax_str)
skipping_date_str = []
accumulate_data = []
for i, day in enumerate(date_ax_str):
print("{}/{}: {}".format(i + 1, N, day))
year = day[2:4]
month = day[4:6]
#ファイル名の生成
wind_file_name = "../data/wind_data/ecm" + day[2:] + ".csv"
ice_file_name = "../data/ice_wind_data/" + day[2:] + ".csv"
ic0_145_file_name = "../data/IC0_csv/2" + day + "A.csv"
ic0_900_file_name = "../data/IC0_csv/2" + day + "A.csv"
coeff_file_name = "../data/A_csv/ssc_amsr_ads" + str(year) + str(
month) + "_" + str(span) + "_fin.csv"
wind10m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
t2m_file_name = "../data/netcdf4/" + day[2:] + ".csv"
if not all([
os.path.isfile(wind_file_name),
os.path.isfile(ice_file_name),
os.path.isfile(ic0_145_file_name),
os.path.isfile(coeff_file_name)
]):
print("\tSkipping " + day + " file...")
date_ax_str.remove(day)
date_ax.remove(datetime(day[:4] + "-" + day[4:6] + "-" + day[6:]))
skipping_date_str.append(day)
continue
data = pd.DataFrame({"data_idx": np.zeros(145 * 145)})
iw_idx_t, ic0_idx_t = np.array([-1]), np.array([-1])
if "ex_1" in get_columns:
tmp = calc_data.get_w_regression_data(wind_file_name,
ice_file_name,
coeff_file_name)
iw_idx_t = np.array(tmp["data_idx"])
data = pd.concat([data, tmp.loc[:, ["A_by_day", "theta_by_day"]]],
axis=1)
if "w" in get_columns:
tmp = calc_data.get_1day_w_data(wind_file_name)
data = pd.concat([data, tmp], axis=1)
if "iw" in get_columns:
tmp = calc_data.get_1day_ice_data(ice_file_name)
iw_idx_t = np.array(tmp["iw_idx_t"])
data = pd.concat([data, tmp.loc[:, ["iw_u", "iw_v", "iw_speed"]]],
axis=1)
if "ic0_145" in get_columns:
tmp = calc_data.get_1day_ic0_data(ic0_file_name,
grid900to145_file_name)
ic0_idx_t = np.array(tmp["ic0_idx_t"])
data = pd.concat([data, tmp.loc[:, ["ic0_145"]]], axis=1)
if "coeff" in get_columns:
tmp = calc_data.get_1month_coeff_data(coeff_file_name)
data = pd.concat([data, tmp], axis=1)
if "w10m" in get_columns:
tmp = calc_data.get_1day_w10m_data(wind10m_file_name)
data = pd.concat([data, tmp], axis=1)
if "t2m" in get_columns:
tmp = calc_data.get_1day_t2m_data(t2m_file_name)
data = pd.concat([data, tmp], axis=1)
mat = iw_idx_t.ravel().tolist() + ic0_idx_t.ravel().tolist()
data_t_idx = calc_data.get_non_nan_idx(mat, ocean_idx, strict=True)
data.loc[data.data_idx, data_t_idx] = 1
data = calc_data.get_masked_region_data(data, region)
if ("coeff" in get_columns) and (len(get_columns) == 1):
print("\tSelected only coeff data. Getting out of the loop...")
continue
if accumulate == True:
data = data.loc[:, get_columns]
print("\t{}".format(data.columns))
accumulate_data.append(np.array(data))
if accumulate == True:
print("accumulate: True\tdata type: array")
return date_ax, date_ax_str, skipping_date_str, accumulate_data
else:
print("accumulate: False\tdata type: DataFrame")
return date_ax, date_ax_str, skipping_date_str, data
def test_w_iw_by_year():
dirs_1 = "../result_h/test/w_iw_by_year/"
dirs_2 = "../result_h/test/w_iw_by_year_without_ocean/"
if not os.path.exists(dirs_1):
os.makedirs(dirs_1)
if not os.path.exists(dirs_2):
os.makedirs(dirs_2)
y_list = [
"03", "04", "05", "06", "07", "08", "09", "10", "13", "14", "15", "16"
]
month_list = [
"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"
]
for month in month_list:
print("************" + month + "************")
helmert_30_30_fname = "../data/csv_Helmert_both_30/Helmert_both_30_2003" + month + ".csv"
data_30 = pd.read_csv(helmert_30_30_fname)
#area_16_index = np.array(data_30.loc[data_30.area_label==16, :].index)
area_16_index = np.array(data_30[(data_30.area_label.isin(
list(range(17))))].dropna().index).tolist()
plot_grids = random.sample(area_16_index, 15)
print(plot_grids)
gw_list = []
iw_list = []
iw_ocean_list = []
for year in y_list:
helmert_file_name = "../data/csv_Helmert_both_30/Helmert_both_30_20" + year + month + ".csv"
gw_file_name = "../data/csv_w/ecm" + year + month + "15.csv"
iw_file_name = "../data/csv_iw/" + year + month + "15.csv"
helmert_data = calc_data.get_1month_helmert_data(helmert_file_name)
gw_data = calc_data.get_1day_w_data(gw_file_name)
iw_data = calc_data.get_1day_iw_data(iw_file_name)
gw_speed = np.array(gw_data.loc[plot_grids, "w_speed"])
iw_speed = np.array(iw_data.loc[plot_grids, "iw_speed"])
iw_ocean_u = np.array(iw_data.loc[plot_grids, "iw_u"] -
helmert_data.loc[plot_grids, "ocean_u"])
iw_ocean_v = np.array(iw_data.loc[plot_grids, "iw_v"] -
helmert_data.loc[plot_grids, "ocean_v"])
iw_speed_ocean = np.sqrt(iw_ocean_u**2 + iw_ocean_v**2)
gw_list.append(gw_speed)
iw_list.append(iw_speed)
iw_ocean_list.append(iw_speed_ocean)
gw_array = np.array(gw_list)
iw_array = np.array(iw_list)
iw_ocean_array = np.array(iw_ocean_list)
for i in range(len(plot_grids)):
gw = gw_array[:, i]
iw = iw_array[:, i]
iw_ocean = iw_ocean_array[:, i]
save_name_1 = dirs_1 + month + "15_grid_" + str(
plot_grids[i]) + ".png"
save_name_2 = dirs_2 + month + "15_grid_" + str(
plot_grids[i]) + ".png"
try:
sns.set_style("darkgrid")
sns.jointplot(x=gw, y=iw, kind="reg")
plt.savefig(save_name_1)
plt.close()
sns.set_style("darkgrid")
sns.jointplot(x=gw, y=iw_ocean, kind="reg")
plt.savefig(save_name_2)
plt.close()
except:
continue