def pdf_on_bad():
data_dir = "D:/ice_recovered_from_hybrid/ice_tests/bad/"
sq3 = []
sq9 = []
for nc_file in glob.iglob(data_dir + "*/" + "*201309*.nc", recursive=True):
idx = 0
for y in range(0, IMAGE_SIZE['y'], SQUARE_SIZE):
for x in range(0, IMAGE_SIZE['x'], SQUARE_SIZE):
if is_inside(x, y):
if idx == 3:
nc = NCFile(nc_file)
conc = nc.variables['iceconc'][:][0]
avg = np.average(conc[y:y + 100, x:x + 100])
sq3.append(avg)
nc.close()
if idx == 9:
nc = NCFile(nc_file)
nc.variables['iceconc'][:][0]
avg = np.average(conc[y:y + 100, x:x + 100])
sq9.append(avg)
nc.close()
idx += 1
pdf = []
for i in range(len(sq3)):
params_x, params_y = load_dist_params("sept_dist_another")
pdf.append(
join_distribution(params_x=params_x,
params_y=params_y,
x=sq3[i],
y=sq9[i]))
pdf_bad_avg = np.average(pdf)
print(pdf_bad_avg)
def big_grid():
data = read_samples("samples/ice_samples.csv")
train, test = split_data(data.samples, 0.2)
# train_middle = int(len(train) / 250)
# test_middle = int(len(test) / 100)
# train_batch_size = calc_batch_size(len(train), int(train_middle - 0.5 * train_middle),
# int(train_middle + 0.5 * train_middle))
# test_batch_size = calc_batch_size(len(test), int(test_middle - 0.5 * test_middle),
# int(test_middle + 0.5 * test_middle))
train_batch_size = 50
test_batch_size = 80
print(len(train))
print(len(test))
print(train_batch_size)
print(test_batch_size)
train_idx = []
for sample in train:
train_idx.append(sample.index - 1)
train_idx = keras.utils.to_categorical(train_idx, 44)
test_idx = []
for sample in test:
test_idx.append(sample.index - 1)
test_idx = keras.utils.to_categorical(test_idx, 44)
tr_samples = []
for idx in range(len(train)):
tr_samples.append([train[idx], train_idx[idx]])
tt_samples = []
for idx in range(len(test)):
tt_samples.append([test[idx], test_idx[idx]])
mask_file = NCFile("samples/bathy_meter_mask.nc")
coastline_mask = mask_file.variables['Bathymetry'][:]
mask_file.close()
epochs = 60
container = VarsContainer()
model = init_model()
history = AccuracyHistory()
model.fit_generator(data_generator(tr_samples, train_batch_size, container, coastline_mask),
steps_per_epoch=train_batch_size,
callbacks=[history],
epochs=epochs)
model.save("samples/model.h5")
# model = load_model("samples/model.h5")
# scores = model.predict_generator(data_generator(tt_samples, test_batch_size, d),
# steps=int(len(test) / test_batch_size))
t = model.evaluate_generator(data_generator(tt_samples, test_batch_size, container, coastline_mask),
steps=int(len(test) / test_batch_size))
print(t)
def values(self, file_name):
if file_name in self.files_by_counts:
self.files_by_counts[file_name] += 1
else:
print(file_name + " was loaded")
self.files_by_counts[file_name] = 1
nc = NCFile(file_name)
# TODO: this should be params list-like
if "satellite" in file_name:
# TODO: check .filled and fix this anywhere
conc = nc.variables['ice_conc'][:].filled(0) / 100.0
thic = np.empty((1, 400, 100), np.float32)
else:
conc = nc.variables['iceconc'][:]
thic = nc.variables['icethic_cea'][:]
conc = reduce_conc(conc)
self.conc_dic[file_name] = conc
self.thic_dic[file_name] = thic
if self.files_by_counts[file_name] == self.limit:
conc_tmp = self.conc_dic[file_name]
thic_tmp = self.thic_dic[file_name]
del self.files_by_counts[file_name]
del self.conc_dic[file_name]
del self.thic_dic[file_name]
return conc_tmp, thic_tmp
else:
return self.conc_dic[file_name], self.thic_dic[file_name]
def load_mask():
mask_file = NCFile("bathy_meter_mask.nc")
mask = mask_file.variables['Bathymetry'][:]
mask_file.close()
mask = 1 - mask
return mask
def avg_of_square(dataset_file):
dataset = Dataset.from_csv(dataset_file)
avg = []
for sample in dataset.samples:
if dataset.samples.index(sample) % 50 == 0:
print("%d/%d done" %
(dataset.samples.index(sample), len(dataset.samples)))
nc = NCFile(sample.nc_file)
conc = nc.variables['ice_conc'][:].filled(0) / 100.0
conc_square = sample.ice_conc(conc)
avg.append(np.average(conc_square))
return avg
def check_month():
in_dir = "samples/conc_satellite/"
out_dir = "samples/aug_check/"
samples = []
for file in glob.iglob(in_dir + "*/" + "08" + "/*0814*.nc",
recursive=True):
samples.append(file)
for sample in samples:
year = sample.split("/")[2]
print(year)
nc = NCFile(sample)
lat = nc.variables['nav_lat'][:]
lon = nc.variables['nav_lon'][:]
conc = nc.variables['ice_conc'][:].filled(0) / 100.0
conc = conc[0]
nc.close()
lat_left_bottom = lat[-1][-1]
lon_left_bottom = lon[-1][-1]
lat_right_top = lat[0][0]
lon_right_top = lon[0][0]
lat_center = 90
lon_center = 110
m = Basemap(projection='stere',
lon_0=lon_center,
lat_0=lat_center,
resolution='l',
llcrnrlat=lat_left_bottom,
llcrnrlon=lon_left_bottom,
urcrnrlat=lat_right_top,
urcrnrlon=lon_right_top)
m.pcolormesh(lon, lat, conc, latlon=True, cmap='RdYlBu_r', vmax=1)
m.drawcoastlines()
m.drawcountries()
m.fillcontinents(color='#cc9966', lake_color='#99ffff')
ax = plt.gca()
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(cax=cax, label="Ice concentration")
plt.savefig(out_dir + year + ".png", dpi=500)
def conditional_probs(dataset_file):
dataset = Dataset.from_csv(dataset_file)
x_idx, y_idx = 3, 9
intervals = np.linspace(0, 1, 11)
x_intervals = [(intervals[idx], intervals[idx + 1])
for idx in range(0,
len(intervals) - 1)]
samples_grouped = dict()
for sample in dataset.samples:
if sample.index in (x_idx, y_idx):
nc = NCFile(sample.nc_file)
conc = nc.variables['ice_conc'][:].filled(0) / 100.0
avg = np.average(sample.ice_conc(conc))
if sample.nc_file not in samples_grouped:
samples_grouped[sample.nc_file] = dict()
samples_grouped[sample.nc_file][sample.index] = avg
else:
samples_grouped[sample.nc_file][sample.index] = avg
x_count = []
y_count = [[] for _ in range(len(x_intervals))]
for file in samples_grouped.keys():
x_value = samples_grouped[file][x_idx]
x_count.append(x_value)
y_value = samples_grouped[file][y_idx]
interval = interval_idx(x_value, x_intervals)
y_count[interval].append(y_value)
x_dist_params = best_fit_distribution(x_count)
y_dist_params = []
for idx in range(len(y_count)):
y_dist_params.append(best_fit_distribution(y_count[idx]))
return x_dist_params, y_dist_params
def check_filter():
file_name = "samples/conc_satellite/2013/08/ice_conc_nh_ease2-250_cdr-v2p0_201308181200.nc"
nc = NCFile(file_name)
lat = nc.variables['nav_lat'][:]
lon = nc.variables['nav_lon'][:]
conc = nc.variables['ice_conc'][:].filled(0) / 100.0
conc = conc[0]
conc = reduce_conc(conc)
initial_conc = conc
nc.close()
for filter in range(1, 2, 1):
conc = uniform_filter(initial_conc, filter)
lat_left_bottom = lat[-1][-1]
lon_left_bottom = lon[-1][-1]
lat_right_top = lat[0][0]
lon_right_top = lon[0][0]
lat_center = 90
# 110, 119
lon_center = 110
m = Basemap(projection='stere',
lon_0=lon_center,
lat_0=lat_center,
resolution='l',
llcrnrlat=lat_left_bottom,
llcrnrlon=lon_left_bottom,
urcrnrlat=lat_right_top,
urcrnrlon=lon_right_top)
m.pcolormesh(lon, lat, conc, latlon=True, cmap='RdYlBu_r', vmax=1)
ax = plt.gca()
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(cax=cax, label="Ice concentration")
plt.savefig("samples/filters/" + str(filter) + ".png", dpi=500)