def label_bad_samples():
file_name = "samples/bad_samples.csv"
with open(file_name, 'w', newline='') as csvfile:
writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['file, label'])
vel_dir = "samples/bad/vel/"
if not os.path.exists(vel_dir):
os.makedirs(vel_dir)
for file_name in os.listdir("samples/bad"):
arctic_file = file_name.split(".")[0]
for x in range(0, 1100, 100):
for y in range(0, 400, 100):
u_file_name = "samples/bad/" + arctic_file + "_vozocrtx_" + str(
x) + "_" + str(y)
v_file_name = "samples/bad/" + arctic_file + "_vomecrty_" + str(
x) + "_" + str(y)
u = img.load_square_from_file(u_file_name)
v = img.load_square_from_file(v_file_name)
u[u < -30000.0] = 0.0
v[v < -30000.0] = 0.0
vel = calculate_velocity_magnitude_matrix(u, v)
show_velocity_square(vel)
velocity_file_name = vel_dir + arctic_file + "_" + str(
x) + "_" + str(y)
img.save_square_to_file(vel, velocity_file_name)
result = bad_samples[len(bad_samples) - 1]
writer.writerow([velocity_file_name, result])
def get_min_max_of_dataset(dataset_file_name):
min_vel = 1000.0
max_vel = -1000.0
with open(dataset_file_name, 'r', newline='') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
square = img.load_square_from_file(row[0])
min_vel = min([min_vel, np.min(square)])
max_vel = max([max_vel, np.max(square)])
return min_vel, max_vel
def data_generator(samples, batch_size):
while 1:
for sample_index in range(0, len(samples), batch_size):
x = np.zeros((batch_size, 100, 100, 1), dtype=np.float32)
y = np.zeros((batch_size,))
for index in range(sample_index, sample_index + batch_size):
file_name = samples[index][0]
square = image.load_square_from_file(file_name)
expanded = np.expand_dims(square, axis=2)
x[index - sample_index] = expanded
y[index - sample_index] = samples[index][1]
yield (x, y)
def label_good_samples():
file_name = "samples/good_samples.csv"
with open(file_name, 'w', newline='') as csvfile:
writer = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
writer.writerow(['file, label'])
images_amount = 30
vel_dir = "samples/good/"
if not os.path.exists(vel_dir):
os.makedirs(vel_dir)
for image_index in range(0, images_amount):
square_index = 1
for x in range(0, 1100, 100):
for y in range(0, 400, 100):
u_file_name = "samples/out/rea" + str(
image_index) + "_u_" + str(x) + "_" + str(y)
v_file_name = "samples/out/rea" + str(
image_index) + "_v_" + str(x) + "_" + str(y)
u = img.load_square_from_file(u_file_name)
v = img.load_square_from_file(v_file_name)
# remove nan-values
u[u < -30000.0] = 0.0
v[v < -30000.0] = 0.0
vel = calculate_velocity_magnitude_matrix(u, v)
velocity_file_name = vel_dir + "rea" + str(
image_index) + "_" + str(x) + "_" + str(y)
img.save_square_to_file(vel, velocity_file_name)
writer.writerow([velocity_file_name, '0'])
print("squares: " + str(square_index) + "/44 done")
square_index += 1
print("images: " + str(image_index + 1) + "/" +
str(images_amount) + " done")
def show_average_vel_by_squares(dataset_file_name):
with open(dataset_file_name, 'r', newline='') as csvfile:
reader = csv.reader(csvfile)
groups = group_squares(reader)
for index in range(0, len(groups)):
squares_vel = np.zeros((len(groups[index]), 100, 100), dtype=float)
for sample_index in range(0, len(groups[index])):
square = img.load_square_from_file(
groups[index][sample_index][0])
squares_vel[sample_index] = square
average_vel = np.average(squares_vel, axis=0)
print(average_vel)
plt.imshow(average_vel)
plt.colorbar()
plt.show()
def init_velocity_field(nc_file_name):
nc = NetCDFFile(nc_file_name, 'r+')
lat = nc.variables['nav_lat_grid_U'][:]
lon = nc.variables['nav_lon_grid_U'][:]
nc_name = nc_file_name.split("/")[4].split(".")[0]
velocity = np.zeros((400, 1100), dtype=np.float32)
vel_dir = "samples/bad/vel/"
for file_name in os.listdir(vel_dir):
if nc_name in file_name:
square = img.load_square_from_file(vel_dir +
file_name.split(".")[0])
square_index = data.extract_square_index(vel_dir +
file_name.split(".")[0])
x = int(square_index.split("_")[0])
y = int(square_index.split("_")[1])
velocity[y:y + 100, x:x + 100] = square
nc.close()
lat_left_bottom = lat[-1][-1]
lon_left_bottom = lon[-1][-1]
lat_right_top = lat[0][0]
lon_right_top = lon[0][0]
lat_center = 90
# 110, 119
lon_center = 110
m = Basemap(projection='stere',
lon_0=lon_center,
lat_0=lat_center,
resolution='l',
llcrnrlat=lat_left_bottom,
llcrnrlon=lon_left_bottom,
urcrnrlat=lat_right_top,
urcrnrlon=lon_right_top)
m.pcolormesh(lon, lat, velocity, latlon=True, cmap='RdYlBu_r', vmax=0.6)
m.drawcoastlines()
m.drawcountries()
m.fillcontinents(color='#cc9966', lake_color='#99ffff')
ax = plt.gca()
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(cax=cax, label="Sea current velocity")
return ax, lat, lon, m
def add_characteristics_of_samples(dataset_file_name):
rows = []
with open(dataset_file_name, 'r', newline='') as old_csv_file:
reader = csv.reader(old_csv_file, delimiter=',')
for row in reader:
rows.append(row)
# Add min && max && average values of samples
for row in rows:
square = img.load_square_from_file(row[0])
row.append(np.min(square))
row.append(np.max(square))
row.append(np.average(square))
with open(dataset_file_name, 'w', newline='') as new_csv_file:
writer = csv.writer(new_csv_file, delimiter=',')
for row in rows:
writer.writerow(row)
def count_outlier_squares(dataset_file_name):
with open(dataset_file_name, 'r', newline='') as csvfile:
reader = csv.reader(csvfile)
square_by_count = {}
max_900_0 = 0.0
for row in reader:
idx = extract_square_index(row[0])
if row[1] == "1":
print(str(idx) + " " + row[3])
if idx in square_by_count:
square_by_count[idx] += 1
else:
square_by_count[idx] = 1
if idx == "100_0" and row[1] == "1":
max_900_0 = max(max_900_0, float(row[3]))
square = img.load_square_from_file(row[0])
plt.figure(figsize=(10, 8))
plt.imshow(square.transpose())
plt.title("Sample: " + row[0] + " with label: " + row[1])
plt.colorbar()
plt.show()
print(square_by_count)
print(max_900_0)
def draw_map(nc_file_name):
nc = NetCDFFile(nc_file_name, 'r+')
lat = nc.variables['nav_lat_grid_U'][:]
lon = nc.variables['nav_lon_grid_U'][:]
nc_name = nc_file_name.split("/")[4].split(".")[0]
velocity = np.zeros((400, 1100), dtype=np.float32)
vel_dir = "samples/bad/vel/"
for file_name in os.listdir(vel_dir):
if nc_name in file_name:
square = img.load_square_from_file(vel_dir +
file_name.split(".")[0])
print(file_name)
print(str(np.min(square)) + "; " + str(np.max(square)))
square_index = data.extract_square_index(vel_dir +
file_name.split(".")[0])
x = int(square_index.split("_")[0])
y = int(square_index.split("_")[1])
print(str(x) + " " + str(y))
velocity[y:y + 100, x:x + 100] = square
nc.close()
lat_left_bottom = lat[-1][-1]
lon_left_bottom = lon[-1][-1]
lat_right_top = lat[0][0]
lon_right_top = lon[0][0]
lat_center = 90
# 110, 119
lon_center = 110
m = Basemap(projection='stere',
lon_0=lon_center,
lat_0=lat_center,
resolution='l',
llcrnrlat=lat_left_bottom,
llcrnrlon=lon_left_bottom,
urcrnrlat=lat_right_top,
urcrnrlon=lon_right_top)
m.pcolormesh(lon, lat, velocity, latlon=True, cmap='RdYlBu_r', vmax=0.6)
m.drawcoastlines()
m.drawcountries()
m.fillcontinents(color='#cc9966', lake_color='#99ffff')
ax = plt.gca()
# ax.tick_params(labelsize=10)
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(cax=cax, label="Sea current velocity")
# ax = plt.gca()
# divider = make_axes_locatable(ax)
# cax = divider.append_axes("right", size="5%", pad=0.05)
# plt.colorbar().set_label(label='Sea current velocity', size=15)
# with open("samples/valid_samples.csv", 'r', newline='') as csvfile:
# samples = []
#
# reader = csv.reader(csvfile, delimiter=',')
# for row in reader:
# if "samples/bad/vel/" + nc_name in row[0]:
# print(row)
# samples.append(row)
# square_index = data.extract_square_index(row[0])
#
# x = int(square_index.split("_")[0])
# y = int(square_index.split("_")[1])
#
# if (x >= 100) and (x < 1000) and (y < 300):
#
# sample = np.zeros((1, 100, 100, 1), dtype=np.float32)
# square = np.expand_dims(img.load_square_from_file(row[0]), axis=2)
# sample[0] = square
# lat_poly = np.array([lat[y][x], lat[y][x + 99], lat[y + 99][x + 99], lat[y + 99][x]])
# lon_poly = np.array([lon[y][x], lon[y][x + 99], lon[y + 99][x + 99], lon[y + 99][x]])
# mapx, mapy = m(lon_poly, lat_poly)
# points = np.zeros((4, 2), dtype=np.float32)
# for j in range(0, 4):
# points[j][0] = mapx[j]
# points[j][1] = mapy[j]
# poly = Polygon(points, color='black', fill=False, linewidth=3)
# ax.add_patch(poly)
# plt.savefig("levels" + "_!!!.png", dpi=500)
plt.show()
def draw_velocity_map(nc_file_name):
nc = NetCDFFile(nc_file_name, 'r+')
lat = nc.variables['nav_lat_grid_U'][:]
lon = nc.variables['nav_lon_grid_U'][:]
vel_dir = "samples/bad/vel/"
nc_name = nc_file_name.split("/")[4].split(".")[0]
velocity = np.zeros((400, 1100), dtype=np.float32)
for file_name in os.listdir(vel_dir):
if nc_name in file_name:
square = img.load_square_from_file(vel_dir +
file_name.split(".")[0])
print(file_name)
print(str(np.min(square)) + "; " + str(np.max(square)))
square_index = data.extract_square_index(vel_dir +
file_name.split(".")[0])
x = int(square_index.split("_")[0])
y = int(square_index.split("_")[1])
print(str(x) + " " + str(y))
velocity[y:y + 100, x:x + 100] = square
nc.close()
lat_left_bottom = lat[-1][-1]
lon_left_bottom = lon[-1][-1]
lat_right_top = lat[0][0]
lon_right_top = lon[0][0]
lat_center = 90
# 110, 119
lon_center = 110
m = Basemap(projection='stere',
lon_0=lon_center,
lat_0=lat_center,
resolution='l',
llcrnrlat=lat_left_bottom,
llcrnrlon=lon_left_bottom,
urcrnrlat=lat_right_top,
urcrnrlon=lon_right_top)
m.pcolormesh(lon, lat, velocity, latlon=True, cmap='RdYlBu_r', vmax=0.6)
m.drawcoastlines()
m.drawcountries()
m.fillcontinents(color='#cc9966', lake_color='#99ffff')
# plt.rcParams.update({'font.size': 22})
ax = plt.gca()
# ax.tick_params(labelsize=10)
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(cax=cax, label="Sea current velocity")
# plt.title("Anomalies detection results for: " + "20 March, 2013")
model = load_model("samples/current_model/model.h5")
valid_squares = [[*list(range(1, 10))]]
with open("samples/valid_samples.csv", 'r', newline='') as csvfile:
samples = []
reader = csv.reader(csvfile, delimiter=',')
for row in reader:
if "samples/bad/vel/" + nc_name in row[0]:
print(row)
samples.append(row)
square_index = data.extract_square_index(row[0])
x = int(square_index.split("_")[0])
y = int(square_index.split("_")[1])
if (x >= 100) and (x < 1000) and (y < 300):
sample = np.zeros((1, 100, 100, 1), dtype=np.float32)
square = np.expand_dims(img.load_square_from_file(row[0]),
axis=2)
sample[0] = square
result = model.predict(sample)
result_x, result_y = m(lon[y + 50][x + 50],
lat[y + 50][x + 50])
ax.text(result_x,
result_y,
"%0.3f" % result[0][0],
ha='center',
size=7,
color="yellow",
path_effects=[
PathEffects.withStroke(linewidth=3,
foreground='black')
])
if row[1] == "1":
print("outlier!")
lat_poly = np.array([
lat[y][x], lat[y][x + 99], lat[y + 99][x + 99],
lat[y + 99][x]
])
lon_poly = np.array([
lon[y][x], lon[y][x + 99], lon[y + 99][x + 99],
lon[y + 99][x]
])
mapx, mapy = m(lon_poly, lat_poly)
points = np.zeros((4, 2), dtype=np.float32)
for j in range(0, 4):
points[j][0] = mapx[j]
points[j][1] = mapy[j]
if result[0][0] > 0.5:
poly = Polygon(points,
color='green',
fill=False,
linewidth=3)
ax.add_patch(poly)
else:
poly = Polygon(points,
color='red',
fill=False,
linewidth=3)
ax.add_patch(poly)
else:
if result[0][0] > 0.5:
lat_poly = np.array([
lat[y][x], lat[y][x + 99], lat[y + 99][x + 99],
lat[y + 99][x]
])
lon_poly = np.array([
lon[y][x], lon[y][x + 99], lon[y + 99][x + 99],
lon[y + 99][x]
])
mapx, mapy = m(lon_poly, lat_poly)
points = np.zeros((4, 2), dtype=np.float32)
for j in range(0, 4):
points[j][0] = mapx[j]
points[j][1] = mapy[j]
poly = Polygon(points,
color='red',
fill=False,
linewidth=3)
ax.add_patch(poly)
print(result)
# plt.show()
red = Patch(color='red', label='Error')
green = Patch(color='green', label='Correct')
plt.legend(loc='lower right',
fontsize='medium',
bbox_to_anchor=(1, 1),
handles=[green, red])
plt.savefig("test" + "_bad_result.png", dpi=500)