def clusters_basic(ax1,year,var,noclust,cl_this, colors, legend = True, markersize = 20, legfontsize = 12 ):
import pickle
import numpy as np
import cmocean as cm
import sys
sys.path.append('./extraction_scripts')
import map_fxn as mf
from salishsea_tools import (
viz_tools)
bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
tpkl = f'./pkls/{var}_clustmat_{year}.pkl'
cl = pickle.load(open(tpkl, 'rb'))
viz_tools.set_aspect(ax1)
fmask = (grid.fmask[0,0,:,:])
mesh = ax1.pcolormesh(fmask, vmin=0, vmax=1, cmap = cm.cm.deep)
ax1.set_ylim([0,898])
ax1.set_xlim([0,398])
stn_x, stn_y = mf.make_stns(10)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x,stn_y,fmask)
d_stn_xar = np.array(d_stn_x)
d_stn_yar = np.array(d_stn_y)
edge = np.where((d_stn_xar<=10) | (d_stn_yar>=888))
np.squeeze(edge)
np.shape(edge)
edgear = np.array(edge)
edgear = edgear[0]
d_stn_xar =np.delete(d_stn_xar,edge)
d_stn_yar = np.delete(d_stn_yar,edge)
for j in range(1,noclust+1):
cluster = np.where(cl_this == j)
cluster = np.squeeze(cluster)
c1_x = np.take(d_stn_xar,cluster)
c1_y = np.take(d_stn_yar,cluster)
pts = ax1.scatter(c1_x,c1_y,s=markersize,c=colors[j], label=str(j), marker='o')
ax1.set_xticklabels( () )
ax1.set_yticklabels( () )
tit = 'year ' + year # + ' \n n. clusters = '+ str(noclust)
if legend:
ax1.legend(bbox_to_anchor=(1.06, 1), fontsize = legfontsize)
ax1.set_title(tit,fontsize = legfontsize + 2)
tdate = arrow_array1[i][0]
ymd = tdate.format('YYYYMMDD')
tstr = (str_to_C + ymd + '*carp_T.nc') #'*carp_T.nc'
#print(tstr)
tstr = glob.glob(str_to_C + ymd + '*carp_T.nc')
#print(tstr)
#print(tstr)
carpT.append(tstr[0])
bath = '/data/tjarniko/MEOPAR/grid/mesh_mask201702.nc'
grid = mf.import_bathy(bath)
fmask = (grid.fmask[0, 0, :, :])
spacing = 10
stn_x, stn_y = mf.make_stns(spacing)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask)
print('complete')
year = '2013'
LL = 0.1
def getVEDEuph(ii, jj, LL, fileK, filePAR):
#print(filePAR)
# print(ii)
# print(jj)
dailyPAR = np.mean(filePAR.variables['PAR'][:, :, jj, ii], 0)
kk = np.sum(dailyPAR > LL * dailyPAR[0])
#print(fileK)
kprof = (fileK.variables['vert_eddy_diff'][:, :, jj, ii])
#print(kprof)
def bio_de(spacing, stn_start, stn_end, year):
import arrow
import time
import xarray as xr
import numpy as np
import map_fxn as mf
import warnings
warnings.filterwarnings("ignore")
if year == 2016:
noday = 366
else:
noday = 365
day_start = 0
day_end = noday
print('Spacing between stations: ' + str(spacing))
print('First day: ' + str(day_start))
print('Last day: ' + str(day_end))
print('testing reload')
print(year)
dirname = '/data/tjarniko/MEOPAR/analysis_tereza/notebooks/CLUSTER_PAPER/CLEAN/NC_HINDCAST/' + str(
year) + '/BIO_TS/'
bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
fmask = (grid.fmask[0, 0, :, :])
stn_x, stn_y = mf.make_stns(spacing)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask)
no_stns = len(d_stn_x)
# a list of all the model outputs (tracers) for this year
trace_list = mf.create_tracelist(year)
print("Number of stations:" + str(no_stns))
stn_names = list()
for i in range(0, no_stns):
stn_name = 'stn_' + str(i)
stn_names.append(stn_name)
#print(stn_name)
for n in range(stn_start, stn_end):
print('stn is: ' + str(n))
start_time = time.time()
MYRI_ar = np.zeros([noday, 40])
MICZ_ar = np.zeros([noday, 40])
PHY2_ar = np.zeros([noday, 40])
PHY_ar = np.zeros([noday, 40])
for i in range(day_start, day_end):
if (i % 20 == 0):
print('day is:' + str(i))
b = i
e = i + 1
start_time = time.time()
trc = (trace_list[i])
nemo = xr.open_mfdataset(trc)
t1 = b
t2 = e
x1 = d_stn_x[n]
#print(x1)
x2 = d_stn_x[n] + 1
y1 = d_stn_y[n]
#print(y1)
y2 = d_stn_y[n] + 1
z1 = 0
z2 = 40
MYRI_d, MICZ_d, PHY2_d, PHY_d = int_phyt_fxn(
grid, nemo, t1, t2, x1, x2, y1, y2, z1, z2)
#print(np.shape(PHY_d))
MYRI_ar[i, :] = (MYRI_d[0, :, 0, 0])
MICZ_ar[i, :] = (MICZ_d[0, :, 0, 0])
PHY2_ar[i, :] = (PHY2_d[0, :, 0, 0])
#print(PHY2_d[0,:,0,0])
PHY_ar[i, :] = (PHY_d[0, :, 0, 0])
jan = xr.Dataset({
'MYRI': (['t', 'z'], MYRI_ar),
'MICZ': (['t', 'z'], MICZ_ar),
'PHY2': (['t', 'z'], PHY2_ar),
'PHY': (['t', 'z'], PHY_ar)
})
stn_name = dirname + stn_names[n] + '_DP_sp' + str(spacing) + '.nc'
#print(stn_name)
jan.to_netcdf(stn_name)
def clusters(ax1, year, var, no_clusters, markersize=20, legfontsize=12):
colors = [
'deepskyblue', 'red', 'goldenrod', 'forestgreen', 'midnightblue',
'orchid', 'gray', 'peru', 'olive', 'sandybrown', 'teal', 'pink', 'tan',
'yellow', 'thistle'
]
import pickle
import numpy as np
import cmocean as cm
import sys
sys.path.append('./extraction_scripts')
import map_fxn as mf
from salishsea_tools import (viz_tools)
bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
tpkl = f'./pkls/{var}_clustmat_{year}.pkl'
cl = pickle.load(open(tpkl, 'rb'))
cl_this = cl[no_clusters - 1, :]
np.shape(cl_this)
viz_tools.set_aspect(ax1)
fmask = (grid.fmask[0, 0, :, :])
mesh = ax1.pcolormesh(fmask, vmin=0, vmax=1, cmap=cm.cm.deep)
ax1.set_ylim([0, 898])
ax1.set_xlim([0, 398])
stn_x, stn_y = mf.make_stns(10)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask)
d_stn_xar = np.array(d_stn_x)
d_stn_yar = np.array(d_stn_y)
edge = np.where((d_stn_xar <= 10) | (d_stn_yar >= 888))
np.squeeze(edge)
np.shape(edge)
edgear = np.array(edge)
edgear = edgear[0]
d_stn_xar = np.delete(d_stn_xar, edge)
d_stn_yar = np.delete(d_stn_yar, edge)
#print(np.size(d_stn_xar))
###sort clusters by size
cluster_ids = np.arange(1, no_clusters + 1, 1)
cluster_sizes = np.zeros_like(cluster_ids)
#retrieve cluster sizes
for j in range(1, no_clusters + 1):
cluster = np.where(cl_this == j)
cluster = np.squeeze(cluster)
cluster_sizes[j - 1] = (np.size(cluster))
#sort cluster size matrix biggest to smallest
cs = np.argsort(-cluster_sizes)
#print(cs)
#use those sizes to sort the cluster id list to corresponde to a list 'clust id, largest to smallest'
new_cidlist = np.zeros_like(cluster_ids)
for j in range(0, len(new_cidlist)):
new_cidlist[j] = cluster_ids[cs[j]]
#start plotting, plotting biggest cluster first, to keep colour order the same
for j in range(0, np.size(new_cidlist)):
cluster = np.where(cl_this == new_cidlist[j])
cluster = np.squeeze(cluster)
c1_x = np.take(d_stn_xar, cluster)
c1_y = np.take(d_stn_yar, cluster)
pts = ax1.scatter(c1_x,
c1_y,
s=markersize,
c=colors[j],
label=str(new_cidlist[j]),
marker='o')
ax1.set_xticklabels(())
ax1.set_yticklabels(())
tit = var + ', year: ' + year + ' \n n. clusters = ' + str(no_clusters)
ax1.legend(bbox_to_anchor=(1.1, 1), fontsize=legfontsize)
ax1.set_title(tit, fontsize=legfontsize + 2)
def yearhalo_de(spacing,stn_b,stn_e,year):
import map_fxn as mf
import time
import xarray as xr
import numpy as np
print('Spacing between stations: ' + str(spacing))
bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
fmask = (grid.fmask[0,0,:,:])
stn_x, stn_y = mf.make_stns(spacing)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x,stn_y,fmask)
no_stns = len(d_stn_x)
monlist = ['jan','feb','mar','apr','may','jun','jul','aug','sep',
'oct','nov','dec']
if year == 2016:
daylist = [31,29,31,30,31,30,31,31,30,31,30,31]
noday = 366
else:
daylist = [31,28,31,30,31,30,31,31,30,31,30,31]
noday = 365
print(year)
print(noday)
# a list of all the model outputs (tracers) for this year
trace_list = mf.create_physlist(year)
print("Number of stations:" + str(no_stns))
bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
fmask = (grid.fmask[0,0,:,:])
for stn in range(stn_b,stn_e):
print('station is: ' ,str(stn))
print('x is :', d_stn_x[stn])
print('y is :', d_stn_y[stn])
ts_x = d_stn_x[stn]
ts_y = d_stn_y[stn]
daily_halocline = np.zeros(noday)
end_day = noday
for day in range(1,end_day+1):
if (day%20 ==0) :
print(day)
trc = trace_list[day-1]
#print(trc)
nemo = xr.open_mfdataset(trc)
halo = halo_de(nemo,grid,ts_x,ts_y)
q = isinstance(halo, (np.ndarray))
if (q == True) :
print('found an array!')
daily_halocline[day-1] = halo[0]
else:
daily_halocline[day-1] = halo
halo = xr.Dataset({'halocline_depth':(['t'], daily_halocline)})
stn_name = '/data/tjarniko/MEOPAR/analysis-tereza/notebooks/CLUSTER_201905/NC_HINDCAST/' + str(year)+ '/HALO_TS/stn_' + str(stn) + 'halo_depth_sp' + str(spacing)+ '.nc'
halo.to_netcdf(stn_name)
def yearved_de(spacing, stn_b, stn_e, year):
import netCDF4 as nc
import map_fxn as mf
import time
import xarray as xr
import numpy as np
print('Spacing between stations: ' + str(spacing))
bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
fmask = (grid.fmask[0, 0, :, :])
grid2 = nc.Dataset('/data/tjarniko/MEOPAR/grid/mesh_mask201702.nc')
stn_x, stn_y = mf.make_stns(spacing)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask)
no_stns = len(d_stn_x)
monlist = [
'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct',
'nov', 'dec'
]
if year == 2016:
daylist = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
noday = 366
print(noday)
if year != 2016:
daylist = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
noday = 365
print(noday)
# a list of all the model outputs (tracers) for this year
trace_list = mf.create_physlist_W(year)
print("Number of stations:" + str(no_stns))
for stn in range(stn_b, stn_e):
print('station is: ', str(stn))
print('x is :', d_stn_x[stn])
print('y is :', d_stn_y[stn])
ts_x = d_stn_x[stn]
ts_y = d_stn_y[stn]
daily_ved = np.zeros(noday)
end_day = noday
for day in range(1, end_day + 1):
if (day % 20 == 0):
print(day)
trc = trace_list[day - 1]
nemo = nc.Dataset(trc)
#print(nemo)
#w = np.array(nemo.variables['vert_eddy_diff'])
ved = nemo['vert_eddy_diff'][0, :, ts_y, ts_x]
where0 = (np.where(ved == 0))
#first out of bounds cell
stop = where0[0][1]
#get only applicable veds
veds = ved[0:stop]
#get applicable depths
depths = grid2['e3w_0'][0, 0:stop, ts_y, ts_x]
#get total depth to fially divide by
totdepth = np.sum(depths)
#weight veds by depths - ie size of cell
veds_depths = veds * depths
#get avg
avg_ved = np.sum(veds_depths / totdepth)
daily_ved[day - 1] = avg_ved
ved = xr.Dataset({'daily_ved': (['t'], daily_ved)})
stn_name = '/data/tjarniko/MEOPAR/analysis_tereza/notebooks/CLUSTER_PAPER/CLEAN/NC_HINDCAST/' + str(
year) + '/VED_TS/stn_' + str(stn) + 'avg_ved_sp' + str(
spacing) + '.nc'
ved.to_netcdf(stn_name)
def yearnit_de(spacing, stn_b, stn_e, year):
import map_fxn as mf
import time
import xarray as xr
import numpy as np
import netCDF4 as nc
print('Spacing between stations: ' + str(spacing))
bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
fmask = (grid.fmask[0, 0, :, :])
stn_x, stn_y = mf.make_stns(spacing)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask)
no_stns = len(d_stn_x)
monlist = [
'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct',
'nov', 'dec'
]
if year == 2016:
daylist = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
noday = 366
else:
daylist = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
noday = 365
# a list of all the model outputs (tracers) for this year
trace_list = mf.create_tracelist(year)
print("Number of stations:" + str(no_stns))
for stn in range(stn_b, stn_e):
print('station is: ', str(stn))
print('x is :', d_stn_x[stn])
print('y is :', d_stn_y[stn])
ts_x = d_stn_x[stn]
ts_y = d_stn_y[stn]
daily_nit = np.zeros(noday)
end_day = noday
for day in range(1, end_day + 1):
# if (day%20 ==0) :
# print(day)
# trc = trace_list[day-1]
# #print(trc)
# nemo = nc.Dataset(trc)
# #print(nemo)
# no3 = np.array(nemo['nitrate'])
# no3 = np.squeeze(no3)
# surf_no3 = no3[0:3,ts_y,ts_x]
# surf_no3 = np.nanmean(surf_no3)
# daily_nit[day-1] = surf_no3
if day % 20 == 0:
print(day)
trc = trace_list[day - 1]
tnc = nc.Dataset(trc)
rivmouth = (tnc['nitrate'][0, 0:3, ts_y, ts_x])
#rivmouth[rivmouth == 0] = np.nan
daily_nit[day - 1] = np.nanmean(rivmouth)
nit = xr.Dataset({'surface_nitrogen': (['t'], daily_nit)})
stn_name = '/data/tjarniko/MEOPAR/analysis_tereza/notebooks/CLUSTER_PAPER/NC_HINDCAST/' + str(
year) + '/NIT_TS/stn_' + str(stn) + 'surfacenitrate_sp' + str(
spacing) + '.nc'
nit.to_netcdf(stn_name)
def yearwinds_de(spacing,stn_b,stn_e,year):
'''POINT: for a given set of stations, produces 3 timeseries of wind forcing data (magnitude, stress, energy) station as lists in a single netcdf file (one netcdf file per station) | CALLS: import_bathy, make_stns, filster_station_in_domain (from map_fxn), reads in netcdf files produced by produce_interpolated_wind_netcdf | NOTES: days start at 0, stations start at 0, for a spacing of 10 there should be 580 stations, the start and end station option is so that it can be run in parallel if necessary, directory into which nc files are deposited is hardcoded under stn_name | TAKES : spacing (has been decided as 10 for this project), year, start station, end station | GIVES: netcdf files that contain 3 windrelated timeseries for each station, in a given repo | USAGE: yearwinds_de(10,1,580,2016) | ROLE IN CLUSTER PROJECT: | written 2017, tjšj '''
import sys
sys.path.append('./extraction_scripts')
import map_fxn as mf
import time
import xarray as xr
import numpy as np
print('Spacing between stations: ' + str(spacing))
noday = 365
if year == 2016:
noday = 366
print('no. days, from yearwinds_de fxn')
print(noday)
bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
fmask = (grid.fmask[0,0,:,:])
stn_x, stn_y = mf.make_stns(spacing)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x,stn_y,fmask)
print('number of stns: ' + str(len(d_stn_x)))
for stn in range(stn_b,stn_e):
print('station is: ' ,str(stn))
print('x is :', d_stn_x[stn])
print('y is :', d_stn_y[stn])
ts_x = d_stn_x[stn]
ts_y = d_stn_y[stn]
daily_windmag = np.zeros(noday)
daily_windstress = np.zeros(noday)
daily_windenergy = np.zeros(noday)
end_day = noday
start = f'{str(year)}-01-01'
end = f'{str(year)}-12-31'
start_run = arrow.get(start)
end_run = arrow.get(end)
arrow_array = []
ncar = []
for r in arrow.Arrow.span_range('day', start_run, end_run):
arrow_array.append(r)
dayslen = len(arrow_array)
#print(dayslen)
ts = np.zeros(dayslen)
#print(arrow_array)
# ops_y2014m09d12.nc
for i in range(0,dayslen):
tdate = arrow_array[i][0]
yy = tdate.format('YYYYMMDD')
print(i)
# for day in range(0,end_day):
# print(day)
# ##open the winds file
# #dayfile
filestart = f'./WINDFILES_interp/windint_{yy}.nc'
#print(day)
#print(filestart)
wf = xr.open_dataset(filestart)
## extract wm, ws, we for the station for the day
## attach it to the corresponding array above
## save the resulting 3 lists to a netcdf file as below
#avg_daily_windmag_windstress_energy
daily_we = wf.daily_avg_windenergy[ts_y,ts_x]
daily_we = daily_we.values
daily_ws = wf.daily_avg_windstress[ts_y,ts_x]
daily_ws = daily_ws.values
daily_wm = wf.daily_avg_windmag[ts_y,ts_x]
daily_wm = daily_wm.values
##be gentle about OB1 errors!!!!
daily_windmag[i] = daily_wm
daily_windstress[i] = daily_ws
daily_windenergy[i] = daily_we
winds = xr.Dataset({'wind_mags':(['t'], daily_windmag), 'wind_stresses':(['t'], daily_windstress),
'wind_energy':(['t'], daily_windenergy)})
stn_name = '/data/tjarniko/MEOPAR/analysis_tereza/notebooks/CLUSTER_PAPER/CLEAN/NC_HINDCAST/' + str(year) + '/WIND_TS/stn_' + str(stn) + '_wind_data_sp' + str(spacing) + '.nc'
winds.to_netcdf(stn_name)
def map_clusters(tit, no_clusters, cl, fsx, fsy, markersize, titfontsize,
legfontsize, fname, colors):
"""argmuents: map_clusters(tit,no_clusters,cl,fsx,fsy,markersize,titfontsize,legfontsize)
cl is list of clusters"""
import map_fxn as mf
import matplotlib.pyplot as plt
import numpy as np
import cmocean
from salishsea_tools import (
nc_tools,
viz_tools,
geo_tools,
tidetools,
visualisations,
)
bath = '/data/tjarniko/MEOPAR/NEMO-forcing/grid/mesh_mask_SalishSea2.nc'
grid = mf.import_bathy(bath)
fmask = (grid.fmask[0, 0, :, :])
spacing = 10
stn_x, stn_y = mf.make_stns(spacing)
d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask)
no_stns = len(d_stn_x)
fig = plt.figure(figsize=(fsx, fsy))
plt.rcParams['image.cmap'] = 'YlGnBu'
for i in range(1, 2):
ax = fig.add_subplot(1, 1, i)
viz_tools.set_aspect(ax)
fmask = (grid.fmask[0, 0, :, :])
mesh = ax.pcolormesh(fmask, vmin=0, vmax=1)
out = ax.set(title='Domain Extent')
ax.set_ylim([0, 898])
ax.set_xlim([0, 398])
if i == 1:
#colors = plt.cm.hsv(np.linspace(0, 1, no_clusters))
for j in range(1, no_clusters + 1):
cluster = np.where(cl == j)
cluster = np.squeeze(cluster)
#find the xs and ys of the stations in a given cluster
c1_x = np.take(d_stn_x, cluster)
c1_y = np.take(d_stn_y, cluster)
print(j)
print(colors[j - 1])
pts = ax.scatter(c1_x,
c1_y,
s=markersize,
c=colors[j - 1],
label=j,
marker='o')
#pts = ax.scatter(c1_x,c1_y,s=markersize,c='xkcd:dust', label=j ,marker='o')
ax.set_title(tit, fontsize=titfontsize)
ax.set_xticklabels(())
ax.set_yticklabels(())
plt.legend(loc=1, fontsize=legfontsize)
plt.savefig(fname, transparent=True)
plt.show()
