def clusters_basic(ax1,year,var,noclust,cl_this, colors, legend = True, markersize = 20, legfontsize = 12 ): import pickle import numpy as np import cmocean as cm import sys sys.path.append('./extraction_scripts') import map_fxn as mf from salishsea_tools import ( viz_tools) bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) tpkl = f'./pkls/{var}_clustmat_{year}.pkl' cl = pickle.load(open(tpkl, 'rb')) viz_tools.set_aspect(ax1) fmask = (grid.fmask[0,0,:,:]) mesh = ax1.pcolormesh(fmask, vmin=0, vmax=1, cmap = cm.cm.deep) ax1.set_ylim([0,898]) ax1.set_xlim([0,398]) stn_x, stn_y = mf.make_stns(10) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x,stn_y,fmask) d_stn_xar = np.array(d_stn_x) d_stn_yar = np.array(d_stn_y) edge = np.where((d_stn_xar<=10) | (d_stn_yar>=888)) np.squeeze(edge) np.shape(edge) edgear = np.array(edge) edgear = edgear[0] d_stn_xar =np.delete(d_stn_xar,edge) d_stn_yar = np.delete(d_stn_yar,edge) for j in range(1,noclust+1): cluster = np.where(cl_this == j) cluster = np.squeeze(cluster) c1_x = np.take(d_stn_xar,cluster) c1_y = np.take(d_stn_yar,cluster) pts = ax1.scatter(c1_x,c1_y,s=markersize,c=colors[j], label=str(j), marker='o') ax1.set_xticklabels( () ) ax1.set_yticklabels( () ) tit = 'year ' + year # + ' \n n. clusters = '+ str(noclust) if legend: ax1.legend(bbox_to_anchor=(1.06, 1), fontsize = legfontsize) ax1.set_title(tit,fontsize = legfontsize + 2)
carpT = [] str_to_C = '/results2/SalishSea/hindcast.201905/*/*' for i in range(0, dayslen): tdate = arrow_array1[i][0] ymd = tdate.format('YYYYMMDD') tstr = (str_to_C + ymd + '*carp_T.nc') #'*carp_T.nc' #print(tstr) tstr = glob.glob(str_to_C + ymd + '*carp_T.nc') #print(tstr) #print(tstr) carpT.append(tstr[0]) bath = '/data/tjarniko/MEOPAR/grid/mesh_mask201702.nc' grid = mf.import_bathy(bath) fmask = (grid.fmask[0, 0, :, :]) spacing = 10 stn_x, stn_y = mf.make_stns(spacing) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask) print('complete') year = '2013' LL = 0.1 def getVEDEuph(ii, jj, LL, fileK, filePAR): #print(filePAR) # print(ii) # print(jj) dailyPAR = np.mean(filePAR.variables['PAR'][:, :, jj, ii], 0)
def bio_de(spacing, stn_start, stn_end, year): import arrow import time import xarray as xr import numpy as np import map_fxn as mf import warnings warnings.filterwarnings("ignore") if year == 2016: noday = 366 else: noday = 365 day_start = 0 day_end = noday print('Spacing between stations: ' + str(spacing)) print('First day: ' + str(day_start)) print('Last day: ' + str(day_end)) print('testing reload') print(year) dirname = '/data/tjarniko/MEOPAR/analysis_tereza/notebooks/CLUSTER_PAPER/CLEAN/NC_HINDCAST/' + str( year) + '/BIO_TS/' bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) fmask = (grid.fmask[0, 0, :, :]) stn_x, stn_y = mf.make_stns(spacing) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask) no_stns = len(d_stn_x) # a list of all the model outputs (tracers) for this year trace_list = mf.create_tracelist(year) print("Number of stations:" + str(no_stns)) stn_names = list() for i in range(0, no_stns): stn_name = 'stn_' + str(i) stn_names.append(stn_name) #print(stn_name) for n in range(stn_start, stn_end): print('stn is: ' + str(n)) start_time = time.time() MYRI_ar = np.zeros([noday, 40]) MICZ_ar = np.zeros([noday, 40]) PHY2_ar = np.zeros([noday, 40]) PHY_ar = np.zeros([noday, 40]) for i in range(day_start, day_end): if (i % 20 == 0): print('day is:' + str(i)) b = i e = i + 1 start_time = time.time() trc = (trace_list[i]) nemo = xr.open_mfdataset(trc) t1 = b t2 = e x1 = d_stn_x[n] #print(x1) x2 = d_stn_x[n] + 1 y1 = d_stn_y[n] #print(y1) y2 = d_stn_y[n] + 1 z1 = 0 z2 = 40 MYRI_d, MICZ_d, PHY2_d, PHY_d = int_phyt_fxn( grid, nemo, t1, t2, x1, x2, y1, y2, z1, z2) #print(np.shape(PHY_d)) MYRI_ar[i, :] = (MYRI_d[0, :, 0, 0]) MICZ_ar[i, :] = (MICZ_d[0, :, 0, 0]) PHY2_ar[i, :] = (PHY2_d[0, :, 0, 0]) #print(PHY2_d[0,:,0,0]) PHY_ar[i, :] = (PHY_d[0, :, 0, 0]) jan = xr.Dataset({ 'MYRI': (['t', 'z'], MYRI_ar), 'MICZ': (['t', 'z'], MICZ_ar), 'PHY2': (['t', 'z'], PHY2_ar), 'PHY': (['t', 'z'], PHY_ar) }) stn_name = dirname + stn_names[n] + '_DP_sp' + str(spacing) + '.nc' #print(stn_name) jan.to_netcdf(stn_name)
def clusters(ax1, year, var, no_clusters, markersize=20, legfontsize=12): colors = [ 'deepskyblue', 'red', 'goldenrod', 'forestgreen', 'midnightblue', 'orchid', 'gray', 'peru', 'olive', 'sandybrown', 'teal', 'pink', 'tan', 'yellow', 'thistle' ] import pickle import numpy as np import cmocean as cm import sys sys.path.append('./extraction_scripts') import map_fxn as mf from salishsea_tools import (viz_tools) bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) tpkl = f'./pkls/{var}_clustmat_{year}.pkl' cl = pickle.load(open(tpkl, 'rb')) cl_this = cl[no_clusters - 1, :] np.shape(cl_this) viz_tools.set_aspect(ax1) fmask = (grid.fmask[0, 0, :, :]) mesh = ax1.pcolormesh(fmask, vmin=0, vmax=1, cmap=cm.cm.deep) ax1.set_ylim([0, 898]) ax1.set_xlim([0, 398]) stn_x, stn_y = mf.make_stns(10) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask) d_stn_xar = np.array(d_stn_x) d_stn_yar = np.array(d_stn_y) edge = np.where((d_stn_xar <= 10) | (d_stn_yar >= 888)) np.squeeze(edge) np.shape(edge) edgear = np.array(edge) edgear = edgear[0] d_stn_xar = np.delete(d_stn_xar, edge) d_stn_yar = np.delete(d_stn_yar, edge) #print(np.size(d_stn_xar)) ###sort clusters by size cluster_ids = np.arange(1, no_clusters + 1, 1) cluster_sizes = np.zeros_like(cluster_ids) #retrieve cluster sizes for j in range(1, no_clusters + 1): cluster = np.where(cl_this == j) cluster = np.squeeze(cluster) cluster_sizes[j - 1] = (np.size(cluster)) #sort cluster size matrix biggest to smallest cs = np.argsort(-cluster_sizes) #print(cs) #use those sizes to sort the cluster id list to corresponde to a list 'clust id, largest to smallest' new_cidlist = np.zeros_like(cluster_ids) for j in range(0, len(new_cidlist)): new_cidlist[j] = cluster_ids[cs[j]] #start plotting, plotting biggest cluster first, to keep colour order the same for j in range(0, np.size(new_cidlist)): cluster = np.where(cl_this == new_cidlist[j]) cluster = np.squeeze(cluster) c1_x = np.take(d_stn_xar, cluster) c1_y = np.take(d_stn_yar, cluster) pts = ax1.scatter(c1_x, c1_y, s=markersize, c=colors[j], label=str(new_cidlist[j]), marker='o') ax1.set_xticklabels(()) ax1.set_yticklabels(()) tit = var + ', year: ' + year + ' \n n. clusters = ' + str(no_clusters) ax1.legend(bbox_to_anchor=(1.1, 1), fontsize=legfontsize) ax1.set_title(tit, fontsize=legfontsize + 2)
def yearhalo_de(spacing,stn_b,stn_e,year): import map_fxn as mf import time import xarray as xr import numpy as np print('Spacing between stations: ' + str(spacing)) bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) fmask = (grid.fmask[0,0,:,:]) stn_x, stn_y = mf.make_stns(spacing) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x,stn_y,fmask) no_stns = len(d_stn_x) monlist = ['jan','feb','mar','apr','may','jun','jul','aug','sep', 'oct','nov','dec'] if year == 2016: daylist = [31,29,31,30,31,30,31,31,30,31,30,31] noday = 366 else: daylist = [31,28,31,30,31,30,31,31,30,31,30,31] noday = 365 print(year) print(noday) # a list of all the model outputs (tracers) for this year trace_list = mf.create_physlist(year) print("Number of stations:" + str(no_stns)) bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) fmask = (grid.fmask[0,0,:,:]) for stn in range(stn_b,stn_e): print('station is: ' ,str(stn)) print('x is :', d_stn_x[stn]) print('y is :', d_stn_y[stn]) ts_x = d_stn_x[stn] ts_y = d_stn_y[stn] daily_halocline = np.zeros(noday) end_day = noday for day in range(1,end_day+1): if (day%20 ==0) : print(day) trc = trace_list[day-1] #print(trc) nemo = xr.open_mfdataset(trc) halo = halo_de(nemo,grid,ts_x,ts_y) q = isinstance(halo, (np.ndarray)) if (q == True) : print('found an array!') daily_halocline[day-1] = halo[0] else: daily_halocline[day-1] = halo halo = xr.Dataset({'halocline_depth':(['t'], daily_halocline)}) stn_name = '/data/tjarniko/MEOPAR/analysis-tereza/notebooks/CLUSTER_201905/NC_HINDCAST/' + str(year)+ '/HALO_TS/stn_' + str(stn) + 'halo_depth_sp' + str(spacing)+ '.nc' halo.to_netcdf(stn_name)
def yearved_de(spacing, stn_b, stn_e, year): import netCDF4 as nc import map_fxn as mf import time import xarray as xr import numpy as np print('Spacing between stations: ' + str(spacing)) bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) fmask = (grid.fmask[0, 0, :, :]) grid2 = nc.Dataset('/data/tjarniko/MEOPAR/grid/mesh_mask201702.nc') stn_x, stn_y = mf.make_stns(spacing) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask) no_stns = len(d_stn_x) monlist = [ 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec' ] if year == 2016: daylist = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] noday = 366 print(noday) if year != 2016: daylist = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] noday = 365 print(noday) # a list of all the model outputs (tracers) for this year trace_list = mf.create_physlist_W(year) print("Number of stations:" + str(no_stns)) for stn in range(stn_b, stn_e): print('station is: ', str(stn)) print('x is :', d_stn_x[stn]) print('y is :', d_stn_y[stn]) ts_x = d_stn_x[stn] ts_y = d_stn_y[stn] daily_ved = np.zeros(noday) end_day = noday for day in range(1, end_day + 1): if (day % 20 == 0): print(day) trc = trace_list[day - 1] nemo = nc.Dataset(trc) #print(nemo) #w = np.array(nemo.variables['vert_eddy_diff']) ved = nemo['vert_eddy_diff'][0, :, ts_y, ts_x] where0 = (np.where(ved == 0)) #first out of bounds cell stop = where0[0][1] #get only applicable veds veds = ved[0:stop] #get applicable depths depths = grid2['e3w_0'][0, 0:stop, ts_y, ts_x] #get total depth to fially divide by totdepth = np.sum(depths) #weight veds by depths - ie size of cell veds_depths = veds * depths #get avg avg_ved = np.sum(veds_depths / totdepth) daily_ved[day - 1] = avg_ved ved = xr.Dataset({'daily_ved': (['t'], daily_ved)}) stn_name = '/data/tjarniko/MEOPAR/analysis_tereza/notebooks/CLUSTER_PAPER/CLEAN/NC_HINDCAST/' + str( year) + '/VED_TS/stn_' + str(stn) + 'avg_ved_sp' + str( spacing) + '.nc' ved.to_netcdf(stn_name)
def yearnit_de(spacing, stn_b, stn_e, year): import map_fxn as mf import time import xarray as xr import numpy as np import netCDF4 as nc print('Spacing between stations: ' + str(spacing)) bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) fmask = (grid.fmask[0, 0, :, :]) stn_x, stn_y = mf.make_stns(spacing) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask) no_stns = len(d_stn_x) monlist = [ 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec' ] if year == 2016: daylist = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] noday = 366 else: daylist = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] noday = 365 # a list of all the model outputs (tracers) for this year trace_list = mf.create_tracelist(year) print("Number of stations:" + str(no_stns)) for stn in range(stn_b, stn_e): print('station is: ', str(stn)) print('x is :', d_stn_x[stn]) print('y is :', d_stn_y[stn]) ts_x = d_stn_x[stn] ts_y = d_stn_y[stn] daily_nit = np.zeros(noday) end_day = noday for day in range(1, end_day + 1): # if (day%20 ==0) : # print(day) # trc = trace_list[day-1] # #print(trc) # nemo = nc.Dataset(trc) # #print(nemo) # no3 = np.array(nemo['nitrate']) # no3 = np.squeeze(no3) # surf_no3 = no3[0:3,ts_y,ts_x] # surf_no3 = np.nanmean(surf_no3) # daily_nit[day-1] = surf_no3 if day % 20 == 0: print(day) trc = trace_list[day - 1] tnc = nc.Dataset(trc) rivmouth = (tnc['nitrate'][0, 0:3, ts_y, ts_x]) #rivmouth[rivmouth == 0] = np.nan daily_nit[day - 1] = np.nanmean(rivmouth) nit = xr.Dataset({'surface_nitrogen': (['t'], daily_nit)}) stn_name = '/data/tjarniko/MEOPAR/analysis_tereza/notebooks/CLUSTER_PAPER/NC_HINDCAST/' + str( year) + '/NIT_TS/stn_' + str(stn) + 'surfacenitrate_sp' + str( spacing) + '.nc' nit.to_netcdf(stn_name)
def yearwinds_de(spacing,stn_b,stn_e,year): '''POINT: for a given set of stations, produces 3 timeseries of wind forcing data (magnitude, stress, energy) station as lists in a single netcdf file (one netcdf file per station) | CALLS: import_bathy, make_stns, filster_station_in_domain (from map_fxn), reads in netcdf files produced by produce_interpolated_wind_netcdf | NOTES: days start at 0, stations start at 0, for a spacing of 10 there should be 580 stations, the start and end station option is so that it can be run in parallel if necessary, directory into which nc files are deposited is hardcoded under stn_name | TAKES : spacing (has been decided as 10 for this project), year, start station, end station | GIVES: netcdf files that contain 3 windrelated timeseries for each station, in a given repo | USAGE: yearwinds_de(10,1,580,2016) | ROLE IN CLUSTER PROJECT: | written 2017, tjšj ''' import sys sys.path.append('./extraction_scripts') import map_fxn as mf import time import xarray as xr import numpy as np print('Spacing between stations: ' + str(spacing)) noday = 365 if year == 2016: noday = 366 print('no. days, from yearwinds_de fxn') print(noday) bath = '/results/nowcast-sys/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) fmask = (grid.fmask[0,0,:,:]) stn_x, stn_y = mf.make_stns(spacing) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x,stn_y,fmask) print('number of stns: ' + str(len(d_stn_x))) for stn in range(stn_b,stn_e): print('station is: ' ,str(stn)) print('x is :', d_stn_x[stn]) print('y is :', d_stn_y[stn]) ts_x = d_stn_x[stn] ts_y = d_stn_y[stn] daily_windmag = np.zeros(noday) daily_windstress = np.zeros(noday) daily_windenergy = np.zeros(noday) end_day = noday start = f'{str(year)}-01-01' end = f'{str(year)}-12-31' start_run = arrow.get(start) end_run = arrow.get(end) arrow_array = [] ncar = [] for r in arrow.Arrow.span_range('day', start_run, end_run): arrow_array.append(r) dayslen = len(arrow_array) #print(dayslen) ts = np.zeros(dayslen) #print(arrow_array) # ops_y2014m09d12.nc for i in range(0,dayslen): tdate = arrow_array[i][0] yy = tdate.format('YYYYMMDD') print(i) # for day in range(0,end_day): # print(day) # ##open the winds file # #dayfile filestart = f'./WINDFILES_interp/windint_{yy}.nc' #print(day) #print(filestart) wf = xr.open_dataset(filestart) ## extract wm, ws, we for the station for the day ## attach it to the corresponding array above ## save the resulting 3 lists to a netcdf file as below #avg_daily_windmag_windstress_energy daily_we = wf.daily_avg_windenergy[ts_y,ts_x] daily_we = daily_we.values daily_ws = wf.daily_avg_windstress[ts_y,ts_x] daily_ws = daily_ws.values daily_wm = wf.daily_avg_windmag[ts_y,ts_x] daily_wm = daily_wm.values ##be gentle about OB1 errors!!!! daily_windmag[i] = daily_wm daily_windstress[i] = daily_ws daily_windenergy[i] = daily_we winds = xr.Dataset({'wind_mags':(['t'], daily_windmag), 'wind_stresses':(['t'], daily_windstress), 'wind_energy':(['t'], daily_windenergy)}) stn_name = '/data/tjarniko/MEOPAR/analysis_tereza/notebooks/CLUSTER_PAPER/CLEAN/NC_HINDCAST/' + str(year) + '/WIND_TS/stn_' + str(stn) + '_wind_data_sp' + str(spacing) + '.nc' winds.to_netcdf(stn_name)
def map_clusters(tit, no_clusters, cl, fsx, fsy, markersize, titfontsize, legfontsize, fname, colors): """argmuents: map_clusters(tit,no_clusters,cl,fsx,fsy,markersize,titfontsize,legfontsize) cl is list of clusters""" import map_fxn as mf import matplotlib.pyplot as plt import numpy as np import cmocean from salishsea_tools import ( nc_tools, viz_tools, geo_tools, tidetools, visualisations, ) bath = '/data/tjarniko/MEOPAR/NEMO-forcing/grid/mesh_mask_SalishSea2.nc' grid = mf.import_bathy(bath) fmask = (grid.fmask[0, 0, :, :]) spacing = 10 stn_x, stn_y = mf.make_stns(spacing) d_stn_x, d_stn_y = mf.filter_stn_in_domain(stn_x, stn_y, fmask) no_stns = len(d_stn_x) fig = plt.figure(figsize=(fsx, fsy)) plt.rcParams['image.cmap'] = 'YlGnBu' for i in range(1, 2): ax = fig.add_subplot(1, 1, i) viz_tools.set_aspect(ax) fmask = (grid.fmask[0, 0, :, :]) mesh = ax.pcolormesh(fmask, vmin=0, vmax=1) out = ax.set(title='Domain Extent') ax.set_ylim([0, 898]) ax.set_xlim([0, 398]) if i == 1: #colors = plt.cm.hsv(np.linspace(0, 1, no_clusters)) for j in range(1, no_clusters + 1): cluster = np.where(cl == j) cluster = np.squeeze(cluster) #find the xs and ys of the stations in a given cluster c1_x = np.take(d_stn_x, cluster) c1_y = np.take(d_stn_y, cluster) print(j) print(colors[j - 1]) pts = ax.scatter(c1_x, c1_y, s=markersize, c=colors[j - 1], label=j, marker='o') #pts = ax.scatter(c1_x,c1_y,s=markersize,c='xkcd:dust', label=j ,marker='o') ax.set_title(tit, fontsize=titfontsize) ax.set_xticklabels(()) ax.set_yticklabels(()) plt.legend(loc=1, fontsize=legfontsize) plt.savefig(fname, transparent=True) plt.show()