def mooring_1dvar(file_in):
"""Standard EcoFOCI Mooring .nc files with one dimensional parameters as a function of time
such as temperature at a particular level
Timestep of data is assumed to be in fractions of a day"""
#dir_path = os.path.dirname(os.path.abspath(__file__))
ncfile = file_in
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = constants.nc_vars_moor()
ncdata = ncutil.ncreadfile(nchandle, params)
ncutil.ncclose(nchandle)
###data massaging
#time_all = ncdata[:,0] + ncdata[:,1]
xx = ncutil.nc_missing(ncdata[:,9], flag=1e35, setfill='Zero')
pytime = util.EPICdate2udunits(ncdata[:,0], ncdata[:,1])
dt = 1. / pytime['interval_min'] #data is 4 times daily
time_base = 'days'
time = pytime['timeint']
#time = util.subsample(time, int(pytime.get('interval_min')) / 4)
variance = np.var(xx)
#normalize
print 'Variance = %s ' % (variance)
x = (xx - np.mean(xx)) / np.sqrt(variance)
variance = np.var(x)
return (xx, x,dt,np.array(time), variance, time_base)
def mooring_1dvar(file_in):
"""Standard EcoFOCI Mooring .nc files with one dimensional parameters as a function of time
such as temperature at a particular level
Timestep of data is assumed to be in fractions of a day"""
#dir_path = os.path.dirname(os.path.abspath(__file__))
ncfile = file_in
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = constants.nc_vars_moor()
ncdata = ncutil.ncreadfile(nchandle, params)
ncutil.ncclose(nchandle)
###data massaging
#time_all = ncdata[:,0] + ncdata[:,1]
xx = ncutil.nc_missing(ncdata[:, 9], flag=1e35, setfill='Zero')
pytime = util.EPICdate2udunits(ncdata[:, 0], ncdata[:, 1])
dt = 1. / pytime['interval_min'] #data is 4 times daily
time_base = 'days'
time = pytime['timeint']
#time = util.subsample(time, int(pytime.get('interval_min')) / 4)
variance = np.var(xx)
#normalize
print 'Variance = %s ' % (variance)
x = (xx - np.mean(xx)) / np.sqrt(variance)
variance = np.var(x)
return (xx, x, dt, np.array(time), variance, time_base)
def reanaly_sfc_press(file_in):
ncfile = file_in
###nc readin/out
nchandle = Dataset(ncfile,'r')
params = ['time','lat', 'lon', 'pres']
time = nchandle.variables[params[0]][:]
lat = nchandle.variables[params[1]][:]
lon = nchandle.variables[params[2]][:]
data = nchandle.variables[params[3]][:,:]
ncutil.ncclose(nchandle)
site_loc = [52.5, 190.0] #52.8, -169.3 for M6 Samalga mooring
ind_lat = np.where(lat == site_loc[0])[0][0]
ind_lon = np.where(lon == site_loc[1])[0][0]
xx = data[:,ind_lat,ind_lon] / 100 #pa to hPa/mbar
# convert time to serial date
base_date = datetime.datetime.fromordinal(1) #time is in hours since 1-1-1
time_delta = datetime.timedelta(hours = 1)
# there is a -2 correction factor in the following
# conversion to calculate the date (unknown why its needed)
time = [base_date + (int(t - 48) * time_delta) for t in time] #convert to integer for datetime calculation
time = [t.toordinal() for t in time ]
variance = np.var(xx)
#normalize
print 'Variance = %s ' % (variance)
x = (xx - np.mean(xx)) / np.sqrt(variance)
variance = np.var(x)
return (xx, x,dt,np.array(time), variance, time_base)
def mooring_1dvar(file_in):
""" TODO: make file choice and var choice more transparent"""
#dir_path = os.path.dirname(os.path.abspath(__file__))
ncfile = file_in
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = constants.nc_vars_moor()
ncdata = ncutil.ncreadfile(nchandle, params)
ncutil.ncclose(nchandle)
###data massaging
#time_all = ncdata[:,0] + ncdata[:,1]
xx = ncutil.nc_missing(ncdata[:,9], flag=1e35, setfill='Zero')
pytime = util.EPICdate2udunits(ncdata[:,0], ncdata[:,1])
dt = 1. / pytime['interval_min'] #data is 4 times daily
time_base = 'days'
time = pytime['timeint']
#time = util.subsample(time, int(pytime.get('interval_min')) / 4)
variance = np.var(xx)
#normalize
print 'Variance = %s ' % (variance)
x = (xx - np.mean(xx)) / np.sqrt(variance)
variance = np.var(x)
return (xx, x,dt,np.array(time), variance, time_base)
def reanaly_sfc_press(file_in):
ncfile = file_in
###nc readin/out
nchandle = Dataset(ncfile, 'r')
params = ['time', 'lat', 'lon', 'pres']
time = nchandle.variables[params[0]][:]
lat = nchandle.variables[params[1]][:]
lon = nchandle.variables[params[2]][:]
data = nchandle.variables[params[3]][:, :]
ncutil.ncclose(nchandle)
site_loc = [52.5, 190.0] #52.8, -169.3 for M6 Samalga mooring
ind_lat = np.where(lat == site_loc[0])[0][0]
ind_lon = np.where(lon == site_loc[1])[0][0]
xx = data[:, ind_lat, ind_lon] / 100 #pa to hPa/mbar
# convert time to serial date
base_date = datetime.datetime.fromordinal(1) #time is in hours since 1-1-1
time_delta = datetime.timedelta(hours=1)
# there is a -2 correction factor in the following
# conversion to calculate the date (unknown why its needed)
time = [base_date + (int(t - 48) * time_delta)
for t in time] #convert to integer for datetime calculation
time = [t.toordinal() for t in time]
variance = np.var(xx)
#normalize
print 'Variance = %s ' % (variance)
x = (xx - np.mean(xx)) / np.sqrt(variance)
variance = np.var(x)
return (xx, x, dt, np.array(time), variance, time_base)
def mooring_1dvar(file_in):
""" TODO: make file choice and var choice more transparent"""
#dir_path = os.path.dirname(os.path.abspath(__file__))
ncfile = file_in
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = constants.nc_vars_moor()
ncdata = ncutil.ncreadfile(nchandle, params)
ncutil.ncclose(nchandle)
###data massaging
#time_all = ncdata[:,0] + ncdata[:,1]
xx = ncutil.nc_missing(ncdata[:, 9], flag=1e35, setfill='Zero')
pytime = util.EPICdate2udunits(ncdata[:, 0], ncdata[:, 1])
dt = 1. / pytime['interval_min'] #data is 4 times daily
time_base = 'days'
time = pytime['timeint']
#time = util.subsample(time, int(pytime.get('interval_min')) / 4)
variance = np.var(xx)
#normalize
print 'Variance = %s ' % (variance)
x = (xx - np.mean(xx)) / np.sqrt(variance)
variance = np.var(x)
return (xx, x, dt, np.array(time), variance, time_base)
def from_netcdf(infile, parameter):
""" Uses ncreadfile_dic which returns a dictionary of all data from netcdf"""
ncfile = infile
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = ncutil.get_vars(nchandle) #gets all of them
if not parameter in params:
### specified parameter doesn't exist
return ('no param')
ncdata = ncutil.ncreadfile_dic(nchandle, params)
ncutil.ncclose(nchandle)
###data massaging
pytime = util.EPICdate2udunits(ncdata['time'], ncdata['time2'])
dt = 1. / pytime['interval_min'] #data is 4 times daily
time_base = 'days'
time = pytime['timeint']
#time = util.subsample(time, int(pytime.get('interval_min')) / 4)
# signal
data = ncdata[parameter][:,0,0,0]
variance = np.var(data)
#normalize
print 'Variance = %s ' % (variance)
x = (data - np.mean(data)) / np.sqrt(variance)
variance = np.var(x)
return_data = {'data': data, 'anom': x, 'dt': dt, 'time':np.array(time), 'variance':variance, 'time_base':time_base}
return (return_data)
def from_2netcdf(infile, parameter):
""" For moorings with a, and b deployments
Ingest and concatenate both files
"""
ncfile = infile[0]
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = ncutil.get_vars(nchandle) #gets all of them
if not parameter in params:
### specified parameter doesn't exist
return ('no param')
ncdata1 = ncutil.ncreadfile_dic(nchandle, params)
ncutil.ncclose(nchandle)
ncfile = infile[1]
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = ncutil.get_vars(nchandle) #gets all of them
if not parameter in params:
### specified parameter doesn't exist
return ('no param')
ncdata2 = ncutil.ncreadfile_dic(nchandle, params)
ncutil.ncclose(nchandle)
###data massaging
pytime1 = util.EPICdate2udunits(ncdata1['time'], ncdata1['time2'])
pytime2 = util.EPICdate2udunits(ncdata2['time'], ncdata2['time2'])
dt = 1. / pytime1['interval_min'] #data is 4 times daily
time_base = 'days'
### fill time gap
blanktime = np.array([])
for tg in np.arange(np.max(pytime1['timeint'])+dt,np.min(pytime2['timeint']),dt):
blanktime = np.hstack((blanktime,tg))
time = np.hstack((pytime1['timeint'], blanktime))
time = np.hstack((time,pytime2['timeint']))
# signal
data_interp = np.interp(blanktime,[np.max(pytime1['timeint']),np.min(pytime2['timeint'])],[ncdata1[parameter][-1,0,0,0], ncdata2[parameter][0,0,0,0]])
data = np.hstack((ncdata1[parameter][:,0,0,0], data_interp))
data = np.hstack((data, ncdata2[parameter][:,0,0,0]))
variance = np.var(data)
#normalize
print 'Variance = %s ' % (variance)
x = (data - np.mean(data)) / np.sqrt(variance)
variance = np.var(x)
return_data = {'data': data, 'anom': x, 'dt': dt, 'time':np.array(time), 'variance':variance, 'time_base':time_base}
return (return_data)
def reanaly_sfc_press_multifile(files_in):
time = []
xx = []
for ncfile in files_in:
###nc readin/out
nchandle = Dataset(ncfile, "r")
params = ["time", "lat", "lon", "pres"]
time = np.hstack((time, nchandle.variables[params[0]][:]))
lat = nchandle.variables[params[1]][:]
lon = nchandle.variables[params[2]][:]
data = nchandle.variables[params[3]][:, :]
ncutil.ncclose(nchandle)
site_loc = [52.5, 190.0] # 52.8, -169.3 for M6 Samalga mooring
ind_lat = np.where(lat == site_loc[0])[0][0]
ind_lon = np.where(lon == site_loc[1])[0][0]
xx = np.hstack((xx, data[:, ind_lat, ind_lon] / 100)) # pa to hPa/mbar
dt = 1.0 # data is daily
time_base = "days"
# convert time to serial date
base_date = datetime.datetime.fromordinal(1) # time is in hours since 1-1-1
time_delta = datetime.timedelta(hours=1)
# there is a -2 correction factor in the following
# conversion to calculate the date (unknown why its needed)
time = [base_date + (int(t - 48) * time_delta) for t in time] # convert to integer for datetime calculation
time = [t.toordinal() for t in time]
variance = np.var(xx)
# normalize
print "Variance = %s " % (variance)
x = (xx - np.mean(xx)) / np.sqrt(variance)
variance = np.var(x)
return (xx, x, dt, np.array(time), variance, time_base)
def from_2netcdf(infile, parameter):
""" For moorings with a, and b deployments
Ingest and concatenate both files
"""
ncfile = infile[0]
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = ncutil.get_vars(nchandle) #gets all of them
if not parameter in params:
### specified parameter doesn't exist
return ('no param')
ncdata1 = ncutil.ncreadfile_dic(nchandle, params)
ncutil.ncclose(nchandle)
ncfile = infile[1]
###nc readin/out
nchandle = ncutil.ncopen(ncfile)
params = ncutil.get_vars(nchandle) #gets all of them
if not parameter in params:
### specified parameter doesn't exist
return ('no param')
ncdata2 = ncutil.ncreadfile_dic(nchandle, params)
ncutil.ncclose(nchandle)
###data massaging
pytime1 = util.EPICdate2udunits(ncdata1['time'], ncdata1['time2'])
pytime2 = util.EPICdate2udunits(ncdata2['time'], ncdata2['time2'])
dt = 1. / pytime1['interval_min'] #data is 4 times daily
time_base = 'days'
### fill time gap
blanktime = np.array([])
for tg in np.arange(
np.max(pytime1['timeint']) + dt, np.min(pytime2['timeint']), dt):
blanktime = np.hstack((blanktime, tg))
time = np.hstack((pytime1['timeint'], blanktime))
time = np.hstack((time, pytime2['timeint']))
# signal
data_interp = np.interp(
blanktime, [np.max(pytime1['timeint']),
np.min(pytime2['timeint'])],
[ncdata1[parameter][-1, 0, 0, 0], ncdata2[parameter][0, 0, 0, 0]])
data = np.hstack((ncdata1[parameter][:, 0, 0, 0], data_interp))
data = np.hstack((data, ncdata2[parameter][:, 0, 0, 0]))
variance = np.var(data)
#normalize
print 'Variance = %s ' % (variance)
x = (data - np.mean(data)) / np.sqrt(variance)
variance = np.var(x)
return_data = {
'data': data,
'anom': x,
'dt': dt,
'time': np.array(time),
'variance': variance,
'time_base': time_base
}
return (return_data)