def butterworth_ts(fs = .1666666666, cutoff = .25):
'''Proving stevens data phase shift is corrected'''
fig = plt.figure(figsize=(12,4))
ax = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
x = nc.get_pressure('pressure3.nc')
t = nc.get_time('pressure3.nc')
lowcut = cutoff / (.5 * fs)
# # # highcut = 1.5 / (.5 * fs)
b, a = signal.butter(4, [lowcut], btype='lowpass')
y = signal.lfilter(b, a, x)
ax.plot(t,x,color="red", alpha=0.5)
ax.plot(t,y,color="blue", alpha=0.5)
z = signal.filtfilt(b, a, x)
ax2.plot(t,x,color="red", alpha=0.5)
ax2.plot(t,z,color="blue", alpha=0.5)
plt.show()
def butterworth_test3(fs = 4, cutoff = .00166666665, mode = 'phase'):
'''Showing filtering and phase shift of created tide data (M2 and S2)'''
fig = plt.figure(figsize=(12,4))
x = nc.get_depth('M2_S2_plus4.nc')
t = nc.get_time('M2_S2_plus4.nc')
if mode == 'freq':
ax = fig.add_subplot(111)
hat = np.fft.rfft(x)
freqs = np.fft.rfftfreq(len(x), d= 1.0/fs)
ax.plot(freqs,np.abs(hat), 'b', alpha=.5)
lowcut = cutoff / (.5 * fs)
b, a = signal.butter(4, [lowcut], btype='lowpass')
y = signal.filtfilt(b, a, x)
yhat1 = np.fft.rfft(y)
ax.plot(freqs,np.abs(yhat1),'r', alpha=.5)
ax.set_title('4th order ButterWorth - 10 minute cutoff')
plt.show()
else:
ax = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
lowcut = cutoff / (.5 * fs)
# # # highcut = 1.5 / (.5 * fs)
b, a = signal.butter(4, [lowcut], btype='lowpass')
y = signal.lfilter(b, a, x)
ax.plot(t,x,color="red", alpha=0.5)
ax.plot(t,y,color="blue", alpha=0.5)
z = signal.filtfilt(b, a, x)
ax2.plot(t,x,color="red", alpha=0.5)
ax2.plot(t,z,color="blue", alpha=0.5)
plt.show()
def change_netCDFTime(in_file_name, out_file_name, start_ms):
shutil.copy(in_file_name, out_file_name)
time_len = len(nc.get_time(out_file_name))
new_time = unit_conversion.generate_ms(start_ms, time_len, 1/900)
nc.set_variable_data(out_file_name, 'time', new_time)
def extract_time(self, fname):
return nc.get_time(fname)
def extract_time(self, fname):
return nc.get_time(fname)
def process_file(args):
daylight_savings = False
if args['daylight_savings'].lower() == 'true':
daylight_savings = True
inputs = {
'in_filename':
args['in_fname'],
'out_filename':
args['out_fname'],
'creator_name':
args['creator_name'],
'creator_email':
args['creator_email'],
'creator_url':
args['creator_url'],
'instrument_name':
args['instrument_name'],
'stn_station_number':
args['stn_station_number'],
'stn_instrument_id':
args['stn_instrument_id'],
'latitude':
args['latitude'],
'longitude':
args['longitude'],
'tz_info':
args['tz_info'],
'daylight_savings':
daylight_savings,
'datum':
args['datum'],
'initial_sensor_orifice_elevation':
args['initial_sensor_orifice_elevation'],
'final_sensor_orifice_elevation':
args['final_sensor_orifice_elevation'],
'salinity':
args['salinity'],
'initial_land_surface_elevation':
args['initial_land_surface_elevation'],
'final_land_surface_elevation':
args['final_land_surface_elevation'],
'deployment_time':
args['deployment_time'],
'retrieval_time':
args['retrieval_time'],
'sea_name':
args['sea_name'],
'pressure_type':
args['pressure_type'],
'good_start_date':
args['good_start_date'],
'good_end_date':
args['good_end_date'],
}
#checks for the correct file type
if check_file_type(inputs['in_filename']) == False:
return (2, None)
#check for dates in chronological order if sea pressure file
if inputs['pressure_type'] == 'Sea Pressure':
inputs['deployment_time'] = uc.datestring_to_ms(inputs['deployment_time'], '%Y%m%d %H%M', \
inputs['tz_info'],
inputs['daylight_savings'])
inputs['retrieval_time'] = uc.datestring_to_ms(inputs['retrieval_time'], '%Y%m%d %H%M', \
inputs['tz_info'],
inputs['daylight_savings'])
if inputs['retrieval_time'] <= inputs['deployment_time']:
return (3, None)
try:
data_issues = convert_to_netcdf(inputs)
except:
return (5, None)
time = nc.get_time(inputs['out_filename'])
start_index = find_index(time,uc.datestring_to_ms(inputs['good_start_date'], '%Y%m%d %H%M', \
inputs['tz_info'],
inputs['daylight_savings']))
end_index = find_index(time,uc.datestring_to_ms(inputs['good_end_date'], '%Y%m%d %H%M', \
inputs['tz_info'],
inputs['daylight_savings']))
#checks for chronological order of dates
if end_index <= start_index:
return (3, None)
air_pressure = False
if args['pressure_type'] == 'Air Pressure':
air_pressure = True
try:
nc.chop_netcdf(inputs['out_filename'],
''.join([inputs['out_filename'], 'chop.nc']),
start_index, end_index, air_pressure)
except:
return (5, None)
if data_issues:
return (1, ''.join([inputs['out_filename'], 'chop.nc']))
else:
return (0, ''.join([inputs['out_filename'], 'chop.nc']))
def change_netCDFTime(in_file_name, out_file_name, start_ms):
shutil.copy(in_file_name, out_file_name)
time_len = len(nc.get_time(out_file_name))
new_time = unit_conversion.generate_ms(start_ms, time_len, 1 / 900)
nc.set_variable_data(out_file_name, 'time', new_time)
def plot_pressure(self):
#check to see if there is already a graph if so destroy it
if self.canvas != None:
self.toolbar.destroy()
self.canvas.get_tk_widget().destroy()
font = {'family': 'Bitstream Vera Sans', 'size': 11}
matplotlib.rc('font', **font)
plt.rcParams['figure.figsize'] = (12, 7)
plt.rcParams['figure.facecolor'] = 'silver'
#get date times from netCDF file
self.t_dates = nc.get_time(self.fname)
self.first_date = uc.convert_ms_to_date(self.t_dates[0], pytz.UTC)
self.last_date = uc.convert_ms_to_date(self.t_dates[-1], pytz.UTC)
self.new_dates = uc.adjust_from_gmt([self.first_date,self.last_date], \
self.tzstringvar.get(),self.daylightSavings.get())
self.first_date = mdates.date2num(self.new_dates[0])
self.last_date = mdates.date2num(self.new_dates[1])
#get sea or air pressure depending on the radio button
if self.methodvar.get() == "sea":
p = nc.get_pressure(self.fname)
else:
p = nc.get_air_pressure(self.fname)
#get quality control data
# qc = nc.get_flags(self.fname)
self.fig = fig = plt.figure(figsize=(12, 7))
self.ax = fig.add_subplot(111)
#title
self.ax.set_title('Chop Pressure File\n(Timezone in %s time)' %
self.tzstringvar.get())
#x axis formatter for dates (function format_date() below)
self.ax.xaxis.set_major_formatter(
ticker.FuncFormatter(self.format_date))
#converts dates to numbers for matplotlib to consume
self.time_nums = np.linspace(self.first_date, self.last_date,
len(self.t_dates))
#labels, and plot time series
self.line = self.ax.plot(self.time_nums, p, color='blue')
# plt.xlabel('Time (s)')
plt.ylabel('Pressure (dBar)')
#all points that were flagged in the qc data are stored in
#bad_points and bad_times to draw a red x in the graph
# data = {'Pressure': pd.Series(p,index=self.time_nums),
# 'PressureQC': pd.Series(qc, index=self.time_nums)}
# df = pd.DataFrame(data)
#
# df.Pressure[(df['PressureQC'] == 11111111) | (df['PressureQC'] == 1111011)
# | (df['PressureQC'] == 11111110) | (df['PressureQC'] == 11110110)] = np.NaN;
#
#
# self.redx = self.ax.plot(df.index, df.Pressure, 'rx')
#saves state for reloading
#set initial bounds for the time series selection
x1 = self.time_nums[0]
x2 = self.time_nums[-1]
self.left = self.ax.axvline(x1, color='black')
self.right = self.ax.axvline(x2, color='black')
#yellow highlight for selected area in graph
self.patch = self.ax.axvspan(x1,
x2,
alpha=0.25,
color='yellow',
linewidth=0)
#initial set of datestring values
temp_date = mdates.num2date(x1, tz=pytz.timezone(
"GMT")) #tz=pytz.timezone(str(self.tzstringvar.get())))
datestring = temp_date.strftime('%m/%d/%y %H:%M:%S')
self.date1.set(datestring)
temp_date2 = mdates.num2date(x2, tz=pytz.timezone(
"GMT")) #tz=pytz.timezone(str(self.tzstringvar.get())))
datestring2 = temp_date2.strftime('%m/%d/%y %H:%M:%S')
self.date2.set(datestring2)
events = []
def on_click(event):
#capture events and button pressed
events.append(event)
if event.button == 1:
l = self.left
elif event.button == 3:
l = self.right
l.set_xdata([event.xdata, event.xdata])
#get the left slice time data, convert matplotlib num to date
#format date string for the GUI start date field
x1 = self.left.get_xdata()[0]
temp_date = mdates.num2date(x1, tz=pytz.timezone(
"GMT")) #tz=pytz.timezone(str(self.tzstringvar.get())))
datestring = temp_date.strftime('%m/%d/%y %H:%M:%S')
self.plot_event = True
self.date1.set(datestring)
#get the left slice time data, convert matplotlib num to date
#format date string for the GUI start date field
x2 = self.right.get_xdata()[0]
temp_date2 = mdates.num2date(x2, tz=pytz.timezone(
"GMT")) #tz=pytz.timezone(str(self.tzstringvar.get())))
datestring2 = temp_date2.strftime('%m/%d/%y %H:%M:%S')
self.plot_event = True
self.date2.set(datestring2)
xy = [[x1, 0], [x1, 1], [x2, 1], [x2, 0], [x1, 0]]
self.patch.set_xy(xy)
self.canvas.draw()
def patch2(sv):
if self.plot_event == True:
self.plot_event = False
else:
try:
if sv == 1:
date = self.date1.get()
tz = pytz.timezone(
"GMT"
) #tz=pytz.timezone(str(self.tzstringvar.get())))
temp_dt = datetime.strptime(date, '%m/%d/%y %H:%M:%S')
#temp_dt = tz.localize(temp_dt, is_dst=None).astimezone(pytz.UTC)
x1 = mdates.date2num(temp_dt)
self.left.set_xdata([x1, x1])
x2 = self.right.get_xdata()[0]
xy = [[x1, 0], [x1, 1], [x2, 1], [x2, 0], [x1, 0]]
#draw yellow highlight over selected area in graph
self.patch.set_xy(xy)
self.canvas.draw()
else:
x1 = self.left.get_xdata()[0]
date = self.date2.get()
tz = pytz.timezone(
"GMT"
) #tz=pytz.timezone(str(self.tzstringvar.get())))
temp_dt = datetime.strptime(date, '%m/%d/%y %H:%M:%S')
#temp_dt = tz.localize(temp_dt, is_dst=None).astimezone(pytz.UTC)
x2 = mdates.date2num(temp_dt)
self.right.set_xdata([x2, x2])
xy = [[x1, 0], [x1, 1], [x2, 1], [x2, 0], [x1, 0]]
#draw yellow highlight over selected area in graph
#
self.patch.set_xy(xy)
self.canvas.draw()
except:
print('No event occurred')
#add event listener for clicks on the graph
#
self.date1.trace("w", lambda name, index, mode, i=1: patch2(i))
self.date2.trace("w", lambda name, index, mode, i=2: patch2(i))
self.canvas = FigureCanvasTkAgg(self.fig, master=self.root)
self.toolbar = NavigationToolbar2TkAgg(self.canvas, self.root)
self.toolbar.update()
self.canvas.mpl_connect('button_press_event', on_click)
self.canvas.show()
self.canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=2)
def custom_copy(fname, out_fname):
if os.path.exists(out_fname):
os.remove(out_fname)
#get station id for the station_id dimension
stn_site_id = nc.get_variable_data(fname, 'station_id')
t = nc.get_time(fname)[:]
d = Dataset(fname)
output = Dataset(out_fname, 'w', format='NETCDF4_CLASSIC')
output.createDimension('time', len(t))
output.createDimension("timeSeries", 1)
output.createDimension("station_id", len(stn_site_id))
# copy globals
for att in d.ncattrs():
setattr(output, att, d.__dict__[att])
setattr(output, "creator_name", "N/a")
setattr(output, "creator_email", "N/a")
setattr(output, "cdm_data_type", "STATION")
# copy variables
for key in d.variables:
if key == 'wave_wl':
continue
name = key
datatype = d.variables[key].datatype
dim = d.variables[key].dimensions
if datatype == "int32":
var = output.createVariable(name, datatype, dim)
else:
if name == "station_id":
var = output.createVariable("station_name", datatype, dim, fill_value=FILL_VALUE)
data = output.getncattr('stn_station_number')
else:
var = output.createVariable(name, datatype, dim, fill_value=FILL_VALUE)
data = None
for att in d.variables[key].ncattrs():
if att != '_FillValue':
if att == "cf_role":
setattr(var, att, "timeseries_id")
elif att == "long_name" and name == "station_id":
setattr(var, att, data)
else:
setattr(var, att, d.variables[key].__dict__[att])
if key in ['time', 'latitude','longitude','altitude']:
setattr(var, 'featureType', 'Point')
else:
setattr(var, 'featureType', 'Station')
if data is None:
var[:] = d.variables[key][:]
else:
var[:] = list(data)
d.close()
output.close()
def multi_air_pressure(self, path, title):
self.create_header()
ax = self.figure.add_subplot(self.grid_spec[1, 0:])
pos1 = ax.get_position() # get the original position
pos2 = [pos1.x0, pos1.y0, pos1.width, pos1.height + .06]
ax.set_position(pos2) # set a new position
#create the second graph title
first_title = "%s Air Pressure Time Series Comparison" % title
titleText = ax.text(0.5, 1.065,first_title, \
va='center', ha='center', transform=ax.transAxes)
ax.set_ylabel('Air Pressure in Inches of Mercury')
ax.set_xlabel('Timezone GMT')
#plot major grid lines
ax.grid(b=True, which='major', color='grey', linestyle="-")
#x axis formatter for dates (function format_date() below)
ax.xaxis.set_major_formatter(ticker.FuncFormatter(self.format_date))
legend_list, label_list = [], []
file_types = ['.nc']
for root, sub_folders, files in os.walk(path):
for file_in_root in files:
index = file_in_root.rfind('.')
if file_in_root[index:] in file_types:
file = ''.join([root, '\\', file_in_root])
try:
air_pressure = nc.get_air_pressure(
file) * unit_conversion.DBAR_TO_INCHES_OF_MERCURY
time = nc.get_time(file)
first_date = unit_conversion.convert_ms_to_date(
time[0], pytz.UTC)
last_date = unit_conversion.convert_ms_to_date(
time[-1], pytz.UTC)
first_date = mdates.date2num(first_date)
last_date = mdates.date2num(last_date)
self.time_nums = np.linspace(first_date, last_date,
len(time))
lat = nc.get_variable_data(file, 'latitude')
lon = nc.get_variable_data(file, 'longitude')
stn_id = nc.get_global_attribute(
file, 'stn_instrument_id')
stn_station = nc.get_global_attribute(
file, 'stn_station_number')
p1, = ax.plot(self.time_nums, air_pressure, alpha=.5)
legend_list.append(p1)
label_list.append('STN Site ID: %s, STN Instrument ID: %s, Lat: %s, Lon: %s' \
%(stn_id, stn_station, lat, lon))
except:
pass
legend = ax.legend(legend_list,label_list
, \
bbox_to_anchor=(.95, 1.355), loc=1, borderaxespad=0.0, prop={'size':10.3},frameon=False,numpoints=1, \
title="EXPLANATION")
legend.get_title().set_position((-340, 0))
plt.savefig('b')
def multi_air_pressure(self,path, title):
self.create_header()
ax = self.figure.add_subplot(self.grid_spec[1,0:])
pos1 = ax.get_position() # get the original position
pos2 = [pos1.x0, pos1.y0, pos1.width, pos1.height + .06]
ax.set_position(pos2) # set a new position
#create the second graph title
first_title = "%s Air Pressure Time Series Comparison" % title
titleText = ax.text(0.5, 1.065,first_title, \
va='center', ha='center', transform=ax.transAxes)
ax.set_ylabel('Air Pressure in Inches of Mercury')
ax.set_xlabel('Timezone GMT')
#plot major grid lines
ax.grid(b=True, which='major', color='grey', linestyle="-")
#x axis formatter for dates (function format_date() below)
ax.xaxis.set_major_formatter(ticker.FuncFormatter(self.format_date))
legend_list, label_list = [], []
file_types = ['.nc']
for root, sub_folders, files in os.walk(path):
for file_in_root in files:
index = file_in_root.rfind('.')
if file_in_root[index:] in file_types:
file = ''.join([root,'\\',file_in_root])
try:
air_pressure = nc.get_air_pressure(file) * unit_conversion.DBAR_TO_INCHES_OF_MERCURY
time = nc.get_time(file)
first_date = unit_conversion.convert_ms_to_date(time[0], pytz.UTC)
last_date = unit_conversion.convert_ms_to_date(time[-1], pytz.UTC)
first_date = mdates.date2num(first_date)
last_date = mdates.date2num(last_date)
self.time_nums = np.linspace(first_date, last_date, len(time))
lat = nc.get_variable_data(file, 'latitude')
lon = nc.get_variable_data(file, 'longitude')
stn_id = nc.get_global_attribute(file, 'stn_instrument_id')
stn_station = nc.get_global_attribute(file, 'stn_station_number')
p1, = ax.plot(self.time_nums, air_pressure, alpha=.5)
legend_list.append(p1)
label_list.append('STN Site ID: %s, STN Instrument ID: %s, Lat: %s, Lon: %s' \
%(stn_id, stn_station, lat, lon))
except:
pass
legend = ax.legend(legend_list,label_list
, \
bbox_to_anchor=(.95, 1.355), loc=1, borderaxespad=0.0, prop={'size':10.3},frameon=False,numpoints=1, \
title="EXPLANATION")
legend.get_title().set_position((-340, 0))
plt.savefig('b')
def display(self):
# try:
t = nc.get_time(self.filename) / 1000
#get datum
datum = nc.get_geospatial_vertical_reference(self.filename)
# times = nc.get_datetimes(self.file_name)
# d = nc.get_depth(self.filename)
d = nc.get_variable_data(self.filename, 'wave_wl')
tstep = t[1] - t[0]
#STATISTICS FUNCTION AVAILABLE FOR PLOTTING
periodfunc = lambda depth: stats.significant_wave_period(depth, tstep)
sigfunc = stats.significant_wave_height_standard
t10func = lambda depth: stats.ten_percent_wave_height(t, depth)
t1func = lambda depth: stats.one_percent_wave_height(t, depth)
rms_func = lambda depth: stats.rms_wave_height(t, depth)
median_func = lambda depth: stats.median_wave_height(t, depth)
maximum_func = lambda depth: stats.maximum_wave_height(t, depth)
average_func = lambda depth: stats.average_wave_height(t, depth)
average_z_func = lambda depth: stats.average_zero_crossing_period(t, depth)
mean_func = lambda depth: stats.mean_wave_period(t, depth)
crest_func = lambda depth: stats.crest_wave_period(t, depth)
peak_wave_func = lambda depth: stats.peak_wave_period(t, depth)
funcs = {'H1/3': (sigfunc, 'H 1/3 (feet)'), # contains functions and labels
'T1/3': (periodfunc, 'T 1/3 (seconds)'),
'T 10%': (t10func, 'T 10%'),
'T 1%': (t1func, 'T 1%'),
'RMS' : (rms_func, 'RMS'),
'Median': (median_func, 'Median'),
'Maximum': (maximum_func, 'Maximum'),
'Average': (average_func, 'Average'),
'Average Z Cross': (average_z_func, 'Average Z Cross'),
'Mean Wave Period': (mean_func, 'Mean Wave Period'),
'Crest': (crest_func, 'Crest'),
'Peak Wave': (peak_wave_func, 'Peak')}
size = self.sizes[self.chunk_size.get()] * float(self.number.get())
try:
tchunks = stats.split_into_chunks(t, tstep, size)
dchunks = stats.split_into_chunks(d, tstep, size)
except ValueError:
print('no chunks')
tchunks = [t]
dchunks = [d]
t_stat = [np.average(tchunk) for tchunk in tchunks]
# t_stat = [uc.convert_ms_to_datestring(t, pytz.utc) for t in t_stat]
func = funcs[self.stat.get()][0]
label = funcs[self.stat.get()][1]
d_stat = [func(dchunk) for dchunk in dchunks]
print(d_stat)
#The first x axis time conversion
t = [uc.convert_ms_to_date(x * 1000, pytz.UTC) for x in t]
t = uc.adjust_from_gmt(t, self.tzstringvar.get(), self.daylightSavings.get())
t = [mdates.date2num(x) for x in t]
#The second x axis time conversion
t_stat = [uc.convert_ms_to_date(x * 1000, pytz.UTC) for x in t_stat]
t_stat = uc.adjust_from_gmt(t_stat, self.tzstringvar.get(), self.daylightSavings.get())
t_stat = [mdates.date2num(x) for x in t_stat]
self.plot_stats(t, d, t_stat, d_stat, label, datum)
plt.show()
def get_points(fname):
time = nc.get_time(fname)
return (time[0], time[:-1])
def process_file(args):
daylight_savings = False
if args.daylight_savings.lower() == 'true':
daylight_savings = True
inputs = {
'in_filename' : args.in_file_name,
'out_filename' : args.out_file_name,
'creator_name' : args.creator_name,
'creator_email' : args.creator_email,
'creator_url' : args.creator_url,
'instrument_name' : args.instrument_name,
'stn_station_number': args.stn_station_number,
'stn_instrument_id': args.stn_instrument_id,
'latitude' : args.latitude,
'longitude' : args.longitude,
'tz_info' : args.tz_info,
'daylight_savings': daylight_savings,
'datum': args.datum,
'initial_sensor_orifice_elevation': args.initial_sensor_orifice_elevation,
'final_sensor_orifice_elevation': args.final_sensor_orifice_elevation,
'salinity' : args.salinity,
'initial_land_surface_elevation': args.initial_land_surface_elevation,
'final_land_surface_elevation': args.final_land_surface_elevation,
'deployment_time' : args.deployment_time,
'retrieval_time' : args.retrieval_time,
'sea_name' : args.sea_name,
'pressure_type' : args.pressure_type,
'good_start_date': args.good_start_date,
'good_end_date': args.good_end_date
}
#checks for the correct file type
if check_file_type(inputs['in_filename']) == False:
return 2
#check for dates in chronological order if sea pressure file
if inputs['pressure_type'] == 'Sea Pressure':
inputs['deployment_time'] = uc.datestring_to_ms(inputs['deployment_time'], '%Y%m%d %H%M', \
inputs['tz_info'],
inputs['daylight_savings'])
inputs['retrieval_time'] = uc.datestring_to_ms(inputs['retrieval_time'], '%Y%m%d %H%M', \
inputs['tz_info'],
inputs['daylight_savings'])
if inputs['retrieval_time'] <= inputs['deployment_time']:
return 4
data_issues = convert_to_netcdf(inputs)
time = nc.get_time(inputs['out_filename'])
start_index = find_index(time,uc.datestring_to_ms(inputs['good_start_date'], '%Y%m%d %H%M', \
inputs['tz_info'],
inputs['daylight_savings']))
end_index = find_index(time,uc.datestring_to_ms(inputs['good_end_date'], '%Y%m%d %H%M', \
inputs['tz_info'],
inputs['daylight_savings']))
#checks for chronological order of dates
if end_index <= start_index:
return 4
air_pressure = False
if args.pressure_type == 'Air Pressure':
air_pressure = True
try:
nc.chop_netcdf(inputs['out_filename'], ''.join([inputs['out_filename'],'chop.nc']),
start_index, end_index, air_pressure)
except:
return 5
if data_issues:
return 1
else:
return 0