def average_ensembles(self,ens_to_avg):
""" Extra variables must be averaged for this subclass
"""
a = super(ADCPRdiWorkhorseData,self).average_ensembles(ens_to_avg)
n2 = a.n_ensembles
nn = range(n2*ens_to_avg)
if a.heading is not None:
head = a.heading[nn].reshape(n2,ens_to_avg)
a.heading = np.zeros(n2,np.float64)
for i in range(n2):
a.heading[i] = ssm.circmean(head[i,:]*np.pi/180)*180/np.pi
if a.bt_depth is not None:
a.bt_depth = au.average_vector(self.bt_depth[0,nn],(n2,ens_to_avg))
a.bt_depth = np.array([a.bt_depth]) # reformat into downward vector
if a.adcp_depth is not None:
a.adcp_depth = au.average_vector(self.adcp_depth[nn],(n2,ens_to_avg))
if a.bt_velocity is not None:
a.bt_velocity = np.zeros((n2,2),np.float64)
for i in range(2):
a.bt_velocity[:,i] = au.average_array(self.bt_velocity[nn,i],(n2,ens_to_avg),axis=0)
a.bt_velocity = au.average_array(self.bt_velocity[nn,:],(n2,ens_to_avg),axis=0)
if a.error_vel is not None:
a.error_vel = au.average_array(self.error_vel[nn,:],(n2,ens_to_avg),axis=0)
return a
def average_ensembles(self, ens_to_avg):
""" Extra variables must be averaged for this subclass
"""
a = super(ADCPRdiWorkhorseData, self).average_ensembles(ens_to_avg)
n2 = a.n_ensembles
nn = range(n2 * ens_to_avg)
if a.heading is not None:
head = a.heading[nn].reshape(n2, ens_to_avg)
a.heading = np.zeros(n2, np.float64)
for i in range(n2):
a.heading[i] = ssm.circmean(
head[i, :] * np.pi / 180) * 180 / np.pi
if a.bt_depth is not None:
a.bt_depth = au.average_vector(self.bt_depth[0, nn],
(n2, ens_to_avg))
a.bt_depth = np.array([a.bt_depth
]) # reformat into downward vector
if a.adcp_depth is not None:
a.adcp_depth = au.average_vector(self.adcp_depth[nn],
(n2, ens_to_avg))
if a.bt_velocity is not None:
a.bt_velocity = np.zeros((n2, 2), np.float64)
for i in range(2):
a.bt_velocity[:, i] = au.average_array(self.bt_velocity[nn, i],
(n2, ens_to_avg),
axis=0)
a.bt_velocity = au.average_array(self.bt_velocity[nn, :],
(n2, ens_to_avg),
axis=0)
if a.error_vel is not None:
a.error_vel = au.average_array(self.error_vel[nn, :],
(n2, ens_to_avg),
axis=0)
return a
def calculate_circular_mean(values_in_hours):
"""
Calculates circular mean for timestamps, expressed as number of hours since
midnight.
Returns a float, representing a circular mean.
values_in_hours -- list of timestamps, expressed as number of hours since
midnight.
"""
if not isinstance(values_in_hours, list):
raise ValueError("Timestamps should be a list!")
for i in values_in_hours:
if not isinstance(i, (int, long, float)):
raise ValueError("{value} is not a number!".format(value=i))
values_in_radians = []
for value in values_in_hours:
values_in_radians.append((float(value) / 24) * (math.pi * 2))
return morestats.circmean(np.array(values_in_radians)) / (math.pi * 2) * 24
def calculate_circular_mean(values_in_hours):
"""
Calculates circular mean for timestamps, expressed as number of hours since
midnight.
Returns a float, representing a circular mean.
values_in_hours -- list of timestamps, expressed as number of hours since
midnight.
"""
if not isinstance(values_in_hours, list):
raise ValueError("Timestamps should be a list!")
for i in values_in_hours:
if not isinstance(i, (int, long, float)):
raise ValueError("{value} is not a number!".format(value=i))
values_in_radians = []
for value in values_in_hours:
values_in_radians.append((float(value) / 24) * (math.pi * 2))
return morestats.circmean(np.array(values_in_radians)) / (math.pi * 2) * 24
def transect_preprocessor(option_file=None):
"""
The method finds, loads, pre-preprocess, and returns ADCPData ojects
for ADCPRdiWorkhorseData compatible raw/netcdf files. It returns
a list of ADCPData objects.
See the default options file 'transect_preprocessor_input.py' for the
input options.
"""
np.seterr(all='ignore')
data_files = None
data_files_nc = None
if option_file is None:
option_file = default_option_file
try:
options, fileExtension = os.path.splitext(option_file)
exec('import %s' % options)
exec('reload(%s)' % options)
exec("from %s import *" % options)
except:
print "Could not load options file: %s" % option_file
raise
if os.path.exists(working_directory):
if (os.path.isdir(working_directory)):
if file_list is not None:
data_files = []
for f in file_list:
fileName, fileExtension = os.path.splitext(f)
if (('R.000' in f and fileExtension == '.000')
or ('r.000' in f and fileExtension == '.000')):
data_files.append(os.path.join(working_directory, f))
elif fileExtension == '.nc':
data_files.append(os.path.join(working_directory, f))
else:
print "Filename '%s' does not appear to be a valid raw (*r.000) or netcdf (*.nc) file - skipping." % f
else:
data_files = glob.glob(
os.path.join(working_directory, '*[rR].000'))
data_files += glob.glob(os.path.join(working_directory,
'*.nc'))
data_path = working_directory
else:
print "Could not open working_directory '%s' - exiting." % working_directory
exit()
else:
print "working_directory '%s' not found - exiting." % working_directory
exit()
outpath = os.path.join(data_path, 'adcpy')
if not os.path.exists(outpath):
os.makedirs(outpath)
# copy processing options
shutil.copyfile(option_file, os.path.join(outpath, option_file))
hc_count = 0
if head_correct_spanning and (len(data_files) > 1):
# bin data rquired for head_correct form input files
mtime_a = None
mtime_b = None
reference_heading = None
is_a = list()
print 'Gathering data from data_files for head_correct spanning...'
for data_file in data_files:
path, fname = os.path.split(data_file)
# try:
a = adcpy.open_adcp(data_file,
file_type="ADCPRdiWorkhorseData",
num_av=1,
adcp_depth=adcp_depth)
m1, h1, bt1, xy1 = a.copy_head_correct_vars(xy_srs=xy_projection)
if reference_heading is None:
reference_heading = ssm.circmean(
h1 * np.pi / 180.) * 180. / np.pi
current_heading = ssm.circmean(h1 * np.pi / 180.) * 180. / np.pi
if mtime_a is None:
mtime_a = m1
heading_a = h1
bt_vel_a = bt1
xy_a = xy1
else:
mtime_a = np.concatenate((mtime_a, m1))
heading_a = np.concatenate((heading_a, h1))
bt_vel_a = np.row_stack((bt_vel_a, bt1))
xy_a = np.row_stack((xy_a, xy1))
print '+', fname, fmt_dnum(a.mtime[0])
if debug_stop_after_n_transects:
hc_count += 1
if hc_count >= debug_stop_after_n_transects:
break
# except:
# print 'Failure reading %s for head_correct spanning!'%fname
# is_a.append('True')
print 'Number direction a headings:', np.shape(mtime_a)
# this method is independent of self/a
#try:
heading_correction_a = adcpy.util.fit_head_correct(
mtime_in=mtime_a,
hdg_in=heading_a,
bt_vel_in=bt_vel_a,
xy_in=xy_a,
u_min_bt=u_min_bt,
hdg_bin_size=hdg_bin_size,
hdg_bin_min_samples=hdg_bin_min_samples)
# except:
# print 'Failure fitting head_correct spanning!'
# if mag_declination is not None:
# print 'Using simple magnetic declination correction instead'
# heading_correction_a = None
# else:
# print 'No magnetic declination value found - head_correct failure.'
# print 'exiting'
# exit()
# setup compression option
if use_netcdf_data_compression:
zlib = True
else:
zlib = None
# begin cycling/processing input files
adcp_preprocessed = []
for data_file in data_files:
# try:
a = adcpy.open_adcp(data_file,
file_type='ADCPRdiWorkhorseData',
num_av=1,
adcp_depth=adcp_depth)
path, fname = os.path.split(data_file)
fname, ext = os.path.splitext(fname)
outname = os.path.join(outpath, fname)
print 'Processing data_file:', outname
if save_raw_data_to_netcdf:
fname = outname + '.nc'
a.write_nc(fname, zlib=zlib)
# setup for heading correction based
if head_correct_spanning and (len(data_files) > 1):
heading_cf = heading_correction_a
else:
heading_cf = None
a.lonlat_to_xy(xy_srs=xy_projection)
a.heading_correct(cf=heading_cf,
u_min_bt=u_min_bt,
hdg_bin_size=hdg_bin_size,
hdg_bin_min_samples=hdg_bin_min_samples,
mag_dec=mag_declination)
if sidelobe_drop != 0:
a.remove_sidelobes(fsidelobe=sidelobe_drop)
if std_drop > 0:
a.sd_drop(sd=std_drop, sd_axis='elevation', interp_holes=True)
a.sd_drop(sd=std_drop, sd_axis='ensemble', interp_holes=True)
if average_ens > 1:
a = a.average_ensembles(ens_to_avg=average_ens)
if regrid_horiz_m is not None:
a.xy_regrid(dxy=regrid_horiz_m,
dz=regrid_vert_m,
xy_srs=xy_projection,
pline=None,
sort=False)
if smooth_kernel > 2:
a.kernel_smooth(kernel_size=smooth_kernel)
if extrap_boundaries:
a.extrapolate_boundaries()
if (save_preprocessed_data_to_netcdf):
fname = outname + '.preprocessed.nc'
a.write_nc(fname, zlib=zlib)
adcp_preprocessed.append(a)
if debug_stop_after_n_transects:
if len(adcp_preprocessed) >= debug_stop_after_n_transects:
return (adcp_preprocessed, outpath)
return (adcp_preprocessed, outpath)
def analyze_file(sky,fname=None):
"""
example:
power,angle,intensity=polarimetry.analyze_file(sky)
"""
WAIT_TIME = 20 #seconds after changeTimes to start polarimetry
ROT180 = True #because camera returns rotated image
if fname is None:
fileName = sky['fileName']
else:
fileName = fname
dirName = sky['dirName']
N = len(sky['changeTimes'][:-1])
fmf = FMF.FlyMovie(os.path.join(dirName,fileName))
frame,timestamp = fmf.get_frame(0)
if not sky.has_key('times'):
timestamps = fmf.get_all_timestamps()
else:
timestamps = sky['times']
for i, startTime in enumerate(sky['changeTimes'][:-1]):
startInd = np.argmin(abs(startTime + WAIT_TIME - timestamps))
sys.stdout.write(time.ctime(startTime + WAIT_TIME)+'\n')
#sys.stdout.write("%s\n" % (str(i)))
sys.stdout.flush()
pwr, phs, ints = do_polarimetry(fmf,firstFrame=startInd,nFrames=500)
phs = phs - circmean(np.ravel(phs[222:261,222:272]),high=np.pi,low=-np.pi)
ang = phs/2.0 # because phase offset of intensity values is twice angle between overlapping polarizers
if ROT180:
pwr = np.rot90(pwr,2)
ang = np.rot90(ang,2)
ints = np.rot90(ints,2)
"""
trueUpDirection = filename[3]
if trueUpDirection == 'E':
pwr = np.rot90(pwr,1)
ang = np.rot90(ang,1)
ints = np.rot90(ints,1)
ang = ang + np.pi/2
elif trueUpDirection == 'S':
pwr = np.rot90(pwr,2)
ang = np.rot90(ang,2)
ints = np.rot90(ints,2)
ang = ang + np.pi
elif trueUpDirection == 'W':
pwr = np.rot90(pwr,3)
ang = np.rot90(ang,3)
ints = np.rot90(ints,3)
ang = ang + 3*np.pi/2
"""
mask = ints>(np.mean(ints)-.45*np.std(ints)) #hack
mask[100:300,100:300] = True #not sure if central dot (polarizer) should be in or not... if so - threshold should be ~1 std below mean intensity
pwr[~mask] = np.nan
ang[~mask] = np.nan
ang = np.mod(ang+np.pi/2,2*np.pi)-np.pi/2
ang = cmt.add_colordisc(ang,width=71)
#ang = np.mod(ang+np.pi,2*np.pi)-np.pi
ang = np.mod(ang,2*np.pi)
if i==0:
w,h=ang.shape
power = np.empty([w,h,N])
power.fill(np.nan)
angle = np.empty([w,h,N])
angle.fill(np.nan)
intensity = np.empty([w,h,N])
intensity.fill(np.nan)
power[:,:,i],angle[:,:,i],intensity[:,:,i]=pwr,ang,ints
return power, angle, intensity
inds = (times > TOTAL_TIME + times[0])
orientations[inds] = numpy.nan
pylab.draw()
fig = pylab.figure()
fig.set_facecolor('w')
pylab.suptitle(fly['fileName'])
for i, cT in enumerate(sky['changeTimes'][:-1]):
pylab.subplot(5,4,1+i,polar=True)
ax=pylab.gca()
inds = (times > cT) & (times < sky['changeTimes'][i+1])
ors = orientations[inds]
ors = ors[~numpy.isnan(ors)]
if len(ors)>0:
orw,n,b,bc,ax = flypod.rose(ors,360)
m=circmean(ors,high=180,low=-180)
v=circvar(ors*numpy.pi/180,high=numpy.pi,low=-numpy.pi)
hold('on')
polar([0,numpy.pi/2-m*numpy.pi/180],[0,ax.get_rmax()*(1-v)])
ax.set_rmax(.4)
ax.set_rgrids([1],'')
ax.set_thetagrids([0,90,180,270],['','','',''])
title(sky['directions'][i])
#ax.set_axis_bgcolor(COLORS[sky['directions'][i]])
ax.axesPatch.set_facecolor(COLORS[sky['directions'][i]])
ax.axesPatch.set_alpha(0.4)
totals[sky['directions'][i]] = numpy.concatenate((totals[sky['directions'][i]],ors))
pylab.draw()
totalTimeRecording[dNum] = (times[-1] - times[0])
inds = (times - times[0]) > MAX_TIME
orientations[inds] = numpy.nan
for i, cT in enumerate(sky['changeTimes'][:-1]):
inds = (times > cT+POSTCHANGE_BUFFER) & (times < sky['changeTimes'][i+1]-PRECHANGE_BUFFER)
ors = orientations[inds]
ors = ors[~numpy.isnan(ors)]
if len(ors)>0:
totals[sky['directions'][i]] = numpy.concatenate((totals[sky['directions'][i]],ors))
for i, d in enumerate(COLORS):
worldTotals = numpy.concatenate((worldTotals,totals[d]+ROTATIONS[d]))
if totalTimeRecording[dNum] > MIN_TIME and (totalTimeRecording[dNum] - timeStopped[dNum]) > MIN_TIME*.8:
M[dNum]=circmean(worldTotals,high=180,low=-180)
V[dNum]=circvar(worldTotals*numpy.pi/180,high=numpy.pi,low=-numpy.pi)
#meanAngSp[dNum] = numpy.mean(abs(numpy.diff(numpy.unwrap(orientations[~numpy.isnan(orientations)],180))))
#V[dNum]=numpy.mean(abs(numpy.diff(worldTotals)))
#fig = pylab.figure()
pylab.subplot(4,5,dNum+1,polar=True)
plotArgs = dict(color='k',linewidth=2)
orw,n,b,bc,ax = flypod.rose(worldTotals,plotArgs=plotArgs)
pylab.hold('on')
#n={}
for i, d in enumerate(COLORS):
plotArgs = dict(color=COLORS[d],linewidth=.5)
flypod.rose(totals[d]+ROTATIONS[d],plotArgs=plotArgs)
#NUMBINS = 8
#n[d], bins, patches = pylab.hist(numpy.mod(totals[d]+ROTATIONS[d],360),bins=numpy.arange(NUMBINS+1)*360/NUMBINS,range=(0,360),normed=True,visible=False)
def transect_preprocessor(option_file=None):
"""
The method finds, loads, pre-preprocess, and returns ADCPData ojects
for ADCPRdiWorkhorseData compatible raw/netcdf files. It returns
a list of ADCPData objects.
See the default options file 'transect_preprocessor_input.py' for the
input options.
"""
np.seterr(all='ignore')
data_files = None
data_files_nc = None
if option_file is None:
option_file = default_option_file
try:
options, fileExtension = os.path.splitext(option_file)
exec('import %s'%options)
exec('reload(%s)'%options)
exec("from %s import *"%options)
except:
print "Could not load options file: %s"%option_file
raise
if os.path.exists(working_directory):
if (os.path.isdir(working_directory)):
if file_list is not None:
data_files = []
for f in file_list:
fileName, fileExtension = os.path.splitext(f)
if (('R.000' in f and fileExtension == '.000') or
('r.000' in f and fileExtension == '.000')):
data_files.append(os.path.join(working_directory,f))
elif fileExtension == '.nc':
data_files.append(os.path.join(working_directory,f))
else:
print "Filename '%s' does not appear to be a valid raw (*r.000) or netcdf (*.nc) file - skipping."%f
else:
data_files = glob.glob(os.path.join(working_directory,'*[rR].000'))
data_files += glob.glob(os.path.join(working_directory,'*.nc'))
data_path = working_directory
else:
print "Could not open working_directory '%s' - exiting."%working_directory
exit()
else:
print "working_directory '%s' not found - exiting."%working_directory
exit()
outpath = os.path.join(data_path,'adcpy')
if not os.path.exists(outpath):
os.makedirs(outpath)
# copy processing options
shutil.copyfile(option_file, os.path.join(outpath,option_file))
hc_count = 0
if head_correct_spanning and (len(data_files) > 1):
# bin data rquired for head_correct form input files
mtime_a = None
mtime_b = None
reference_heading = None
is_a = list()
print 'Gathering data from data_files for head_correct spanning...'
for data_file in data_files:
path, fname = os.path.split(data_file)
# try:
a = adcpy.open_adcp(data_file,
file_type="ADCPRdiWorkhorseData",
num_av=1,
adcp_depth=adcp_depth)
m1,h1,bt1,xy1 = a.copy_head_correct_vars(xy_srs=xy_projection)
if reference_heading is None:
reference_heading = ssm.circmean(h1*np.pi/180.)*180./np.pi
current_heading = ssm.circmean(h1*np.pi/180.)*180./np.pi
if mtime_a is None:
mtime_a = m1
heading_a = h1
bt_vel_a = bt1
xy_a = xy1
else:
mtime_a = np.concatenate((mtime_a,m1))
heading_a = np.concatenate((heading_a,h1))
bt_vel_a = np.row_stack((bt_vel_a,bt1))
xy_a = np.row_stack((xy_a,xy1))
print '+',fname,fmt_dnum(a.mtime[0])
if debug_stop_after_n_transects:
hc_count += 1
if hc_count >= debug_stop_after_n_transects:
break
# except:
# print 'Failure reading %s for head_correct spanning!'%fname
# is_a.append('True')
print 'Number direction a headings:',np.shape(mtime_a)
# this method is independent of self/a
#try:
heading_correction_a = adcpy.util.fit_head_correct(mtime_in=mtime_a,
hdg_in=heading_a,
bt_vel_in=bt_vel_a,
xy_in=xy_a,
u_min_bt=u_min_bt,
hdg_bin_size=hdg_bin_size,
hdg_bin_min_samples=hdg_bin_min_samples)
# except:
# print 'Failure fitting head_correct spanning!'
# if mag_declination is not None:
# print 'Using simple magnetic declination correction instead'
# heading_correction_a = None
# else:
# print 'No magnetic declination value found - head_correct failure.'
# print 'exiting'
# exit()
# setup compression option
if use_netcdf_data_compression:
zlib = True
else:
zlib = None
# begin cycling/processing input files
adcp_preprocessed = []
for data_file in data_files:
# try:
a = adcpy.open_adcp(data_file,
file_type='ADCPRdiWorkhorseData',
num_av=1,
adcp_depth=adcp_depth)
path, fname = os.path.split(data_file)
fname, ext = os.path.splitext(fname)
outname = os.path.join(outpath,fname)
print 'Processing data_file:', outname
if save_raw_data_to_netcdf:
fname = outname + '.nc'
a.write_nc(fname,zlib=zlib)
# setup for heading correction based
if head_correct_spanning and (len(data_files) > 1):
heading_cf = heading_correction_a
else:
heading_cf = None
a.lonlat_to_xy(xy_srs=xy_projection)
a.heading_correct(cf=heading_cf,
u_min_bt=u_min_bt,
hdg_bin_size=hdg_bin_size,
hdg_bin_min_samples=hdg_bin_min_samples,
mag_dec=mag_declination)
if sidelobe_drop != 0:
a.remove_sidelobes(fsidelobe=sidelobe_drop)
if std_drop > 0:
a.sd_drop(sd=std_drop,
sd_axis='elevation',
interp_holes=True)
a.sd_drop(sd=std_drop,
sd_axis='ensemble',
interp_holes=True)
if average_ens > 1:
a = a.average_ensembles(ens_to_avg=average_ens)
if regrid_horiz_m is not None:
a.xy_regrid(dxy=regrid_horiz_m,
dz=regrid_vert_m,
xy_srs=xy_projection,
pline=None,
sort=False)
if smooth_kernel > 2:
a.kernel_smooth(kernel_size = smooth_kernel)
if extrap_boundaries:
a.extrapolate_boundaries()
if (save_preprocessed_data_to_netcdf):
fname = outname + '.preprocessed.nc'
a.write_nc(fname,zlib=zlib)
adcp_preprocessed.append(a)
if debug_stop_after_n_transects:
if len(adcp_preprocessed) >= debug_stop_after_n_transects:
return (adcp_preprocessed,outpath)
return (adcp_preprocessed,outpath)
import arcpy
from scipy.stats import morestats
raster_path = "data/aspect_raster.tif"
r = arcpy.RasterToNumPyArray(raster_path)
# can't take a log of 0, offset values
r += 0.01
print("""Circular Mean:\t\t\t{}
Circular Std. Dev.:\t\t\t{}
Circular Variance:\t\t\t{}
""".format(
morestats.circmean(r),
morestats.circstd(r),
morestats.circvar(r)))
import arcpy
from scipy.stats import morestats
raster_path = "data/aspect_raster.tif"
r = arcpy.RasterToNumPyArray(raster_path)
# can't take a log of 0, offset values
r += 0.01
print("""Circular Mean:\t\t\t{}
Circular Std. Dev.:\t\t\t{}
Circular Variance:\t\t\t{}
""".format(morestats.circmean(r), morestats.circstd(r), morestats.circvar(r)))