Exemple #1
0
 def average_ensembles(self,ens_to_avg):
     """ Extra variables must be averaged for this subclass
     """
     a = super(ADCPRdiWorkhorseData,self).average_ensembles(ens_to_avg)
     n2 = a.n_ensembles
     nn = range(n2*ens_to_avg)
     if a.heading is not None:
         head = a.heading[nn].reshape(n2,ens_to_avg)
         a.heading = np.zeros(n2,np.float64)
         for i in range(n2):
             a.heading[i] = ssm.circmean(head[i,:]*np.pi/180)*180/np.pi
     if a.bt_depth is not None:
         a.bt_depth = au.average_vector(self.bt_depth[0,nn],(n2,ens_to_avg))
         a.bt_depth = np.array([a.bt_depth]) # reformat into downward vector
     if a.adcp_depth is not None:
         a.adcp_depth = au.average_vector(self.adcp_depth[nn],(n2,ens_to_avg))
     if a.bt_velocity is not None:                
         a.bt_velocity = np.zeros((n2,2),np.float64)
         for i in range(2):
             a.bt_velocity[:,i] = au.average_array(self.bt_velocity[nn,i],(n2,ens_to_avg),axis=0)  
             a.bt_velocity = au.average_array(self.bt_velocity[nn,:],(n2,ens_to_avg),axis=0)
     if a.error_vel is not None:
         a.error_vel = au.average_array(self.error_vel[nn,:],(n2,ens_to_avg),axis=0)
 
     return a
    def average_ensembles(self, ens_to_avg):
        """ Extra variables must be averaged for this subclass
        """
        a = super(ADCPRdiWorkhorseData, self).average_ensembles(ens_to_avg)
        n2 = a.n_ensembles
        nn = range(n2 * ens_to_avg)
        if a.heading is not None:
            head = a.heading[nn].reshape(n2, ens_to_avg)
            a.heading = np.zeros(n2, np.float64)
            for i in range(n2):
                a.heading[i] = ssm.circmean(
                    head[i, :] * np.pi / 180) * 180 / np.pi
        if a.bt_depth is not None:
            a.bt_depth = au.average_vector(self.bt_depth[0, nn],
                                           (n2, ens_to_avg))
            a.bt_depth = np.array([a.bt_depth
                                   ])  # reformat into downward vector
        if a.adcp_depth is not None:
            a.adcp_depth = au.average_vector(self.adcp_depth[nn],
                                             (n2, ens_to_avg))
        if a.bt_velocity is not None:
            a.bt_velocity = np.zeros((n2, 2), np.float64)
            for i in range(2):
                a.bt_velocity[:, i] = au.average_array(self.bt_velocity[nn, i],
                                                       (n2, ens_to_avg),
                                                       axis=0)
                a.bt_velocity = au.average_array(self.bt_velocity[nn, :],
                                                 (n2, ens_to_avg),
                                                 axis=0)
        if a.error_vel is not None:
            a.error_vel = au.average_array(self.error_vel[nn, :],
                                           (n2, ens_to_avg),
                                           axis=0)

        return a
Exemple #3
0
	def calculate_circular_mean(values_in_hours):
		"""
		Calculates circular mean for timestamps, expressed as number of hours since
		midnight.

		Returns a float, representing a circular mean.

		values_in_hours -- list of timestamps, expressed as number of hours since 
		midnight.
		"""
		if not isinstance(values_in_hours, list):
			raise ValueError("Timestamps should be a list!")
		for i in values_in_hours:
			if not isinstance(i, (int, long, float)):
				raise ValueError("{value} is not a number!".format(value=i))
		values_in_radians = []
		for value in values_in_hours:
			values_in_radians.append((float(value) / 24) * (math.pi * 2))
		return morestats.circmean(np.array(values_in_radians)) / (math.pi * 2) * 24
    def calculate_circular_mean(values_in_hours):
        """
		Calculates circular mean for timestamps, expressed as number of hours since
		midnight.

		Returns a float, representing a circular mean.

		values_in_hours -- list of timestamps, expressed as number of hours since 
		midnight.
		"""
        if not isinstance(values_in_hours, list):
            raise ValueError("Timestamps should be a list!")
        for i in values_in_hours:
            if not isinstance(i, (int, long, float)):
                raise ValueError("{value} is not a number!".format(value=i))
        values_in_radians = []
        for value in values_in_hours:
            values_in_radians.append((float(value) / 24) * (math.pi * 2))
        return morestats.circmean(np.array(values_in_radians)) / (math.pi * 2) * 24
def transect_preprocessor(option_file=None):
    """
    The method finds, loads, pre-preprocess, and returns ADCPData ojects
    for ADCPRdiWorkhorseData compatible raw/netcdf files.  It returns
    a list of ADCPData objects.
    
    See the default options file 'transect_preprocessor_input.py' for the 
    input options.
    """
    np.seterr(all='ignore')

    data_files = None
    data_files_nc = None

    if option_file is None:
        option_file = default_option_file
    try:
        options, fileExtension = os.path.splitext(option_file)
        exec('import %s' % options)
        exec('reload(%s)' % options)
        exec("from %s import *" % options)
    except:
        print "Could not load options file: %s" % option_file
        raise

    if os.path.exists(working_directory):
        if (os.path.isdir(working_directory)):
            if file_list is not None:
                data_files = []
                for f in file_list:
                    fileName, fileExtension = os.path.splitext(f)
                    if (('R.000' in f and fileExtension == '.000')
                            or ('r.000' in f and fileExtension == '.000')):
                        data_files.append(os.path.join(working_directory, f))
                    elif fileExtension == '.nc':
                        data_files.append(os.path.join(working_directory, f))
                    else:
                        print "Filename '%s' does not appear to be a valid raw (*r.000) or netcdf (*.nc) file - skipping." % f
            else:
                data_files = glob.glob(
                    os.path.join(working_directory, '*[rR].000'))
                data_files += glob.glob(os.path.join(working_directory,
                                                     '*.nc'))
            data_path = working_directory
        else:
            print "Could not open working_directory '%s' - exiting." % working_directory
            exit()
    else:
        print "working_directory '%s' not found - exiting." % working_directory
        exit()

    outpath = os.path.join(data_path, 'adcpy')
    if not os.path.exists(outpath):
        os.makedirs(outpath)

    # copy processing options
    shutil.copyfile(option_file, os.path.join(outpath, option_file))

    hc_count = 0
    if head_correct_spanning and (len(data_files) > 1):
        # bin data rquired for head_correct form input files
        mtime_a = None
        mtime_b = None
        reference_heading = None
        is_a = list()
        print 'Gathering data from data_files for head_correct spanning...'
        for data_file in data_files:
            path, fname = os.path.split(data_file)

            #            try:
            a = adcpy.open_adcp(data_file,
                                file_type="ADCPRdiWorkhorseData",
                                num_av=1,
                                adcp_depth=adcp_depth)
            m1, h1, bt1, xy1 = a.copy_head_correct_vars(xy_srs=xy_projection)

            if reference_heading is None:
                reference_heading = ssm.circmean(
                    h1 * np.pi / 180.) * 180. / np.pi

            current_heading = ssm.circmean(h1 * np.pi / 180.) * 180. / np.pi

            if mtime_a is None:
                mtime_a = m1
                heading_a = h1
                bt_vel_a = bt1
                xy_a = xy1
            else:
                mtime_a = np.concatenate((mtime_a, m1))
                heading_a = np.concatenate((heading_a, h1))
                bt_vel_a = np.row_stack((bt_vel_a, bt1))
                xy_a = np.row_stack((xy_a, xy1))

            print '+', fname, fmt_dnum(a.mtime[0])

            if debug_stop_after_n_transects:
                hc_count += 1
                if hc_count >= debug_stop_after_n_transects:
                    break
#            except:

#                print 'Failure reading %s for head_correct spanning!'%fname
#                is_a.append('True')

        print 'Number direction a headings:', np.shape(mtime_a)

        # this method is independent of self/a
        #try:
        heading_correction_a = adcpy.util.fit_head_correct(
            mtime_in=mtime_a,
            hdg_in=heading_a,
            bt_vel_in=bt_vel_a,
            xy_in=xy_a,
            u_min_bt=u_min_bt,
            hdg_bin_size=hdg_bin_size,
            hdg_bin_min_samples=hdg_bin_min_samples)
#        except:
#                print 'Failure fitting head_correct spanning!'
#                if mag_declination is not None:
#                    print 'Using simple magnetic declination correction instead'
#                    heading_correction_a = None
#                else:
#                    print 'No magnetic declination value found - head_correct failure.'
#                    print 'exiting'
#                    exit()

# setup compression option
    if use_netcdf_data_compression:
        zlib = True
    else:
        zlib = None

    # begin cycling/processing input files
    adcp_preprocessed = []
    for data_file in data_files:
        #       try:
        a = adcpy.open_adcp(data_file,
                            file_type='ADCPRdiWorkhorseData',
                            num_av=1,
                            adcp_depth=adcp_depth)
        path, fname = os.path.split(data_file)
        fname, ext = os.path.splitext(fname)
        outname = os.path.join(outpath, fname)

        print 'Processing data_file:', outname

        if save_raw_data_to_netcdf:
            fname = outname + '.nc'
            a.write_nc(fname, zlib=zlib)

        # setup for heading correction based
        if head_correct_spanning and (len(data_files) > 1):
            heading_cf = heading_correction_a
        else:
            heading_cf = None

        a.lonlat_to_xy(xy_srs=xy_projection)
        a.heading_correct(cf=heading_cf,
                          u_min_bt=u_min_bt,
                          hdg_bin_size=hdg_bin_size,
                          hdg_bin_min_samples=hdg_bin_min_samples,
                          mag_dec=mag_declination)
        if sidelobe_drop != 0:
            a.remove_sidelobes(fsidelobe=sidelobe_drop)
        if std_drop > 0:
            a.sd_drop(sd=std_drop, sd_axis='elevation', interp_holes=True)
            a.sd_drop(sd=std_drop, sd_axis='ensemble', interp_holes=True)
        if average_ens > 1:
            a = a.average_ensembles(ens_to_avg=average_ens)
        if regrid_horiz_m is not None:
            a.xy_regrid(dxy=regrid_horiz_m,
                        dz=regrid_vert_m,
                        xy_srs=xy_projection,
                        pline=None,
                        sort=False)
        if smooth_kernel > 2:
            a.kernel_smooth(kernel_size=smooth_kernel)
        if extrap_boundaries:
            a.extrapolate_boundaries()

        if (save_preprocessed_data_to_netcdf):
            fname = outname + '.preprocessed.nc'
            a.write_nc(fname, zlib=zlib)

        adcp_preprocessed.append(a)

        if debug_stop_after_n_transects:
            if len(adcp_preprocessed) >= debug_stop_after_n_transects:
                return (adcp_preprocessed, outpath)

    return (adcp_preprocessed, outpath)
Exemple #6
0
def analyze_file(sky,fname=None):
    """
    example:
    power,angle,intensity=polarimetry.analyze_file(sky)
    """
    WAIT_TIME = 20 #seconds after changeTimes to start polarimetry
    ROT180 = True #because camera returns rotated image
    
    if fname is None:
        fileName = sky['fileName']
    else:
        fileName = fname
    dirName = sky['dirName']
    
    N = len(sky['changeTimes'][:-1])
    
    fmf = FMF.FlyMovie(os.path.join(dirName,fileName))
    frame,timestamp = fmf.get_frame(0)
    
    if not sky.has_key('times'):
        timestamps = fmf.get_all_timestamps()
    else:
        timestamps = sky['times']
    
    for i, startTime in enumerate(sky['changeTimes'][:-1]):
        startInd = np.argmin(abs(startTime + WAIT_TIME - timestamps))
        sys.stdout.write(time.ctime(startTime + WAIT_TIME)+'\n')
        #sys.stdout.write("%s\n" % (str(i)))
        sys.stdout.flush()
        pwr, phs, ints = do_polarimetry(fmf,firstFrame=startInd,nFrames=500)
                        
        phs = phs - circmean(np.ravel(phs[222:261,222:272]),high=np.pi,low=-np.pi)
        
        ang = phs/2.0 # because phase offset of intensity values is twice angle between overlapping polarizers
        
        if ROT180:
            pwr = np.rot90(pwr,2)
            ang = np.rot90(ang,2)
            ints = np.rot90(ints,2)
        """
        trueUpDirection = filename[3]
        if trueUpDirection == 'E':
            pwr = np.rot90(pwr,1)
            ang = np.rot90(ang,1)
            ints = np.rot90(ints,1)
            ang = ang + np.pi/2
        elif trueUpDirection == 'S':
            pwr = np.rot90(pwr,2)
            ang = np.rot90(ang,2)
            ints = np.rot90(ints,2)
            ang = ang + np.pi
        elif trueUpDirection == 'W':
            pwr = np.rot90(pwr,3)
            ang = np.rot90(ang,3)
            ints = np.rot90(ints,3)
            ang = ang + 3*np.pi/2
        """
        
        mask = ints>(np.mean(ints)-.45*np.std(ints)) #hack
        mask[100:300,100:300] = True #not sure if central dot (polarizer) should be in or not... if so - threshold should be ~1 std below mean intensity
        pwr[~mask] = np.nan
        ang[~mask] = np.nan
        
        ang = np.mod(ang+np.pi/2,2*np.pi)-np.pi/2
        ang = cmt.add_colordisc(ang,width=71)
        #ang = np.mod(ang+np.pi,2*np.pi)-np.pi
        ang = np.mod(ang,2*np.pi)
        
        if i==0:
            w,h=ang.shape
            power = np.empty([w,h,N])
            power.fill(np.nan)
            angle = np.empty([w,h,N])
            angle.fill(np.nan)
            intensity = np.empty([w,h,N])
            intensity.fill(np.nan)
            
        power[:,:,i],angle[:,:,i],intensity[:,:,i]=pwr,ang,ints
    return power, angle, intensity
Exemple #7
0
    inds = (times > TOTAL_TIME + times[0])
    orientations[inds] = numpy.nan

    pylab.draw()
    fig = pylab.figure()
    fig.set_facecolor('w')
    pylab.suptitle(fly['fileName'])
    for i, cT in enumerate(sky['changeTimes'][:-1]):
        pylab.subplot(5,4,1+i,polar=True)
        ax=pylab.gca()
        inds = (times > cT) & (times < sky['changeTimes'][i+1])
        ors = orientations[inds]
        ors = ors[~numpy.isnan(ors)]
        if len(ors)>0:
            orw,n,b,bc,ax = flypod.rose(ors,360)
            m=circmean(ors,high=180,low=-180)
            v=circvar(ors*numpy.pi/180,high=numpy.pi,low=-numpy.pi)
            hold('on')
            polar([0,numpy.pi/2-m*numpy.pi/180],[0,ax.get_rmax()*(1-v)])
            ax.set_rmax(.4)
            ax.set_rgrids([1],'')
            ax.set_thetagrids([0,90,180,270],['','','',''])
            title(sky['directions'][i])
            #ax.set_axis_bgcolor(COLORS[sky['directions'][i]])
            ax.axesPatch.set_facecolor(COLORS[sky['directions'][i]])
            ax.axesPatch.set_alpha(0.4)
            
            totals[sky['directions'][i]] = numpy.concatenate((totals[sky['directions'][i]],ors))
        
    pylab.draw()
        
Exemple #8
0
 totalTimeRecording[dNum] = (times[-1] - times[0])
 inds = (times - times[0]) > MAX_TIME
 orientations[inds] = numpy.nan
 
 for i, cT in enumerate(sky['changeTimes'][:-1]):
     inds = (times > cT+POSTCHANGE_BUFFER) & (times < sky['changeTimes'][i+1]-PRECHANGE_BUFFER)
     ors = orientations[inds]
     ors = ors[~numpy.isnan(ors)]
     if len(ors)>0:
         totals[sky['directions'][i]] = numpy.concatenate((totals[sky['directions'][i]],ors))
 
 for i, d in enumerate(COLORS):
     worldTotals = numpy.concatenate((worldTotals,totals[d]+ROTATIONS[d]))
 
 if totalTimeRecording[dNum] > MIN_TIME and (totalTimeRecording[dNum] - timeStopped[dNum]) > MIN_TIME*.8:
     M[dNum]=circmean(worldTotals,high=180,low=-180)
     V[dNum]=circvar(worldTotals*numpy.pi/180,high=numpy.pi,low=-numpy.pi)
     #meanAngSp[dNum] = numpy.mean(abs(numpy.diff(numpy.unwrap(orientations[~numpy.isnan(orientations)],180))))
     #V[dNum]=numpy.mean(abs(numpy.diff(worldTotals)))
     #fig = pylab.figure()
     pylab.subplot(4,5,dNum+1,polar=True)
     plotArgs = dict(color='k',linewidth=2)
     orw,n,b,bc,ax = flypod.rose(worldTotals,plotArgs=plotArgs)
     pylab.hold('on')
     
     #n={}
     for i, d in enumerate(COLORS):
         plotArgs = dict(color=COLORS[d],linewidth=.5)
         flypod.rose(totals[d]+ROTATIONS[d],plotArgs=plotArgs)
         #NUMBINS = 8
         #n[d], bins, patches = pylab.hist(numpy.mod(totals[d]+ROTATIONS[d],360),bins=numpy.arange(NUMBINS+1)*360/NUMBINS,range=(0,360),normed=True,visible=False)
def transect_preprocessor(option_file=None):
    """
    The method finds, loads, pre-preprocess, and returns ADCPData ojects
    for ADCPRdiWorkhorseData compatible raw/netcdf files.  It returns
    a list of ADCPData objects.
    
    See the default options file 'transect_preprocessor_input.py' for the 
    input options.
    """
    np.seterr(all='ignore')
    
    data_files = None
    data_files_nc = None
    
    if option_file is None:
        option_file = default_option_file
    try:
        options, fileExtension = os.path.splitext(option_file)
        exec('import %s'%options)
        exec('reload(%s)'%options)
        exec("from %s import *"%options)
    except:
        print "Could not load options file: %s"%option_file
        raise

    if os.path.exists(working_directory):
        if (os.path.isdir(working_directory)):
            if file_list is not None:
                data_files = []
                for f in file_list:
                    fileName, fileExtension = os.path.splitext(f)
                    if (('R.000' in f and fileExtension == '.000') or
                        ('r.000' in f and fileExtension == '.000')):
                        data_files.append(os.path.join(working_directory,f))
                    elif fileExtension == '.nc':
                        data_files.append(os.path.join(working_directory,f))
                    else:
                        print "Filename '%s' does not appear to be a valid raw (*r.000) or netcdf (*.nc) file - skipping."%f
            else:
                data_files = glob.glob(os.path.join(working_directory,'*[rR].000'))
                data_files += glob.glob(os.path.join(working_directory,'*.nc'))            
            data_path = working_directory
        else:
            print "Could not open working_directory '%s' - exiting."%working_directory
            exit()
    else:
        print "working_directory '%s' not found - exiting."%working_directory
        exit()
    
    outpath = os.path.join(data_path,'adcpy')
    if not os.path.exists(outpath):
        os.makedirs(outpath)
   
    # copy processing options
    shutil.copyfile(option_file, os.path.join(outpath,option_file))

    hc_count = 0
    if head_correct_spanning and (len(data_files) > 1):
        # bin data rquired for head_correct form input files
        mtime_a = None
        mtime_b = None
        reference_heading = None
        is_a = list()
        print 'Gathering data from data_files for head_correct spanning...'
        for data_file in data_files:
            path, fname = os.path.split(data_file)
            
#            try:
            a = adcpy.open_adcp(data_file,
                            file_type="ADCPRdiWorkhorseData",
                            num_av=1,
                            adcp_depth=adcp_depth)
            m1,h1,bt1,xy1 = a.copy_head_correct_vars(xy_srs=xy_projection)
            
            if reference_heading is None:
                reference_heading = ssm.circmean(h1*np.pi/180.)*180./np.pi

            current_heading = ssm.circmean(h1*np.pi/180.)*180./np.pi
                        
            if mtime_a is None:
                mtime_a = m1
                heading_a = h1
                bt_vel_a = bt1
                xy_a = xy1
            else:
                mtime_a = np.concatenate((mtime_a,m1))
                heading_a = np.concatenate((heading_a,h1))
                bt_vel_a = np.row_stack((bt_vel_a,bt1))
                xy_a = np.row_stack((xy_a,xy1))
                    
            print '+',fname,fmt_dnum(a.mtime[0])
            
            
            if debug_stop_after_n_transects:
                hc_count += 1
                if hc_count >= debug_stop_after_n_transects:
                    break           
#            except:
                
#                print 'Failure reading %s for head_correct spanning!'%fname                    
#                is_a.append('True')
               
        print 'Number direction a headings:',np.shape(mtime_a)

        # this method is independent of self/a
        #try: 
        heading_correction_a = adcpy.util.fit_head_correct(mtime_in=mtime_a,
                                   hdg_in=heading_a,
                                   bt_vel_in=bt_vel_a,
                                   xy_in=xy_a,
                                   u_min_bt=u_min_bt,
                                   hdg_bin_size=hdg_bin_size,
                                   hdg_bin_min_samples=hdg_bin_min_samples)
#        except:
#                print 'Failure fitting head_correct spanning!'
#                if mag_declination is not None:
#                    print 'Using simple magnetic declination correction instead'
#                    heading_correction_a = None
#                else:
#                    print 'No magnetic declination value found - head_correct failure.'
#                    print 'exiting'
#                    exit()
                  
    # setup compression option
    if use_netcdf_data_compression:
        zlib = True
    else:
        zlib = None
   
    # begin cycling/processing input files 
    adcp_preprocessed = []
    for data_file in data_files:    
 #       try:
        a = adcpy.open_adcp(data_file,
                            file_type='ADCPRdiWorkhorseData',
                            num_av=1,
                            adcp_depth=adcp_depth)
        path, fname = os.path.split(data_file)
        fname, ext = os.path.splitext(fname)
        outname = os.path.join(outpath,fname)
        
        print 'Processing data_file:', outname
        
        if save_raw_data_to_netcdf:
            fname = outname + '.nc'
            a.write_nc(fname,zlib=zlib)

        # setup for heading correction based
        if head_correct_spanning and (len(data_files) > 1):        
            heading_cf = heading_correction_a
        else:
            heading_cf = None
        
        a.lonlat_to_xy(xy_srs=xy_projection)
        a.heading_correct(cf=heading_cf,
                          u_min_bt=u_min_bt,
                          hdg_bin_size=hdg_bin_size,
                          hdg_bin_min_samples=hdg_bin_min_samples,
                          mag_dec=mag_declination)
        if sidelobe_drop != 0:
            a.remove_sidelobes(fsidelobe=sidelobe_drop)
        if std_drop > 0:
            a.sd_drop(sd=std_drop,
                      sd_axis='elevation',
                      interp_holes=True)
            a.sd_drop(sd=std_drop,
                      sd_axis='ensemble',
                      interp_holes=True)
        if average_ens > 1:
            a = a.average_ensembles(ens_to_avg=average_ens)
        if regrid_horiz_m is not None:
            a.xy_regrid(dxy=regrid_horiz_m,
                        dz=regrid_vert_m,
                        xy_srs=xy_projection,
                        pline=None,
                        sort=False)
        if smooth_kernel > 2:
            a.kernel_smooth(kernel_size = smooth_kernel)
        if extrap_boundaries:
            a.extrapolate_boundaries()

        if (save_preprocessed_data_to_netcdf):
            fname = outname + '.preprocessed.nc'
            a.write_nc(fname,zlib=zlib)
            
        adcp_preprocessed.append(a)
        
        if debug_stop_after_n_transects:
            if len(adcp_preprocessed) >= debug_stop_after_n_transects:
                return (adcp_preprocessed,outpath)

    return (adcp_preprocessed,outpath)
import arcpy
from scipy.stats import morestats

raster_path = "data/aspect_raster.tif"
r = arcpy.RasterToNumPyArray(raster_path)

# can't take a log of 0, offset values
r += 0.01

print("""Circular Mean:\t\t\t{}
Circular Std. Dev.:\t\t\t{}
Circular Variance:\t\t\t{}
""".format(
    morestats.circmean(r),
    morestats.circstd(r),
    morestats.circvar(r)))
Exemple #11
0
import arcpy
from scipy.stats import morestats

raster_path = "data/aspect_raster.tif"
r = arcpy.RasterToNumPyArray(raster_path)

# can't take a log of 0, offset values
r += 0.01

print("""Circular Mean:\t\t\t{}
Circular Std. Dev.:\t\t\t{}
Circular Variance:\t\t\t{}
""".format(morestats.circmean(r), morestats.circstd(r), morestats.circvar(r)))