コード例 #1
0
def analyse_data(input_file):
    df = pd.read_csv(input_file)
    labels = df['board_size'].unique()
    to_write = []

    for l in labels:
        rows_values = df.loc[(df.board_size == l)]
        step_values = rows_values.loc[rows_values.solved ==
                                      'yes'].steps_taken.values
        if not len(step_values):
            to_write.append(
                [l, input_file[:-4],
                 len(rows_values), 0, 0, 0, 0, 0, 0, 0])
        else:
            analys = stats.describe(step_values)
            median = nanmedian(step_values)
            percentile = [
                analys[1][0],
                np.percentile(step_values, 25),
                np.percentile(step_values, 50),
                np.percentile(step_values, 75), analys[1][1]
            ]
            to_write.append([
                l, input_file[:-4],
                len(rows_values), analys[0],
                round(np.mean(step_values), 3)
            ] + percentile)
    return to_write
コード例 #2
0
def SRIRAW2iono(flist,outdir,desrange=[100.,650.],inttime=5*60.,timelim=1200.):
    """ 
        This will take a list of files and save out radar data
        Inputs
            flist - A list of files that will read and turned into the format 
                for the simulator.
            outdir - The directory that that will hold the data in the format
                from radardatasim. A directory will be created called RadarData
                and the data will be saved there.
            inttime - The number of seconds for the integration time.
    """

    # make directory structure
    if not os.path.isdir(outdir):
        os.mkdir(outdir)
    radardatadir = os.path.join(outdir,'Radardata')

    if not os.path.isdir(radardatadir):
        os.mkdir(radardatadir)
    pulsetimes=[]
    outfilelist = []
    pulsenumbers = []
    beam_list_all = []
    noisetimes=[]
    for ifile,filename in enumerate(flist):
        outdict={}
        fullfile = h5file(filename)
        fullfiledict = fullfile.readWholeh5file()

        if fullfiledict['/Site']['Name'] =='Resolute North':
            radarname='risr'
            Pcal=fullfiledict['/Rx']['Bandwidth']*fullfiledict['/Rx']['CalTemp']*v_Boltz
        else:
            radarname='pfisr'
            Pcal=fullfiledict['/Rx']['Bandwidth']*fullfiledict['/Rx']['CalTemp']*v_Boltz

        bco = fullfiledict['/S/Data']['Beamcodes']
        beamlist = bco[0]
        if ifile==0:
            bstart = sp.zeros(len(beamlist),dtype=sp.int32)
        time= fullfiledict['/Time']['UnixTime']
#
        fullfile = h5file(filename)
        fullfiledict = fullfile.readWholeh5file()
        print('Main file being operated on: '+os.path.split(filename)[-1])
        
        acflags = fullfiledict['/S/Data/Acf']['Lags'].flatten()
        nlags = len(acflags)
        lagdown = sp.arange(-sp.floor(float(nlags-1)/2.),0).astype(int)
        lagup = sp.arange(1,sp.ceil(float(nlags+1)/2.)).astype(int)
        # Get the raw samples
        all_data = fullfiledict['/Raw11/Raw/Samples']['Data']
        rawrange = fullfiledict['/Raw11/Raw/Samples']['Range'][0]*1e-3
        rngkeep= sp.where(sp.logical_and(rawrange>=desrange[0],rawrange<desrange[1]))[0]
        
        rngkeep = sp.hstack((lagdown+rngkeep[0],rngkeep,lagup+rngkeep[-1]))
        rawrange=rawrange[rngkeep]
        pulse_times = fullfiledict['/Raw11/Raw/RadacHeader']['RadacTime']
        rawsamps = all_data[:,:,:,0]+1j*all_data[:,:,:,1]
        rawsamps = rawsamps[:,:,rngkeep]
        (nrecs,  np_rec,nrng)=rawsamps.shape
        
        all_beams_mat = fullfiledict['/Raw11/Raw/RadacHeader']['BeamCode']
#        
        pwidth = float(fullfiledict['/S/Data']['Pulsewidth'])
#        
#        # Pull in call and noise material because these will needed for fitting
        beamcodes_cal = fullfiledict['/S/Cal']['Beamcodes']
        cal_pint = fullfiledict['/S/Cal']['PulsesIntegrated']
        caldata = fullfiledict['/S/Cal/Power']['Data']
        noise_pint = fullfiledict['/S/Noise']['PulsesIntegrated']
        noise_pwer = fullfiledict['/S/Noise/Power']['Data']
        noise_data =fullfiledict['/S/Noise/Acf']['Data']
        noise_acf = noise_data[:,:,:,:,0]+1j*noise_data[:,:,:,:,1]
        noise_acf2 = sp.transpose(noise_acf,(0,1,3,2))
        (nbeams,nnrng,nlags)=noise_acf2.shape[1:]
#        
        # use median to avoid large jumps. The difference between the mean and median estimator
        # goes to zero after enough pulses have been originally integrated. From what Im seeing you're
        # close to 64 pulses integrated you this will be off by only a 1/2 % of the true value.
        n_pow_e = sp.nanmedian(noise_pwer,axis=-1)/noise_pint
        
        c_pow_e = sp.nanmedian(caldata,axis=-1)/cal_pint
        
        # Need to adjust for cal and noise
        powmult = Pcal/(c_pow_e-n_pow_e)
        noise_mult = sp.tile(powmult[:,:,sp.newaxis,sp.newaxis],noise_acf2.shape[2:])
        npint_mat = sp.tile(noise_pint[:,:,sp.newaxis,sp.newaxis],noise_acf2.shape[2:])
        noise_acf_out = noise_acf2*noise_mult/npint_mat
        datamult = sp.zeros_like(rawsamps)
        
        for irec,ibeamlist in enumerate(beamcodes_cal):
            for ibpos,ibeam in enumerate(ibeamlist):
                b_idx = sp.where(ibeam==all_beams_mat[irec])[0]
                datamult[irec,b_idx]=sp.sqrt(powmult[irec,ibpos])

        outraw= rawsamps*datamult
        timep =  pulse_times.reshape(nrecs*np_rec)
        beamn = all_beams_mat.reshape(nrecs*np_rec)
        beamnrs = sp.zeros_like(beamn)
        pulsen = sp.ones(beamn.shape,dtype=sp.int32)
        for ibn, ibeam in enumerate(beamlist):
            curlocs = sp.where(beamn==ibeam)[0]
            beamnrs[curlocs] = ibn
            curtime = timep[curlocs]
            cursort = sp.argsort(curtime)
            curlocs_sort = curlocs[cursort]
            pulsen[curlocs_sort]= sp.arange(len(curlocs)) +bstart[ibn]
            bstart[ibn]=bstart[ibn]+len(curlocs)
            
        outdict['RawData']=outraw.reshape(nrecs*np_rec,nrng)
        outdict['RawDatanonoise'] = outdict['RawData']
        outdict['AddedNoise'] = (1./sp.sqrt(2.))*(sp.randn(*outdict['RawData'].shape)+1j*sp.randn(*outdict['RawData'].shape))
        outdict['NoiseDataACF'] = noise_acf_out
        outdict['BeamsNoise'] = bco[0]
        outdict['Beams']= beamnrs
        outdict['Time'] =timep
        outdict['NoiseTime'] = time
        outdict['Pulses']= pulsen
#
        fname = '{0:d} RawData.h5'.format(ifile)
        newfn = os.path.join(radardatadir,fname)
        outfilelist.append(newfn)
        pulsetimes.append(timep)
        pulsenumbers.append(pulsen)
        noisetimes.append(time)
        beam_list_all.append(beamnrs)
        dict2h5(newfn,outdict)
#    # save the information file
    infodict = {'Files':outfilelist,'Time':pulsetimes,'Beams':beam_list_all,'Pulses':pulsenumbers,'NoiseTime':noisetimes,'Range':rawrange}
    dict2h5(os.path.join(radardatadir,'INFO.h5'),infodict)
#
    ts = fullfiledict['/Rx']['SampleTime']
    sumrule = makesumrule('long',fullfiledict['/S/Data']['Pulsewidth'],ts)
#    minrg = -sumrule[0].min()
#    maxrg = len(rng_vec)-sumrule[1].max()
    maxrg = len(rawrange)+sumrule[0].min()
    minrg = sumrule[1].max()
    rng_lims = [rawrange[minrg],rawrange[maxrg]]# limits of the range gates
    IPP = .0087 #interpulse period in seconds
#   
    simparams =   {'IPP':IPP, #interpulse period
                   'TimeLim':timelim, # length of simulation
                   'RangeLims':rng_lims, # range swath limit
#                   'Pulse':pulse, # pulse shape
                   'Pulselength':pwidth,
                   'FitType' :'acf',
                   't_s': ts,
                   'Pulsetype':'long', # type of pulse can be long or barker,
                   'Tint':inttime, #Integration time for each fitting
                   'Fitinter':inttime, # time interval between fitted params
                   'NNs': nnrng+nlags-1,# number of noise samples per pulse
                   'NNp':100, # number of noise pulses
                   'dtype':sp.complex128, #type of numbers used for simulation
                   'ambupsamp':1, # up sampling factor for ambiguity function
                   'species':['O+','e-'], # type of ion species used in simulation
                   'numpoints':128,
                   'startfile':'startfile.h5'
                   } # number of points for each spectrum
    makeconfigfile(os.path.join(outdir,'sriconfig1.ini'),beamlist,radarname,simparams)
コード例 #3
0
def SRIACF2iono(flist):
    """ 
        This will take the ACF files and save them as Ionofiles
    """

                   
    for iflistn, iflist in enumerate(flist):

        for ifile,filename in enumerate(iflist):
            fullfile = h5file(filename)
            fullfiledict = fullfile.readWholeh5file()

            if fullfiledict['/Site']['Name'] =='Resolute North':
                radarname='risr'
            else:
                radarname='pfisr'
            Pcal=fullfiledict['/Rx']['Bandwidth']*fullfiledict['/Rx']['CalTemp']*v_Boltz

            bco = fullfiledict['/S/Data']['Beamcodes']
            beamlist = bco[0]
            time= fullfiledict['/Time']['UnixTime']
#            if ifile==0 and iflistn==0:
#                time0=time[0,0]
#            time=time-time0
            #nt x nbeams x lags x range x 2
            acfs = fullfiledict['/S/Data/Acf']['Data']
            acfs = acfs[:,:,:,:,0]+1j*acfs[:,:,:,:,1]
            (nt,nbeams,nlags,nrng) =acfs.shape
            acfrng = fullfiledict['/S/Data/Acf']['Range'].flatten()
            acf_pint = fullfiledict['/S/Data']['PulsesIntegrated']


            pwidth = fullfiledict['/S/Data']['Pulsewidth']

            # Pull in call and noise material because these will needed for fitting

            cal_pint = fullfiledict['/S/Cal']['PulsesIntegrated']
            #nt x nbeams x range
            caldata = fullfiledict['/S/Cal/Power']['Data']
            caldata = sp.nanmedian(caldata,axis=-1)
            caldata=caldata/cal_pint

            noise_pint = fullfiledict['/S/Noise']['PulsesIntegrated']
            noise_pwer = fullfiledict['/S/Noise/Power']['Data']
            noise_pwer = sp.nanmedian(noise_pwer,axis=-1)
            noise_pwer = noise_pwer/noise_pint


            caldata = caldata-noise_pwer
            caldataD = sp.tile(caldata[:,:,sp.newaxis,sp.newaxis],(1,1,nlags,nrng))
            #nt x nbeams x lags x range x 2
            noise_data = fullfiledict['/S/Noise/Acf']['Data']
            noise_data = noise_data[:,:,:,:,0]+1j*noise_data[:,:,:,:,1]
            nnrg = noise_data.shape[-1]
            caldataN = sp.tile(caldata[:,:,sp.newaxis,sp.newaxis],(1,1,nlags,nnrg))
            # subtract noise and
            acfs = Pcal*acfs/caldataD
            acfs = acfs.transpose((0,1,3,2))
            noise_data = Pcal*noise_data/caldataN
            noise_data = noise_data.transpose((0,1,3,2))
            # Create output dictionaries and output data
            if ifile==0:
                acfs_sum=acfs.copy()
                acf_pint_sum = acf_pint.copy()
                noise_data_sum = noise_data.copy()
                noise_pint_sum = noise_pint.copy()

            else:
                acfs_sum=acfs_sum+acfs
                acf_pint_sum = acf_pint_sum+acf_pint
                noise_data_sum = noise_data_sum+ noise_data
                noise_pint_sum = noise_pint_sum + noise_pint
        if iflistn==0:

            acf_acum=acfs_sum.copy()
            acf_pint_acum = acf_pint_sum.copy()
            noise_data_acum = noise_data_sum.copy()
            noise_pint_acum = noise_pint_sum.copy()
            Time_acum =time.copy()
        else:
            acf_acum=sp.vstack((acf_acum,acfs_sum))
            acf_pint_acum = sp.vstack((acf_pint_acum,acf_pint_sum))
            noise_data_acum = sp.vstack((noise_data_acum,noise_data_sum))
            noise_pint_acum= sp.vstack((noise_pint_acum,noise_pint_sum))
            Time_acum = sp.vstack((Time_acum,time))
    
    DataLags = {'ACF':acf_acum,'Pow':acf_acum[:,:,:,0].real,'Pulses':acf_pint_acum,'Time':Time_acum}
    NoiseLags = {'ACF':noise_data_acum,'Pow':noise_data_acum[:,:,:,0].real,'Pulses':noise_pint_acum,'Time':Time_acum}

    rng_vec = acfrng*1e-3
    ts = fullfiledict['/Rx']['SampleTime']
    sumrule = makesumrule('long',fullfiledict['/S/Data']['Pulsewidth'],ts)
    minrg = -sumrule[0].min()
    maxrg = len(rng_vec)-sumrule[1].max()
    rng_lims = [rng_vec[minrg],rng_vec[maxrg]]# limits of the range gates
    IPP = .0087 #interpulse period in seconds


    simparams =   {'IPP':IPP, #interpulse period
                   'TimeLim':time[-1,1], # length of simulation
                   'RangeLims':rng_lims, # range swath limit
#                   'Pulse':pulse, # pulse shape
                   'Pulselength':pwidth,
                   'FitType' :'acf',
                   't_s': ts,
                   'Pulsetype':'long', # type of pulse can be long or barker,
                   'Tint':time[0,1]-time[0,0], #Integration time for each fitting
                   'Fitinter':time[1,0]-time[0,0], # time interval between fitted params
                   'NNs': 100,# number of noise samples per pulse
                   'NNp':100, # number of noise pulses
                   'dtype':sp.complex128, #type of numbers used for simulation
                   'ambupsamp':1, # up sampling factor for ambiguity function
                   'species':['O+','e-'], # type of ion species used in simulation
                   'numpoints':128} # number of points for each spectrum

    (sensdict,simparams) = makeparamdicts(beamlist,radarname,simparams)
    simparams['Rangegates'] =rng_vec
    simparams['Rangegatesfinal']=rng_vec[minrg:maxrg]

    ionolag, ionosigs = lagdict2ionocont(DataLags,NoiseLags,sensdict,simparams,Time_acum)

    return (ionolag,ionosigs,simparams,sensdict)