def analyse_data(input_file):
df = pd.read_csv(input_file)
labels = df['board_size'].unique()
to_write = []
for l in labels:
rows_values = df.loc[(df.board_size == l)]
step_values = rows_values.loc[rows_values.solved ==
'yes'].steps_taken.values
if not len(step_values):
to_write.append(
[l, input_file[:-4],
len(rows_values), 0, 0, 0, 0, 0, 0, 0])
else:
analys = stats.describe(step_values)
median = nanmedian(step_values)
percentile = [
analys[1][0],
np.percentile(step_values, 25),
np.percentile(step_values, 50),
np.percentile(step_values, 75), analys[1][1]
]
to_write.append([
l, input_file[:-4],
len(rows_values), analys[0],
round(np.mean(step_values), 3)
] + percentile)
return to_write
def SRIRAW2iono(flist,outdir,desrange=[100.,650.],inttime=5*60.,timelim=1200.):
"""
This will take a list of files and save out radar data
Inputs
flist - A list of files that will read and turned into the format
for the simulator.
outdir - The directory that that will hold the data in the format
from radardatasim. A directory will be created called RadarData
and the data will be saved there.
inttime - The number of seconds for the integration time.
"""
# make directory structure
if not os.path.isdir(outdir):
os.mkdir(outdir)
radardatadir = os.path.join(outdir,'Radardata')
if not os.path.isdir(radardatadir):
os.mkdir(radardatadir)
pulsetimes=[]
outfilelist = []
pulsenumbers = []
beam_list_all = []
noisetimes=[]
for ifile,filename in enumerate(flist):
outdict={}
fullfile = h5file(filename)
fullfiledict = fullfile.readWholeh5file()
if fullfiledict['/Site']['Name'] =='Resolute North':
radarname='risr'
Pcal=fullfiledict['/Rx']['Bandwidth']*fullfiledict['/Rx']['CalTemp']*v_Boltz
else:
radarname='pfisr'
Pcal=fullfiledict['/Rx']['Bandwidth']*fullfiledict['/Rx']['CalTemp']*v_Boltz
bco = fullfiledict['/S/Data']['Beamcodes']
beamlist = bco[0]
if ifile==0:
bstart = sp.zeros(len(beamlist),dtype=sp.int32)
time= fullfiledict['/Time']['UnixTime']
#
fullfile = h5file(filename)
fullfiledict = fullfile.readWholeh5file()
print('Main file being operated on: '+os.path.split(filename)[-1])
acflags = fullfiledict['/S/Data/Acf']['Lags'].flatten()
nlags = len(acflags)
lagdown = sp.arange(-sp.floor(float(nlags-1)/2.),0).astype(int)
lagup = sp.arange(1,sp.ceil(float(nlags+1)/2.)).astype(int)
# Get the raw samples
all_data = fullfiledict['/Raw11/Raw/Samples']['Data']
rawrange = fullfiledict['/Raw11/Raw/Samples']['Range'][0]*1e-3
rngkeep= sp.where(sp.logical_and(rawrange>=desrange[0],rawrange<desrange[1]))[0]
rngkeep = sp.hstack((lagdown+rngkeep[0],rngkeep,lagup+rngkeep[-1]))
rawrange=rawrange[rngkeep]
pulse_times = fullfiledict['/Raw11/Raw/RadacHeader']['RadacTime']
rawsamps = all_data[:,:,:,0]+1j*all_data[:,:,:,1]
rawsamps = rawsamps[:,:,rngkeep]
(nrecs, np_rec,nrng)=rawsamps.shape
all_beams_mat = fullfiledict['/Raw11/Raw/RadacHeader']['BeamCode']
#
pwidth = float(fullfiledict['/S/Data']['Pulsewidth'])
#
# # Pull in call and noise material because these will needed for fitting
beamcodes_cal = fullfiledict['/S/Cal']['Beamcodes']
cal_pint = fullfiledict['/S/Cal']['PulsesIntegrated']
caldata = fullfiledict['/S/Cal/Power']['Data']
noise_pint = fullfiledict['/S/Noise']['PulsesIntegrated']
noise_pwer = fullfiledict['/S/Noise/Power']['Data']
noise_data =fullfiledict['/S/Noise/Acf']['Data']
noise_acf = noise_data[:,:,:,:,0]+1j*noise_data[:,:,:,:,1]
noise_acf2 = sp.transpose(noise_acf,(0,1,3,2))
(nbeams,nnrng,nlags)=noise_acf2.shape[1:]
#
# use median to avoid large jumps. The difference between the mean and median estimator
# goes to zero after enough pulses have been originally integrated. From what Im seeing you're
# close to 64 pulses integrated you this will be off by only a 1/2 % of the true value.
n_pow_e = sp.nanmedian(noise_pwer,axis=-1)/noise_pint
c_pow_e = sp.nanmedian(caldata,axis=-1)/cal_pint
# Need to adjust for cal and noise
powmult = Pcal/(c_pow_e-n_pow_e)
noise_mult = sp.tile(powmult[:,:,sp.newaxis,sp.newaxis],noise_acf2.shape[2:])
npint_mat = sp.tile(noise_pint[:,:,sp.newaxis,sp.newaxis],noise_acf2.shape[2:])
noise_acf_out = noise_acf2*noise_mult/npint_mat
datamult = sp.zeros_like(rawsamps)
for irec,ibeamlist in enumerate(beamcodes_cal):
for ibpos,ibeam in enumerate(ibeamlist):
b_idx = sp.where(ibeam==all_beams_mat[irec])[0]
datamult[irec,b_idx]=sp.sqrt(powmult[irec,ibpos])
outraw= rawsamps*datamult
timep = pulse_times.reshape(nrecs*np_rec)
beamn = all_beams_mat.reshape(nrecs*np_rec)
beamnrs = sp.zeros_like(beamn)
pulsen = sp.ones(beamn.shape,dtype=sp.int32)
for ibn, ibeam in enumerate(beamlist):
curlocs = sp.where(beamn==ibeam)[0]
beamnrs[curlocs] = ibn
curtime = timep[curlocs]
cursort = sp.argsort(curtime)
curlocs_sort = curlocs[cursort]
pulsen[curlocs_sort]= sp.arange(len(curlocs)) +bstart[ibn]
bstart[ibn]=bstart[ibn]+len(curlocs)
outdict['RawData']=outraw.reshape(nrecs*np_rec,nrng)
outdict['RawDatanonoise'] = outdict['RawData']
outdict['AddedNoise'] = (1./sp.sqrt(2.))*(sp.randn(*outdict['RawData'].shape)+1j*sp.randn(*outdict['RawData'].shape))
outdict['NoiseDataACF'] = noise_acf_out
outdict['BeamsNoise'] = bco[0]
outdict['Beams']= beamnrs
outdict['Time'] =timep
outdict['NoiseTime'] = time
outdict['Pulses']= pulsen
#
fname = '{0:d} RawData.h5'.format(ifile)
newfn = os.path.join(radardatadir,fname)
outfilelist.append(newfn)
pulsetimes.append(timep)
pulsenumbers.append(pulsen)
noisetimes.append(time)
beam_list_all.append(beamnrs)
dict2h5(newfn,outdict)
# # save the information file
infodict = {'Files':outfilelist,'Time':pulsetimes,'Beams':beam_list_all,'Pulses':pulsenumbers,'NoiseTime':noisetimes,'Range':rawrange}
dict2h5(os.path.join(radardatadir,'INFO.h5'),infodict)
#
ts = fullfiledict['/Rx']['SampleTime']
sumrule = makesumrule('long',fullfiledict['/S/Data']['Pulsewidth'],ts)
# minrg = -sumrule[0].min()
# maxrg = len(rng_vec)-sumrule[1].max()
maxrg = len(rawrange)+sumrule[0].min()
minrg = sumrule[1].max()
rng_lims = [rawrange[minrg],rawrange[maxrg]]# limits of the range gates
IPP = .0087 #interpulse period in seconds
#
simparams = {'IPP':IPP, #interpulse period
'TimeLim':timelim, # length of simulation
'RangeLims':rng_lims, # range swath limit
# 'Pulse':pulse, # pulse shape
'Pulselength':pwidth,
'FitType' :'acf',
't_s': ts,
'Pulsetype':'long', # type of pulse can be long or barker,
'Tint':inttime, #Integration time for each fitting
'Fitinter':inttime, # time interval between fitted params
'NNs': nnrng+nlags-1,# number of noise samples per pulse
'NNp':100, # number of noise pulses
'dtype':sp.complex128, #type of numbers used for simulation
'ambupsamp':1, # up sampling factor for ambiguity function
'species':['O+','e-'], # type of ion species used in simulation
'numpoints':128,
'startfile':'startfile.h5'
} # number of points for each spectrum
makeconfigfile(os.path.join(outdir,'sriconfig1.ini'),beamlist,radarname,simparams)
def SRIACF2iono(flist):
"""
This will take the ACF files and save them as Ionofiles
"""
for iflistn, iflist in enumerate(flist):
for ifile,filename in enumerate(iflist):
fullfile = h5file(filename)
fullfiledict = fullfile.readWholeh5file()
if fullfiledict['/Site']['Name'] =='Resolute North':
radarname='risr'
else:
radarname='pfisr'
Pcal=fullfiledict['/Rx']['Bandwidth']*fullfiledict['/Rx']['CalTemp']*v_Boltz
bco = fullfiledict['/S/Data']['Beamcodes']
beamlist = bco[0]
time= fullfiledict['/Time']['UnixTime']
# if ifile==0 and iflistn==0:
# time0=time[0,0]
# time=time-time0
#nt x nbeams x lags x range x 2
acfs = fullfiledict['/S/Data/Acf']['Data']
acfs = acfs[:,:,:,:,0]+1j*acfs[:,:,:,:,1]
(nt,nbeams,nlags,nrng) =acfs.shape
acfrng = fullfiledict['/S/Data/Acf']['Range'].flatten()
acf_pint = fullfiledict['/S/Data']['PulsesIntegrated']
pwidth = fullfiledict['/S/Data']['Pulsewidth']
# Pull in call and noise material because these will needed for fitting
cal_pint = fullfiledict['/S/Cal']['PulsesIntegrated']
#nt x nbeams x range
caldata = fullfiledict['/S/Cal/Power']['Data']
caldata = sp.nanmedian(caldata,axis=-1)
caldata=caldata/cal_pint
noise_pint = fullfiledict['/S/Noise']['PulsesIntegrated']
noise_pwer = fullfiledict['/S/Noise/Power']['Data']
noise_pwer = sp.nanmedian(noise_pwer,axis=-1)
noise_pwer = noise_pwer/noise_pint
caldata = caldata-noise_pwer
caldataD = sp.tile(caldata[:,:,sp.newaxis,sp.newaxis],(1,1,nlags,nrng))
#nt x nbeams x lags x range x 2
noise_data = fullfiledict['/S/Noise/Acf']['Data']
noise_data = noise_data[:,:,:,:,0]+1j*noise_data[:,:,:,:,1]
nnrg = noise_data.shape[-1]
caldataN = sp.tile(caldata[:,:,sp.newaxis,sp.newaxis],(1,1,nlags,nnrg))
# subtract noise and
acfs = Pcal*acfs/caldataD
acfs = acfs.transpose((0,1,3,2))
noise_data = Pcal*noise_data/caldataN
noise_data = noise_data.transpose((0,1,3,2))
# Create output dictionaries and output data
if ifile==0:
acfs_sum=acfs.copy()
acf_pint_sum = acf_pint.copy()
noise_data_sum = noise_data.copy()
noise_pint_sum = noise_pint.copy()
else:
acfs_sum=acfs_sum+acfs
acf_pint_sum = acf_pint_sum+acf_pint
noise_data_sum = noise_data_sum+ noise_data
noise_pint_sum = noise_pint_sum + noise_pint
if iflistn==0:
acf_acum=acfs_sum.copy()
acf_pint_acum = acf_pint_sum.copy()
noise_data_acum = noise_data_sum.copy()
noise_pint_acum = noise_pint_sum.copy()
Time_acum =time.copy()
else:
acf_acum=sp.vstack((acf_acum,acfs_sum))
acf_pint_acum = sp.vstack((acf_pint_acum,acf_pint_sum))
noise_data_acum = sp.vstack((noise_data_acum,noise_data_sum))
noise_pint_acum= sp.vstack((noise_pint_acum,noise_pint_sum))
Time_acum = sp.vstack((Time_acum,time))
DataLags = {'ACF':acf_acum,'Pow':acf_acum[:,:,:,0].real,'Pulses':acf_pint_acum,'Time':Time_acum}
NoiseLags = {'ACF':noise_data_acum,'Pow':noise_data_acum[:,:,:,0].real,'Pulses':noise_pint_acum,'Time':Time_acum}
rng_vec = acfrng*1e-3
ts = fullfiledict['/Rx']['SampleTime']
sumrule = makesumrule('long',fullfiledict['/S/Data']['Pulsewidth'],ts)
minrg = -sumrule[0].min()
maxrg = len(rng_vec)-sumrule[1].max()
rng_lims = [rng_vec[minrg],rng_vec[maxrg]]# limits of the range gates
IPP = .0087 #interpulse period in seconds
simparams = {'IPP':IPP, #interpulse period
'TimeLim':time[-1,1], # length of simulation
'RangeLims':rng_lims, # range swath limit
# 'Pulse':pulse, # pulse shape
'Pulselength':pwidth,
'FitType' :'acf',
't_s': ts,
'Pulsetype':'long', # type of pulse can be long or barker,
'Tint':time[0,1]-time[0,0], #Integration time for each fitting
'Fitinter':time[1,0]-time[0,0], # time interval between fitted params
'NNs': 100,# number of noise samples per pulse
'NNp':100, # number of noise pulses
'dtype':sp.complex128, #type of numbers used for simulation
'ambupsamp':1, # up sampling factor for ambiguity function
'species':['O+','e-'], # type of ion species used in simulation
'numpoints':128} # number of points for each spectrum
(sensdict,simparams) = makeparamdicts(beamlist,radarname,simparams)
simparams['Rangegates'] =rng_vec
simparams['Rangegatesfinal']=rng_vec[minrg:maxrg]
ionolag, ionosigs = lagdict2ionocont(DataLags,NoiseLags,sensdict,simparams,Time_acum)
return (ionolag,ionosigs,simparams,sensdict)
