def main():
fname="/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0300.data"
b=bmx.BMXFile(fname)
b.getRadar()
mspec=b.data[np.logical_not(b.radarOn)]['chan1_0'].mean(axis=0)
mspec2=b.data[np.logical_not(b.radarOn)]['chan2_0'].mean(axis=0)
mspecR=b.data[np.logical_not(b.radarOn)]['chanXR_0'].mean(axis=0)
mspecI=b.data[np.logical_not(b.radarOn)]['chanXI_0'].mean(axis=0)
mspecRr=b.data[b.radarOn]['chanXR_0'].mean(axis=0)
mspecIr=b.data[b.radarOn]['chanXI_0'].mean(axis=0)
for i in range(4):
plt.subplot(4,1,i+1)
imin=1024*i
imax=imin+1024
plt.plot(b.freq[0][imin:imax],mspec[imin:imax])
plt.plot(b.freq[0][imin:imax],mspec2[imin:imax])
plt.xlabel("MHz")
plt.ylabel("P")
plt.semilogy()
plt.show()
for i in range(4):
plt.subplot(4,1,i+1)
imin=1024*i
imax=imin+1024
plt.plot(b.freq[0][imin:imax],mspecR[imin:imax])
plt.plot(b.freq[0][imin:imax],mspecI[imin:imax])
plt.xlabel("MHz")
plt.ylabel("P")
plt.show()
return
Nfft=4096*4
da=np.zeros(Nfft,complex)
da[4096*2:4096*3]=mspec
#da[:4096]=1.0+0j
xi=np.fft.irfft(da)[:Nfft]
dt=3e8/3.3e9
me=dt*np.arange(len(xi))
plt.plot(me,xi)
plt.semilogx()
plt.show()
def paddata(self):
"""Concatenate data before and after if it exists and is
consecutive. Not sure if this currently works."""
ind = np.where(dm.tags == self.tag)[0][0]
# Store start/stop time of data
self.mjdspan = [self.d.data['mjd'][[0,-1]]]
if ind >= 1:
# Load tag before
tag0 = dm.tags[ind-1]
da = bmxdata.BMXFile(dm.getrawfname(tag0))
self.d.data=np.hstack((self.d.data, da.data))
self.d.nSamples=len(self.d.data)
def __init__(self, tag):
""" Init with a tag string E.g.
r = reduce('170928_1000')
"""
# Store tag string and get filename
self.tag = tag
self.rawfname = dm.getrawfname(tag)
self.redfname = dm.getreducedfname(tag)
print('loading data...')
t = time.time()
if np.int(self.tag[0:2]) >= 19:
self.d = bmxdata.BMXFile(self.rawfname, loadD2=True, loadRFI=True)
else:
self.d = bmxdata.BMXFile(self.rawfname)
print(('...took {:0.1f} sec'.format(time.time()-t)))
# Channel names to plot
self.chnames = np.array(self.d.names)[np.array(
['chan' in k for k in self.d.names])]
self.nChan = len(self.chnames)
# Undo RFI
if hasattr(self.d, 'rfi'):
self.undorfi()
# Get frequency array
self.f = self.d.freq[0]
self.dtraw = self.d.deltaT[0]
# Now concatenate on either side with data from adjacent tags if the
# data are consecutive
# self.paddata()
self.getind()
def __init__(self, tag):
""" Init with a tag string E.g.
r = reduce('170928_1000')
"""
# Store tag string and get filename
self.tag = tag
self.rawfname = dm.getrawfname(tag)
self.redfname = dm.getreducedfname(tag)
print('loading data...')
t = time.time()
self.d = bmxdata.BMXFile(self.rawfname)
print('...took {:0.1f} sec'.format(time.time()-t))
# Get frequency array
self.f = self.d.freq[0]
self.dtraw = self.d.deltaT[0]
def main():
fname="/gpfs01/astro/workarea/bmxdata/170719_first/flight_rfi_20_4_170719_0300.data"
b=bmx.BMXFile(fname)
b.getRadar()
t=0
tsince=[]
for i,r in enumerate(b.radarOn):
if r:
t=0
else:
t+=1
tsince.append(t)
tsince=np.array(tsince)
chan='chan1_0'
Ron=b.data[b.radarOn]['chan1_0'].mean(axis=0)
Roff=[]
st=3
nst=98/st+1
for i in range(nst):
imin=i*st
imax=(i+1)*st
w=(tsince>=imin) & (tsince<imax)
print i,w.sum()
Roff.append(b.data[w]['chan1_0'].mean(axis=0))
Rmin=np.array(Roff).min(axis=0)
for sens in range(1):
plt.figure()
plt.plot(b.freq[0],Ron/Rmin-1,'g-',label='Radar on')
for i in range(nst):
plt.plot(b.freq[0],Roff[i]/Rmin-1,label='Dt=%3.2f s'%(0.122*(st*i+5)),color=(1.0*i/nst,0,1.-i*1.0/nst))
plt.xlabel('MHz')
plt.ylabel('Delta P')
plt.legend(fontsize=10)
#plt.ylim(0,2e16/10**sens)
plt.tight_layout()
plt.semilogy()
plt.show()
plt.colorbar()
plt.title('data R')
plt.subplot(2,2,2)
plt.imshow(x.CrossModel(*Rx)[0],aspect='auto',interpolation='nearest')
plt.title('model R')
plt.colorbar()
plt.subplot(2,2,3)
plt.imshow(x.xi,aspect='auto',interpolation='nearest')
plt.colorbar()
plt.title('data I')
plt.subplot(2,2,4)
plt.imshow(x.CrossModel(*Rx)[1],aspect='auto',interpolation='nearest')
plt.title('model I')
plt.colorbar()
plt.savefig(rootfn+"_cross.png")
print "expected omega at 5m:",
#d=bmxdata.BMXFile("/home/anze/180214_1900.data")
#getFitsAndPlots(dataCut(d,1167,1185),'cr1')
#getFitsAndPlots(dataCut(d,1215,1240),'cr2')
d=bmxdata.BMXFile("/home/anze/180227_1800.data")
getFitsAndPlots(dataCut(d,1167,1185),'cn1')
getFitsAndPlots(dataCut(d,1215,1240),'cn2')
def loadex():
d = bmxdata.BMXFile("/home/anze/180214_1900.data")
return dataCut(d, 1220, 1240)
#!/usr/bin/env python
import bmxdata as bmx
import matplotlib.pyplot as plt
import numpy as np
import sys
for fn in sys.argv[1:]:
d = bmx.BMXFile(fn)
def rnm(x, N):
return np.convolve(x, np.ones((N, )) / N, mode='valid')[(N - 1):]
if False:
for i, n in enumerate(d.names):
plt.subplot(2, 2, i + 1)
y = d.data[n].mean(axis=0)
plt.plot(d.freq, y)
plt.xlabel('delta freq [kHz] ' + n)
if True:
plt.figure(figsize=(10, 8))
for i, n in enumerate(d.names[:-1]):
print i, n
mxf = []
mx = []
for line in d.data[n]:
#print line
i = abs(line).argmax()
mxf.append(i)
#print (line.sum(), line[i-20:i+20].sum())
#plt.plot(range(len(line)),line)
def main():
f = []
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170718_2000.data"
)
if True:
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170718_2100.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170718_2200.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170718_2300.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0000.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0100.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0200.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0300.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0400.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0500.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0600.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0700.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0800.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_0900.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_1000.data"
)
f.append(
"/gpfs01/astro/workarea/anze/bmxdata/170719_first/flight_rfi_20_4_170719_1100.data"
)
bigar = [None, None]
#fmin,fmax=1400., 1440.
fmin, fmax = None, None
#fnormmin,fnormmax=1320.,1340.
fnormmin, fnormmax = None, None
binSize = 4
cut = 0
tavg = 600
minmax = 0.3
for fname in f:
b = bmx.BMXFile(fname)
if False:
b.plotWaterfall(nsamples=300, binSize=4)
stop()
b.getRadar()
b.filterRadar()
if fnormmin is not None:
b.normalizeOnRegion(fnormmin, fnormmax)
imin, imax, fminp, fmaxp = b.f2iminmax(fmin, fmax, binSize=binSize)
for n in range(2):
arr = []
for i in range(b.nSamples):
arr.append(b.data[i]['chan' + str(n + 1) + '_' +
str(cut)][imin:imax])
arr[i] = np.reshape(
arr[i], (-1, binSize)) #bin frequencies into groups of 4
arr[i] = np.mean(arr[i], axis=1) #average the 4
arr = np.array(arr)
if bigar[n] is None:
bigar[n] = arr
else:
bigar[n] = np.vstack((bigar[n], arr))
if tavg > 1:
nsize = len(bigar[0]) / tavg
for ch in [0, 1]:
for i in range(nsize):
bigar[ch][i, :] = bigar[ch][tavg * i:tavg *
(i + 1), :].mean(axis=0)
bigar[ch] = bigar[ch][:nsize, :]
print '---------'
for ch in [0, 1]:
print bigar[ch].shape
mean = bigar[ch].mean(axis=0)
if True:
for i in range(len(bigar[ch])):
bigar[ch][i] /= mean
bigar[ch] -= 1.
## fft filter
print "FFT"
if True:
t = np.fft.rfft2(bigar[ch])
t[5:-5, :] = 0
t[:, :50] = 0
bigar[ch] = np.fft.irfft2(t)
plt.subplot(2, 1, ch + 1)
if True:
plt.imshow(bigar[ch],
interpolation="nearest",
aspect="auto",
extent=[
fminp, fmaxp,
len(bigar[0]) * .122016 * tavg / 3600, 0
],
vmin=-minmax,
vmax=+minmax)
else:
plt.imshow(bigar[ch],
norm=colors.LogNorm(),
interpolation="nearest",
aspect="auto",
extent=[
fminp, fmaxp,
len(bigar[0]) * .122016 * tavg / 3600, 0
])
plt.xlabel("MHz")
plt.ylabel("t[hours] since 8pm")
plt.colorbar()
plt.show()
def stage_one(self):
debug=False
if debug:
self.tags=self.tags[:1]+self.tags[-1:]
self.set_logto("stage1")
self.log("Starting Stage1 reduction on %s"%self.timenow_string())
cutdirs={}
for ext in 'D1','D2':
data=[]
for fn in [self.dm.getrawfname(tag) for tag in self.tags]:
if ext=='D2':
fn=fn.replace('D1','D2')
self.log("Loading %s ..."%fn)
data.append(bmxdata.BMXFile(fn))
if (hasattr(data[0],'wires')):
self.log("Writing wires file ...")
with open(self.root+"/wires",'wb' if ext=='D1' else 'ab') as f:
for i in range(1,5):
f.write (data[0].wires[i]+b"\n")
ncuts=data[0].ncuts
for cut in range(ncuts):
if cut not in cutdirs:
cdir=self.root+"/cut%i"%cut
cutdirs[cut]=cdir
if not os.path.isdir(cdir):
self.log("Creating %s ..."%cdir)
os.mkdir(cdir)
assert (data[0].nChan==2)
assert (data[0].nCards==2)
## first check for continuity of mjd
mjd=data[0].data['mjd']
deltamjd=data[0].deltaT/(24*3600)
#for i in range(len(data)-1):
# if data[i+1].data['mjd'][0]-data[i].data['mjd'][-1]>deltamjd*1.1:
# self.log("Tags %s %s discontinous in MJD."%(self.tags[i],self.tags[i+1]))
# if not debug: ## during debug we work with discontinous files
# self.log("Quitting!")
# return False
mjd=np.hstack([d.data['mjd'] for d in data])
if data[0].version>=10:
temp_fgpa = np.vstack([d.data['temp_fgpa'] for d in data])
temp_adc = np.vstack([d.data['temp_adc'] for d in data])
temp_frontend = np.vstack([d.data['temp_frontend'] for d in data])
have_temps = True
else:
have_temps = False
if ext=='D1':
mjd_d1=mjd
if have_temps:
temp_fgpa_D1 = temp_fgpa
temp_adc_D1 = temp_adc
temp_frontend_D1 =temp_frontend
##
## Now caclulate ra/dec so that we can get cut into our passage
##
self.log('Calculating celestical coords...')
mjdlist=np.arange(mjd[0]-1e-5,mjd[-1]+0.005,0.005)## every 7 mins or so
ra,dec,gall,galb=telescope.mjd2coords(mjdlist)
## set up interpolator
## we now need to make sure ra is always rising
while (np.any( (ra[1:]-ra[:-1])<0)):
i=np.argmax((ra[1:]-ra[:-1])<0)
ra[:i+1]-=2*np.pi
raint=interp1d(mjdlist,ra)
decint=interp1d(mjdlist,dec)
gallint=interp1d(mjdlist,gall)
galbint=interp1d(mjdlist,galb)
radeg=raint(mjd)*180/np.pi
if self.typ=='pas':
istart=np.where(radeg>270.0-360)[0][0]-1 ## we should be safely away from corner cases
iend=np.where(radeg<330.0)[0][-1]+1
else:
istart=0
iend=len(mjd)-1 ## cut one sample at the end to make sure D2 is OK
print ("Cutting %i %i samples at beginning / end..."%(istart,len(mjd)-iend))
print ("Number of samples: ",iend-istart)
else:
## we now have both, let's combine
if (mjd_d1[0]-mjd[0])>deltamjd*0.2: ## should be aligned at least by 20%
self.log("Computers not aligned in MJD.")
self.log("Quitting!")
mjd_d1=mjd_d1[istart:iend]
mjd_d2=mjd[istart:iend]
mjd=0.5*(mjd_d1+mjd_d2)
## Let's fix MJD, to be really continous
mjdfix=np.arange(len(mjd))*deltamjd
mjdfix+=mjd.mean()-mjdfix.mean()
#print (deltamjd, mjdfix-mjd)
# up to 0.5s discrepancy
if np.any(np.abs(mjdfix-mjd)>max(deltamjd,0.5/(3600*24))):
self.log("Warning: Tag perhaps discontinous in MJD.")
#if not debug: ## during debug we work with discontinous files
# self.log("Quitting!")
# return False
mjd=mjdfix
ra=raint(mjd)
dec=decint(mjd)
gall=gallint(mjd)
galb=galbint(mjd)
self.log("Writing MJD file...",mjd.shape,mjd_d1.shape,mjd_d2.shape)
fitsio.write(self.root+"/mjd.fits", np.rec.fromarrays([mjd,mjd_d1,mjd_d2],
dtype=[('mjd','f8'),('mjd_d1','f8'),('mjd_d2','f8')]),
clobber=True)
self.log("Writing coordinate file...")
fitsio.write(self.root+"/coords.fits",
np.rec.fromarrays([ra,dec,gall,galb],
dtype=[('ra','f4'),('dec','f4'),('lgal','f4'),('bgal','f4')]),
clobber=True)
## now also write the labjack
labjack=np.hstack([d.data['lj_diode'] for d in data])[istart:iend]
if labjack.sum()==0:
self.log("No labjack diode...")
self.add_meta("no_diode")
else:
outfn=self.root+"/diode.fits"
self.log("Writing %s ... "%outfn)
fitsio.write(outfn,labjack,clobber=True)
if have_temps:
self.add_meta("have_temperatures")
self.log("Writing temperature file")
fitsio.write(self.root+"/temperatures.fits",
np.rec.fromarrays(
[np.hstack((temp_fgpa_D1[istart:iend,:],temp_fgpa[istart:iend,:])),
np.hstack((temp_adc_D1[istart:iend,:],temp_adc[istart:iend,:])),
np.hstack((temp_frontend_D1[istart:iend,:],temp_frontend[istart:iend,:]))],
dtype=[('fgpa','4f4'),('adc','4f4'),('frontend','4f4')]),
clobber=True)
## Now let's dump our guys
for cut in range(ncuts):
nfreq=data[0].nP[cut]
if (ext=='D1'):
outfn=cutdirs[cut]+"/freq.fits"
self.log("Writing %s ... "%outfn)
fitsio.write(outfn,data[0].freq[cut],clobber=True)
if (cut==1) and abs(data[0].freq[cut].mean()-1420.0)<5:
self.log("Galactic 21cm cut present ...")
self.add_meta("galactic21_cut")
for ch1 in range(1,5): ## hardcoded, yes, but can become more flexible later
for ch2 in range(ch1,5):
if (ch1==ch2):
name="chan%i_%i"%(ch1,cut)
dataR=np.vstack([d.data[name] for d in data])[istart:iend]
outfn=cutdirs[cut]+'/auto_%i.fits'%(ch1+4*(ext=='D2'))
self.log("Writing %s ... "%outfn)
fitsio.write(outfn,dataR,clobber=True)
else:
nameR='chan%ix%iR_%i'%(ch1,ch2,cut)
dataR=np.vstack([d.data[nameR] for d in data])[istart:iend]
nameI='chan%ix%iI_%i'%(ch1,ch2,cut)
dataI=np.vstack([d.data[nameI] for d in data])[istart:iend]
outfn=cutdirs[cut]+'/cross_%i%i.fits'%(ch1+4*(ext=='D2'),ch2+4*(ext=='D2'))
self.log("Writing %s ..."%outfn)
fitsio.write(outfn,dataR,clobber=True)
fitsio.write(outfn,dataI)
#done
self.log("Complete.")
return True
return ind
# Get cooldown data
datadir = './data/'
g = []
n = []
# Channel X, Y
for j in range(2):
if j == 0:
figure(1)
clf()
'Xpol, channel 1'
d = bmxdata.BMXFile(datadir + '170816_1502.data')
p = d.data['chan1_0']
t290 = [60, 80]
t77 = [300, 310]
elif j == 1:
figure(2)
clf()
'Ypol, channel 2'
d = bmxdata.BMXFile(datadir + '170816_1538.data')
p = d.data['chan2_0']
t290 = [150, 160]
t77 = [350, 360]
f = d.freq[0] + 1100
# Get regions of constant temperature
161205_0100.tone.data
161205_0200.tone.data
161205_0300.tone.data
161205_0400.tone.data
161205_0500.tone.data
161205_0600.tone.data
161205_0700.tone.data
161205_0800.tone.data
161205_0900.tone.data
161205_1000.tone.data""".split("\n")
sw=0.0
#filelist=filelist[0:2]
for fname in filelist:
d=bmx.BMXFile('../bmxdaq/data/'+fname)
da=d.data['chan1_0']
da=np.log(da)-35.14
da[:,1638]=0.0
w=len(da)
cov=np.cov(da,rowvar=False,bias=True)
cmean=da.mean(axis=0)
cov=cov+np.outer(cmean,cmean)
###print (cov[0,0],cmean[0])
###print (cmean[:10])
if sw==0:
mean=cmean*w
tcov=cov*w
sw=w
else:
mean+=cmean*w
