def move_temperatre_data():
"""
Moving chunk of temperaure data stored in switch previously to switch_data and change scio to raw files
"""
temp_dirs = glob.glob('/data/marion2017/switch_data/temperatures/*')
for subdir in temp_dirs:
fields = glob.glob(subdir + '/*')
for f in fields:
try:
pi_data = scio.read(f + '/pi_temp.scio')
snapbox_data = scio.read(f + '/snapbox_temp.scio')
f_pi_temp = open(f + '/pi_temp.raw', 'w')
f_pi_time = open(f + '/pi_time.raw', 'w')
f_snapbox_temp = open(f + '/snapbox_temp.raw', 'w')
f_snapbox_time = open(f + '/snapbox_time.raw', 'w')
pi_temp = numpy.int32(pi_data[:, 1])
pi_temp.tofile(f_pi_time)
pi_data[:, 1].tofile(f_pi_temp)
snapbox_data[:, 0].tofile(f_snapbox_time)
snapbox_data[:, 1].tofile(f_snapbox_temp)
f_pi_temp.flush()
f_pi_time.flush()
f_snapbox_time.flush()
f_snapbox_time.flush()
except IOError:
print 'tempearatures files not found'
continue
future_dir = '/data/marion2017/switch_data/' + '/' + f.split(
'/')[-2] + '/' + f.split('/')[-1]
os.system('mv %s/*.raw %s/' % (f, future_dir))
os.system('rm %s/*.scio' % future_dir)
def read_channel_from_dirs(name,dirs,dtype='float64'):
#assumes dirs is a list of directories that
#contain possibly useful data. You probably want
#them sorted by ctime.
crap=[None]*len(dirs)
if name[-3:]=='npy':
#read numpy arrays
for ii,dr in enumerate(dirs):
fname=dr+'/'+name
crap[ii]=numpy.load(fname)
#if reading data, make sure it comes back suitable stacked
if len(crap[0].shape)==3:
return numpy.hstack(crap)
return numpy.hstack(crap)
if name[-4:]=='scio':
for ii,dr in enumerate(dirs):
fname=dr+'/'+name
crap[ii]=scio.read(fname)
#print crap[0].shape
crap=numpy.vstack(crap)
return numpy.swapaxes(crap,0,1)
#assume we have a raw type now
for ii,dr in enumerate(dirs):
fname=dr+'/'+name
f=open(fname)
crap[ii]=numpy.fromfile(f,dtype=dtype)
f.close()
#print crap[ii].shape
#print type(crap[ii])
return numpy.hstack(crap)
# Deal with time stamps
tstamp = ctime2timestamp(int(lastsubdir))
tstamp_outfile = tstamp.strftime(
'%Y%m%d_%H%M%S') # YYYYMMMDD_HHMMSS format for outfile
# Plotting routines for PRIZM antennas
if antname == '100MHz' or antname == '70MHz':
# Deal with time stamps
t1 = nm.fromfile(d + '/time_sys_start.raw')
t2 = nm.fromfile(d + '/time_sys_stop.raw')
time = 0.5 * (t1 + t2)
time = nm.asarray(ctime2timestamp(time, string=True))
fields = ['pol0.scio', 'pol1.scio']
pylab.figure(figsize=(15, 10))
for ifig, field in enumerate(fields):
ps = scio.read(d + '/' + field)
freq = nm.arange(nm.shape(ps)[1]) * 250. / nm.shape(ps)[1]
# Plot waterfall
pylab.subplot(2, 2, ifig * 2 + 1)
ps10 = nm.log10(ps)
med = nm.median(ps10[~nm.isnan(ps10)])
std = nm.std(ps10[~nm.isnan(ps10)])
vmin = med - std
vmax = med + std
# pylab.imshow(ps10, vmin=vmin, vmax=vmax, extent=[0, freq[-1], 0, nm.shape(ps10)[0]])
pylab.imshow(ps10,
aspect='auto',
vmin=vmin,
vmax=vmax,
extent=[0, freq[-1], 0,
nm.shape(ps10)[0]])
import numpy, scio
from matplotlib import pyplot as plt
aa = scio.read('pol0_1490282305.scio')
bb = scio.read('pol0_1490283209.scio')
cc = scio.read('pol0_1490284113.scio')
ee = scio.read(
'pol0_1490298177.scio') #this one has the filter over the Valon output
ff = scio.read(
'pol0_1490299078.scio') #this one has the filter over the Valon output
tmp = 0 * ee[0:10, :]
dd = numpy.vstack((aa, bb, cc, tmp, ee, ff))
delt = aa[107, :] - ee[
103, :] #this is a pair of nearly-matched before/after scans at same frequency
nu = ((numpy.arange(dd.shape[1]) + 0.5) / (1.0 * dd.shape[1])) * 250
tvec = (numpy.arange(dd.shape[0])) * 5.0
plt.ion()
plt.imshow(numpy.log10(dd),
aspect='auto',
extent=(0, 250, 5.0 * dd.shape[0], 0),
vmin=7.0,
vmax=8.0)
plt.xlabel('Frequency')
plt.colorbar()
plt.title('Log10(Power)')
plt.savefig('sweep_wfilter.png')
#plt.imshow(numpy.log10(dd),aspect='auto')
import numpy, scio
from matplotlib import pyplot as plt
aa = scio.read('pol1_1490391144.scio')
bb = scio.read('pol1_1490392045.scio')
cc = scio.read('pol1_1490392947.scio')
dd = numpy.vstack((aa, bb, cc))
nu = (numpy.arange(dd.shape[1]) / (1.0 * dd.shape[1] - 1) + 0.5) * 250
tvec = (numpy.arange(dd.shape[0])) * 5.0
plt.ion()
plt.imshow(numpy.log10(dd),
aspect='auto',
extent=(0, 250, 5.0 * dd.shape[0], 0))
plt.xlabel('Frequency')
plt.colorbar()
plt.title('Log10(Power)')
plt.savefig('durban_70mhz.png')
#plt.imshow(numpy.log10(dd),aspect='auto')
ii = dd[:, 1166] > 2e11
ii2 = dd[:, 1166] < 2e11
spec1 = numpy.median(dd[ii, :], axis=0)
spec2 = numpy.median(dd[ii2, :], axis=0)
import time
from matplotlib import pyplot as plt
import read_4bit
from importlib import reload
import scio
plt.ion()
fnames = glob.glob('../../baseband/15634/*.raw')
fnames.sort()
fnames = fnames[3:-2]
#fnames=fnames[:5]
#fnames_autocross=glob.glob('../../data_auto_cross/15634/*')
#fnames_autocross.sort()
froot = '../../data_auto_cross/15634/1563419861/'
spec0_snap = scio.read(froot + 'pol00.scio')
spec1_snap = scio.read(froot + 'pol11.scio')
spec01_snap = scio.read(froot +
'pol01r.scio') + 1J * scio.read(froot + 'pol01i.scio')
t_start = np.fromfile(froot + 'time_gps_start.raw', dtype='double')
t_stop = np.fromfile(froot + 'time_gps_stop.raw', dtype='double')
dt = t_stop - t_start
#dt=np.diff(t_start) #this seems be more accurate than t_stop-t_start
t_per_spec = 2048 * 2 / 250e6
nspec_approx = np.median(dt) / t_per_spec
items = -1
bin_size = np.int(nspec_approx)
for file_name in fnames:
import numpy,scio
n=50
m=100;
x=numpy.random.randn(n,m)
f=scio.scio('test1.scio')
ff=scio.scio('test2.scio',diff=True)
for i in range(n):
f.append(x[i,:])
ff.append(x[i,:])
f.close()
ff.close()
print x.shape
xx=scio.read('test1.scio')
xx2=scio.read('test2.scio')
import numpy, scio
from matplotlib import pyplot as plt
aa = scio.read('pol1_1490375185.scio')
bb = scio.read('pol1_1490376086.scio')
cc = scio.read('pol1_1490376991.scio')
dd = numpy.vstack((aa, bb, cc))
nu = (numpy.arange(dd.shape[1]) / (1.0 * dd.shape[1] - 1) + 0.5) * 250
tvec = (numpy.arange(dd.shape[0])) * 5.0
plt.ion()
plt.imshow(numpy.log10(dd),
aspect='auto',
extent=(0, 250, 5.0 * dd.shape[0], 0))
plt.xlabel('Frequency')
plt.colorbar()
plt.title('Log10(Power)')
plt.savefig('durban.png')
#plt.imshow(numpy.log10(dd),aspect='auto')
ii = dd[:, 1166] > 2e11
ii2 = dd[:, 1166] < 2e11
spec1 = numpy.median(dd[ii, :], axis=0)
spec2 = numpy.median(dd[ii2, :], axis=0)
f_sys_clk1, f_sys_clk2 = numpy.array([]), numpy.array([])
f_sync_cnt1, f_sync_cnt2 = numpy.array([]), numpy.array([])
f_pi_temp, f_fpga_temp = numpy.array([]), numpy.array([])
utctime = []
for dname in args:
try:
ctime = dname.split('/')[-1]
utctime.append(ct_to_utc(float(ctime)))
print 'Opening ', dname
print 'Uncompressing files'
os.system('bzip2 -dk ' + dname + '/pol0.scio.bz2')
os.system('bzip2 -dk ' + dname + '/pol1.scio.bz2')
os.system('bzip2 -dk ' + dname + '/cross_real.scio.bz2')
os.system('bzip2 -dk ' + dname + '/cross_imag.scio.bz2')
print 'Reading scio files'
pol0 = scio.read(dname + '/pol0.scio')
pol1 = scio.read(dname + '/pol1.scio')
cross_real = scio.read(dname + '/cross_real.scio')
cross_imag = scio.read(dname + '/cross_imag.scio')
cross = cross_real + 1j * cross_imag
print 'Reading raw files'
t_start = numpy.fromfile(dname + '/time_start.raw')
t_stop = numpy.fromfile(dname + '/time_stop.raw')
acc_cnt1 = numpy.fromfile(dname + '/acc_cnt1.raw', dtype='int32')
acc_cnt2 = numpy.fromfile(dname + '/acc_cnt2.raw', dtype='int32')
fft_of_cnt = numpy.fromfile(dname + '/fft_of_cnt.raw',
dtype='int32')
fft_shift = numpy.fromfile(dname + '/fft_shift.raw', dtype='int32')
sys_clk1 = numpy.fromfile(dname + '/sys_clk1.raw', dtype='int32')
sys_clk2 = numpy.fromfile(dname + '/sys_clk2.raw', dtype='int32')
import numpy, scio
from matplotlib import pyplot as plt
aa = scio.read('pol1_1490380508.scio')
dd = aa.copy()
nu = ((numpy.arange(dd.shape[1]) + 0.5) / (1.0 * dd.shape[1])) * 250
tvec = (numpy.arange(dd.shape[0])) * 5.0
short_spec = numpy.median(aa, 0)
bb = scio.read('pol1_1490378695.scio')
load_spec = numpy.median(bb, 0)
aa = scio.read('pol1_1490375185.scio')
bb = scio.read('pol1_1490376086.scio')
cc = scio.read('pol1_1490376991.scio')
dd = numpy.vstack((aa, bb, cc))
sky_spec = numpy.median(dd, 0)
plt.ion()
plt.clf()
plt.plot(nu[1:], short_spec[1:])
plt.plot(nu[1:], load_spec[1:])
plt.plot(nu[1:], sky_spec[1:])
plt.yscale('log')
plt.legend(['Short', '50 Ohm load', 'Antenna'])
plt.ylabel('power')
plt.xlabel('Frequency (MHz)')
def read_scihi_data(time_start,
time_stop,
dir_top,
subdir_100='data_100MHz',
subdir_70='data_70MHz',
subdir_switch='switch_data/data/switch',
subdir_temp='switch_data/data/temperatures',
read_100=True,
read_70=True,
read_switch=True,
read_temp=True,
trim=True,
verbose=False):
"""Read SCI-HI data within a specified time range.
- time_start, time_stop = start/stop times in ctime
- dir_top = top level data directory
- subdir_100 = 100 MHz subdirectory within the top level directory
- subdir_70 = 70 MHz subdirectory within the top level directory
- subdir_switch = switch state subdirectory within the top level directory
- subdir_temp = temperature data subdirectory within the top level directory
- read_100, read_70, read_switch, read_temp = Booleans to select which data to (not) read
- trim = trim data exactly to time range, or read everything in matching subdirectories if false
- verboase = speak to the human?
Return dictionaries with all the datas!!!
"""
scihi_dat = {}
scio_fields = [
'pol0.scio', 'pol1.scio', 'cross_real.scio', 'cross_imag.scio'
]
raw_fields = [('acc_cnt1.raw', 'int32'), ('acc_cnt2.raw', 'int32'),
('fft_of_cnt.raw', 'int32'), ('fft_shift.raw', 'int64'),
('fpga_temp.raw', 'float'), ('pi_temp.raw', 'int32'),
('sync_cnt1.raw', 'int32'), ('sync_cnt2.raw', 'int32'),
('sys_clk1.raw', 'int32'), ('sys_clk2.raw', 'int32'),
('time_start.raw', 'float'), ('time_stop.raw', 'float')]
# Start by reading primary data products for specified antennas
keys = []
subdirs = {}
if read_100:
key = '100'
keys.append(key)
subdirs[key] = subdir_100
if read_70:
key = '70'
keys.append(key)
subdirs[key] = subdir_70
for key in keys:
scihi_dat[key] = {}
dirs = time2dirs(time_start, time_stop, dir_top + '/' + subdirs[key])
for d in dirs:
# Keep track of field lengths just in case there's a mismatch somewhere
field_lengths = []
# Read scio files, and decompress only if it hasn't been done already
# Note: might want to add option later to read .bz2 files directly if we're low on disk space
for field in scio_fields:
if not os.path.exists(d + '/' + field):
if verbose:
print 'read_scihi_data: decompressing', d + '/' + field
os.system('bzip2 -dk ' + d + '/' + field + '.bz2')
if verbose:
print 'read_scihi_data: reading scio files in', d
for field in scio_fields:
dat = scio.read(d + '/' + field)
field_lengths.append(len(dat))
if d is dirs[0]:
scihi_dat[key][field.split('.')[0]] = dat
else:
scihi_dat[key][field.split('.')[0]] = nm.append(
scihi_dat[key][field.split('.')[0]], dat, axis=0)
# Read raw files
if verbose:
print 'read_scihi_data: reading raw files in', d
for field in raw_fields:
dat = nm.fromfile(d + '/' + field[0], dtype=field[1])
field_lengths.append(len(dat))
if d is dirs[0]:
scihi_dat[key][field[0].split('.')[0]] = dat
else:
scihi_dat[key][field[0].split('.')[0]] = nm.append(
scihi_dat[key][field[0].split('.')[0]], dat)
# Check consistency of field lengths
if len(nm.where(nm.diff(field_lengths) != 0)[0]) > 0:
print 'read_scihi_data: warning, found inconsistent field lengths in', d, ' -- OMITTING THIS DATA'
print 'read_scihi_data: field lengths are', field_lengths
# Remove the data if it's borked
subkeys = [field.split('.')[0] for field in scio_fields] + \
[field[0].split('.')[0] for field in raw_fields]
for field_length, subkey in zip(field_lengths, subkeys):
scihi_dat[key][subkey] = scihi_dat[key][
subkey][:-field_length]
# Trim to desired time range
if trim:
t0 = scihi_dat[key]['time_start'] # Start of FPGA read
t1 = scihi_dat[key]['time_stop'] # End of FPGA read
inds = nm.where((t0 >= time_start) & (t1 <= time_stop))[0]
if len(inds) == 0:
print 'read_scihi_data: warning, found no data in requested time range'
subkeys = scihi_dat[key].keys()
for subkey in subkeys:
scihi_dat[key][subkey] = scihi_dat[key][subkey][inds]
# Now move on to auxiliary data
if read_switch:
if verbose:
print 'read_scihi_data: reading switch data'
key = 'switch'
scihi_dat[key] = {}
dirs = time2dirs(time_start, time_stop, dir_top + '/' + subdir_switch)
scio_fields = [
'antenna.scio', 'res100.scio', 'res50.scio', 'short.scio'
]
for d in dirs:
for field in scio_fields:
# Switch fields are slow, so sometimes the files are empty. Check this before reading.
st = os.stat(d + '/' + field)
# If file is empty, just move along
if st.st_size == 0:
continue
dat = scio.read(d + '/' + field)
if d is dirs[0]:
scihi_dat[key][field.split('.')[0]] = dat
else:
scihi_dat[key][field.split('.')[0]] = nm.append(
scihi_dat[key][field.split('.')[0]], dat, axis=0)
# Trim to desired time range
subkeys = scihi_dat[key].keys()
for subkey in subkeys:
inds = nm.where((scihi_dat[key][subkey][:, 1] >= time_start)
& (scihi_dat[key][subkey][:, 1] <= time_stop))[0]
scihi_dat[key][subkey] = scihi_dat[key][subkey][inds]
if read_temp:
if verbose:
print 'read_scihi_data: reading temperature data'
key = 'temp'
scihi_dat[key] = {}
dirs = time2dirs(time_start, time_stop, dir_top + '/' + subdir_temp)
scio_fields = ['pi_temp.scio', 'snapbox_temp.scio']
for d in dirs:
for field in scio_fields:
dat = scio.read(d + '/' + field)
if d is dirs[0]:
scihi_dat[key][field.split('.')[0]] = dat
else:
scihi_dat[key][field.split('.')[0]] = nm.append(
scihi_dat[key][field.split('.')[0]], dat, axis=0)
# Trim to desired time range
# Write this code later, after we have data products that are functioning
if trim:
print 'Warning, trimming not implemented for aux temperatures yet'
return scihi_dat
"Collecting calibration data... WILL RUN FOR {} SECONDS"
.format(args.runtime))
cmd = "python dump_spectra.py -c " + args.configfile
cmd = cmd.split(' ')
pp = subprocess.Popen(cmd)
time.sleep(args.runtime)
pp.terminate()
while True:
tstamp = raw_input(bcolors.UNDERLINE +
"Timestamp:".format(freq, pwr) +
bcolors.ENDC + " ")
data_dir = spectra_output_dir + '/' + tstamp[:
5] + '/' + tstamp
if adc == 0:
if os.path.isfile(data_dir + '/pol00.scio'):
poldata = scio.read(data_dir +
'/pol00.scio')
break
elif os.path.isfile(data_dir +
'/pol00.scio.bzip2'):
poldata = scio.read(data_dir +
'/pol00.scio.bzip2')
break
elif adc == 3:
if os.path.isfile(data_dir + '/pol11.scio'):
poldata = scio.read(data_dir +
'/pol11.scio')
break
elif os.path.isfile(data_dir +
'/pol11.scio.bzip2'):
poldata = scio.read(data_dir +
'/pol11.scio.bzip2')
import numpy, scio
from matplotlib import pyplot as plt
aa = scio.read('pol0_1490282305.scio')
bb = scio.read('pol0_1490283209.scio')
cc = scio.read('pol0_1490284113.scio')
dd = numpy.vstack((aa, bb, cc))
nu = (numpy.arange(dd.shape[1]) / (1.0 * dd.shape[1] - 1) + 0.5) * 250
tvec = (numpy.arange(dd.shape[0])) * 5.0
plt.ion()
plt.imshow(numpy.log10(dd),
aspect='auto',
extent=(0, 250, 5.0 * dd.shape[0], 0))
plt.xlabel('Frequency')
plt.colorbar()
plt.title('Log10(Power)')
plt.savefig('sweep.png')
#plt.imshow(numpy.log10(dd),aspect='auto')
