def move_temperatre_data(): """ Moving chunk of temperaure data stored in switch previously to switch_data and change scio to raw files """ temp_dirs = glob.glob('/data/marion2017/switch_data/temperatures/*') for subdir in temp_dirs: fields = glob.glob(subdir + '/*') for f in fields: try: pi_data = scio.read(f + '/pi_temp.scio') snapbox_data = scio.read(f + '/snapbox_temp.scio') f_pi_temp = open(f + '/pi_temp.raw', 'w') f_pi_time = open(f + '/pi_time.raw', 'w') f_snapbox_temp = open(f + '/snapbox_temp.raw', 'w') f_snapbox_time = open(f + '/snapbox_time.raw', 'w') pi_temp = numpy.int32(pi_data[:, 1]) pi_temp.tofile(f_pi_time) pi_data[:, 1].tofile(f_pi_temp) snapbox_data[:, 0].tofile(f_snapbox_time) snapbox_data[:, 1].tofile(f_snapbox_temp) f_pi_temp.flush() f_pi_time.flush() f_snapbox_time.flush() f_snapbox_time.flush() except IOError: print 'tempearatures files not found' continue future_dir = '/data/marion2017/switch_data/' + '/' + f.split( '/')[-2] + '/' + f.split('/')[-1] os.system('mv %s/*.raw %s/' % (f, future_dir)) os.system('rm %s/*.scio' % future_dir)
def read_channel_from_dirs(name,dirs,dtype='float64'): #assumes dirs is a list of directories that #contain possibly useful data. You probably want #them sorted by ctime. crap=[None]*len(dirs) if name[-3:]=='npy': #read numpy arrays for ii,dr in enumerate(dirs): fname=dr+'/'+name crap[ii]=numpy.load(fname) #if reading data, make sure it comes back suitable stacked if len(crap[0].shape)==3: return numpy.hstack(crap) return numpy.hstack(crap) if name[-4:]=='scio': for ii,dr in enumerate(dirs): fname=dr+'/'+name crap[ii]=scio.read(fname) #print crap[0].shape crap=numpy.vstack(crap) return numpy.swapaxes(crap,0,1) #assume we have a raw type now for ii,dr in enumerate(dirs): fname=dr+'/'+name f=open(fname) crap[ii]=numpy.fromfile(f,dtype=dtype) f.close() #print crap[ii].shape #print type(crap[ii]) return numpy.hstack(crap)
# Deal with time stamps tstamp = ctime2timestamp(int(lastsubdir)) tstamp_outfile = tstamp.strftime( '%Y%m%d_%H%M%S') # YYYYMMMDD_HHMMSS format for outfile # Plotting routines for PRIZM antennas if antname == '100MHz' or antname == '70MHz': # Deal with time stamps t1 = nm.fromfile(d + '/time_sys_start.raw') t2 = nm.fromfile(d + '/time_sys_stop.raw') time = 0.5 * (t1 + t2) time = nm.asarray(ctime2timestamp(time, string=True)) fields = ['pol0.scio', 'pol1.scio'] pylab.figure(figsize=(15, 10)) for ifig, field in enumerate(fields): ps = scio.read(d + '/' + field) freq = nm.arange(nm.shape(ps)[1]) * 250. / nm.shape(ps)[1] # Plot waterfall pylab.subplot(2, 2, ifig * 2 + 1) ps10 = nm.log10(ps) med = nm.median(ps10[~nm.isnan(ps10)]) std = nm.std(ps10[~nm.isnan(ps10)]) vmin = med - std vmax = med + std # pylab.imshow(ps10, vmin=vmin, vmax=vmax, extent=[0, freq[-1], 0, nm.shape(ps10)[0]]) pylab.imshow(ps10, aspect='auto', vmin=vmin, vmax=vmax, extent=[0, freq[-1], 0, nm.shape(ps10)[0]])
import numpy, scio from matplotlib import pyplot as plt aa = scio.read('pol0_1490282305.scio') bb = scio.read('pol0_1490283209.scio') cc = scio.read('pol0_1490284113.scio') ee = scio.read( 'pol0_1490298177.scio') #this one has the filter over the Valon output ff = scio.read( 'pol0_1490299078.scio') #this one has the filter over the Valon output tmp = 0 * ee[0:10, :] dd = numpy.vstack((aa, bb, cc, tmp, ee, ff)) delt = aa[107, :] - ee[ 103, :] #this is a pair of nearly-matched before/after scans at same frequency nu = ((numpy.arange(dd.shape[1]) + 0.5) / (1.0 * dd.shape[1])) * 250 tvec = (numpy.arange(dd.shape[0])) * 5.0 plt.ion() plt.imshow(numpy.log10(dd), aspect='auto', extent=(0, 250, 5.0 * dd.shape[0], 0), vmin=7.0, vmax=8.0) plt.xlabel('Frequency') plt.colorbar() plt.title('Log10(Power)') plt.savefig('sweep_wfilter.png') #plt.imshow(numpy.log10(dd),aspect='auto')
import numpy, scio from matplotlib import pyplot as plt aa = scio.read('pol1_1490391144.scio') bb = scio.read('pol1_1490392045.scio') cc = scio.read('pol1_1490392947.scio') dd = numpy.vstack((aa, bb, cc)) nu = (numpy.arange(dd.shape[1]) / (1.0 * dd.shape[1] - 1) + 0.5) * 250 tvec = (numpy.arange(dd.shape[0])) * 5.0 plt.ion() plt.imshow(numpy.log10(dd), aspect='auto', extent=(0, 250, 5.0 * dd.shape[0], 0)) plt.xlabel('Frequency') plt.colorbar() plt.title('Log10(Power)') plt.savefig('durban_70mhz.png') #plt.imshow(numpy.log10(dd),aspect='auto') ii = dd[:, 1166] > 2e11 ii2 = dd[:, 1166] < 2e11 spec1 = numpy.median(dd[ii, :], axis=0) spec2 = numpy.median(dd[ii2, :], axis=0)
import time from matplotlib import pyplot as plt import read_4bit from importlib import reload import scio plt.ion() fnames = glob.glob('../../baseband/15634/*.raw') fnames.sort() fnames = fnames[3:-2] #fnames=fnames[:5] #fnames_autocross=glob.glob('../../data_auto_cross/15634/*') #fnames_autocross.sort() froot = '../../data_auto_cross/15634/1563419861/' spec0_snap = scio.read(froot + 'pol00.scio') spec1_snap = scio.read(froot + 'pol11.scio') spec01_snap = scio.read(froot + 'pol01r.scio') + 1J * scio.read(froot + 'pol01i.scio') t_start = np.fromfile(froot + 'time_gps_start.raw', dtype='double') t_stop = np.fromfile(froot + 'time_gps_stop.raw', dtype='double') dt = t_stop - t_start #dt=np.diff(t_start) #this seems be more accurate than t_stop-t_start t_per_spec = 2048 * 2 / 250e6 nspec_approx = np.median(dt) / t_per_spec items = -1 bin_size = np.int(nspec_approx) for file_name in fnames:
import numpy,scio n=50 m=100; x=numpy.random.randn(n,m) f=scio.scio('test1.scio') ff=scio.scio('test2.scio',diff=True) for i in range(n): f.append(x[i,:]) ff.append(x[i,:]) f.close() ff.close() print x.shape xx=scio.read('test1.scio') xx2=scio.read('test2.scio')
import numpy, scio from matplotlib import pyplot as plt aa = scio.read('pol1_1490375185.scio') bb = scio.read('pol1_1490376086.scio') cc = scio.read('pol1_1490376991.scio') dd = numpy.vstack((aa, bb, cc)) nu = (numpy.arange(dd.shape[1]) / (1.0 * dd.shape[1] - 1) + 0.5) * 250 tvec = (numpy.arange(dd.shape[0])) * 5.0 plt.ion() plt.imshow(numpy.log10(dd), aspect='auto', extent=(0, 250, 5.0 * dd.shape[0], 0)) plt.xlabel('Frequency') plt.colorbar() plt.title('Log10(Power)') plt.savefig('durban.png') #plt.imshow(numpy.log10(dd),aspect='auto') ii = dd[:, 1166] > 2e11 ii2 = dd[:, 1166] < 2e11 spec1 = numpy.median(dd[ii, :], axis=0) spec2 = numpy.median(dd[ii2, :], axis=0)
f_sys_clk1, f_sys_clk2 = numpy.array([]), numpy.array([]) f_sync_cnt1, f_sync_cnt2 = numpy.array([]), numpy.array([]) f_pi_temp, f_fpga_temp = numpy.array([]), numpy.array([]) utctime = [] for dname in args: try: ctime = dname.split('/')[-1] utctime.append(ct_to_utc(float(ctime))) print 'Opening ', dname print 'Uncompressing files' os.system('bzip2 -dk ' + dname + '/pol0.scio.bz2') os.system('bzip2 -dk ' + dname + '/pol1.scio.bz2') os.system('bzip2 -dk ' + dname + '/cross_real.scio.bz2') os.system('bzip2 -dk ' + dname + '/cross_imag.scio.bz2') print 'Reading scio files' pol0 = scio.read(dname + '/pol0.scio') pol1 = scio.read(dname + '/pol1.scio') cross_real = scio.read(dname + '/cross_real.scio') cross_imag = scio.read(dname + '/cross_imag.scio') cross = cross_real + 1j * cross_imag print 'Reading raw files' t_start = numpy.fromfile(dname + '/time_start.raw') t_stop = numpy.fromfile(dname + '/time_stop.raw') acc_cnt1 = numpy.fromfile(dname + '/acc_cnt1.raw', dtype='int32') acc_cnt2 = numpy.fromfile(dname + '/acc_cnt2.raw', dtype='int32') fft_of_cnt = numpy.fromfile(dname + '/fft_of_cnt.raw', dtype='int32') fft_shift = numpy.fromfile(dname + '/fft_shift.raw', dtype='int32') sys_clk1 = numpy.fromfile(dname + '/sys_clk1.raw', dtype='int32') sys_clk2 = numpy.fromfile(dname + '/sys_clk2.raw', dtype='int32')
import numpy, scio from matplotlib import pyplot as plt aa = scio.read('pol1_1490380508.scio') dd = aa.copy() nu = ((numpy.arange(dd.shape[1]) + 0.5) / (1.0 * dd.shape[1])) * 250 tvec = (numpy.arange(dd.shape[0])) * 5.0 short_spec = numpy.median(aa, 0) bb = scio.read('pol1_1490378695.scio') load_spec = numpy.median(bb, 0) aa = scio.read('pol1_1490375185.scio') bb = scio.read('pol1_1490376086.scio') cc = scio.read('pol1_1490376991.scio') dd = numpy.vstack((aa, bb, cc)) sky_spec = numpy.median(dd, 0) plt.ion() plt.clf() plt.plot(nu[1:], short_spec[1:]) plt.plot(nu[1:], load_spec[1:]) plt.plot(nu[1:], sky_spec[1:]) plt.yscale('log') plt.legend(['Short', '50 Ohm load', 'Antenna']) plt.ylabel('power') plt.xlabel('Frequency (MHz)')
def read_scihi_data(time_start, time_stop, dir_top, subdir_100='data_100MHz', subdir_70='data_70MHz', subdir_switch='switch_data/data/switch', subdir_temp='switch_data/data/temperatures', read_100=True, read_70=True, read_switch=True, read_temp=True, trim=True, verbose=False): """Read SCI-HI data within a specified time range. - time_start, time_stop = start/stop times in ctime - dir_top = top level data directory - subdir_100 = 100 MHz subdirectory within the top level directory - subdir_70 = 70 MHz subdirectory within the top level directory - subdir_switch = switch state subdirectory within the top level directory - subdir_temp = temperature data subdirectory within the top level directory - read_100, read_70, read_switch, read_temp = Booleans to select which data to (not) read - trim = trim data exactly to time range, or read everything in matching subdirectories if false - verboase = speak to the human? Return dictionaries with all the datas!!! """ scihi_dat = {} scio_fields = [ 'pol0.scio', 'pol1.scio', 'cross_real.scio', 'cross_imag.scio' ] raw_fields = [('acc_cnt1.raw', 'int32'), ('acc_cnt2.raw', 'int32'), ('fft_of_cnt.raw', 'int32'), ('fft_shift.raw', 'int64'), ('fpga_temp.raw', 'float'), ('pi_temp.raw', 'int32'), ('sync_cnt1.raw', 'int32'), ('sync_cnt2.raw', 'int32'), ('sys_clk1.raw', 'int32'), ('sys_clk2.raw', 'int32'), ('time_start.raw', 'float'), ('time_stop.raw', 'float')] # Start by reading primary data products for specified antennas keys = [] subdirs = {} if read_100: key = '100' keys.append(key) subdirs[key] = subdir_100 if read_70: key = '70' keys.append(key) subdirs[key] = subdir_70 for key in keys: scihi_dat[key] = {} dirs = time2dirs(time_start, time_stop, dir_top + '/' + subdirs[key]) for d in dirs: # Keep track of field lengths just in case there's a mismatch somewhere field_lengths = [] # Read scio files, and decompress only if it hasn't been done already # Note: might want to add option later to read .bz2 files directly if we're low on disk space for field in scio_fields: if not os.path.exists(d + '/' + field): if verbose: print 'read_scihi_data: decompressing', d + '/' + field os.system('bzip2 -dk ' + d + '/' + field + '.bz2') if verbose: print 'read_scihi_data: reading scio files in', d for field in scio_fields: dat = scio.read(d + '/' + field) field_lengths.append(len(dat)) if d is dirs[0]: scihi_dat[key][field.split('.')[0]] = dat else: scihi_dat[key][field.split('.')[0]] = nm.append( scihi_dat[key][field.split('.')[0]], dat, axis=0) # Read raw files if verbose: print 'read_scihi_data: reading raw files in', d for field in raw_fields: dat = nm.fromfile(d + '/' + field[0], dtype=field[1]) field_lengths.append(len(dat)) if d is dirs[0]: scihi_dat[key][field[0].split('.')[0]] = dat else: scihi_dat[key][field[0].split('.')[0]] = nm.append( scihi_dat[key][field[0].split('.')[0]], dat) # Check consistency of field lengths if len(nm.where(nm.diff(field_lengths) != 0)[0]) > 0: print 'read_scihi_data: warning, found inconsistent field lengths in', d, ' -- OMITTING THIS DATA' print 'read_scihi_data: field lengths are', field_lengths # Remove the data if it's borked subkeys = [field.split('.')[0] for field in scio_fields] + \ [field[0].split('.')[0] for field in raw_fields] for field_length, subkey in zip(field_lengths, subkeys): scihi_dat[key][subkey] = scihi_dat[key][ subkey][:-field_length] # Trim to desired time range if trim: t0 = scihi_dat[key]['time_start'] # Start of FPGA read t1 = scihi_dat[key]['time_stop'] # End of FPGA read inds = nm.where((t0 >= time_start) & (t1 <= time_stop))[0] if len(inds) == 0: print 'read_scihi_data: warning, found no data in requested time range' subkeys = scihi_dat[key].keys() for subkey in subkeys: scihi_dat[key][subkey] = scihi_dat[key][subkey][inds] # Now move on to auxiliary data if read_switch: if verbose: print 'read_scihi_data: reading switch data' key = 'switch' scihi_dat[key] = {} dirs = time2dirs(time_start, time_stop, dir_top + '/' + subdir_switch) scio_fields = [ 'antenna.scio', 'res100.scio', 'res50.scio', 'short.scio' ] for d in dirs: for field in scio_fields: # Switch fields are slow, so sometimes the files are empty. Check this before reading. st = os.stat(d + '/' + field) # If file is empty, just move along if st.st_size == 0: continue dat = scio.read(d + '/' + field) if d is dirs[0]: scihi_dat[key][field.split('.')[0]] = dat else: scihi_dat[key][field.split('.')[0]] = nm.append( scihi_dat[key][field.split('.')[0]], dat, axis=0) # Trim to desired time range subkeys = scihi_dat[key].keys() for subkey in subkeys: inds = nm.where((scihi_dat[key][subkey][:, 1] >= time_start) & (scihi_dat[key][subkey][:, 1] <= time_stop))[0] scihi_dat[key][subkey] = scihi_dat[key][subkey][inds] if read_temp: if verbose: print 'read_scihi_data: reading temperature data' key = 'temp' scihi_dat[key] = {} dirs = time2dirs(time_start, time_stop, dir_top + '/' + subdir_temp) scio_fields = ['pi_temp.scio', 'snapbox_temp.scio'] for d in dirs: for field in scio_fields: dat = scio.read(d + '/' + field) if d is dirs[0]: scihi_dat[key][field.split('.')[0]] = dat else: scihi_dat[key][field.split('.')[0]] = nm.append( scihi_dat[key][field.split('.')[0]], dat, axis=0) # Trim to desired time range # Write this code later, after we have data products that are functioning if trim: print 'Warning, trimming not implemented for aux temperatures yet' return scihi_dat
"Collecting calibration data... WILL RUN FOR {} SECONDS" .format(args.runtime)) cmd = "python dump_spectra.py -c " + args.configfile cmd = cmd.split(' ') pp = subprocess.Popen(cmd) time.sleep(args.runtime) pp.terminate() while True: tstamp = raw_input(bcolors.UNDERLINE + "Timestamp:".format(freq, pwr) + bcolors.ENDC + " ") data_dir = spectra_output_dir + '/' + tstamp[: 5] + '/' + tstamp if adc == 0: if os.path.isfile(data_dir + '/pol00.scio'): poldata = scio.read(data_dir + '/pol00.scio') break elif os.path.isfile(data_dir + '/pol00.scio.bzip2'): poldata = scio.read(data_dir + '/pol00.scio.bzip2') break elif adc == 3: if os.path.isfile(data_dir + '/pol11.scio'): poldata = scio.read(data_dir + '/pol11.scio') break elif os.path.isfile(data_dir + '/pol11.scio.bzip2'): poldata = scio.read(data_dir + '/pol11.scio.bzip2')
import numpy, scio from matplotlib import pyplot as plt aa = scio.read('pol0_1490282305.scio') bb = scio.read('pol0_1490283209.scio') cc = scio.read('pol0_1490284113.scio') dd = numpy.vstack((aa, bb, cc)) nu = (numpy.arange(dd.shape[1]) / (1.0 * dd.shape[1] - 1) + 0.5) * 250 tvec = (numpy.arange(dd.shape[0])) * 5.0 plt.ion() plt.imshow(numpy.log10(dd), aspect='auto', extent=(0, 250, 5.0 * dd.shape[0], 0)) plt.xlabel('Frequency') plt.colorbar() plt.title('Log10(Power)') plt.savefig('sweep.png') #plt.imshow(numpy.log10(dd),aspect='auto')