def get_diff(dio_cross, feedtype, **kwargs): ''' Returns ON-OFF for all Stokes parameters given a cross_pols noise diode measurement ''' #Get Stokes parameters, frequencies, and time sample length obs = Waterfall(dio_cross, max_load=150) freqs = obs.populate_freqs() tsamp = obs.header['tsamp'] data = obs.data obs = None I, Q, U, V = get_stokes(data, feedtype) #Fold noise diode data I_OFF, I_ON = foldcal(I, tsamp, **kwargs) Q_OFF, Q_ON = foldcal(Q, tsamp, **kwargs) U_OFF, U_ON = foldcal(U, tsamp, **kwargs) V_OFF, V_ON = foldcal(V, tsamp, **kwargs) #Do ON-OFF subtraction Idiff = I_ON - I_OFF Qdiff = Q_ON - Q_OFF Udiff = U_ON - U_OFF Vdiff = V_ON - V_OFF return Idiff, Qdiff, Udiff, Vdiff, freqs
def __init__(self, filename, f_start=None, f_stop=None, t_start=None, t_stop=None, coarse_chan=1, n_coarse_chan=None): """ :param filename: string name of file :param f_start: float start frequency in MHz :param f_stop: float stop frequency in MHz :param t_start: int start integration ID :param t_stop: int stop integration ID :param coarse_chan: int :param n_coarse_chan: int """ self.filename = filename self.closed = False self.f_start = f_start self.f_stop = f_stop self.t_start = t_start self.t_stop = t_stop self.n_coarse_chan = n_coarse_chan #Instancing file. try: self.fil_file = Waterfall(filename, f_start=self.f_start, f_stop=self.f_stop, t_start=self.t_start, t_stop=self.t_stop, load_data=False) except: errmsg = "Error encountered when trying to open file: {}".format(filename) logger.error(errmsg) raise IOError(errmsg) #Getting header try: if self.n_coarse_chan: header = self.__make_data_header(self.fil_file.header, coarse=True) else: header = self.__make_data_header(self.fil_file.header) except: errmsg = "Error accessing header from file: {}".format(self.fil_file.header) logger.error(errmsg) raise IOError(errmsg) self.header = header self.fftlen = header['NAXIS1'] #EE To check if swapping tsteps_valid and tsteps is more appropriate. self.tsteps_valid = header['NAXIS2'] self.tsteps = int(math.pow(2, math.ceil(np.log2(math.floor(self.tsteps_valid))))) self.obs_length = self.tsteps_valid * header['DELTAT'] self.drift_rate_resolution = (1e6 * np.abs(header['DELTAF'])) / self.obs_length # in Hz/sec self.header['baryv'] = 0.0 self.header['barya'] = 0.0 self.header['coarse_chan'] = coarse_chan #EE For now I'm not using a shoulder. This is ok as long as ## I'm analyzing each coarse channel individually. #EE In general this parameter is an integer (even number). #This gives two regions, each of n*steps, around spectra[i] self.shoulder_size = 0 self.tdwidth = self.fftlen + self.shoulder_size * self.tsteps
def get_Tsys(calON_obs, calOFF_obs, calflux, calfreq, spec_in, oneflux=False, **kwargs): ''' Returns frequency dependent system temperature given observations on and off a calibrator source Parameters ---------- (See diode_spec()) ''' obs = Waterfall(calON_obs, max_load=150) ncoarse = obs.calc_n_coarse_chan() chan_per_coarse = obs.header['nchans'] / ncoarse freqs = obs.populate_freqs() cfreqs = get_centerfreqs(freqs, chan_per_coarse) S_sys = diode_spec(calON_obs, calOFF_obs, calflux, calfreq, spec_in, average=False, oneflux=False, **kwargs)[1] T_sys = Jy_to_Kelvin(S_sys, cfreqs) return T_sys, cfreqs
def plot_hit(fil_filename, dat_filename, hit_id, bw=None, offset=0): r"""Plot a candidate from a .dat file Args: fil_filename(str): Path to filterbank file to plot dat_filename(str): Path to turbosSETI generated .dat output file of events hit_id(int): ID of hit in the dat file to plot (TopHitNum) offset(float, optional): Offset drift line on plot. Default 0. bw: (Default value = None) Returns: """ # Load hit details dat = find_event.make_table(dat_filename) hit = dat.iloc[hit_id] f0 = hit['Freq'] if bw is None: bw_mhz = np.abs(hit['FreqStart'] - hit['FreqEnd']) else: bw_mhz = bw * 1e-6 fil = Waterfall(fil_filename, f_start=f0 - bw_mhz / 2, f_stop=f0 + bw_mhz / 2) t_duration = (fil.n_ints_in_file - 1) * fil.header['tsamp'] fil.plot_waterfall() plot_event.overlay_drift(f0, f0, f0, hit['DriftRate'], t_duration, offset)
def __split_h5(self, size_limit=SIZE_LIM): '''Creates a plan to select data from single coarse channels. ''' data_list = [] #Instancing file. try: fil_file = Waterfall(self.filename) except: logger.error("Error encountered when trying to open file: %s" % self.filename) raise IOError("Error encountered when trying to open file: %s" % self.filename) #Finding lowest freq in file. try: f_delt = fil_file.header[u'foff'] f0 = fil_file.header[u'fch1'] except: f_delt = fil_file.header[b'foff'] f0 = fil_file.header[b'fch1'] #Looping over the number of coarse channels. n_coarse_chan = int(fil_file.calc_n_coarse_chan()) if n_coarse_chan != fil_file.calc_n_coarse_chan(): logger.warning( 'The file/selection is not an integer number of coarse channels. This could have unexpected consequences. Let op!' ) for chan in range(n_coarse_chan): #Calculate freq range for given course channel. f_start = f0 + chan * ( f_delt) * fil_file.n_channels_in_file / n_coarse_chan f_stop = f0 + (chan + 1) * ( f_delt) * fil_file.n_channels_in_file / n_coarse_chan if f_start > f_stop: f_start, f_stop = f_stop, f_start data_obj = DATAH5(self.filename, f_start=f_start, f_stop=f_stop, coarse_chan=chan, tn_coarse_chan=n_coarse_chan) #---------------------------------------------------------------- # fn = filename[filename.rfind('/')+1:filename.rfind('.fil')] # deltaf = header[u'DELTAF'] # fftlen = header[u'NAXIS1'] # fcntr = header[u'FCNTR'] # frange = [fcntr - fftlen*deltaf/2, fcntr + fftlen*deltaf/2] #This appends to a list of all data instance selections. So that all get processed later. data_list.append(data_obj) return data_list
def __init__(self, filename, size_limit = SIZE_LIM,f_start=None, f_stop=None,t_start=None, t_stop=None,coarse_chan=1,tn_coarse_chan=None): self.filename = filename self.closed = False self.f_start = f_start self.f_stop = f_stop self.t_start = t_start self.t_stop = t_stop self.tn_coarse_chan = tn_coarse_chan #Instancing file. try: self.fil_file = Waterfall(filename,f_start=self.f_start, f_stop=self.f_stop,t_start=self.t_start, t_stop=self.t_stop,load_data=False) except: logger.error("Error encountered when trying to open file %s"%filename) raise IOError("Error encountered when trying to open file %s"%filename) #Getting header try: if self.tn_coarse_chan: header = self.__make_data_header(self.fil_file.header,coarse=True) else: header = self.__make_data_header(self.fil_file.header) except: logger.debug('The fil_file.header is ' % self.fil_file.header) raise IOError("Error accessing header from file: %s." % self.filename) self.header = header try: self.fftlen = header[u'NAXIS1'] #EE To check if swapping tsteps_valid and tsteps is more appropriate. self.tsteps_valid = header[u'NAXIS2'] self.tsteps = int(math.pow(2, math.ceil(np.log2(math.floor(self.tsteps_valid))))) self.obs_length = self.tsteps_valid * header[u'DELTAT'] self.drift_rate_resolution = (1e6 * np.abs(header[u'DELTAF'])) / self.obs_length # in Hz/sec self.header[u'baryv'] = 0.0 self.header[u'barya'] = 0.0 self.header[u'coarse_chan'] = coarse_chan except: self.fftlen = header[b'NAXIS1'] #EE To check if swapping tsteps_valid and tsteps is more appropriate. self.tsteps_valid = header[b'NAXIS2'] self.tsteps = int(math.pow(2, math.ceil(np.log2(math.floor(self.tsteps_valid))))) self.obs_length = self.tsteps_valid * header[b'DELTAT'] self.drift_rate_resolution = (1e6 * np.abs(header[b'DELTAF'])) / self.obs_length # in Hz/sec self.header[b'baryv'] = 0.0 self.header[b'barya'] = 0.0 self.header[b'coarse_chan'] = coarse_chan #EE For now I'm not using a shoulder. This is ok as long as I'm analyzing each coarse channel individually. #EE In general this parameter is an integer (even number). #This gives two regions, each of n*steps, around spectra[i] self.shoulder_size = 0 self.tdwidth = self.fftlen + self.shoulder_size*self.tsteps
def __make_h5_file(self,): ''' Converts file to h5 format. Saves output in current dir. ''' fil_file = Waterfall(self.filename) new_filename = self.out_dir+self.filename.replace('.fil','.h5').split('/')[-1] fil_file.write_to_hdf5(new_filename) self.filename = new_filename
def __split_h5(self, size_limit=SIZE_LIM): """ Creates a plan to select data from single coarse channels. :param size_limit: float, maximum size in MB that the file is allowed to be :return: list[DATAH5], the list contains a DATAH5 object for each of the coarse channels in the file """ data_list = [] #Instancing file. try: fil_file = Waterfall(self.filename) except: logger.error("Error encountered when trying to open file: %s" % self.filename) raise IOError("Error encountered when trying to open file: %s" % self.filename) #Finding lowest freq in file. f_delt = fil_file.header['foff'] f0 = fil_file.header['fch1'] #Looping over the number of coarse channels. if self.n_coarse_chan is not None: n_coarse_chan = self.n_coarse_chan elif fil_file.header.get('n_coarse_chan', None) is not None: n_coarse_chan = fil_file.header['n_coarse_chan'] else: n_coarse_chan = int(fil_file.calc_n_coarse_chan()) # Only load coarse chans of interest -- or do all if not specified if self.coarse_chans in (None, ''): self.coarse_chans = range(n_coarse_chan) for chan in self.coarse_chans: #Calculate freq range for given course channel. f_start = f0 + chan * ( f_delt) * fil_file.n_channels_in_file / n_coarse_chan f_stop = f0 + (chan + 1) * ( f_delt) * fil_file.n_channels_in_file / n_coarse_chan if f_start > f_stop: f_start, f_stop = f_stop, f_start data_obj = DATAH5(self.filename, f_start=f_start, f_stop=f_stop, coarse_chan=chan, tn_coarse_chan=n_coarse_chan) #---------------------------------------------------------------- #This appends to a list of all data instance selections. So that all get processed later. data_list.append(data_obj) return data_list
def __split_h5(self): """ Creates a plan to select data from single coarse channels. :return: list[DATAH5], where each list member contains a DATAH5 object for each of the coarse channels in the file. """ data_list = [] #Instancing file. try: fil_file = Waterfall(self.filename) except: errmsg = "Error encountered when trying to open file: {}".format( self.filename) logger.error(errmsg) raise IOError(errmsg) #Finding lowest freq in file. f_delt = fil_file.header['foff'] f0 = fil_file.header['fch1'] #Looping over the number of coarse channels. if self.n_coarse_chan is not None: n_coarse_chan = self.n_coarse_chan elif fil_file.header.get('n_coarse_chan', None) is not None: n_coarse_chan = fil_file.header['n_coarse_chan'] else: n_coarse_chan = int(fil_file.calc_n_coarse_chan()) # Only load coarse chans of interest -- or do all if not specified if self.coarse_chans in (None, ''): self.coarse_chans = range(n_coarse_chan) for chan in self.coarse_chans: #Calculate freq range for given course channel. f_start = f0 + chan * ( f_delt) * fil_file.n_channels_in_file / n_coarse_chan f_stop = f0 + (chan + 1) * ( f_delt) * fil_file.n_channels_in_file / n_coarse_chan if f_start > f_stop: f_start, f_stop = f_stop, f_start data_obj = { 'filename': self.filename, 'f_start': f_start, 'f_stop': f_stop, 'coarse_chan': chan, 'n_coarse_chan': n_coarse_chan } #This appends to a list of all data instance selections. So that all get processed later. data_list.append(data_obj) return data_list
def calibrate_fluxes(main_obs_name,dio_name,dspec,Tsys,fullstokes=False,**kwargs): ''' Produce calibrated Stokes I for an observation given a noise diode measurement on the source and a diode spectrum with the same number of coarse channels Parameters ---------- main_obs_name : str Path to filterbank file containing final data to be calibrated dio_name : str Path to filterbank file for observation on the target source with flickering noise diode dspec : 1D Array (float) or float Coarse channel spectrum (or average) of the noise diode in Jy (obtained from diode_spec()) Tsys : 1D Array (float) or float Coarse channel spectrum (or average) of the system temperature in Jy fullstokes: boolean Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if it is in cross_pols format ''' #Find folded spectra of the target source with the noise diode ON and OFF main_obs = Waterfall(main_obs_name,max_load=150) ncoarse = main_obs.calc_n_coarse_chan() dio_obs = Waterfall(dio_name,max_load=150) dio_chan_per_coarse = dio_obs.header['nchans']/ncoarse dOFF,dON = integrate_calib(dio_name,dio_chan_per_coarse,fullstokes,**kwargs) #Find Jy/count for each coarse channel using the diode spectrum main_dat = main_obs.data scale_facs = dspec/(dON-dOFF) print(scale_facs) nchans = main_obs.header['nchans'] obs_chan_per_coarse = nchans/ncoarse ax0_size = np.size(main_dat,0) ax1_size = np.size(main_dat,1) #Reshape data array of target observation and multiply coarse channels by the scale factors main_dat = np.reshape(main_dat,(ax0_size,ax1_size,ncoarse,obs_chan_per_coarse)) main_dat = np.swapaxes(main_dat,2,3) main_dat = main_dat*scale_facs main_dat = main_dat-Tsys main_dat = np.swapaxes(main_dat,2,3) main_dat = np.reshape(main_dat,(ax0_size,ax1_size,nchans)) #Write calibrated data to a new filterbank file with ".fluxcal" extension main_obs.data = main_dat main_obs.write_to_filterbank(main_obs_name[:-4]+'.fluxcal.fil') print('Finished: calibrated product written to ' + main_obs_name[:-4]+'.fluxcal.fil')
def __make_h5_file(self): r""" Converts file to h5 format, saved in current directory. Sets the filename attribute of the calling DATAHandle. to the (new) filename. """ fil_file = Waterfall(self.filename) bn = os.path.basename(self.filename) new_filename = os.path.join(self.out_dir, bn.replace('.fil', '.h5')) fil_file.write_to_hdf5(new_filename) self.filename = new_filename
def test_ncc_chan_bw(): wf = Waterfall(voyager_h5) print("test_bcc_chan_bw: telescope_id:", wf.header['telescope_id']) print("test_ncc_chan_bw: nchans:", wf.header['nchans']) n_coarse_chan = wf.calc_n_coarse_chan(16) print("test_ncc_chan_bw: n_coarse_chan [chan_bw=16]:", n_coarse_chan) assert n_coarse_chan == 64 n_coarse_chan = wf.calc_n_coarse_chan(1) print("test_ncc_chan_bw: n_coarse_chan [chan_bw=1]:", n_coarse_chan) assert n_coarse_chan > 2.9 and n_coarse_chan < 3.0
def __make_h5_file(self, ): """ Converts file to h5 format. Saves output in current dir. Sets the filename attribute of the calling DATAHandle to the (new) filename. :return: void """ fil_file = Waterfall(self.filename) bn = os.path.basename(self.filename) new_filename = os.path.join(self.out_dir, bn.replace('.fil', '.h5')) fil_file.write_to_hdf5(new_filename) self.filename = new_filename
def get_NT(FILE): ''' Returns number of time integrations in fil FILE; loads one frequency channel (across all time) of file to do this ''' fbank = Waterfall(FILE, load_data=False) fch1 = fbank.header['fch1'] f_off = fbank.header['foff'] fbank.read_data( f_start=fch1 + f_off, f_stop=fch1) # Read one channel of file to determine n_ints n_ints = fbank.data.shape[0] return n_ints
def diode_spec(calON_obs, calOFF_obs, calflux, calfreq, spec_in, average=True, oneflux=False, **kwargs): ''' Calculate the coarse channel spectrum and system temperature of the noise diode in Jy given two noise diode measurements ON and OFF the calibrator source with the same frequency and time resolution Parameters ---------- calON_obs : str (see f_ratios() above) calOFF_obs : str (see f_ratios() above) calflux : float Known flux of calibrator source at a particular frequency calfreq : float Frequency where calibrator source has flux calflux (see above) spec_in : float Known power-law spectral index of calibrator source. Use convention flux(frequency) = constant * frequency^(spec_in) average : boolean Use average=True to return noise diode and Tsys spectra averaged over frequencies ''' #Load frequencies and calculate number of channels per coarse channel obs = Waterfall(calON_obs, max_load=150) freqs = obs.populate_freqs() ncoarse = obs.calc_n_coarse_chan() nchans = obs.header['nchans'] chan_per_coarse = nchans / ncoarse f_ON, f_OFF = f_ratios(calON_obs, calOFF_obs, chan_per_coarse, **kwargs) #Obtain spectrum of the calibrator source for the given frequency range centerfreqs = get_centerfreqs(freqs, chan_per_coarse) calfluxes = get_calfluxes(calflux, calfreq, spec_in, centerfreqs, oneflux) #C_o and Tsys as defined in van Straten et al. 2012 C_o = calfluxes / (1 / f_ON - 1 / f_OFF) Tsys = C_o / f_OFF #return coarse channel diode spectrum if average == True: return np.mean(C_o), np.mean(Tsys) else: return C_o, Tsys
def _update_waterfall(self): # Set fil with sample data; (1.4 Hz, 1.4 s) res if self.waterfall is None: my_path = os.path.abspath(os.path.dirname(__file__)) path = os.path.join(my_path, 'assets/sample.fil') self.waterfall = Waterfall(path) self.waterfall.header[b'source_name'] = b'Synthetic' self.waterfall.header[b'foff'] = self.df * -1e-6 self.waterfall.header[b'tsamp'] = self.dt self.waterfall.header[b'nchans'] = self.fchans self.waterfall.header[b'fch1'] = self.fmax # Have to manually flip in the frequency direction + add an extra # dimension for polarization to work with Waterfall self.waterfall.data = self.data[:, np.newaxis, ::-1]
def gen_pulse_corpus(FILE, N, OUTDIR, NT=256, band='L', fullband=True): ''' Calls get_pulse in parallel to generate a corpus of N batches of pulses in OUTDIR ''' if band == 'C': fullband = False fbank = Waterfall(FILE, load_data=False) fheader = fbank.header print("HEADER INFO:") print(fheader) if fullband: fband_range = [ fheader['fch1'] + fheader['foff'] * fheader['nchans'], fheader['fch1'] ] nu = np.arange(fband_range[0], fband_range[-1], np.abs( fheader['foff'])) * 1.e-3 else: fband_range = FREQ_BAND_TO_INFO[band]['freq_range'] fc1, fc2, nus = get_chans(fheader['fch1'], fheader['foff'], fheader['nchans'], f1=fband_range[1], f2=fband_range[0]) nu = nus[::-1] * 1.e-3 # to GHz t = np.arange(0, NT, 1) * fbank.header['tsamp'] * 1.e3 if not os.path.exists(OUTDIR): os.makedirs(OUTDIR) _ = Parallel(n_jobs=8)(delayed(get_pulse)(t, nu, i, OUTDIR, 1, NT, band) for i in range(N))
def get_info(self): """ :return: dict, header of the blimpy file """ fil_file = Waterfall(self.filename, load_data=False) return fil_file.header
def write_fil(data, times, freqs_GHz, hdr, hotpotato): # Reshape data array. data = data.T.reshape( (data.T.shape[0], 1, data.T.shape[1]) ) # New shape = (No. of time samples, No. of polarizations, No. of channels) # Construct a Waterfall object that will be written to disk as a filterbank file. base_fileout = hotpotato['basename'] + '_t%.2fto%.2f_freqs%.2fto%.2f' % ( times[0], times[-1], freqs_GHz[0], freqs_GHz[-1]) + '.fil' filename_out = hotpotato['OUTPUT_DIR'] + '/' + base_fileout wat = Waterfall() # Empty Waterfall object wat.header = hdr.primary with open(filename_out, 'wb') as fh: print('Writing smoothed data to %s' % (base_fileout)) fh.write(generate_sigproc_header( wat)) # Trick Blimpy into writing a sigproc header. np.float32(data.ravel()).tofile(fh)
def get_data(waterfall, use_db=False): """ Get time-frequency data from filterbank file as a 2d NumPy array. Note: when multiple Stokes parameters are supported, this will have to be expanded. Parameters ---------- waterfall : str or Waterfall Name of filterbank file or Waterfall object Returns ------- data : ndarray Time-frequency data """ if isinstance(waterfall, str): waterfall = Waterfall(waterfall) elif not isinstance(waterfall, Waterfall): sys.exit('Invalid data file!') if use_db: return 10 * np.log10(waterfall.data[:, 0, :]) return waterfall.data[:, 0, :]
def get_drift_rate(file): # open the file obs = Waterfall(file) # get the time of obs + make array 15 min into future jdate = obs.header['tstart'] + 2400000.5 JDUTC = np.linspace(jdate, jdate + (60.0 * 15.0/86400.), num=100) # get the pointing of the obs c = SkyCoord(obs.header['src_raj'], obs.header['src_dej']) s = c.to_string('decimal') ra_probe, dec_probe = [float(string) for string in s.split()] # other needed params obsname = 'GBT' epoch = 2451545.0 rv = 0.0 zmeas = 0.0 ephemeris='https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/a_old_versions/de405.bsp' # get the BC vel baryvel = get_BC_vel(JDUTC=JDUTC, ra=ra_probe, dec=dec_probe, obsname='GBT', rv=rv, zmeas=zmeas, epoch=epoch, ephemeris=ephemeris, leap_update=True) # take the derivative of velocity to get acceleration (i.e., drift) diffT = np.diff(JDUTC) * 86400.0 diffV = np.diff(baryvel[0]) drift = diffV/diffT # convert m/s^2 to Hz/s and take the max drift calculated drift *= (obs.container.f_stop * 1e6 / 3e8) # TODO check SIGN!!! return np.max(np.abs(drift))
def plot_diodespec(ON_obs,OFF_obs,calflux,calfreq,spec_in,units='mJy',**kwargs): """ Plots the full-band Stokes I spectrum of the noise diode (ON-OFF) """ dspec = diode_spec(ON_obs,OFF_obs,calflux,calfreq,spec_in,**kwargs) obs = Waterfall(ON_obs,max_load=150) freqs = obs.populate_freqs() chan_per_coarse = obs.header['nchans']/obs.calc_n_coarse_chan() coarse_freqs = convert_to_coarse(freqs,chan_per_coarse) plt.ion() plt.figure() plt.plot(coarse_freqs,dspec) plt.xlabel('Frequency (MHz)') plt.ylabel('Flux Density ('+units+')') plt.title('Noise Diode Spectrum')
def get_Tsys_nodiode(calON_obs_name, calOFF_obs_name, calflux, calfreq, spec_in, plot=False, **kwargs): ''' Calculates system temperature from two flux calibrator scans taken without noise diode flickering. CURRENTLY ONLY IMPLEMENTED FOR STOKES I DATA. Parameters ---------- calON_obs_name : str Path to filterbank file for scan ON calibrator target calOFF_obs_name : str Path to filterbank file for scan OFF calibrator calflux : float Flux in Jy of the calibrator source calfreq : float or int Frequency at which calflux was taken (MHz) spec_in : float Spectral index of this calibrator ''' calON_obs = Waterfall(calON_obs_name, max_load=150) calOFF_obs = Waterfall(calOFF_obs_name, max_load=150) ncoarse = calON_obs.calc_n_coarse_chan() chan_per_coarse = calON_obs.header['nchans'] / ncoarse ONobs_spec = np.squeeze(np.mean(calON_obs.data, axis=0)) OFFobs_spec = np.squeeze(np.mean(calOFF_obs.data, axis=0)) freqs = calON_obs.populate_freqs() ON_coarse_spec = integrate_chans(ONobs_spec, freqs, chan_per_coarse) OFF_coarse_spec = integrate_chans(OFFobs_spec, freqs, chan_per_coarse) cfreqs = get_centerfreqs(freqs, chan_per_coarse) cal_fluxes = get_calfluxes(calflux, calfreq, spec_in, cfreqs, oneflux=False) S_sys = (OFF_coarse_spec) / (ON_coarse_spec - OFF_coarse_spec) * cal_fluxes #Convert to Kelvins T_sys, cfreqs = Jy_to_Kelvin(S_sys, cfreqs) return T_sys, cfreqs
def integrate_calib(name,chan_per_coarse,fullstokes=False,**kwargs): ''' Folds Stokes I noise diode data and integrates along coarse channels Parameters ---------- name : str Path to noise diode filterbank file chan_per_coarse : int Number of frequency bins per coarse channel fullstokes : boolean Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if it is in cross_pols format ''' #Load data obs = Waterfall(name,max_load=150) data = obs.data #If the data has cross_pols format calculate Stokes I if fullstokes==False and data.shape[1]>1: data = data[:,0,:]+data[:,1,:] data = np.expand_dims(data,axis=1) #If the data has IQUV format get Stokes I if fullstokes==True: data = data[:,0,:] data = np.expand_dims(data,axis=1) tsamp = obs.header['tsamp'] #Calculate ON and OFF values OFF,ON = foldcal(data,tsamp,**kwargs) freqs = obs.populate_freqs() #Find ON and OFF spectra by coarse channel ON_int = integrate_chans(ON,freqs,chan_per_coarse) OFF_int = integrate_chans(OFF,freqs,chan_per_coarse) #If "ON" is actually "OFF" switch them if np.sum(ON_int)<np.sum(OFF_int): temp = ON_int ON_int = OFF_int OFF_int = temp #Return coarse channel spectrum of OFF and ON return OFF_int,ON_int
def read_filterbank(self, tab, startbin, chunksize): """ Read a chunk of filterbank data :param int tab: TAB index :param int startbin: Index of first time sample to read :param int chunksize: Number of time samples to read :return: chunk of data with shape (nfreq, chunksize) """ fil = Waterfall(self.fnames[tab], load_data=False) # read chunk of data fil.read_data(None, None, startbin, startbin + chunksize) # keep only time and freq axes, transpose to have frequency first data = fil.data[:, 0, :].T.astype(float) if self.median_filter: data -= np.median(data, axis=1, keepdims=True) return data
def integrate_calib(name, chan_per_coarse, fullstokes=False, **kwargs): ''' Folds Stokes I noise diode data and integrates along coarse channels Parameters ---------- name : str Path to noise diode filterbank file chan_per_coarse : int Number of frequency bins per coarse channel fullstokes : boolean Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if it is in cross_pols format ''' #Load data obs = Waterfall(name, max_load=150) data = obs.data #If the data has cross_pols format calculate Stokes I if fullstokes == False and data.shape[1] > 1: data = data[:, 0, :] + data[:, 1, :] data = np.expand_dims(data, axis=1) #If the data has IQUV format get Stokes I if fullstokes == True: data = data[:, 0, :] data = np.expand_dims(data, axis=1) tsamp = obs.header['tsamp'] #Calculate ON and OFF values OFF, ON = foldcal(data, tsamp, **kwargs) freqs = obs.populate_freqs() #Find ON and OFF spectra by coarse channel ON_int = integrate_chans(ON, freqs, chan_per_coarse) OFF_int = integrate_chans(OFF, freqs, chan_per_coarse) #If "ON" is actually "OFF" switch them if np.sum(ON_int) < np.sum(OFF_int): temp = ON_int ON_int = OFF_int OFF_int = temp #Return coarse channel spectrum of OFF and ON return OFF_int, ON_int
def plot_gain_offsets(dio_cross, dio_chan_per_coarse=8, feedtype='l', ax1=None, ax2=None, legend=True, **kwargs): ''' Plots the calculated gain offsets of each coarse channel along with the time averaged power spectra of the X and Y feeds ''' #Get ON-OFF ND spectra Idiff, Qdiff, Udiff, Vdiff, freqs = get_diff(dio_cross, feedtype, **kwargs) obs = Waterfall(dio_cross, max_load=150) tsamp = obs.header['tsamp'] data = obs.data obs = None I, Q, U, V = get_stokes(data, feedtype) #Get phase offsets and convert to degrees coarse_G = gain_offsets(I, Q, U, V, tsamp, dio_chan_per_coarse, feedtype, **kwargs) coarse_freqs = convert_to_coarse(freqs, dio_chan_per_coarse) #Get X and Y spectra for the noise diode ON and OFF #If using circular feeds these correspond to LL and RR XX_OFF, XX_ON = foldcal(np.expand_dims(data[:, 0, :], axis=1), tsamp, **kwargs) YY_OFF, YY_ON = foldcal(np.expand_dims(data[:, 1, :], axis=1), tsamp, **kwargs) if ax1 == None: plt.subplot(211) else: axG = plt.axes(ax1) plt.setp(axG.get_xticklabels(), visible=False) plt.plot(coarse_freqs, coarse_G, 'ko', markersize=2) plt.ylabel(r'$\frac{\Delta G}{2}$', rotation=90) if feedtype == 'l': plt.title('XY Gain Difference') if feedtype == 'c': plt.title('LR Gain Difference') plt.grid(True) if ax2 == None: plt.subplot(212) else: axXY = plt.axes(ax2, sharex=axG) if feedtype == 'l': plt.plot(freqs, XX_OFF, 'b-', label='XX') plt.plot(freqs, YY_OFF, 'r-', label='YY') if feedtype == 'c': plt.plot(freqs, XX_OFF, 'b-', label='LL') plt.plot(freqs, YY_OFF, 'r-', label='RR') plt.xlabel('Frequency (MHz)') plt.ylabel('Power (Counts)') if legend == True: plt.legend()
def plot_calibrated_diode(dio_cross, chan_per_coarse=8, feedtype='l', **kwargs): ''' Plots the corrected noise diode spectrum for a given noise diode measurement after application of the inverse Mueller matrix for the electronics chain. ''' #Get full stokes data for the ND observation obs = Waterfall(dio_cross, max_load=150) freqs = obs.populate_freqs() tsamp = obs.header['tsamp'] data = obs.data obs = None I, Q, U, V = get_stokes(data, feedtype) data = None #Calculate Mueller Matrix variables for each coarse channel psis = phase_offsets(I, Q, U, V, tsamp, chan_per_coarse, feedtype, **kwargs) G = gain_offsets(I, Q, U, V, tsamp, chan_per_coarse, feedtype, **kwargs) #Apply the Mueller matrix to original noise diode data and refold I, Q, U, V = apply_Mueller(I, Q, U, V, G, psis, chan_per_coarse, feedtype) I_OFF, I_ON = foldcal(I, tsamp, **kwargs) Q_OFF, Q_ON = foldcal(Q, tsamp, **kwargs) U_OFF, U_ON = foldcal(U, tsamp, **kwargs) V_OFF, V_ON = foldcal(V, tsamp, **kwargs) #Delete data arrays for space I = None Q = None U = None V = None #Plot new ON-OFF spectra plt.plot(freqs, I_ON - I_OFF, 'k-', label='I') plt.plot(freqs, Q_ON - Q_OFF, 'r-', label='Q') plt.plot(freqs, U_ON - U_OFF, 'g-', label='U') plt.plot(freqs, V_ON - V_OFF, 'm-', label='V') plt.legend() plt.xlabel('Frequency (MHz)') plt.title('Calibrated Full Stokes Noise Diode Spectrum') plt.ylabel('Power (Counts)')
def split_file_generator(filename, frequency_dimension=512, count = None, NUM_PARTS = 100): wf = Waterfall(filename, load_data = False) f_begin, f_end, f_delta = get_f_iter_bounds(wf.header, frequency_dimension) leftover = None #read NUM_PARTS from wf at a time, to be split after for iteration, (f_start, f_stop) in enumerate(pairwise(np.arange(f_begin, f_end, f_delta * NUM_PARTS))): wf.read_data(f_start=f_start, f_stop = f_stop) parts_together = wf.data.squeeze() #squeeze brings shape from (16, 1, 512 * NUM_PARTS) -> (16, 512 * NUM_PARTS) if not leftover is None: parts_together = np.append(leftover, parts_together, axis = 1) parts_lst, leftover = split_parts(parts_together, frequency_dimension) yield parts_lst if count and iteration * NUM_PARTS >= count: break if not count and not leftover is None: yield leftover
def plot_phase_offsets(dio_cross, chan_per_coarse=8, feedtype='l', ax1=None, ax2=None, legend=True, **kwargs): ''' Plots the calculated phase offsets of each coarse channel along with the UV (or QU) noise diode spectrum for comparison ''' #Get ON-OFF ND spectra Idiff, Qdiff, Udiff, Vdiff, freqs = get_diff(dio_cross, feedtype, **kwargs) obs = Waterfall(dio_cross, max_load=150) tsamp = obs.header['tsamp'] data = obs.data obs = None I, Q, U, V = get_stokes(data, feedtype) #Get phase offsets and convert to degrees coarse_psis = phase_offsets(I, Q, U, V, tsamp, chan_per_coarse, feedtype, **kwargs) coarse_freqs = convert_to_coarse(freqs, chan_per_coarse) coarse_degs = np.degrees(coarse_psis) #Plot phase offsets if ax2 == None: plt.subplot(211) else: axPsi = plt.axes(ax2) plt.setp(axPsi.get_xticklabels(), visible=False) plt.plot(coarse_freqs, coarse_degs, 'ko', markersize=2, label='Coarse Channel $\psi$') plt.ylabel('Degrees') plt.grid(True) plt.title('Phase Offsets') if legend == True: plt.legend() #Plot U and V spectra if ax1 == None: plt.subplot(212) else: axUV = plt.axes(ax1) plt.plot(freqs, Udiff, 'g-', label='U') if feedtype == 'l': plt.plot(freqs, Vdiff, 'm-', label='V') if feedtype == 'c': plt.plot(freqs, Qdiff, 'r-', label='Q') plt.xlabel('Frequency (MHz)') plt.ylabel('Power (Counts)') plt.grid(True) if legend == True: plt.legend()
def calibrate_fluxes(main_obs_name, dio_name, dspec, Tsys, fullstokes=False, **kwargs): ''' Produce calibrated Stokes I for an observation given a noise diode measurement on the source and a diode spectrum with the same number of coarse channels Parameters ---------- main_obs_name : str Path to filterbank file containing final data to be calibrated dio_name : str Path to filterbank file for observation on the target source with flickering noise diode dspec : 1D Array (float) or float Coarse channel spectrum (or average) of the noise diode in Jy (obtained from diode_spec()) Tsys : 1D Array (float) or float Coarse channel spectrum (or average) of the system temperature in Jy fullstokes: boolean Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if it is in cross_pols format ''' #Find folded spectra of the target source with the noise diode ON and OFF main_obs = Waterfall(main_obs_name, max_load=150) ncoarse = main_obs.calc_n_coarse_chan() dio_obs = Waterfall(dio_name, max_load=150) dio_chan_per_coarse = dio_obs.header['nchans'] / ncoarse dOFF, dON = integrate_calib(dio_name, dio_chan_per_coarse, fullstokes, **kwargs) #Find Jy/count for each coarse channel using the diode spectrum main_dat = main_obs.data scale_facs = dspec / (dON - dOFF) print(scale_facs) nchans = main_obs.header['nchans'] obs_chan_per_coarse = nchans / ncoarse ax0_size = np.size(main_dat, 0) ax1_size = np.size(main_dat, 1) #Reshape data array of target observation and multiply coarse channels by the scale factors main_dat = np.reshape(main_dat, (ax0_size, ax1_size, ncoarse, obs_chan_per_coarse)) main_dat = np.swapaxes(main_dat, 2, 3) main_dat = main_dat * scale_facs main_dat = main_dat - Tsys main_dat = np.swapaxes(main_dat, 2, 3) main_dat = np.reshape(main_dat, (ax0_size, ax1_size, nchans)) #Write calibrated data to a new filterbank file with ".fluxcal" extension main_obs.data = main_dat main_obs.write_to_filterbank(main_obs_name[:-4] + '.fluxcal.fil') print 'Finished: calibrated product written to ' + main_obs_name[:-4] + '.fluxcal.fil'
def plot_diode_fold(dio_cross,bothfeeds=True,feedtype='l',min_samp=-500,max_samp=7000,legend=True,**kwargs): """ Plots the calculated average power and time sampling of ON (red) and OFF (blue) for a noise diode measurement over the observation time series """ #Get full stokes data of ND measurement obs = Waterfall(dio_cross,max_load=150) tsamp = obs.header['tsamp'] data = obs.data obs = None I,Q,U,V = get_stokes(data,feedtype) #Calculate time series, OFF and ON averages, and time samples for each tseriesI = np.squeeze(np.mean(I,axis=2)) I_OFF,I_ON,OFFints,ONints = foldcal(I,tsamp,inds=True,**kwargs) if bothfeeds==True: if feedtype=='l': tseriesQ = np.squeeze(np.mean(Q,axis=2)) tseriesX = (tseriesI+tseriesQ)/2 tseriesY = (tseriesI-tseriesQ)/2 if feedtype=='c': tseriesV = np.squeeze(np.mean(V,axis=2)) tseriesR = (tseriesI+tseriesV)/2 tseriesL = (tseriesI-tseriesV)/2 stop = ONints[-1,1] #Plot time series and calculated average for ON and OFF if bothfeeds==False: plt.plot(tseriesI[0:stop],'k-',label='Total Power') for i in ONints: plt.plot(np.arange(i[0],i[1]),np.full((i[1]-i[0]),np.mean(I_ON)),'r-') for i in OFFints: plt.plot(np.arange(i[0],i[1]),np.full((i[1]-i[0]),np.mean(I_OFF)),'b-') else: if feedtype=='l': diff = np.mean(tseriesX)-np.mean(tseriesY) plt.plot(tseriesX[0:stop],'b-',label='XX') plt.plot(tseriesY[0:stop]+diff,'r-',label='YY (shifted)') if feedtype=='c': diff = np.mean(tseriesL)-np.mean(tseriesR) plt.plot(tseriesL[0:stop],'b-',label='LL') plt.plot(tseriesR[0:stop]+diff,'r-',label='RR (shifted)') #Calculate plotting limits if bothfeeds==False: lowlim = np.mean(I_OFF)-(np.mean(I_ON)-np.mean(I_OFF))/2 hilim = np.mean(I_ON)+(np.mean(I_ON)-np.mean(I_OFF))/2 plt.ylim((lowlim,hilim)) plt.xlim((min_samp,max_samp)) plt.xlabel('Time Sample Number') plt.ylabel('Power (Counts)') plt.title('Noise Diode Fold') if legend==True: plt.legend()
def diode_spec(calON_obs,calOFF_obs,calflux,calfreq,spec_in,average=True,oneflux=False,**kwargs): ''' Calculate the coarse channel spectrum and system temperature of the noise diode in Jy given two noise diode measurements ON and OFF the calibrator source with the same frequency and time resolution Parameters ---------- calON_obs : str (see f_ratios() above) calOFF_obs : str (see f_ratios() above) calflux : float Known flux of calibrator source at a particular frequency calfreq : float Frequency where calibrator source has flux calflux (see above) spec_in : float Known power-law spectral index of calibrator source. Use convention flux(frequency) = constant * frequency^(spec_in) average : boolean Use average=True to return noise diode and Tsys spectra averaged over frequencies ''' #Load frequencies and calculate number of channels per coarse channel obs = Waterfall(calON_obs,max_load=150) freqs = obs.populate_freqs() ncoarse = obs.calc_n_coarse_chan() nchans = obs.header['nchans'] chan_per_coarse = nchans/ncoarse f_ON, f_OFF = f_ratios(calON_obs,calOFF_obs,chan_per_coarse,**kwargs) #Obtain spectrum of the calibrator source for the given frequency range centerfreqs = get_centerfreqs(freqs,chan_per_coarse) calfluxes = get_calfluxes(calflux,calfreq,spec_in,centerfreqs,oneflux) #C_o and Tsys as defined in van Straten et al. 2012 C_o = calfluxes/(1/f_ON-1/f_OFF) Tsys = C_o/f_OFF #return coarse channel diode spectrum if average==True: return np.mean(C_o),np.mean(Tsys) else: return C_o,Tsys
def write_polfils(str, str_I, **kwargs): '''Writes two new filterbank files containing fractional linear and circular polarization data''' lin,circ=fracpols(str, **kwargs) obs = Waterfall(str_I, max_load=150) obs.data = lin obs.write_to_fil(str[:-15]+'.linpol.fil') #assuming file is named *.cross_pols.fil obs.data = circ obs.write_to_fil(str[:-15]+'.circpol.fil') #assuming file is named *.cross_pols.fil
def calibrate_pols(cross_pols,diode_cross,obsI=None,onefile=True,feedtype='l',**kwargs): ''' Write Stokes-calibrated filterbank file for a given observation with a calibrator noise diode measurement on the source Parameters ---------- cross_pols : string Path to cross polarization filterbank file (rawspec output) for observation to be calibrated diode_cross : string Path to cross polarization filterbank file of noise diode measurement ON the target obsI : string Path to Stokes I filterbank file of main observation (only needed if onefile=False) onefile : boolean True writes all calibrated Stokes parameters to a single filterbank file, False writes four separate files feedtype : 'l' or 'c' Basis of antenna dipoles. 'c' for circular, 'l' for linear ''' #Obtain time sample length, frequencies, and noise diode data obs = Waterfall(diode_cross,max_load=150) cross_dat = obs.data tsamp = obs.header['tsamp'] #Calculate number of coarse channels in the noise diode measurement (usually 8) dio_ncoarse = obs.calc_n_coarse_chan() dio_nchans = obs.header['nchans'] dio_chan_per_coarse = dio_nchans/dio_ncoarse obs = None Idat,Qdat,Udat,Vdat = get_stokes(cross_dat,feedtype) cross_dat = None #Calculate differential gain and phase from noise diode measurements print('Calculating Mueller Matrix variables') gams = gain_offsets(Idat,Qdat,Udat,Vdat,tsamp,dio_chan_per_coarse,feedtype,**kwargs) psis = phase_offsets(Idat,Qdat,Udat,Vdat,tsamp,dio_chan_per_coarse,feedtype,**kwargs) #Clear data arrays to save memory Idat = None Qdat = None Udat = None Vdat = None #Get corrected Stokes parameters print('Opening '+cross_pols) cross_obs = Waterfall(cross_pols,max_load=150) obs_ncoarse = cross_obs.calc_n_coarse_chan() obs_nchans = cross_obs.header['nchans'] obs_chan_per_coarse = obs_nchans/obs_ncoarse print('Grabbing Stokes parameters') I,Q,U,V = get_stokes(cross_obs.data,feedtype) print('Applying Mueller Matrix') I,Q,U,V = apply_Mueller(I,Q,U,V,gams,psis,obs_chan_per_coarse,feedtype) #Use onefile (default) to produce one filterbank file containing all Stokes information if onefile==True: cross_obs.data[:,0,:] = np.squeeze(I) cross_obs.data[:,1,:] = np.squeeze(Q) cross_obs.data[:,2,:] = np.squeeze(U) cross_obs.data[:,3,:] = np.squeeze(V) cross_obs.write_to_fil(cross_pols[:-15]+'.SIQUV.polcal.fil') print('Calibrated Stokes parameters written to '+cross_pols[:-15]+'.SIQUV.polcal.fil') return #Write corrected Stokes parameters to four filterbank files if onefile==False obs = Waterfall(obs_I,max_load=150) obs.data = I obs.write_to_fil(cross_pols[:-15]+'.SI.polcal.fil') #assuming file is named *.cross_pols.fil print('Calibrated Stokes I written to '+cross_pols[:-15]+'.SI.polcal.fil') obs.data = Q obs.write_to_fil(cross_pols[:-15]+'.Q.polcal.fil') #assuming file is named *.cross_pols.fil print('Calibrated Stokes Q written to '+cross_pols[:-15]+'.Q.polcal.fil') obs.data = U obs.write_to_fil(cross_pols[:-15]+'.U.polcal.fil') #assuming file is named *.cross_pols.fil print('Calibrated Stokes U written to '+cross_pols[:-15]+'.U.polcal.fil') obs.data = V obs.write_to_fil(cross_pols[:-15]+'.V.polcal.fil') #assuming file is named *.cross_pols.fil print('Calibrated Stokes V written to '+cross_pols[:-15]+'.V.polcal.fil')
def write_stokefils(str, str_I, Ifil=False, Qfil=False, Ufil=False, Vfil=False, Lfil=False, **kwargs): '''Writes up to 5 new filterbank files corresponding to each Stokes parameter (and total linear polarization L) for a given cross polarization .fil file''' I,Q,U,V,L=get_stokes(str, **kwargs) obs = Waterfall(str_I, max_load=150) #Load filterbank file to write stokes data to if Ifil: obs.data = I obs.write_to_fil(str[:-15]+'.I.fil') #assuming file is named *.cross_pols.fil if Qfil: obs.data = Q obs.write_to_fil(str[:-15]+'.Q.fil') #assuming file is named *.cross_pols.fil if Ufil: obs.data = U obs.write_to_fil(str[:-15]+'.U.fil') #assuming file is named *.cross_pols.fil if Vfil: obs.data = V obs.write_to_fil(str[:-15]+'.V.fil') #assuming file is named *.cross_pols.fil if Lfil: obs.data = L obs.write_to_fil(str[:-15]+'.L.fil') #assuming file is named *.cross_pols.fil