def get_diff(dio_cross, feedtype, **kwargs):
'''
Returns ON-OFF for all Stokes parameters given a cross_pols noise diode measurement
'''
#Get Stokes parameters, frequencies, and time sample length
obs = Waterfall(dio_cross, max_load=150)
freqs = obs.populate_freqs()
tsamp = obs.header['tsamp']
data = obs.data
obs = None
I, Q, U, V = get_stokes(data, feedtype)
#Fold noise diode data
I_OFF, I_ON = foldcal(I, tsamp, **kwargs)
Q_OFF, Q_ON = foldcal(Q, tsamp, **kwargs)
U_OFF, U_ON = foldcal(U, tsamp, **kwargs)
V_OFF, V_ON = foldcal(V, tsamp, **kwargs)
#Do ON-OFF subtraction
Idiff = I_ON - I_OFF
Qdiff = Q_ON - Q_OFF
Udiff = U_ON - U_OFF
Vdiff = V_ON - V_OFF
return Idiff, Qdiff, Udiff, Vdiff, freqs
def __init__(self, filename, f_start=None, f_stop=None, t_start=None, t_stop=None,
coarse_chan=1, n_coarse_chan=None):
"""
:param filename: string name of file
:param f_start: float start frequency in MHz
:param f_stop: float stop frequency in MHz
:param t_start: int start integration ID
:param t_stop: int stop integration ID
:param coarse_chan: int
:param n_coarse_chan: int
"""
self.filename = filename
self.closed = False
self.f_start = f_start
self.f_stop = f_stop
self.t_start = t_start
self.t_stop = t_stop
self.n_coarse_chan = n_coarse_chan
#Instancing file.
try:
self.fil_file = Waterfall(filename, f_start=self.f_start, f_stop=self.f_stop,
t_start=self.t_start, t_stop=self.t_stop, load_data=False)
except:
errmsg = "Error encountered when trying to open file: {}".format(filename)
logger.error(errmsg)
raise IOError(errmsg)
#Getting header
try:
if self.n_coarse_chan:
header = self.__make_data_header(self.fil_file.header, coarse=True)
else:
header = self.__make_data_header(self.fil_file.header)
except:
errmsg = "Error accessing header from file: {}".format(self.fil_file.header)
logger.error(errmsg)
raise IOError(errmsg)
self.header = header
self.fftlen = header['NAXIS1']
#EE To check if swapping tsteps_valid and tsteps is more appropriate.
self.tsteps_valid = header['NAXIS2']
self.tsteps = int(math.pow(2, math.ceil(np.log2(math.floor(self.tsteps_valid)))))
self.obs_length = self.tsteps_valid * header['DELTAT']
self.drift_rate_resolution = (1e6 * np.abs(header['DELTAF'])) / self.obs_length # in Hz/sec
self.header['baryv'] = 0.0
self.header['barya'] = 0.0
self.header['coarse_chan'] = coarse_chan
#EE For now I'm not using a shoulder. This is ok as long as
## I'm analyzing each coarse channel individually.
#EE In general this parameter is an integer (even number).
#This gives two regions, each of n*steps, around spectra[i]
self.shoulder_size = 0
self.tdwidth = self.fftlen + self.shoulder_size * self.tsteps
def get_Tsys(calON_obs,
calOFF_obs,
calflux,
calfreq,
spec_in,
oneflux=False,
**kwargs):
'''
Returns frequency dependent system temperature given observations on and off a calibrator source
Parameters
----------
(See diode_spec())
'''
obs = Waterfall(calON_obs, max_load=150)
ncoarse = obs.calc_n_coarse_chan()
chan_per_coarse = obs.header['nchans'] / ncoarse
freqs = obs.populate_freqs()
cfreqs = get_centerfreqs(freqs, chan_per_coarse)
S_sys = diode_spec(calON_obs,
calOFF_obs,
calflux,
calfreq,
spec_in,
average=False,
oneflux=False,
**kwargs)[1]
T_sys = Jy_to_Kelvin(S_sys, cfreqs)
return T_sys, cfreqs
def plot_hit(fil_filename, dat_filename, hit_id, bw=None, offset=0):
r"""Plot a candidate from a .dat file
Args:
fil_filename(str): Path to filterbank file to plot
dat_filename(str): Path to turbosSETI generated .dat output file of events
hit_id(int): ID of hit in the dat file to plot (TopHitNum)
offset(float, optional): Offset drift line on plot. Default 0.
bw: (Default value = None)
Returns:
"""
# Load hit details
dat = find_event.make_table(dat_filename)
hit = dat.iloc[hit_id]
f0 = hit['Freq']
if bw is None:
bw_mhz = np.abs(hit['FreqStart'] - hit['FreqEnd'])
else:
bw_mhz = bw * 1e-6
fil = Waterfall(fil_filename,
f_start=f0 - bw_mhz / 2,
f_stop=f0 + bw_mhz / 2)
t_duration = (fil.n_ints_in_file - 1) * fil.header['tsamp']
fil.plot_waterfall()
plot_event.overlay_drift(f0, f0, f0, hit['DriftRate'], t_duration, offset)
def __split_h5(self, size_limit=SIZE_LIM):
'''Creates a plan to select data from single coarse channels.
'''
data_list = []
#Instancing file.
try:
fil_file = Waterfall(self.filename)
except:
logger.error("Error encountered when trying to open file: %s" %
self.filename)
raise IOError("Error encountered when trying to open file: %s" %
self.filename)
#Finding lowest freq in file.
try:
f_delt = fil_file.header[u'foff']
f0 = fil_file.header[u'fch1']
except:
f_delt = fil_file.header[b'foff']
f0 = fil_file.header[b'fch1']
#Looping over the number of coarse channels.
n_coarse_chan = int(fil_file.calc_n_coarse_chan())
if n_coarse_chan != fil_file.calc_n_coarse_chan():
logger.warning(
'The file/selection is not an integer number of coarse channels. This could have unexpected consequences. Let op!'
)
for chan in range(n_coarse_chan):
#Calculate freq range for given course channel.
f_start = f0 + chan * (
f_delt) * fil_file.n_channels_in_file / n_coarse_chan
f_stop = f0 + (chan + 1) * (
f_delt) * fil_file.n_channels_in_file / n_coarse_chan
if f_start > f_stop:
f_start, f_stop = f_stop, f_start
data_obj = DATAH5(self.filename,
f_start=f_start,
f_stop=f_stop,
coarse_chan=chan,
tn_coarse_chan=n_coarse_chan)
#----------------------------------------------------------------
# fn = filename[filename.rfind('/')+1:filename.rfind('.fil')]
# deltaf = header[u'DELTAF']
# fftlen = header[u'NAXIS1']
# fcntr = header[u'FCNTR']
# frange = [fcntr - fftlen*deltaf/2, fcntr + fftlen*deltaf/2]
#This appends to a list of all data instance selections. So that all get processed later.
data_list.append(data_obj)
return data_list
def __init__(self, filename, size_limit = SIZE_LIM,f_start=None, f_stop=None,t_start=None, t_stop=None,coarse_chan=1,tn_coarse_chan=None):
self.filename = filename
self.closed = False
self.f_start = f_start
self.f_stop = f_stop
self.t_start = t_start
self.t_stop = t_stop
self.tn_coarse_chan = tn_coarse_chan
#Instancing file.
try:
self.fil_file = Waterfall(filename,f_start=self.f_start, f_stop=self.f_stop,t_start=self.t_start, t_stop=self.t_stop,load_data=False)
except:
logger.error("Error encountered when trying to open file %s"%filename)
raise IOError("Error encountered when trying to open file %s"%filename)
#Getting header
try:
if self.tn_coarse_chan:
header = self.__make_data_header(self.fil_file.header,coarse=True)
else:
header = self.__make_data_header(self.fil_file.header)
except:
logger.debug('The fil_file.header is ' % self.fil_file.header)
raise IOError("Error accessing header from file: %s." % self.filename)
self.header = header
try:
self.fftlen = header[u'NAXIS1']
#EE To check if swapping tsteps_valid and tsteps is more appropriate.
self.tsteps_valid = header[u'NAXIS2']
self.tsteps = int(math.pow(2, math.ceil(np.log2(math.floor(self.tsteps_valid)))))
self.obs_length = self.tsteps_valid * header[u'DELTAT']
self.drift_rate_resolution = (1e6 * np.abs(header[u'DELTAF'])) / self.obs_length # in Hz/sec
self.header[u'baryv'] = 0.0
self.header[u'barya'] = 0.0
self.header[u'coarse_chan'] = coarse_chan
except:
self.fftlen = header[b'NAXIS1']
#EE To check if swapping tsteps_valid and tsteps is more appropriate.
self.tsteps_valid = header[b'NAXIS2']
self.tsteps = int(math.pow(2, math.ceil(np.log2(math.floor(self.tsteps_valid)))))
self.obs_length = self.tsteps_valid * header[b'DELTAT']
self.drift_rate_resolution = (1e6 * np.abs(header[b'DELTAF'])) / self.obs_length # in Hz/sec
self.header[b'baryv'] = 0.0
self.header[b'barya'] = 0.0
self.header[b'coarse_chan'] = coarse_chan
#EE For now I'm not using a shoulder. This is ok as long as I'm analyzing each coarse channel individually.
#EE In general this parameter is an integer (even number).
#This gives two regions, each of n*steps, around spectra[i]
self.shoulder_size = 0
self.tdwidth = self.fftlen + self.shoulder_size*self.tsteps
def __make_h5_file(self,):
''' Converts file to h5 format. Saves output in current dir.
'''
fil_file = Waterfall(self.filename)
new_filename = self.out_dir+self.filename.replace('.fil','.h5').split('/')[-1]
fil_file.write_to_hdf5(new_filename)
self.filename = new_filename
def __split_h5(self, size_limit=SIZE_LIM):
"""
Creates a plan to select data from single coarse channels.
:param size_limit: float, maximum size in MB that the file is allowed to be
:return: list[DATAH5], the list contains a DATAH5 object for each of the coarse channels in
the file
"""
data_list = []
#Instancing file.
try:
fil_file = Waterfall(self.filename)
except:
logger.error("Error encountered when trying to open file: %s" %
self.filename)
raise IOError("Error encountered when trying to open file: %s" %
self.filename)
#Finding lowest freq in file.
f_delt = fil_file.header['foff']
f0 = fil_file.header['fch1']
#Looping over the number of coarse channels.
if self.n_coarse_chan is not None:
n_coarse_chan = self.n_coarse_chan
elif fil_file.header.get('n_coarse_chan', None) is not None:
n_coarse_chan = fil_file.header['n_coarse_chan']
else:
n_coarse_chan = int(fil_file.calc_n_coarse_chan())
# Only load coarse chans of interest -- or do all if not specified
if self.coarse_chans in (None, ''):
self.coarse_chans = range(n_coarse_chan)
for chan in self.coarse_chans:
#Calculate freq range for given course channel.
f_start = f0 + chan * (
f_delt) * fil_file.n_channels_in_file / n_coarse_chan
f_stop = f0 + (chan + 1) * (
f_delt) * fil_file.n_channels_in_file / n_coarse_chan
if f_start > f_stop:
f_start, f_stop = f_stop, f_start
data_obj = DATAH5(self.filename,
f_start=f_start,
f_stop=f_stop,
coarse_chan=chan,
tn_coarse_chan=n_coarse_chan)
#----------------------------------------------------------------
#This appends to a list of all data instance selections. So that all get processed later.
data_list.append(data_obj)
return data_list
def __split_h5(self):
"""
Creates a plan to select data from single coarse channels.
:return: list[DATAH5],
where each list member contains a DATAH5 object
for each of the coarse channels in the file.
"""
data_list = []
#Instancing file.
try:
fil_file = Waterfall(self.filename)
except:
errmsg = "Error encountered when trying to open file: {}".format(
self.filename)
logger.error(errmsg)
raise IOError(errmsg)
#Finding lowest freq in file.
f_delt = fil_file.header['foff']
f0 = fil_file.header['fch1']
#Looping over the number of coarse channels.
if self.n_coarse_chan is not None:
n_coarse_chan = self.n_coarse_chan
elif fil_file.header.get('n_coarse_chan', None) is not None:
n_coarse_chan = fil_file.header['n_coarse_chan']
else:
n_coarse_chan = int(fil_file.calc_n_coarse_chan())
# Only load coarse chans of interest -- or do all if not specified
if self.coarse_chans in (None, ''):
self.coarse_chans = range(n_coarse_chan)
for chan in self.coarse_chans:
#Calculate freq range for given course channel.
f_start = f0 + chan * (
f_delt) * fil_file.n_channels_in_file / n_coarse_chan
f_stop = f0 + (chan + 1) * (
f_delt) * fil_file.n_channels_in_file / n_coarse_chan
if f_start > f_stop:
f_start, f_stop = f_stop, f_start
data_obj = {
'filename': self.filename,
'f_start': f_start,
'f_stop': f_stop,
'coarse_chan': chan,
'n_coarse_chan': n_coarse_chan
}
#This appends to a list of all data instance selections. So that all get processed later.
data_list.append(data_obj)
return data_list
def calibrate_fluxes(main_obs_name,dio_name,dspec,Tsys,fullstokes=False,**kwargs):
'''
Produce calibrated Stokes I for an observation given a noise diode
measurement on the source and a diode spectrum with the same number of
coarse channels
Parameters
----------
main_obs_name : str
Path to filterbank file containing final data to be calibrated
dio_name : str
Path to filterbank file for observation on the target source with flickering noise diode
dspec : 1D Array (float) or float
Coarse channel spectrum (or average) of the noise diode in Jy (obtained from diode_spec())
Tsys : 1D Array (float) or float
Coarse channel spectrum (or average) of the system temperature in Jy
fullstokes: boolean
Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if
it is in cross_pols format
'''
#Find folded spectra of the target source with the noise diode ON and OFF
main_obs = Waterfall(main_obs_name,max_load=150)
ncoarse = main_obs.calc_n_coarse_chan()
dio_obs = Waterfall(dio_name,max_load=150)
dio_chan_per_coarse = dio_obs.header['nchans']/ncoarse
dOFF,dON = integrate_calib(dio_name,dio_chan_per_coarse,fullstokes,**kwargs)
#Find Jy/count for each coarse channel using the diode spectrum
main_dat = main_obs.data
scale_facs = dspec/(dON-dOFF)
print(scale_facs)
nchans = main_obs.header['nchans']
obs_chan_per_coarse = nchans/ncoarse
ax0_size = np.size(main_dat,0)
ax1_size = np.size(main_dat,1)
#Reshape data array of target observation and multiply coarse channels by the scale factors
main_dat = np.reshape(main_dat,(ax0_size,ax1_size,ncoarse,obs_chan_per_coarse))
main_dat = np.swapaxes(main_dat,2,3)
main_dat = main_dat*scale_facs
main_dat = main_dat-Tsys
main_dat = np.swapaxes(main_dat,2,3)
main_dat = np.reshape(main_dat,(ax0_size,ax1_size,nchans))
#Write calibrated data to a new filterbank file with ".fluxcal" extension
main_obs.data = main_dat
main_obs.write_to_filterbank(main_obs_name[:-4]+'.fluxcal.fil')
print('Finished: calibrated product written to ' + main_obs_name[:-4]+'.fluxcal.fil')
def __make_h5_file(self):
r"""
Converts file to h5 format, saved in current directory.
Sets the filename attribute of the calling DATAHandle.
to the (new) filename.
"""
fil_file = Waterfall(self.filename)
bn = os.path.basename(self.filename)
new_filename = os.path.join(self.out_dir, bn.replace('.fil', '.h5'))
fil_file.write_to_hdf5(new_filename)
self.filename = new_filename
def test_ncc_chan_bw():
wf = Waterfall(voyager_h5)
print("test_bcc_chan_bw: telescope_id:", wf.header['telescope_id'])
print("test_ncc_chan_bw: nchans:", wf.header['nchans'])
n_coarse_chan = wf.calc_n_coarse_chan(16)
print("test_ncc_chan_bw: n_coarse_chan [chan_bw=16]:", n_coarse_chan)
assert n_coarse_chan == 64
n_coarse_chan = wf.calc_n_coarse_chan(1)
print("test_ncc_chan_bw: n_coarse_chan [chan_bw=1]:", n_coarse_chan)
assert n_coarse_chan > 2.9 and n_coarse_chan < 3.0
def __make_h5_file(self, ):
"""
Converts file to h5 format. Saves output in current dir. Sets the filename attribute of the calling DATAHandle
to the (new) filename.
:return: void
"""
fil_file = Waterfall(self.filename)
bn = os.path.basename(self.filename)
new_filename = os.path.join(self.out_dir, bn.replace('.fil', '.h5'))
fil_file.write_to_hdf5(new_filename)
self.filename = new_filename
def get_NT(FILE):
'''
Returns number of time integrations in fil FILE; loads one
frequency channel (across all time) of file to do this
'''
fbank = Waterfall(FILE, load_data=False)
fch1 = fbank.header['fch1']
f_off = fbank.header['foff']
fbank.read_data(
f_start=fch1 + f_off,
f_stop=fch1) # Read one channel of file to determine n_ints
n_ints = fbank.data.shape[0]
return n_ints
def diode_spec(calON_obs,
calOFF_obs,
calflux,
calfreq,
spec_in,
average=True,
oneflux=False,
**kwargs):
'''
Calculate the coarse channel spectrum and system temperature of the noise diode in Jy given two noise diode
measurements ON and OFF the calibrator source with the same frequency and time resolution
Parameters
----------
calON_obs : str
(see f_ratios() above)
calOFF_obs : str
(see f_ratios() above)
calflux : float
Known flux of calibrator source at a particular frequency
calfreq : float
Frequency where calibrator source has flux calflux (see above)
spec_in : float
Known power-law spectral index of calibrator source. Use convention flux(frequency) = constant * frequency^(spec_in)
average : boolean
Use average=True to return noise diode and Tsys spectra averaged over frequencies
'''
#Load frequencies and calculate number of channels per coarse channel
obs = Waterfall(calON_obs, max_load=150)
freqs = obs.populate_freqs()
ncoarse = obs.calc_n_coarse_chan()
nchans = obs.header['nchans']
chan_per_coarse = nchans / ncoarse
f_ON, f_OFF = f_ratios(calON_obs, calOFF_obs, chan_per_coarse, **kwargs)
#Obtain spectrum of the calibrator source for the given frequency range
centerfreqs = get_centerfreqs(freqs, chan_per_coarse)
calfluxes = get_calfluxes(calflux, calfreq, spec_in, centerfreqs, oneflux)
#C_o and Tsys as defined in van Straten et al. 2012
C_o = calfluxes / (1 / f_ON - 1 / f_OFF)
Tsys = C_o / f_OFF
#return coarse channel diode spectrum
if average == True:
return np.mean(C_o), np.mean(Tsys)
else:
return C_o, Tsys
def _update_waterfall(self):
# Set fil with sample data; (1.4 Hz, 1.4 s) res
if self.waterfall is None:
my_path = os.path.abspath(os.path.dirname(__file__))
path = os.path.join(my_path, 'assets/sample.fil')
self.waterfall = Waterfall(path)
self.waterfall.header[b'source_name'] = b'Synthetic'
self.waterfall.header[b'foff'] = self.df * -1e-6
self.waterfall.header[b'tsamp'] = self.dt
self.waterfall.header[b'nchans'] = self.fchans
self.waterfall.header[b'fch1'] = self.fmax
# Have to manually flip in the frequency direction + add an extra
# dimension for polarization to work with Waterfall
self.waterfall.data = self.data[:, np.newaxis, ::-1]
def gen_pulse_corpus(FILE, N, OUTDIR, NT=256, band='L', fullband=True):
'''
Calls get_pulse in parallel to generate a corpus of N batches
of pulses in OUTDIR
'''
if band == 'C':
fullband = False
fbank = Waterfall(FILE, load_data=False)
fheader = fbank.header
print("HEADER INFO:")
print(fheader)
if fullband:
fband_range = [
fheader['fch1'] + fheader['foff'] * fheader['nchans'],
fheader['fch1']
]
nu = np.arange(fband_range[0], fband_range[-1], np.abs(
fheader['foff'])) * 1.e-3
else:
fband_range = FREQ_BAND_TO_INFO[band]['freq_range']
fc1, fc2, nus = get_chans(fheader['fch1'],
fheader['foff'],
fheader['nchans'],
f1=fband_range[1],
f2=fband_range[0])
nu = nus[::-1] * 1.e-3 # to GHz
t = np.arange(0, NT, 1) * fbank.header['tsamp'] * 1.e3
if not os.path.exists(OUTDIR):
os.makedirs(OUTDIR)
_ = Parallel(n_jobs=8)(delayed(get_pulse)(t, nu, i, OUTDIR, 1, NT, band)
for i in range(N))
def get_info(self):
"""
:return: dict, header of the blimpy file
"""
fil_file = Waterfall(self.filename, load_data=False)
return fil_file.header
def write_fil(data, times, freqs_GHz, hdr, hotpotato):
# Reshape data array.
data = data.T.reshape(
(data.T.shape[0], 1, data.T.shape[1])
) # New shape = (No. of time samples, No. of polarizations, No. of channels)
# Construct a Waterfall object that will be written to disk as a filterbank file.
base_fileout = hotpotato['basename'] + '_t%.2fto%.2f_freqs%.2fto%.2f' % (
times[0], times[-1], freqs_GHz[0], freqs_GHz[-1]) + '.fil'
filename_out = hotpotato['OUTPUT_DIR'] + '/' + base_fileout
wat = Waterfall() # Empty Waterfall object
wat.header = hdr.primary
with open(filename_out, 'wb') as fh:
print('Writing smoothed data to %s' % (base_fileout))
fh.write(generate_sigproc_header(
wat)) # Trick Blimpy into writing a sigproc header.
np.float32(data.ravel()).tofile(fh)
def get_data(waterfall, use_db=False):
"""
Get time-frequency data from filterbank file as a 2d NumPy array.
Note: when multiple Stokes parameters are supported, this will have to
be expanded.
Parameters
----------
waterfall : str or Waterfall
Name of filterbank file or Waterfall object
Returns
-------
data : ndarray
Time-frequency data
"""
if isinstance(waterfall, str):
waterfall = Waterfall(waterfall)
elif not isinstance(waterfall, Waterfall):
sys.exit('Invalid data file!')
if use_db:
return 10 * np.log10(waterfall.data[:, 0, :])
return waterfall.data[:, 0, :]
def get_drift_rate(file):
# open the file
obs = Waterfall(file)
# get the time of obs + make array 15 min into future
jdate = obs.header['tstart'] + 2400000.5
JDUTC = np.linspace(jdate, jdate + (60.0 * 15.0/86400.), num=100)
# get the pointing of the obs
c = SkyCoord(obs.header['src_raj'], obs.header['src_dej'])
s = c.to_string('decimal')
ra_probe, dec_probe = [float(string) for string in s.split()]
# other needed params
obsname = 'GBT'
epoch = 2451545.0
rv = 0.0
zmeas = 0.0
ephemeris='https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/a_old_versions/de405.bsp'
# get the BC vel
baryvel = get_BC_vel(JDUTC=JDUTC, ra=ra_probe, dec=dec_probe, obsname='GBT',
rv=rv, zmeas=zmeas, epoch=epoch, ephemeris=ephemeris,
leap_update=True)
# take the derivative of velocity to get acceleration (i.e., drift)
diffT = np.diff(JDUTC) * 86400.0
diffV = np.diff(baryvel[0])
drift = diffV/diffT
# convert m/s^2 to Hz/s and take the max drift calculated
drift *= (obs.container.f_stop * 1e6 / 3e8)
# TODO check SIGN!!!
return np.max(np.abs(drift))
def plot_diodespec(ON_obs,OFF_obs,calflux,calfreq,spec_in,units='mJy',**kwargs):
"""
Plots the full-band Stokes I spectrum of the noise diode (ON-OFF)
"""
dspec = diode_spec(ON_obs,OFF_obs,calflux,calfreq,spec_in,**kwargs)
obs = Waterfall(ON_obs,max_load=150)
freqs = obs.populate_freqs()
chan_per_coarse = obs.header['nchans']/obs.calc_n_coarse_chan()
coarse_freqs = convert_to_coarse(freqs,chan_per_coarse)
plt.ion()
plt.figure()
plt.plot(coarse_freqs,dspec)
plt.xlabel('Frequency (MHz)')
plt.ylabel('Flux Density ('+units+')')
plt.title('Noise Diode Spectrum')
def get_Tsys_nodiode(calON_obs_name,
calOFF_obs_name,
calflux,
calfreq,
spec_in,
plot=False,
**kwargs):
'''
Calculates system temperature from two flux calibrator scans taken without noise diode flickering.
CURRENTLY ONLY IMPLEMENTED FOR STOKES I DATA.
Parameters
----------
calON_obs_name : str
Path to filterbank file for scan ON calibrator target
calOFF_obs_name : str
Path to filterbank file for scan OFF calibrator
calflux : float
Flux in Jy of the calibrator source
calfreq : float or int
Frequency at which calflux was taken (MHz)
spec_in : float
Spectral index of this calibrator
'''
calON_obs = Waterfall(calON_obs_name, max_load=150)
calOFF_obs = Waterfall(calOFF_obs_name, max_load=150)
ncoarse = calON_obs.calc_n_coarse_chan()
chan_per_coarse = calON_obs.header['nchans'] / ncoarse
ONobs_spec = np.squeeze(np.mean(calON_obs.data, axis=0))
OFFobs_spec = np.squeeze(np.mean(calOFF_obs.data, axis=0))
freqs = calON_obs.populate_freqs()
ON_coarse_spec = integrate_chans(ONobs_spec, freqs, chan_per_coarse)
OFF_coarse_spec = integrate_chans(OFFobs_spec, freqs, chan_per_coarse)
cfreqs = get_centerfreqs(freqs, chan_per_coarse)
cal_fluxes = get_calfluxes(calflux,
calfreq,
spec_in,
cfreqs,
oneflux=False)
S_sys = (OFF_coarse_spec) / (ON_coarse_spec - OFF_coarse_spec) * cal_fluxes
#Convert to Kelvins
T_sys, cfreqs = Jy_to_Kelvin(S_sys, cfreqs)
return T_sys, cfreqs
def integrate_calib(name,chan_per_coarse,fullstokes=False,**kwargs):
'''
Folds Stokes I noise diode data and integrates along coarse channels
Parameters
----------
name : str
Path to noise diode filterbank file
chan_per_coarse : int
Number of frequency bins per coarse channel
fullstokes : boolean
Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if
it is in cross_pols format
'''
#Load data
obs = Waterfall(name,max_load=150)
data = obs.data
#If the data has cross_pols format calculate Stokes I
if fullstokes==False and data.shape[1]>1:
data = data[:,0,:]+data[:,1,:]
data = np.expand_dims(data,axis=1)
#If the data has IQUV format get Stokes I
if fullstokes==True:
data = data[:,0,:]
data = np.expand_dims(data,axis=1)
tsamp = obs.header['tsamp']
#Calculate ON and OFF values
OFF,ON = foldcal(data,tsamp,**kwargs)
freqs = obs.populate_freqs()
#Find ON and OFF spectra by coarse channel
ON_int = integrate_chans(ON,freqs,chan_per_coarse)
OFF_int = integrate_chans(OFF,freqs,chan_per_coarse)
#If "ON" is actually "OFF" switch them
if np.sum(ON_int)<np.sum(OFF_int):
temp = ON_int
ON_int = OFF_int
OFF_int = temp
#Return coarse channel spectrum of OFF and ON
return OFF_int,ON_int
def read_filterbank(self, tab, startbin, chunksize):
"""
Read a chunk of filterbank data
:param int tab: TAB index
:param int startbin: Index of first time sample to read
:param int chunksize: Number of time samples to read
:return: chunk of data with shape (nfreq, chunksize)
"""
fil = Waterfall(self.fnames[tab], load_data=False)
# read chunk of data
fil.read_data(None, None, startbin, startbin + chunksize)
# keep only time and freq axes, transpose to have frequency first
data = fil.data[:, 0, :].T.astype(float)
if self.median_filter:
data -= np.median(data, axis=1, keepdims=True)
return data
def integrate_calib(name, chan_per_coarse, fullstokes=False, **kwargs):
'''
Folds Stokes I noise diode data and integrates along coarse channels
Parameters
----------
name : str
Path to noise diode filterbank file
chan_per_coarse : int
Number of frequency bins per coarse channel
fullstokes : boolean
Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if
it is in cross_pols format
'''
#Load data
obs = Waterfall(name, max_load=150)
data = obs.data
#If the data has cross_pols format calculate Stokes I
if fullstokes == False and data.shape[1] > 1:
data = data[:, 0, :] + data[:, 1, :]
data = np.expand_dims(data, axis=1)
#If the data has IQUV format get Stokes I
if fullstokes == True:
data = data[:, 0, :]
data = np.expand_dims(data, axis=1)
tsamp = obs.header['tsamp']
#Calculate ON and OFF values
OFF, ON = foldcal(data, tsamp, **kwargs)
freqs = obs.populate_freqs()
#Find ON and OFF spectra by coarse channel
ON_int = integrate_chans(ON, freqs, chan_per_coarse)
OFF_int = integrate_chans(OFF, freqs, chan_per_coarse)
#If "ON" is actually "OFF" switch them
if np.sum(ON_int) < np.sum(OFF_int):
temp = ON_int
ON_int = OFF_int
OFF_int = temp
#Return coarse channel spectrum of OFF and ON
return OFF_int, ON_int
def plot_gain_offsets(dio_cross,
dio_chan_per_coarse=8,
feedtype='l',
ax1=None,
ax2=None,
legend=True,
**kwargs):
'''
Plots the calculated gain offsets of each coarse channel along with
the time averaged power spectra of the X and Y feeds
'''
#Get ON-OFF ND spectra
Idiff, Qdiff, Udiff, Vdiff, freqs = get_diff(dio_cross, feedtype, **kwargs)
obs = Waterfall(dio_cross, max_load=150)
tsamp = obs.header['tsamp']
data = obs.data
obs = None
I, Q, U, V = get_stokes(data, feedtype)
#Get phase offsets and convert to degrees
coarse_G = gain_offsets(I, Q, U, V, tsamp, dio_chan_per_coarse, feedtype,
**kwargs)
coarse_freqs = convert_to_coarse(freqs, dio_chan_per_coarse)
#Get X and Y spectra for the noise diode ON and OFF
#If using circular feeds these correspond to LL and RR
XX_OFF, XX_ON = foldcal(np.expand_dims(data[:, 0, :], axis=1), tsamp,
**kwargs)
YY_OFF, YY_ON = foldcal(np.expand_dims(data[:, 1, :], axis=1), tsamp,
**kwargs)
if ax1 == None:
plt.subplot(211)
else:
axG = plt.axes(ax1)
plt.setp(axG.get_xticklabels(), visible=False)
plt.plot(coarse_freqs, coarse_G, 'ko', markersize=2)
plt.ylabel(r'$\frac{\Delta G}{2}$', rotation=90)
if feedtype == 'l':
plt.title('XY Gain Difference')
if feedtype == 'c':
plt.title('LR Gain Difference')
plt.grid(True)
if ax2 == None:
plt.subplot(212)
else:
axXY = plt.axes(ax2, sharex=axG)
if feedtype == 'l':
plt.plot(freqs, XX_OFF, 'b-', label='XX')
plt.plot(freqs, YY_OFF, 'r-', label='YY')
if feedtype == 'c':
plt.plot(freqs, XX_OFF, 'b-', label='LL')
plt.plot(freqs, YY_OFF, 'r-', label='RR')
plt.xlabel('Frequency (MHz)')
plt.ylabel('Power (Counts)')
if legend == True:
plt.legend()
def plot_calibrated_diode(dio_cross,
chan_per_coarse=8,
feedtype='l',
**kwargs):
'''
Plots the corrected noise diode spectrum for a given noise diode measurement
after application of the inverse Mueller matrix for the electronics chain.
'''
#Get full stokes data for the ND observation
obs = Waterfall(dio_cross, max_load=150)
freqs = obs.populate_freqs()
tsamp = obs.header['tsamp']
data = obs.data
obs = None
I, Q, U, V = get_stokes(data, feedtype)
data = None
#Calculate Mueller Matrix variables for each coarse channel
psis = phase_offsets(I, Q, U, V, tsamp, chan_per_coarse, feedtype,
**kwargs)
G = gain_offsets(I, Q, U, V, tsamp, chan_per_coarse, feedtype, **kwargs)
#Apply the Mueller matrix to original noise diode data and refold
I, Q, U, V = apply_Mueller(I, Q, U, V, G, psis, chan_per_coarse, feedtype)
I_OFF, I_ON = foldcal(I, tsamp, **kwargs)
Q_OFF, Q_ON = foldcal(Q, tsamp, **kwargs)
U_OFF, U_ON = foldcal(U, tsamp, **kwargs)
V_OFF, V_ON = foldcal(V, tsamp, **kwargs)
#Delete data arrays for space
I = None
Q = None
U = None
V = None
#Plot new ON-OFF spectra
plt.plot(freqs, I_ON - I_OFF, 'k-', label='I')
plt.plot(freqs, Q_ON - Q_OFF, 'r-', label='Q')
plt.plot(freqs, U_ON - U_OFF, 'g-', label='U')
plt.plot(freqs, V_ON - V_OFF, 'm-', label='V')
plt.legend()
plt.xlabel('Frequency (MHz)')
plt.title('Calibrated Full Stokes Noise Diode Spectrum')
plt.ylabel('Power (Counts)')
def split_file_generator(filename, frequency_dimension=512, count = None, NUM_PARTS = 100): wf = Waterfall(filename, load_data = False) f_begin, f_end, f_delta = get_f_iter_bounds(wf.header, frequency_dimension) leftover = None #read NUM_PARTS from wf at a time, to be split after for iteration, (f_start, f_stop) in enumerate(pairwise(np.arange(f_begin, f_end, f_delta * NUM_PARTS))): wf.read_data(f_start=f_start, f_stop = f_stop) parts_together = wf.data.squeeze() #squeeze brings shape from (16, 1, 512 * NUM_PARTS) -> (16, 512 * NUM_PARTS) if not leftover is None: parts_together = np.append(leftover, parts_together, axis = 1) parts_lst, leftover = split_parts(parts_together, frequency_dimension) yield parts_lst if count and iteration * NUM_PARTS >= count: break if not count and not leftover is None: yield leftover
def plot_phase_offsets(dio_cross,
chan_per_coarse=8,
feedtype='l',
ax1=None,
ax2=None,
legend=True,
**kwargs):
'''
Plots the calculated phase offsets of each coarse channel along with
the UV (or QU) noise diode spectrum for comparison
'''
#Get ON-OFF ND spectra
Idiff, Qdiff, Udiff, Vdiff, freqs = get_diff(dio_cross, feedtype, **kwargs)
obs = Waterfall(dio_cross, max_load=150)
tsamp = obs.header['tsamp']
data = obs.data
obs = None
I, Q, U, V = get_stokes(data, feedtype)
#Get phase offsets and convert to degrees
coarse_psis = phase_offsets(I, Q, U, V, tsamp, chan_per_coarse, feedtype,
**kwargs)
coarse_freqs = convert_to_coarse(freqs, chan_per_coarse)
coarse_degs = np.degrees(coarse_psis)
#Plot phase offsets
if ax2 == None:
plt.subplot(211)
else:
axPsi = plt.axes(ax2)
plt.setp(axPsi.get_xticklabels(), visible=False)
plt.plot(coarse_freqs,
coarse_degs,
'ko',
markersize=2,
label='Coarse Channel $\psi$')
plt.ylabel('Degrees')
plt.grid(True)
plt.title('Phase Offsets')
if legend == True:
plt.legend()
#Plot U and V spectra
if ax1 == None:
plt.subplot(212)
else:
axUV = plt.axes(ax1)
plt.plot(freqs, Udiff, 'g-', label='U')
if feedtype == 'l':
plt.plot(freqs, Vdiff, 'm-', label='V')
if feedtype == 'c':
plt.plot(freqs, Qdiff, 'r-', label='Q')
plt.xlabel('Frequency (MHz)')
plt.ylabel('Power (Counts)')
plt.grid(True)
if legend == True:
plt.legend()
def calibrate_fluxes(main_obs_name,
dio_name,
dspec,
Tsys,
fullstokes=False,
**kwargs):
'''
Produce calibrated Stokes I for an observation given a noise diode
measurement on the source and a diode spectrum with the same number of
coarse channels
Parameters
----------
main_obs_name : str
Path to filterbank file containing final data to be calibrated
dio_name : str
Path to filterbank file for observation on the target source with flickering noise diode
dspec : 1D Array (float) or float
Coarse channel spectrum (or average) of the noise diode in Jy (obtained from diode_spec())
Tsys : 1D Array (float) or float
Coarse channel spectrum (or average) of the system temperature in Jy
fullstokes: boolean
Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if
it is in cross_pols format
'''
#Find folded spectra of the target source with the noise diode ON and OFF
main_obs = Waterfall(main_obs_name, max_load=150)
ncoarse = main_obs.calc_n_coarse_chan()
dio_obs = Waterfall(dio_name, max_load=150)
dio_chan_per_coarse = dio_obs.header['nchans'] / ncoarse
dOFF, dON = integrate_calib(dio_name, dio_chan_per_coarse, fullstokes,
**kwargs)
#Find Jy/count for each coarse channel using the diode spectrum
main_dat = main_obs.data
scale_facs = dspec / (dON - dOFF)
print(scale_facs)
nchans = main_obs.header['nchans']
obs_chan_per_coarse = nchans / ncoarse
ax0_size = np.size(main_dat, 0)
ax1_size = np.size(main_dat, 1)
#Reshape data array of target observation and multiply coarse channels by the scale factors
main_dat = np.reshape(main_dat,
(ax0_size, ax1_size, ncoarse, obs_chan_per_coarse))
main_dat = np.swapaxes(main_dat, 2, 3)
main_dat = main_dat * scale_facs
main_dat = main_dat - Tsys
main_dat = np.swapaxes(main_dat, 2, 3)
main_dat = np.reshape(main_dat, (ax0_size, ax1_size, nchans))
#Write calibrated data to a new filterbank file with ".fluxcal" extension
main_obs.data = main_dat
main_obs.write_to_filterbank(main_obs_name[:-4] + '.fluxcal.fil')
print 'Finished: calibrated product written to ' + main_obs_name[:-4] + '.fluxcal.fil'
def plot_diode_fold(dio_cross,bothfeeds=True,feedtype='l',min_samp=-500,max_samp=7000,legend=True,**kwargs):
"""
Plots the calculated average power and time sampling of ON (red) and
OFF (blue) for a noise diode measurement over the observation time series
"""
#Get full stokes data of ND measurement
obs = Waterfall(dio_cross,max_load=150)
tsamp = obs.header['tsamp']
data = obs.data
obs = None
I,Q,U,V = get_stokes(data,feedtype)
#Calculate time series, OFF and ON averages, and time samples for each
tseriesI = np.squeeze(np.mean(I,axis=2))
I_OFF,I_ON,OFFints,ONints = foldcal(I,tsamp,inds=True,**kwargs)
if bothfeeds==True:
if feedtype=='l':
tseriesQ = np.squeeze(np.mean(Q,axis=2))
tseriesX = (tseriesI+tseriesQ)/2
tseriesY = (tseriesI-tseriesQ)/2
if feedtype=='c':
tseriesV = np.squeeze(np.mean(V,axis=2))
tseriesR = (tseriesI+tseriesV)/2
tseriesL = (tseriesI-tseriesV)/2
stop = ONints[-1,1]
#Plot time series and calculated average for ON and OFF
if bothfeeds==False:
plt.plot(tseriesI[0:stop],'k-',label='Total Power')
for i in ONints:
plt.plot(np.arange(i[0],i[1]),np.full((i[1]-i[0]),np.mean(I_ON)),'r-')
for i in OFFints:
plt.plot(np.arange(i[0],i[1]),np.full((i[1]-i[0]),np.mean(I_OFF)),'b-')
else:
if feedtype=='l':
diff = np.mean(tseriesX)-np.mean(tseriesY)
plt.plot(tseriesX[0:stop],'b-',label='XX')
plt.plot(tseriesY[0:stop]+diff,'r-',label='YY (shifted)')
if feedtype=='c':
diff = np.mean(tseriesL)-np.mean(tseriesR)
plt.plot(tseriesL[0:stop],'b-',label='LL')
plt.plot(tseriesR[0:stop]+diff,'r-',label='RR (shifted)')
#Calculate plotting limits
if bothfeeds==False:
lowlim = np.mean(I_OFF)-(np.mean(I_ON)-np.mean(I_OFF))/2
hilim = np.mean(I_ON)+(np.mean(I_ON)-np.mean(I_OFF))/2
plt.ylim((lowlim,hilim))
plt.xlim((min_samp,max_samp))
plt.xlabel('Time Sample Number')
plt.ylabel('Power (Counts)')
plt.title('Noise Diode Fold')
if legend==True:
plt.legend()
def diode_spec(calON_obs,calOFF_obs,calflux,calfreq,spec_in,average=True,oneflux=False,**kwargs):
'''
Calculate the coarse channel spectrum and system temperature of the noise diode in Jy given two noise diode
measurements ON and OFF the calibrator source with the same frequency and time resolution
Parameters
----------
calON_obs : str
(see f_ratios() above)
calOFF_obs : str
(see f_ratios() above)
calflux : float
Known flux of calibrator source at a particular frequency
calfreq : float
Frequency where calibrator source has flux calflux (see above)
spec_in : float
Known power-law spectral index of calibrator source. Use convention flux(frequency) = constant * frequency^(spec_in)
average : boolean
Use average=True to return noise diode and Tsys spectra averaged over frequencies
'''
#Load frequencies and calculate number of channels per coarse channel
obs = Waterfall(calON_obs,max_load=150)
freqs = obs.populate_freqs()
ncoarse = obs.calc_n_coarse_chan()
nchans = obs.header['nchans']
chan_per_coarse = nchans/ncoarse
f_ON, f_OFF = f_ratios(calON_obs,calOFF_obs,chan_per_coarse,**kwargs)
#Obtain spectrum of the calibrator source for the given frequency range
centerfreqs = get_centerfreqs(freqs,chan_per_coarse)
calfluxes = get_calfluxes(calflux,calfreq,spec_in,centerfreqs,oneflux)
#C_o and Tsys as defined in van Straten et al. 2012
C_o = calfluxes/(1/f_ON-1/f_OFF)
Tsys = C_o/f_OFF
#return coarse channel diode spectrum
if average==True:
return np.mean(C_o),np.mean(Tsys)
else:
return C_o,Tsys
def write_polfils(str, str_I, **kwargs):
'''Writes two new filterbank files containing fractional linear and
circular polarization data'''
lin,circ=fracpols(str, **kwargs)
obs = Waterfall(str_I, max_load=150)
obs.data = lin
obs.write_to_fil(str[:-15]+'.linpol.fil') #assuming file is named *.cross_pols.fil
obs.data = circ
obs.write_to_fil(str[:-15]+'.circpol.fil') #assuming file is named *.cross_pols.fil
def calibrate_pols(cross_pols,diode_cross,obsI=None,onefile=True,feedtype='l',**kwargs):
'''
Write Stokes-calibrated filterbank file for a given observation
with a calibrator noise diode measurement on the source
Parameters
----------
cross_pols : string
Path to cross polarization filterbank file (rawspec output) for observation to be calibrated
diode_cross : string
Path to cross polarization filterbank file of noise diode measurement ON the target
obsI : string
Path to Stokes I filterbank file of main observation (only needed if onefile=False)
onefile : boolean
True writes all calibrated Stokes parameters to a single filterbank file,
False writes four separate files
feedtype : 'l' or 'c'
Basis of antenna dipoles. 'c' for circular, 'l' for linear
'''
#Obtain time sample length, frequencies, and noise diode data
obs = Waterfall(diode_cross,max_load=150)
cross_dat = obs.data
tsamp = obs.header['tsamp']
#Calculate number of coarse channels in the noise diode measurement (usually 8)
dio_ncoarse = obs.calc_n_coarse_chan()
dio_nchans = obs.header['nchans']
dio_chan_per_coarse = dio_nchans/dio_ncoarse
obs = None
Idat,Qdat,Udat,Vdat = get_stokes(cross_dat,feedtype)
cross_dat = None
#Calculate differential gain and phase from noise diode measurements
print('Calculating Mueller Matrix variables')
gams = gain_offsets(Idat,Qdat,Udat,Vdat,tsamp,dio_chan_per_coarse,feedtype,**kwargs)
psis = phase_offsets(Idat,Qdat,Udat,Vdat,tsamp,dio_chan_per_coarse,feedtype,**kwargs)
#Clear data arrays to save memory
Idat = None
Qdat = None
Udat = None
Vdat = None
#Get corrected Stokes parameters
print('Opening '+cross_pols)
cross_obs = Waterfall(cross_pols,max_load=150)
obs_ncoarse = cross_obs.calc_n_coarse_chan()
obs_nchans = cross_obs.header['nchans']
obs_chan_per_coarse = obs_nchans/obs_ncoarse
print('Grabbing Stokes parameters')
I,Q,U,V = get_stokes(cross_obs.data,feedtype)
print('Applying Mueller Matrix')
I,Q,U,V = apply_Mueller(I,Q,U,V,gams,psis,obs_chan_per_coarse,feedtype)
#Use onefile (default) to produce one filterbank file containing all Stokes information
if onefile==True:
cross_obs.data[:,0,:] = np.squeeze(I)
cross_obs.data[:,1,:] = np.squeeze(Q)
cross_obs.data[:,2,:] = np.squeeze(U)
cross_obs.data[:,3,:] = np.squeeze(V)
cross_obs.write_to_fil(cross_pols[:-15]+'.SIQUV.polcal.fil')
print('Calibrated Stokes parameters written to '+cross_pols[:-15]+'.SIQUV.polcal.fil')
return
#Write corrected Stokes parameters to four filterbank files if onefile==False
obs = Waterfall(obs_I,max_load=150)
obs.data = I
obs.write_to_fil(cross_pols[:-15]+'.SI.polcal.fil') #assuming file is named *.cross_pols.fil
print('Calibrated Stokes I written to '+cross_pols[:-15]+'.SI.polcal.fil')
obs.data = Q
obs.write_to_fil(cross_pols[:-15]+'.Q.polcal.fil') #assuming file is named *.cross_pols.fil
print('Calibrated Stokes Q written to '+cross_pols[:-15]+'.Q.polcal.fil')
obs.data = U
obs.write_to_fil(cross_pols[:-15]+'.U.polcal.fil') #assuming file is named *.cross_pols.fil
print('Calibrated Stokes U written to '+cross_pols[:-15]+'.U.polcal.fil')
obs.data = V
obs.write_to_fil(cross_pols[:-15]+'.V.polcal.fil') #assuming file is named *.cross_pols.fil
print('Calibrated Stokes V written to '+cross_pols[:-15]+'.V.polcal.fil')
def write_stokefils(str, str_I, Ifil=False, Qfil=False, Ufil=False, Vfil=False, Lfil=False, **kwargs):
'''Writes up to 5 new filterbank files corresponding to each Stokes
parameter (and total linear polarization L) for a given cross polarization .fil file'''
I,Q,U,V,L=get_stokes(str, **kwargs)
obs = Waterfall(str_I, max_load=150) #Load filterbank file to write stokes data to
if Ifil:
obs.data = I
obs.write_to_fil(str[:-15]+'.I.fil') #assuming file is named *.cross_pols.fil
if Qfil:
obs.data = Q
obs.write_to_fil(str[:-15]+'.Q.fil') #assuming file is named *.cross_pols.fil
if Ufil:
obs.data = U
obs.write_to_fil(str[:-15]+'.U.fil') #assuming file is named *.cross_pols.fil
if Vfil:
obs.data = V
obs.write_to_fil(str[:-15]+'.V.fil') #assuming file is named *.cross_pols.fil
if Lfil:
obs.data = L
obs.write_to_fil(str[:-15]+'.L.fil') #assuming file is named *.cross_pols.fil
