def action(self, Data) :
params = self.params
# Keep track of how many pre existing flags there are for feedback
# purposes.
already_flagged = ma.count_masked(Data.data)
if params["rotate"]:
if (tuple(Data.field['CRVAL4']) == (1, 2, 3, 4)):
rotate_pol.rotate(Data, (-5,-7,-8,-6))
Data.add_history('Rotated to XX,XY,YX,YY')
# Few operations to be performed before flagging.
if params["perform_hanning"] :
hanning.hanning_smooth(Data)
Data.add_history('Hanning smoothed.')
if params["cal_scale"] :
cal_scale.scale_by_cal(Data, True, False, False)
Data.add_history('Converted to units of noise cal temperture.')
# Flag the data.
apply_cuts(Data, sigma_thres=params['sigma_thres'],
badness_thres=params['badness_thres'],
time_cut=params['time_cut'])
Data.add_history('Flagged Bad Data.', ('Sigma threshold: '
+ str(self.params['sigma_thres']), 'Badness threshold: '
+ str(self.params['badness_thres']), 'Time mask size: '
+ str(self.params['time_cut'])))
# Report the number of new flags.
new_flags = ma.count_masked(Data.data) - already_flagged
self.block_feedback = str(new_flags) + ', '
return Data
def load_moonscan(filename):
cal_coords = ephem.Equatorial("05:42:36.155", "+49:51:07.28",
epoch=ephem.B1950)
# convert cal to a Body object.
cal_source = ephem.FixedBody()
cal_source._ra = cal_coords.ra
cal_source._dec = cal_coords.dec
cal_source._epoch = cal_coords.epoch
Reader = fitsGBT.Reader(filename)
moon_dataobj = Reader.read(0,0)
rotate_pol.rotate(moon_dataobj, (-5, -7, -8, -6))
cal_scale.scale_by_cal(moon_dataobj, scale_t_ave=True, scale_f_ave=False,
sub_med=False, scale_f_ave_mod=False, rotate=True)
flag_data.flag_data(moon_dataobj, 5, 0.1, 40)
rebin_freq.rebin(moon_dataobj, 16, True, True)
#rebin_time.rebin(moon_dataobj, 4)
fgc_mueler_file = '/mnt/raid-project/gmrt/tcv/diff_gain_params/GBT12A_418/22_diff_gain_calc.txt'
fgc_RM_file = ' '
fgc_R_to_sky = True
fgc_DP_correct = False # this is already handled in scale_by_cal's rotate
fgc_RM_correct = False
from time_stream import flux_diff_gain_cal as fdg
m_total = fdg.flux_dg(fgc_mueler_file)
fdg.calibrate_pol(moon_dataobj, m_total, fgc_RM_file,
fgc_R_to_sky, fgc_DP_correct, fgc_RM_correct)
return moon_dataobj
def action(self, Data):
"""Prepares Data and flags RFI.
Parameters
----------
Data : DataBlock
Contains information in a usable format direct from GBT.
Returns
-------
Data : DataBlock
The input `Data` with RFI flagged. Will also be cal scaled and
rotated to XX,YY... if so chosen.
"""
params = self.params
if params["rotate"]:
if tuple(Data.field["CRVAL4"]) == (1, 2, 3, 4):
rotate_pol.rotate(Data, (-5, -7, -8, -6))
Data.add_history("Rotated to XX,XY,YX,YY")
print "Rotated to XX,XY,YX,YY"
if params["cal_scale"] or params["cal_phase"]:
cal_scale.scale_by_cal(Data, params["cal_scale"], False, False, False, rotate=params["cal_phase"])
Data.add_history("Converted to units of noise cal temperture.")
print "Converted to units of noise cal temperture."
if params["cal_rm"]:
cal_xx = np.ma.mean(Data.data[:, 0, 0, :] - Data.data[:, 0, 1, :], axis=0)
cal_yy = np.ma.mean(Data.data[:, 3, 0, :] - Data.data[:, 3, 1, :], axis=0)
if np.ma.median(cal_xx) > 0:
# print np.median(cal_xx)
Data.data[:, 0, 0, :] -= cal_xx[None, :]
Data.data[:, 3, 0, :] -= cal_yy[None, :]
Data.data[:, 1, 0, :] -= np.ma.sqrt(cal_xx * cal_yy)[None, :]
Data.data[:, 2, 0, :] -= np.ma.sqrt(cal_xx * cal_yy)[None, :]
else:
Data.data[:, 0, 1, :] -= cal_xx[None, :]
Data.data[:, 3, 1, :] -= cal_yy[None, :]
Data.data[:, 1, 1, :] -= np.ma.sqrt(cal_xx * cal_yy)[None, :]
Data.data[:, 2, 1, :] -= np.ma.sqrt(cal_xx * cal_yy)[None, :]
Data.add_history("Remove noise cal signal.")
print "Remove noise cal signal."
# return corr_svd(Data, params, self.file_ind)
# get freq axis and time axis
Data.calc_time()
time = Data.time
freq = np.arange(Data.data.shape[-1]) - Data.field["CRPIX1"] + 1
freq *= Data.field["CDELT1"]
freq += Data.field["CRVAL1"]
# Data.data = svd_spec_time(Data.data, params, self.file_ind, freq, time)
Data.data = corr_svd(Data.data, params, self.file_ind, freq, time)
return Data
def action(self, Data):
'''Prepares Data and flags RFI.
Parameters
----------
Data : DataBlock
Contains information in a usable format direct from GBT.
Returns
-------
Data : DataBlock
The input `Data` with RFI flagged. Will also be cal scaled and
rotated to XX,YY... if so chosen.
'''
params = self.params
# Keep track of how many pre existing flags there are for feedback
# purposes.
already_flagged = ma.count_masked(Data.data)
if params["rotate"]:
if (tuple(Data.field['CRVAL4']) == (1, 2, 3, 4)):
rotate_pol.rotate(Data, (-5, -7, -8, -6))
Data.add_history('Rotated to XX,XY,YX,YY')
# Few operations to be performed before flagging.
if params["perform_hanning"]:
hanning.hanning_smooth(Data)
Data.add_history('Hanning smoothed.')
if params["cal_scale"] or params["cal_phase"]:
cal_scale.scale_by_cal(Data,
params['cal_scale'],
False,
False,
False,
rotate=params['cal_phase'])
Data.add_history('Converted to units of noise cal temperture.')
# Flag the data.
apply_cuts(Data,
sigma_thres=params['sigma_thres'],
badness_thres=params['badness_thres'],
time_cut=params['time_cut'])
Data.add_history(
'Flagged Bad Data.',
('Sigma threshold: ' + str(self.params['sigma_thres']),
'Badness threshold: ' + str(self.params['badness_thres']),
'Time mask size: ' + str(self.params['time_cut'])))
# Report the number of new flags.
new_flags = ma.count_masked(Data.data) - already_flagged
percent = float(new_flags) / Data.data.size * 100
self.block_feedback = '%d (%f%%), ' % (new_flags, percent)
return Data
def action(self, Data):
'''Prepares Data and flags RFI.
Parameters
----------
Data : DataBlock
Contains information in a usable format direct from GBT.
Returns
-------
Data : DataBlock
The input `Data` with RFI flagged. Will also be cal scaled and
rotated to XX,YY... if so chosen.
'''
params = self.params
# Keep track of how many pre existing flags there are for feedback
# purposes.
already_flagged = ma.count_masked(Data.data)
if params["rotate"]:
if (tuple(Data.field['CRVAL4']) == (1, 2, 3, 4)):
rotate_pol.rotate(Data, (-5,-7,-8,-6))
Data.add_history('Rotated to XX,XY,YX,YY')
# Few operations to be performed before flagging.
if params["perform_hanning"] :
hanning.hanning_smooth(Data)
Data.add_history('Hanning smoothed.')
if params["cal_scale"] or params["cal_phase"]:
cal_scale.scale_by_cal(Data, params['cal_scale'], False, False,
False, rotate=params['cal_phase'])
Data.add_history('Converted to units of noise cal temperture.')
# Flag the data.
apply_cuts(Data, sigma_thres=params['sigma_thres'],
badness_thres=params['badness_thres'],
time_cut=params['time_cut'],
submean=params['submean'])
Data.add_history('Flagged Bad Data.', ('Sigma threshold: '
+ str(self.params['sigma_thres']), 'Badness threshold: '
+ str(self.params['badness_thres']), 'Time mask size: '
+ str(self.params['time_cut'])))
# Report the number of new flags.
new_flags = ma.count_masked(Data.data) - already_flagged
percent = float(new_flags) / Data.data.size * 100
self.block_feedback = '%d (%f%%), ' % (new_flags, percent)
return Data
def load_moonscan(filename, rotate_moon=True):
cal_coords = ephem.Equatorial("05:42:36.155",
"+49:51:07.28",
epoch=ephem.B1950)
# convert cal to a Body object.
cal_source = ephem.FixedBody()
cal_source._ra = cal_coords.ra
cal_source._dec = cal_coords.dec
cal_source._epoch = cal_coords.epoch
Reader = fitsGBT.Reader(filename)
moon_dataobj = Reader.read(0, 0)
rotate_pol.rotate(moon_dataobj, (-5, -7, -8, -6))
cal_scale.scale_by_cal(moon_dataobj,
scale_t_ave=True,
scale_f_ave=False,
sub_med=False,
scale_f_ave_mod=False,
rotate=True)
flag_data.flag_data(moon_dataobj, 5, 0.1, 40)
rebin_freq.rebin(moon_dataobj, 16, True, True)
#rebin_time.rebin(moon_dataobj, 4)
if rotate_moon:
moon_rotation.rotate_pol_moon(moon_dataobj)
fgc_mueler_file = '/mnt/raid-project/gmrt/tcv/diff_gain_params/GBT12A_418/22_diff_gain_calc.txt'
fgc_RM_file = ' '
fgc_R_to_sky = True
fgc_DP_correct = False # this is already handled in scale_by_cal's rotate
fgc_RM_correct = False
from time_stream import flux_diff_gain_cal as fdg
m_total = fdg.flux_dg(fgc_mueler_file)
fdg.calibrate_pol(moon_dataobj, m_total, fgc_RM_file, fgc_R_to_sky,
fgc_DP_correct, fgc_RM_correct)
return moon_dataobj
def test_dims_cal_ave(self) :
dims = self.Data.dims
rotate_pol.rotate(self.Data, (1,), True)
dims = dims[:1] + (1,1) + dims[3:]
self.assertEqual(self.Data.dims, dims)
self.Data.verify()
def test_dims(self) :
dims = self.Data.dims
rotate_pol.rotate(self.Data, (1,))
dims = dims[:1] + (1,) + dims[2:]
self.assertEqual(self.Data.dims, dims)
def test_runs_all(self) :
rotate_pol.rotate(self.Data, (1,2,3,4))
self.Data.verify()
def test_runs_I(self) :
rotate_pol.rotate(self.Data, (1,))
self.Data.verify()
def test_runs_all(self):
rotate_pol.rotate(self.Data, (1, 2, 3, 4))
self.Data.verify()
Blocks = Reader.read((0, 1), None)
for Data in Blocks:
rebin_freq.rebin(Data, 32, True, True)
rebin_time.rebin(Data, 2)
split_Blocks = ()
for Data in Blocks:
split_Blocks += split_bands.split(Data, 2, 32, 25)
comb_Blocks = copy.deepcopy(split_Blocks)
for Data in comb_Blocks:
combine_cal.combine(Data, sub_mean=False)
rot_Blocks = copy.deepcopy(comb_Blocks)
for Data in rot_Blocks:
rotate_pol.rotate(Data)
# Measure some parameters from the noise.
# Sort into 2 bands
file_pars = {}
band1_Blocks = []
band2_Blocks = []
band1 = 0
band2 = 0
for Data in rot_Blocks:
band = int(round(Data.field["CRVAL1"] / 1e6))
if band == band1:
band1_Blocks.append(Data)
elif band == band2:
band2_Blocks.append(Data)
elif not band1:
Blocks = Reader.read((0,1), None)
for Data in Blocks:
rebin_freq.rebin(Data, 32, True, True)
rebin_time.rebin(Data, 2)
split_Blocks = ()
for Data in Blocks:
split_Blocks += split_bands.split(Data, 2, 32, 25)
comb_Blocks = copy.deepcopy(split_Blocks)
for Data in comb_Blocks:
combine_cal.combine(Data, sub_mean=False)
rot_Blocks = copy.deepcopy(comb_Blocks)
for Data in rot_Blocks:
rotate_pol.rotate(Data)
# Measure some parameters from the noise.
# Sort into 2 bands
file_pars = {}
band1_Blocks = []
band2_Blocks = []
band1 = 0
band2 = 0
for Data in rot_Blocks:
band = int(round(Data.field["CRVAL1"]/1e6))
if band == band1:
band1_Blocks.append(Data)
elif band == band2:
band2_Blocks.append(Data)
elif not band1:
def test_dims_cal_ave(self):
dims = self.Data.dims
rotate_pol.rotate(self.Data, (1, ), True)
dims = dims[:1] + (1, 1) + dims[3:]
self.assertEqual(self.Data.dims, dims)
self.Data.verify()
def test_dims(self):
dims = self.Data.dims
rotate_pol.rotate(self.Data, (1, ))
dims = dims[:1] + (1, ) + dims[2:]
self.assertEqual(self.Data.dims, dims)
def process_file(self, middle) :
"""Split off to fix pyfits memory leak."""
params = self.params
# Construct the file name and read in all scans.
file_name = params["input_root"] + middle + ".fits"
Reader = fitsGBT.Reader(file_name)
Blocks = Reader.read((), (), force_tuple=True)
# Plotting limits need to be adjusted for on-off scans.
if file_name.find("onoff") != -1 :
onoff=True
else :
onoff=False
# Initialize a few variables.
counts = 0
cal_sum_unscaled = 0
cal_sum = 0
cal_time = ma.zeros((0, 4))
sys_time = ma.zeros((0, 4))
cal_noise_spec = 0
# Get the number of times in the first block and shorten to a
# number that should be smaller than all blocks.
nt = int(Blocks[0].dims[0]*.9)
# Get the frequency axis. Must be before loop because the data is
# rebined in the loop.
Blocks[0].calc_freq()
f = Blocks[0].freq
for Data in Blocks :
# Rotate to XX, YY etc.
rotate_pol.rotate(Data, (-5, -7, -8, -6))
this_count = ma.count(Data.data[:,:,0,:]
+ Data.data[:,:,1,:], 0)
cal_sum_unscaled += ma.sum(Data.data[:,:,0,:] +
Data.data[:,:,1,:], 0)
# Time series of the cal temperture.
cal_time = sp.concatenate((cal_time, ma.mean(Data.data[:,:,0,:]
- Data.data[:,:,1,:], -1).filled(-1)), 0)
# Everything else done in cal units.
cal_scale.scale_by_cal(Data)
# Time serise of the system temperture.
sys_time = sp.concatenate((sys_time, ma.mean(Data.data[:,:,0,:]
+ Data.data[:,:,1,:], -1).filled(-5)), 0)
# Accumulate variouse sums.
counts += this_count
cal_sum += ma.sum(Data.data[:,:,0,:] + Data.data[:,:,1,:], 0)
# Take power spectrum of on-off/on+off.
rebin_freq.rebin(Data, 512, mean=True, by_nbins=True)
cal_diff = ((Data.data[:,[0,-1],0,:]
- Data.data[:,[0,-1],1,:])
/ (Data.data[:,[0,-1],0,:]
+ Data.data[:,[0,-1],1,:]))
cal_diff -= ma.mean(cal_diff, 0)
cal_diff = cal_diff.filled(0)[0:nt,...]
power = abs(fft.fft(cal_diff, axis=0)[range(nt//2+1)])
power = power**2/nt
cal_noise_spec += power
# Normalize.
cal_sum_unscaled /= 2*counts
cal_sum /= 2*counts
# Get time steps and frequency wdith for noise power normalization.
Data = Blocks[0]
Data.calc_time()
dt = abs(sp.mean(sp.diff(Data.time)))
# Note that Data was rebined in the loop.
dnu = abs(Data.field["CDELT1"])
cal_noise_spec *= dt*dnu/len(Blocks)
# Power spectrum independant axis.
ps_freqs = sp.arange(nt//2 + 1, dtype=float)
ps_freqs /= (nt//2 + 1)*dt*2
# Long time axis.
t_total = sp.arange(cal_time.shape[0])*dt
# Make plots.
h = plt.figure(figsize=(10,10))
# Unscaled temperature spectrum.
plt.subplot(3, 2, 1)
plt.plot(f/1e6, sp.rollaxis(cal_sum_unscaled, -1))
plt.xlim((7e2, 9e2))
plt.xlabel("frequency (MHz)")
plt.title("System temperature - mean over time")
# Temperture spectrum in terms of noise cal. 4 Polarizations.
plt.subplot(3, 2, 2)
plt.plot(f/1e6, sp.rollaxis(cal_sum, -1))
if onoff :
plt.ylim((-1, 60))
else :
plt.ylim((-10, 40))
plt.xlim((7e2, 9e2))
plt.xlabel("frequency (MHz)")
plt.title("System temperature in cal units")
# Time serise of cal T.
plt.subplot(3, 2, 3)
plt.plot(t_total, cal_time)
if onoff :
plt.xlim((0,dt*900))
else :
plt.xlim((0,dt*3500))
plt.xlabel("time (s)")
plt.title("Noise cal temperature - mean over frequency")
# Time series of system T.
plt.subplot(3, 2, 4)
plt.plot(t_total, sys_time)
plt.xlabel("time (s)")
if onoff :
plt.ylim((-4, 90))
plt.xlim((0,dt*900))
else :
plt.ylim((-4, 35))
plt.xlim((0,dt*3500))
plt.title("System temperature in cal units")
# XX cal PS.
plt.subplot(3, 2, 5)
plt.loglog(ps_freqs, cal_noise_spec[:,0,:])
plt.xlim((1.0/60, 1/(2*dt)))
plt.ylim((1e-1, 1e3))
plt.xlabel("frequency (Hz)")
plt.title("XX cal power spectrum")
# YY cal PS.
plt.subplot(3, 2, 6)
plt.loglog(ps_freqs, cal_noise_spec[:,1,:])
plt.xlim((1.0/60, 1/(2*dt)))
plt.ylim((1e-1, 1e3))
plt.xlabel("frequency (Hz)")
plt.title("YY cal power spectrum")
# Adjust spacing.
plt.subplots_adjust(hspace=.4)
# Save the figure.
plt.savefig(params['output_root'] + middle
+ params['output_end'])
cal_files = ['21_3C286_track_' + str(ii) for ii in range(18, 26)]
# The following two loops is a standard set of things we do to our raw data
# when it comes from the telescope. Our data format is roughly SDfits.
# Read and preprocess the Data.
cal_Blocks = []
for fname in cal_files:
# Read.
fpath = data_root + fname + end
Reader = fitsGBT.Reader(fpath)
Data = Reader.read(0, 0)
cal_Blocks.append(Data)
for Data in cal_Blocks:
# Preprocess.
rotate_pol.rotate(Data, (-5, -7, -8, -6))
cal_scale.scale_by_cal(Data, True, False, False, False, True)
flag_data.flag_data(Data, 5, 0.1, 40)
#rebin_freq.rebin(Data, 16, True, True)
rebin_freq.rebin(Data, 16, True, True)
#combine_cal.combine(Data, (0.5, 0.5), False, True)
combine_cal.combine(Data, (0., 1.), False, True)
#rebin_time.rebin(Data, 4)
Data.calc_freq()
# Put all the data into a same format for the fits.
BeamData = beam_fit.FormattedData(cal_Blocks)
# Source object. This just calculates the ephemeris of the source compared to
# where the telescope is pointing.
fnames = ['02_3C295_track_104.fits', '02_3C295_track_48.fits']
#fnames = ['02_3C295_track_104.fits'] #, '02_3C295_track_48.fits']
root = os.getenv('GBT_DATA') + 'GBT12A_418/'
Blocks = []
for fname in fnames:
# Read.
fpath = root + fname
Reader = fitsGBT.Reader(fpath)
Data = Reader.read(0,0)
Blocks.append(Data)
for Data in Blocks:
# Preprocess.
rotate_pol.rotate(Data, (-5, -7, -8, -6))
cal_scale.scale_by_cal(Data, True, False, False, False, True)
flag_data.flag_data(Data, 5, 0.1, 40)
rebin_freq.rebin(Data, 8, True, True)
rebin_time.rebin(Data, 4)
#rotate_pol.rotate(Data, (1, 2, 3, 4))
def model(n_time, centre, width, amp_xx, amp_yy, amp_xy, amp_yx,
off_xx, off_yy, off_yx, off_xy, slope_xx, slope_yy, slope_xy,
slope_yx, gain_xx, gain_yy, quad_xx, quad_yy):
# Preliminaries.
time = np.arange(n_time, dtype=float) - centre
out = np.empty((4, 2, n_time), dtype=float)
# Generate a unit gaussian.
gauss = np.exp(- time**2 / (2 * width**2))
def test_runs_I(self):
rotate_pol.rotate(self.Data, (1, ))
self.Data.verify()