def bandpass_I_from_onoff(filename):
reader = fitsGBT.Reader(f)
scans = reader.read()
scans[0].calc_freq()
freq = scans[0].freq
src = scans[0].field['OBJECT']
on_spectrum = I_in_cal_units(scans[0])
off_spectrum = I_in_cal_units(scans[1])
src_spectrum = on_spectrum - off_spectrum
src_nominal = calibrator_spectrum(src, freq)
cal_spectrum = src_nominal / src_spectrum
out = np.empty(len(freq),
dtype=[('freq', np.float64), ('cal_T', np.float64)])
out['freq'] = freq
out['cal_T'] = cal_spectrum
#plt.figure()
#plt.plot(freq, on_spectrum)
#plt.plot(freq, off_spectrum)
#plt.figure()
#plt.plot(freq, cal_spectrum)
return out
def execute(self):
"""Process all data."""
# You have access to the input parameters through the dictionary
# self.params.
params = self.params
# If you have output files, make parent directories if need be.
utils.mkparents(params['output_root'])
# Write the input parameters to file so you can go back and look at
# them.
parse_ini.write_params(params,
params['output_root'] + 'params.ini',
prefix=self.prefix)
# Loop over the files to process.
for file_middle in params['file_middles']:
input_fname = (params['input_root'] + file_middle +
params['input_end'])
# Read in the data. The reader is an object that can read
# DataBlock objects out of a fits file.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
# Some examples of how you would read DataBlock Objects:
first_scan_and_IF_DataBlock = Reader.read(scans=0, IFs=0)
second_scan_and_first_IF_DataBlock = Reader.read(scans=1, IFs=0)
list_of_a_few_data_blocks = Reader.read(scans=(1, 2, 3), IFs=0)
list_of_all_data_blocks = Reader.read(scans=(), IFs=())
def setUp(self):
# Read in just to fiugre out the band structure.
this_test_file = 'testdata/testfile_guppi_rotated.fits'
Reader = fitsGBT.Reader(this_test_file, feedback=0)
Blocks = Reader.read((0, ), ())
bands = ()
for Data in Blocks:
n_chan = Data.dims[3]
Data.calc_freq()
freq = Data.freq
delta = abs(sp.mean(sp.diff(freq)))
centre = freq[n_chan // 2]
band = int(centre / 1e6)
bands += (band, )
map = sp.zeros((n_chan, 15, 11))
map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec'))
map.set_axis_info('freq', centre, -delta)
map.set_axis_info('ra', 218, -0.2)
map.set_axis_info('dec', 2, 0.2)
algebra.save('./testout_clean_map_I_' + str(band) + '.npy', map)
self.params = {
'sm_input_root': 'testdata/',
'sm_file_middles': ("testfile", ),
'sm_input_end': "_guppi_rotated.fits",
'sm_output_root': "./testout_",
'sm_output_end': "_sub.fits",
'sm_solve_for_gain': True,
'sm_gain_output_end': 'gain.pickle',
'sm_map_input_root': './testout_',
'sm_map_type': 'clean_map_',
'sm_map_polarizations': ('I', ),
'sm_map_bands': bands
}
def setUp(self):
Reader = fitsGBT.Reader("./testdata/testfile_guppi_combined.fits",
feedback=0)
self.Blocks = Reader.read((), 1)
Data = self.Blocks[0]
Data.calc_freq()
params = {'dm_deweight_time_slope': True}
Maker = dirty_map.DirtyMapMaker(params, feedback=0)
n_chan = Data.dims[-1]
Maker.n_chan = n_chan
Maker.pols = (1, 2, 3, 4)
Maker.pol_ind = 0
Maker.band_centres = (Data.freq[Data.dims[-1] // 2], )
Maker.band_ind = 0
map = sp.zeros((Data.dims[-1], 32, 15))
map = al.make_vect(map, ('freq', 'ra', 'dec'))
map.set_axis_info('freq', Data.freq[Data.dims[-1] // 2],
Data.field['CRVAL1'])
map.set_axis_info('ra', 218, 0.075)
map.set_axis_info('dec', 2, 0.075)
Maker.map = map
self.Maker = Maker
# The variances of each channel.
self.norms = (sp.arange(1., 2., 0.25)[:, None] *
(sp.arange(1., 2., 1. / n_chan)[None, :]))
for Data in self.Blocks:
Data.data[...] = random.randn(*Data.data.shape)
Data.data *= sp.sqrt(self.norms[:, None, :])
Data.data += 50.
def setUp(self) :
Reader = fitsGBT.Reader(test_file, feedback=0)
blocks = Reader.read((),())
for Data in blocks :
rotate_pol.rotate(Data, (1,2,3,4))
Writer = fitsGBT.Writer(blocks, feedback=0)
Writer.write('test_rot.testout.fits')
self.params = {'pc_file_middles' : ("test",),
'pc_input_end' : "_rot.testout.fits",
'pc_output_end' : "_polcal.testout.fits",
'pc_mueler_file' : test_mueler_file
}
def setUp(self) :
Reader = fitsGBT.Reader(test_filename, 0)
self.Data = Reader.read(0,0)
self.Data.calc_freq()
calT = self.Data.freq[sp.newaxis,sp.newaxis,sp.newaxis,:]
calT = sp.concatenate((calT, 5*calT), 1)
self.CalData = data_block.DataBlock(calT)
for field in ('CRVAL1', 'CDELT1', 'CRPIX1'):
self.CalData.set_field(field, self.Data.field[field],
self.Data.field_axes[field], self.Data.field_formats[field])
self.CalData.set_field('CAL', ['R'], ('cal',), '1A')
self.CalData.set_field('CRVAL4', [-5, -6], ('pol',), '1I')
self.CalData.verify()
def setUp(self):
Reader = fitsGBT.Reader(test_file, feedback=0)
self.blocks = Reader.read((), ())
b = self.blocks[0]
b.calc_freq()
map = sp.zeros((150, 10, 11))
map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec'))
map.set_axis_info('freq', 700.0e6, -2.0e6)
map.set_axis_info('ra', 325.6, -0.2)
map.set_axis_info('dec', 0, 0.2)
self.map = map
for Data in self.blocks:
rotate_pol.rotate(Data, (1, 2, 3, 4))
def __init__(self, parameter_file_or_dict=None, feedback=2):
# Call the base_single init.
base_single.BaseSingle.__init__(self, parameter_file_or_dict, feedback)
# Read in calibration table.
file_names = self.params['cal_temperature_files']
if len(file_names) > 1:
raise NotImplementedError('Can use a single calibration file'
' at a time.')
Reader = fitsGBT.Reader(file_names[0], feedback=self.feedback)
self.CalData = Reader.read([], [])
if type(self.CalData) is tuple:
raise ce.DataError('Expected calibration file to have only a'
' single scan and IF.')
def replace(in_dir, out_dir):
kiyopy.utils.mkdir_p(out_dir)
files = glob.glob(in_dir + '/*.fits')
data_dir = os.getenv('GBT_DATA')
for file in files:
out_fname = out_dir + file.split('/')[-1]
data_fname = data_dir + '/'.join(file.split('/')[-2:])
if not os.path.isfile(data_fname):
msg = "Raw data file: %s missing" % data_fname
raise ValueError(msg)
raw_Reader = fitsGBT.Reader(data_fname)
raw_data_list = raw_Reader.read()
Reader = fitsGBT.Reader(file)
data_list = Reader.read()
if len(raw_data_list) != len(data_list):
raise ValueError("Wrong number of scans in raw data.")
for ii in range(len(data_list)):
raw_data = raw_data_list[ii]
ra = raw_data.field['RA']
ra_format = raw_data.field_formats['RA']
ra_axes = raw_data.field_axes['RA']
data_list[ii].set_field('RA',
ra,
axis_names=ra_axes,
format=ra_format)
dec = raw_data.field['DEC']
dec_format = raw_data.field_formats['DEC']
dec_axes = raw_data.field_axes['DEC']
data_list[ii].set_field('DEC',
dec,
axis_names=dec_axes,
format=dec_format)
Writer = fitsGBT.Writer(data_list)
Writer.write(out_fname)
def setUp(self):
Reader = fitsGBT.Reader(test_file, feedback=0)
Data = Reader.read(0, 0)
Data.data[...] = 0
Data.calc_freq()
freq = Data.freq
n_chan = len(freq)
centre = freq[n_chan // 2]
delta = -abs(np.mean(np.diff(freq)))
map = np.zeros((n_chan, 20, 10))
map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec'))
map.set_axis_info('freq', centre, delta)
map.set_axis_info('ra', 0, -0.1)
map.set_axis_info('dec', 0, 0.1)
self.Data = Data
self.map = map
def load_moonscan(filename, rotate_moon=True):
cal_coords = ephem.Equatorial("05:42:36.155",
"+49:51:07.28",
epoch=ephem.B1950)
# convert cal to a Body object.
cal_source = ephem.FixedBody()
cal_source._ra = cal_coords.ra
cal_source._dec = cal_coords.dec
cal_source._epoch = cal_coords.epoch
Reader = fitsGBT.Reader(filename)
moon_dataobj = Reader.read(0, 0)
rotate_pol.rotate(moon_dataobj, (-5, -7, -8, -6))
cal_scale.scale_by_cal(moon_dataobj,
scale_t_ave=True,
scale_f_ave=False,
sub_med=False,
scale_f_ave_mod=False,
rotate=True)
flag_data.flag_data(moon_dataobj, 5, 0.1, 40)
rebin_freq.rebin(moon_dataobj, 16, True, True)
#rebin_time.rebin(moon_dataobj, 4)
if rotate_moon:
moon_rotation.rotate_pol_moon(moon_dataobj)
fgc_mueler_file = '/mnt/raid-project/gmrt/tcv/diff_gain_params/GBT12A_418/22_diff_gain_calc.txt'
fgc_RM_file = ' '
fgc_R_to_sky = True
fgc_DP_correct = False # this is already handled in scale_by_cal's rotate
fgc_RM_correct = False
from time_stream import flux_diff_gain_cal as fdg
m_total = fdg.flux_dg(fgc_mueler_file)
fdg.calibrate_pol(moon_dataobj, m_total, fgc_RM_file, fgc_R_to_sky,
fgc_DP_correct, fgc_RM_correct)
return moon_dataobj
def process_file(self, file_ind):
"""Process on file from the list to be processed based on the passed
index.
"""
self.file_ind = file_ind
params = self.params
file_middle = params['file_middles'][file_ind]
input_fname = (params['input_root'] + file_middle +
params['input_end'])
output_fname = (params['output_root'] + file_middle +
params['output_end'])
Writer = fitsGBT.Writer(feedback=self.feedback)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
if hasattr(self, 'feedback_title') and self.feedback > 1:
print self.feedback_title,
# Get the number of scans if asked for all of them.
scan_inds = params['scans']
if len(scan_inds) == 0 or scan_inds is None:
scan_inds = range(len(Reader.scan_set))
# Loop over scans.
for thisscan in scan_inds:
Blocks = Reader.read(thisscan, params['IFs'], force_tuple=True)
# Function that loops over DataBlocks within a scan.
NewBlocks = self.scan_action(Blocks)
del Blocks
Writer.add_data(NewBlocks)
# Go to a new line if we are printing statistics.
if hasattr(self, 'feedback_title') and self.feedback > 1:
print ''
# Finally write the data back to file.
utils.mkparents(output_fname)
Writer.write(output_fname)
def setUp(self):
test_file = 'testdata/testfile_guppi_rebinned.fits'
Reader = fitsGBT.Reader(test_file, feedback=0)
self.Data = Reader.read(0, 0)
brackets.append((lower_ind, upper_ind))
new_n = len(brackets)
out = np.zeros((new_n, ) + s, dtype=power.dtype)
for ii in range(new_n):
lower_ind = brackets[ii][0]
upper_ind = brackets[ii][1]
out[ii, ...] = np.mean(power[lower_ind:upper_ind, ...], 0)
return out
#### Analysis and plots ####
file_name = data_root + middle + end
# Initial read.
Reader = fitsGBT.Reader(file_name)
Data = Reader.read(0, 0)
# Preprocess.
rotate_pol.rotate(Data, (-5, -7, -8, -6))
cal_scale.scale_by_cal(Data, True, False, False, False, True)
data_preflag = Data.data.copy()
flag_data.flag_data(Data, 5, 0.1, 40)
data_postflag = Data.data.copy()
Data.calc_freq()
nu_full = Data.freq
rebin_freq.rebin(Data, 16, True, True)
#rotate_pol.rotate(Data, (1,))
combine_cal.combine(Data, (0., 1.), False, True)
data_preflag2 = Data.data.copy()
reflag.flag(Data,
source = '3C286'
# These files we will use to calibrate.
# Slow scans.
#cal_files = ['22_3C147_track_' + str(ii) for ii in range(27, 35)]
#cal_files = ['22_3C295_track_' + str(ii) for ii in range(59, 67)]
cal_files = ['21_3C286_track_' + str(ii) for ii in range(18, 26)]
# The following two loops is a standard set of things we do to our raw data
# when it comes from the telescope. Our data format is roughly SDfits.
# Read and preprocess the Data.
cal_Blocks = []
for fname in cal_files:
# Read.
fpath = data_root + fname + end
Reader = fitsGBT.Reader(fpath)
Data = Reader.read(0, 0)
cal_Blocks.append(Data)
for Data in cal_Blocks:
# Preprocess.
rotate_pol.rotate(Data, (-5, -7, -8, -6))
cal_scale.scale_by_cal(Data, True, False, False, False, True)
flag_data.flag_data(Data, 5, 0.1, 40)
#rebin_freq.rebin(Data, 16, True, True)
rebin_freq.rebin(Data, 16, True, True)
#combine_cal.combine(Data, (0.5, 0.5), False, True)
combine_cal.combine(Data, (0., 1.), False, True)
#rebin_time.rebin(Data, 4)
Data.calc_freq()
def setUp(self):
Reader = fitsGBT.Reader(test_file, feedback=0)
self.Data = Reader.read(1, 0)
rotate_pol.rotate(self.Data, (1, ))
self.DataSub = Reader.read(1, 0)
rotate_pol.rotate(self.DataSub, (1, ))
def setUp(self):
Reader = fitsGBT.Reader('testfile_GBTfits.fits', feedback=0)
self.data_blocks = Reader.read(0, ())
def setUp(self):
Reader = fitsGBT.Reader(test_file, feedback=0)
self.Data = Reader.read(1, 0)
def test_runs(self) :
ExampleProcessor(input_parameters, feedback=0).execute()
Reader = fitsGBT.Reader('temp_test_.fits', feedback=0)
Data = Reader.read(1, 1)
self.assertTrue(Data.history.has_key('001: Did Nothing.'))
self.assertEqual(Data.history['001: Did Nothing.'][0], 'nothing')
"""Script to run profile rebin_freq module."""
import os
import cProfile
import numpy.ma as ma
from core import fitsGBT
import time_stream.flag_data as flag_data
data_file_name = (os.getenv('GBT10B_OUT') + 'guppi_data/' +
'44_wigglez15hrst_ralongmap_297-304.fits')
out_file_name = 'flag_data.prof'
Reader = fitsGBT.Reader(data_file_name)
Data = Reader.read(5, 0)
#Data.data = ma.copy(Data.data, 'F')
cProfile.run('flag_data.apply_cuts(Data, 5, 5)', out_file_name)
scans_all==thescan))
inds_sif = list(inds_sif[times])
inds_total += inds_sif
inds_total.sort()
# Seems to be nessisary for fitsdata[inds] to be the right type
inds = sp.array(inds_total)
testhdulist[1].data = fitsdata[inds]
testhdulist.writeto(test_file_name)
#### A series of test data files created from guppi data.
guppi_file_name = (os.getenv('GBT_DATA') +
'/GBT10B_036/42_wigglez15hrst_ralongmap_230-237.fits')
Reader = fitsGBT.Reader(guppi_file_name)
Blocks = Reader.read((0,1), None)
for Data in Blocks:
rebin_freq.rebin(Data, 32, True, True)
rebin_time.rebin(Data, 2)
split_Blocks = ()
for Data in Blocks:
split_Blocks += split_bands.split(Data, 2, 32, 25)
comb_Blocks = copy.deepcopy(split_Blocks)
for Data in comb_Blocks:
combine_cal.combine(Data, sub_mean=False)
rot_Blocks = copy.deepcopy(comb_Blocks)
for Data in rot_Blocks:
def test_history(self):
smd.Subtract(self.params, feedback=0).execute()
Data = fitsGBT.Reader('./testout_testfile_sub.fits',
feedback=0).read(0, 0)
self.assertTrue(Data.history.has_key('002: Subtracted map from data.'))
def setUp(self):
self.Reader = fitsGBT.Reader(testfile, feedback=0)
self.Blocks = self.Reader.read(None, 0)
def setUp(self) :
Reader = fitsGBT.Reader(test_file, feedback=0)
self.blocks = Reader.read((),())
self.mueler = pol_cal.mueller()
def build_hitmap(file_middles, input_end, output_root, scans, IFs,
field_centre, map_shape, pixel_spacing):
# Rename some commonly used parameters.
ra_spacing = -spacing / sp.cos(field_centre[1] * sp.pi / 180.)
if len(IFs) != 1:
raise ce.FileParameterTypeError('Can only process a single IF.')
all_file_names = []
# Flag for the first block processed (will allowcate memory on the
# first iteration).
first_block = True
# Loop over the files to process.
try:
for file_middle in file_middles:
input_fname = (input_root + file_middle + input_end)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=feedback)
Blocks = Reader.read(scans, IFs)
# Calculate the time varience at each frequency. This will
# be used as weights in most algorithms.
for Data in Blocks:
dims = Data.dims
# On first pass set up the map parameters.
if first_block:
shape = map_shape + (dims[-1], )
Data.calc_freq()
centre_freq = Data.freq[dims[-1] // 2]
delta_freq = Data.field['CDELT1']
# Allocate memory for the map.
map_data = sp.zeros(shape, dtype=float)
map_data = algebra.make_vect(map_data,
axis_names=('ra', 'dec',
'freq'))
first_block = False
# Figure out the pointing pixel index and the frequency
# indicies.
Data.calc_pointing()
ra_inds = tools.calc_inds(Data.ra, field_centre[0], shape[0],
ra_spacing)
dec_inds = tools.calc_inds(Data.dec, field_centre[1], shape[1],
pixel_spacing)
data = Data.data[:, pol_ind, 0, :]
pixel_hits[:] = 0
pixel_list = pixel_counts(data,
ra_inds,
dec_inds,
pixel_hits,
map_shape=shape[0:2])
# End Blocks for loop.
# Free up all that memory and flush memory maps to file.
del map_data
# Save the file names for the history.
all_file_names.append(
kiyopy.utils.abbreviate_file_path(map_file_name))
except ce.NextIteration:
pass
# End polarization for loop.
history.add("Made dirty map.", all_file_names)
h_file_name = (output_root + 'history.hist')
history.write(h_file_name)
def process_file(self, file_ind):
params = self.params
file_middle = params['file_middles'][file_ind]
input_fname = (params['input_root'] + file_middle +
params['input_end'])
sub_input_fname = (params['subtracted_input_root'] + file_middle +
params['input_end'])
output_fname = (params['output_root'] + file_middle +
params['output_end'])
sub_output_fname = (params['subtracted_output_root'] + file_middle +
params['output_end'])
Writer = fitsGBT.Writer(feedback=self.feedback)
SubWriter = fitsGBT.Writer(feedback=self.feedback)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
SubReader = fitsGBT.Reader(sub_input_fname, feedback=self.feedback)
if (sp.any(Reader.scan_set != SubReader.scan_set)
or sp.any(Reader.IF_set != SubReader.IF_set)):
raise ce.DataError("IFs and scans don't match signal subtracted"
" data.")
# Get the number of scans if asked for all of them.
scan_inds = params['scans']
if len(scan_inds) == 0 or scan_inds is None:
scan_inds = range(len(Reader.scan_set))
if_inds = params['IFs']
if len(if_inds) == 0 or scan_inds is None:
if_inds = range(len(Reader.IF_set))
if self.feedback > 1:
print "New flags each block:",
# Loop over scans and IFs
for thisscan in scan_inds:
for thisIF in if_inds:
Data = Reader.read(thisscan, thisIF)
SubData = SubReader.read(thisscan, thisIF)
# Make sure they have agreeing masks to start.
SubData.data[ma.getmaskarray(Data.data)] = ma.masked
Data.data[ma.getmaskarray(SubData.data)] = ma.masked
# Get initial number of flags.
n_flags = ma.count_masked(Data.data)
# Now do the flagging.
flag(Data, SubData, params['thres'],
params['max_noise_factor'],
params['smooth_modes_subtract'], params['filter_type'])
Data.add_history(
"Reflaged for outliers.",
("Used file: " +
utils.abbreviate_file_path(sub_input_fname), ))
SubData.add_history("Reflaged for outliers.")
Writer.add_data(Data)
SubWriter.add_data(SubData)
# Report the number of new flags.
n_flags = ma.count_masked(Data.data) - n_flags
if self.feedback > 1:
print n_flags,
if self.feedback > 1:
print ''
# Finally write the data back to file.
utils.mkparents(output_fname)
utils.mkparents(sub_output_fname)
Writer.write(output_fname)
SubWriter.write(sub_output_fname)
def setUp(self):
Reader = fitsGBT.Reader(test_file, feedback=0)
self.Blocks = Reader.read((), 1)
