def bandpass_I_from_onoff(filename): reader = fitsGBT.Reader(f) scans = reader.read() scans[0].calc_freq() freq = scans[0].freq src = scans[0].field['OBJECT'] on_spectrum = I_in_cal_units(scans[0]) off_spectrum = I_in_cal_units(scans[1]) src_spectrum = on_spectrum - off_spectrum src_nominal = calibrator_spectrum(src, freq) cal_spectrum = src_nominal / src_spectrum out = np.empty(len(freq), dtype=[('freq', np.float64), ('cal_T', np.float64)]) out['freq'] = freq out['cal_T'] = cal_spectrum #plt.figure() #plt.plot(freq, on_spectrum) #plt.plot(freq, off_spectrum) #plt.figure() #plt.plot(freq, cal_spectrum) return out
def execute(self): """Process all data.""" # You have access to the input parameters through the dictionary # self.params. params = self.params # If you have output files, make parent directories if need be. utils.mkparents(params['output_root']) # Write the input parameters to file so you can go back and look at # them. parse_ini.write_params(params, params['output_root'] + 'params.ini', prefix=self.prefix) # Loop over the files to process. for file_middle in params['file_middles']: input_fname = (params['input_root'] + file_middle + params['input_end']) # Read in the data. The reader is an object that can read # DataBlock objects out of a fits file. Reader = fitsGBT.Reader(input_fname, feedback=self.feedback) # Some examples of how you would read DataBlock Objects: first_scan_and_IF_DataBlock = Reader.read(scans=0, IFs=0) second_scan_and_first_IF_DataBlock = Reader.read(scans=1, IFs=0) list_of_a_few_data_blocks = Reader.read(scans=(1, 2, 3), IFs=0) list_of_all_data_blocks = Reader.read(scans=(), IFs=())
def setUp(self): # Read in just to fiugre out the band structure. this_test_file = 'testdata/testfile_guppi_rotated.fits' Reader = fitsGBT.Reader(this_test_file, feedback=0) Blocks = Reader.read((0, ), ()) bands = () for Data in Blocks: n_chan = Data.dims[3] Data.calc_freq() freq = Data.freq delta = abs(sp.mean(sp.diff(freq))) centre = freq[n_chan // 2] band = int(centre / 1e6) bands += (band, ) map = sp.zeros((n_chan, 15, 11)) map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec')) map.set_axis_info('freq', centre, -delta) map.set_axis_info('ra', 218, -0.2) map.set_axis_info('dec', 2, 0.2) algebra.save('./testout_clean_map_I_' + str(band) + '.npy', map) self.params = { 'sm_input_root': 'testdata/', 'sm_file_middles': ("testfile", ), 'sm_input_end': "_guppi_rotated.fits", 'sm_output_root': "./testout_", 'sm_output_end': "_sub.fits", 'sm_solve_for_gain': True, 'sm_gain_output_end': 'gain.pickle', 'sm_map_input_root': './testout_', 'sm_map_type': 'clean_map_', 'sm_map_polarizations': ('I', ), 'sm_map_bands': bands }
def setUp(self): Reader = fitsGBT.Reader("./testdata/testfile_guppi_combined.fits", feedback=0) self.Blocks = Reader.read((), 1) Data = self.Blocks[0] Data.calc_freq() params = {'dm_deweight_time_slope': True} Maker = dirty_map.DirtyMapMaker(params, feedback=0) n_chan = Data.dims[-1] Maker.n_chan = n_chan Maker.pols = (1, 2, 3, 4) Maker.pol_ind = 0 Maker.band_centres = (Data.freq[Data.dims[-1] // 2], ) Maker.band_ind = 0 map = sp.zeros((Data.dims[-1], 32, 15)) map = al.make_vect(map, ('freq', 'ra', 'dec')) map.set_axis_info('freq', Data.freq[Data.dims[-1] // 2], Data.field['CRVAL1']) map.set_axis_info('ra', 218, 0.075) map.set_axis_info('dec', 2, 0.075) Maker.map = map self.Maker = Maker # The variances of each channel. self.norms = (sp.arange(1., 2., 0.25)[:, None] * (sp.arange(1., 2., 1. / n_chan)[None, :])) for Data in self.Blocks: Data.data[...] = random.randn(*Data.data.shape) Data.data *= sp.sqrt(self.norms[:, None, :]) Data.data += 50.
def setUp(self) : Reader = fitsGBT.Reader(test_file, feedback=0) blocks = Reader.read((),()) for Data in blocks : rotate_pol.rotate(Data, (1,2,3,4)) Writer = fitsGBT.Writer(blocks, feedback=0) Writer.write('test_rot.testout.fits') self.params = {'pc_file_middles' : ("test",), 'pc_input_end' : "_rot.testout.fits", 'pc_output_end' : "_polcal.testout.fits", 'pc_mueler_file' : test_mueler_file }
def setUp(self) : Reader = fitsGBT.Reader(test_filename, 0) self.Data = Reader.read(0,0) self.Data.calc_freq() calT = self.Data.freq[sp.newaxis,sp.newaxis,sp.newaxis,:] calT = sp.concatenate((calT, 5*calT), 1) self.CalData = data_block.DataBlock(calT) for field in ('CRVAL1', 'CDELT1', 'CRPIX1'): self.CalData.set_field(field, self.Data.field[field], self.Data.field_axes[field], self.Data.field_formats[field]) self.CalData.set_field('CAL', ['R'], ('cal',), '1A') self.CalData.set_field('CRVAL4', [-5, -6], ('pol',), '1I') self.CalData.verify()
def setUp(self): Reader = fitsGBT.Reader(test_file, feedback=0) self.blocks = Reader.read((), ()) b = self.blocks[0] b.calc_freq() map = sp.zeros((150, 10, 11)) map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec')) map.set_axis_info('freq', 700.0e6, -2.0e6) map.set_axis_info('ra', 325.6, -0.2) map.set_axis_info('dec', 0, 0.2) self.map = map for Data in self.blocks: rotate_pol.rotate(Data, (1, 2, 3, 4))
def __init__(self, parameter_file_or_dict=None, feedback=2): # Call the base_single init. base_single.BaseSingle.__init__(self, parameter_file_or_dict, feedback) # Read in calibration table. file_names = self.params['cal_temperature_files'] if len(file_names) > 1: raise NotImplementedError('Can use a single calibration file' ' at a time.') Reader = fitsGBT.Reader(file_names[0], feedback=self.feedback) self.CalData = Reader.read([], []) if type(self.CalData) is tuple: raise ce.DataError('Expected calibration file to have only a' ' single scan and IF.')
def replace(in_dir, out_dir): kiyopy.utils.mkdir_p(out_dir) files = glob.glob(in_dir + '/*.fits') data_dir = os.getenv('GBT_DATA') for file in files: out_fname = out_dir + file.split('/')[-1] data_fname = data_dir + '/'.join(file.split('/')[-2:]) if not os.path.isfile(data_fname): msg = "Raw data file: %s missing" % data_fname raise ValueError(msg) raw_Reader = fitsGBT.Reader(data_fname) raw_data_list = raw_Reader.read() Reader = fitsGBT.Reader(file) data_list = Reader.read() if len(raw_data_list) != len(data_list): raise ValueError("Wrong number of scans in raw data.") for ii in range(len(data_list)): raw_data = raw_data_list[ii] ra = raw_data.field['RA'] ra_format = raw_data.field_formats['RA'] ra_axes = raw_data.field_axes['RA'] data_list[ii].set_field('RA', ra, axis_names=ra_axes, format=ra_format) dec = raw_data.field['DEC'] dec_format = raw_data.field_formats['DEC'] dec_axes = raw_data.field_axes['DEC'] data_list[ii].set_field('DEC', dec, axis_names=dec_axes, format=dec_format) Writer = fitsGBT.Writer(data_list) Writer.write(out_fname)
def setUp(self): Reader = fitsGBT.Reader(test_file, feedback=0) Data = Reader.read(0, 0) Data.data[...] = 0 Data.calc_freq() freq = Data.freq n_chan = len(freq) centre = freq[n_chan // 2] delta = -abs(np.mean(np.diff(freq))) map = np.zeros((n_chan, 20, 10)) map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec')) map.set_axis_info('freq', centre, delta) map.set_axis_info('ra', 0, -0.1) map.set_axis_info('dec', 0, 0.1) self.Data = Data self.map = map
def load_moonscan(filename, rotate_moon=True): cal_coords = ephem.Equatorial("05:42:36.155", "+49:51:07.28", epoch=ephem.B1950) # convert cal to a Body object. cal_source = ephem.FixedBody() cal_source._ra = cal_coords.ra cal_source._dec = cal_coords.dec cal_source._epoch = cal_coords.epoch Reader = fitsGBT.Reader(filename) moon_dataobj = Reader.read(0, 0) rotate_pol.rotate(moon_dataobj, (-5, -7, -8, -6)) cal_scale.scale_by_cal(moon_dataobj, scale_t_ave=True, scale_f_ave=False, sub_med=False, scale_f_ave_mod=False, rotate=True) flag_data.flag_data(moon_dataobj, 5, 0.1, 40) rebin_freq.rebin(moon_dataobj, 16, True, True) #rebin_time.rebin(moon_dataobj, 4) if rotate_moon: moon_rotation.rotate_pol_moon(moon_dataobj) fgc_mueler_file = '/mnt/raid-project/gmrt/tcv/diff_gain_params/GBT12A_418/22_diff_gain_calc.txt' fgc_RM_file = ' ' fgc_R_to_sky = True fgc_DP_correct = False # this is already handled in scale_by_cal's rotate fgc_RM_correct = False from time_stream import flux_diff_gain_cal as fdg m_total = fdg.flux_dg(fgc_mueler_file) fdg.calibrate_pol(moon_dataobj, m_total, fgc_RM_file, fgc_R_to_sky, fgc_DP_correct, fgc_RM_correct) return moon_dataobj
def process_file(self, file_ind): """Process on file from the list to be processed based on the passed index. """ self.file_ind = file_ind params = self.params file_middle = params['file_middles'][file_ind] input_fname = (params['input_root'] + file_middle + params['input_end']) output_fname = (params['output_root'] + file_middle + params['output_end']) Writer = fitsGBT.Writer(feedback=self.feedback) # Read in the data, and loop over data blocks. Reader = fitsGBT.Reader(input_fname, feedback=self.feedback) if hasattr(self, 'feedback_title') and self.feedback > 1: print self.feedback_title, # Get the number of scans if asked for all of them. scan_inds = params['scans'] if len(scan_inds) == 0 or scan_inds is None: scan_inds = range(len(Reader.scan_set)) # Loop over scans. for thisscan in scan_inds: Blocks = Reader.read(thisscan, params['IFs'], force_tuple=True) # Function that loops over DataBlocks within a scan. NewBlocks = self.scan_action(Blocks) del Blocks Writer.add_data(NewBlocks) # Go to a new line if we are printing statistics. if hasattr(self, 'feedback_title') and self.feedback > 1: print '' # Finally write the data back to file. utils.mkparents(output_fname) Writer.write(output_fname)
def setUp(self): test_file = 'testdata/testfile_guppi_rebinned.fits' Reader = fitsGBT.Reader(test_file, feedback=0) self.Data = Reader.read(0, 0)
brackets.append((lower_ind, upper_ind)) new_n = len(brackets) out = np.zeros((new_n, ) + s, dtype=power.dtype) for ii in range(new_n): lower_ind = brackets[ii][0] upper_ind = brackets[ii][1] out[ii, ...] = np.mean(power[lower_ind:upper_ind, ...], 0) return out #### Analysis and plots #### file_name = data_root + middle + end # Initial read. Reader = fitsGBT.Reader(file_name) Data = Reader.read(0, 0) # Preprocess. rotate_pol.rotate(Data, (-5, -7, -8, -6)) cal_scale.scale_by_cal(Data, True, False, False, False, True) data_preflag = Data.data.copy() flag_data.flag_data(Data, 5, 0.1, 40) data_postflag = Data.data.copy() Data.calc_freq() nu_full = Data.freq rebin_freq.rebin(Data, 16, True, True) #rotate_pol.rotate(Data, (1,)) combine_cal.combine(Data, (0., 1.), False, True) data_preflag2 = Data.data.copy() reflag.flag(Data,
source = '3C286' # These files we will use to calibrate. # Slow scans. #cal_files = ['22_3C147_track_' + str(ii) for ii in range(27, 35)] #cal_files = ['22_3C295_track_' + str(ii) for ii in range(59, 67)] cal_files = ['21_3C286_track_' + str(ii) for ii in range(18, 26)] # The following two loops is a standard set of things we do to our raw data # when it comes from the telescope. Our data format is roughly SDfits. # Read and preprocess the Data. cal_Blocks = [] for fname in cal_files: # Read. fpath = data_root + fname + end Reader = fitsGBT.Reader(fpath) Data = Reader.read(0, 0) cal_Blocks.append(Data) for Data in cal_Blocks: # Preprocess. rotate_pol.rotate(Data, (-5, -7, -8, -6)) cal_scale.scale_by_cal(Data, True, False, False, False, True) flag_data.flag_data(Data, 5, 0.1, 40) #rebin_freq.rebin(Data, 16, True, True) rebin_freq.rebin(Data, 16, True, True) #combine_cal.combine(Data, (0.5, 0.5), False, True) combine_cal.combine(Data, (0., 1.), False, True) #rebin_time.rebin(Data, 4) Data.calc_freq()
def setUp(self): Reader = fitsGBT.Reader(test_file, feedback=0) self.Data = Reader.read(1, 0) rotate_pol.rotate(self.Data, (1, )) self.DataSub = Reader.read(1, 0) rotate_pol.rotate(self.DataSub, (1, ))
def setUp(self): Reader = fitsGBT.Reader('testfile_GBTfits.fits', feedback=0) self.data_blocks = Reader.read(0, ())
def setUp(self): Reader = fitsGBT.Reader(test_file, feedback=0) self.Data = Reader.read(1, 0)
def test_runs(self) : ExampleProcessor(input_parameters, feedback=0).execute() Reader = fitsGBT.Reader('temp_test_.fits', feedback=0) Data = Reader.read(1, 1) self.assertTrue(Data.history.has_key('001: Did Nothing.')) self.assertEqual(Data.history['001: Did Nothing.'][0], 'nothing')
"""Script to run profile rebin_freq module.""" import os import cProfile import numpy.ma as ma from core import fitsGBT import time_stream.flag_data as flag_data data_file_name = (os.getenv('GBT10B_OUT') + 'guppi_data/' + '44_wigglez15hrst_ralongmap_297-304.fits') out_file_name = 'flag_data.prof' Reader = fitsGBT.Reader(data_file_name) Data = Reader.read(5, 0) #Data.data = ma.copy(Data.data, 'F') cProfile.run('flag_data.apply_cuts(Data, 5, 5)', out_file_name)
scans_all==thescan)) inds_sif = list(inds_sif[times]) inds_total += inds_sif inds_total.sort() # Seems to be nessisary for fitsdata[inds] to be the right type inds = sp.array(inds_total) testhdulist[1].data = fitsdata[inds] testhdulist.writeto(test_file_name) #### A series of test data files created from guppi data. guppi_file_name = (os.getenv('GBT_DATA') + '/GBT10B_036/42_wigglez15hrst_ralongmap_230-237.fits') Reader = fitsGBT.Reader(guppi_file_name) Blocks = Reader.read((0,1), None) for Data in Blocks: rebin_freq.rebin(Data, 32, True, True) rebin_time.rebin(Data, 2) split_Blocks = () for Data in Blocks: split_Blocks += split_bands.split(Data, 2, 32, 25) comb_Blocks = copy.deepcopy(split_Blocks) for Data in comb_Blocks: combine_cal.combine(Data, sub_mean=False) rot_Blocks = copy.deepcopy(comb_Blocks) for Data in rot_Blocks:
def test_history(self): smd.Subtract(self.params, feedback=0).execute() Data = fitsGBT.Reader('./testout_testfile_sub.fits', feedback=0).read(0, 0) self.assertTrue(Data.history.has_key('002: Subtracted map from data.'))
def setUp(self): self.Reader = fitsGBT.Reader(testfile, feedback=0) self.Blocks = self.Reader.read(None, 0)
def setUp(self) : Reader = fitsGBT.Reader(test_file, feedback=0) self.blocks = Reader.read((),()) self.mueler = pol_cal.mueller()
def build_hitmap(file_middles, input_end, output_root, scans, IFs, field_centre, map_shape, pixel_spacing): # Rename some commonly used parameters. ra_spacing = -spacing / sp.cos(field_centre[1] * sp.pi / 180.) if len(IFs) != 1: raise ce.FileParameterTypeError('Can only process a single IF.') all_file_names = [] # Flag for the first block processed (will allowcate memory on the # first iteration). first_block = True # Loop over the files to process. try: for file_middle in file_middles: input_fname = (input_root + file_middle + input_end) # Read in the data, and loop over data blocks. Reader = fitsGBT.Reader(input_fname, feedback=feedback) Blocks = Reader.read(scans, IFs) # Calculate the time varience at each frequency. This will # be used as weights in most algorithms. for Data in Blocks: dims = Data.dims # On first pass set up the map parameters. if first_block: shape = map_shape + (dims[-1], ) Data.calc_freq() centre_freq = Data.freq[dims[-1] // 2] delta_freq = Data.field['CDELT1'] # Allocate memory for the map. map_data = sp.zeros(shape, dtype=float) map_data = algebra.make_vect(map_data, axis_names=('ra', 'dec', 'freq')) first_block = False # Figure out the pointing pixel index and the frequency # indicies. Data.calc_pointing() ra_inds = tools.calc_inds(Data.ra, field_centre[0], shape[0], ra_spacing) dec_inds = tools.calc_inds(Data.dec, field_centre[1], shape[1], pixel_spacing) data = Data.data[:, pol_ind, 0, :] pixel_hits[:] = 0 pixel_list = pixel_counts(data, ra_inds, dec_inds, pixel_hits, map_shape=shape[0:2]) # End Blocks for loop. # Free up all that memory and flush memory maps to file. del map_data # Save the file names for the history. all_file_names.append( kiyopy.utils.abbreviate_file_path(map_file_name)) except ce.NextIteration: pass # End polarization for loop. history.add("Made dirty map.", all_file_names) h_file_name = (output_root + 'history.hist') history.write(h_file_name)
def process_file(self, file_ind): params = self.params file_middle = params['file_middles'][file_ind] input_fname = (params['input_root'] + file_middle + params['input_end']) sub_input_fname = (params['subtracted_input_root'] + file_middle + params['input_end']) output_fname = (params['output_root'] + file_middle + params['output_end']) sub_output_fname = (params['subtracted_output_root'] + file_middle + params['output_end']) Writer = fitsGBT.Writer(feedback=self.feedback) SubWriter = fitsGBT.Writer(feedback=self.feedback) # Read in the data, and loop over data blocks. Reader = fitsGBT.Reader(input_fname, feedback=self.feedback) SubReader = fitsGBT.Reader(sub_input_fname, feedback=self.feedback) if (sp.any(Reader.scan_set != SubReader.scan_set) or sp.any(Reader.IF_set != SubReader.IF_set)): raise ce.DataError("IFs and scans don't match signal subtracted" " data.") # Get the number of scans if asked for all of them. scan_inds = params['scans'] if len(scan_inds) == 0 or scan_inds is None: scan_inds = range(len(Reader.scan_set)) if_inds = params['IFs'] if len(if_inds) == 0 or scan_inds is None: if_inds = range(len(Reader.IF_set)) if self.feedback > 1: print "New flags each block:", # Loop over scans and IFs for thisscan in scan_inds: for thisIF in if_inds: Data = Reader.read(thisscan, thisIF) SubData = SubReader.read(thisscan, thisIF) # Make sure they have agreeing masks to start. SubData.data[ma.getmaskarray(Data.data)] = ma.masked Data.data[ma.getmaskarray(SubData.data)] = ma.masked # Get initial number of flags. n_flags = ma.count_masked(Data.data) # Now do the flagging. flag(Data, SubData, params['thres'], params['max_noise_factor'], params['smooth_modes_subtract'], params['filter_type']) Data.add_history( "Reflaged for outliers.", ("Used file: " + utils.abbreviate_file_path(sub_input_fname), )) SubData.add_history("Reflaged for outliers.") Writer.add_data(Data) SubWriter.add_data(SubData) # Report the number of new flags. n_flags = ma.count_masked(Data.data) - n_flags if self.feedback > 1: print n_flags, if self.feedback > 1: print '' # Finally write the data back to file. utils.mkparents(output_fname) utils.mkparents(sub_output_fname) Writer.write(output_fname) SubWriter.write(sub_output_fname)
def setUp(self): Reader = fitsGBT.Reader(test_file, feedback=0) self.Blocks = Reader.read((), 1)