def write(self, file_name) :
"""Write stored data to file.
Take all the data stored in the Writer (from added DataBlocks) and
write it to a fits file with the passed file name.
"""
# Add the data
Col = pyfits.Column(name='DATA', format=self.data_format,
array=self.data)
columns = [Col,]
# Add all the other stored fields.
for field_name in self.field.iterkeys() :
Col = pyfits.Column(name=field_name,
format=self.formats[field_name],
array=self.field[field_name])
columns.append(Col)
coldefs = pyfits.ColDefs(columns)
# Creat fits header data units, one for the table and the mandatory
# primary.
tbhdu = pyfits.new_table(coldefs)
prihdu = pyfits.PrimaryHDU()
# Add the write history.
fname_abbr = ku.abbreviate_file_path(file_name)
self.history.add('Written to file.', ('File name: ' + fname_abbr,))
# Add the history to the header.
bf.write_history_header(prihdu.header, self.history)
# Combine the HDUs and write to file.
hdulist = pyfits.HDUList([prihdu, tbhdu])
hdulist.writeto(file_name, clobber=True)
if self.feedback > 0 :
print 'Wrote data to file: ' + fname_abbr
def write(self, file_name) :
"""Write stored data to file.
Take all the data stored in the Writer (from added DataBlocks) and
write it to a fits file with the passed file name.
"""
# Add the data
Col = pyfits.Column(name='DATA', format=self.data_format,
array=self.data)
columns = [Col,]
# Add all the other stored fields.
for field_name in self.field.iterkeys() :
Col = pyfits.Column(name=field_name,
format=self.formats[field_name],
array=self.field[field_name])
columns.append(Col)
coldefs = pyfits.ColDefs(columns)
# Creat fits header data units, one for the table and the mandatory
# primary.
tbhdu = pyfits.new_table(coldefs)
prihdu = pyfits.PrimaryHDU()
# Add the write history.
fname_abbr = ku.abbreviate_file_path(file_name)
self.history.add('Written to file.', ('File name: ' + fname_abbr,))
# Add the history to the header.
bf.write_history_header(prihdu.header, self.history)
# Combine the HDUs and write to file.
hdulist = pyfits.HDUList([prihdu, tbhdu])
hdulist.writeto(file_name, clobber=True)
if self.feedback > 0 :
print 'Wrote data to file: ' + fname_abbr
def process_file(self, file_ind) :
params = self.params
file_middle = params['file_middles'][file_ind]
input_fname = (params['input_root'] + file_middle +
params['input_end'])
sub_input_fname = (params['subtracted_input_root'] + file_middle
+ params['input_end'])
output_fname = (params['output_root']
+ file_middle + params['output_end'])
sub_output_fname = (params['subtracted_output_root']
+ file_middle + params['output_end'])
Writer = fitsGBT.Writer(feedback=self.feedback)
SubWriter = fitsGBT.Writer(feedback=self.feedback)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
SubReader = fitsGBT.Reader(sub_input_fname, feedback=self.feedback)
if (sp.any(Reader.scan_set != SubReader.scan_set)
or sp.any(Reader.IF_set != SubReader.IF_set)) :
raise ce.DataError("IFs and scans don't match signal subtracted"
" data.")
# Get the number of scans if asked for all of them.
scan_inds = params['scans']
if len(scan_inds) == 0 or scan_inds is None :
scan_inds = range(len(Reader.scan_set))
if_inds = params['IFs']
if len(if_inds) == 0 or scan_inds is None :
if_inds = range(len(Reader.IF_set))
if self.feedback > 1 :
print "New flags each block:",
# Loop over scans and IFs
for thisscan in scan_inds :
for thisIF in if_inds :
Data = Reader.read(thisscan, thisIF)
SubData = SubReader.read(thisscan, thisIF)
n_flags = ma.count_masked(Data.data)
# Now do the flagging.
flag(Data, SubData, params['thres'])
Data.add_history("Reflaged for outliers.", ("Used file: "
+ utils.abbreviate_file_path(sub_input_fname),))
SubData.add_history("Reflaged for outliers.")
Writer.add_data(Data)
SubWriter.add_data(SubData)
# Report the numbe of new flags.
n_flags = ma.count_masked(Data.data) - n_flags
if self.feedback > 1 :
print n_flags,
if self.feedback > 1 :
print ''
# Finally write the data back to file.
utils.mkparents(output_fname)
utils.mkparents(sub_output_fname)
Writer.write(output_fname)
SubWriter.write(sub_output_fname)
def action(self, Data) :
if (not self.params['solve_for_gain'] or
self.params['gain_output_end'] is '') :
sub_map(Data, self.Map, self.params['solve_for_gain'])
else :
block_gain = {}
Data.calc_freq()
block_gain['freq'] = sp.copy(Data.freq)
block_gain['time'] = Data.field['DATE-OBS'][0]
block_gain['scan'] = Data.field['SCAN']
block_gain['gain'] = sub_map(Data, self.Map, True)
self.gain_list.append(block_gain)
Data.add_history('Subtracted map from data.',
('Map file: ' + ku.abbreviate_file_path(self.params['map_file']),))
return Data
def __init__(self, fname, feedback=2, checking=1) :
"""Init script for the fitsGBT Reader class.
The reader is initialised with the fits file name to be read.
Optionally, one can supply up to two integers: feedback (default 2)
indicating how much junk to print, and checking (default 1) specifying
to what level the input file is checked for sensible data.
"""
self.feedback = feedback
self.checking = checking
if checking > 0 :
self.verify_ordering = 1
else :
self.verify_ordering = 0
self.fname = fname
# The passed file name is assumed to be a GBT spectrometer fits file.
self.hdulist = pyfits.open(self.fname, 'readonly')
if len(self.hdulist) < 2 :
raise ce.DataError("File missing data extension")
if self.feedback > 0 :
print "Opened GBT fits file: ", ku.abbreviate_file_path(fname)
# Separate in to the useful sub objects. These assignments are all
# done by reference, so this is efficient.
self.fitsdata = self.hdulist[1].data
# The records in fitsdata are not guaranteed to be in proper order.
# Mostly the IFs are all out of whack. However, once you isolate an
# IF everything should be well ordered.
# Get the scans and IF of all records. This is later used to isolate a
# single IF and scan. Also get the set of unique IFs and scans, so we
# know what is in the file.
self._scans_all = self.fitsdata.field('SCAN')
self.scan_set = sp.unique(self._scans_all)
# Sort scans for reapeatable ordering.
self.scan_set.sort()
self._IFs_all = self.fitsdata.field('CRVAL1')/1E6 # MHz
# Round the frequencies as we only need to tell the difference between
# one IF and the other.
self._IFs_all = self._IFs_all.round(0)
self.IF_set = sp.unique(self._IFs_all)
self.IF_set.sort()
def __init__(self, fname, feedback=2, checking=1, memmap=False) :
"""Init script for the fitsGBT Reader class.
The reader is initialised with the fits file name to be read.
Optionally, one can supply up to two integers: feedback (default 2)
indicating how much junk to print, and checking (default 1) specifying
to what level the input file is checked for sensible data.
"""
self.feedback = feedback
self.checking = checking
if checking > 0 :
self.verify_ordering = 1
else :
self.verify_ordering = 0
self.fname = fname
# The passed file name is assumed to be a GBT spectrometer fits file.
self.hdulist = pyfits.open(self.fname, 'readonly', memmap=memmap)
if len(self.hdulist) < 2 :
raise ce.DataError("File missing data extension")
if self.feedback > 0 :
print "Opened GBT fits file: ", ku.abbreviate_file_path(fname)
# Separate in to the useful sub objects. These assignments are all
# done by reference, so this is efficient.
self.fitsdata = self.hdulist[1].data
# The records in fitsdata are not guaranteed to be in proper order.
# Mostly the IFs are all out of whack. However, once you isolate an
# IF everything should be well ordered.
# Get the scans and IF of all records. This is later used to isolate a
# single IF and scan. Also get the set of unique IFs and scans, so we
# know what is in the file.
self._scans_all = self.fitsdata.field('SCAN')
self.scan_set = sp.unique(self._scans_all)
# Sort scans for reapeatable ordering.
self.scan_set.sort()
self._IFs_all = self.fitsdata.field('CRVAL1')/1E6 # MHz
# Round the frequencies as we only need to tell the difference between
# one IF and the other.
self._IFs_all = self._IFs_all.round(0)
self.IF_set = sp.unique(self._IFs_all)
self.IF_set.sort()
def execute(self, nprocesses=1) :
params = self.params
model = params["model"]
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params, params['output_root'] + 'params.ini',
prefix=prefix)
# Loop over files to process.
for file_middle in params['file_middles'] :
input_fname = (params['input_root'] + file_middle +
params['input_end'])
Reader = core.fitsGBT.Reader(input_fname, feedback=self.feedback)
output_fname = params["output_root"] + file_middle + ".npy"
if model == "scan_var" :
n_scans = len(Reader.scan_set)
n_IFs = len(Reader.IF_set)
first_block = True
for jj in range(n_IFs) :
# These all become arrays on the first iteration.
var = 0.0
mean = 0.0
counts = 0
for ii in range(n_scans) :
Data = Reader.read(ii, jj)
if first_block :
out_shape = (n_IFs,) + Data.dims[1:]
out_arr = sp.empty(out_shape, dtype=float)
first_block = False
var += ma.sum(Data.data**2, 0).filled(0)
mean += ma.sum(Data.data, 0).filled(0)
counts += ma.count(Data.data, 0)
# If we didn't get at least 5 good hits, throw aways the
# scan.
counts[counts < 5] = -1
var = var/counts - (mean/counts)**2
var[counts < 5] = 1.0e10
out_arr[jj, ...] = var
sp.save(output_fname, out_arr)
if self.feedback > 1 :
print ("Wrote noise parameters to file: "
+ utils.abbreviate_file_path(output_fname))
else :
raise ValueError("Invalid noise model: " + model)
def read(file_name, map_inds=None, feedback=2):
"""Read a map from an image fits file.
"""
fname_abbr = ku.abbreviate_file_path(file_name)
if feedback > 0:
print 'Opening file: ' + fname_abbr
# Open the fits file.
hdulist = pyfits.open(file_name)
history = bf.get_history_header(hdulist[0].header)
history.add('Read from file.', 'File name: ' + fname_abbr)
map_list = []
if map_inds is None:
map_inds = range(1, len(hdulist))
elif not hasattr(map_inds, '__iter__'):
map_inds = (map_inds, )
elif len(scans) == 0:
map_inds = range(1, len(hdulist))
for ii in map_inds:
data = hdulist[ii].data
# Set the data attriute
Map = data_map.DataMap()
Map.set_data(sp.swapaxes(data, 0, 2))
# Masked data is stored in FITS files as float('nan')
Map.data[sp.logical_not(sp.isfinite(Map.data))] = ma.masked
Map.history = history
# Set the other fields.
for field_name in fields:
if not field_name in hdulist[1].header.keys():
continue
value = hdulist[1].header[field_name]
Map.set_field(field_name, value)
map_list.append(Map)
if len(map_list) == 1:
map_list = map_list[0]
return map_list
def read(file_name, map_inds=None, feedback=2):
"""Read a map from an image fits file.
"""
fname_abbr = ku.abbreviate_file_path(file_name)
if feedback > 0:
print "Opening file: " + fname_abbr
# Open the fits file.
hdulist = pyfits.open(file_name)
history = bf.get_history_header(hdulist[0].header)
history.add("Read from file.", "File name: " + fname_abbr)
map_list = []
if map_inds is None:
map_inds = range(1, len(hdulist))
elif not hasattr(map_inds, "__iter__"):
map_inds = (map_inds,)
elif len(scans) == 0:
map_inds = range(1, len(hdulist))
for ii in map_inds:
data = hdulist[ii].data
# Set the data attriute
Map = data_map.DataMap()
Map.set_data(sp.swapaxes(data, 0, 2))
# Masked data is stored in FITS files as float('nan')
Map.data[sp.logical_not(sp.isfinite(Map.data))] = ma.masked
Map.history = history
# Set the other fields.
for field_name in fields:
if not field_name in hdulist[1].header.keys():
continue
value = hdulist[1].header[field_name]
Map.set_field(field_name, value)
map_list.append(Map)
if len(map_list) == 1:
map_list = map_list[0]
return map_list
def action(self, Data):
# Figure out what map band this corresponds to.
Data.calc_freq()
freq = Data.freq
# We are going to look for an exact match in for the map frequencies.
# This could be made more general since the sub_map function can handle
# partial overlap, but this will be fine for now.
for band_maps in self.maps:
maps_freq = band_maps[0].get_axis('freq')
if sp.allclose(maps_freq, freq):
maps = band_maps
break
else:
raise NotImplementedError('No maps with frequency axis exactly'
' matching data.')
# Now make sure we have the polarizations in the right order.
data_pols = Data.field['CRVAL4'].copy()
for ii in range(len(data_pols)):
if (misc.polint2str(data_pols[ii])
!= self.params['map_polarizations'][ii]):
raise NotImplementedError('Map polarizations not in same order'
' as data polarizations.')
if (not self.params['solve_for_gain'] or
self.params['gain_output_end'] is '') :
sub_map(Data, maps, self.params['solve_for_gain'],
interpolation=self.params['interpolation'])
else :
block_gain = {}
Data.calc_freq()
block_gain['freq'] = sp.copy(Data.freq)
block_gain['time'] = Data.field['DATE-OBS'][0]
block_gain['scan'] = Data.field['SCAN']
block_gain['gain'] = sub_map(Data, maps, True,
interpolation=self.params['interpolation'])
self.gain_list.append(block_gain)
Data.add_history('Subtracted map from data.',
('Map root: ' + ku.abbreviate_file_path(
self.params['map_input_root']),))
return Data
def write(Maps, file_name, feedback=2):
"""Write a map to fits file.
Map should be a map_data.MapData object.
"""
# If only a single Map was passed, make it iterable.
if not hasattr(Maps, '__iter__'):
Maps = (Maps, )
# First create a primary with the history and such:
prihdu = pyfits.PrimaryHDU()
# Add history to the primary.
fname_abbr = ku.abbreviate_file_path(file_name)
# Add final history entry and store in the primary header.
history = base_data.merge_histories(*Maps)
history.add('Written to file.', 'File name: ' + fname_abbr)
bf.write_history_header(prihdu.header, history)
list_of_hdus = [prihdu]
for ii, Map in enumerate(Maps):
# Creat an image HDU.
map = sp.array(ma.filled(Map.data, float('nan')))
imhdu = pyfits.ImageHDU(sp.swapaxes(map, 0, 2), name='MAP%d' % ii)
# Add extra data to the HDU
for key in Map.field.iterkeys():
if Map.field_axes[key] != ():
raise ce.DataError('Only 0D data can be written to a Fits Map '
'Header.')
card = pyfits.Card(key, Map.field[key].item())
imhdu.header.ascardlist().append(card)
list_of_hdus.append(imhdu)
# Creat the HDU list and write to file.
hdulist = pyfits.HDUList(list_of_hdus)
hdulist.writeto(file_name, clobber=True)
if feedback > 0:
print 'Wrote data to file: ' + fname_abbr
def write(Maps, file_name, feedback=2):
"""Write a map to fits file.
Map should be a map_data.MapData object.
"""
# If only a single Map was passed, make it iterable.
if not hasattr(Maps, "__iter__"):
Maps = (Maps,)
# First create a primary with the history and such:
prihdu = pyfits.PrimaryHDU()
# Add history to the primary.
fname_abbr = ku.abbreviate_file_path(file_name)
# Add final history entry and store in the primary header.
history = base_data.merge_histories(*Maps)
history.add("Written to file.", "File name: " + fname_abbr)
bf.write_history_header(prihdu.header, history)
list_of_hdus = [prihdu]
for ii, Map in enumerate(Maps):
# Creat an image HDU.
map = sp.array(ma.filled(Map.data, float("nan")))
imhdu = pyfits.ImageHDU(sp.swapaxes(map, 0, 2), name="MAP%d" % ii)
# Add extra data to the HDU
for key in Map.field.iterkeys():
if Map.field_axes[key] != ():
raise ce.DataError("Only 0D data can be written to a Fits Map " "Header.")
card = pyfits.Card(key, Map.field[key].item())
imhdu.header.ascardlist().append(card)
list_of_hdus.append(imhdu)
# Creat the HDU list and write to file.
hdulist = pyfits.HDUList(list_of_hdus)
hdulist.writeto(file_name, clobber=True)
if feedback > 0:
print "Wrote data to file: " + fname_abbr
def processfile(self, scan, Pipe) :
params = self.params
scan_log = self.scan_log
log_dir = params["fits_log_dir"]
# Whethar we will fold the data to look for the cal or not.
get_cal = params["partition_cal"]
if get_cal :
ncal = 2
else :
ncal = 1
# First find the GO fits file.
scan_log_files = scan_log.field('FILEPATH')[
scan_log.field('SCAN')==scan]
go_file = log_dir + get_filename_from_key(scan_log_files, "GO")
# If we are combining multiple scans from one proceedure, make sure
# that they all came from the same one (ie that the proceedure
# wasn't aborted).
if params["combine_map_scans"] :
go_hdu = pyfits.open(go_file)[0].header
if (self.n_scans_proc != go_hdu["PROCSIZE"] or go_hdu["PROCSEQN"]
- self.initial_scan_ind != scan - self.initial_scan) :
# Rather than crashing here, send an error code and crash the
# main program.
Pipe.send(-1)
return
# Now get the pointing data from the antenna fits file.
antenna_file = (log_dir + get_filename_from_key(scan_log_files,
"/Antenna"))
az_interp, el_interp, ant_time_range = get_antenna_data(antenna_file)
# Get the guppi file name. We check all of the guppi input roots to
# see if there is a corresponding file.
# Sometimes guppi files are just missing. This shouldn't crash the
# program.
for in_root in params["guppi_input_roots"] :
guppi_file = (in_root + "%04d"%scan +
params["guppi_input_end"])
if os.path.isfile(guppi_file) :
break
else :
Pipe.send(None)
return
print "Converting file: " + guppi_file,
if params['combine_map_scans'] :
print (" (scan " + str(go_hdu["PROCSEQN"]) + " of " +
str(self.n_scans_proc) + ')')
else :
print
try :
psrhdu_list = pyfits.open(guppi_file, 'readonly')
except IOError :
# Currupted guppi file. This happens occationally.
Pipe.send(None)
return
# A record array with each row holding about 2 seconds of data.
psrdata = psrhdu_list[1].data
psrheader = psrhdu_list[1].header
psrmain = psrhdu_list[0].header
# Some numbers we need to pull from the header.
if not psrheader["TTYPE17"] == "DATA" :
raise ce.DataError("Expected to find DATA as the 17th field")
data_shape = eval(psrheader["TDIM17"])
if data_shape[0] != 1:
raise ce.DataError("Data not shaped as expected.")
# Change data shape to C ordering.
data_shape = (list(data_shape[1:]))
data_shape.reverse()
data_shape = tuple(data_shape)
# Number of times in a record (only a fraction of the total
# number of times in the fits file).
ntime = data_shape[0]
npol = data_shape[1]
nfreq = data_shape[2]
# Number of times to average to together when rebinning.
n_bins_ave = params["time_bins_to_average"]
if ntime%n_bins_ave != 0 :
raise ValueError("Number of time bins to average must divide "
"the number of time bins in a sub integration.")
# Number of time bins AFTER rebinning
n_bins = ntime//n_bins_ave
# Figure out the file start time
start_string = psrmain["DATE-OBS"] # UTC string including date.
# Since 00h00 UTC.
start_seconds = psrmain["STT_SMJD"] + psrmain["STT_OFFS"]
# String that will be converted to the time at each sample.
time_string = start_string.split('T')[0] + 'T'
time_string = time_string + '%02d:%02d:%06.3f'
# Make a time string for the next day in the off chance that the scan
# strattles 00:00:00 UTC. This calculation is very rarely needed and
# suficiently complex that I won't write the whole algorithm more
# generally, lest I make a mistake.
time_obj = datetime.datetime.strptime(start_string.split('T')[0],
"%Y-%M-%d")
time_obj = time_obj + datetime.timedelta(days=1)
time_string_next_day = (time_obj.strftime("%Y-%M-%d") +
'T%02d:%02d:%06.3f')
# Figure out the sample period after rebinning.
sample_time = psrheader["TBIN"]
resampled_time = sample_time*n_bins_ave
# In current scan startegy, Zenith angle is approximatly
# constant over a file. We will verify that this matches the antenna
# fits file as a good (but scan strategy specific) check.
if self.proceedure == 'ralongmap' :
zenith_angle = psrdata[0]["TEL_ZEN"]
if not sp.allclose(90.0 - zenith_angle, el_interp.y, atol=0.1) :
raise ce.DataError("Antenna el disagrees with guppi el.")
# Occationally, the guppi integrates longer than the scan and the
# antenna fits file won't cover the first or last few records.
# Check for this and trunkate the rocords appropriately.
n_records = len(psrdata)
start_record = 0
record_offsets = psrdata.field("OFFS_SUB")
for ii in range(3) :
earliest = (start_seconds + record_offsets[start_record]
- sample_time*ntime/2)
latest = (start_seconds + record_offsets[n_records-1]
+ sample_time*ntime/2)
if earliest < ant_time_range[0] :
start_record -= 1
print "Discarded first record due to antenna time mismatch."
if latest > ant_time_range[1] :
n_records -= 1
print "Discarded last record due to antenna time mismatch."
# Allowcate memory for this whole scan.
# final_shape is ordered (ntime, npol, ncal, nfreq) where ntime
# is after rebining and combing the records.
final_shape = (n_bins*(n_records - start_record), npol, ncal,
nfreq)
Data = data_block.DataBlock(sp.empty(final_shape, dtype=float))
# Allowcate memory for the the fields that are time dependant.
Data.set_field('CRVAL2', sp.empty(final_shape[0]),
('time',), '1D')
Data.set_field('CRVAL3', sp.empty(final_shape[0]),
('time',), '1D')
Data.set_field('DATE-OBS', sp.empty(final_shape[0], dtype='S22'),
('time',), '22A')
# Copy as much field information as we can without looping.
crpix1 = nfreq//2 + 1
crval1 = psrdata.field("DAT_FREQ")[0, crpix1 - 1]*1e6
Data.set_field('CRVAL1', crval1, (), '1D')
Data.set_field('CRPIX1', crpix1, (), '1I')
Data.set_field('SCAN', scan, (), '1I')
Data.set_field('BANDWID', psrmain['OBSBW']*1e6, (), '1D')
Data.set_field('CDELT1', psrheader['CHAN_BW']*1e6, (), '1D')
Data.set_field('OBJECT', psrmain['SRC_NAME'], (), '32A')
Data.set_field('EXPOSURE', resampled_time/ncal, (), '1D')
if psrheader["POL_TYPE"].strip() == 'IQUV' :
Data.set_field('CRVAL4', [1,2,3,4], ('pol',), '1I')
else :
raise ce.DataError("Unsupported polarizations.")
if get_cal :
Data.set_field('CAL', ['T', 'F'], ('cal',), '1A')
else :
Data.set_field('CAL', ['F'], ('cal',), '1A')
Data.verify()
# We will devided the data into params["cal_fold_time"] chunks and
# solve for the cal phase in each one. Figure out how many records go
# in each chunk.
# Figure out the number of time bins in a cal period.
if get_cal:
try :
cal_period = 1.0/psrmain["CAL_FREQ"]
except ZeroDivisionError :
# Default value that we usually use.
cal_period = 0.065536
n_bins_cal = cal_period/sample_time
if n_bins_cal%1.0 > 0.00001 :
raise NotImplementedError("Need an integer number of "
"samples per cal period.")
n_bins_cal = int(round(n_bins_cal))
if n_bins_ave%n_bins_cal != 0 :
raise ValueError("You should rebin on a multiple of "
"the cal period. Change parameter "
"time_bins_to_average.")
total_time = (n_records - start_record)*ntime*sample_time
n_divisions = total_time/params['cal_fold_time']
n_divisions = int(round(n_divisions))
n_records_div = float(n_records - start_record)/n_divisions
division_starts = [int(round(start_record + n_records_div*ii))
for ii in range(n_divisions+1)]
# Loop over the records and modify the data.
if self.feedback > 1 :
print "Record: ",
for jj in range(start_record, n_records) :
record = psrdata[jj]
# Print the record we are currently on and force a flush to
# standard out (so we can watch the progress update).
if self.feedback > 1 :
print jj,
sys.stdout.flush()
# Do one pass through the current division of the data to find the
# folding offsets.
if get_cal and jj in division_starts :
div_num = division_starts.index(jj)
# Loop over the data and accumulate it into one array profile.
record_profile = sp.zeros((ntime, nfreq), dtype=float)
for kk in range(division_starts[div_num],
division_starts[div_num+1]):
record_tmp = psrdata[kk]
# Data comes from telescope as wierd unsigned and signed
# ints. Handle carefully. Only interested in stokes I.
tmp_data = record_tmp["DATA"]
tmp_data.shape = (ntime, npol, nfreq)
# The 'I' polarization should already be correctly formated
# as an uint8.
tmp_data = tmp_data[:,0,:]
# There are actually time intependant a frequency dependant
# scalings that are ignored here. They shouldn't affect
# the shape of the profile much.
record_profile += tmp_data
# `get_cal_mask` expects 3D data.
record_profile.shape = (ntime, 1, nfreq)
# Try to get the profile of this division. If we fail, set the
# Phase info to None, which cause the phase to be solved for on
# a record by record basis.
try :
cal_phase_info = get_cal_mask(record_profile, n_bins_cal)
except ce.DataError :
cal_phase_info = None
# Get the data field.
data = record["DATA"]
# Convert the Data to the proper shape and type.
data = format_data(data, ntime, npol, nfreq)
if get_cal :
data = separate_cal(data, n_bins_cal, cal_phase_info)
# Down sample from the cal period to the final number of time
# bins.
data.shape = (n_bins, ntime//n_bins_cal//n_bins, npol,
ncal, nfreq)
# Use mean not median due to discritization.
data = sp.mean(data, 1)
else :
# Just rebin in time and add a length 1 cal index.
data.shape = (n_bins, n_bins_ave, npol, ncal, nfreq)
data = sp.mean(data, 1)
# Now get the scale and offsets for the data and apply them.
scls = sp.array(record["DAT_SCL"], dtype=sp.float32)
scls.shape = (1, npol, 1, nfreq)
offs = sp.array(record["DAT_OFFS"], dtype=sp.float32)
offs.shape = (1, npol, 1, nfreq)
data = scls*data + offs
# Now figure out the timing information.
time = start_seconds + record["OFFS_SUB"]
time = time + (sp.arange(-n_bins/2.0 + 0.5, n_bins/2.0 + 0.5)
* resampled_time)
# Make sure that there are no time gaps between records.
# This is nessisary for the cal separation to be valid.
if jj > start_record:
record_time = record["OFFS_SUB"] - last_offs_sub
if not sp.allclose(record_time, n_bins*resampled_time):
msg = "Time gaps between records"
raise ce.DataError(msg)
last_offs_sub = record["OFFS_SUB"]
# Make sure that our pointing data covers the guppi data time
# range within the 0.1 second accuracy.
if ((max(time) - ant_time_range[1] > 0.) or
(min(time) - ant_time_range[0] < 0.)) :
message = ("Guppi data time range not covered by antenna."
" Antenna: file " + antenna_file + " range "
+ str((ant_time_range[0], ant_time_range[1]))
+ " guppi: file " + guppi_file + " range "
+ str((min(time), max(time))))
raise ce.DataError(message)
az = az_interp(time)
el = el_interp(time)
# Convert the time to hours, minutes and seconds.
hours = time//3600
minutes = (time%3600)//60
seconds = time%60
# Put all the data into the DataBlock for writing out.
Data.data[jj*n_bins:(jj+1)*n_bins,:,:,:] = data
Data.field['CRVAL2'][jj*n_bins:(jj+1)*n_bins] = az
Data.field['CRVAL3'][jj*n_bins:(jj+1)*n_bins] = el
# If the scan strattles midnight UTC, this is extra
# complecated.
for mm, kk in enumerate(xrange(jj*n_bins, (jj+1)*n_bins)) :
if hours[mm] >= 24 :
tmp_time = (time_string_next_day
% (hours[mm]-24, minutes[mm], seconds[mm]))
else :
tmp_time = time_string%(hours[mm], minutes[mm],
seconds[mm])
Data.field["DATE-OBS"][kk] = tmp_time
Data.add_history('Converted from a guppi sub-integration fits file.',
(utils.abbreviate_file_path(guppi_file),
utils.abbreviate_file_path(antenna_file),
utils.abbreviate_file_path(go_file)))
# End loop over records.
if self.feedback > 1 :
print
# Send Data back on the pipe.
Pipe.send(Data)
def read(self, scans=None, IFs=None, force_tuple=False) :
"""Read in data from the fits file.
This method reads data from the fits file including the files history
and basically every peice of data that could be needed. It is done,
one scan and one IF at a time. Each scan and IF is returned in an
instance of the DataBlock class (defined in another module of this
package).
Arguments:
scans: Which scans in the file to be processed. A list of
integers, with 0 corresponding to the lowest numbered scan.
Default is all of them.
IFs: Which intermediate frequencies (also called frequency windows)
to process. A list of integers with 0 coorsponding to the
lowest frequency present. Default is all of them.
TODO : Overlapping frequency windows stiched together somehow.
force_tuple: By default, if there is only a single output Data
Block, it is returned not wraped in a tuple, but if we want to
loop over the output we can force the output to be a tuple,
even if it only has one element.
Returns: Instance of the DataBlock class, or a tuple of these instances
(if asked for multiple scans and IFs).
"""
# We want scans and IFs to be a sequence of indicies.
if scans is None :
scans = range(len(self.scan_set))
elif not hasattr(scans, '__iter__') :
scans = (scans, )
elif len(scans) == 0 :
scans = range(len(self.scan_set))
if IFs is None :
IFs = range(len(self.IF_set))
elif not hasattr(IFs, '__iter__') :
IFs = (IFs, )
elif len(IFs) == 0 :
IFs = range(len(self.IF_set))
# Sequence of output DataBlock objects.
output = ()
for scan_ind in scans :
for IF_ind in IFs :
# Choose the appropriate records from the file, get that data.
inds_sif = self.get_scan_IF_inds(scan_ind, IF_ind)
Data_sif = db.DataBlock(self.fitsdata.field('DATA')[inds_sif])
# Masked data is stored in FITS files as float('nan')
Data_sif.data[sp.logical_not(sp.isfinite(
Data_sif.data))] = ma.masked
# Now iterate over the fields and add them
# to the data block.
for field, axis in fields_and_axes.iteritems() :
# See if this fits file has the key we are looking for.
if not field in self.fitsdata._names :
continue
# First get the 'FITS' format string.
field_format = self.hdulist[1].columns.formats[
self.hdulist[1].columns.names.index(field)]
if axis :
# From the indices in inds_sif, we only need a
# subset: which_data will subscript inds_sif.
temp_data = self.fitsdata.field(field)[inds_sif]
# For reshaping at the end.
field_shape = []
for ii, single_axis in enumerate(Data_sif.axes[0:-1]) :
# For each axis, slice out all the data except the
# stuff we need.
which_data = [slice(None)] * 3
if single_axis in axis :
field_shape.append(Data_sif.dims[ii])
else :
which_data[ii] = [0]
temp_data = temp_data[tuple(which_data)]
temp_data.shape = tuple(field_shape)
Data_sif.set_field(field, temp_data, axis, field_format)
else :
Data_sif.set_field(field, self.fitsdata.field(field)
[inds_sif[0,0,0]], axis, field_format)
if hasattr(self, 'history') :
Data_sif.history = db.History(self.history)
else :
self.history =bf.get_history_header(self.hdulist[0].header)
#self.set_history(Data_sif)
fname_abbr = ku.abbreviate_file_path(self.fname)
self.history.add('Read from file.', ('File name: ' +
fname_abbr, ))
Data_sif.history = db.History(self.history)
Data_sif.verify()
output = output + (Data_sif, )
if self.feedback > 2 :
print 'Read finished.'
if len(output) == 1 and not force_tuple :
return output[0]
else :
return output
def process_session(self, session) :
params = self.params
# Get all the files names and such.
number = session["number"]
try :
guppi_root = session["guppi_input_root"]
except KeyError :
guppi_root = params["default_guppi_input_root"]
# Some observations are spread over multiple directories, in which case
# session["guppi_dir"] will be a list.
if isinstance(session["guppi_dir"], list) :
session_dirs = session["guppi_dir"]
else :
session_dirs = [session["guppi_dir"]]
# Version of session_dirs with absolute path.
guppi_dirs = [guppi_root + dir + "/"
for dir in session_dirs]
fits_root = params["fits_log_root"] + "%02d"%number + "/"
outroot = params["output_root"]
print ("Processing sesson " + str(number) + ", in guppi directories "
+ str(guppi_dirs))
# Check that all the directories are present.
for dir in guppi_dirs :
if (not os.path.isdir(dir) or not os.path.isdir(fits_root)) :
print "Skipped do to missing data."
return
# First thing we do is start to rsync to the archive.
if number in params['sessions_to_archive'] and not params["dry_run"]:
# Construct the multiple line command that will tarnsfer all
# guppi directories to teh archive.
program = "rsync -av -essh "
command = ""
for ii in range(len(session_dirs)) :
command += program
command += guppi_dirs[ii] + " "
command += (params["archive_root"] + str(number) + "_"
+ session_dirs[ii] + "/")
command += ";"
print command
SyncProc = subprocess.Popen(command, shell=True,
stdout=self.r_out, stderr=subprocess.STDOUT)
# Check if this session is in the list of force even if output file
# exist.
if number in params['sessions_to_force'] :
force_session = True
else :
force_session = False
# Now loop over the fields for this session.
for source in session["sources"] :
field = source[0]
# scans is a list of integers containing at least one scan number
# from each process.
scans = source[1]
# File name pattern that output files should match.
converted_pattern = (params["output_root"] + "%02d"%number
+ '_' + field + '*.fits')
if force_session :
scans_to_convert = scans
else :
scans_to_convert = []
for scan in scans :
# Check if the scan has already been convereted to SDfits
# by looking for the output file.
matched_file = check_file_matching_scan(scan,
converted_pattern)
if not matched_file is None :
print ("Skipped because scan already converted "
"in file: "
+ utils.abbreviate_file_path(matched_file))
else :
scans_to_convert.append(scan)
if len(scans_to_convert) > 0 :
# Set up the converter for the scans that havn't been converted
# yet.
# Probably shouldn't hard code these.
converter_params = {"guppi_input_end" : "_0001.fits",
"combine_map_scans" : True,
"partition_cal" : True,
"time_bins_to_average" : 128,
"cal_fold_time" : 30
}
# The parameters that are acctually assigned dynamically.
converter_params["scans"] = tuple(scans_to_convert)
converter_params["fits_log_dir"] = fits_root
converter_params["output_root"] = outroot
try :
blacklist = session["blacklist"]
except KeyError :
blacklist = []
converter_params["blacklist"] = blacklist
# Guppi input root should be a list of all possible roots for
# this session. All permutations of guppi directory and of
# session number.
if isinstance(session["guppi_session"], list) :
guppi_sessions = session["guppi_session"]
else :
guppi_sessions = [session["guppi_session"]]
converter_params["guppi_input_roots"] = []
for dir in guppi_dirs :
for se in guppi_sessions :
converter_params["guppi_input_roots"].append(dir
+ "guppi_" + se + "_" + field + "_")
# Convert the files from this session.
if not params["dry_run"] :
C = Converter(converter_params, feedback=1)
C.execute(params['nprocesses'])
# Check that the new files are in fact present.
# Make a list of these files for the next section of the pipeline.
if params["dry_run"] :
# The files we need to set up the rest of the proceedure will
# be missing, just return.
return
file_middles = []
for scan in scans :
converted_file = check_file_matching_scan(scan,
converted_pattern)
if converted_file is None :
raise RuntimeError("Scan not converted: " + str(scan))
# Get the middle (unique) part of the file name.
converted_file = converted_file.split('/')[-1]
converted_file = converted_file.split('.')[0]
file_middles.append(converted_file)
# Now figure out which scans have had data quality checks
# already run.
if force_session :
files_to_check = file_middles
else :
files_to_check = []
for file_middle in file_middles :
# Figure out which files have already been checked by
# looking for the output file.
checked_out_fname = (params["quality_check_root"] +
file_middle + ".pdf")
if os.path.isfile(checked_out_fname) :
print ("Skipped because file already checked "
"in file: " + utils.abbreviate_file_path(
checked_out_fname))
else :
files_to_check.append(file_middle)
if len(files_to_check) != 0 :
# Build up input parameters for DataChecker.
checker_params = {
"output_end" : ".pdf",
"file_middles" : tuple(files_to_check),
"input_root" : params["output_root"],
"output_root" : params["quality_check_root"]
}
# Execute the data checker.
DataChecker(checker_params).execute(params["nprocesses"])
# Wait for the rsync to terminate:
if number in params['sessions_to_archive']:
SyncProc.wait()
if SyncProc.returncode :
raise RuntimeError("rsync failed with exit code: "
+ str(SyncProc.returncode))
def process_file(self, file_ind):
params = self.params
file_middle = params['file_middles'][file_ind]
input_fname = (params['input_root'] + file_middle +
params['input_end'])
sub_input_fname = (params['subtracted_input_root'] + file_middle +
params['input_end'])
output_fname = (params['output_root'] + file_middle +
params['output_end'])
sub_output_fname = (params['subtracted_output_root'] + file_middle +
params['output_end'])
Writer = fitsGBT.Writer(feedback=self.feedback)
SubWriter = fitsGBT.Writer(feedback=self.feedback)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
SubReader = fitsGBT.Reader(sub_input_fname, feedback=self.feedback)
if (sp.any(Reader.scan_set != SubReader.scan_set)
or sp.any(Reader.IF_set != SubReader.IF_set)):
raise ce.DataError("IFs and scans don't match signal subtracted"
" data.")
# Get the number of scans if asked for all of them.
scan_inds = params['scans']
if len(scan_inds) == 0 or scan_inds is None:
scan_inds = range(len(Reader.scan_set))
if_inds = params['IFs']
if len(if_inds) == 0 or scan_inds is None:
if_inds = range(len(Reader.IF_set))
if self.feedback > 1:
print "New flags each block:",
# Loop over scans and IFs
for thisscan in scan_inds:
for thisIF in if_inds:
Data = Reader.read(thisscan, thisIF)
SubData = SubReader.read(thisscan, thisIF)
# Make sure they have agreeing masks to start.
SubData.data[ma.getmaskarray(Data.data)] = ma.masked
Data.data[ma.getmaskarray(SubData.data)] = ma.masked
# Get initial number of flags.
n_flags = ma.count_masked(Data.data)
# Now do the flagging.
flag(Data, SubData, params['thres'],
params['max_noise_factor'],
params['smooth_modes_subtract'], params['filter_type'])
Data.add_history(
"Reflaged for outliers.",
("Used file: " +
utils.abbreviate_file_path(sub_input_fname), ))
SubData.add_history("Reflaged for outliers.")
Writer.add_data(Data)
SubWriter.add_data(SubData)
# Report the number of new flags.
n_flags = ma.count_masked(Data.data) - n_flags
if self.feedback > 1:
print n_flags,
if self.feedback > 1:
print ''
# Finally write the data back to file.
utils.mkparents(output_fname)
utils.mkparents(sub_output_fname)
Writer.write(output_fname)
SubWriter.write(sub_output_fname)
def read(self, scans=None, bands=None, force_tuple=False, IFs=None) :
"""Read in data from the fits file.
This method reads data from the fits file including the files history
and basically every peice of data that could be needed. It is done,
one scan and one IF at a time. Each scan and IF is returned in an
instance of the DataBlock class (defined in another module of this
package).
Parameters
----------
scans : tuple of integers
Which scans in the file to be processed. A list of
integers, with 0 corresponding to the lowest numbered scan.
Default is all of them.
bands : tuple of integers
Which intermediate frequencies (also called frequency windows)
to process. A list of integers with 0 coorsponding to the
lowest frequency present. Default is all of them.
TODO : Overlapping frequency windows stiched together somehow.
force_tuple: By default, if there is only a single output Data
Block, it is returned not wraped in a tuple, but if we want to
loop over the output we can force the output to be a tuple,
even if it only has one element.
IFs : tuple of integers
Depricated, use `bands`.
Returns
-------
Instance of the DataBlock class, or a tuple of these instances
(if asked for multiple scans and IFs).
"""
# `bands` and `IFs` is are two names for the same parameter.
if not bands is None and IFs is None:
IFs = bands
# We want scans and IFs to be a sequence of indicies.
if scans is None :
scans = range(len(self.scan_set))
elif not hasattr(scans, '__iter__') :
scans = (scans, )
elif len(scans) == 0 :
scans = range(len(self.scan_set))
if IFs is None :
IFs = range(len(self.IF_set))
elif not hasattr(IFs, '__iter__') :
IFs = (IFs, )
elif len(IFs) == 0 :
IFs = range(len(self.IF_set))
# Sequence of output DataBlock objects.
output = ()
for scan_ind in scans :
for IF_ind in IFs :
# Choose the appropriate records from the file, get that data.
inds_sif = self.get_scan_IF_inds(scan_ind, IF_ind)
Data_sif = db.DataBlock(self.fitsdata.field('DATA')[inds_sif])
# Masked data is stored in FITS files as float('nan')
Data_sif.data[sp.logical_not(sp.isfinite(
Data_sif.data))] = ma.masked
# Now iterate over the fields and add them
# to the data block.
for field, axis in fields_and_axes.iteritems() :
# See if this fits file has the key we are looking for.
try:
names = self.fitsdata.names
except AttributeError:
names = self.fitsdata._names
if not field in names :
continue
# First get the 'FITS' format string.
field_format = self.hdulist[1].columns.formats[
self.hdulist[1].columns.names.index(field)]
if axis :
# From the indices in inds_sif, we only need a
# subset: which_data will subscript inds_sif.
temp_data = self.fitsdata.field(field)[inds_sif]
# For reshaping at the end.
field_shape = []
for ii, single_axis in enumerate(Data_sif.axes[0:-1]) :
# For each axis, slice out all the data except the
# stuff we need.
which_data = [slice(None)] * 3
if single_axis in axis :
field_shape.append(Data_sif.dims[ii])
else :
which_data[ii] = [0]
temp_data = temp_data[tuple(which_data)]
temp_data.shape = tuple(field_shape)
Data_sif.set_field(field, temp_data, axis, field_format)
else :
Data_sif.set_field(field, self.fitsdata.field(field)
[inds_sif[0,0,0]], axis, field_format)
if hasattr(self, 'history') :
Data_sif.history = db.History(self.history)
else :
self.history =bf.get_history_header(self.hdulist[0].header)
#self.set_history(Data_sif)
fname_abbr = ku.abbreviate_file_path(self.fname)
self.history.add('Read from file.', ('File name: ' +
fname_abbr, ))
Data_sif.history = db.History(self.history)
Data_sif.verify()
output = output + (Data_sif, )
if self.feedback > 2 :
print 'Read finished.'
if len(output) == 1 and not force_tuple :
return output[0]
else :
return output