def preview_zenith_tau(log, row_list, cl_params, feeds, windows, pols):
# if using the weather database
if not cl_params.zenithtau:
foo = row_list.get(cl_params.mapscans[0], feeds[0], windows[0],
pols[0])
ff = fitsio.FITS(cl_params.infilename)
extension = foo['EXTENSION']
row = foo['ROW'][0]
bar = ff[extension]['OBSFREQ', 'DATE-OBS'][row]
dateobs = bar['DATE-OBS'][0]
obsfreq = bar['OBSFREQ'][0]
ff.close()
weather = Weather()
pu = Pipeutils()
mjd = pu.dateToMjd(dateobs)
zenithtau = weather.retrieve_zenith_opacity(mjd, obsfreq)
log.doMessage('INFO',
'Zenith opacity for map: {0:.3f}'.format(zenithtau))
# else if set at the command line
else:
log.doMessage('INFO',
'Zenith opacity for '
'map: {0:.3f}'.format(cl_params.zenithtau))
def test_dateToMjd():
putils = Pipeutils()
sdfits_date_string = '2009-02-10T21:09:00.10'
result = putils.dateToMjd( sdfits_date_string )
expected_result = 54872.8812512
assert_almost_equal( result, expected_result )
def test_dateToMjd():
putils = Pipeutils()
sdfits_date_string = '2009-02-10T21:09:00.10'
result = putils.dateToMjd(sdfits_date_string)
expected_result = 54872.8812512
assert_almost_equal(result, expected_result)
def preview_zenith_tau(log, row_list, cl_params, feeds, windows, pols):
foo = None
# if using the weather database
if cl_params.zenithtau is None:
for feed in feeds:
for window in windows:
for pol in pols:
try:
foo = row_list.get(cl_params.mapscans[0], feed, window,
pol)
break # if we found a row move on, otherwise try another feed/win/pol
except KeyError:
continue
if not foo:
log.doMessage('ERR',
'Could not find scan for zenith opacity preview')
return
ff = fitsio.FITS(cl_params.infilename)
extension = foo['EXTENSION']
row = foo['ROW'][0]
bar = ff[extension]['OBSFREQ', 'DATE-OBS'][row]
dateobs = bar['DATE-OBS'][0]
obsfreq = bar['OBSFREQ'][0]
ff.close()
weather = Weather()
pu = Pipeutils()
mjd = pu.dateToMjd(dateobs)
zenithtau = weather.retrieve_zenith_opacity(mjd, obsfreq, log)
if zenithtau:
log.doMessage(
'INFO', 'Approximate zenith opacity for map: {0:.3f}'.format(
zenithtau))
else:
log.doMessage(
'ERR', 'Not able to retrieve integration '
'zenith opacity for calibration to:', cl_params.units,
'\n Please supply a zenith opacity or calibrate to Ta.')
sys.exit(9)
# else if set at the command line
else:
log.doMessage(
'INFO', 'Zenith opacity for '
'map: {0:.3f}'.format(cl_params.zenithtau))
def preview_zenith_tau(log, row_list, cl_params, feeds, windows, pols):
foo = None
# if using the weather database
if cl_params.zenithtau is None:
for feed in feeds:
for window in windows:
for pol in pols:
try:
foo = row_list.get(cl_params.mapscans[0], feed, window, pol)
break # if we found a row move on, otherwise try another feed/win/pol
except KeyError:
continue
if not foo:
log.doMessage('ERR', 'Could not find scan for zenith opacity preview')
return
ff = fitsio.FITS(cl_params.infilename)
extension = foo['EXTENSION']
row = foo['ROW'][0]
bar = ff[extension]['OBSFREQ', 'DATE-OBS'][row]
dateobs = bar['DATE-OBS'][0]
obsfreq = bar['OBSFREQ'][0]
ff.close()
weather = Weather()
pu = Pipeutils()
mjd = pu.dateToMjd(dateobs)
zenithtau = weather.retrieve_zenith_opacity(mjd, obsfreq, log)
if zenithtau:
log.doMessage('INFO',
'Approximate zenith opacity for map: {0:.3f}'.format(zenithtau))
else:
log.doMessage('ERR', 'Not able to retrieve integration '
'zenith opacity for calibration to:', cl_params.units,
'\n Please supply a zenith opacity or calibrate to Ta.')
sys.exit(9)
# else if set at the command line
else:
log.doMessage('INFO',
'Zenith opacity for '
'map: {0:.3f}'.format(cl_params.zenithtau))
class SdFits:
"""Class contains methods to read and write to the GBT SdFits format.
This includes code for both the FITS files and associated index files.
A description (but not a definition) of the SD FITS is here:
https://safe.nrao.edu/wiki/bin/view/Main/SdfitsDetails
"""
def __init__(self):
self.pu = Pipeutils()
def find_maps(self, indexfile, debug=False):
"""Find mapping blocks. Also find samplers used in each map
Args:
indexfile: input required to search for maps and samplers
debug: optional debug flag
Returns:
a (list) of map blocks, with each entry a (tuple) of the form:
(int) reference 1,
(list of ints) mapscans,
(int) reference 2
"""
map_scans = {}
observation, summary = self.parseSdfitsIndex(indexfile)
feed = observation.feeds()[0]
window = observation.windows()[0]
pol = observation.pols()[0]
# print results
if debug:
print '------------------------- All scans'
for scanid in sorted(observation.scans()):
scanstruct = observation.get(scanid, feed, window, pol)
print(
'scan \'{0}\' obsid \'{1}\' procname \'{2}\' procscan \'{3}\''
.format(scanid, scanstruct['OBSID'],
scanstruct['PROCNAME'], scanstruct['PROCSCAN']))
for scanid in observation.scans():
scanstruct = observation.get(scanid, feed, window, pol)
obsid = scanstruct['OBSID'].upper()
procname = scanstruct['PROCNAME'].upper()
procscan = scanstruct['PROCSCAN'].upper()
# keyword check should depend on presence of PROCSCAN key, which is an
# alternative to checking SDFITVER.
# OBSID is the old way, PROCSCAN is the new way MR8Q312
# create a new list that only has 'MAP' and 'OFF' scans
if not procscan and (obsid == 'MAP' or obsid == 'OFF'):
map_scans[scanid] = obsid
elif (procscan == 'MAP' or procname == 'TRACK'
or (procname == 'ONOFF' and procscan == 'OFF')
or (procname == 'OFFON' and procscan == 'OFF')):
map_scans[scanid] = procscan
mapkeys = map_scans.keys()
mapkeys.sort()
if debug:
print '------------------------- Relavant scans'
for scanid in mapkeys:
print 'scan', scanid, map_scans[scanid]
maps = [] # final list of maps
ref1 = None
ref2 = None
prev_ref2 = None
mapscans = [] # temporary list of map scans for a single map
if debug:
print 'mapkeys', mapkeys
MapParams = namedtuple("MapParams", "refscan1 mapscans refscan2")
for idx, scan in enumerate(mapkeys):
# look for the reference scans
if (map_scans[scan]).upper() == 'OFF' or (
map_scans[scan]).upper() == 'ON':
# if there is no ref1 or this is another ref1
if not ref1 or (ref1 and bool(mapscans) == False):
ref1 = scan
else:
ref2 = scan
prev_ref2 = ref2
elif (map_scans[scan]).upper() == 'MAP':
if not ref1 and prev_ref2:
ref1 = prev_ref2
mapscans.append(scan)
# see if this scan is the last one in the relevant scan list
# or see if we have a ref2
# if so, close out
if ref2 or idx == len(mapkeys) - 1:
maps.append(MapParams(ref1, mapscans, ref2))
ref1 = False
ref2 = False
mapscans = []
if debug:
import pprint
pprint.pprint(maps)
for idx, mm in enumerate(maps):
print "Map", idx
if mm.refscan2:
print "\tReference scans.....", mm.refscan1, mm.refscan2
else:
print "\tReference scan......", mm.refscan1
print "\tMap scans...........", mm.mapscans
return maps
def parseSdfitsIndex(self, infile, mapscans=[]):
try:
ifile = open(infile)
except IOError:
print(
"ERROR: Could not open file: {0}\n"
"Please check and try again.".format(infile))
raise
observation = ObservationRows()
while True:
line = ifile.readline()
# look for start of row data or EOF (i.e. not line)
if '[rows]' in line or not line:
break
lookup_table = {}
header = ifile.readline()
fields = [xx.lstrip() for xx in re.findall(r' *\S+', header)]
iterator = re.finditer(r' *\S+', header)
for idx, mm in enumerate(iterator):
lookup_table[fields[idx]] = slice(mm.start(), mm.end())
rr = SdFitsIndexRowReader(lookup_table)
summary = {'WINDOWS': set([]), 'FEEDS': set([])}
# keep a list of suspect scans so we can know if the
# user has already been warned
suspectScans = set()
for row in ifile:
rr.setrow(row)
scanid = int(rr['SCAN'])
# have a look at the procedure
# if it is "Unknown", the data is suspect, so skip it
procname = rr['PROCEDURE']
if scanid in suspectScans:
continue
if ((scanid not in suspectScans)
and procname.lower() == 'unknown'):
suspectScans.add(scanid)
if scanid in mapscans:
print 'WARNING: scan', scanid, 'has "Unknown" procedure. Skipping.'
continue
feed = int(rr['FDNUM'])
windowNum = int(rr['IFNUM'])
pol = int(rr['PLNUM'])
fitsExtension = int(rr['EXT'])
rowOfFitsFile = int(rr['ROW'])
obsid = rr['OBSID']
procscan = rr['PROCSCAN']
nchans = rr['NUMCHN']
summary['WINDOWS'].add((windowNum, float(rr['RESTFREQ']) / 1e9))
summary['FEEDS'].add(rr['FDNUM'])
# we can assume all integrations of a single scan are within the same
# FITS extension
observation.addRow(scanid, feed, windowNum, pol, fitsExtension,
rowOfFitsFile, obsid, procname, procscan,
nchans)
try:
ifile.close()
except NameError:
raise
return observation, summary
def getReferenceIntegration(self, cal_on, cal_off, scale):
cal = Calibration()
cal_ondata = cal_on['DATA']
cal_offdata = cal_off['DATA']
cref, exposure = cal.total_power(cal_ondata, cal_offdata,
cal_on['EXPOSURE'],
cal_off['EXPOSURE'])
tcal = cal_off['TCAL'] * scale
tsys = cal.tsys(tcal, cal_ondata, cal_offdata)
dateobs = cal_off['DATE-OBS']
timestamp = self.pu.dateToMjd(dateobs)
tambient = cal_off['TAMBIENT']
elevation = cal_off['ELEVATIO']
return cref, tsys, exposure, timestamp, tambient, elevation
def nameIndexFile(self, pathname):
# ------------------------------------------------- name index file
if not os.path.exists(pathname):
print(
'ERROR: Path does not exist {0}.\n'
' Please check and try again'.format(pathname))
sys.exit(9)
if os.path.isdir(pathname):
bn = os.path.basename(pathname.rstrip('/'))
return '{0}/{1}.index'.format(pathname, bn)
elif os.path.isfile(pathname) and pathname.endswith('.fits'):
return os.path.splitext(pathname)[0] + '.index'
else:
# doMessage(logger,msg.ERR,'input file not recognized as a fits file.',\
# ' Please check the file extension and change to \'fits\' if necessary.')
print 'ERROR: Input file does not end with .fits:', pathname
sys.exit(9)
class SdFits:
"""Class contains methods to read and write to the GBT SdFits format.
This includes code for both the FITS files and associated index files.
A description (but not a definition) of the SD FITS is here:
https://safe.nrao.edu/wiki/bin/view/Main/SdfitsDetails
"""
def __init__(self):
self.pu = Pipeutils()
def find_maps(self, indexfile, debug=False):
"""Find mapping blocks. Also find samplers used in each map
Keywords:
indexfile -- input required to search for maps and samplers
debug -- optional debug flag
Returns:
a (list) of map blocks, with each entry a (tuple) of the form:
(int) reference 1,
(list of ints) mapscans,
(int) reference 2
"""
map_scans = {}
observation = self.parseSdfitsIndex(indexfile)
feed = observation.feeds()[0]
window = observation.windows()[0]
pol = observation.pols()[0]
# print results
if debug:
print '------------------------- All scans'
for scanid in observation.scans():
scanstruct = observation.get(scanid, feed, window, pol)
print 'scan', scanid, scanstruct['OBSID']
print '------------------------- Relavant scans'
for scanid in observation.scans():
scanstruct = observation.get(scanid, feed, window, pol)
obsid = scanstruct['OBSID'].upper()
procscan = scanstruct['PROCSCAN'].upper()
# keyword check should depend on presence of PROCSCAN key, which is an
# alternative to checking SDFITVER.
# OBSID is the old way, PROCSCAN is the new way MR8Q312
# only do the following for an "OLD" SDFITS version
# create a new list that only has 'MAP' and 'OFF' scans
if not procscan and (obsid=='MAP' or obsid=='OFF'):
map_scans[scanid] = obsid
elif procscan=='MAP' or procscan=='OFF':
map_scans[scanid] = procscan
mapkeys = map_scans.keys()
mapkeys.sort()
if debug:
for scanid in mapkeys:
print 'scan', scanid, map_scans[scanid]
maps = [] # final list of maps
ref1 = None
ref2 = None
prev_ref2 = None
mapscans = [] # temporary list of map scans for a single map
if debug:
print 'mapkeys', mapkeys
MapParams = namedtuple("MapParams", "refscan1 mapscans refscan2")
for idx,scan in enumerate(mapkeys):
# look for the offs
if (map_scans[scan]).upper()=='OFF':
# if there is no ref1 or this is another ref1
if not ref1 or (ref1 and bool(mapscans)==False):
ref1 = scan
else:
ref2 = scan
prev_ref2 = ref2
elif (map_scans[scan]).upper()=='MAP':
if not ref1 and prev_ref2:
ref1 = prev_ref2
mapscans.append(scan)
# see if this scan is the last one in the relevant scan list
# or see if we have a ref2
# if so, close out
if ref2 or idx==len(mapkeys)-1:
maps.append(MapParams(ref1,mapscans,ref2))
ref1 = False
ref2 = False
mapscans = []
if debug:
import pprint; pprint.pprint(maps)
for idx,mm in enumerate(maps):
print "Map", idx
if mm.refscan2:
print "\tReference scans.....", mm.refscan1, mm.refscan2
else:
print "\tReference scan......", mm.refscan1
print "\tMap scans...........", mm.mapscans
return maps
def parseSdfitsIndex(self, infile):
try:
ifile = open(infile)
except IOError:
print "ERROR: Could not open file. Please check and try again."
raise
observation = ObservationRows()
while True:
line = ifile.readline()
# look for start of row data or EOF (i.e. not line)
if '[rows]' in line or not line:
break
lookup_table = {}
header = ifile.readline()
fields = [xx.lstrip() for xx in re.findall(r' *\S+',header)]
iterator = re.finditer(r' *\S+',header)
for idx,mm in enumerate(iterator):
lookup_table[fields[idx]] = slice(mm.start(),mm.end())
rr = SdFitsIndexRowReader(lookup_table)
for row in ifile:
rr.setrow(row)
scanid = int(rr['SCAN'])
feed = int(rr['FDNUM'])
windowNum = int(rr['IFNUM'])
pol = int(rr['PLNUM'])
fitsExtension = int(rr['EXT'])
rowOfFitsFile = int(rr['ROW'])
typeOfScan = rr['PROCEDURE']
obsid = rr['OBSID']
procscan = rr['PROCSCAN']
# we can assume all integrations of a single scan are within the same
# FITS extension
observation.addRow(scanid, feed, windowNum, pol,
fitsExtension, rowOfFitsFile, typeOfScan, obsid, procscan)
try:
ifile.close()
except NameError:
raise
return observation
def getReferenceIntegration(self, cal_on, cal_off):
cal = Calibration()
cal_ondata = cal_on['DATA']
cal_offdata = cal_off['DATA']
cref, exposure = cal.total_power(cal_ondata, cal_offdata, cal_on['EXPOSURE'], cal_off['EXPOSURE'])
tcal = cal_off['TCAL']
tsys = cal.tsys( tcal, cal_ondata, cal_offdata )
dateobs = cal_off['DATE-OBS']
timestamp = self.pu.dateToMjd(dateobs)
tambient = cal_off['TAMBIENT']
elevation = cal_off['ELEVATIO']
return cref, tsys, exposure, timestamp, tambient, elevation
def nameIndexFile(self, fitsfile):
# ------------------------------------------------- name index file
if fitsfile.endswith('.fits'):
return os.path.splitext(fitsfile)[0]+'.index'
else:
#doMessage(logger,msg.ERR,'input file not recognized as a fits file.',\
# ' Please check the file extension and change to \'fits\' if necessary.')
print 'ERROR: Input file does not end with .fits:', fitsfile
sys.exit(9)
class SdFits:
"""Class contains methods to read and write to the GBT SdFits format.
This includes code for both the FITS files and associated index files.
A description (but not a definition) of the SD FITS is here:
https://safe.nrao.edu/wiki/bin/view/Main/SdfitsDetails
"""
def __init__(self):
self.pu = Pipeutils()
def find_maps(self, indexfile, debug=False):
"""Find mapping blocks. Also find samplers used in each map
Args:
indexfile: input required to search for maps and samplers
debug: optional debug flag
Returns:
a (list) of map blocks, with each entry a (tuple) of the form:
(int) reference 1,
(list of ints) mapscans,
(int) reference 2
"""
map_scans = {}
observation, summary = self.parseSdfitsIndex(indexfile)
feed = observation.feeds()[0]
window = observation.windows()[0]
pol = observation.pols()[0]
# print results
if debug:
print '------------------------- All scans'
for scanid in sorted(observation.scans()):
scanstruct = observation.get(scanid, feed, window, pol)
print('scan \'{0}\' obsid \'{1}\' procname \'{2}\' procscan \'{3}\''.format(scanid,
scanstruct['OBSID'],
scanstruct['PROCNAME'],
scanstruct['PROCSCAN']))
for scanid in observation.scans():
scanstruct = observation.get(scanid, feed, window, pol)
obsid = scanstruct['OBSID'].upper()
procname = scanstruct['PROCNAME'].upper()
procscan = scanstruct['PROCSCAN'].upper()
# keyword check should depend on presence of PROCSCAN key, which is an
# alternative to checking SDFITVER.
# OBSID is the old way, PROCSCAN is the new way MR8Q312
# create a new list that only has 'MAP' and 'OFF' scans
if not procscan and (obsid == 'MAP' or obsid == 'OFF'):
map_scans[scanid] = obsid
elif (procscan == 'MAP' or
procname == 'TRACK' or
(procname == 'ONOFF' and procscan == 'OFF') or
(procname == 'OFFON' and procscan == 'OFF')):
map_scans[scanid] = procscan
mapkeys = map_scans.keys()
mapkeys.sort()
if debug:
print '------------------------- Relavant scans'
for scanid in mapkeys:
print 'scan', scanid, map_scans[scanid]
maps = [] # final list of maps
ref1 = None
ref2 = None
prev_ref2 = None
mapscans = [] # temporary list of map scans for a single map
if debug:
print 'mapkeys', mapkeys
MapParams = namedtuple("MapParams", "refscan1 mapscans refscan2")
for idx, scan in enumerate(mapkeys):
# look for the reference scans
if (map_scans[scan]).upper() == 'OFF' or (map_scans[scan]).upper() == 'ON':
# if there is no ref1 or this is another ref1
if not ref1 or (ref1 and bool(mapscans) == False):
ref1 = scan
else:
ref2 = scan
prev_ref2 = ref2
elif (map_scans[scan]).upper() == 'MAP':
if not ref1 and prev_ref2:
ref1 = prev_ref2
mapscans.append(scan)
# see if this scan is the last one in the relevant scan list
# or see if we have a ref2
# if so, close out
if ref2 or idx == len(mapkeys)-1:
maps.append(MapParams(ref1, mapscans, ref2))
ref1 = False
ref2 = False
mapscans = []
if debug:
import pprint
pprint.pprint(maps)
for idx, mm in enumerate(maps):
print "Map", idx
if mm.refscan2:
print "\tReference scans.....", mm.refscan1, mm.refscan2
else:
print "\tReference scan......", mm.refscan1
print "\tMap scans...........", mm.mapscans
return maps
def parseSdfitsIndex(self, infile, mapscans=[]):
try:
ifile = open(infile)
except IOError:
print("ERROR: Could not open file: {0}\n"
"Please check and try again.".format(infile))
raise
observation = ObservationRows()
while True:
line = ifile.readline()
# look for start of row data or EOF (i.e. not line)
if '[rows]' in line or not line:
break
lookup_table = {}
header = ifile.readline()
fields = [xx.lstrip() for xx in re.findall(r' *\S+', header)]
iterator = re.finditer(r' *\S+', header)
for idx, mm in enumerate(iterator):
lookup_table[fields[idx]] = slice(mm.start(), mm.end())
rr = SdFitsIndexRowReader(lookup_table)
summary = {'WINDOWS': set([]), 'FEEDS': set([])}
# keep a list of suspect scans so we can know if the
# user has already been warned
suspectScans = set()
for row in ifile:
rr.setrow(row)
scanid = int(rr['SCAN'])
# have a look at the procedure
# if it is "Unknown", the data is suspect, so skip it
procname = rr['PROCEDURE']
if scanid in suspectScans:
continue
if ((scanid not in suspectScans) and procname.lower() == 'unknown'):
suspectScans.add(scanid)
if scanid in mapscans:
print 'WARNING: scan', scanid, 'has "Unknown" procedure. Skipping.'
continue
feed = int(rr['FDNUM'])
windowNum = int(rr['IFNUM'])
pol = int(rr['PLNUM'])
fitsExtension = int(rr['EXT'])
rowOfFitsFile = int(rr['ROW'])
obsid = rr['OBSID']
procscan = rr['PROCSCAN']
nchans = rr['NUMCHN']
summary['WINDOWS'].add((windowNum, float(rr['RESTFREQ'])/1e9))
summary['FEEDS'].add(rr['FDNUM'])
# we can assume all integrations of a single scan are within the same
# FITS extension
observation.addRow(scanid, feed, windowNum, pol,
fitsExtension, rowOfFitsFile, obsid,
procname, procscan, nchans)
try:
ifile.close()
except NameError:
raise
return observation, summary
def getReferenceIntegration(self, cal_on, cal_off, scale):
cal = Calibration()
cal_ondata = cal_on['DATA']
cal_offdata = cal_off['DATA']
cref, exposure = cal.total_power(cal_ondata, cal_offdata, cal_on['EXPOSURE'], cal_off['EXPOSURE'])
tcal = cal_off['TCAL'] * scale
tsys = cal.tsys(tcal, cal_ondata, cal_offdata)
dateobs = cal_off['DATE-OBS']
timestamp = self.pu.dateToMjd(dateobs)
tambient = cal_off['TAMBIENT']
elevation = cal_off['ELEVATIO']
return cref, tsys, exposure, timestamp, tambient, elevation
def nameIndexFile(self, pathname):
# ------------------------------------------------- name index file
if not os.path.exists(pathname):
print ('ERROR: Path does not exist {0}.\n'
' Please check and try again'.format(pathname))
sys.exit(9)
if os.path.isdir(pathname):
bn = os.path.basename(pathname.rstrip('/'))
return '{0}/{1}.index'.format(pathname, bn)
elif os.path.isfile(pathname) and pathname.endswith('.fits'):
return os.path.splitext(pathname)[0]+'.index'
else:
# doMessage(logger,msg.ERR,'input file not recognized as a fits file.',\
# ' Please check the file extension and change to \'fits\' if necessary.')
print 'ERROR: Input file does not end with .fits:', pathname
sys.exit(9)
