def find_calibrator(obsid,
maxtimediff=TimeDelta(1 * u.d),
maxseparation=180 * u.deg,
sourcename=None,
notsourcename=['FDS'],
matchproject=True,
matchnight=True,
priority='time',
all=False):
"""
obsid, timediff, distance=find_calibrator(obsid,
maxtimediff=TimeDelta(1*u.d),
maxseparation=180*u.deg,
sourcename=None,
notsourcename=['FDS'],
matchproject=True,
matchnight=True,
priority='time',
all=False):
timediff is in days
distance is in degrees
"""
assert priority in ['time', 'distance']
starttime = Time(obsid, format='gps', scale='utc') - maxtimediff
stoptime = Time(obsid, format='gps', scale='utc') + maxtimediff
try:
baseobs = metadata.MWA_Observation(obsid)
except Exception, e:
logger.error('Cannot fetch info for observation %d:\n\t%s' %
(obsid, e))
return None
def __init__(self,
obsid,
basedir='./',
clobber=False,
cleanfits=False,
cleanms=False,
checkfree=True,
copycommand='rsync -aruvP'):
self.obsid = obsid
self.basedir = basedir
self.downloadedfiles = None
self.metafits = None
self.cottercommand = None
self.msfile = None
self.clobber = clobber
self.copycommand = copycommand
self.cleanfits = cleanfits
self.cleanms = cleanms
self.destination = None
self.rawsize = None
self.fitssize = None
self.mssize = None
self.checkfree = checkfree
self.outputdir = os.path.join(self.basedir, str(self.obsid), '')
os.chdir(basedir)
logger.info('##################################################')
logger.info('%d' % self.obsid)
self.observation = metadata.MWA_Observation(self.obsid)
logger.debug('\n' + str(self.observation))
obsinfo = metadata.fetch_obsinfo(self.obsid)
self.rawsize = sum(
[obsinfo['files'][f]['size'] for f in obsinfo['files'].keys()])
self.fitssize = sum([
obsinfo['files'][f]['size'] *
(obsinfo['files'][f]['filetype'] == 8)
for f in obsinfo['files'].keys()
])
logger.info('Raw data size is %.1f MB' %
(self.rawsize / 1024. / 1024.))
def main():
usage = "Usage: %prog [options] <obsid>\n"
o = optparse.OptionParser(usage=usage,
version=mwapy.__version__ + ' ' + mwapy.__date__)
o.add_option('--separation',
dest='separation',
default=180,
type='float',
help='Maximum separation (deg) [default=%default]')
o.add_option(
'--timediff',
dest='timediff',
default='1d',
help='Maximum time difference (d, h, m, or s) [default=%default]')
o.add_option('--matchproject',
dest='matchproject',
default=False,
action='store_true',
help='Match project to original ObsID?')
o.add_option('--matchnight',
dest='matchnight',
default=False,
action='store_true',
help='Match night to original ObsID?')
o.add_option('--source',
dest='source',
default=None,
help='Calibrator source name [default=None]')
o.add_option(
'--notsource',
dest='notsource',
default='FDS',
help=
'Comma-separated list of source names to exclude [default=%default]')
o.add_option(
'--priority',
dest='priority',
default='time',
type='choice',
choices=['time', 'distance'],
help='Return the closest in time or distance? [default=%default]')
o.add_option('--all',
dest='all',
default=False,
action='store_true',
help='Return all possible matches?')
o.add_option('-v',
'--verbose',
dest='verbose',
default=False,
action='store_true',
help='Give verbose output?')
options, args = o.parse_args()
if len(args) == 0:
logger.error('Must specify >1 obsids')
sys.exit(1)
if 'd' in options.timediff:
maxtimediff = TimeDelta(float(options.timediff[:-1]) * u.d)
elif 'h' in options.timediff:
maxtimediff = TimeDelta(float(options.timediff[:-1]) * u.h)
elif 'm' in options.timediff:
maxtimediff = TimeDelta(float(options.timediff[:-1]) * u.m)
elif 's' in options.timediff:
maxtimediff = TimeDelta(float(options.timediff[:-1]) * u.s)
maxseparation = options.separation * u.deg
matchproject = options.matchproject
matchnight = options.matchnight
priority = options.priority
for obsid in args:
result = find_calibrator(int(obsid),
maxtimediff=maxtimediff,
maxseparation=maxseparation,
matchproject=matchproject,
matchnight=matchnight,
sourcename=options.source,
notsourcename=options.notsource.split(','),
priority=priority,
all=options.all)
if result is None:
print "None"
elif not options.all:
print '# Obsid\t\tCalObsid\tTimediff(day)\tDistance(deg)\tSource'
print '%s\t%s\t%.3f\t\t%.1f\t\t%s' % (obsid, result[0], result[1],
result[2], result[3])
if options.verbose:
o = metadata.MWA_Observation(int(result[0]))
print o
else:
print '# Obsid\t\tCalObsid\tTimediff(day)\tDistance(deg)\tSource'
for i in xrange(len(result[0])):
print '%s\t%s\t%.3f\t\t%.1f\t\t%s' % (
obsid, result[0][i], result[1][i], result[2][i],
result[3][i])
if options.verbose:
o = metadata.MWA_Observation(int(result[0][i]))
print o
sys.exit(0)
def main():
usage="Usage: %prog [options]\n"
usage+="\tFinds MWA Observations based on a set of query parameters\n"
usage+="\tWill return just the obsids(default) or more information (if verbose)\n"
o = optparse.OptionParser(usage=usage,version=mwapy.__version__ + ' ' + mwapy.__date__)
o.add_option('--limit', type='int', default=100,
help='Limit on number of results returned [default=%default]')
o.add_option('--proj', type='str', default=None,
help = 'Project ID')
#o.add_option('--list_projects', action='store_true',
# help = 'List available project ids and exit')
o.add_option('--start', type='str', default=None,
help='Start time (GPSseconds or ISO UT date)')
o.add_option('--stop', type='str', default=None,
help='Stop time (GPSseconds or ISO UT date)')
o.add_option('--racenter', type='str', default=None,
help='Center RA for search (decimal degrees or hms)')
o.add_option('--deccenter', type='str', default=None,
help='Center Dec for search (decimal degrees or dms)')
o.add_option('--rawidth', type='str', default='10d',
help='Width of RA search box (degrees assumed; will not handle 360d boundary) [default=%default]')
o.add_option('--decwidth', type='str', default='10d',
help='Width of Dec search box (degrees) [default=%default]')
o.add_option('--chan',type='int', default=None,
help='Center channel')
o.add_option('--anychan',type='int',default=None,
help='Match to any channel')
o.add_option('--contiguous', action='store_true',dest='contiguous',default=None,
help='Only include observations with contiguous frequency channels')
o.add_option('--discontiguous', action='store_false',dest='contiguous',default=None,
help='Only include observations with discontiguous frequency channels')
o.add_option('--inttime', type='int', default=None,
help='Integration time (s)')
o.add_option('--freqres', type='int', default=None,
help='Frequency resolution (kHz)')
#o.add_option('--withdata*',action='store_true',
# help="Only include observations with data.")
o.add_option('--cal',action='store_true', default=None,dest='withcal',
help="Only get data that are marked as calibrations")
o.add_option('--nocal',action='store_false', default=None,dest='withcal',
help="Only get data that are not marked as calibrations")
o.add_option('--future',action='store_true', default=None,dest='future',
help="Only get observations that are in the future")
o.add_option('--past',action='store_false', default=None,dest='future',
help="Only get observations that are in the past")
o.add_option('--creator',type='str',default=None,
help='Creator')
o.add_option('--mode',type='str',default=None,
help='Observation mode')
o.add_option('--obsname',type='str',
help='Match any part of an observation name')
o.add_option('--close',default=False, action='store_true',
help='Sort by observations closest in time')
o.add_option('--maxdiff',default='1d',
help='Maximum time difference for proximity search [default=%default]')
o.add_option('-v','--verbose',action="store_true",dest="verbose",default=False,
help="Increase verbosity of output")
o.add_option('--full',action="store_true",dest="full",default=False,
help="Print full info for each observation")
o.add_option('-u','--url',default=metadata._BASEURL,
help="URL for metadata retrieval [default=%default]")
options, args = o.parse_args()
if (options.verbose):
logger.setLevel(logging.INFO)
cal=None
if options.withcal is not None:
if options.withcal:
cal=1
else:
cal=0
contiguous=None
if options.contiguous is not None:
if options.contiguous:
contiguous=1
else:
contiguous=0
future=None
if options.future is not None:
if options.future:
future=1
else:
future=0
if options.start is not None:
GPSstart=parse_time(options.start)
if GPSstart is None:
sys.exit(1)
else:
GPSstart=None
if options.stop is not None:
GPSstop=parse_time(options.stop)
if GPSstop is None:
sys.exit(1)
else:
GPSstop=None
racenter,deccenter=None,None
rawidth,decwidth=None,None
if options.racenter is not None:
try:
racenter=Angle(options.racenter)
except UnitsError:
try:
racenter=Angle(float(options.racenter), unit=u.degree)
except ValueError:
try:
racenter=Angle(options.racenter, unit=u.hourangle)
except:
logger.error('Unable to parse input RA %s' % options.racenter)
sys.exit(1)
except:
print 'othererror'
logger.error('Unable to parse input RA %s' % options.racenter)
sys.exit(1)
if options.deccenter is not None:
try:
deccenter=Angle(options.deccenter, unit=u.degree)
except:
logger.error('Unable to parse input Dec %s' % options.deccenter)
sys.exit(1)
if options.rawidth is not None:
try:
rawidth=Angle(options.rawidth)
except UnitsError:
try:
rawidth=Angle(options.rawidth, unit=u.degree)
except:
logger.error('Unable to parse RA width %s' % options.rawidth)
sys.exit(1)
except:
logger.error('Unable to parse RA width %s' % options.rawidth)
sys.exit(1)
if options.decwidth is not None:
try:
decwidth=Angle(options.decwidth, unit=u.degree)
except:
logger.error('Unable to parse input Dec %s' % options.decwidth)
sys.exit(1)
maxdiff=None
if options.maxdiff is not None:
# try seconds
try:
maxdiff=int(options.maxdiff)
except ValueError:
if options.maxdiff[-1]=='s':
maxdiff=int(options.maxdiff[:-1])
elif options.maxdiff[-1]=='m':
maxdiff=int(float(options.maxdiff[:-1])*60)
elif options.maxdiff[-1]=='h':
maxdiff=int(float(options.maxdiff[:-1])*60*60)
elif options.maxdiff[-1]=='d':
maxdiff=int(float(options.maxdiff[:-1])*86400)
else:
logger.error('Unable to parse time difference %s' % options.maxdiff)
sys.exit(1)
if GPSstart is not None and GPSstop is not None:
logger.info('Searching from %d to %d...' % (GPSstart,GPSstop))
elif GPSstart is not None:
logger.info('Searching from %d...' % (GPSstart))
elif GPSstop is not None:
logger.info('Searching until %d...' % (GPSstop))
if options.proj is not None:
logger.info('Require projectid=%s' % options.proj)
if options.withcal is not None:
if options.withcal:
logger.info('Require calibrator observations')
else:
logger.info('Require noncalibrator observations')
if options.creator is not None:
logger.info('Require creator=%s' % options.creator)
if options.mode is not None:
logger.info('Require mode=%s' % options.mode)
if options.chan is not None:
logger.info('Require center channel=%d' % options.chan)
if options.anychan is not None:
logger.info('Require any channel=%d' % options.anychan)
if options.contiguous is not None:
if options.contiguous:
logger.info('Require contiguous frequency channels')
else:
logger.info('Require discontiguous frequency channels')
if options.inttime:
logger.info('Require integration time=%d s' % options.inttime)
if options.freqres:
logger.info('Require frequency resolution=%d kHz' % options.freqres)
if racenter is not None:
ramin=(max(racenter-rawidth/2,Angle(0,unit=u.degree))).degree
ramax=(min(racenter+rawidth/2,Angle(360,unit=u.degree))).degree
logger.info('Require %.1fd < RA < %.1fd' % (ramin,
ramax))
else:
ramin,ramax=None,None
if deccenter is not None:
decmin=(max(deccenter-decwidth/2,Angle(-90,unit=u.degree))).degree
decmax=(min(deccenter+decwidth/2,Angle(90,unit=u.degree))).degree
logger.info('Require %.1fd < Dec < %.1fd' % (decmin,
decmax))
else:
decmin,decmax=None,None
if options.obsname is not None:
logger.info('Require obsname=%s' % options.obsname)
if options.future is not None:
if options.future:
logger.info('Require observation in the future')
else:
logger.info('Require observation in the past')
if options.close:
if GPSstart is None:
logger.error('Require start time with --close')
sys.exit(1)
logger.info('Will sort observations within %ds of %d...' % (
maxdiff,
GPSstart))
if not options.close:
results=metadata.fetch_observations(URL=options.url,
mintime=GPSstart,
maxtime=GPSstop,
minra=ramin,
maxra=ramax,
mindec=decmin,
maxdec=decmax,
limit=options.limit,
projectid=tokenize(options.proj),
calibration=cal,
contigfreq=contiguous,
cenchan=options.chan,
anychan=options.anychan,
int_time=options.inttime,
freq_res=options.freqres,
creator=tokenize(options.creator),
mode=tokenize(options.mode),
obsname=tokenize(options.obsname),
future=future)
if results is None or len(results)==0:
print 'Query return no results'
sys.exit(0)
if options.verbose:
print metadata.MWA_Observation_Summary.string_header()
else:
print '# starttime'
for item in results:
o=metadata.MWA_Observation_Summary(item)
if options.verbose:
print o
else:
print o.obsid
if options.full:
observation=metadata.MWA_Observation(o.obsid,
rfstream=0,
ionex=False, url=options.url)
print observation
else:
GPSstop=GPSstart+maxdiff
results1=metadata.fetch_observations(URL=options.url,
mintime=GPSstart,
maxtime=GPSstop,
minra=ramin,
maxra=ramax,
mindec=decmin,
maxdec=decmax,
limit=options.limit/2,
projectid=tokenize(options.proj),
calibration=cal,
contigfreq=contiguous,
cenchan=options.chan,
anychan=options.anychan,
int_time=options.inttime,
freq_res=options.freqres,
creator=tokenize(options.creator),
mode=tokenize(options.mode),
obsname=tokenize(options.obsname),
future=future)
GPSstop=GPSstart-maxdiff
try:
results2=metadata.fetch_observations(URL=options.url,
mintime=GPSstop,
maxtime=GPSstart,
minra=ramin,
maxra=ramax,
mindec=decmin,
maxdec=decmax,
limit=options.limit/2,
projectid=tokenize(options.proj),
calibration=cal,
contigfreq=contiguous,
cenchan=options.chan,
anychan=options.anychan,
int_time=options.inttime,
freq_res=options.freqres,
creator=tokenize(options.creator),
mode=tokenize(options.mode),
obsname=tokenize(options.obsname),
future=future)[::-1]
results=results2+results1
except:
results=results1
if results is None or len(results)==0:
print 'Query return no results'
sys.exit(0)
if options.verbose:
print metadata.MWA_Observation_Summary.string_header()
else:
print '# starttime'
last=None
for item in results:
o=metadata.MWA_Observation_Summary(item)
if (last is None or last < GPSstart) and o.obsid >= GPSstart:
print '# ***** %d\t%s' % (GPSstart,
Time(GPSstart,format='gps',
scale='utc').datetime.strftime('%Y-%m-%dT%H:%M:%S'))
if options.verbose:
print o
else:
print o.obsid
last=o.obsid
if options.full:
observation=metadata.MWA_Observation(o.obsid,
rfstream=0,
ionex=False, url=options.url)
print observation
sys.exit(0)
def fluxmatch(image,
catalog='GLEAMIDR3.fits',
fluxcolumn=None,
fluxerrcolumn=None,
racolumn='RAJ2000',
deccolumn='DECJ2000',
nsigma=10,
rmsfactor=3,
matchradius=120,
rejectsigma=3,
maxdistance=20,
minbeam=0.5,
psfextent=1.1,
limit=10,
refineposition=False,
update=False,
prefix=None,
otherimages=[],
updatepoln=False,
updatebane=False,
plot=True,
region=True,
cores=1):
"""
catalog='GLEAMIDR3.fits',
fluxcolumn=None,
fluxerrcolumn=None,
racolumn='RAJ2000',
deccolumn='DECJ2000'
signal-to-noise for source finding
nsigma=10,
ratio of local rms to minimum image RMS that is OK
rmsfactor=3,
distance between catalog source and source in image (arcsec)
matchradius=120,
rejection sigma for flux ratio outliers
rejectsigma=3,
distance from the image center that is OK (deg)
maxdistance=20,
minimum beam power compared to max
minbeam=0.5,
area of source/area of psf threshold
psfextent=1.1,
max ratio of new to old fluxes (or reciprocal)
limit=10,
"""
if not isinstance(matchradius, astropy.units.quantity.Quantity):
matchradius = matchradius * u.arcsec
if not isinstance(maxdistance, astropy.units.quantity.Quantity):
maxdistance = maxdistance * u.deg
if not os.path.exists(image):
logger.error('Cannot find input image %s' % image)
return None
if updatepoln:
for stokes in ['Q', 'U', 'V']:
if os.path.exists(image.replace('-I.fits', '-%s.fits' % stokes)):
otherimages.append(
image.replace('-I.fits', '-%s.fits' % stokes))
logger.info('Will also scale %s' % otherimages[-1])
if updatebane:
for ext in ['rms', 'bkg']:
if os.path.exists(image.replace('-I.fits', '-I_%s.fits' % ext)):
otherimages.append(image.replace('-I.fits',
'-I_%s.fits' % ext))
logger.info('Will also scale %s' % otherimages[-1])
if updatepoln:
for stokes in ['Q', 'U', 'V']:
if os.path.exists(
image.replace('-I.fits',
'-%s_%s.fits' % (stokes, ext))):
otherimages.append(
image.replace('-I.fits',
'-%s_%s.fits' % (stokes, ext)))
logger.info('Will also scale %s' % otherimages[-1])
if not os.path.exists(catalog):
logger.error('Cannot find GLEAM catalog %s' % catalog)
return None
beam = find_beam(image)
if beam is None:
logger.warning('Did not generate primary beam: will ignore')
minbeam = None
if beam is not None and not os.path.exists(beam):
logger.warning('Cannot find primary beam %s: will ignore' % beam)
minbeam = None
beam = None
outbase = os.path.splitext(image)[0]
sources, rmsimage, bgimage = find_sources_in_image(image,
nsigma=nsigma,
cores=cores)
logger.info('Found %d sources above %d sigma in %s' %
(len(sources), nsigma, image))
logger.info('Wrote %s and %s' % (rmsimage, bgimage))
# convert to astropy table
sourcesTable = aegean2table(sources)
fimage = fits.open(image)
frequency = fimage[0].header['CRVAL3']
w = WCS(fimage[0].header, naxis=2)
frmsimage = fits.open(rmsimage)
minrms = numpy.nanmin(frmsimage[0].data)
logger.info('Minimum RMS in image is %.1f mJy' % (minrms * 1e3))
if beam is not None:
fbeam = fits.open(beam)
x, y = w.wcs_world2pix(sourcesTable['RA'], sourcesTable['Dec'], 0)
sourcesTable.add_column(Column(x, name='X'))
sourcesTable.add_column(Column(y, name='Y'))
if 'RA' in fimage[0].header.keys():
pointingcenter = SkyCoord(fimage[0].header['RA'],
fimage[0].header['DEC'],
unit=('deg', 'deg'))
else:
# get the pointing center from the metadata
logger.warning(
'Pointing metadata not present in header; retrieving...')
try:
obs = metadata.MWA_Observation(fimage[0].header['GPSTIME'])
logger.info('Found pointing center %f,%f' % (obs.RA, obs.Dec))
pointingcenter = SkyCoord(obs.RA, obs.Dec, unit=('deg', 'deg'))
except:
logger.warning('Using CRVAL1/CRVAL2 for pointing center')
pointingcenter = SkyCoord(fimage[0].header['CRVAL1'],
fimage[0].header['CRVAL2'],
unit=('deg', 'deg'))
coords = SkyCoord(sourcesTable['RA'],
sourcesTable['Dec'],
unit=(u.deg, u.deg))
sourcesTable.add_column(
Column(coords.separation(pointingcenter).to(u.deg), name='SOURCEDIST'))
if beam is not None:
pixelx, pixely = numpy.int16(x), numpy.int16(y)
pixelx[pixelx < 0] = 0
pixely[pixely < 0] = 0
pixelx[pixelx >= fbeam[0].data.shape[-1]] = fbeam[0].data.shape[-1] - 1
pixely[pixely >= fbeam[0].data.shape[-2]] = fbeam[0].data.shape[-2] - 1
sourcesTable.add_column(
Column(fbeam[0].data[0, 0, pixelx, pixely], name='BEAM'))
else:
sourcesTable.add_column(Column(0 * x, name='BEAM'))
if '.fits' in catalog:
# this seems to be faster than going straight to the Table.read()
try:
fcatalog = fits.open(catalog)
except:
logger.error('Unable to open FITS catalog %s' % catalog)
return None
catalogTable = Table(fcatalog[1].data)
else:
try:
catalogTable = Table.read(catalog)
except:
logger.error('Unable to read catalog %s' % catalog)
return None
try:
bandfrequencies = numpy.array([
int(s.split('_')[-1])
for s in numpy.array(catalogTable.colnames)[numpy.nonzero(
numpy.array([('int_flux' in c) and not ('deep' in c)
and not ('wide' in c)('fit' in c)
for c in catalogTable.colnames]))[0]]
])
except:
bandfrequencies = []
if len(bandfrequencies) > 0:
# find the indices of the bands just above and below the observation
# linearly weight the fluxes just above and below to match
# the observation frequency
indexplus = (bandfrequencies >= frequency / 1e6).nonzero()[0].min()
indexminus = (bandfrequencies < frequency / 1e6).nonzero()[0].max()
logger.info(
'Observation frequency of %.1f MHz: interpolating between %d MHz and %d MHz'
% (frequency / 1e6, bandfrequencies[indexminus],
bandfrequencies[indexplus]))
weightplus = (frequency / 1e6 - bandfrequencies[indexminus]) / (
bandfrequencies[indexplus] - bandfrequencies[indexminus])
weightminus = 1 - weightplus
gleamflux = catalogTable[
'int_flux_%03d' %
bandfrequencies[indexminus]] * weightminus + catalogTable[
'int_flux_%03d' % bandfrequencies[indexplus]] * weightplus
try:
gleamfluxerr = numpy.sqrt(
(catalogTable['err_fit_flux_%03d' %
bandfrequencies[indexminus]] * weightminus)**2 +
(catalogTable['err_fit_flux_%03d' %
bandfrequencies[indexplus]] * weightplus)**2)
except KeyError:
gleamfluxerr = numpy.sqrt(
(catalogTable['err_int_flux_%03d' %
bandfrequencies[indexminus]] * weightminus)**2 +
(catalogTable['err_int_flux_%03d' %
bandfrequencies[indexplus]] * weightplus)**2)
else:
logger.warning('Could not identify GLEAM band fluxes')
if fluxcolumn is None:
logger.error('Could not identify flux columns to use')
return None
if fluxcolumn in catalogTable.colnames and fluxerrcolumn in catalogTable.colnames:
logger.warning('Using %s and %s columns' %
(fluxcolumn, fluxerrcolumn))
gleamflux = catalogTable[fluxcolumn]
gleamfluxerr = catalogTable[fluxerrcolumn]
else:
logger.error('Could not identify flux columns to use')
return None
try:
catalogcoords = SkyCoord(catalogTable[racolumn],
catalogTable[deccolumn],
unit=(u.deg, u.deg))
except KeyError:
catalogcoords = SkyCoord(catalogTable['RAJ2000'],
catalogTable['DEJ2000'],
unit=(u.deg, u.deg))
racolumn = 'RAJ2000'
deccolumn = 'DEJ2000'
# match the catalog to the data
idx, sep2d, sep3d = coords.match_to_catalog_sky(catalogcoords)
# add the matched columns to the soure table
try:
sourcesTable.add_column(Column(catalogTable['Name'][idx], name='Name'))
except:
pass
sourcesTable.add_column(Column(catalogTable[racolumn][idx],
name='GLEAMRA'))
sourcesTable.add_column(
Column(catalogTable[deccolumn][idx], name='GLEAMDEC'))
sourcesTable.add_column(Column(sep2d.to(u.arcsec), name='GLEAMSep'))
sourcesTable.add_column(Column(gleamflux[idx], name='GLEAMFlux'))
sourcesTable.add_column(Column(gleamfluxerr[idx], name='GLEAMFluxErr'))
try:
sourcesTable.add_column(
Column(
catalogTable['psf_a_%03d' % bandfrequencies[indexplus]][idx] *
catalogTable['psf_b_%03d' % bandfrequencies[indexplus]][idx],
name='PSFAREA'))
sourcesTable.add_column(
Column(catalogTable['a_%03d' % bandfrequencies[indexplus]][idx] *
catalogTable['b_%03d' % bandfrequencies[indexplus]][idx],
name='SOURCEAREA'))
except:
pass
dRA = (sourcesTable['RA'] - sourcesTable['GLEAMRA'])
dDEC = (sourcesTable['Dec'] - sourcesTable['GLEAMDEC'])
iterations = 1
if refineposition:
iterations = 2
for iter in xrange(iterations):
# determine the good matches
# first criterion is separation
good = (sourcesTable['GLEAMSep'] < matchradius)
logger.info('%04d/%04d sources are within %.1f arcsec' %
(good.sum(), len(good), matchradius.to(u.arcsec).value))
# only point sources
if psfextent is not None and psfextent > 0:
good = good & (sourcesTable['SOURCEAREA'] <=
psfextent * sourcesTable['PSFAREA'])
logger.info(
'%04d/%04d sources also have source a*b < %.1f * psf a*b' %
(good.sum(), len(good), psfextent))
# cut on the local rms compared to the minimum in the image
if rmsfactor is not None and rmsfactor > 0:
good = good & (sourcesTable['RMS'] <= rmsfactor * minrms)
logger.info('%04d/%04d sources also have RMS < %.1f mJy' %
(good.sum(), len(good), rmsfactor * minrms * 1e3))
# distance from pointing center
if maxdistance is not None and maxdistance > 0:
good = good & (sourcesTable['SOURCEDIST'] < maxdistance)
logger.info(
'%04d/%04d sources also are within %.1f deg of pointing center'
% (good.sum(), len(good), maxdistance.to(u.deg).value))
# primary beam power
if minbeam is not None and minbeam > 0:
good = good & (sourcesTable['BEAM'] >
minbeam * fbeam[0].data.max())
logger.info(
'%04d/%04d sources also are at primary beam power > %.2f' %
(good.sum(), len(good), minbeam))
# require that all sources are > 5 sigma detections
# and that flux uncertainties are > 0
ignorefluxerrs = True
if numpy.all(sourcesTable['IntFluxErr'] < 0) or ignorefluxerrs:
logger.warning('All source uncertainties are < 0: will ignore')
else:
good = good & (sourcesTable['IntFluxErr'] <
0.2 * sourcesTable['IntFlux']) & (
sourcesTable['IntFluxErr'] >
0) & (sourcesTable['GLEAMFluxErr'] >
0) & (sourcesTable['GLEAMFluxErr'] <
0.2 * sourcesTable['GLEAMFlux'])
try:
good = good & (sourcesTable['GLEAMFlux'] >=
sourcesTable['IntFlux'][good].min())
pass
except ValueError:
logger.warning('No good sources left')
good = numpy.array([False] * len(good))
logger.info('%04d/%04d sources match all cuts' %
(good.sum(), len(good)))
if good.sum() < 5:
logger.error('Insufficient sources for flux scaling')
return None
fitres = numpy.polyfit(sourcesTable['GLEAMFlux'][good],
sourcesTable['IntFlux'][good],
deg=1,
w=1 / sourcesTable['IntFluxErr'][good]**2)
ratio = sourcesTable['IntFlux'] / sourcesTable['GLEAMFlux']
ratioerr = numpy.sqrt(
(sourcesTable['IntFluxErr'] / sourcesTable['GLEAMFlux'])**2 +
(sourcesTable['IntFlux'] * sourcesTable['GLEAMFluxErr'] /
sourcesTable['GLEAMFlux']**2)**2)
if rejectsigma is not None:
# do a bit of sigma clipping just in case
good = (good) & (numpy.abs(ratio - numpy.median(ratio[good])) <=
ratioerr * rejectsigma)
fittedratio = (ratio[good] / ratioerr[good]**2).sum() / (
1 / ratioerr[good]**2).sum()
fittedratioerr = numpy.sqrt(1 / (1 / ratioerr[good]**2).sum())
chisq = (((ratio[good] - fittedratio) / ratioerr[good])**2).sum()
ndof = good.sum() - 1
logger.info('Found ratio of %s / %s = %.3f +/- %.3f' %
(image, catalog, fittedratio, fittedratioerr))
if refineposition and iter == 0:
sourcesTable['RA'] -= dRA[good].mean()
sourcesTable['Dec'] -= dDEC[good].mean()
logger.info('Applied shift of (%.1f sec, %.1f arcsec)' %
(dRA[good].mean() * 3600, dDEC[good].mean() * 3600))
coords = SkyCoord(sourcesTable['RA'],
sourcesTable['Dec'],
unit=(u.deg, u.deg))
idx, sep2d, sep3d = coords.match_to_catalog_sky(catalogcoords)
sourcesTable['GLEAMSep'] = sep2d.to(u.arcsec)
sourcesTable.add_column(Column(good, name='GOOD'))
sourcesTable.meta['ratio'] = fittedratio
sourcesTable.meta['ratio_err'] = fittedratioerr
sourcesTable.meta['chisq'] = chisq
sourcesTable.meta['ndof'] = ndof
sourcesTable.meta['slope'] = fitres[0]
sourcesTable.meta['intercept'] = fitres[1]
if refineposition:
sourcesTable.meta['rashift'] = dRA[good].mean() * 3600
sourcesTable.meta['decshift'] = dDEC[good].mean() * 3600
if os.path.exists(outbase + '_fluxmatch.hdf5'):
os.remove(outbase + '_fluxmatch.hdf5')
sourcesTable.write(outbase + '_fluxmatch.hdf5', path='data')
logger.info('Wrote %s_fluxmatch.hdf5' % outbase)
if region:
outreg = outbase + '_fluxmatch.reg'
if os.path.exists(outreg):
os.remove(outreg)
foutreg = open(outreg, 'w')
for i in xrange(len(sourcesTable)):
if sourcesTable[i]['GOOD']:
foutreg.write(
'icrs;circle(%f,%f,60") # text={%03d} color={green}\n' %
(sourcesTable[i]['RA'], sourcesTable[i]['Dec'], i))
else:
foutreg.write(
'icrs;box(%f,%f,60",60",0) # text={%03d} color={red}\n' %
(sourcesTable[i]['RA'], sourcesTable[i]['Dec'], i))
logger.info('Wrote %s' % outreg)
foutreg.close()
if update:
if fittedratio > limit or fittedratio < 1.0 / limit:
logger.warning('Ratio exceeds reasonable limits; skipping...')
else:
fimage = fits.open(image, 'update')
if not 'BEAM' in fimage[0].header.keys():
fimage[0].header['BEAM'] = beam
fimage[0].data /= fittedratio
fimage[0].header['FLUXSCAL'] = (fittedratio,
'Flux scaling relative to catalog')
fimage[0].header['FLUX_ERR'] = (
fittedratioerr, 'Flux scaling uncertainty relative to catalog')
fimage[0].header['FLUXCAT'] = (catalog, 'Flux scaling catalog')
fimage[0].header['NFLUXSRC'] = (
good.sum(), 'Number of sources used for flux scaling')
fimage[0].header['FLUXCHI2'] = (chisq, 'Flux scaling chi-squared')
fimage[0].header['FLUXSLOP'] = (fitres[0], 'Flux scaling slope')
if refineposition:
fimage[0].header['RASHIFT'] = (
dRA[good].mean() * 3600, '[s] RA Shift for catalog match')
fimage[0].header['DECSHIFT'] = (
dDEC[good].mean() * 3600,
'[arcsec] DEC Shift for catalog match')
fimage[0].header['CRVAL1'] -= dRA[good].mean()
fimage[0].header['CRVAL2'] -= dDEC[good].mean()
if 'IMAGERMS' in fimage[0].header.keys():
fimage[0].header['IMAGERMS'] /= fittedratio
if prefix is None:
fimage.flush()
logger.info('Scaled %s by %.3f' % (image, fittedratio))
else:
p, f = os.path.split(image)
outfile = os.path.join(p, prefix + f)
if os.path.exists(outfile):
os.remove(outfile)
fimage.writeto(outfile)
logger.info('Scaled %s by %.3f and wrote to %s' %
(image, fittedratio, outfile))
if otherimages is not None and len(otherimages) > 0:
# also update some other images
for otherimage in otherimages:
foimage = fits.open(otherimage, 'update')
foimage[0].data /= fittedratio
foimage[0].header['FLUXIMG'] = (
image, 'Image used for flux scaling')
foimage[0].header['FLUXSCAL'] = (
fittedratio, 'Flux scaling relative to catalog')
foimage[0].header['FLUX_ERR'] = (
fittedratioerr,
'Flux scaling uncertainty relative to catalog')
foimage[0].header['FLUXCAT'] = (catalog,
'Flux scaling catalog')
foimage[0].header['NFLUXSRC'] = (
good.sum(), 'Number of sources used for flux scaling')
foimage[0].header['FLUXCHI2'] = (
chisq, 'Flux scaling chi-squared')
foimage[0].header['FLUXSLOP'] = (fitres[0],
'Flux scaling slope')
if refineposition:
foimage[0].header['RASHIFT'] = (
dRA[good].mean() * 3600,
'[s] RA Shift for catalog match')
foimage[0].header['DECSHIFT'] = (
dDEC[good].mean() * 3600,
'[arcsec] DEC Shift for catalog match')
foimage[0].header['CRVAL1'] -= dRA[good].mean()
foimage[0].header['CRVAL2'] -= dDEC[good].mean()
if 'IMAGERMS' in fimage[0].header.keys():
foimage[0].header['IMAGERMS'] /= fittedratio
if prefix is None:
foimage.flush()
logger.info('Scaled %s by %.3f' %
(otherimage, fittedratio))
else:
p, f = os.path.split(otherimage)
outfile = os.path.join(p, prefix + f)
if os.path.exists(outfile):
os.remove(outfile)
fimage.writeto(outfile)
logger.info('Scaled %s by %.3f and wrote to %s' %
(otherimage, fittedratio, outfile))
if plot:
imagename = image.replace('_', '\_')
plt.clf()
xx = numpy.logspace(-2, 10)
plt.loglog(xx, xx * fittedratio, 'r')
plt.loglog(xx, numpy.polyval(fitres, xx), 'r--')
plt.errorbar(sourcesTable[good]['GLEAMFlux'],
sourcesTable[good]['IntFlux'],
xerr=sourcesTable[good]['GLEAMFluxErr'],
yerr=sourcesTable[good]['IntFluxErr'],
fmt='b.')
#plt.gca().set_xscale('log')
#plt.gca().set_yscale('log')
plt.axis([0.1, 100, 0.1, 100])
plt.xlabel('Flux Density in %s (Jy)' % catalog.replace('_', '\_'),
fontsize=16)
plt.ylabel('Flux Density in %s (Jy)' % imagename.replace('_', '\_'),
fontsize=16)
plt.gca().tick_params(labelsize=16)
plt.savefig('%s_fluxflux.pdf' % outbase)
logger.info('Wrote %s_fluxflux.pdf' % outbase)
plt.clf()
plt.hist(ratio[good], 30)
plt.xlabel('Flux Density in %s / Flux Density in %s' %
(imagename.replace('_', '\_'), catalog.replace('_', '\_')),
fontsize=16)
plt.ylabel('Number of Sources', fontsize=16)
plt.plot(fittedratio * numpy.array([1, 1]), plt.gca().get_ylim(), 'r-')
plt.gca().tick_params(labelsize=16)
plt.savefig('%s_hist.pdf' % outbase)
logger.info('Wrote %s_hist.pdf' % outbase)
plt.clf()
plt.plot(x, y, 'k.')
h = plt.scatter(x[good],
y[good],
s=60,
c=ratio[good],
norm=matplotlib.colors.LogNorm(vmin=0.5, vmax=2),
cmap=plt.cm.BrBG)
plt.xlabel('X', fontsize=16)
plt.ylabel('Y', fontsize=16)
cbar = plt.gcf().colorbar(h, ticks=[0.5, 1, 2])
plt.gca().tick_params(labelsize=16)
plt.savefig('%s_scatter.pdf' % outbase)
logger.info('Wrote %s_scatter.pdf' % outbase)
plt.clf()
plt.plot(
(sourcesTable['RA'][good] - sourcesTable['GLEAMRA'][good]) * 3600,
(sourcesTable['Dec'][good] - sourcesTable['GLEAMDEC'][good]) *
3600, 'ro')
plt.plot(plt.gca().get_xlim(), [0, 0], 'k--')
plt.plot([0, 0], plt.gca().get_ylim(), 'k--')
plt.xlabel('$\\alpha$(%s)-$\\alpha$(%s)' %
(imagename.replace('_', '\_'), catalog.replace('_', '\_')),
fontsize=16)
plt.ylabel('$\\delta$(%s)-$\\delta$(%s)' %
(imagename.replace('_', '\_'), catalog.replace('_', '\_')),
fontsize=16)
plt.gca().tick_params(labelsize=16)
plt.savefig('%s_position.pdf' % outbase)
logger.info('Wrote %s_position.pdf' % outbase)
plt.clf()
xx = numpy.linspace(0, 300, 50)
plt.hist(sourcesTable['GLEAMSep'].to(u.arcsec).value[~good],
xx,
color='b',
alpha=0.5)
plt.hist(sourcesTable['GLEAMSep'].to(u.arcsec).value[good],
xx,
color='r',
alpha=0.5)
plt.plot(
matchradius.to(u.arcsec).value * numpy.array([1, 1]),
plt.gca().get_ylim(), 'k--')
plt.xlabel('Separation %s vs. %s (arcsec)' %
(imagename.replace('_', '\_'), catalog.replace('_', '\_')),
fontsize=16)
plt.ylabel('Number of sources', fontsize=16)
plt.gca().tick_params(labelsize=16)
plt.savefig('%s_separation.pdf' % outbase)
logger.info('Wrote %s_separation.pdf' % outbase)
return fittedratio, fittedratioerr, chisq, ndof, fitres[0], fitres[1]
def main():
localswarpconfig = os.path.join(os.path.split(__file__)[0], swarpconfig)
usage = "Usage: %prog [options] <files>\n"
usage += "\tMosaic together MWA driftscan data\n"
#usage+="\tImages should be either raw XX,YY or primary-beam corrected I (or Q or U or V)\n"
usage += "\tImages should be either raw XX,YY or primary-beam corrected I\n"
usage += "\tNormal operation:\n"
usage += "\tXX,YY data will be converted into Stokes I\n"
usage += "\tStokes I beams will be computed for optimal weighting\n"
usage += "\tExcept if --separate and then only XX,YY will be processed and mosaiced separately\n"
usage += "\tswarp is used for mosaicing\n"
parser = OptionParser(usage=usage,
version=mwapy.__version__ + ' ' + mwapy.__date__)
parser.add_option('--metafits',
action="store_true",
dest="metafits",
default=False,
help="Force creation of metafits files?")
parser.add_option(
'--copybeam',
action="store_true",
dest="copy_beam",
default=False,
help="Copy the beam for the driftscan instead of creating separately?")
parser.add_option(
'--analytic',
action="store_true",
dest="analytic_model",
default=False,
help=
"Use the old analytic dipole model, instead of the default Sutinjo 2014 model."
)
parser.add_option(
'--noprecess',
action='store_false',
dest='precess',
default=True,
help=
'Do not precess coordinates to current epoch (faster but less accurate) [default=False]'
)
parser.add_option('-t',
'--threshold',
dest='threshold',
default=0.05,
type='float',
help='Beam threshold [default=%default]')
parser.add_option('--reduce',
dest='reduce',
default=False,
action='store_true',
help='Reduce dimensionality of input images?')
parser.add_option('--oversample',
dest='oversample',
default=2,
type='int',
help='Swarp oversampling [default=%default]')
parser.add_option(
'--ra',
dest='ra',
default=None,
help='Center RA of mosaic in hours or "middle" [default=auto]')
parser.add_option('--dec',
dest='dec',
default=None,
help='Center Dec of mosaic in degrees [default=auto]')
parser.add_option('--proj',
dest='projection',
default='MOL',
type='choice',
choices=['MOL', 'ZEA', 'SIN', 'AIT'],
help='Image projection [default=%default]')
parser.add_option(
'--scale',
'--scalar',
dest='scalarweight',
default=None,
help=
'Additional scalar weight header keyword for each image [default=None]'
)
parser.add_option(
'--function',
dest='weightfunction',
default='inversesquare',
type='choice',
choices=['inversesquare', 'inverse', 'linear', 'square'],
help=
'Function applied to scalar weight (higher weight is better) [default=%default]'
)
parser.add_option(
'--separate',
dest='separate',
default=False,
action='store_true',
help='Mosaic XX and YY separately as well [default=%default]')
parser.add_option('--arguments',
dest='swarpcommands',
default='',
help='Additional arguments for Swarp')
parser.add_option('--out',
dest='root',
default='mosaic',
help='SWARP output root [default=%default]')
parser.add_option('--swarp',
dest='swarp',
default=None,
help='Path to Swarp executable [default=search path]')
parser.add_option('--config',
dest='config',
default=None,
help='Path to separate Swarp config file [default=%s]' %
localswarpconfig)
parser.add_option('--clean',
action="store_true",
dest="clean",
default=False,
help="Clean temporary files?")
parser.add_option('-v',
'--verbose',
action="store_true",
dest="verbose",
default=False,
help="Increase verbosity of output")
(options, args) = parser.parse_args()
if (options.verbose):
logger.setLevel(logging.INFO)
if options.config is not None:
localswarpconfig = options.config
if not os.path.exists(localswarpconfig):
logger.error('Cannot find swarp config file %s' % localswarpconfig)
sys.exit(1)
logger.info('Will use config file %s...' % localswarpconfig)
inputfiles = args
if len(inputfiles) == 0:
logger.error('Must supply >=1 input images')
sys.exit(1)
Imagelist = {}
if options.swarp is None:
# python 2.6 compatibility
try:
swarp = subprocess.check_output(['which', 'swarp']).strip()
except AttributeError:
swarp = subprocess.Popen(
['which', 'swarp'],
stdout=subprocess.PIPE).communicate()[0].strip()
else:
swarp = options.swarp
##############################
# go through the images
# figure out which is for which polarization
# and see if there is also a metafits file
##############################
times = {}
times['start'] = time.time()
for file in inputfiles:
polaxis = None
try:
obsid = int(file.split('_')[0])
except:
try:
# allow for wsclean-style output
obsid = int(file.split('-')[0])
except:
logger.error('Unable to interpret filename %s' % file)
sys.exit(0)
if not Imagelist.has_key(obsid):
Imagelist[obsid] = MWAImage(obsid)
mwaimage = Imagelist[obsid]
currentstokes = None
try:
f = pyfits.open(file)
if f[0].header['CTYPE3'] == 'STOKES':
polaxis = 3
if f[0].header['CTYPE4'] == 'STOKES':
polaxis = 4
if not polaxis is None:
if f[0].header['CRVAL%d' % polaxis] == -5:
mwaimage.ximage = file
currentstokes = 'XX'
elif f[0].header['CRVAL%d' % polaxis] == -6:
mwaimage.yimage = file
currentstokes = 'YY'
elif f[0].header['CRVAL%d' % polaxis] >= 1 and f[0].header[
'CRVAL%d' % polaxis] <= 4:
mwaimage.iimage = file
currentstokes = 'I'
else:
logger.warning(
'Do not know how to process Stokes=%d for %s; skipping...'
% (f[0].header['CRVAL%d' % polaxis], file))
continue
else:
if 'XX' in file:
mwaimage.ximage = file
currentstokes = 'XX'
elif 'YY' in file:
mwaimage.yimage = file
currentstokes = 'YY'
# see if beams already exist
# would be identified through header keywords
if 'BEAM' in f[0].header.keys():
if currentstokes == 'I' and os.path.exists(
f[0].header['BEAM']):
mwaimage.ibeam = f[0].header['BEAM']
elif currentstokes == 'XX' and os.path.exists(
f[0].header['BEAM']):
mwaimage.xbeam = f[0].header['BEAM']
elif currentstokes == 'YY' and os.path.exists(
f[0].header['BEAM']):
mwaimage.ybeam = f[0].header['BEAM']
except:
if 'XX' in file:
mwaimage.ximage = file
currentstokes = 'XX'
elif 'YY' in file:
mwaimage.yimage = file
currentstokes = 'YY'
if os.path.exists('%d.metafits' % obsid):
mwaimage.metafits = '%d.metafits' % obsid
logger.info('Found %d input images for %d observations\n' %
(len(inputfiles), len(Imagelist.keys())))
times['beams'] = time.time()
##############################
# now loop through and do what needs to be done
##############################
beams = {}
tempfiles = []
for obsid in sorted(Imagelist.keys()):
logger.info('Processing %d...' % obsid)
if Imagelist[obsid].ximage is None and Imagelist[
obsid].yimage is not None:
logger.error('Obsid %d has YY image but no XX image' % obsid)
sys.exit(1)
if Imagelist[obsid].yimage is None and Imagelist[
obsid].ximage is not None:
logger.error('Obsid %d has XX image but no YY image' % obsid)
sys.exit(1)
s = '%d: ' % obsid
if Imagelist[obsid].ximage is not None:
s += '(XX,YY)=(%s, %s)' % (Imagelist[obsid].ximage,
Imagelist[obsid].yimage)
if Imagelist[obsid].iimage is not None:
s += ' I=%s' % Imagelist[obsid].iimage
if Imagelist[obsid].metafits is not None:
s += ' meta=%s' % Imagelist[obsid].metafits
logger.info(s)
if Imagelist[obsid].xbeam is not None or Imagelist[
obsid].ibeam is not None:
s = '%d: ' % obsid
if Imagelist[obsid].xbeam is not None:
s += '(XXbeam,YYbeam)=(%s, %s)' % (Imagelist[obsid].xbeam,
Imagelist[obsid].ybeam)
if Imagelist[obsid].ibeam is not None:
s += ' Ibeam=%s' % Imagelist[obsid].ibeam
logger.info(s)
if Imagelist[obsid].iimage is None:
# no I image exists: deal with XX, YY
fx = pyfits.open(Imagelist[obsid].ximage)
fy = pyfits.open(Imagelist[obsid].yimage)
# Get scaling from imageheader
if options.scalarweight is not None:
try:
Imagelist[obsid].xscalarweight = weightfunction(
fx[0].header[options.scalarweight],
options.weightfunction)
except:
logger.warning(
'Unable to read header keyword "%s" from image %s; setting I scalar weight to 1'
% (options.scalarweight, Imagelist[obsid].ximage))
Imagelist[obsid].xscalarweight = 1
try:
Imagelist[obsid].yscalarweight = weightfunction(
fy[0].header[options.scalarweight],
options.weightfunction)
except:
logger.warning(
'Unable to read header keyword "%s" from image %s; setting I scalar weight to 1'
% (options.scalarweight, Imagelist[obsid].yimage))
Imagelist[obsid].yscalarweight = 1
try:
result = fx[0].header['DELAYS'] is not None and fx[0].header[
'DATE-OBS'] is not None
xheader = True
except KeyError:
xheader = False
try:
result = fy[0].header['DELAYS'] is not None and fy[0].header[
'DATE-OBS'] is not None
yheader = True
except KeyError:
yheader = False
if not (xheader or yheader) or options.metafits:
# does not have the necessary keys
# need to make metafits
if Imagelist[obsid].metafits is None:
logger.warning(
'FITS header keywords not present for obsid %d; creating metafits...'
% obsid)
if not _newmetadata:
hh = make_metafiles.Corr2UVFITSHeader(obsid, db=db)
T = make_metafiles.instrument_configuration(
gpstime=obsid, duration=hh.obs.duration, db=db)
result = T.make_instr_config(quick=True)
if result is None:
logger.error(
'Error making instr_config file for %d' %
obsid)
sys.exit(1)
hh.make_header()
T.corr2uvfitsheader = hh
h = T.make_metafits(quick=True)
else:
# use newer interface
obs = metadata.MWA_Observation(obsid)
if obs.observation_number is None:
logger.error('No observation found for %s' % obsid)
sys.exit(1)
m = metadata.instrument_configuration(obsid)
h = m.make_metafits(quick=True)
Imagelist[obsid].metafits = '%d.metafits' % obsid
if os.path.exists(Imagelist[obsid].metafits):
os.remove(Imagelist[obsid].metafits)
try:
h.writeto(Imagelist[obsid].metafits)
logger.info('Metafits for %d written to %s' %
(obsid, Imagelist[obsid].metafits))
except Exception, e:
logger.error(
'Unable to write metafits file for %d:\n%s' %
(obsid, e))
sys.exit(1)
if not xheader:
fx.close()
if not Imagelist[obsid].metafits is None:
logger.warning(
'Copying metafits from %s to %s' %
(Imagelist[obsid].metafits, Imagelist[obsid].ximage))
result = copy_metafits(Imagelist[obsid].ximage,
Imagelist[obsid].metafits)
else:
# assume we can get it from the YY image?
logger.warning(
'Copying metafits from %s to %s' %
(Imagelist[obsid].yimage, Imagelist[obsid].ximage))
result = copy_metafits(Imagelist[obsid].ximage,
Imagelist[obsid].yimage,
keys=['DELAYS', 'DATE-OBS'])
if not result:
logger.error('Error updating FITS header of %s' %
Imagelist[obsid].ximage)
sys.exit(1)
if not yheader:
fy.close()
if not Imagelist[obsid].metafits is None:
logger.warning(
'Copying metafits from %s to %s' %
(Imagelist[obsid].metafits, Imagelist[obsid].yimage))
result = copy_metafits(Imagelist[obsid].yimage,
Imagelist[obsid].metafits)
else:
# assume we can get it from the YY image?
logger.warning(
'Copying metafits from %s to %s' %
(Imagelist[obsid].ximage, Imagelist[obsid].yimage))
result = copy_metafits(Imagelist[obsid].yimage,
Imagelist[obsid].ximage,
keys=['DELAYS', 'DATE-OBS'])
if not result:
logger.error('Error updating FITS header of %s' %
Imagelist[obsid].yimage)
sys.exit(1)
##############################
# now create the beams
##############################
fx = pyfits.open(Imagelist[obsid].ximage, 'update')
fy = pyfits.open(Imagelist[obsid].yimage, 'update')
delays = [int(x) for x in fx[0].header['DELAYS'].split(',')]
if Imagelist[obsid].xbeam is not None and Imagelist[
obsid].ybeam is not None:
logger.info(
'XX,YY beams already exists for %d; using %s,%s...' %
(obsid, Imagelist[obsid].xbeam, Imagelist[obsid].ybeam))
elif not options.copy_beam or (not beams.has_key(
fx[0].header['DELAYS'])):
logger.info('Creating primary beams for %d' % obsid)
out = make_beam.make_beam(
Imagelist[obsid].ximage,
ext=0,
delays=delays,
analytic_model=options.analytic_model,
jones=False,
precess=options.precess)
if out is None:
logger.error('Problem creating primary beam for %s' %
obsid)
sys.exit(1)
Imagelist[obsid].xbeam, Imagelist[obsid].ybeam = out
beams[fx[0].header['DELAYS']] = obsid
logger.info('Wrote %s,%s' % (out[0], out[1]))
fx[0].header['BEAM'] = Imagelist[obsid].xbeam
fy[0].header['BEAM'] = Imagelist[obsid].ybeam
fx.flush()
fy.flush()
tempfiles += out
else:
logger.info('Copying beam from %s to %s' %
(Imagelist[beams[fx[0].header['DELAYS']]].ximage,
Imagelist[obsid].ximage))
result = copy_driftscanbeam(
Imagelist[beams[fx[0].header['DELAYS']]].ximage,
Imagelist[obsid].ximage)
if result is None:
sys.exit(1)
Imagelist[obsid].xbeam, Imagelist[obsid].ybeam = result
fx[0].header['BEAM'] = Imagelist[obsid].xbeam
fy[0].header['BEAM'] = Imagelist[obsid].ybeam
fx.flush()
fy.flush()
tempfiles += result
fx.close()
fy.close()
fx = pyfits.open(Imagelist[obsid].ximage)
fy = pyfits.open(Imagelist[obsid].yimage)
if not options.separate:
# and primary-beam correct
fxB = pyfits.open(Imagelist[obsid].xbeam)
fyB = pyfits.open(Imagelist[obsid].ybeam)
fx[0].data = (fx[0].data / fxB[0].data +
fy[0].data / fyB[0].data) / 2.0
fx[0].header['CRVAL%d' % polaxis] = 1
Imagelist[obsid].iimage = Imagelist[obsid].ximage.replace(
'_XX', '_I')
if os.path.exists(Imagelist[obsid].iimage):
os.remove(Imagelist[obsid].iimage)
fx.writeto(Imagelist[obsid].iimage)
logger.info('Wrote I image %s' % Imagelist[obsid].iimage)
tempfiles.append(Imagelist[obsid].iimage)
fxB[0].data = (fxB[0].data + fyB[0].data) / 2.0
fxB[0].header['CRVAL%d' % polaxis] = 1
Imagelist[obsid].ibeam = Imagelist[obsid].xbeam.replace(
'_beamXX', '_beamI')
Imagelist[obsid].ibeam = Imagelist[obsid].ibeam.replace(
'_XX', '_I')
if os.path.exists(Imagelist[obsid].ibeam):
os.remove(Imagelist[obsid].ibeam)
fxB.writeto(Imagelist[obsid].ibeam)
logger.info('Wrote I beam %s' % Imagelist[obsid].ibeam)
fi = pyfits.open(Imagelist[obsid].iimage, 'update')
fi[0].header['BEAM'] = Imagelist[obsid].ibeam
fi.flush()
fi.close()
tempfiles.append(Imagelist[obsid].ibeam)
else:
# want to make corrected XX and YY
fxB = pyfits.open(Imagelist[obsid].xbeam)
fyB = pyfits.open(Imagelist[obsid].ybeam)
fx[0].data = (fx[0].data / fxB[0].data)
Imagelist[obsid].ximage = Imagelist[obsid].ximage.replace(
'_XX', '_XXcorr')
if os.path.exists(Imagelist[obsid].ximage):
os.remove(Imagelist[obsid].ximage)
fx.writeto(Imagelist[obsid].ximage)
logger.info('Wrote corrected XX image %s' %
Imagelist[obsid].ximage)
tempfiles.append(Imagelist[obsid].ximage)
# and copy the XX beam to correspond to the new name
shutil.copy(
Imagelist[obsid].xbeam, Imagelist[obsid].xbeam.replace(
'_XX', '_XXcorr').replace('_YY', '_YYcorr'))
logger.info(
'Copied XX beam from %s -> %s' %
(Imagelist[obsid].xbeam, Imagelist[obsid].xbeam.replace(
'_XX', '_XXcorr').replace('_YY', '_YYcorr')))
Imagelist[obsid].xbeam = Imagelist[obsid].xbeam.replace(
'_XX', '_XXcorr').replace('_YY', '_YYcorr')
tempfiles.append(Imagelist[obsid].xbeam)
fy[0].data = (fy[0].data / fyB[0].data)
Imagelist[obsid].yimage = Imagelist[obsid].yimage.replace(
'_YY', '_YYcorr')
if os.path.exists(Imagelist[obsid].yimage):
os.remove(Imagelist[obsid].yimage)
fy.writeto(Imagelist[obsid].yimage)
logger.info('Wrote corrected YY image %s' %
Imagelist[obsid].yimage)
tempfiles.append(Imagelist[obsid].yimage)
# and copy the YY beam to correspond to the new name
shutil.copy(
Imagelist[obsid].ybeam, Imagelist[obsid].ybeam.replace(
'_XX', '_XXcorr').replace('_YY', '_YYcorr'))
logger.info(
'Copied YY beam from %s -> %s' %
(Imagelist[obsid].ybeam, Imagelist[obsid].ybeam.replace(
'_XX', '_XXcorr').replace('_YY', '_YYcorr')))
Imagelist[obsid].ybeam = Imagelist[obsid].ybeam.replace(
'_XX', '_XXcorr').replace('_YY', '_YYcorr')
tempfiles.append(Imagelist[obsid].ybeam)
else:
# Stokes I (corrected) exists
# now just make I beam
fi = pyfits.open(Imagelist[obsid].iimage)
try:
result = fi[0].header['DELAYS'] is not None and fi[0].header[
'DATE-OBS'] is not None
iheader = True
except KeyError:
iheader = False
if not (iheader) or options.metafits:
# does not have the necessary keys
# need to make metafits
if Imagelist[obsid].metafits is None:
logger.warning(
'FITS header keywords not present for obsid %d; creating metafits...'
% obsid)
if not _newmetadata:
hh = make_metafiles.Corr2UVFITSHeader(obsid, db=db)
T = make_metafiles.instrument_configuration(
gpstime=obsid, duration=hh.obs.duration, db=db)
result = T.make_instr_config(quick=True)
if result is None:
logger.error(
'Error making instr_config file for %d' %
obsid)
sys.exit(1)
hh.make_header()
T.corr2uvfitsheader = hh
h = T.make_metafits(quick=True)
else:
# use newer interface
obs = metadata.MWA_Observation(obsid)
if obs.observation_number is None:
logger.error('No observation found for %s' % obsid)
sys.exit(1)
m = metadata.instrument_configuration(obsid)
h = m.make_metafits(quick=True)
Imagelist[obsid].metafits = '%d.metafits' % obsid
if os.path.exists(Imagelist[obsid].metafits):
os.remove(Imagelist[obsid].metafits)
try:
h.writeto(Imagelist[obsid].metafits)
logger.info('Metafits for %d written to %s' %
(obsid, Imagelist[obsid].metafits))
except Exception, e:
logger.error(
'Unable to write metafits file for %d:\n%s' %
(obsid, e))
sys.exit(1)
if not iheader:
fi.close()
if not Imagelist[obsid].metafits is None:
logger.warning(
'Copying metafits from %s to %s' %
(Imagelist[obsid].metafits, Imagelist[obsid].iimage))
result = copy_metafits(Imagelist[obsid].iimage,
Imagelist[obsid].metafits)
if not result:
logger.error('Error updating FITS header of %s' %
Imagelist[obsid].iimage)
sys.exit(1)
##############################
# now create the beams
##############################
fi = pyfits.open(Imagelist[obsid].iimage, 'update')
delays = [int(x) for x in fi[0].header['DELAYS'].split(',')]
if Imagelist[obsid].ibeam is not None:
logger.info('I beam already exists for %d; using %s...' %
(obsid, Imagelist[obsid].ibeam))
else:
if not options.copy_beam or (not beams.has_key(
fi[0].header['DELAYS'])):
logger.info('Creating primary beams for %d' % obsid)
out = make_beam.make_beam(
Imagelist[obsid].iimage,
ext=0,
delays=delays,
analytic_model=options.analytic_model,
jones=False,
precess=options.precess)
if out is None:
logger.error('Problem creating primary beam for %s' %
obsid)
sys.exit(1)
Imagelist[obsid].xbeam, Imagelist[obsid].ybeam = out
beams[fi[0].header['DELAYS']] = obsid
logger.info('Wrote %s,%s' % (out[0], out[1]))
tempfiles += out
else:
logger.info(
'Copying beam from %s to %s' %
(Imagelist[beams[fi[0].header['DELAYS']]].iimage,
Imagelist[obsid].iimage))
result = copy_driftscanbeam(
Imagelist[beams[fi[0].header['DELAYS']]].iimage,
Imagelist[obsid].iimage)
if result is None:
sys.exit(1)
Imagelist[obsid].xbeam, Imagelist[obsid].ybeam = result
tempfiles += result
# create the I beam
fxB = pyfits.open(Imagelist[obsid].xbeam)
fyB = pyfits.open(Imagelist[obsid].ybeam)
fxB[0].data = (fxB[0].data + fyB[0].data) / 2.0
fxB[0].header['CRVAL%d' % polaxis] = 1
Imagelist[obsid].ibeam = Imagelist[obsid].xbeam.replace(
'_beamXX', '_beamI')
Imagelist[obsid].ibeam = Imagelist[obsid].ibeam.replace(
'_XX', '_I')
if os.path.exists(Imagelist[obsid].ibeam):
os.remove(Imagelist[obsid].ibeam)
fxB.writeto(Imagelist[obsid].ibeam)
logger.info('Wrote I beam %s' % Imagelist[obsid].ibeam)
# update the header so that the beam can be re-used in the future
fi[0].header['BEAM'] = Imagelist[obsid].ibeam
fi.flush()
fi.close()
tempfiles.append(Imagelist[obsid].ibeam)
def main():
observation_num = None
usage = "Usage: %prog [options]\n"
usage += '\tMakes metadata FITS file\n'
parser = OptionParser(usage=usage,
version=mwapy.__version__ + ' ' + mwapy.__date__)
parser.add_option('-f',
'--filename',
dest="filename",
help="Create metafiles for <FILE>",
metavar="FILE")
parser.add_option('-d',
'--datetime',
dest="datetimestring",
help="Create metafiles for <DATETIME> (YYYYMMDDhhmmss)",
metavar="DATETIME")
parser.add_option('-g',
'--gps',
dest="gpstime",
help="Create metafiles for <GPS>",
type='int',
metavar="GPS")
parser.add_option(
'--contiguous',
dest="contiguous",
action="store_true",
default=False,
help="Output separate files for contiguous channels only")
parser.add_option(
'--quick',
dest='quick',
default=False,
action='store_true',
help='Quick output (ignores all Rx connections)? [default=%default]')
parser.add_option(
'--minbaddipoles',
dest='min_bad_dipoles',
default=2,
type='int',
help=
'Minimum number of bad dipoles in a single pol required to flag the entire tile [default=%default]'
)
parser.add_option('-m',
'--maxdiff',
dest="maxtimediff",
type='int',
help="Maximum time difference for search (in sec)",
default=10)
parser.add_option(
'--channels',
dest="channels",
default=24,
type=int,
help=
"Number of coarse channels [default=%default, select automatically]")
parser.add_option(
'--dt',
dest="dt",
default=0,
type=float,
help="[sec] Integration time [default=%default, select by gpstime]")
parser.add_option(
'--df',
dest="df",
default=0,
type=int,
help="[kHz] Fine channel width [default=%default, select by gpstime]")
parser.add_option(
'--timeoffset',
dest="timeoffset",
default=0,
type=int,
help=
"[sec] Time offset between filename and start of data [default=%default]"
)
parser.add_option(
'-l',
'--lock',
dest='lock',
action='store_true',
default=False,
help=
'Use \"-l\" option to create header file for locked phase center (RTS input)'
)
parser.add_option(
'-o',
'--output',
type=str,
default=None,
help='Name of output FITS file [default=<gpstime>.metafits]')
parser.add_option('-u',
'--url',
default=metadata._BASEURL,
help="URL for metadata retrieval [default=%default]")
parser.add_option('-v',
'--verbose',
action="store_true",
dest="verbose",
default=False,
help="Increase verbosity of output")
(options, args) = parser.parse_args()
if (options.verbose):
logger.setLevel(logging.INFO)
observation_num = options.gpstime
if options.output is None:
options.output = '%d.metafits' % observation_num
if observation_num is not None:
obs = metadata.MWA_Observation(observation_num, url=options.url)
if obs.observation_number is None:
logger.error('No observation found for %s' % observation_num)
sys.exit(1)
if numpy.diff(obs.channels).max() > 1 and options.contiguous:
# the frequencies are not contiguous
# determine the contiguous ranges
df = numpy.diff(obs.channels)
frequency_indices = numpy.where(df > 1)[0]
channel_selections = []
istart = 0
for istop in frequency_indices:
channel_selections.append(numpy.arange(istart, istop + 1))
istart = istop + 1
channel_selections.append(numpy.arange(istart, len(obs.channels)))
else:
channel_selections = [numpy.arange(len(obs.channels))]
m = metadata.instrument_configuration(
observation_num,
min_bad_dipoles=options.min_bad_dipoles,
lock=options.lock)
for i, channel_selection in zip(range(len(channel_selections)),
channel_selections):
logger.info('Creating output for channels %s' % channel_selection)
h = m.make_metafits(quick=options.quick)
output = options.output
if len(channel_selections) > 1:
output = output.replace('.metafits', '.%02d.metafits' % i)
if os.path.exists(output):
os.remove(output)
try:
h.writeto(output)
logger.info('Metafits written to %s' % (output))
except Exception, e:
logger.error('Unable to write metafits file %s:\n%s' %
(output, e))
sys.exit(1)
def main():
observation_num = None
usage = "Usage: %prog [options] <obsids>\n"
parser = OptionParser(usage=usage)
parser.add_option(
'--start',
dest='starttime',
default=None,
type='str',
help='Starting time for processing (GPSseconds or ISO UT date)')
parser.add_option(
'--stop',
dest='stoptime',
default=None,
type='str',
help='Final time for processing (GPSseconds or ISO UT date)')
parser.add_option('--proj',
dest='project',
default=None,
help='Project ID')
parser.add_option('--destination',
dest='destination',
default=None,
help='Final destination to (remote) copy the MS file')
parser.add_option('--summaryname',
dest='summaryname',
default=None,
help='Name for output summary table')
parser.add_option(
'--downloads',
dest='downloads',
default=4,
type='int',
help='Number of simultaneous NGAS downloads [default=%default]')
parser.add_option('--nomissing',
dest='allowmissing',
default=True,
action='store_false',
help='Do not allow missing GPU box files')
parser.add_option('--timeres',
dest='timeres',
default=4,
type='float',
help='Output time resolution (s) [default=%default]')
parser.add_option(
'--freqres',
dest='freqres',
default=40,
type='int',
help='Output frequency resolution (kHz) [default=%default]')
parser.add_option(
'--caltimeres',
dest='caltimeres',
default=0,
type='float',
help='Output time resolution for calibrators (s) [default=<timeres>]')
parser.add_option(
'--calfreqres',
dest='calfreqres',
default=0,
type='int',
help='Output frequency resolution (kHz) [default=<freqres>]')
parser.add_option('--ra',
dest='ra',
default=None,
help='Output RA phase center (deg) [default=metafits]')
parser.add_option('--dec',
dest='dec',
default=None,
help='Output Dec phase center (deg) [default=metafits]')
parser.add_option('--cal',
dest='cal',
default=False,
action='store_true',
help='Always include a calibrator observation?')
parser.add_option('--cpus',
dest='cottercpus',
default=4,
type='int',
help='Number of CPUs for cotter [default %default]')
parser.add_option('--clobber',
dest='clobber',
default=False,
action='store_true',
help='Clobber existing MS file? [default=False]')
parser.add_option(
'--cleanall',
dest='cleanall',
default=False,
action='store_true',
help=
'Delete downloaded FITS files and original MS file when done? [default=False]'
)
parser.add_option(
'--cleanfits',
dest='cleanfits',
default=False,
action='store_true',
help='Delete downloaded FITS files when done? [default=False]')
parser.add_option(
'--cleanms',
dest='cleanms',
default=False,
action='store_true',
help='Delete original MS file when done? [default=False]')
parser.add_option('--copycommand',
dest='copycommand',
default='rsync -aruvP',
help='Command for (remote) copying [default=%default]')
parser.add_option(
'--summaryformat',
dest='summaryformat',
default='ascii.commented_header',
help=
'Format for output summary table (from astropy.table) [default=%default]'
)
parser.add_option('--email',
dest='notifyemail',
default=None,
help='Notification email [default=no email]')
parser.add_option(
'--ignorespace',
dest='ignorespace',
default=False,
action='store_true',
help='Ignore free-space for file download, processing, and copying?')
parser.add_option(
"-v",
"--verbose",
dest="loudness",
default=0,
action="count",
help="Each -v option produces more informational/debugging output")
parser.add_option(
"-q",
"--quiet",
dest="quietness",
default=0,
action="count",
help="Each -q option produces less error/warning/informational output")
(options, args) = parser.parse_args()
if options.cleanall:
options.cleanfits = True
options.cleanms = True
loglevels = {
0: [logging.DEBUG, 'DEBUG'],
1: [logging.INFO, 'INFO'],
2: [logging.WARNING, 'WARNING'],
3: [logging.ERROR, 'ERROR'],
4: [logging.CRITICAL, 'CRITICAL']
}
logdefault = 2 # WARNING
level = max(min(logdefault - options.loudness + options.quietness, 4), 0)
logging.getLogger('').handlers[1].setLevel(loglevels[level][0])
#logger.info('Log level set: messages that are %s or higher will be shown.' % loglevels[level][1])
eh = extra_utils.ExitHandler(os.path.split(sys.argv[0])[-1],
email=options.notifyemail)
logger.info('**************************************************')
logger.info('%s starting at %s UT on host %s with user %s' %
(sys.argv[0], datetime.datetime.now(), socket.gethostname(),
os.environ['USER']))
logger.info('**************************************************')
if options.ra is not None and options.dec is not None:
phasecenter = SkyCoord(options.ra,
options.dec,
frame='icrs',
unit='deg')
else:
phasecenter = None
if len(args) == 0:
if options.starttime is None:
logger.error('Must supply starttime')
eh.exit(1)
if options.stoptime is None:
logger.error('Must supply stoptime')
eh.exit(1)
logger.debug('Using mwapy version %s' % mwapy.__version__)
result = subprocess.Popen(['cotter', '-version'],
stdout=subprocess.PIPE).communicate()[0].strip()
cotterversion = result.split('\n')[0].split('version')[1].strip()
AOFlaggerversion = result.split('\n')[1].split('AOFlagger')[1].strip()
logger.debug('Using cotter version %s' % cotterversion)
logger.debug('Using AOFlagger version %s' % AOFlaggerversion)
results = []
havecalibrator = {}
if not (options.starttime is None and options.stoptime is None):
GPSstart = parse_time(options.starttime)
GPSstop = parse_time(options.stoptime)
if options.project is not None:
logger.info(
'Will preprocess observations from %s to %s with project=%s' %
(GPSstart, GPSstop, options.project))
else:
logger.info('Will preprocess observations from %s to %s' %
(GPSstart, GPSstop))
if options.project is None:
results = metadata.fetch_observations(mintime=GPSstart - 1,
maxtime=GPSstop + 1)
else:
results = metadata.fetch_observations(mintime=GPSstart - 1,
maxtime=GPSstop + 1,
projectid=tokenize(
options.project))
if len(args) > 0:
# add in some additional obsids
for arg in args:
results += metadata.fetch_observations(mintime=int(arg) - 1,
maxtime=int(arg) + 1)
if results is None or len(results) == 0:
logger.error('No observations found')
eh.exit(1)
logger.info(metadata.MWA_Observation_Summary.string_header())
# check if there is a calibrator present
observations = []
for item in results:
o = metadata.MWA_Observation_Summary(item)
logger.info(str(o))
observations.append(metadata.MWA_Observation(item[0]))
if havecalibrator.has_key(observations[-1].center_channel):
havecalibrator[observations[-1].center_channel] = havecalibrator[
observations[-1].
center_channel] or observations[-1].calibration
else:
havecalibrator[
observations[-1].center_channel] = observations[-1].calibration
if not all(havecalibrator.values()):
for k in havecalibrator.keys():
if not havecalibrator[k]:
logger.warning(
'No calibrator observation found for center channel %d' %
k)
if options.cal:
cals = []
for i in xrange(len(results)):
channel = observations[i].center_channel
if not havecalibrator[channel]:
cal = find_calibrator.find_calibrator(
observations[i].observation_number,
matchproject=False,
priority='time',
all=False)
if cal is None:
logger.error(
'Unable to find an appropriate calibrator scan')
eh.exit(1)
logger.info(
'Found calibrator observation %d for center channel %d'
% (cal[0], channel))
cals += metadata.fetch_observations(mintime=cal[0] - 1,
maxtime=cal[0] + 1)
logger.info(str(metadata.MWA_Observation_Summary(
cals[-1])))
havecalibrator[channel] = True
results += cals
data = None
logger.info('Processing observations...\n\n')
basedir = os.path.abspath(os.curdir)
for obs in results:
timeres, freqres = None, None
processstart = datetime.datetime.now()
observation = Observation(obs[0],
basedir=basedir,
copycommand=options.copycommand,
clobber=options.clobber,
cleanms=options.cleanms,
cleanfits=options.cleanfits,
checkfree=not options.ignorespace)
downloadedfiles = observation.download(numdownload=options.downloads)
if downloadedfiles is None:
break
if observation.makemetafits() is None:
break
if observation.observation.calibration is not None and observation.observation.calibration:
if options.caltimeres == 0:
timeres = options.timeres
else:
timeres = options.caltimeres
if options.calfreqres == 0:
freqres = options.freqres
else:
freqres = options.calfreqres
else:
timeres, freqres = options.timeres, options.freqres
msfilesize = observation.cotter(cottercpus=options.cottercpus,
timeres=timeres,
freqres=freqres,
phasecenter=phasecenter,
allowmissing=options.allowmissing)
if msfilesize is None:
break
if options.destination is not None and observation.copy(
options.destination) is None:
break
row = collections.OrderedDict({
'date':
processstart.strftime('%Y-%m-%dT%H:%M:%S'),
'obsid':
obs[0],
'user':
os.environ['USER'],
'host':
socket.gethostname(),
'cotter':
cotterversion.replace(' ', '_'),
'mwapy':
mwapy.__version__,
'rawfiles':
len(observation.downloadedfiles),
'msfilesize':
msfilesize
})
if options.destination is not None:
row['msfile'] = os.path.join(options.destination,
observation.msfile)
else:
row['msfile'] = os.path.join(observation.outputdir,
observation.msfile)
if data is None:
data = Table([row])
else:
data.add_row(row)
os.chdir(basedir)
if data is not None and options.summaryname is not None:
data = data[('date', 'obsid', 'user', 'host', 'cotter', 'mwapy',
'rawfiles', 'msfilesize', 'msfile')]
try:
data.write(options.summaryname, format=options.summaryformat)
logger.info('Summary table written to %s' % options.summaryname)
except Exception, e:
logger.error(
'Unable to write summary table %s with format %s:\n%s' %
(options.summaryname, options.summaryformat, e))
def main():
usage = "Usage: %prog [options]\n"
usage += "\tFinds MWA Observations based on a set of query parameters\n"
usage += "\tWill return just the obsids(default) or more information (if verbose)\n"
usage += '\tFor strings (--proj, --creator, --obsname) can use wildcards "*" and "?" to match\n'
o = optparse.OptionParser(usage=usage,
version=mwapy.__version__ + ' ' + mwapy.__date__)
o.add_option(
'--limit',
type='int',
default=100,
help='Limit on number of results returned. [default=%default]')
o.add_option('--proj', type='str', default=None, help='Project ID')
#o.add_option('--list_projects', action='store_true',
# help = 'List available project ids and exit')
o.add_option('--start',
type='str',
default=None,
help='Start time (GPSseconds or ISO UT date)')
o.add_option('--stop',
type='str',
default=None,
help='Stop time (GPSseconds or ISO UT date)')
o.add_option(
'--GPSrange',
type='str',
default=None,
dest='gpsrange',
help='<start>_<stop> in GPSseconds (overrides --start and --stop)')
o.add_option('--racenter',
type='str',
default=None,
help='Center RA for search (decimal degrees or hms)')
o.add_option('--deccenter',
type='str',
default=None,
help='Center Dec for search (decimal degrees or dms)')
o.add_option(
'--rawidth',
type='str',
default='10d',
help=
'Width of RA search box (degrees assumed; will not handle 360d boundary) [default=%default]'
)
o.add_option('--decwidth',
type='str',
default='10d',
help='Width of Dec search box (degrees) [default=%default]')
o.add_option('--chan', type='int', default=None, help='Center channel')
o.add_option('--anychan', type='int', default=None, help='Any channel')
o.add_option(
'--contiguous',
action='store_true',
dest='contiguous',
default=None,
help='Only include observations with contiguous frequency channels')
o.add_option(
'--discontiguous',
action='store_false',
dest='contiguous',
default=None,
help='Only include observations with discontiguous frequency channels')
o.add_option('--inttime',
type='int',
default=None,
help='Integration time (s)')
o.add_option('--freqres',
type='int',
default=None,
help='Frequency resolution (kHz)')
#o.add_option('--withdata*',action='store_true',
# help="Only include observations with data.")
o.add_option('--cal',
action='store_true',
default=None,
dest='withcal',
help="Only get data that are marked as calibrations")
o.add_option('--nocal',
action='store_false',
default=None,
dest='withcal',
help="Only get data that are not marked as calibrations")
o.add_option('--future',
action='store_true',
default=None,
dest='future',
help="Only get observations that are in the future")
o.add_option('--past',
action='store_false',
default=None,
dest='future',
help="Only get observations that are in the past")
o.add_option('--creator', type='str', default=None, help='Creator')
o.add_option('--mode', type='str', default=None, help='Observation mode')
o.add_option('--obsname', type='str', help='Observation name')
o.add_option('--close',
default=False,
action='store_true',
help='Sort by observations closest in time to --start')
o.add_option(
'--maxdiff',
default='1d',
help=
'Maximum time difference for proximity search with --close [default=%default]'
)
# Each -v will increase the log output by one level, each -q will decrease it by one level
o.add_option(
"-v",
"--verbose",
dest="loudness",
default=0,
action="count",
help="Each -v option produces more informational/debugging output")
o.add_option(
"-q",
"--quiet",
dest="quietness",
default=0,
action="count",
help="Each -q option produces less error/warning/informational output")
#o.add_option('-v','--verbose',action="store_true",dest="verbose",default=False,
# help="Increase verbosity of output")
o.add_option('--full',
action="store_true",
dest="full",
default=False,
help="Print full info for each observation")
o.add_option('-u',
'--url',
default=metadata._BASEURL,
help="URL for metadata retrieval [default=%default]")
o.add_option(
'--gridpoint',
type='int',
default=None,
dest='gridpoint',
help=
'Pointing gridpoint number. For zenith scans gridpoint=0 [default=%default]'
)
o.add_option('--uvfits',
action='store_true',
default=None,
dest='withuvfits',
help="Only get observations that have a uvfits file")
options, args = o.parse_args()
loglevels = {
0: [logging.DEBUG, 'DEBUG'],
1: [logging.INFO, 'INFO'],
2: [logging.WARNING, 'WARNING'],
3: [logging.ERROR, 'ERROR'],
4: [logging.CRITICAL, 'CRITICAL']
}
logdefault = 2 # WARNING
level = max(min(logdefault - options.loudness + options.quietness, 4), 0)
logger.setLevel(loglevels[level][0])
logger.info(
'Log level set: messages that are %s or higher will be shown.' %
loglevels[level][1])
cal = None
if options.withcal is not None:
if options.withcal:
cal = 1
else:
cal = 0
filetype = None
if options.withuvfits is not None:
if options.withuvfits:
filetype = 13
contiguous = None
if options.contiguous is not None:
if options.contiguous:
contiguous = 1
else:
contiguous = 0
future = None
if options.future is not None:
if options.future:
future = 1
else:
future = 0
if options.start is not None:
GPSstart = parse_time(options.start)
if GPSstart is None:
sys.exit(1)
else:
GPSstart = None
if options.stop is not None:
GPSstop = parse_time(options.stop)
if GPSstop is None:
sys.exit(1)
else:
GPSstop = None
if options.gpsrange is not None:
GPSstart, GPSstop = map(parse_time, options.gpsrange.split('_'))
if GPSstart is None or GPSstop is None:
sys.exit(1)
racenter, deccenter = None, None
rawidth, decwidth = None, None
if options.racenter is not None:
try:
racenter = Angle(options.racenter)
except UnitsError:
try:
racenter = Angle(float(options.racenter), unit=u.degree)
except ValueError:
try:
racenter = Angle(options.racenter, unit=u.hourangle)
except:
logger.error('Unable to parse input RA %s' %
options.racenter)
sys.exit(1)
except:
logger.error('Unable to parse input RA %s' % options.racenter)
sys.exit(1)
if options.deccenter is not None:
try:
deccenter = Angle(options.deccenter, unit=u.degree)
except:
logger.error('Unable to parse input Dec %s' % options.deccenter)
sys.exit(1)
if options.rawidth is not None:
try:
rawidth = Angle(options.rawidth)
except UnitsError:
try:
rawidth = Angle(options.rawidth, unit=u.degree)
except:
logger.error('Unable to parse RA width %s' % options.rawidth)
sys.exit(1)
except:
logger.error('Unable to parse RA width %s' % options.rawidth)
sys.exit(1)
if options.decwidth is not None:
try:
decwidth = Angle(options.decwidth, unit=u.degree)
except:
logger.error('Unable to parse input Dec %s' % options.decwidth)
sys.exit(1)
maxdiff = None
if options.maxdiff is not None:
# try seconds
try:
maxdiff = int(options.maxdiff)
except ValueError:
if options.maxdiff[-1] == 's':
maxdiff = int(options.maxdiff[:-1])
elif options.maxdiff[-1] == 'm':
maxdiff = int(float(options.maxdiff[:-1]) * 60)
elif options.maxdiff[-1] == 'h':
maxdiff = int(float(options.maxdiff[:-1]) * 60 * 60)
elif options.maxdiff[-1] == 'd':
maxdiff = int(float(options.maxdiff[:-1]) * 86400)
else:
logger.error('Unable to parse time difference %s' %
options.maxdiff)
sys.exit(1)
if GPSstart is not None and GPSstop is not None:
logger.info('Searching from %d to %d...' % (GPSstart, GPSstop))
elif GPSstart is not None:
logger.info('Searching from %d...' % (GPSstart))
elif GPSstop is not None:
logger.info('Searching until %d...' % (GPSstop))
if options.proj is not None:
logger.info('Require projectid=%s' % options.proj)
if options.withcal is not None:
if options.withcal:
logger.info('Require calibrator observations')
else:
logger.info('Require noncalibrator observations')
if options.creator is not None:
logger.info('Require creator=%s' % options.creator)
if options.mode is not None:
logger.info('Require mode=%s' % options.mode)
if options.chan is not None:
logger.info('Require center channel=%d' % options.chan)
if options.anychan is not None:
logger.info('Require any channel=%d' % options.anychan)
if options.contiguous is not None:
if options.contiguous:
logger.info('Require contiguous frequency channels')
else:
logger.info('Require discontiguous frequency channels')
if options.inttime:
logger.info('Require integration time=%d s' % options.inttime)
if options.freqres:
logger.info('Require frequency resolution=%d kHz' % options.freqres)
if racenter is not None:
ramin = (max(racenter - rawidth / 2, Angle(0, unit=u.degree))).degree
ramax = (min(racenter + rawidth / 2, Angle(360, unit=u.degree))).degree
logger.info('Require %.1fd < RA < %.1fd' % (ramin, ramax))
else:
ramin, ramax = None, None
if deccenter is not None:
decmin = (max(deccenter - decwidth / 2, Angle(-90,
unit=u.degree))).degree
decmax = (min(deccenter + decwidth / 2, Angle(90,
unit=u.degree))).degree
logger.info('Require %.1fd < Dec < %.1fd' % (decmin, decmax))
else:
decmin, decmax = None, None
if options.gridpoint is not None:
logger.info('Require gridpoint=%d' % options.gridpoint)
if options.obsname is not None:
logger.info('Require obsname=%s' % options.obsname)
if options.future is not None:
if options.future:
logger.info('Require observation in the future')
else:
logger.info('Require observation in the past')
if options.close:
if GPSstart is None:
logger.error('Require start time with --close')
sys.exit(1)
logger.info('Will sort observations within %ds of %d...' %
(maxdiff, GPSstart))
if options.withuvfits:
logger.info('Require observations with UVFITS files')
if not options.close:
results = metadata.fetch_observations(
URL=options.url,
mintime=GPSstart,
maxtime=GPSstop,
minra=ramin,
maxra=ramax,
mindec=decmin,
maxdec=decmax,
limit=options.limit,
projectid=tokenize(options.proj),
calibration=cal,
contigfreq=contiguous,
cenchan=options.chan,
anychan=options.anychan,
int_time=options.inttime,
freq_res=options.freqres,
creator=tokenize(options.creator),
mode=tokenize(options.mode),
obsname=tokenize(options.obsname),
future=future,
gridpoint=options.gridpoint)
if results is None or len(results) == 0:
print 'Query return no results'
sys.exit(0)
if loglevels[level][0] <= logging.INFO:
print metadata.MWA_Observation_Summary.string_header()
elif loglevels[level][0] <= logging.WARNING:
print '# starttime'
if filetype is not None:
uvfitslist = metadata.query_filetype(str(filetype))
results = [obs for obs in results if [obs[0]] in uvfitslist]
for item in results:
o = metadata.MWA_Observation_Summary(item)
if loglevels[level][0] <= logging.INFO:
print o
else:
print o.obsid
if options.full:
observation = metadata.MWA_Observation(o.obsid,
rfstream=0,
ionex=False,
url=options.url)
print observation
else:
GPSstop = GPSstart + maxdiff
results1 = metadata.fetch_observations(
URL=options.url,
mintime=GPSstart,
maxtime=GPSstop,
minra=ramin,
maxra=ramax,
mindec=decmin,
maxdec=decmax,
limit=options.limit / 2,
projectid=tokenize(options.proj),
calibration=cal,
contigfreq=contiguous,
cenchan=options.chan,
anychan=options.anychan,
int_time=options.inttime,
freq_res=options.freqres,
creator=tokenize(options.creator),
mode=tokenize(options.mode),
obsname=tokenize(options.obsname),
future=future)
GPSstop = GPSstart - maxdiff
try:
results2 = metadata.fetch_observations(
URL=options.url,
mintime=GPSstop,
maxtime=GPSstart,
minra=ramin,
maxra=ramax,
mindec=decmin,
maxdec=decmax,
limit=options.limit / 2,
projectid=tokenize(options.proj),
calibration=cal,
contigfreq=contiguous,
cenchan=options.chan,
anychan=options.anychan,
int_time=options.inttime,
freq_res=options.freqres,
creator=tokenize(options.creator),
mode=tokenize(options.mode),
obsname=tokenize(options.obsname),
future=future,
gridpoint=options.gridpoint)
results = results2 + results1
except:
results = results1
if results is None or len(results) == 0:
print 'Query return no results'
sys.exit(0)
if loglevels[level][0] <= logging.INFO:
print metadata.MWA_Observation_Summary.string_header()
else:
print '# starttime'
last = None
if filetype is not None:
uvfitslist = metadata.query_filetype(str(filetype))
results = [obs for obs in results if [obs[0]] in uvfitslist]
for item in results:
o = metadata.MWA_Observation_Summary(item)
if (last is None or last < GPSstart) and o.obsid >= GPSstart:
print '# ***** %d\t%s' % (
GPSstart, Time(
GPSstart, format='gps',
scale='utc').datetime.strftime('%Y-%m-%dT%H:%M:%S'))
if loglevels[level][0] <= logging.INFO:
print o
else:
print o.obsid
last = o.obsid
if options.full:
observation = metadata.MWA_Observation(o.obsid,
rfstream=0,
ionex=False,
url=options.url)
obsid = str(GPSstart)
lst = str(observation.LST)
ra = str(observation.RA)
dec = str(observation.Dec)
azimuth = str(observation.azimuth)
elevation = str(observation.elevation)
obsinfo_string = obsid + ", " + lst + ", " + ra + ", " + dec + ", " + azimuth + ", " + elevation + "\n"
write_filename = "/nfs/eor-00/h1/rbyrne/sidelobe_survey_obsinfo.txt"
openfile = open(write_filename, "a")
openfile.write('obsid, LST, RA, Dec, Az, El\n')
openfile.write(obsinfo_string)
openfile.close()
sys.exit(0)
except ValueError, e:
logger.error('Unable to parse input time %s: %s' %
(options.datetimestring, e))
sys.exit(1)
results = metadata.find_closest_observation(int(t.gps),
options.maxtimediff,
url=options.url)
if results is None:
logger.error('No observation found within %ds of s' %
(options.maxtimediff, t))
sys.exit(1)
observation_num = results
if observation_num is not None:
observation = metadata.MWA_Observation(observation_num,
rfstream=options.rfstream,
ionex=options.ionex,
url=options.url)
if observation.observation_number is None:
logger.error('No observation found for %s' % observation_num)
sys.exit(1)
print observation
if (options.image):
if (not _useplotting):
logger.warning(
'Unable to import primarybeammap to generate image\n')
return
datetimestring = '%04d%02d%02d%02d%02d%02d' % (
observation.year, observation.month, observation.day,
observation.hour, observation.minute, observation.second)
logger.info(