def find_calibrator(obsid, maxtimediff=TimeDelta(1 * u.d), maxseparation=180 * u.deg, sourcename=None, notsourcename=['FDS'], matchproject=True, matchnight=True, priority='time', all=False): """ obsid, timediff, distance=find_calibrator(obsid, maxtimediff=TimeDelta(1*u.d), maxseparation=180*u.deg, sourcename=None, notsourcename=['FDS'], matchproject=True, matchnight=True, priority='time', all=False): timediff is in days distance is in degrees """ assert priority in ['time', 'distance'] starttime = Time(obsid, format='gps', scale='utc') - maxtimediff stoptime = Time(obsid, format='gps', scale='utc') + maxtimediff try: baseobs = metadata.MWA_Observation(obsid) except Exception, e: logger.error('Cannot fetch info for observation %d:\n\t%s' % (obsid, e)) return None
def __init__(self, obsid, basedir='./', clobber=False, cleanfits=False, cleanms=False, checkfree=True, copycommand='rsync -aruvP'): self.obsid = obsid self.basedir = basedir self.downloadedfiles = None self.metafits = None self.cottercommand = None self.msfile = None self.clobber = clobber self.copycommand = copycommand self.cleanfits = cleanfits self.cleanms = cleanms self.destination = None self.rawsize = None self.fitssize = None self.mssize = None self.checkfree = checkfree self.outputdir = os.path.join(self.basedir, str(self.obsid), '') os.chdir(basedir) logger.info('##################################################') logger.info('%d' % self.obsid) self.observation = metadata.MWA_Observation(self.obsid) logger.debug('\n' + str(self.observation)) obsinfo = metadata.fetch_obsinfo(self.obsid) self.rawsize = sum( [obsinfo['files'][f]['size'] for f in obsinfo['files'].keys()]) self.fitssize = sum([ obsinfo['files'][f]['size'] * (obsinfo['files'][f]['filetype'] == 8) for f in obsinfo['files'].keys() ]) logger.info('Raw data size is %.1f MB' % (self.rawsize / 1024. / 1024.))
def main(): usage = "Usage: %prog [options] <obsid>\n" o = optparse.OptionParser(usage=usage, version=mwapy.__version__ + ' ' + mwapy.__date__) o.add_option('--separation', dest='separation', default=180, type='float', help='Maximum separation (deg) [default=%default]') o.add_option( '--timediff', dest='timediff', default='1d', help='Maximum time difference (d, h, m, or s) [default=%default]') o.add_option('--matchproject', dest='matchproject', default=False, action='store_true', help='Match project to original ObsID?') o.add_option('--matchnight', dest='matchnight', default=False, action='store_true', help='Match night to original ObsID?') o.add_option('--source', dest='source', default=None, help='Calibrator source name [default=None]') o.add_option( '--notsource', dest='notsource', default='FDS', help= 'Comma-separated list of source names to exclude [default=%default]') o.add_option( '--priority', dest='priority', default='time', type='choice', choices=['time', 'distance'], help='Return the closest in time or distance? [default=%default]') o.add_option('--all', dest='all', default=False, action='store_true', help='Return all possible matches?') o.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true', help='Give verbose output?') options, args = o.parse_args() if len(args) == 0: logger.error('Must specify >1 obsids') sys.exit(1) if 'd' in options.timediff: maxtimediff = TimeDelta(float(options.timediff[:-1]) * u.d) elif 'h' in options.timediff: maxtimediff = TimeDelta(float(options.timediff[:-1]) * u.h) elif 'm' in options.timediff: maxtimediff = TimeDelta(float(options.timediff[:-1]) * u.m) elif 's' in options.timediff: maxtimediff = TimeDelta(float(options.timediff[:-1]) * u.s) maxseparation = options.separation * u.deg matchproject = options.matchproject matchnight = options.matchnight priority = options.priority for obsid in args: result = find_calibrator(int(obsid), maxtimediff=maxtimediff, maxseparation=maxseparation, matchproject=matchproject, matchnight=matchnight, sourcename=options.source, notsourcename=options.notsource.split(','), priority=priority, all=options.all) if result is None: print "None" elif not options.all: print '# Obsid\t\tCalObsid\tTimediff(day)\tDistance(deg)\tSource' print '%s\t%s\t%.3f\t\t%.1f\t\t%s' % (obsid, result[0], result[1], result[2], result[3]) if options.verbose: o = metadata.MWA_Observation(int(result[0])) print o else: print '# Obsid\t\tCalObsid\tTimediff(day)\tDistance(deg)\tSource' for i in xrange(len(result[0])): print '%s\t%s\t%.3f\t\t%.1f\t\t%s' % ( obsid, result[0][i], result[1][i], result[2][i], result[3][i]) if options.verbose: o = metadata.MWA_Observation(int(result[0][i])) print o sys.exit(0)
def main(): usage="Usage: %prog [options]\n" usage+="\tFinds MWA Observations based on a set of query parameters\n" usage+="\tWill return just the obsids(default) or more information (if verbose)\n" o = optparse.OptionParser(usage=usage,version=mwapy.__version__ + ' ' + mwapy.__date__) o.add_option('--limit', type='int', default=100, help='Limit on number of results returned [default=%default]') o.add_option('--proj', type='str', default=None, help = 'Project ID') #o.add_option('--list_projects', action='store_true', # help = 'List available project ids and exit') o.add_option('--start', type='str', default=None, help='Start time (GPSseconds or ISO UT date)') o.add_option('--stop', type='str', default=None, help='Stop time (GPSseconds or ISO UT date)') o.add_option('--racenter', type='str', default=None, help='Center RA for search (decimal degrees or hms)') o.add_option('--deccenter', type='str', default=None, help='Center Dec for search (decimal degrees or dms)') o.add_option('--rawidth', type='str', default='10d', help='Width of RA search box (degrees assumed; will not handle 360d boundary) [default=%default]') o.add_option('--decwidth', type='str', default='10d', help='Width of Dec search box (degrees) [default=%default]') o.add_option('--chan',type='int', default=None, help='Center channel') o.add_option('--anychan',type='int',default=None, help='Match to any channel') o.add_option('--contiguous', action='store_true',dest='contiguous',default=None, help='Only include observations with contiguous frequency channels') o.add_option('--discontiguous', action='store_false',dest='contiguous',default=None, help='Only include observations with discontiguous frequency channels') o.add_option('--inttime', type='int', default=None, help='Integration time (s)') o.add_option('--freqres', type='int', default=None, help='Frequency resolution (kHz)') #o.add_option('--withdata*',action='store_true', # help="Only include observations with data.") o.add_option('--cal',action='store_true', default=None,dest='withcal', help="Only get data that are marked as calibrations") o.add_option('--nocal',action='store_false', default=None,dest='withcal', help="Only get data that are not marked as calibrations") o.add_option('--future',action='store_true', default=None,dest='future', help="Only get observations that are in the future") o.add_option('--past',action='store_false', default=None,dest='future', help="Only get observations that are in the past") o.add_option('--creator',type='str',default=None, help='Creator') o.add_option('--mode',type='str',default=None, help='Observation mode') o.add_option('--obsname',type='str', help='Match any part of an observation name') o.add_option('--close',default=False, action='store_true', help='Sort by observations closest in time') o.add_option('--maxdiff',default='1d', help='Maximum time difference for proximity search [default=%default]') o.add_option('-v','--verbose',action="store_true",dest="verbose",default=False, help="Increase verbosity of output") o.add_option('--full',action="store_true",dest="full",default=False, help="Print full info for each observation") o.add_option('-u','--url',default=metadata._BASEURL, help="URL for metadata retrieval [default=%default]") options, args = o.parse_args() if (options.verbose): logger.setLevel(logging.INFO) cal=None if options.withcal is not None: if options.withcal: cal=1 else: cal=0 contiguous=None if options.contiguous is not None: if options.contiguous: contiguous=1 else: contiguous=0 future=None if options.future is not None: if options.future: future=1 else: future=0 if options.start is not None: GPSstart=parse_time(options.start) if GPSstart is None: sys.exit(1) else: GPSstart=None if options.stop is not None: GPSstop=parse_time(options.stop) if GPSstop is None: sys.exit(1) else: GPSstop=None racenter,deccenter=None,None rawidth,decwidth=None,None if options.racenter is not None: try: racenter=Angle(options.racenter) except UnitsError: try: racenter=Angle(float(options.racenter), unit=u.degree) except ValueError: try: racenter=Angle(options.racenter, unit=u.hourangle) except: logger.error('Unable to parse input RA %s' % options.racenter) sys.exit(1) except: print 'othererror' logger.error('Unable to parse input RA %s' % options.racenter) sys.exit(1) if options.deccenter is not None: try: deccenter=Angle(options.deccenter, unit=u.degree) except: logger.error('Unable to parse input Dec %s' % options.deccenter) sys.exit(1) if options.rawidth is not None: try: rawidth=Angle(options.rawidth) except UnitsError: try: rawidth=Angle(options.rawidth, unit=u.degree) except: logger.error('Unable to parse RA width %s' % options.rawidth) sys.exit(1) except: logger.error('Unable to parse RA width %s' % options.rawidth) sys.exit(1) if options.decwidth is not None: try: decwidth=Angle(options.decwidth, unit=u.degree) except: logger.error('Unable to parse input Dec %s' % options.decwidth) sys.exit(1) maxdiff=None if options.maxdiff is not None: # try seconds try: maxdiff=int(options.maxdiff) except ValueError: if options.maxdiff[-1]=='s': maxdiff=int(options.maxdiff[:-1]) elif options.maxdiff[-1]=='m': maxdiff=int(float(options.maxdiff[:-1])*60) elif options.maxdiff[-1]=='h': maxdiff=int(float(options.maxdiff[:-1])*60*60) elif options.maxdiff[-1]=='d': maxdiff=int(float(options.maxdiff[:-1])*86400) else: logger.error('Unable to parse time difference %s' % options.maxdiff) sys.exit(1) if GPSstart is not None and GPSstop is not None: logger.info('Searching from %d to %d...' % (GPSstart,GPSstop)) elif GPSstart is not None: logger.info('Searching from %d...' % (GPSstart)) elif GPSstop is not None: logger.info('Searching until %d...' % (GPSstop)) if options.proj is not None: logger.info('Require projectid=%s' % options.proj) if options.withcal is not None: if options.withcal: logger.info('Require calibrator observations') else: logger.info('Require noncalibrator observations') if options.creator is not None: logger.info('Require creator=%s' % options.creator) if options.mode is not None: logger.info('Require mode=%s' % options.mode) if options.chan is not None: logger.info('Require center channel=%d' % options.chan) if options.anychan is not None: logger.info('Require any channel=%d' % options.anychan) if options.contiguous is not None: if options.contiguous: logger.info('Require contiguous frequency channels') else: logger.info('Require discontiguous frequency channels') if options.inttime: logger.info('Require integration time=%d s' % options.inttime) if options.freqres: logger.info('Require frequency resolution=%d kHz' % options.freqres) if racenter is not None: ramin=(max(racenter-rawidth/2,Angle(0,unit=u.degree))).degree ramax=(min(racenter+rawidth/2,Angle(360,unit=u.degree))).degree logger.info('Require %.1fd < RA < %.1fd' % (ramin, ramax)) else: ramin,ramax=None,None if deccenter is not None: decmin=(max(deccenter-decwidth/2,Angle(-90,unit=u.degree))).degree decmax=(min(deccenter+decwidth/2,Angle(90,unit=u.degree))).degree logger.info('Require %.1fd < Dec < %.1fd' % (decmin, decmax)) else: decmin,decmax=None,None if options.obsname is not None: logger.info('Require obsname=%s' % options.obsname) if options.future is not None: if options.future: logger.info('Require observation in the future') else: logger.info('Require observation in the past') if options.close: if GPSstart is None: logger.error('Require start time with --close') sys.exit(1) logger.info('Will sort observations within %ds of %d...' % ( maxdiff, GPSstart)) if not options.close: results=metadata.fetch_observations(URL=options.url, mintime=GPSstart, maxtime=GPSstop, minra=ramin, maxra=ramax, mindec=decmin, maxdec=decmax, limit=options.limit, projectid=tokenize(options.proj), calibration=cal, contigfreq=contiguous, cenchan=options.chan, anychan=options.anychan, int_time=options.inttime, freq_res=options.freqres, creator=tokenize(options.creator), mode=tokenize(options.mode), obsname=tokenize(options.obsname), future=future) if results is None or len(results)==0: print 'Query return no results' sys.exit(0) if options.verbose: print metadata.MWA_Observation_Summary.string_header() else: print '# starttime' for item in results: o=metadata.MWA_Observation_Summary(item) if options.verbose: print o else: print o.obsid if options.full: observation=metadata.MWA_Observation(o.obsid, rfstream=0, ionex=False, url=options.url) print observation else: GPSstop=GPSstart+maxdiff results1=metadata.fetch_observations(URL=options.url, mintime=GPSstart, maxtime=GPSstop, minra=ramin, maxra=ramax, mindec=decmin, maxdec=decmax, limit=options.limit/2, projectid=tokenize(options.proj), calibration=cal, contigfreq=contiguous, cenchan=options.chan, anychan=options.anychan, int_time=options.inttime, freq_res=options.freqres, creator=tokenize(options.creator), mode=tokenize(options.mode), obsname=tokenize(options.obsname), future=future) GPSstop=GPSstart-maxdiff try: results2=metadata.fetch_observations(URL=options.url, mintime=GPSstop, maxtime=GPSstart, minra=ramin, maxra=ramax, mindec=decmin, maxdec=decmax, limit=options.limit/2, projectid=tokenize(options.proj), calibration=cal, contigfreq=contiguous, cenchan=options.chan, anychan=options.anychan, int_time=options.inttime, freq_res=options.freqres, creator=tokenize(options.creator), mode=tokenize(options.mode), obsname=tokenize(options.obsname), future=future)[::-1] results=results2+results1 except: results=results1 if results is None or len(results)==0: print 'Query return no results' sys.exit(0) if options.verbose: print metadata.MWA_Observation_Summary.string_header() else: print '# starttime' last=None for item in results: o=metadata.MWA_Observation_Summary(item) if (last is None or last < GPSstart) and o.obsid >= GPSstart: print '# ***** %d\t%s' % (GPSstart, Time(GPSstart,format='gps', scale='utc').datetime.strftime('%Y-%m-%dT%H:%M:%S')) if options.verbose: print o else: print o.obsid last=o.obsid if options.full: observation=metadata.MWA_Observation(o.obsid, rfstream=0, ionex=False, url=options.url) print observation sys.exit(0)
def fluxmatch(image, catalog='GLEAMIDR3.fits', fluxcolumn=None, fluxerrcolumn=None, racolumn='RAJ2000', deccolumn='DECJ2000', nsigma=10, rmsfactor=3, matchradius=120, rejectsigma=3, maxdistance=20, minbeam=0.5, psfextent=1.1, limit=10, refineposition=False, update=False, prefix=None, otherimages=[], updatepoln=False, updatebane=False, plot=True, region=True, cores=1): """ catalog='GLEAMIDR3.fits', fluxcolumn=None, fluxerrcolumn=None, racolumn='RAJ2000', deccolumn='DECJ2000' signal-to-noise for source finding nsigma=10, ratio of local rms to minimum image RMS that is OK rmsfactor=3, distance between catalog source and source in image (arcsec) matchradius=120, rejection sigma for flux ratio outliers rejectsigma=3, distance from the image center that is OK (deg) maxdistance=20, minimum beam power compared to max minbeam=0.5, area of source/area of psf threshold psfextent=1.1, max ratio of new to old fluxes (or reciprocal) limit=10, """ if not isinstance(matchradius, astropy.units.quantity.Quantity): matchradius = matchradius * u.arcsec if not isinstance(maxdistance, astropy.units.quantity.Quantity): maxdistance = maxdistance * u.deg if not os.path.exists(image): logger.error('Cannot find input image %s' % image) return None if updatepoln: for stokes in ['Q', 'U', 'V']: if os.path.exists(image.replace('-I.fits', '-%s.fits' % stokes)): otherimages.append( image.replace('-I.fits', '-%s.fits' % stokes)) logger.info('Will also scale %s' % otherimages[-1]) if updatebane: for ext in ['rms', 'bkg']: if os.path.exists(image.replace('-I.fits', '-I_%s.fits' % ext)): otherimages.append(image.replace('-I.fits', '-I_%s.fits' % ext)) logger.info('Will also scale %s' % otherimages[-1]) if updatepoln: for stokes in ['Q', 'U', 'V']: if os.path.exists( image.replace('-I.fits', '-%s_%s.fits' % (stokes, ext))): otherimages.append( image.replace('-I.fits', '-%s_%s.fits' % (stokes, ext))) logger.info('Will also scale %s' % otherimages[-1]) if not os.path.exists(catalog): logger.error('Cannot find GLEAM catalog %s' % catalog) return None beam = find_beam(image) if beam is None: logger.warning('Did not generate primary beam: will ignore') minbeam = None if beam is not None and not os.path.exists(beam): logger.warning('Cannot find primary beam %s: will ignore' % beam) minbeam = None beam = None outbase = os.path.splitext(image)[0] sources, rmsimage, bgimage = find_sources_in_image(image, nsigma=nsigma, cores=cores) logger.info('Found %d sources above %d sigma in %s' % (len(sources), nsigma, image)) logger.info('Wrote %s and %s' % (rmsimage, bgimage)) # convert to astropy table sourcesTable = aegean2table(sources) fimage = fits.open(image) frequency = fimage[0].header['CRVAL3'] w = WCS(fimage[0].header, naxis=2) frmsimage = fits.open(rmsimage) minrms = numpy.nanmin(frmsimage[0].data) logger.info('Minimum RMS in image is %.1f mJy' % (minrms * 1e3)) if beam is not None: fbeam = fits.open(beam) x, y = w.wcs_world2pix(sourcesTable['RA'], sourcesTable['Dec'], 0) sourcesTable.add_column(Column(x, name='X')) sourcesTable.add_column(Column(y, name='Y')) if 'RA' in fimage[0].header.keys(): pointingcenter = SkyCoord(fimage[0].header['RA'], fimage[0].header['DEC'], unit=('deg', 'deg')) else: # get the pointing center from the metadata logger.warning( 'Pointing metadata not present in header; retrieving...') try: obs = metadata.MWA_Observation(fimage[0].header['GPSTIME']) logger.info('Found pointing center %f,%f' % (obs.RA, obs.Dec)) pointingcenter = SkyCoord(obs.RA, obs.Dec, unit=('deg', 'deg')) except: logger.warning('Using CRVAL1/CRVAL2 for pointing center') pointingcenter = SkyCoord(fimage[0].header['CRVAL1'], fimage[0].header['CRVAL2'], unit=('deg', 'deg')) coords = SkyCoord(sourcesTable['RA'], sourcesTable['Dec'], unit=(u.deg, u.deg)) sourcesTable.add_column( Column(coords.separation(pointingcenter).to(u.deg), name='SOURCEDIST')) if beam is not None: pixelx, pixely = numpy.int16(x), numpy.int16(y) pixelx[pixelx < 0] = 0 pixely[pixely < 0] = 0 pixelx[pixelx >= fbeam[0].data.shape[-1]] = fbeam[0].data.shape[-1] - 1 pixely[pixely >= fbeam[0].data.shape[-2]] = fbeam[0].data.shape[-2] - 1 sourcesTable.add_column( Column(fbeam[0].data[0, 0, pixelx, pixely], name='BEAM')) else: sourcesTable.add_column(Column(0 * x, name='BEAM')) if '.fits' in catalog: # this seems to be faster than going straight to the Table.read() try: fcatalog = fits.open(catalog) except: logger.error('Unable to open FITS catalog %s' % catalog) return None catalogTable = Table(fcatalog[1].data) else: try: catalogTable = Table.read(catalog) except: logger.error('Unable to read catalog %s' % catalog) return None try: bandfrequencies = numpy.array([ int(s.split('_')[-1]) for s in numpy.array(catalogTable.colnames)[numpy.nonzero( numpy.array([('int_flux' in c) and not ('deep' in c) and not ('wide' in c)('fit' in c) for c in catalogTable.colnames]))[0]] ]) except: bandfrequencies = [] if len(bandfrequencies) > 0: # find the indices of the bands just above and below the observation # linearly weight the fluxes just above and below to match # the observation frequency indexplus = (bandfrequencies >= frequency / 1e6).nonzero()[0].min() indexminus = (bandfrequencies < frequency / 1e6).nonzero()[0].max() logger.info( 'Observation frequency of %.1f MHz: interpolating between %d MHz and %d MHz' % (frequency / 1e6, bandfrequencies[indexminus], bandfrequencies[indexplus])) weightplus = (frequency / 1e6 - bandfrequencies[indexminus]) / ( bandfrequencies[indexplus] - bandfrequencies[indexminus]) weightminus = 1 - weightplus gleamflux = catalogTable[ 'int_flux_%03d' % bandfrequencies[indexminus]] * weightminus + catalogTable[ 'int_flux_%03d' % bandfrequencies[indexplus]] * weightplus try: gleamfluxerr = numpy.sqrt( (catalogTable['err_fit_flux_%03d' % bandfrequencies[indexminus]] * weightminus)**2 + (catalogTable['err_fit_flux_%03d' % bandfrequencies[indexplus]] * weightplus)**2) except KeyError: gleamfluxerr = numpy.sqrt( (catalogTable['err_int_flux_%03d' % bandfrequencies[indexminus]] * weightminus)**2 + (catalogTable['err_int_flux_%03d' % bandfrequencies[indexplus]] * weightplus)**2) else: logger.warning('Could not identify GLEAM band fluxes') if fluxcolumn is None: logger.error('Could not identify flux columns to use') return None if fluxcolumn in catalogTable.colnames and fluxerrcolumn in catalogTable.colnames: logger.warning('Using %s and %s columns' % (fluxcolumn, fluxerrcolumn)) gleamflux = catalogTable[fluxcolumn] gleamfluxerr = catalogTable[fluxerrcolumn] else: logger.error('Could not identify flux columns to use') return None try: catalogcoords = SkyCoord(catalogTable[racolumn], catalogTable[deccolumn], unit=(u.deg, u.deg)) except KeyError: catalogcoords = SkyCoord(catalogTable['RAJ2000'], catalogTable['DEJ2000'], unit=(u.deg, u.deg)) racolumn = 'RAJ2000' deccolumn = 'DEJ2000' # match the catalog to the data idx, sep2d, sep3d = coords.match_to_catalog_sky(catalogcoords) # add the matched columns to the soure table try: sourcesTable.add_column(Column(catalogTable['Name'][idx], name='Name')) except: pass sourcesTable.add_column(Column(catalogTable[racolumn][idx], name='GLEAMRA')) sourcesTable.add_column( Column(catalogTable[deccolumn][idx], name='GLEAMDEC')) sourcesTable.add_column(Column(sep2d.to(u.arcsec), name='GLEAMSep')) sourcesTable.add_column(Column(gleamflux[idx], name='GLEAMFlux')) sourcesTable.add_column(Column(gleamfluxerr[idx], name='GLEAMFluxErr')) try: sourcesTable.add_column( Column( catalogTable['psf_a_%03d' % bandfrequencies[indexplus]][idx] * catalogTable['psf_b_%03d' % bandfrequencies[indexplus]][idx], name='PSFAREA')) sourcesTable.add_column( Column(catalogTable['a_%03d' % bandfrequencies[indexplus]][idx] * catalogTable['b_%03d' % bandfrequencies[indexplus]][idx], name='SOURCEAREA')) except: pass dRA = (sourcesTable['RA'] - sourcesTable['GLEAMRA']) dDEC = (sourcesTable['Dec'] - sourcesTable['GLEAMDEC']) iterations = 1 if refineposition: iterations = 2 for iter in xrange(iterations): # determine the good matches # first criterion is separation good = (sourcesTable['GLEAMSep'] < matchradius) logger.info('%04d/%04d sources are within %.1f arcsec' % (good.sum(), len(good), matchradius.to(u.arcsec).value)) # only point sources if psfextent is not None and psfextent > 0: good = good & (sourcesTable['SOURCEAREA'] <= psfextent * sourcesTable['PSFAREA']) logger.info( '%04d/%04d sources also have source a*b < %.1f * psf a*b' % (good.sum(), len(good), psfextent)) # cut on the local rms compared to the minimum in the image if rmsfactor is not None and rmsfactor > 0: good = good & (sourcesTable['RMS'] <= rmsfactor * minrms) logger.info('%04d/%04d sources also have RMS < %.1f mJy' % (good.sum(), len(good), rmsfactor * minrms * 1e3)) # distance from pointing center if maxdistance is not None and maxdistance > 0: good = good & (sourcesTable['SOURCEDIST'] < maxdistance) logger.info( '%04d/%04d sources also are within %.1f deg of pointing center' % (good.sum(), len(good), maxdistance.to(u.deg).value)) # primary beam power if minbeam is not None and minbeam > 0: good = good & (sourcesTable['BEAM'] > minbeam * fbeam[0].data.max()) logger.info( '%04d/%04d sources also are at primary beam power > %.2f' % (good.sum(), len(good), minbeam)) # require that all sources are > 5 sigma detections # and that flux uncertainties are > 0 ignorefluxerrs = True if numpy.all(sourcesTable['IntFluxErr'] < 0) or ignorefluxerrs: logger.warning('All source uncertainties are < 0: will ignore') else: good = good & (sourcesTable['IntFluxErr'] < 0.2 * sourcesTable['IntFlux']) & ( sourcesTable['IntFluxErr'] > 0) & (sourcesTable['GLEAMFluxErr'] > 0) & (sourcesTable['GLEAMFluxErr'] < 0.2 * sourcesTable['GLEAMFlux']) try: good = good & (sourcesTable['GLEAMFlux'] >= sourcesTable['IntFlux'][good].min()) pass except ValueError: logger.warning('No good sources left') good = numpy.array([False] * len(good)) logger.info('%04d/%04d sources match all cuts' % (good.sum(), len(good))) if good.sum() < 5: logger.error('Insufficient sources for flux scaling') return None fitres = numpy.polyfit(sourcesTable['GLEAMFlux'][good], sourcesTable['IntFlux'][good], deg=1, w=1 / sourcesTable['IntFluxErr'][good]**2) ratio = sourcesTable['IntFlux'] / sourcesTable['GLEAMFlux'] ratioerr = numpy.sqrt( (sourcesTable['IntFluxErr'] / sourcesTable['GLEAMFlux'])**2 + (sourcesTable['IntFlux'] * sourcesTable['GLEAMFluxErr'] / sourcesTable['GLEAMFlux']**2)**2) if rejectsigma is not None: # do a bit of sigma clipping just in case good = (good) & (numpy.abs(ratio - numpy.median(ratio[good])) <= ratioerr * rejectsigma) fittedratio = (ratio[good] / ratioerr[good]**2).sum() / ( 1 / ratioerr[good]**2).sum() fittedratioerr = numpy.sqrt(1 / (1 / ratioerr[good]**2).sum()) chisq = (((ratio[good] - fittedratio) / ratioerr[good])**2).sum() ndof = good.sum() - 1 logger.info('Found ratio of %s / %s = %.3f +/- %.3f' % (image, catalog, fittedratio, fittedratioerr)) if refineposition and iter == 0: sourcesTable['RA'] -= dRA[good].mean() sourcesTable['Dec'] -= dDEC[good].mean() logger.info('Applied shift of (%.1f sec, %.1f arcsec)' % (dRA[good].mean() * 3600, dDEC[good].mean() * 3600)) coords = SkyCoord(sourcesTable['RA'], sourcesTable['Dec'], unit=(u.deg, u.deg)) idx, sep2d, sep3d = coords.match_to_catalog_sky(catalogcoords) sourcesTable['GLEAMSep'] = sep2d.to(u.arcsec) sourcesTable.add_column(Column(good, name='GOOD')) sourcesTable.meta['ratio'] = fittedratio sourcesTable.meta['ratio_err'] = fittedratioerr sourcesTable.meta['chisq'] = chisq sourcesTable.meta['ndof'] = ndof sourcesTable.meta['slope'] = fitres[0] sourcesTable.meta['intercept'] = fitres[1] if refineposition: sourcesTable.meta['rashift'] = dRA[good].mean() * 3600 sourcesTable.meta['decshift'] = dDEC[good].mean() * 3600 if os.path.exists(outbase + '_fluxmatch.hdf5'): os.remove(outbase + '_fluxmatch.hdf5') sourcesTable.write(outbase + '_fluxmatch.hdf5', path='data') logger.info('Wrote %s_fluxmatch.hdf5' % outbase) if region: outreg = outbase + '_fluxmatch.reg' if os.path.exists(outreg): os.remove(outreg) foutreg = open(outreg, 'w') for i in xrange(len(sourcesTable)): if sourcesTable[i]['GOOD']: foutreg.write( 'icrs;circle(%f,%f,60") # text={%03d} color={green}\n' % (sourcesTable[i]['RA'], sourcesTable[i]['Dec'], i)) else: foutreg.write( 'icrs;box(%f,%f,60",60",0) # text={%03d} color={red}\n' % (sourcesTable[i]['RA'], sourcesTable[i]['Dec'], i)) logger.info('Wrote %s' % outreg) foutreg.close() if update: if fittedratio > limit or fittedratio < 1.0 / limit: logger.warning('Ratio exceeds reasonable limits; skipping...') else: fimage = fits.open(image, 'update') if not 'BEAM' in fimage[0].header.keys(): fimage[0].header['BEAM'] = beam fimage[0].data /= fittedratio fimage[0].header['FLUXSCAL'] = (fittedratio, 'Flux scaling relative to catalog') fimage[0].header['FLUX_ERR'] = ( fittedratioerr, 'Flux scaling uncertainty relative to catalog') fimage[0].header['FLUXCAT'] = (catalog, 'Flux scaling catalog') fimage[0].header['NFLUXSRC'] = ( good.sum(), 'Number of sources used for flux scaling') fimage[0].header['FLUXCHI2'] = (chisq, 'Flux scaling chi-squared') fimage[0].header['FLUXSLOP'] = (fitres[0], 'Flux scaling slope') if refineposition: fimage[0].header['RASHIFT'] = ( dRA[good].mean() * 3600, '[s] RA Shift for catalog match') fimage[0].header['DECSHIFT'] = ( dDEC[good].mean() * 3600, '[arcsec] DEC Shift for catalog match') fimage[0].header['CRVAL1'] -= dRA[good].mean() fimage[0].header['CRVAL2'] -= dDEC[good].mean() if 'IMAGERMS' in fimage[0].header.keys(): fimage[0].header['IMAGERMS'] /= fittedratio if prefix is None: fimage.flush() logger.info('Scaled %s by %.3f' % (image, fittedratio)) else: p, f = os.path.split(image) outfile = os.path.join(p, prefix + f) if os.path.exists(outfile): os.remove(outfile) fimage.writeto(outfile) logger.info('Scaled %s by %.3f and wrote to %s' % (image, fittedratio, outfile)) if otherimages is not None and len(otherimages) > 0: # also update some other images for otherimage in otherimages: foimage = fits.open(otherimage, 'update') foimage[0].data /= fittedratio foimage[0].header['FLUXIMG'] = ( image, 'Image used for flux scaling') foimage[0].header['FLUXSCAL'] = ( fittedratio, 'Flux scaling relative to catalog') foimage[0].header['FLUX_ERR'] = ( fittedratioerr, 'Flux scaling uncertainty relative to catalog') foimage[0].header['FLUXCAT'] = (catalog, 'Flux scaling catalog') foimage[0].header['NFLUXSRC'] = ( good.sum(), 'Number of sources used for flux scaling') foimage[0].header['FLUXCHI2'] = ( chisq, 'Flux scaling chi-squared') foimage[0].header['FLUXSLOP'] = (fitres[0], 'Flux scaling slope') if refineposition: foimage[0].header['RASHIFT'] = ( dRA[good].mean() * 3600, '[s] RA Shift for catalog match') foimage[0].header['DECSHIFT'] = ( dDEC[good].mean() * 3600, '[arcsec] DEC Shift for catalog match') foimage[0].header['CRVAL1'] -= dRA[good].mean() foimage[0].header['CRVAL2'] -= dDEC[good].mean() if 'IMAGERMS' in fimage[0].header.keys(): foimage[0].header['IMAGERMS'] /= fittedratio if prefix is None: foimage.flush() logger.info('Scaled %s by %.3f' % (otherimage, fittedratio)) else: p, f = os.path.split(otherimage) outfile = os.path.join(p, prefix + f) if os.path.exists(outfile): os.remove(outfile) fimage.writeto(outfile) logger.info('Scaled %s by %.3f and wrote to %s' % (otherimage, fittedratio, outfile)) if plot: imagename = image.replace('_', '\_') plt.clf() xx = numpy.logspace(-2, 10) plt.loglog(xx, xx * fittedratio, 'r') plt.loglog(xx, numpy.polyval(fitres, xx), 'r--') plt.errorbar(sourcesTable[good]['GLEAMFlux'], sourcesTable[good]['IntFlux'], xerr=sourcesTable[good]['GLEAMFluxErr'], yerr=sourcesTable[good]['IntFluxErr'], fmt='b.') #plt.gca().set_xscale('log') #plt.gca().set_yscale('log') plt.axis([0.1, 100, 0.1, 100]) plt.xlabel('Flux Density in %s (Jy)' % catalog.replace('_', '\_'), fontsize=16) plt.ylabel('Flux Density in %s (Jy)' % imagename.replace('_', '\_'), fontsize=16) plt.gca().tick_params(labelsize=16) plt.savefig('%s_fluxflux.pdf' % outbase) logger.info('Wrote %s_fluxflux.pdf' % outbase) plt.clf() plt.hist(ratio[good], 30) plt.xlabel('Flux Density in %s / Flux Density in %s' % (imagename.replace('_', '\_'), catalog.replace('_', '\_')), fontsize=16) plt.ylabel('Number of Sources', fontsize=16) plt.plot(fittedratio * numpy.array([1, 1]), plt.gca().get_ylim(), 'r-') plt.gca().tick_params(labelsize=16) plt.savefig('%s_hist.pdf' % outbase) logger.info('Wrote %s_hist.pdf' % outbase) plt.clf() plt.plot(x, y, 'k.') h = plt.scatter(x[good], y[good], s=60, c=ratio[good], norm=matplotlib.colors.LogNorm(vmin=0.5, vmax=2), cmap=plt.cm.BrBG) plt.xlabel('X', fontsize=16) plt.ylabel('Y', fontsize=16) cbar = plt.gcf().colorbar(h, ticks=[0.5, 1, 2]) plt.gca().tick_params(labelsize=16) plt.savefig('%s_scatter.pdf' % outbase) logger.info('Wrote %s_scatter.pdf' % outbase) plt.clf() plt.plot( (sourcesTable['RA'][good] - sourcesTable['GLEAMRA'][good]) * 3600, (sourcesTable['Dec'][good] - sourcesTable['GLEAMDEC'][good]) * 3600, 'ro') plt.plot(plt.gca().get_xlim(), [0, 0], 'k--') plt.plot([0, 0], plt.gca().get_ylim(), 'k--') plt.xlabel('$\\alpha$(%s)-$\\alpha$(%s)' % (imagename.replace('_', '\_'), catalog.replace('_', '\_')), fontsize=16) plt.ylabel('$\\delta$(%s)-$\\delta$(%s)' % (imagename.replace('_', '\_'), catalog.replace('_', '\_')), fontsize=16) plt.gca().tick_params(labelsize=16) plt.savefig('%s_position.pdf' % outbase) logger.info('Wrote %s_position.pdf' % outbase) plt.clf() xx = numpy.linspace(0, 300, 50) plt.hist(sourcesTable['GLEAMSep'].to(u.arcsec).value[~good], xx, color='b', alpha=0.5) plt.hist(sourcesTable['GLEAMSep'].to(u.arcsec).value[good], xx, color='r', alpha=0.5) plt.plot( matchradius.to(u.arcsec).value * numpy.array([1, 1]), plt.gca().get_ylim(), 'k--') plt.xlabel('Separation %s vs. %s (arcsec)' % (imagename.replace('_', '\_'), catalog.replace('_', '\_')), fontsize=16) plt.ylabel('Number of sources', fontsize=16) plt.gca().tick_params(labelsize=16) plt.savefig('%s_separation.pdf' % outbase) logger.info('Wrote %s_separation.pdf' % outbase) return fittedratio, fittedratioerr, chisq, ndof, fitres[0], fitres[1]
def main(): localswarpconfig = os.path.join(os.path.split(__file__)[0], swarpconfig) usage = "Usage: %prog [options] <files>\n" usage += "\tMosaic together MWA driftscan data\n" #usage+="\tImages should be either raw XX,YY or primary-beam corrected I (or Q or U or V)\n" usage += "\tImages should be either raw XX,YY or primary-beam corrected I\n" usage += "\tNormal operation:\n" usage += "\tXX,YY data will be converted into Stokes I\n" usage += "\tStokes I beams will be computed for optimal weighting\n" usage += "\tExcept if --separate and then only XX,YY will be processed and mosaiced separately\n" usage += "\tswarp is used for mosaicing\n" parser = OptionParser(usage=usage, version=mwapy.__version__ + ' ' + mwapy.__date__) parser.add_option('--metafits', action="store_true", dest="metafits", default=False, help="Force creation of metafits files?") parser.add_option( '--copybeam', action="store_true", dest="copy_beam", default=False, help="Copy the beam for the driftscan instead of creating separately?") parser.add_option( '--analytic', action="store_true", dest="analytic_model", default=False, help= "Use the old analytic dipole model, instead of the default Sutinjo 2014 model." ) parser.add_option( '--noprecess', action='store_false', dest='precess', default=True, help= 'Do not precess coordinates to current epoch (faster but less accurate) [default=False]' ) parser.add_option('-t', '--threshold', dest='threshold', default=0.05, type='float', help='Beam threshold [default=%default]') parser.add_option('--reduce', dest='reduce', default=False, action='store_true', help='Reduce dimensionality of input images?') parser.add_option('--oversample', dest='oversample', default=2, type='int', help='Swarp oversampling [default=%default]') parser.add_option( '--ra', dest='ra', default=None, help='Center RA of mosaic in hours or "middle" [default=auto]') parser.add_option('--dec', dest='dec', default=None, help='Center Dec of mosaic in degrees [default=auto]') parser.add_option('--proj', dest='projection', default='MOL', type='choice', choices=['MOL', 'ZEA', 'SIN', 'AIT'], help='Image projection [default=%default]') parser.add_option( '--scale', '--scalar', dest='scalarweight', default=None, help= 'Additional scalar weight header keyword for each image [default=None]' ) parser.add_option( '--function', dest='weightfunction', default='inversesquare', type='choice', choices=['inversesquare', 'inverse', 'linear', 'square'], help= 'Function applied to scalar weight (higher weight is better) [default=%default]' ) parser.add_option( '--separate', dest='separate', default=False, action='store_true', help='Mosaic XX and YY separately as well [default=%default]') parser.add_option('--arguments', dest='swarpcommands', default='', help='Additional arguments for Swarp') parser.add_option('--out', dest='root', default='mosaic', help='SWARP output root [default=%default]') parser.add_option('--swarp', dest='swarp', default=None, help='Path to Swarp executable [default=search path]') parser.add_option('--config', dest='config', default=None, help='Path to separate Swarp config file [default=%s]' % localswarpconfig) parser.add_option('--clean', action="store_true", dest="clean", default=False, help="Clean temporary files?") parser.add_option('-v', '--verbose', action="store_true", dest="verbose", default=False, help="Increase verbosity of output") (options, args) = parser.parse_args() if (options.verbose): logger.setLevel(logging.INFO) if options.config is not None: localswarpconfig = options.config if not os.path.exists(localswarpconfig): logger.error('Cannot find swarp config file %s' % localswarpconfig) sys.exit(1) logger.info('Will use config file %s...' % localswarpconfig) inputfiles = args if len(inputfiles) == 0: logger.error('Must supply >=1 input images') sys.exit(1) Imagelist = {} if options.swarp is None: # python 2.6 compatibility try: swarp = subprocess.check_output(['which', 'swarp']).strip() except AttributeError: swarp = subprocess.Popen( ['which', 'swarp'], stdout=subprocess.PIPE).communicate()[0].strip() else: swarp = options.swarp ############################## # go through the images # figure out which is for which polarization # and see if there is also a metafits file ############################## times = {} times['start'] = time.time() for file in inputfiles: polaxis = None try: obsid = int(file.split('_')[0]) except: try: # allow for wsclean-style output obsid = int(file.split('-')[0]) except: logger.error('Unable to interpret filename %s' % file) sys.exit(0) if not Imagelist.has_key(obsid): Imagelist[obsid] = MWAImage(obsid) mwaimage = Imagelist[obsid] currentstokes = None try: f = pyfits.open(file) if f[0].header['CTYPE3'] == 'STOKES': polaxis = 3 if f[0].header['CTYPE4'] == 'STOKES': polaxis = 4 if not polaxis is None: if f[0].header['CRVAL%d' % polaxis] == -5: mwaimage.ximage = file currentstokes = 'XX' elif f[0].header['CRVAL%d' % polaxis] == -6: mwaimage.yimage = file currentstokes = 'YY' elif f[0].header['CRVAL%d' % polaxis] >= 1 and f[0].header[ 'CRVAL%d' % polaxis] <= 4: mwaimage.iimage = file currentstokes = 'I' else: logger.warning( 'Do not know how to process Stokes=%d for %s; skipping...' % (f[0].header['CRVAL%d' % polaxis], file)) continue else: if 'XX' in file: mwaimage.ximage = file currentstokes = 'XX' elif 'YY' in file: mwaimage.yimage = file currentstokes = 'YY' # see if beams already exist # would be identified through header keywords if 'BEAM' in f[0].header.keys(): if currentstokes == 'I' and os.path.exists( f[0].header['BEAM']): mwaimage.ibeam = f[0].header['BEAM'] elif currentstokes == 'XX' and os.path.exists( f[0].header['BEAM']): mwaimage.xbeam = f[0].header['BEAM'] elif currentstokes == 'YY' and os.path.exists( f[0].header['BEAM']): mwaimage.ybeam = f[0].header['BEAM'] except: if 'XX' in file: mwaimage.ximage = file currentstokes = 'XX' elif 'YY' in file: mwaimage.yimage = file currentstokes = 'YY' if os.path.exists('%d.metafits' % obsid): mwaimage.metafits = '%d.metafits' % obsid logger.info('Found %d input images for %d observations\n' % (len(inputfiles), len(Imagelist.keys()))) times['beams'] = time.time() ############################## # now loop through and do what needs to be done ############################## beams = {} tempfiles = [] for obsid in sorted(Imagelist.keys()): logger.info('Processing %d...' % obsid) if Imagelist[obsid].ximage is None and Imagelist[ obsid].yimage is not None: logger.error('Obsid %d has YY image but no XX image' % obsid) sys.exit(1) if Imagelist[obsid].yimage is None and Imagelist[ obsid].ximage is not None: logger.error('Obsid %d has XX image but no YY image' % obsid) sys.exit(1) s = '%d: ' % obsid if Imagelist[obsid].ximage is not None: s += '(XX,YY)=(%s, %s)' % (Imagelist[obsid].ximage, Imagelist[obsid].yimage) if Imagelist[obsid].iimage is not None: s += ' I=%s' % Imagelist[obsid].iimage if Imagelist[obsid].metafits is not None: s += ' meta=%s' % Imagelist[obsid].metafits logger.info(s) if Imagelist[obsid].xbeam is not None or Imagelist[ obsid].ibeam is not None: s = '%d: ' % obsid if Imagelist[obsid].xbeam is not None: s += '(XXbeam,YYbeam)=(%s, %s)' % (Imagelist[obsid].xbeam, Imagelist[obsid].ybeam) if Imagelist[obsid].ibeam is not None: s += ' Ibeam=%s' % Imagelist[obsid].ibeam logger.info(s) if Imagelist[obsid].iimage is None: # no I image exists: deal with XX, YY fx = pyfits.open(Imagelist[obsid].ximage) fy = pyfits.open(Imagelist[obsid].yimage) # Get scaling from imageheader if options.scalarweight is not None: try: Imagelist[obsid].xscalarweight = weightfunction( fx[0].header[options.scalarweight], options.weightfunction) except: logger.warning( 'Unable to read header keyword "%s" from image %s; setting I scalar weight to 1' % (options.scalarweight, Imagelist[obsid].ximage)) Imagelist[obsid].xscalarweight = 1 try: Imagelist[obsid].yscalarweight = weightfunction( fy[0].header[options.scalarweight], options.weightfunction) except: logger.warning( 'Unable to read header keyword "%s" from image %s; setting I scalar weight to 1' % (options.scalarweight, Imagelist[obsid].yimage)) Imagelist[obsid].yscalarweight = 1 try: result = fx[0].header['DELAYS'] is not None and fx[0].header[ 'DATE-OBS'] is not None xheader = True except KeyError: xheader = False try: result = fy[0].header['DELAYS'] is not None and fy[0].header[ 'DATE-OBS'] is not None yheader = True except KeyError: yheader = False if not (xheader or yheader) or options.metafits: # does not have the necessary keys # need to make metafits if Imagelist[obsid].metafits is None: logger.warning( 'FITS header keywords not present for obsid %d; creating metafits...' % obsid) if not _newmetadata: hh = make_metafiles.Corr2UVFITSHeader(obsid, db=db) T = make_metafiles.instrument_configuration( gpstime=obsid, duration=hh.obs.duration, db=db) result = T.make_instr_config(quick=True) if result is None: logger.error( 'Error making instr_config file for %d' % obsid) sys.exit(1) hh.make_header() T.corr2uvfitsheader = hh h = T.make_metafits(quick=True) else: # use newer interface obs = metadata.MWA_Observation(obsid) if obs.observation_number is None: logger.error('No observation found for %s' % obsid) sys.exit(1) m = metadata.instrument_configuration(obsid) h = m.make_metafits(quick=True) Imagelist[obsid].metafits = '%d.metafits' % obsid if os.path.exists(Imagelist[obsid].metafits): os.remove(Imagelist[obsid].metafits) try: h.writeto(Imagelist[obsid].metafits) logger.info('Metafits for %d written to %s' % (obsid, Imagelist[obsid].metafits)) except Exception, e: logger.error( 'Unable to write metafits file for %d:\n%s' % (obsid, e)) sys.exit(1) if not xheader: fx.close() if not Imagelist[obsid].metafits is None: logger.warning( 'Copying metafits from %s to %s' % (Imagelist[obsid].metafits, Imagelist[obsid].ximage)) result = copy_metafits(Imagelist[obsid].ximage, Imagelist[obsid].metafits) else: # assume we can get it from the YY image? logger.warning( 'Copying metafits from %s to %s' % (Imagelist[obsid].yimage, Imagelist[obsid].ximage)) result = copy_metafits(Imagelist[obsid].ximage, Imagelist[obsid].yimage, keys=['DELAYS', 'DATE-OBS']) if not result: logger.error('Error updating FITS header of %s' % Imagelist[obsid].ximage) sys.exit(1) if not yheader: fy.close() if not Imagelist[obsid].metafits is None: logger.warning( 'Copying metafits from %s to %s' % (Imagelist[obsid].metafits, Imagelist[obsid].yimage)) result = copy_metafits(Imagelist[obsid].yimage, Imagelist[obsid].metafits) else: # assume we can get it from the YY image? logger.warning( 'Copying metafits from %s to %s' % (Imagelist[obsid].ximage, Imagelist[obsid].yimage)) result = copy_metafits(Imagelist[obsid].yimage, Imagelist[obsid].ximage, keys=['DELAYS', 'DATE-OBS']) if not result: logger.error('Error updating FITS header of %s' % Imagelist[obsid].yimage) sys.exit(1) ############################## # now create the beams ############################## fx = pyfits.open(Imagelist[obsid].ximage, 'update') fy = pyfits.open(Imagelist[obsid].yimage, 'update') delays = [int(x) for x in fx[0].header['DELAYS'].split(',')] if Imagelist[obsid].xbeam is not None and Imagelist[ obsid].ybeam is not None: logger.info( 'XX,YY beams already exists for %d; using %s,%s...' % (obsid, Imagelist[obsid].xbeam, Imagelist[obsid].ybeam)) elif not options.copy_beam or (not beams.has_key( fx[0].header['DELAYS'])): logger.info('Creating primary beams for %d' % obsid) out = make_beam.make_beam( Imagelist[obsid].ximage, ext=0, delays=delays, analytic_model=options.analytic_model, jones=False, precess=options.precess) if out is None: logger.error('Problem creating primary beam for %s' % obsid) sys.exit(1) Imagelist[obsid].xbeam, Imagelist[obsid].ybeam = out beams[fx[0].header['DELAYS']] = obsid logger.info('Wrote %s,%s' % (out[0], out[1])) fx[0].header['BEAM'] = Imagelist[obsid].xbeam fy[0].header['BEAM'] = Imagelist[obsid].ybeam fx.flush() fy.flush() tempfiles += out else: logger.info('Copying beam from %s to %s' % (Imagelist[beams[fx[0].header['DELAYS']]].ximage, Imagelist[obsid].ximage)) result = copy_driftscanbeam( Imagelist[beams[fx[0].header['DELAYS']]].ximage, Imagelist[obsid].ximage) if result is None: sys.exit(1) Imagelist[obsid].xbeam, Imagelist[obsid].ybeam = result fx[0].header['BEAM'] = Imagelist[obsid].xbeam fy[0].header['BEAM'] = Imagelist[obsid].ybeam fx.flush() fy.flush() tempfiles += result fx.close() fy.close() fx = pyfits.open(Imagelist[obsid].ximage) fy = pyfits.open(Imagelist[obsid].yimage) if not options.separate: # and primary-beam correct fxB = pyfits.open(Imagelist[obsid].xbeam) fyB = pyfits.open(Imagelist[obsid].ybeam) fx[0].data = (fx[0].data / fxB[0].data + fy[0].data / fyB[0].data) / 2.0 fx[0].header['CRVAL%d' % polaxis] = 1 Imagelist[obsid].iimage = Imagelist[obsid].ximage.replace( '_XX', '_I') if os.path.exists(Imagelist[obsid].iimage): os.remove(Imagelist[obsid].iimage) fx.writeto(Imagelist[obsid].iimage) logger.info('Wrote I image %s' % Imagelist[obsid].iimage) tempfiles.append(Imagelist[obsid].iimage) fxB[0].data = (fxB[0].data + fyB[0].data) / 2.0 fxB[0].header['CRVAL%d' % polaxis] = 1 Imagelist[obsid].ibeam = Imagelist[obsid].xbeam.replace( '_beamXX', '_beamI') Imagelist[obsid].ibeam = Imagelist[obsid].ibeam.replace( '_XX', '_I') if os.path.exists(Imagelist[obsid].ibeam): os.remove(Imagelist[obsid].ibeam) fxB.writeto(Imagelist[obsid].ibeam) logger.info('Wrote I beam %s' % Imagelist[obsid].ibeam) fi = pyfits.open(Imagelist[obsid].iimage, 'update') fi[0].header['BEAM'] = Imagelist[obsid].ibeam fi.flush() fi.close() tempfiles.append(Imagelist[obsid].ibeam) else: # want to make corrected XX and YY fxB = pyfits.open(Imagelist[obsid].xbeam) fyB = pyfits.open(Imagelist[obsid].ybeam) fx[0].data = (fx[0].data / fxB[0].data) Imagelist[obsid].ximage = Imagelist[obsid].ximage.replace( '_XX', '_XXcorr') if os.path.exists(Imagelist[obsid].ximage): os.remove(Imagelist[obsid].ximage) fx.writeto(Imagelist[obsid].ximage) logger.info('Wrote corrected XX image %s' % Imagelist[obsid].ximage) tempfiles.append(Imagelist[obsid].ximage) # and copy the XX beam to correspond to the new name shutil.copy( Imagelist[obsid].xbeam, Imagelist[obsid].xbeam.replace( '_XX', '_XXcorr').replace('_YY', '_YYcorr')) logger.info( 'Copied XX beam from %s -> %s' % (Imagelist[obsid].xbeam, Imagelist[obsid].xbeam.replace( '_XX', '_XXcorr').replace('_YY', '_YYcorr'))) Imagelist[obsid].xbeam = Imagelist[obsid].xbeam.replace( '_XX', '_XXcorr').replace('_YY', '_YYcorr') tempfiles.append(Imagelist[obsid].xbeam) fy[0].data = (fy[0].data / fyB[0].data) Imagelist[obsid].yimage = Imagelist[obsid].yimage.replace( '_YY', '_YYcorr') if os.path.exists(Imagelist[obsid].yimage): os.remove(Imagelist[obsid].yimage) fy.writeto(Imagelist[obsid].yimage) logger.info('Wrote corrected YY image %s' % Imagelist[obsid].yimage) tempfiles.append(Imagelist[obsid].yimage) # and copy the YY beam to correspond to the new name shutil.copy( Imagelist[obsid].ybeam, Imagelist[obsid].ybeam.replace( '_XX', '_XXcorr').replace('_YY', '_YYcorr')) logger.info( 'Copied YY beam from %s -> %s' % (Imagelist[obsid].ybeam, Imagelist[obsid].ybeam.replace( '_XX', '_XXcorr').replace('_YY', '_YYcorr'))) Imagelist[obsid].ybeam = Imagelist[obsid].ybeam.replace( '_XX', '_XXcorr').replace('_YY', '_YYcorr') tempfiles.append(Imagelist[obsid].ybeam) else: # Stokes I (corrected) exists # now just make I beam fi = pyfits.open(Imagelist[obsid].iimage) try: result = fi[0].header['DELAYS'] is not None and fi[0].header[ 'DATE-OBS'] is not None iheader = True except KeyError: iheader = False if not (iheader) or options.metafits: # does not have the necessary keys # need to make metafits if Imagelist[obsid].metafits is None: logger.warning( 'FITS header keywords not present for obsid %d; creating metafits...' % obsid) if not _newmetadata: hh = make_metafiles.Corr2UVFITSHeader(obsid, db=db) T = make_metafiles.instrument_configuration( gpstime=obsid, duration=hh.obs.duration, db=db) result = T.make_instr_config(quick=True) if result is None: logger.error( 'Error making instr_config file for %d' % obsid) sys.exit(1) hh.make_header() T.corr2uvfitsheader = hh h = T.make_metafits(quick=True) else: # use newer interface obs = metadata.MWA_Observation(obsid) if obs.observation_number is None: logger.error('No observation found for %s' % obsid) sys.exit(1) m = metadata.instrument_configuration(obsid) h = m.make_metafits(quick=True) Imagelist[obsid].metafits = '%d.metafits' % obsid if os.path.exists(Imagelist[obsid].metafits): os.remove(Imagelist[obsid].metafits) try: h.writeto(Imagelist[obsid].metafits) logger.info('Metafits for %d written to %s' % (obsid, Imagelist[obsid].metafits)) except Exception, e: logger.error( 'Unable to write metafits file for %d:\n%s' % (obsid, e)) sys.exit(1) if not iheader: fi.close() if not Imagelist[obsid].metafits is None: logger.warning( 'Copying metafits from %s to %s' % (Imagelist[obsid].metafits, Imagelist[obsid].iimage)) result = copy_metafits(Imagelist[obsid].iimage, Imagelist[obsid].metafits) if not result: logger.error('Error updating FITS header of %s' % Imagelist[obsid].iimage) sys.exit(1) ############################## # now create the beams ############################## fi = pyfits.open(Imagelist[obsid].iimage, 'update') delays = [int(x) for x in fi[0].header['DELAYS'].split(',')] if Imagelist[obsid].ibeam is not None: logger.info('I beam already exists for %d; using %s...' % (obsid, Imagelist[obsid].ibeam)) else: if not options.copy_beam or (not beams.has_key( fi[0].header['DELAYS'])): logger.info('Creating primary beams for %d' % obsid) out = make_beam.make_beam( Imagelist[obsid].iimage, ext=0, delays=delays, analytic_model=options.analytic_model, jones=False, precess=options.precess) if out is None: logger.error('Problem creating primary beam for %s' % obsid) sys.exit(1) Imagelist[obsid].xbeam, Imagelist[obsid].ybeam = out beams[fi[0].header['DELAYS']] = obsid logger.info('Wrote %s,%s' % (out[0], out[1])) tempfiles += out else: logger.info( 'Copying beam from %s to %s' % (Imagelist[beams[fi[0].header['DELAYS']]].iimage, Imagelist[obsid].iimage)) result = copy_driftscanbeam( Imagelist[beams[fi[0].header['DELAYS']]].iimage, Imagelist[obsid].iimage) if result is None: sys.exit(1) Imagelist[obsid].xbeam, Imagelist[obsid].ybeam = result tempfiles += result # create the I beam fxB = pyfits.open(Imagelist[obsid].xbeam) fyB = pyfits.open(Imagelist[obsid].ybeam) fxB[0].data = (fxB[0].data + fyB[0].data) / 2.0 fxB[0].header['CRVAL%d' % polaxis] = 1 Imagelist[obsid].ibeam = Imagelist[obsid].xbeam.replace( '_beamXX', '_beamI') Imagelist[obsid].ibeam = Imagelist[obsid].ibeam.replace( '_XX', '_I') if os.path.exists(Imagelist[obsid].ibeam): os.remove(Imagelist[obsid].ibeam) fxB.writeto(Imagelist[obsid].ibeam) logger.info('Wrote I beam %s' % Imagelist[obsid].ibeam) # update the header so that the beam can be re-used in the future fi[0].header['BEAM'] = Imagelist[obsid].ibeam fi.flush() fi.close() tempfiles.append(Imagelist[obsid].ibeam)
def main(): observation_num = None usage = "Usage: %prog [options]\n" usage += '\tMakes metadata FITS file\n' parser = OptionParser(usage=usage, version=mwapy.__version__ + ' ' + mwapy.__date__) parser.add_option('-f', '--filename', dest="filename", help="Create metafiles for <FILE>", metavar="FILE") parser.add_option('-d', '--datetime', dest="datetimestring", help="Create metafiles for <DATETIME> (YYYYMMDDhhmmss)", metavar="DATETIME") parser.add_option('-g', '--gps', dest="gpstime", help="Create metafiles for <GPS>", type='int', metavar="GPS") parser.add_option( '--contiguous', dest="contiguous", action="store_true", default=False, help="Output separate files for contiguous channels only") parser.add_option( '--quick', dest='quick', default=False, action='store_true', help='Quick output (ignores all Rx connections)? [default=%default]') parser.add_option( '--minbaddipoles', dest='min_bad_dipoles', default=2, type='int', help= 'Minimum number of bad dipoles in a single pol required to flag the entire tile [default=%default]' ) parser.add_option('-m', '--maxdiff', dest="maxtimediff", type='int', help="Maximum time difference for search (in sec)", default=10) parser.add_option( '--channels', dest="channels", default=24, type=int, help= "Number of coarse channels [default=%default, select automatically]") parser.add_option( '--dt', dest="dt", default=0, type=float, help="[sec] Integration time [default=%default, select by gpstime]") parser.add_option( '--df', dest="df", default=0, type=int, help="[kHz] Fine channel width [default=%default, select by gpstime]") parser.add_option( '--timeoffset', dest="timeoffset", default=0, type=int, help= "[sec] Time offset between filename and start of data [default=%default]" ) parser.add_option( '-l', '--lock', dest='lock', action='store_true', default=False, help= 'Use \"-l\" option to create header file for locked phase center (RTS input)' ) parser.add_option( '-o', '--output', type=str, default=None, help='Name of output FITS file [default=<gpstime>.metafits]') parser.add_option('-u', '--url', default=metadata._BASEURL, help="URL for metadata retrieval [default=%default]") parser.add_option('-v', '--verbose', action="store_true", dest="verbose", default=False, help="Increase verbosity of output") (options, args) = parser.parse_args() if (options.verbose): logger.setLevel(logging.INFO) observation_num = options.gpstime if options.output is None: options.output = '%d.metafits' % observation_num if observation_num is not None: obs = metadata.MWA_Observation(observation_num, url=options.url) if obs.observation_number is None: logger.error('No observation found for %s' % observation_num) sys.exit(1) if numpy.diff(obs.channels).max() > 1 and options.contiguous: # the frequencies are not contiguous # determine the contiguous ranges df = numpy.diff(obs.channels) frequency_indices = numpy.where(df > 1)[0] channel_selections = [] istart = 0 for istop in frequency_indices: channel_selections.append(numpy.arange(istart, istop + 1)) istart = istop + 1 channel_selections.append(numpy.arange(istart, len(obs.channels))) else: channel_selections = [numpy.arange(len(obs.channels))] m = metadata.instrument_configuration( observation_num, min_bad_dipoles=options.min_bad_dipoles, lock=options.lock) for i, channel_selection in zip(range(len(channel_selections)), channel_selections): logger.info('Creating output for channels %s' % channel_selection) h = m.make_metafits(quick=options.quick) output = options.output if len(channel_selections) > 1: output = output.replace('.metafits', '.%02d.metafits' % i) if os.path.exists(output): os.remove(output) try: h.writeto(output) logger.info('Metafits written to %s' % (output)) except Exception, e: logger.error('Unable to write metafits file %s:\n%s' % (output, e)) sys.exit(1)
def main(): observation_num = None usage = "Usage: %prog [options] <obsids>\n" parser = OptionParser(usage=usage) parser.add_option( '--start', dest='starttime', default=None, type='str', help='Starting time for processing (GPSseconds or ISO UT date)') parser.add_option( '--stop', dest='stoptime', default=None, type='str', help='Final time for processing (GPSseconds or ISO UT date)') parser.add_option('--proj', dest='project', default=None, help='Project ID') parser.add_option('--destination', dest='destination', default=None, help='Final destination to (remote) copy the MS file') parser.add_option('--summaryname', dest='summaryname', default=None, help='Name for output summary table') parser.add_option( '--downloads', dest='downloads', default=4, type='int', help='Number of simultaneous NGAS downloads [default=%default]') parser.add_option('--nomissing', dest='allowmissing', default=True, action='store_false', help='Do not allow missing GPU box files') parser.add_option('--timeres', dest='timeres', default=4, type='float', help='Output time resolution (s) [default=%default]') parser.add_option( '--freqres', dest='freqres', default=40, type='int', help='Output frequency resolution (kHz) [default=%default]') parser.add_option( '--caltimeres', dest='caltimeres', default=0, type='float', help='Output time resolution for calibrators (s) [default=<timeres>]') parser.add_option( '--calfreqres', dest='calfreqres', default=0, type='int', help='Output frequency resolution (kHz) [default=<freqres>]') parser.add_option('--ra', dest='ra', default=None, help='Output RA phase center (deg) [default=metafits]') parser.add_option('--dec', dest='dec', default=None, help='Output Dec phase center (deg) [default=metafits]') parser.add_option('--cal', dest='cal', default=False, action='store_true', help='Always include a calibrator observation?') parser.add_option('--cpus', dest='cottercpus', default=4, type='int', help='Number of CPUs for cotter [default %default]') parser.add_option('--clobber', dest='clobber', default=False, action='store_true', help='Clobber existing MS file? [default=False]') parser.add_option( '--cleanall', dest='cleanall', default=False, action='store_true', help= 'Delete downloaded FITS files and original MS file when done? [default=False]' ) parser.add_option( '--cleanfits', dest='cleanfits', default=False, action='store_true', help='Delete downloaded FITS files when done? [default=False]') parser.add_option( '--cleanms', dest='cleanms', default=False, action='store_true', help='Delete original MS file when done? [default=False]') parser.add_option('--copycommand', dest='copycommand', default='rsync -aruvP', help='Command for (remote) copying [default=%default]') parser.add_option( '--summaryformat', dest='summaryformat', default='ascii.commented_header', help= 'Format for output summary table (from astropy.table) [default=%default]' ) parser.add_option('--email', dest='notifyemail', default=None, help='Notification email [default=no email]') parser.add_option( '--ignorespace', dest='ignorespace', default=False, action='store_true', help='Ignore free-space for file download, processing, and copying?') parser.add_option( "-v", "--verbose", dest="loudness", default=0, action="count", help="Each -v option produces more informational/debugging output") parser.add_option( "-q", "--quiet", dest="quietness", default=0, action="count", help="Each -q option produces less error/warning/informational output") (options, args) = parser.parse_args() if options.cleanall: options.cleanfits = True options.cleanms = True loglevels = { 0: [logging.DEBUG, 'DEBUG'], 1: [logging.INFO, 'INFO'], 2: [logging.WARNING, 'WARNING'], 3: [logging.ERROR, 'ERROR'], 4: [logging.CRITICAL, 'CRITICAL'] } logdefault = 2 # WARNING level = max(min(logdefault - options.loudness + options.quietness, 4), 0) logging.getLogger('').handlers[1].setLevel(loglevels[level][0]) #logger.info('Log level set: messages that are %s or higher will be shown.' % loglevels[level][1]) eh = extra_utils.ExitHandler(os.path.split(sys.argv[0])[-1], email=options.notifyemail) logger.info('**************************************************') logger.info('%s starting at %s UT on host %s with user %s' % (sys.argv[0], datetime.datetime.now(), socket.gethostname(), os.environ['USER'])) logger.info('**************************************************') if options.ra is not None and options.dec is not None: phasecenter = SkyCoord(options.ra, options.dec, frame='icrs', unit='deg') else: phasecenter = None if len(args) == 0: if options.starttime is None: logger.error('Must supply starttime') eh.exit(1) if options.stoptime is None: logger.error('Must supply stoptime') eh.exit(1) logger.debug('Using mwapy version %s' % mwapy.__version__) result = subprocess.Popen(['cotter', '-version'], stdout=subprocess.PIPE).communicate()[0].strip() cotterversion = result.split('\n')[0].split('version')[1].strip() AOFlaggerversion = result.split('\n')[1].split('AOFlagger')[1].strip() logger.debug('Using cotter version %s' % cotterversion) logger.debug('Using AOFlagger version %s' % AOFlaggerversion) results = [] havecalibrator = {} if not (options.starttime is None and options.stoptime is None): GPSstart = parse_time(options.starttime) GPSstop = parse_time(options.stoptime) if options.project is not None: logger.info( 'Will preprocess observations from %s to %s with project=%s' % (GPSstart, GPSstop, options.project)) else: logger.info('Will preprocess observations from %s to %s' % (GPSstart, GPSstop)) if options.project is None: results = metadata.fetch_observations(mintime=GPSstart - 1, maxtime=GPSstop + 1) else: results = metadata.fetch_observations(mintime=GPSstart - 1, maxtime=GPSstop + 1, projectid=tokenize( options.project)) if len(args) > 0: # add in some additional obsids for arg in args: results += metadata.fetch_observations(mintime=int(arg) - 1, maxtime=int(arg) + 1) if results is None or len(results) == 0: logger.error('No observations found') eh.exit(1) logger.info(metadata.MWA_Observation_Summary.string_header()) # check if there is a calibrator present observations = [] for item in results: o = metadata.MWA_Observation_Summary(item) logger.info(str(o)) observations.append(metadata.MWA_Observation(item[0])) if havecalibrator.has_key(observations[-1].center_channel): havecalibrator[observations[-1].center_channel] = havecalibrator[ observations[-1]. center_channel] or observations[-1].calibration else: havecalibrator[ observations[-1].center_channel] = observations[-1].calibration if not all(havecalibrator.values()): for k in havecalibrator.keys(): if not havecalibrator[k]: logger.warning( 'No calibrator observation found for center channel %d' % k) if options.cal: cals = [] for i in xrange(len(results)): channel = observations[i].center_channel if not havecalibrator[channel]: cal = find_calibrator.find_calibrator( observations[i].observation_number, matchproject=False, priority='time', all=False) if cal is None: logger.error( 'Unable to find an appropriate calibrator scan') eh.exit(1) logger.info( 'Found calibrator observation %d for center channel %d' % (cal[0], channel)) cals += metadata.fetch_observations(mintime=cal[0] - 1, maxtime=cal[0] + 1) logger.info(str(metadata.MWA_Observation_Summary( cals[-1]))) havecalibrator[channel] = True results += cals data = None logger.info('Processing observations...\n\n') basedir = os.path.abspath(os.curdir) for obs in results: timeres, freqres = None, None processstart = datetime.datetime.now() observation = Observation(obs[0], basedir=basedir, copycommand=options.copycommand, clobber=options.clobber, cleanms=options.cleanms, cleanfits=options.cleanfits, checkfree=not options.ignorespace) downloadedfiles = observation.download(numdownload=options.downloads) if downloadedfiles is None: break if observation.makemetafits() is None: break if observation.observation.calibration is not None and observation.observation.calibration: if options.caltimeres == 0: timeres = options.timeres else: timeres = options.caltimeres if options.calfreqres == 0: freqres = options.freqres else: freqres = options.calfreqres else: timeres, freqres = options.timeres, options.freqres msfilesize = observation.cotter(cottercpus=options.cottercpus, timeres=timeres, freqres=freqres, phasecenter=phasecenter, allowmissing=options.allowmissing) if msfilesize is None: break if options.destination is not None and observation.copy( options.destination) is None: break row = collections.OrderedDict({ 'date': processstart.strftime('%Y-%m-%dT%H:%M:%S'), 'obsid': obs[0], 'user': os.environ['USER'], 'host': socket.gethostname(), 'cotter': cotterversion.replace(' ', '_'), 'mwapy': mwapy.__version__, 'rawfiles': len(observation.downloadedfiles), 'msfilesize': msfilesize }) if options.destination is not None: row['msfile'] = os.path.join(options.destination, observation.msfile) else: row['msfile'] = os.path.join(observation.outputdir, observation.msfile) if data is None: data = Table([row]) else: data.add_row(row) os.chdir(basedir) if data is not None and options.summaryname is not None: data = data[('date', 'obsid', 'user', 'host', 'cotter', 'mwapy', 'rawfiles', 'msfilesize', 'msfile')] try: data.write(options.summaryname, format=options.summaryformat) logger.info('Summary table written to %s' % options.summaryname) except Exception, e: logger.error( 'Unable to write summary table %s with format %s:\n%s' % (options.summaryname, options.summaryformat, e))
def main(): usage = "Usage: %prog [options]\n" usage += "\tFinds MWA Observations based on a set of query parameters\n" usage += "\tWill return just the obsids(default) or more information (if verbose)\n" usage += '\tFor strings (--proj, --creator, --obsname) can use wildcards "*" and "?" to match\n' o = optparse.OptionParser(usage=usage, version=mwapy.__version__ + ' ' + mwapy.__date__) o.add_option( '--limit', type='int', default=100, help='Limit on number of results returned. [default=%default]') o.add_option('--proj', type='str', default=None, help='Project ID') #o.add_option('--list_projects', action='store_true', # help = 'List available project ids and exit') o.add_option('--start', type='str', default=None, help='Start time (GPSseconds or ISO UT date)') o.add_option('--stop', type='str', default=None, help='Stop time (GPSseconds or ISO UT date)') o.add_option( '--GPSrange', type='str', default=None, dest='gpsrange', help='<start>_<stop> in GPSseconds (overrides --start and --stop)') o.add_option('--racenter', type='str', default=None, help='Center RA for search (decimal degrees or hms)') o.add_option('--deccenter', type='str', default=None, help='Center Dec for search (decimal degrees or dms)') o.add_option( '--rawidth', type='str', default='10d', help= 'Width of RA search box (degrees assumed; will not handle 360d boundary) [default=%default]' ) o.add_option('--decwidth', type='str', default='10d', help='Width of Dec search box (degrees) [default=%default]') o.add_option('--chan', type='int', default=None, help='Center channel') o.add_option('--anychan', type='int', default=None, help='Any channel') o.add_option( '--contiguous', action='store_true', dest='contiguous', default=None, help='Only include observations with contiguous frequency channels') o.add_option( '--discontiguous', action='store_false', dest='contiguous', default=None, help='Only include observations with discontiguous frequency channels') o.add_option('--inttime', type='int', default=None, help='Integration time (s)') o.add_option('--freqres', type='int', default=None, help='Frequency resolution (kHz)') #o.add_option('--withdata*',action='store_true', # help="Only include observations with data.") o.add_option('--cal', action='store_true', default=None, dest='withcal', help="Only get data that are marked as calibrations") o.add_option('--nocal', action='store_false', default=None, dest='withcal', help="Only get data that are not marked as calibrations") o.add_option('--future', action='store_true', default=None, dest='future', help="Only get observations that are in the future") o.add_option('--past', action='store_false', default=None, dest='future', help="Only get observations that are in the past") o.add_option('--creator', type='str', default=None, help='Creator') o.add_option('--mode', type='str', default=None, help='Observation mode') o.add_option('--obsname', type='str', help='Observation name') o.add_option('--close', default=False, action='store_true', help='Sort by observations closest in time to --start') o.add_option( '--maxdiff', default='1d', help= 'Maximum time difference for proximity search with --close [default=%default]' ) # Each -v will increase the log output by one level, each -q will decrease it by one level o.add_option( "-v", "--verbose", dest="loudness", default=0, action="count", help="Each -v option produces more informational/debugging output") o.add_option( "-q", "--quiet", dest="quietness", default=0, action="count", help="Each -q option produces less error/warning/informational output") #o.add_option('-v','--verbose',action="store_true",dest="verbose",default=False, # help="Increase verbosity of output") o.add_option('--full', action="store_true", dest="full", default=False, help="Print full info for each observation") o.add_option('-u', '--url', default=metadata._BASEURL, help="URL for metadata retrieval [default=%default]") o.add_option( '--gridpoint', type='int', default=None, dest='gridpoint', help= 'Pointing gridpoint number. For zenith scans gridpoint=0 [default=%default]' ) o.add_option('--uvfits', action='store_true', default=None, dest='withuvfits', help="Only get observations that have a uvfits file") options, args = o.parse_args() loglevels = { 0: [logging.DEBUG, 'DEBUG'], 1: [logging.INFO, 'INFO'], 2: [logging.WARNING, 'WARNING'], 3: [logging.ERROR, 'ERROR'], 4: [logging.CRITICAL, 'CRITICAL'] } logdefault = 2 # WARNING level = max(min(logdefault - options.loudness + options.quietness, 4), 0) logger.setLevel(loglevels[level][0]) logger.info( 'Log level set: messages that are %s or higher will be shown.' % loglevels[level][1]) cal = None if options.withcal is not None: if options.withcal: cal = 1 else: cal = 0 filetype = None if options.withuvfits is not None: if options.withuvfits: filetype = 13 contiguous = None if options.contiguous is not None: if options.contiguous: contiguous = 1 else: contiguous = 0 future = None if options.future is not None: if options.future: future = 1 else: future = 0 if options.start is not None: GPSstart = parse_time(options.start) if GPSstart is None: sys.exit(1) else: GPSstart = None if options.stop is not None: GPSstop = parse_time(options.stop) if GPSstop is None: sys.exit(1) else: GPSstop = None if options.gpsrange is not None: GPSstart, GPSstop = map(parse_time, options.gpsrange.split('_')) if GPSstart is None or GPSstop is None: sys.exit(1) racenter, deccenter = None, None rawidth, decwidth = None, None if options.racenter is not None: try: racenter = Angle(options.racenter) except UnitsError: try: racenter = Angle(float(options.racenter), unit=u.degree) except ValueError: try: racenter = Angle(options.racenter, unit=u.hourangle) except: logger.error('Unable to parse input RA %s' % options.racenter) sys.exit(1) except: logger.error('Unable to parse input RA %s' % options.racenter) sys.exit(1) if options.deccenter is not None: try: deccenter = Angle(options.deccenter, unit=u.degree) except: logger.error('Unable to parse input Dec %s' % options.deccenter) sys.exit(1) if options.rawidth is not None: try: rawidth = Angle(options.rawidth) except UnitsError: try: rawidth = Angle(options.rawidth, unit=u.degree) except: logger.error('Unable to parse RA width %s' % options.rawidth) sys.exit(1) except: logger.error('Unable to parse RA width %s' % options.rawidth) sys.exit(1) if options.decwidth is not None: try: decwidth = Angle(options.decwidth, unit=u.degree) except: logger.error('Unable to parse input Dec %s' % options.decwidth) sys.exit(1) maxdiff = None if options.maxdiff is not None: # try seconds try: maxdiff = int(options.maxdiff) except ValueError: if options.maxdiff[-1] == 's': maxdiff = int(options.maxdiff[:-1]) elif options.maxdiff[-1] == 'm': maxdiff = int(float(options.maxdiff[:-1]) * 60) elif options.maxdiff[-1] == 'h': maxdiff = int(float(options.maxdiff[:-1]) * 60 * 60) elif options.maxdiff[-1] == 'd': maxdiff = int(float(options.maxdiff[:-1]) * 86400) else: logger.error('Unable to parse time difference %s' % options.maxdiff) sys.exit(1) if GPSstart is not None and GPSstop is not None: logger.info('Searching from %d to %d...' % (GPSstart, GPSstop)) elif GPSstart is not None: logger.info('Searching from %d...' % (GPSstart)) elif GPSstop is not None: logger.info('Searching until %d...' % (GPSstop)) if options.proj is not None: logger.info('Require projectid=%s' % options.proj) if options.withcal is not None: if options.withcal: logger.info('Require calibrator observations') else: logger.info('Require noncalibrator observations') if options.creator is not None: logger.info('Require creator=%s' % options.creator) if options.mode is not None: logger.info('Require mode=%s' % options.mode) if options.chan is not None: logger.info('Require center channel=%d' % options.chan) if options.anychan is not None: logger.info('Require any channel=%d' % options.anychan) if options.contiguous is not None: if options.contiguous: logger.info('Require contiguous frequency channels') else: logger.info('Require discontiguous frequency channels') if options.inttime: logger.info('Require integration time=%d s' % options.inttime) if options.freqres: logger.info('Require frequency resolution=%d kHz' % options.freqres) if racenter is not None: ramin = (max(racenter - rawidth / 2, Angle(0, unit=u.degree))).degree ramax = (min(racenter + rawidth / 2, Angle(360, unit=u.degree))).degree logger.info('Require %.1fd < RA < %.1fd' % (ramin, ramax)) else: ramin, ramax = None, None if deccenter is not None: decmin = (max(deccenter - decwidth / 2, Angle(-90, unit=u.degree))).degree decmax = (min(deccenter + decwidth / 2, Angle(90, unit=u.degree))).degree logger.info('Require %.1fd < Dec < %.1fd' % (decmin, decmax)) else: decmin, decmax = None, None if options.gridpoint is not None: logger.info('Require gridpoint=%d' % options.gridpoint) if options.obsname is not None: logger.info('Require obsname=%s' % options.obsname) if options.future is not None: if options.future: logger.info('Require observation in the future') else: logger.info('Require observation in the past') if options.close: if GPSstart is None: logger.error('Require start time with --close') sys.exit(1) logger.info('Will sort observations within %ds of %d...' % (maxdiff, GPSstart)) if options.withuvfits: logger.info('Require observations with UVFITS files') if not options.close: results = metadata.fetch_observations( URL=options.url, mintime=GPSstart, maxtime=GPSstop, minra=ramin, maxra=ramax, mindec=decmin, maxdec=decmax, limit=options.limit, projectid=tokenize(options.proj), calibration=cal, contigfreq=contiguous, cenchan=options.chan, anychan=options.anychan, int_time=options.inttime, freq_res=options.freqres, creator=tokenize(options.creator), mode=tokenize(options.mode), obsname=tokenize(options.obsname), future=future, gridpoint=options.gridpoint) if results is None or len(results) == 0: print 'Query return no results' sys.exit(0) if loglevels[level][0] <= logging.INFO: print metadata.MWA_Observation_Summary.string_header() elif loglevels[level][0] <= logging.WARNING: print '# starttime' if filetype is not None: uvfitslist = metadata.query_filetype(str(filetype)) results = [obs for obs in results if [obs[0]] in uvfitslist] for item in results: o = metadata.MWA_Observation_Summary(item) if loglevels[level][0] <= logging.INFO: print o else: print o.obsid if options.full: observation = metadata.MWA_Observation(o.obsid, rfstream=0, ionex=False, url=options.url) print observation else: GPSstop = GPSstart + maxdiff results1 = metadata.fetch_observations( URL=options.url, mintime=GPSstart, maxtime=GPSstop, minra=ramin, maxra=ramax, mindec=decmin, maxdec=decmax, limit=options.limit / 2, projectid=tokenize(options.proj), calibration=cal, contigfreq=contiguous, cenchan=options.chan, anychan=options.anychan, int_time=options.inttime, freq_res=options.freqres, creator=tokenize(options.creator), mode=tokenize(options.mode), obsname=tokenize(options.obsname), future=future) GPSstop = GPSstart - maxdiff try: results2 = metadata.fetch_observations( URL=options.url, mintime=GPSstop, maxtime=GPSstart, minra=ramin, maxra=ramax, mindec=decmin, maxdec=decmax, limit=options.limit / 2, projectid=tokenize(options.proj), calibration=cal, contigfreq=contiguous, cenchan=options.chan, anychan=options.anychan, int_time=options.inttime, freq_res=options.freqres, creator=tokenize(options.creator), mode=tokenize(options.mode), obsname=tokenize(options.obsname), future=future, gridpoint=options.gridpoint) results = results2 + results1 except: results = results1 if results is None or len(results) == 0: print 'Query return no results' sys.exit(0) if loglevels[level][0] <= logging.INFO: print metadata.MWA_Observation_Summary.string_header() else: print '# starttime' last = None if filetype is not None: uvfitslist = metadata.query_filetype(str(filetype)) results = [obs for obs in results if [obs[0]] in uvfitslist] for item in results: o = metadata.MWA_Observation_Summary(item) if (last is None or last < GPSstart) and o.obsid >= GPSstart: print '# ***** %d\t%s' % ( GPSstart, Time( GPSstart, format='gps', scale='utc').datetime.strftime('%Y-%m-%dT%H:%M:%S')) if loglevels[level][0] <= logging.INFO: print o else: print o.obsid last = o.obsid if options.full: observation = metadata.MWA_Observation(o.obsid, rfstream=0, ionex=False, url=options.url) obsid = str(GPSstart) lst = str(observation.LST) ra = str(observation.RA) dec = str(observation.Dec) azimuth = str(observation.azimuth) elevation = str(observation.elevation) obsinfo_string = obsid + ", " + lst + ", " + ra + ", " + dec + ", " + azimuth + ", " + elevation + "\n" write_filename = "/nfs/eor-00/h1/rbyrne/sidelobe_survey_obsinfo.txt" openfile = open(write_filename, "a") openfile.write('obsid, LST, RA, Dec, Az, El\n') openfile.write(obsinfo_string) openfile.close() sys.exit(0)
except ValueError, e: logger.error('Unable to parse input time %s: %s' % (options.datetimestring, e)) sys.exit(1) results = metadata.find_closest_observation(int(t.gps), options.maxtimediff, url=options.url) if results is None: logger.error('No observation found within %ds of s' % (options.maxtimediff, t)) sys.exit(1) observation_num = results if observation_num is not None: observation = metadata.MWA_Observation(observation_num, rfstream=options.rfstream, ionex=options.ionex, url=options.url) if observation.observation_number is None: logger.error('No observation found for %s' % observation_num) sys.exit(1) print observation if (options.image): if (not _useplotting): logger.warning( 'Unable to import primarybeammap to generate image\n') return datetimestring = '%04d%02d%02d%02d%02d%02d' % ( observation.year, observation.month, observation.day, observation.hour, observation.minute, observation.second) logger.info(