def nights_moving(request, night='all', min_length=100, age=0): db = DATABASES['favor2'] if request.method == 'GET': if request.GET.get('min_length'): min_length = int(request.GET.get('min_length')) if request.GET.get('max_age'): age = int(request.GET.get('max_age')) if age > 24 * 3600: age = 1800 if age > 0: print age, min_length f = Favor2(dbname=db['NAME'], dbuser=db['USER'], dbpassword=db['PASSWORD']) s = dump_satellites(f, night=night, min_length=min_length, max_age=age, crlf=True, use_zip=True) response = HttpResponse(s, content_type='application/x-zip-compressed') if age > 0: # response['Content-Disposition'] = 'attachment; filename=latest.zip' response[ 'Content-Disposition'] = 'attachment; filename=latest_%d_%s.zip' % ( age, datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S')) else: response['Content-Disposition'] = 'attachment; filename=%s.zip' % night return response
def __init__(self, path = '', id = 0, half_width = 10, load = False, wcs_order=2, width=2560, height=2160, rough=False, excess=4, favor2=None): self.favor2 = favor2 or Favor2() self.original_width = width self.original_height = height self.half_width = half_width self.header = None self.frames = [] self.channels = [] self.wcs_order = wcs_order self.rough = rough self.excess = excess if id: res = self.favor2.query('SELECT * FROM realtime_objects WHERE state=0 AND id=%s', (id,)) if res: path = posixpath.join('ARCHIVE', 'meteor', res['night'], str(id)) print path if path: if load: self.loadMeteor(path) else: self.analyzeMeteor(path) else: self.path = None
def blind_match(filename, outname=None): # dir = tempfile.mkdtemp(prefix='match') # wcs = None # binname = None f = Favor2() if not outname: # outname = posixpath.splitext(filename)[0] + '.new' # if outname == filename: # outname = filename + '.new' fd, outname = tempfile.mkstemp(prefix='match', suffix='.new') os.close(fd) f.blind_match_file(filename, outfile=outname) # for path in ['.', '/usr/local', '/opt/local']: # if os.path.isfile(posixpath.join(path, 'astrometry', 'bin', 'solve-field')): # binname = posixpath.join(path, 'astrometry', 'bin', 'solve-field') # break # if binname: # command = "%s -D %s --overwrite --no-fits2fits --no-plots --no-verify --use-sextractor -t 2 %s --new-fits %s" % (binname, dir, filename, outname) # #os.system("%s >/dev/null 2>/dev/null" % (command)) # os.system("%s" % (command)) # shutil.rmtree(dir, ignore_errors=True) if not posixpath.isfile(outname): outname = None return outname
def realtime_object_delete(id=0): try: db = settings.DATABASES['favor2'] f = Favor2(dbname=db['NAME'], dbuser=db['USER'], dbpassword=db['PASSWORD']) f.delete_object(id) except: pass
def check_tycho2(ra, dec, radius=30.0 / 3600, limit=12, favor2=None): favor2 = favor2 or Favor2() res = favor2.get_stars(ra, dec, radius, catalog='tycho2', extra="vt < %g" % limit) if len(res): return True else: return False
def beholder_plot_status(file, time=None, favor2=None, status=None, extra=None): time = time or datetime.datetime.utcnow() if status is None: favor2 = favor2 or Favor2() res = favor2.query("SELECT * FROM beholder_status WHERE time <= %s ORDER BY time DESC LIMIT 1", (time, ), simplify=True) status = res['status'] time = res['time'] channels = [] mounts = [] interests = [] for id in xrange(1, int(status['nchannels'])+1): if status['channel%d' % (id)] == '1': channels.append({'id':id, 'ra':float(status['channel%d_ra' % (id)]), 'dec':float(status['channel%d_dec' % (id)]), 'ra1':float(status['channel%d_ra1' % (id)]), 'dec1':float(status['channel%d_dec1' % (id)]), 'ra2':float(status['channel%d_ra2' % (id)]), 'dec2':float(status['channel%d_dec2' % (id)]), 'ra3':float(status['channel%d_ra3' % (id)]), 'dec3':float(status['channel%d_dec3' % (id)]), 'ra4':float(status['channel%d_ra4' % (id)]), 'dec4':float(status['channel%d_dec4' % (id)]), 'filter':status['channel%d_hw_filter' % (id)]}) for id in xrange(1, int(status['nmounts'])+1): if status['mount%d' % (id)] == '1': mounts.append({'id':id, 'ra':float(status['mount%d_ra' % (id)]), 'dec':float(status['mount%d_dec' % (id)])}) if status.has_key('scheduler_ninterests'): for id in xrange(int(status['scheduler_ninterests'])): interests.append({'name':status['scheduler_interest%d_name' % (id)], 'ra':float(status['scheduler_interest%d_ra' % (id)]), 'dec':float(status['scheduler_interest%d_dec' % (id)]), 'radius':float(status['scheduler_interest%d_radius' % (id)]), 'color':'green', 'marker':'o', 'size':2000}) if status.has_key('scheduler_suggestion_type') and status.has_key('scheduler_suggestion_name'): interests.append({'name':status['scheduler_suggestion_name'], 'ra':float(status['scheduler_suggestion_ra']), 'dec':float(status['scheduler_suggestion_dec']), 'color':'magenta', 'marker':'+', 'size':100, 'alpha':0.6, 'va': 'top'}) return beholder_plot_map(file, channels=channels, mounts=mounts, interests=interests, extra=extra, time=time, status=status)
def find_transients(night='all', type='survey', id=0, reprocess=False, base='.', channel_id=0, verbose=False): favor2 = Favor2() where = [] where_opts = [] if not reprocess: where.append("keywords->'transients_extracted' != '1'") if id: where.append("id = %s") where_opts.append(int(id)) if type: where.append("type = %s") where_opts.append(type) if night and night != 'all': where.append("night = %s") where_opts.append(night) if channel_id: where.append("channel_id = %s") where_opts.append(channel_id) if where: where_string = "WHERE " + " AND ".join(where) res = favor2.query("SELECT * FROM images %s ORDER BY time;" % (where_string), where_opts, simplify=False, debug=True) for r in res: filename = r['filename'] filename = fix_remote_path(filename, channel_id=r['channel_id']) transients = find_transients_in_frame(filename, verbose=verbose, favor2=favor2) for transient in transients: favor2.query( "INSERT INTO survey_transients (channel_id, frame_id)") favor2.query( "UPDATE images SET keywords = keywords || 'transients_extracted=>1' WHERE id=%s", (r['id'], ))
def update_satcat(): favor2 = Favor2() res = favor2.query( 'SELECT * FROM satellites WHERE (catalogue=1 OR catalogue=3) AND (iname IS NULL OR rcs IS NULL)' ) ids = [_['catalogue_id'] for _ in res] print ids res0 = favor2.query( 'SELECT * FROM satellites WHERE (catalogue=1 OR catalogue=3)') ids0 = [_['catalogue_id'] for _ in res0] #print ids0 lines = requests.get('https://celestrak.com/pub/satcat.txt') print "satcat downloaded" for line in lines.text.splitlines(): iname = line[0:11] norad = line[13:18] rcs = line[119:127] iname, norad, rcs = [_.strip() for _ in iname, norad, rcs] norad = int(norad) rcs = float(rcs) if rcs and rcs != 'N/A' else None if norad not in ids0: continue print norad, iname, rcs favor2.query( 'UPDATE satellites SET rcs=%s WHERE (catalogue=1 OR catalogue=3) AND catalogue_id=%s AND (rcs is NULL or rcs!=%s)', (rcs, norad, rcs)) if norad not in ids: continue if iname: favor2.query( 'UPDATE satellites SET iname=%s WHERE (catalogue=1 OR catalogue=3) AND catalogue_id=%s', (iname, norad)) res = favor2.query( 'SELECT * FROM satellites WHERE (catalogue=1 OR catalogue=3) AND (iname IS NULL OR rcs IS NULL)' ) ids = [_['catalogue_id'] for _ in res] print ids
def download_moving(request, id=0): db = DATABASES['favor2'] if request.method == 'GET': if request.GET.get('min_length'): min_length = int(request.GET.get('min_length')) f = Favor2(dbname=db['NAME'], dbuser=db['USER'], dbpassword=db['PASSWORD']) s = dump_satellites(f, id=int(id), crlf=True, use_zip=True) response = HttpResponse(s, content_type='application/x-zip-compressed') response['Content-Disposition'] = 'attachment; filename=moving_%s.zip' % id return response
def check_planets(ra, dec, time, favor2=None): favor2 = favor2 or Favor2() for o in [ephem.Moon(), ephem.Venus(), ephem.Mars(), ephem.Jupiter()]: favor2.obs.date = time o.compute(favor2.obs) sr = { 'Moon': 15.0, 'Venus': 0.5, 'Jupiter': 0.5, 'Mars': 0.1 }.get(o.name) if coords.sphdist(np.rad2deg(o.ra), np.rad2deg(o.dec), ra, dec)[0] < sr: return True return False
def process_dir(dir, dbname='favor2', dbhost=None): favor2 = Favor2(dbname=dbname, dbhost=dbhost) favor2.conn.autocommit = False # Night night = posixpath.split(dir)[-1] print(night, '/', dir) files = glob.glob('%s/20*.fits' % dir) files.sort() res = favor2.query('SELECT filename FROM images WHERE night=%s', (night, ), simplify=False) filenames = [_['filename'] for _ in res] j = 0 for j, filename in enumerate(files): if filename in filenames: continue try: result = process_file(filename, night=night, favor2=favor2, verbose=True) sys.stdout.write('\r %d / %d - %s' % (j, len(files), filename)) sys.stdout.flush() except KeyboardInterrupt: raise except: import traceback print("Exception while processing", filename) traceback.print_exc() pass #break favor2.conn.commit() if j: print
def process_satellite_track(track): f = Favor2() print "Processing satellite track %d" % track['id'] # Find the satellite try: sat = f.query( "SELECT * FROM satellites WHERE catalogue=%s AND catalogue_id=%s;", (track['catalogue'], track['catalogue_id']), simplify=True) if not sat: # Create new one, if not exists sat = f.query( "INSERT INTO satellites (catalogue, catalogue_id, name, launch_date, country, orbit_inclination, orbit_period, orbit_eccentricity) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING *;", (track['catalogue'], track['catalogue_id'], track['name'], track['launch_date'], track['country'], track['orbit_inclination'], track['orbit_period'], track['orbit_eccentricity'])) except Exception, ex: print ex.message return pass
def __init__(self, favor2=None, nside=128, r0=15.0, restore=True, update=True): self.favor2 = favor2 or Favor2() self.nside = nside self.npix = hp.nside2npix(self.nside) #print "Map resolution is %g degrees" % (np.rad2deg(hp.nside2resol(self.nside))) self.coverage = np.zeros(self.npix) self.restrictions = np.zeros(self.npix) self.iweights = np.zeros(self.npix) theta, phi = hp.pix2ang(nside, np.arange(self.npix)) self.ra = np.rad2deg(phi) self.dec = np.rad2deg(0.5 * np.pi - theta) self.r0 = r0 self.monitoring_exposure = 1000.0 self.latest_time = datetime.datetime.min + datetime.timedelta(days=1) self.update_time = datetime.datetime.min + datetime.timedelta(days=1) self.fast_update_time = datetime.datetime.min + datetime.timedelta( days=1) self.store_time = datetime.datetime.min + datetime.timedelta(days=1) self.altaz_time = datetime.datetime.min + datetime.timedelta(days=1) self.interests = {} self.modes = {'targets', 'LVC', 'survey'} self.update(restore=restore, update=update)
def sql_view(request, readonly=True): if not readonly: assert_permission(request, 'modify_data') context = {} if request.method == 'POST': message = "" query = request.POST.get('query') r = None if query: r = favor2_celery.runSqlQuery.apply_async(kwargs={ 'query': query, 'readonly': readonly }, queue="beholder") if r: # Redirect to the task page return redirect('tasks_task', id=r.id) else: # Redirect to the same view, but with no POST args. if readonly: return redirect('sql') else: return redirect('sql_unsafe') else: # Request the database schema db = DATABASES['favor2'] f = Favor2(dbname=db['NAME'], dbuser=db['USER'], dbpassword=db['PASSWORD']) tables = [] functions = [] res = f.query( "SELECT * FROM information_schema.tables WHERE table_schema='public' ORDER BY table_name", simplify=False) for table in res: columns = f.query( "SELECT * FROM information_schema.columns WHERE table_name=%s ORDER BY ordinal_position", (table['table_name'], ), simplify=False) tables.append({ 'name': table['table_name'], 'table': table, 'columns': columns }) context['tables'] = tables context['functions'] = f.query( "SELECT * FROM information_schema.routines WHERE routine_schema='public' ORDER BY routine_name", simplify=False) return TemplateResponse(request, 'sql.html', context=context)
help='Number of threads to use', action='store', dest='nthreads', type='int', default=0) # parser.add_option('-t', '--type', help='Type', action='store', dest='type', default='survey') parser.add_option('-r', '--replace', help='Replace', action='store_true', dest='replace', default=False) (options, args) = parser.parse_args() favor2 = Favor2(dbname=options.db, dbhost=options.dbhost) cfgs0 = favor2.query( 'select * from (select channel,shutter,pos0,pos1,filter,count(*) from images where (type=\'survey\' or type=\'widefield\') and filter=\'Clear\' group by channel,shutter,pos0,pos1,filter order by channel,shutter,pos0,pos1) t where count>100 order by count' ) cfgs = [] for cfg in cfgs0: res0 = favor2.query( 'select night,count(*) from images where channel=%s and shutter=%s and pos0=%s and pos1=%s and filter=%s and (type=\'survey\' or type=\'widefield\') group by night order by night', (cfg['channel'], cfg['shutter'], cfg['pos0'], cfg['pos1'], cfg['filter'])) is_first = True night1, night2, N = None, None, 0
def process_file(filename, favor2=None, verbose=False, replace=False, dbname=None, dbhost=None, photodir='photometry'): #### Some parameters aper = 2.0 bkgann = None order = 4 bg_order = 4 color_order = 2 sn = 5 if not posixpath.exists(filename): return None # Rough but fast checking of whether the file is already processed if not replace and posixpath.exists(photodir + '/' + filename.split('/')[-2] + '/' + posixpath.splitext(posixpath.split(filename)[-1])[0] + '.cat'): return #### Preparation header = fits.getheader(filename, -1) if header['TYPE'] not in ['survey', 'imaging', 'widefield', 'Swift', 'Fermi', 'test']: return channel = header.get('CHANNEL ID') fname = header.get('FILTER', 'unknown') time = parse_time(header['TIME']) shutter = header.get('SHUTTER', -1) if fname not in ['Clear']: return if fname == 'Clear': effective_fname = 'V' else: effective_fname = fname night = get_night(time) dirname = '%s/%s' % (photodir, night) basename = posixpath.splitext(posixpath.split(filename)[-1])[0] basename = dirname + '/' + basename catname = basename + '.cat' if not replace and posixpath.exists(catname): return if verbose: print(filename, channel, night, fname, effective_fname) image = fits.getdata(filename, -1).astype(np.double) if favor2 is None: favor2 = Favor2(dbname=options.db, dbhost=options.dbhost) #### Basic calibration darkname = favor2.find_image('masterdark', header=header, debug=False) flatname = favor2.find_image('masterflat', header=header, debug=False) if darkname: dark = fits.getdata(darkname) else: dark = None if flatname: flat = fits.getdata(flatname) else: flat = None if dark is None or flat is None: survey.save_objects(catname, None) return image,header = calibrate.calibrate(image, header, dark=dark) # Check whether the calibration failed if 'SATURATE' not in header: print('Calibration failed for', filename) survey.save_objects(catname, None) return #### Basic masking mask = image > 0.9*header['SATURATE'] fmask = ~np.isfinite(flat) | (flat < 0.5) dmask = dark > 10.0*mad_std(dark) + np.nanmedian(dark) if header.get('BLEMISHCORRECTION', 1): # We have to mask blemished pixels blemish = fits.getdata('calibrations/blemish_shutter_%d_channel_%d.fits' % (header['SHUTTER'], header['CHANNEL ID'])) if verbose: print(100*np.sum(blemish>0)/blemish.shape[0]/blemish.shape[1], '% pixels blemished') dmask |= blemish > 0 image[~fmask] *= np.median(flat[~fmask])/flat[~fmask] #### WCS wcs = WCS(header) pixscale = np.hypot(wcs.pixel_scale_matrix[0,0], wcs.pixel_scale_matrix[0,1]) gain = 0.67 if header.get('SHUTTER') == 0 else 1.9 #### Background mask mask_bg = np.zeros_like(mask) mask_segm = np.zeros_like(mask) # bg2 = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64) # for _ in xrange(3): # bg1 = sep.Background(image, mask=mask|mask_bg, bw=256, bh=256) # ibg = bg2.back() - bg1.back() # tmp = np.abs(ibg - np.median(ibg)) > 5.0*mad_std(ibg) # mask_bg |= survey.dilate(tmp, np.ones([50, 50])) # mask_bg = survey.dilate(tmp, np.ones([50, 50])) # Large objects?.. bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128) image1 = image - bg.back() obj0,segm = sep.extract(image1, err=bg.rms(), thresh=10, minarea=10, mask=mask|dmask|fmask|mask_bg, filter_kernel=None, clean=False, segmentation_map=True) mask_segm = np.isin(segm, [_+1 for _,npix in enumerate(obj0['npix']) if npix > 500]) mask_segm = survey.dilate(mask_segm, np.ones([20, 20])) if np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1] > 0.4: print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked, skipping', filename) survey.save_objects(catname, None) return elif verbose: print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked') # Frame footprint at +10 pixels from the edge ra,dec = wcs.all_pix2world([10, 10, image.shape[1]-10, image.shape[1]-10], [10, image.shape[0]-10, image.shape[0]-10, 10], 0) footprint = "(" + ",".join(["(%g,%g)" % (_,__) for _,__ in zip(ra, dec)]) + ")" #### Catalogue ra0,dec0,sr0 = survey.get_frame_center(header=header) cat = favor2.get_stars(ra0, dec0, sr0, catalog='gaia', extra=['g<14', 'q3c_poly_query(ra, dec, \'%s\'::polygon)' % footprint], limit=1000000) if verbose: print(len(cat['ra']), 'star positions from Gaia down to g=%.1f mag' % np.max(cat['g'])) ## Detection of blended and not really needed stars in the catalogue h = htm.HTM(10) m = h.match(cat['ra'], cat['dec'], cat['ra'], cat['dec'], 2.0*aper*pixscale, maxmatch=0) m = [_[m[2]>1e-5] for _ in m] blended = np.zeros_like(cat['ra'], dtype=np.bool) notneeded = np.zeros_like(cat['ra'], dtype=np.bool) for i1,i2,dist in zip(*m): if dist*3600 > 0.5*aper*pixscale: if cat['g'][i1] - cat['g'][i2] < 3: blended[i1] = True blended[i2] = True else: # i1 is fainter by more than 3 mag notneeded[i1] = True if dist*3600 < 0.5*aper*pixscale: if cat['g'][i1] > cat['g'][i2]: notneeded[i1] = True cat,blended = [_[~notneeded] for _ in cat,blended] #### Background subtraction bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128) # bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=32, bh=32) image1 = image - bg.back() #### Detection of all objects on the frame obj0,segm = sep.extract(image1, err=bg.rms(), thresh=2, minarea=3, mask=mask|dmask|fmask|mask_bg|mask_segm, filter_kernel=None, clean=False, segmentation_map=True) obj0 = obj0[(obj0['x'] > 10) & (obj0['y'] > 10) & (obj0['x'] < image.shape[1]-10) & (obj0['y'] < image.shape[0]-10)] obj0 = obj0[obj0['flag'] <= 1] # We keep only normal and blended oblects fields = ['ra', 'dec', 'fluxerr', 'mag', 'magerr', 'flags', 'cat'] obj0 = np.lib.recfunctions.append_fields(obj0, fields, [np.zeros_like(obj0['x'], dtype=np.int if _ in ['flags', 'cat'] else np.double) for _ in fields], usemask=False) obj0['ra'],obj0['dec'] = wcs.all_pix2world(obj0['x'], obj0['y'], 0) obj0['flags'] = obj0['flag'] if verbose: print(len(obj0['x']), 'objects detected on the frame') ## Filter out objects not coincident with catalogue positions h = htm.HTM(10) m = h.match(obj0['ra'], obj0['dec'], cat['ra'], cat['dec'], aper*pixscale) nidx = np.isin(np.arange(len(obj0['ra'])), m[0], invert=True) obj0 = obj0[nidx] if verbose: print(len(obj0['x']), 'are outside catalogue apertures') # Catalogue stars xc,yc = wcs.all_world2pix(cat['ra'], cat['dec'], 0) obj = {'x':xc, 'y':yc, 'ra':cat['ra'], 'dec':cat['dec']} obj['flags'] = np.zeros_like(xc, dtype=np.int) obj['flags'][blended] |= FLAG_BLENDED obj['cat'] = np.ones_like(xc, dtype=np.int) for _ in ['mag', 'magerr', 'flux', 'fluxerr']: obj[_] = np.zeros_like(xc) # Merge detected objects for _ in ['x', 'y', 'ra', 'dec', 'flags', 'mag', 'magerr', 'flux', 'fluxerr', 'cat']: obj[_] = np.concatenate((obj[_], obj0[_])) if verbose: print(len(obj['x']), 'objects for photometry') # Simple aperture photometry obj['flux'],obj['fluxerr'],flag = sep.sum_circle(image1, obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann) obj['flags'] |= flag # Normalize flags obj['flags'][obj['flags'] & sep.APER_TRUNC] |= FLAG_TRUNCATED obj['flags'][obj['flags'] & sep.APER_ALLMASKED] |= FLAG_MASKED obj['flags'] &= FLAG_NORMAL | FLAG_BLENDED | FLAG_TRUNCATED | FLAG_MASKED | FLAG_NO_BACKGROUND | FLAG_BAD_CALIBRATION area,_,_ = sep.sum_circle(np.ones_like(image1), obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann) # Simple local background estimation bgflux,bgfluxerr,bgflag = sep.sum_circann(image1, obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0)) bgarea,_,_ = sep.sum_circann(np.ones_like(image1), obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0)) bgidx = np.isfinite(bgarea) & np.isfinite(area) bgidx[bgidx] &= (bgarea[bgidx] > 10) & (area[bgidx] > 1) obj['flux'][bgidx] -= bgflux[bgidx]*area[bgidx]/bgarea[bgidx] obj['flags'][~bgidx] |= FLAG_NO_BACKGROUND # No local background obj['deltabgflux'] = np.zeros_like(obj['x']) obj['deltabgflux'][bgidx] = bgflux[bgidx]*area[bgidx]/bgarea[bgidx] fidx = np.isfinite(obj['flux']) & np.isfinite(obj['fluxerr']) fidx[fidx] &= (obj['flux'][fidx] > 0) obj['mag'][fidx] = -2.5*np.log10(obj['flux'][fidx]) obj['magerr'][fidx] = 2.5/np.log(10)*obj['fluxerr'][fidx]/obj['flux'][fidx] fidx[fidx] &= (obj['magerr'][fidx] > 0) fidx[fidx] &= 1/obj['magerr'][fidx] > sn for _ in obj.keys(): if hasattr(obj[_], '__len__'): obj[_] = obj[_][fidx] obj['aper'] = aper if verbose: print(len(obj['x']), 'objects with S/N >', sn) if len(obj['x']) < 1000: print('Only', len(obj['x']), 'objects on the frame, skipping', filename) survey.save_objects(catname, None) return obj['fwhm'] = 2.0*sep.flux_radius(image1, obj['x'], obj['y'], 2.0*aper*np.ones_like(obj['x']), 0.5, mask=mask|dmask|fmask|mask_bg|mask_segm)[0] #### Check FWHM of all objects and select only 'good' ones idx = obj['flags'] == 0 idx &= obj['magerr'] < 1/20 fwhm0 = survey.fit_2d(obj['x'][idx], obj['y'][idx], obj['fwhm'][idx], obj['x'], obj['y'], weights=1/obj['magerr'][idx]) fwhm_idx = np.abs(obj['fwhm'] - fwhm0 - np.median((obj['fwhm'] - fwhm0)[idx])) < 3.0*mad_std((obj['fwhm'] - fwhm0)[idx]) obj['flags'][~fwhm_idx] |= FLAG_BLENDED #### Catalogue matching idx = obj['flags'] m = htm.HTM(10).match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], 1e-5) fidx = np.in1d(np.arange(len(cat['ra'])), m[1]) # Stars that got successfully measured and not blended cidx = (cat['good'] == 1) & (cat['var'] == 0) cidx &= np.isfinite(cat['B']) & np.isfinite(cat['V']) # & np.isfinite(cat['lum']) cidx[cidx] &= ((cat['B'] - cat['V'])[cidx] > -0.5) & ((cat['B'] - cat['V'])[cidx] < 2.0) # cidx[cidx] &= (cat['lum'][cidx] > 0.3) & (cat['lum'][cidx] < 30) if np.sum(cidx & fidx & (cat['multi_70'] == 0)) > 2000: cidx &= (cat['multi_70'] == 0) obj['cat_multi'] = 70 elif np.sum(cidx & fidx & (cat['multi_45'] == 0)) > 1000: cidx &= (cat['multi_45'] == 0) obj['cat_multi'] = 45 else: cidx &= (cat['multi_30'] == 0) obj['cat_multi'] = 30 if verbose: print(np.sum(obj['flags'] == 0), 'objects without flags') print('Amount of good stars:', np.sum(cidx & fidx & (cat['multi_70'] == 0)), np.sum(cidx & fidx & (cat['multi_45'] == 0)), np.sum(cidx & fidx & (cat['multi_30'] == 0))) print('Using %d arcsec avoidance radius' % obj['cat_multi']) # We match with very small SR to only account for manually placed apertures if verbose: print('Trying full fit:', len(obj['x']), 'objects,', np.sum(cidx), 'stars') match = Match(width=image.shape[1], height=image.shape[0]) prev_ngoodstars = len(obj['x']) for iter in range(10): if not match.match(obj=obj, cat=cat[cidx], sr=1e-5, filter_name='V', order=order, bg_order=bg_order, color_order=color_order, verbose=False) or match.ngoodstars < 500: if verbose: print(match.ngoodstars, 'good matches, matching failed for', filename) survey.save_objects(catname, None) return if verbose: print(match.ngoodstars, 'good matches, std =', match.std) if match.ngoodstars == prev_ngoodstars: if verbose: print('Converged on iteration', iter) break prev_ngoodstars = match.ngoodstars # Match good objects with stars oidx = obj['flags'] == 0 oidx1,cidx1,dist1 = htm.HTM(10).match(obj['ra'][oidx], obj['dec'][oidx], cat['ra'][cidx], cat['dec'][cidx], 1e-5) x = obj['x'][oidx][oidx1] y = obj['y'][oidx][oidx1] cbv = match.color_term[oidx][oidx1] cbv2 = match.color_term2[oidx][oidx1] cbv3 = match.color_term3[oidx][oidx1] bv = (cat['B'] - cat['V'])[cidx][cidx1] cmag = cat[match.cat_filter_name][cidx][cidx1] mag = match.mag[oidx][oidx1] + bv*cbv + bv**2*cbv2 + bv**3*cbv3 magerr = np.hypot(obj['magerr'][oidx][oidx1], 0.02) dmag = mag-cmag ndmag = ((mag-cmag)/magerr) idx = cmag < match.mag_limit[oidx][oidx1] x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag = [_[idx] for _ in [x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag]] # Match all objects with good objects xy = np.array([x,y]).T xy0 = np.array([obj['x'], obj['y']]).T kd = cKDTree(xy) dist,m = kd.query(xy0, 101) dist = dist[:,1:] m = m[:,1:] vchi2 = mad_std(ndmag[m]**2, axis=1) # Mark regions of too sparse or too noisy matches as bad obj['flags'][vchi2 > 5] |= FLAG_BAD_CALIBRATION # obj['flags'][vchi2 > np.median(vchi2) + 5.0*mad_std(vchi2)] |= FLAG_BAD_CALIBRATION obj['flags'][dist[:,10] > np.median(dist[:,10]) + 10.0*mad_std(dist[:,10])] |= FLAG_BAD_CALIBRATION match.good_idx = (obj['flags'] & FLAG_BAD_CALIBRATION) == 0 if verbose: print(np.sum(match.good_idx), 'of', len(match.good_idx), 'stars are good') #### Store objects to file try: os.makedirs(dirname) except: pass obj['mag_limit'] = match.mag_limit obj['color_term'] = match.color_term obj['color_term2'] = match.color_term2 obj['color_term3'] = match.color_term3 obj['filename'] = filename obj['night'] = night obj['channel'] = channel obj['filter'] = fname obj['cat_filter'] = match.cat_filter_name obj['time'] = time obj['mag_id'] = match.mag_id obj['good_idx'] = match.good_idx obj['calib_mag'] = match.mag obj['calib_magerr'] = match.magerr obj['std'] = match.std obj['nstars'] = match.ngoodstars survey.save_objects(catname, obj, header=header)
def make_flats(night='all', imtype='skyflat', flattype='flat', filter='Clear', shutter=-1): print "Making flats for night %s" % night favor2 = Favor2() limit = 0.015 for channel_id in [1, 2, 3, 4, 5, 6, 7, 8, 9]: res = favor2.query( 'SELECT * FROM images WHERE type=%s AND channel_id=%s AND filter=get_filter_id(%s) AND (night=%s OR %s=\'all\') AND ((keywords->\'SHUTTER\')::int=%s OR %s<0) ORDER BY time ASC', (imtype, channel_id, filter, night, night, shutter, shutter)) print "%d flats for channel %d in filter %s" % (len(res), channel_id, filter) flats = [] filenames = [] calib = None for r in res: filename = fix_remote_path(r['filename'], channel_id=r['channel_id']) flat, header = pyfits.getdata(filename, -1), pyfits.getheader(filename, -1) if calib is None: calib = Calibrator(shutter=int(r['keywords']['SHUTTER']), channel_id=r['channel_id']) flat, header = calib.calibrate(flat, header) idx = flat >= header['SATURATE'] if 'skyflat' in r['type']: darkname = favor2.find_image(r['time'], 'dark', channel_id=r['channel_id']) darkname = fix_remote_path(darkname, channel_id=r['channel_id']) dark = pyfits.getdata(darkname, -1) dark = calib.calibrate(dark) flat = 1.0 * flat - dark flat /= np.median(flat) flat[flat == 0] = 0.1 flat[idx] = 0.1 flats.append(flat) filenames.append(filename) mflat = np.median(flats[:20], axis=0) # Initial estimation of the flat mflat1 = None N = 0 flats1 = [] for i, flat in enumerate(flats): frac = flat / mflat print filenames[i], np.mean(frac), np.std(frac) if np.std(frac) < limit: flats1.append(flat) if mflat1 is None: mflat1 = flat N = 1 else: mflat1 += flat N += 1 print "%d good flats" % N if N: if N < 20: mflat1 = np.median(flats1, axis=0) else: mflat1 /= N r = res[0] time = r['time'] header = pyfits.getheader(filenames[0], -1) header['TYPE'] = flattype if posixpath.exists(posixpath.join('AVG', r['night'])): filename = posixpath.join( 'AVG', r['night'], time.strftime('%Y%m%d_%H%M%S') + '_%d.%s.fits' % (r['channel_id'], flattype)) else: filename = posixpath.join( '/MMT/%d' % r['channel_id'], 'AVG', r['night'], time.strftime('%Y%m%d_%H%M%S') + '_%d.%s.fits' % (r['channel_id'], flattype)) print filename pyfits.writeto(filename, mflat1, header, clobber=True) favor2.register_image(filename, imtype=flattype, time=time)
parser.add_option('-b', '--base', help='Base path for storing data', action='store', dest='base_path', default='TRIGGERS') parser.add_option('-H', '--host', help='Host to connect', action='store', dest='host', default='68.169.57.253') parser.add_option('-p', '--port', help='Port to connect', action='store', dest='port', type='int', default=8099) parser.add_option('--scheduler-host', help='Hostname of scheduler', action='store', dest='scheduler_host', default='localhost') parser.add_option('--scheduler-port', help='Port number of scheduler', action='store', dest='scheduler_port', type='int', default=5557) parser.add_option('-f', '--fake', help='Initiate fake trigger', action='store_true', dest='is_fake', default=False) parser.add_option('--ra', help='RA', action='store', dest='ra', type='float', default=0.0) parser.add_option('--dec', help='Dec', action='store', dest='dec', type='float', default=0.0) (options,args) = parser.parse_args() scheduler_host = options.scheduler_host scheduler_port = options.scheduler_port base_path = options.base_path favor2 = Favor2() # TODO: make it configurable gracedb_username='******' gracedb_password='******' if options.is_fake: print "Initiating fake trigger at RA=%g Dec=%g" % (options.ra, options.dec) favor2.query("INSERT INTO scheduler_targets (ra, dec, name, type, timetolive, priority) VALUES (%s, %s, %s, %s, %s, 10) RETURNING id", (options.ra, options.dec, 'test', 'Swift', 3600), simplify=True) # We should also inform the scheduler that new high-priority target has been added message_to_scheduler('reschedule') else: print "Listening for VOEvents at %s:%d" % (options.host, options.port) print "Serving to scheduler at %s:%d" % (scheduler_host, scheduler_port)
def plot_LVC_map(id=0, size=800, favor2=None, horizon=True, file=None): favor2 = favor2 or Favor2() target = favor2.query("SELECT * FROM scheduler_targets WHERE id=%s", (id, ), simplify=True) images = favor2.query("SELECT * FROM images WHERE keywords->'TARGET ID' = %s::text ORDER BY time", (target['id'],), simplify=False) # Trigger base dir base = posixpath.join('TRIGGERS', target['type'], str(target['external_id'])) files = glob.glob(posixpath.join(base, '*.fits.gz')) files.sort(reverse=True) skymap, header = hp.read_map(files[0], h=True, verbose=False) header = dict(header) date = header['DATE'][:19] date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S') # TODO: use time from file header date = target['time_created'] fig = Figure(facecolor='white', dpi=72, figsize=(size/72, 0.6*size/72), tight_layout=True) ax = fig.add_subplot(111) # set up gnomonic map projection map = Basemap(projection='hammer', lat_0 = 0, lon_0 = 0, resolution = None, celestial = True, ax = ax) # draw the edge of the map projection region (the projection limb) map.drawmapboundary() degtoralabel = lambda deg : "$%d^h$" % int(deg/15) degtodeclabel = lambda deg : "%+d$^\circ$" % deg map.drawparallels(np.arange(-90, 90, 30), color='lightgray')#, fmt=degtodeclabel) map.drawmeridians(np.arange(0, 360, 45), color='lightgray') for h in [0,3,6,9,12,15,18,21]: try: x,y = map(h*15,0) if abs(x) < 1e10 and abs(y) < 1e10: ax.text(x, y, degtoralabel(h*15), color='black') except: pass # Rebin the map nside = hp.npix2nside(len(skymap)) ra=np.arange(0, 360, 1) dec=np.arange(-90, 90, 1) theta, phi = np.deg2rad(90 - dec), np.deg2rad(ra) PHI, THETA = np.meshgrid(phi, theta) grid_pix = hp.ang2pix(nside, THETA, PHI) grid_map = skymap[grid_pix] ra0, dec0 = np.meshgrid(ra, dec) m = map.pcolormesh(ra0, dec0, grid_map, latlon=True, cmap='YlOrRd') #map.colorbar(m) for image in images: filename = fix_remote_path(image['filename'], channel_id=image['channel_id']) header = pyfits.getheader(filename, -1) wcs = pywcs.WCS(header) ra1, dec1 = wcs.all_pix2world([0, 0, header['NAXIS1'], header['NAXIS1'], 0], [0, header['NAXIS2'], header['NAXIS2'], 0, 0], 0) try: map.plot(ra1, dec1, '-', latlon=True, color='blue') except: pass fig.suptitle("Trigger %s at %s" % (target['name'], date)) if images: ax.set_title("%d images from %s to %s" % (len(images), images[0]['time'], images[-1]['time'])) if horizon: for iter in [0,1]: if iter == 0: if images: favor2.obs.date = images[0]['time'] else: favor2.obs.date = date color, alpha = 'black', 1 else: favor2.obs.date = datetime.datetime.utcnow() color, alpha = 'black', 0.3 ra_h, dec_h = [], [] for A in np.arange(0, 2*np.pi, 0.1): radec = favor2.obs.radec_of(A, np.deg2rad(0)) ra_h.append(np.rad2deg(radec[0])) dec_h.append(np.rad2deg(radec[1])) map.plot(ra_h, dec_h, '--', latlon=True, color=color, alpha=alpha) favor2.obs.date = date # Sun s = favor2.sun s.compute(favor2.obs) px, py = map(np.rad2deg(s.ra), np.rad2deg(s.dec)) if abs(px) < 1e10 and abs(py) < 1e10: map.scatter(px, py, s=2000, c='lightgray', marker='o', alpha=0.7) ax.text(px, py, "Sun", color='black', va='center', ha='center') # Moon m = favor2.moon m.compute(favor2.obs) px, py = map(np.rad2deg(m.ra), np.rad2deg(m.dec)) if abs(px) < 1e10 and abs(py) < 1e10: map.scatter(px, py, s=1200, c='lightgray', marker='o', alpha=0.7) ax.text(px, py, "Moon", color='black', va='center', ha='center') # Return the image if file: canvas = FigureCanvas(fig) canvas.print_png(file, bbox_inches='tight')
def process_file(filename, night=None, favor2=None, verbose=False): if favor2 is None: favor2 = Favor2() header = fits.getheader(filename, -1) if night is None: time = parse_time(header['TIME']) night = get_night(time) if verbose: print(night, header['TYPE']) # Skip old master calibrations if header['TYPE'] in ['bgflat', 'superflat', 'flat']: return None image = fits.getdata(filename, -1).astype(np.double) # Frame dimensions width, height = header['NAXIS1'], header['NAXIS2'] # image,header = crop_overscans(image, header, subtract=False) if header['TYPE'] not in ['flat', 'masterflat', 'dark', 'masterdark']: image -= header.get('BASELINE', 100.0) # Clean up the header a bit header.remove('HISTORY', remove_all=True, ignore_missing=True) header.remove('COMMENT', remove_all=True, ignore_missing=True) header.remove('', remove_all=True, ignore_missing=True) for _ in header.keys(): if _ and _[0] == '_': header.remove(_, remove_all=True, ignore_missing=True) type = header.get('TYPE', 'unknown') filter = header.get('FILTER', 'unknown') if filter == 'Custom' or 'Pol' in filter: print('Broken filter in', filename) return None if type not in [ 'dark', 'masterdark', 'flat', 'skyflat', 'superflat', 'masterflat' ]: wcs = WCS(header) if header.get('CTYPE1') else None if not wcs or not wcs.sip: print('No WCS information in', filename) return None ra, dec = wcs.all_pix2world([0, image.shape[1], 0.5 * image.shape[1]], [0, image.shape[0], 0.5 * image.shape[0]], 0) ra1, dec1 = wcs.all_pix2world([image.shape[1], 0], [0, image.shape[0]], 0) radius = 0.5 * coords.sphdist(ra[0], dec[0], ra[1], dec[1])[0] radius1 = 0.5 * coords.sphdist(ra1[0], dec1[0], ra1[1], dec1[1])[0] # Sanity checking with some hard-coded values if radius < 7.25 or radius > 7.45 or np.abs(1.0 - radius / radius1) > 0.1: print('Broken WCS information in', filename) return None ra0, dec0 = ra[2], dec[2] # Frame footprint ra, dec = wcs.all_pix2world([0, 0, image.shape[1], image.shape[1]], [0, image.shape[0], image.shape[0], 0], 0) footprint = "(" + ",".join( ["(%g,%g)" % (_, __) for _, __ in zip(ra, dec)]) + ")" # Frame footprint at +10 pixels from the edge ra, dec = wcs.all_pix2world( [10, 10, image.shape[1] - 10, image.shape[1] - 10], [10, image.shape[0] - 10, image.shape[0] - 10, 10], 0) footprint10 = "(" + ",".join( ["(%g,%g)" % (_, __) for _, __ in zip(ra, dec)]) + ")" else: # Should we really discard WCS for non-object frames? ra0, dec0, radius = None, None, None footprint, footprint10 = None, None time = parse_time(header['TIME']) exposure = header.get('EXPOSURE') shutter = header.get('SHUTTER') channel = header.get('CHANNEL ID') pos0 = header.get('MIRROR_POS0') pos1 = header.get('MIRROR_POS1') mean = np.mean(image) keywords = dict(header) favor2.query( 'INSERT INTO images (filename,night,time,channel,type,filter,exposure,shutter,pos0,pos1,ra,dec,radius,width,height,mean,footprint,footprint10,keywords) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (filename) DO NOTHING', (filename, night, time, channel, type, filter, exposure, shutter, pos0, pos1, ra0, dec0, radius, width, height, mean, footprint, footprint10, keywords)) return { 'filename': filename, 'night': night, 'time': time, 'channel': channel, 'type': type, 'filter': filter, 'shutter': shutter, 'ra0': ra0, 'dec0': dec0, 'radius': radius, 'exposure': exposure, 'width': width, 'height': height, 'mean': mean, 'keywords': keywords }
def calibrate_image(filename, dark='', flat='', simple=True, match=True, clean=False, maglim=None): print "Performing photometric calibration of file %s" % filename res = {} image = pyfits.getdata(filename, -1) header = pyfits.getheader(filename, -1) f = Favor2() if match or not has_wcs(filename): print "No WCS info, blindmatching" wcs = f.blind_match_file(filename) else: wcs = pywcs.WCS(header=header) res['wcs'] = wcs if not wcs: print "WCS match failure" return res ra0, dec0, sr0 = get_frame_center(header=header, wcs=wcs) res['ra0'], res['dec0'], res['sr0'] = ra0, dec0, sr0 # Saturated pixels, to be masked after the flatfielding #isat = image == np.max(image) isat = image >= 30000 if dark: idark = pyfits.getdata(dark, -1) image = 1.0 * image - idark if flat: iflat = pyfits.getdata(flat, -1) iflat = 1.0 * iflat - idark image *= np.mean(iflat) / iflat if clean: image = clean_image(image) image[isat] = 65535 # Detect objects obj = get_objects(image=image, wcs=wcs) print "%d objects detected" % (len(obj['flux'])) res['filter'] = header['FILTER'] res['mag_calibrated'] = False if len(obj['flux']) > 1: # Get the catalogue cat = f.get_stars(ra0, dec0, sr0, catalog='pickles') zero = get_zero_point_simple(obj, cat, filter=res['filter'], maglim=maglim) if zero: mag0 = zero['mag0'] if mag0: res['mag_calibrated'] = True else: mag0 = 0 res['mag0'] = mag0 return res
def handle(self, *args, **options): f = Favor2() f.query('REFRESH MATERIALIZED VIEW CONCURRENTLY satellites_view')
from sextractor import Sextractor from favor2 import Favor2 from esutil import coords, htm def breakpoint(): try: from IPython.core.debugger import Tracer Tracer()() except: import pdb pdb.set_trace() f = Favor2() def has_wcs(filename=None, header=None): if not header: header = pyfits.getheader(filename, -1) w = pywcs.WCS(header=header) return w.wcs.has_cd() def get_frame_center(filename=None, header=None): if not header: header = pyfits.getheader(filename, -1)
def refreshViews(): f = Favor2() res = f.query('REFRESH MATERIALIZED VIEW CONCURRENTLY satellites_view') return res
def survey_dbg_image(request, id=0, size=900, cat='pickles'): if request.method == 'GET': if request.GET.has_key('cat'): cat = request.GET.get('cat') image_obj = Images.objects.get(id=id) filename = fix_remote_path(image_obj.filename, image_obj.channel_id) darkname = find_image(image_obj.time, 'dark', image_obj.channel_id) darkname = fix_remote_path(darkname, image_obj.channel_id) flatname = find_image(image_obj.time, 'flat', image_obj.channel_id) flatname = fix_remote_path(flatname, image_obj.channel_id) data = pyfits.getdata(filename, -1) header = pyfits.getheader(filename, -1) # Prepare if posixpath.exists(darkname): dark = pyfits.getdata(darkname, -1) data -= dark if posixpath.exists(flatname): flat = pyfits.getdata(flatname, -1) data *= np.mean(flat) / flat # Calibrate obj = survey.get_objects_from_file(filename, simple=False, filter=True) wcs = pywcs.WCS(header) ra0, dec0, sr0 = survey.get_frame_center(header=header, wcs=wcs) print filename, '-', ra0, dec0, sr0 favor2 = Favor2() if cat == 'apass': cat = favor2.get_stars(ra0, dec0, sr0, catalog='apass', limit=600000, extra=[ "g<14 and g>8.5", "b<20 and v<20 and r<20 and i<20 and g<20", "var = 0", "gerr>0 and rerr>0 and ierr>0" ]) else: cat = favor2.get_stars(ra0, dec0, sr0, catalog='pickles', limit=200000, extra=["var = 0", "rank=1"]) print "%d objects, %d stars" % (len(obj['ra']), len(cat['ra'])) survey.fix_distortion(obj, cat, header=header) Y, YY, corr, oidx = survey.calibrate_objects(obj, cat, sr=10. / 3600, retval='all') # Plot dpi = 72 figsize = (size / dpi, size * 4 * 0.6 / dpi) fig = Figure(facecolor='white', figsize=figsize, tight_layout=True) # 1 ax = fig.add_subplot(411) limits = np.percentile(data, [0.5, 99.5]) i = ax.imshow(data, origin='lower', cmap='hot', interpolation='nearest', vmin=limits[0], vmax=limits[1]) fig.colorbar(i) ax.set_title( "%d / %s / ch=%d at %s" % (image_obj.id, image_obj.type, image_obj.channel_id, image_obj.time)) # 2 ax = fig.add_subplot(412) gmag0, bins, binloc = griddata(obj['x'][oidx], obj['y'][oidx], Y, binsize=100) limits = np.percentile(gmag0[np.isfinite(gmag0)], [0.5, 95.5]) i = ax.imshow(gmag0, origin='lower', extent=[0, header['NAXIS1'], 0, header['NAXIS2']], interpolation='nearest', vmin=limits[0], vmax=limits[1]) fig.colorbar(i) ax.autoscale(False) ax.plot(obj['x'][oidx], obj['y'][oidx], 'b.', alpha=0.3) ax.set_title("Actual zero point") # 3 ax = fig.add_subplot(413) gmag0, bins, binloc = griddata(obj['x'][oidx], obj['y'][oidx], YY, binsize=100) #limits = np.percentile(gmag0[np.isfinite(gmag0)], [0.5, 95.5]) i = ax.imshow(gmag0, origin='lower', extent=[0, header['NAXIS1'], 0, header['NAXIS2']], interpolation='nearest', vmin=limits[0], vmax=limits[1]) fig.colorbar(i) ax.autoscale(False) ax.plot(obj['x'][oidx], obj['y'][oidx], 'b.', alpha=0.3) ax.set_title("Fitted zero point") # 4 ax = fig.add_subplot(414) gmag0, bins, binloc = griddata(obj['x'][oidx], obj['y'][oidx], Y - YY, binsize=100) limits = np.percentile(gmag0[np.isfinite(gmag0)], [0.5, 95.5]) i = ax.imshow(gmag0, origin='lower', extent=[0, header['NAXIS1'], 0, header['NAXIS2']], interpolation='nearest', vmin=limits[0], vmax=limits[1]) fig.colorbar(i) ax.autoscale(False) ax.plot(obj['x'][oidx], obj['y'][oidx], 'b.', alpha=0.3) ax.set_title("Zero point residuals. std=%g" % np.std(Y - YY)) canvas = FigureCanvas(fig) response = HttpResponse(content_type='image/png') canvas.print_png(response) return response
def find_transients_in_frame(filename, refine=True, favor2=None, workdir=None, figure=None, show=False, verbose=False, plot=True): transients = [] favor2 = favor2 or Favor2() ra0, dec0, sr0 = get_frame_center(filename=filename) print "Frame %s at RA=%g Dec=%g with SR=%g" % (filename, ra0, dec0, sr0) # Dir to hold all temporary files basedir = workdir or tempfile.mkdtemp(prefix='survey') filtered_name = posixpath.join(basedir, "filtered.fits") print "Cleaning the frame" cleaned_name = posixpath.join(basedir, "clean.fits") filter_cosmics_in_file(filename, cleaned_name) header = pyfits.getheader(cleaned_name, -1) time = datetime.datetime.strptime(header['TIME'], '%d.%m.%Y %H:%M:%S.%f') print "Pre-processing the frame" os.system("nebuliser %s noconf %s 3 3 --takeout_sky" % (cleaned_name, filtered_name)) if refine: print "Refining the WCS" matched_name = posixpath.join(basedir, "image.fits") if not favor2.blind_match_file(filtered_name, outfile=matched_name): print "Failure refining the WCS" return transients else: matched_name = filtered_name print "Extracting the objects" obj = ss.get_objects(filename=matched_name, options={ 'DETECT_MINAREA': 3.0, 'DETECT_THRESH': 1.5, 'BACKPHOTO_TYPE': 'GLOBAL', 'DEBLEND_NTHRESH': 64, 'DEBLEND_MINCONT': 0.001, 'FILTER': 'Y', 'BACK_TYPE': 'MANUAL', 'BACK_VALUE': 0.0, 'BACK_SIZE': 64, 'BACK_FILTERSIZE': 3 }, fwhm=2, aper_fwhm=2) print "Querying the catalogues" cat = favor2.get_stars(ra0, dec0, sr0, catalog='twomass', limit=1000000) catn = favor2.get_stars(ra0, dec0, sr0, catalog='nomad', limit=1000000) catt = favor2.get_stars(ra0, dec0, sr0, catalog='tycho2', limit=1000000) cattb = favor2.get_stars(ra0, dec0, sr0, catalog='tycho2', limit=1000000, extra='vt < 7') print "%d objects, %d 2MASS stars, %d NOMAD stars, %d bright Tycho2 stars" % ( len(obj['ra']), len(cat['ra']), len(catn['ra']), len(cattb['ra'])) print "Matching the objects" wcs = pywcs.WCS(header) pixscale = wcs.pixel_scale_matrix.max( ) * 3600 # estimate of pixel scale, in arcsec/pix size = 2.0 * obj['A_WORLD'] * pixscale size = np.maximum(size, np.repeat(40.0, len(obj['ra']))) h = htm.HTM(10) m = h.match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], size / 3600, maxmatch=0) mn = h.match(obj['ra'], obj['dec'], catn['ra'], catn['dec'], size / 3600, maxmatch=0) mt = h.match(obj['ra'], obj['dec'], catt['ra'], catt['dec'], size / 3600, maxmatch=0) if len(cattb['ra']): mtb = h.match(obj['ra'], obj['dec'], cattb['ra'], cattb['dec'], (size + 100.0) / 3600, maxmatch=0) uidx = np.setdiff1d(np.arange(len(obj['ra'])), m[0]) print "%d objects not matched with 2MASS" % len(uidx) uidx = np.setdiff1d(uidx, mn[0]) print "%d objects not matched with NOMAD" % len(uidx) uidx = np.setdiff1d(uidx, mt[0]) print "%d objects not matched with Tycho2" % len(uidx) if len(cattb['ra']): uidx = np.setdiff1d(uidx, mtb[0]) print "%d objects not matched with brightest Tycho2 stars" % len(uidx) if len(uidx) < 100: uidx = filter_indices(uidx, obj, time) print "%d objects after line filtering and GSC 2.3.2 matching" % len( uidx) # Get the image and project the catalogues onto it image = pyfits.getdata(filtered_name, -1) cat_x, cat_y = wcs.all_world2pix(cat['ra'], cat['dec'], 0) catn_x, catn_y = wcs.all_world2pix(catn['ra'], catn['dec'], 0) catt_x, catt_y = wcs.all_world2pix(catt['ra'], catt['dec'], 0) if len(cattb['ra']): cattb_x, cattb_y = wcs.all_world2pix(cattb['ra'], cattb['dec'], 0) if len(uidx) > 100: print "Too many unmatched objects (%d), something is wrong!" % len( uidx) else: # Let's iterate over the transients for i in uidx: transient = { 'x': float(obj['x'][i]), 'y': float(obj['y'][i]), 'flux': float(obj['FLUX_AUTO'][i]), 'flux_err': float(obj['FLUXERR_AUTO'][i]), 'flags': int(obj['FLAGS'][i]), 'a': float(obj['A_WORLD'][i]), 'b': float(obj['B_WORLD'][i]), 'theta': float(obj['THETA_WORLD'][i]), 'ra': float(obj['ra'][i]), 'dec': float(obj['dec'][i]), 'filename': filename, 'time': time, 'channel_id': header['CHANNEL ID'] } transient['simbad'] = None transient['mpc'] = None try: res = Simbad.query_region( SkyCoord(obj['ra'][i], obj['dec'][i], unit='deg'), "30 seconds") if res is not None: transient['simbad'] = ", ".join( [r for r in res['MAIN_ID']]) except: pass try: # Minor planets mpc = get_skybot(obj['ra'][i], obj['dec'][i], 30.0 / 3600, time, objFilter=110) # Comets if mpc is None: mpc = get_skybot(obj['ra'][i], obj['dec'][i], 300.0 / 3600, time, objFilter=1) if mpc is not None: transient['mpc'] = ", ".join([ "%s %s %s (Mv=%s)" % (mpc['class'][ii], mpc['num'][ii], mpc['name'][ii], mpc['Mv'][ii]) for ii in xrange(len(mpc['name'])) ]) except: pass if verbose: #print transient print "x = %g y = %g" % (obj['x'][i], obj['y'][i]) print "RA/Dec = %g %g = %s %s" % ( obj['ra'][i], obj['dec'][i], str(ephem.hours(obj['ra'][i] * np.pi / 180)).replace( ":", " "), str(ephem.degrees( obj['dec'][i] * np.pi / 180)).replace(":", " ")) time0 = time.replace(hour=0, minute=0, second=0, microsecond=0) print "Time = %s" % time print "Time = %s + %g" % (time0, (time - time0).total_seconds() / 24 / 3600) if transient['simbad']: print "SIMBAD: " + transient['simbad'] print if plot: icrop, x0, y0 = crop_image(image, obj['x'][i], obj['y'][i], 40) limits = np.percentile(icrop, [0.5, 99.5]) if not figure: size = 400 figure = Figure(facecolor='white', dpi=72, figsize=(size / 72, size / 72), tight_layout=True) figure.clear() ax = figure.add_axes([0, 0, 1, 1]) ax.axis('off') im = ax.imshow(icrop, origin="bottom", interpolation="nearest", cmap="hot", vmin=limits[0], vmax=limits[1]) #figure.colorbar(im) ax.autoscale(False) ax.scatter(obj['x'] - x0, obj['y'] - y0, marker='o', color='green', s=400, facecolors='None', lw=3) ax.scatter(cat_x - x0, cat_y - y0, marker='o', color='blue', s=100, facecolors='None', lw=2) ax.scatter(catt_x - x0, catt_y - y0, marker='x', s=100, color='blue', facecolors='None', lw=2) #ellipse = Ellipse(xy=(obj['x'][i]-x0, obj['y'][i]-y0), width=2.0*obj['A_WORLD'][i], height=2.0*obj['B_WORLD'][i], angle=obj['THETA_WORLD'][i], edgecolor='b', fc='None', axes=ax) #ax.add_patch(ellipse) if show: figure.show(warn=False) else: canvas = FigureCanvas(figure) sio = StringIO.StringIO() canvas.print_png(sio, bbox_inches='tight') transient[ 'preview'] = "data:image/png;base64," + base64.b64encode( sio.getvalue()) pass else: transient['preview'] = None transients.append(transient) # cleanup if not workdir: shutil.rmtree(basedir) print "Done" return transients
def process_cfg(cfg, dbname=None, dbhost=None, replace=False): print( "Processing channel %d / shutter %d / pos %d %d / filter %s / %s - %s: %d frames" % (cfg['channel'], cfg['shutter'], cfg['pos0'], cfg['pos1'], cfg['filter'], cfg['night1'], cfg['night2'], cfg['count'])) favor2 = Favor2(dbname=dbname, dbhost=dbhost) res = favor2.query( 'select time,night,filename from images where channel=%s and shutter=%s and pos0=%s and pos1=%s and filter=%s and (type=\'survey\' or type=\'widefield\') and night>=%s and night<=%s order by time', (cfg['channel'], cfg['shutter'], cfg['pos0'], cfg['pos1'], cfg['filter'], cfg['night1'], cfg['night2'])) night0 = res[0]['night'] header0 = fits.getheader(res[0]['filename'], -1) header0['type'] = 'masterflat' header0['NFRAMES'] = cfg['count'] header0['NIGHT1'] = cfg['night1'] header0['NIGHT2'] = cfg['night2'] darkname = favor2.find_image(res[0]['time'], type='masterdark', shutter=cfg['shutter'], channel=cfg['channel']) dark = fits.getdata(darkname, -1) basename = 'calibrations/masterflats/superflat_channel_%d_shutter_%d_pos_%d_%d_%s_%s.fits' % ( cfg['channel'], cfg['shutter'], cfg['pos0'], cfg['pos1'], cfg['filter'], night0) if posixpath.exists(basename) and not replace: print('Skipping', basename) return random.shuffle(res) images = [] coadd, counts = None, 0 for i, r in enumerate(res): filename = r['filename'] image, header = fits.getdata(filename, -1), fits.getheader(filename, -1) image, header = calibrate(image, header, dark=dark) # Mask stars in the image bg = sep.Background(image) mask = image > bg.back() + 2.0 * bg.rms() image[mask] = np.nan image /= np.nanmedian(image) images.append(image) if len(images) == 5: flat = np.nanmedian(images, axis=0) flat /= np.nanmedian(flat) idx = np.isfinite(flat) images = [] if coadd is None: coadd = np.zeros_like(image) counts = np.zeros_like(image, dtype=np.int) coadd[idx] += flat[idx] counts[idx] += 1 print(i, '/', len(res), basename) # Make masterflat and interpolate the gaps idx = counts > 5 masterflat = np.ones_like(coadd) * np.nan masterflat[idx] = coadd[idx] / counts[idx] idx |= masterflat < 0.1 bg = sep.Background(masterflat, mask=~idx) masterflat[~idx] = bg.back()[~idx] fits.writeto(basename, masterflat, header0, overwrite=True) print(basename)
def realtime_object_change_state(id=0, state='particle'): db = settings.DATABASES['favor2'] f = Favor2(dbname=db['NAME'], dbuser=db['USER'], dbpassword=db['PASSWORD']) f.change_object_state(id, state)
def reportTargetObserved(id=0, favor2=None, lvc=False): favor2 = favor2 or Favor2() target = favor2.query("SELECT * FROM scheduler_targets WHERE id=%s", (id, ), simplify=True) if target: # Send an email if target['type'] in ['Swift', 'Fermi', 'LVC']: subject = 'scheduler: target %d: %s / %s observed' % ( target['id'], target['name'], target['type']) body = subject body += "\n" body += "http://mmt.favor2.info/scheduler/%d" % (target['id']) if target['type'] == 'LVC': dirname = posixpath.join("TRIGGERS", "LVC", str(target['external_id'])) imagename = posixpath.join(dirname, "map.png") plot_LVC_map(target['id'], file=imagename) send_email(body, subject=subject, attachments=[imagename]) else: send_email(body, subject=subject) # Submit EM footprint to LVC GraceDB if target['type'] == 'LVC' and lvc: base = posixpath.join('TRIGGERS', 'LVC', str(target['external_id'])) files = glob.glob(posixpath.join(base, 'voevent_*.xml')) files.sort() with open(files[0]) as file: # FIXME: all these should be extracted and stored to the DB as soon as trigger arrived! root = xml.etree.cElementTree.fromstringlist(file.readlines()) grace_id = root.find( "./What/Param[@name='GraceID']").attrib['value'] # TODO: make it configurable gracedb_username = '******' gracedb_password = '******' group = 'GRA SAO RAS' raList = [] decList = [] timeList = [] exposureList = [] raWidthList = [] decWidthList = [] images = favor2.query( "SELECT * FROM images WHERE keywords->'TARGET ID' = %s::text ORDER BY time", (id, ), simplify=False) for image in images: raList.append(image['ra0']) decList.append(image['dec0']) timeList.append(image['time'].isoformat()) exposureList.append(float(image['keywords']['EXPOSURE'])) raWidthList.append(10) decWidthList.append(10) #print ra, dec, time, exposure client = GraceDbBasic(username=gracedb_username, password=gracedb_password) r = client.writeEMObservation(grace_id, group, raList, raWidthList, decList, decWidthList, timeList, exposureList) if r.status == 201: # 201 means 'Created' print 'Successfully reported the observation to GraceDB' else: print 'Error %d reporting the observation to GraceDB' % r.status
def logMessage(text='', id='news'): f = Favor2() f.log(text, id=id) return True