def nights_moving(request, night='all', min_length=100, age=0):
db = DATABASES['favor2']
if request.method == 'GET':
if request.GET.get('min_length'):
min_length = int(request.GET.get('min_length'))
if request.GET.get('max_age'):
age = int(request.GET.get('max_age'))
if age > 24 * 3600:
age = 1800
if age > 0:
print age, min_length
f = Favor2(dbname=db['NAME'], dbuser=db['USER'], dbpassword=db['PASSWORD'])
s = dump_satellites(f,
night=night,
min_length=min_length,
max_age=age,
crlf=True,
use_zip=True)
response = HttpResponse(s, content_type='application/x-zip-compressed')
if age > 0:
# response['Content-Disposition'] = 'attachment; filename=latest.zip'
response[
'Content-Disposition'] = 'attachment; filename=latest_%d_%s.zip' % (
age, datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S'))
else:
response['Content-Disposition'] = 'attachment; filename=%s.zip' % night
return response
def __init__(self, path = '', id = 0, half_width = 10, load = False, wcs_order=2, width=2560, height=2160, rough=False, excess=4, favor2=None):
self.favor2 = favor2 or Favor2()
self.original_width = width
self.original_height = height
self.half_width = half_width
self.header = None
self.frames = []
self.channels = []
self.wcs_order = wcs_order
self.rough = rough
self.excess = excess
if id:
res = self.favor2.query('SELECT * FROM realtime_objects WHERE state=0 AND id=%s', (id,))
if res:
path = posixpath.join('ARCHIVE', 'meteor', res['night'], str(id))
print path
if path:
if load:
self.loadMeteor(path)
else:
self.analyzeMeteor(path)
else:
self.path = None
def blind_match(filename, outname=None):
# dir = tempfile.mkdtemp(prefix='match')
# wcs = None
# binname = None
f = Favor2()
if not outname:
# outname = posixpath.splitext(filename)[0] + '.new'
# if outname == filename:
# outname = filename + '.new'
fd, outname = tempfile.mkstemp(prefix='match', suffix='.new')
os.close(fd)
f.blind_match_file(filename, outfile=outname)
# for path in ['.', '/usr/local', '/opt/local']:
# if os.path.isfile(posixpath.join(path, 'astrometry', 'bin', 'solve-field')):
# binname = posixpath.join(path, 'astrometry', 'bin', 'solve-field')
# break
# if binname:
# command = "%s -D %s --overwrite --no-fits2fits --no-plots --no-verify --use-sextractor -t 2 %s --new-fits %s" % (binname, dir, filename, outname)
# #os.system("%s >/dev/null 2>/dev/null" % (command))
# os.system("%s" % (command))
# shutil.rmtree(dir, ignore_errors=True)
if not posixpath.isfile(outname):
outname = None
return outname
def realtime_object_delete(id=0):
try:
db = settings.DATABASES['favor2']
f = Favor2(dbname=db['NAME'],
dbuser=db['USER'],
dbpassword=db['PASSWORD'])
f.delete_object(id)
except:
pass
def check_tycho2(ra, dec, radius=30.0 / 3600, limit=12, favor2=None):
favor2 = favor2 or Favor2()
res = favor2.get_stars(ra,
dec,
radius,
catalog='tycho2',
extra="vt < %g" % limit)
if len(res):
return True
else:
return False
def beholder_plot_status(file, time=None, favor2=None, status=None, extra=None):
time = time or datetime.datetime.utcnow()
if status is None:
favor2 = favor2 or Favor2()
res = favor2.query("SELECT * FROM beholder_status WHERE time <= %s ORDER BY time DESC LIMIT 1", (time, ), simplify=True)
status = res['status']
time = res['time']
channels = []
mounts = []
interests = []
for id in xrange(1, int(status['nchannels'])+1):
if status['channel%d' % (id)] == '1':
channels.append({'id':id,
'ra':float(status['channel%d_ra' % (id)]),
'dec':float(status['channel%d_dec' % (id)]),
'ra1':float(status['channel%d_ra1' % (id)]),
'dec1':float(status['channel%d_dec1' % (id)]),
'ra2':float(status['channel%d_ra2' % (id)]),
'dec2':float(status['channel%d_dec2' % (id)]),
'ra3':float(status['channel%d_ra3' % (id)]),
'dec3':float(status['channel%d_dec3' % (id)]),
'ra4':float(status['channel%d_ra4' % (id)]),
'dec4':float(status['channel%d_dec4' % (id)]),
'filter':status['channel%d_hw_filter' % (id)]})
for id in xrange(1, int(status['nmounts'])+1):
if status['mount%d' % (id)] == '1':
mounts.append({'id':id,
'ra':float(status['mount%d_ra' % (id)]),
'dec':float(status['mount%d_dec' % (id)])})
if status.has_key('scheduler_ninterests'):
for id in xrange(int(status['scheduler_ninterests'])):
interests.append({'name':status['scheduler_interest%d_name' % (id)],
'ra':float(status['scheduler_interest%d_ra' % (id)]),
'dec':float(status['scheduler_interest%d_dec' % (id)]),
'radius':float(status['scheduler_interest%d_radius' % (id)]),
'color':'green', 'marker':'o', 'size':2000})
if status.has_key('scheduler_suggestion_type') and status.has_key('scheduler_suggestion_name'):
interests.append({'name':status['scheduler_suggestion_name'],
'ra':float(status['scheduler_suggestion_ra']),
'dec':float(status['scheduler_suggestion_dec']),
'color':'magenta', 'marker':'+', 'size':100, 'alpha':0.6, 'va': 'top'})
return beholder_plot_map(file, channels=channels, mounts=mounts, interests=interests, extra=extra, time=time, status=status)
def find_transients(night='all',
type='survey',
id=0,
reprocess=False,
base='.',
channel_id=0,
verbose=False):
favor2 = Favor2()
where = []
where_opts = []
if not reprocess:
where.append("keywords->'transients_extracted' != '1'")
if id:
where.append("id = %s")
where_opts.append(int(id))
if type:
where.append("type = %s")
where_opts.append(type)
if night and night != 'all':
where.append("night = %s")
where_opts.append(night)
if channel_id:
where.append("channel_id = %s")
where_opts.append(channel_id)
if where:
where_string = "WHERE " + " AND ".join(where)
res = favor2.query("SELECT * FROM images %s ORDER BY time;" %
(where_string),
where_opts,
simplify=False,
debug=True)
for r in res:
filename = r['filename']
filename = fix_remote_path(filename, channel_id=r['channel_id'])
transients = find_transients_in_frame(filename,
verbose=verbose,
favor2=favor2)
for transient in transients:
favor2.query(
"INSERT INTO survey_transients (channel_id, frame_id)")
favor2.query(
"UPDATE images SET keywords = keywords || 'transients_extracted=>1' WHERE id=%s",
(r['id'], ))
def update_satcat():
favor2 = Favor2()
res = favor2.query(
'SELECT * FROM satellites WHERE (catalogue=1 OR catalogue=3) AND (iname IS NULL OR rcs IS NULL)'
)
ids = [_['catalogue_id'] for _ in res]
print ids
res0 = favor2.query(
'SELECT * FROM satellites WHERE (catalogue=1 OR catalogue=3)')
ids0 = [_['catalogue_id'] for _ in res0]
#print ids0
lines = requests.get('https://celestrak.com/pub/satcat.txt')
print "satcat downloaded"
for line in lines.text.splitlines():
iname = line[0:11]
norad = line[13:18]
rcs = line[119:127]
iname, norad, rcs = [_.strip() for _ in iname, norad, rcs]
norad = int(norad)
rcs = float(rcs) if rcs and rcs != 'N/A' else None
if norad not in ids0:
continue
print norad, iname, rcs
favor2.query(
'UPDATE satellites SET rcs=%s WHERE (catalogue=1 OR catalogue=3) AND catalogue_id=%s AND (rcs is NULL or rcs!=%s)',
(rcs, norad, rcs))
if norad not in ids:
continue
if iname:
favor2.query(
'UPDATE satellites SET iname=%s WHERE (catalogue=1 OR catalogue=3) AND catalogue_id=%s',
(iname, norad))
res = favor2.query(
'SELECT * FROM satellites WHERE (catalogue=1 OR catalogue=3) AND (iname IS NULL OR rcs IS NULL)'
)
ids = [_['catalogue_id'] for _ in res]
print ids
def download_moving(request, id=0):
db = DATABASES['favor2']
if request.method == 'GET':
if request.GET.get('min_length'):
min_length = int(request.GET.get('min_length'))
f = Favor2(dbname=db['NAME'], dbuser=db['USER'], dbpassword=db['PASSWORD'])
s = dump_satellites(f, id=int(id), crlf=True, use_zip=True)
response = HttpResponse(s, content_type='application/x-zip-compressed')
response['Content-Disposition'] = 'attachment; filename=moving_%s.zip' % id
return response
def check_planets(ra, dec, time, favor2=None):
favor2 = favor2 or Favor2()
for o in [ephem.Moon(), ephem.Venus(), ephem.Mars(), ephem.Jupiter()]:
favor2.obs.date = time
o.compute(favor2.obs)
sr = {
'Moon': 15.0,
'Venus': 0.5,
'Jupiter': 0.5,
'Mars': 0.1
}.get(o.name)
if coords.sphdist(np.rad2deg(o.ra), np.rad2deg(o.dec), ra,
dec)[0] < sr:
return True
return False
def process_dir(dir, dbname='favor2', dbhost=None):
favor2 = Favor2(dbname=dbname, dbhost=dbhost)
favor2.conn.autocommit = False
# Night
night = posixpath.split(dir)[-1]
print(night, '/', dir)
files = glob.glob('%s/20*.fits' % dir)
files.sort()
res = favor2.query('SELECT filename FROM images WHERE night=%s', (night, ),
simplify=False)
filenames = [_['filename'] for _ in res]
j = 0
for j, filename in enumerate(files):
if filename in filenames:
continue
try:
result = process_file(filename,
night=night,
favor2=favor2,
verbose=True)
sys.stdout.write('\r %d / %d - %s' % (j, len(files), filename))
sys.stdout.flush()
except KeyboardInterrupt:
raise
except:
import traceback
print("Exception while processing", filename)
traceback.print_exc()
pass
#break
favor2.conn.commit()
if j:
print
def process_satellite_track(track):
f = Favor2()
print "Processing satellite track %d" % track['id']
# Find the satellite
try:
sat = f.query(
"SELECT * FROM satellites WHERE catalogue=%s AND catalogue_id=%s;",
(track['catalogue'], track['catalogue_id']),
simplify=True)
if not sat:
# Create new one, if not exists
sat = f.query(
"INSERT INTO satellites (catalogue, catalogue_id, name, launch_date, country, orbit_inclination, orbit_period, orbit_eccentricity) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING *;",
(track['catalogue'], track['catalogue_id'], track['name'],
track['launch_date'], track['country'],
track['orbit_inclination'], track['orbit_period'],
track['orbit_eccentricity']))
except Exception, ex:
print ex.message
return
pass
def __init__(self,
favor2=None,
nside=128,
r0=15.0,
restore=True,
update=True):
self.favor2 = favor2 or Favor2()
self.nside = nside
self.npix = hp.nside2npix(self.nside)
#print "Map resolution is %g degrees" % (np.rad2deg(hp.nside2resol(self.nside)))
self.coverage = np.zeros(self.npix)
self.restrictions = np.zeros(self.npix)
self.iweights = np.zeros(self.npix)
theta, phi = hp.pix2ang(nside, np.arange(self.npix))
self.ra = np.rad2deg(phi)
self.dec = np.rad2deg(0.5 * np.pi - theta)
self.r0 = r0
self.monitoring_exposure = 1000.0
self.latest_time = datetime.datetime.min + datetime.timedelta(days=1)
self.update_time = datetime.datetime.min + datetime.timedelta(days=1)
self.fast_update_time = datetime.datetime.min + datetime.timedelta(
days=1)
self.store_time = datetime.datetime.min + datetime.timedelta(days=1)
self.altaz_time = datetime.datetime.min + datetime.timedelta(days=1)
self.interests = {}
self.modes = {'targets', 'LVC', 'survey'}
self.update(restore=restore, update=update)
def sql_view(request, readonly=True):
if not readonly:
assert_permission(request, 'modify_data')
context = {}
if request.method == 'POST':
message = ""
query = request.POST.get('query')
r = None
if query:
r = favor2_celery.runSqlQuery.apply_async(kwargs={
'query': query,
'readonly': readonly
},
queue="beholder")
if r:
# Redirect to the task page
return redirect('tasks_task', id=r.id)
else:
# Redirect to the same view, but with no POST args.
if readonly:
return redirect('sql')
else:
return redirect('sql_unsafe')
else:
# Request the database schema
db = DATABASES['favor2']
f = Favor2(dbname=db['NAME'],
dbuser=db['USER'],
dbpassword=db['PASSWORD'])
tables = []
functions = []
res = f.query(
"SELECT * FROM information_schema.tables WHERE table_schema='public' ORDER BY table_name",
simplify=False)
for table in res:
columns = f.query(
"SELECT * FROM information_schema.columns WHERE table_name=%s ORDER BY ordinal_position",
(table['table_name'], ),
simplify=False)
tables.append({
'name': table['table_name'],
'table': table,
'columns': columns
})
context['tables'] = tables
context['functions'] = f.query(
"SELECT * FROM information_schema.routines WHERE routine_schema='public' ORDER BY routine_name",
simplify=False)
return TemplateResponse(request, 'sql.html', context=context)
help='Number of threads to use',
action='store',
dest='nthreads',
type='int',
default=0)
# parser.add_option('-t', '--type', help='Type', action='store', dest='type', default='survey')
parser.add_option('-r',
'--replace',
help='Replace',
action='store_true',
dest='replace',
default=False)
(options, args) = parser.parse_args()
favor2 = Favor2(dbname=options.db, dbhost=options.dbhost)
cfgs0 = favor2.query(
'select * from (select channel,shutter,pos0,pos1,filter,count(*) from images where (type=\'survey\' or type=\'widefield\') and filter=\'Clear\' group by channel,shutter,pos0,pos1,filter order by channel,shutter,pos0,pos1) t where count>100 order by count'
)
cfgs = []
for cfg in cfgs0:
res0 = favor2.query(
'select night,count(*) from images where channel=%s and shutter=%s and pos0=%s and pos1=%s and filter=%s and (type=\'survey\' or type=\'widefield\') group by night order by night',
(cfg['channel'], cfg['shutter'], cfg['pos0'], cfg['pos1'],
cfg['filter']))
is_first = True
night1, night2, N = None, None, 0
def process_file(filename, favor2=None, verbose=False, replace=False, dbname=None, dbhost=None, photodir='photometry'):
#### Some parameters
aper = 2.0
bkgann = None
order = 4
bg_order = 4
color_order = 2
sn = 5
if not posixpath.exists(filename):
return None
# Rough but fast checking of whether the file is already processed
if not replace and posixpath.exists(photodir + '/' + filename.split('/')[-2] + '/' + posixpath.splitext(posixpath.split(filename)[-1])[0] + '.cat'):
return
#### Preparation
header = fits.getheader(filename, -1)
if header['TYPE'] not in ['survey', 'imaging', 'widefield', 'Swift', 'Fermi', 'test']:
return
channel = header.get('CHANNEL ID')
fname = header.get('FILTER', 'unknown')
time = parse_time(header['TIME'])
shutter = header.get('SHUTTER', -1)
if fname not in ['Clear']:
return
if fname == 'Clear':
effective_fname = 'V'
else:
effective_fname = fname
night = get_night(time)
dirname = '%s/%s' % (photodir, night)
basename = posixpath.splitext(posixpath.split(filename)[-1])[0]
basename = dirname + '/' + basename
catname = basename + '.cat'
if not replace and posixpath.exists(catname):
return
if verbose:
print(filename, channel, night, fname, effective_fname)
image = fits.getdata(filename, -1).astype(np.double)
if favor2 is None:
favor2 = Favor2(dbname=options.db, dbhost=options.dbhost)
#### Basic calibration
darkname = favor2.find_image('masterdark', header=header, debug=False)
flatname = favor2.find_image('masterflat', header=header, debug=False)
if darkname:
dark = fits.getdata(darkname)
else:
dark = None
if flatname:
flat = fits.getdata(flatname)
else:
flat = None
if dark is None or flat is None:
survey.save_objects(catname, None)
return
image,header = calibrate.calibrate(image, header, dark=dark)
# Check whether the calibration failed
if 'SATURATE' not in header:
print('Calibration failed for', filename)
survey.save_objects(catname, None)
return
#### Basic masking
mask = image > 0.9*header['SATURATE']
fmask = ~np.isfinite(flat) | (flat < 0.5)
dmask = dark > 10.0*mad_std(dark) + np.nanmedian(dark)
if header.get('BLEMISHCORRECTION', 1):
# We have to mask blemished pixels
blemish = fits.getdata('calibrations/blemish_shutter_%d_channel_%d.fits' % (header['SHUTTER'], header['CHANNEL ID']))
if verbose:
print(100*np.sum(blemish>0)/blemish.shape[0]/blemish.shape[1], '% pixels blemished')
dmask |= blemish > 0
image[~fmask] *= np.median(flat[~fmask])/flat[~fmask]
#### WCS
wcs = WCS(header)
pixscale = np.hypot(wcs.pixel_scale_matrix[0,0], wcs.pixel_scale_matrix[0,1])
gain = 0.67 if header.get('SHUTTER') == 0 else 1.9
#### Background mask
mask_bg = np.zeros_like(mask)
mask_segm = np.zeros_like(mask)
# bg2 = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)
# for _ in xrange(3):
# bg1 = sep.Background(image, mask=mask|mask_bg, bw=256, bh=256)
# ibg = bg2.back() - bg1.back()
# tmp = np.abs(ibg - np.median(ibg)) > 5.0*mad_std(ibg)
# mask_bg |= survey.dilate(tmp, np.ones([50, 50]))
# mask_bg = survey.dilate(tmp, np.ones([50, 50]))
# Large objects?..
bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128)
image1 = image - bg.back()
obj0,segm = sep.extract(image1, err=bg.rms(), thresh=10, minarea=10, mask=mask|dmask|fmask|mask_bg, filter_kernel=None, clean=False, segmentation_map=True)
mask_segm = np.isin(segm, [_+1 for _,npix in enumerate(obj0['npix']) if npix > 500])
mask_segm = survey.dilate(mask_segm, np.ones([20, 20]))
if np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1] > 0.4:
print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked, skipping', filename)
survey.save_objects(catname, None)
return
elif verbose:
print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked')
# Frame footprint at +10 pixels from the edge
ra,dec = wcs.all_pix2world([10, 10, image.shape[1]-10, image.shape[1]-10], [10, image.shape[0]-10, image.shape[0]-10, 10], 0)
footprint = "(" + ",".join(["(%g,%g)" % (_,__) for _,__ in zip(ra, dec)]) + ")"
#### Catalogue
ra0,dec0,sr0 = survey.get_frame_center(header=header)
cat = favor2.get_stars(ra0, dec0, sr0, catalog='gaia', extra=['g<14', 'q3c_poly_query(ra, dec, \'%s\'::polygon)' % footprint], limit=1000000)
if verbose:
print(len(cat['ra']), 'star positions from Gaia down to g=%.1f mag' % np.max(cat['g']))
## Detection of blended and not really needed stars in the catalogue
h = htm.HTM(10)
m = h.match(cat['ra'], cat['dec'], cat['ra'], cat['dec'], 2.0*aper*pixscale, maxmatch=0)
m = [_[m[2]>1e-5] for _ in m]
blended = np.zeros_like(cat['ra'], dtype=np.bool)
notneeded = np.zeros_like(cat['ra'], dtype=np.bool)
for i1,i2,dist in zip(*m):
if dist*3600 > 0.5*aper*pixscale:
if cat['g'][i1] - cat['g'][i2] < 3:
blended[i1] = True
blended[i2] = True
else:
# i1 is fainter by more than 3 mag
notneeded[i1] = True
if dist*3600 < 0.5*aper*pixscale:
if cat['g'][i1] > cat['g'][i2]:
notneeded[i1] = True
cat,blended = [_[~notneeded] for _ in cat,blended]
#### Background subtraction
bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128)
# bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=32, bh=32)
image1 = image - bg.back()
#### Detection of all objects on the frame
obj0,segm = sep.extract(image1, err=bg.rms(), thresh=2, minarea=3, mask=mask|dmask|fmask|mask_bg|mask_segm, filter_kernel=None, clean=False, segmentation_map=True)
obj0 = obj0[(obj0['x'] > 10) & (obj0['y'] > 10) & (obj0['x'] < image.shape[1]-10) & (obj0['y'] < image.shape[0]-10)]
obj0 = obj0[obj0['flag'] <= 1] # We keep only normal and blended oblects
fields = ['ra', 'dec', 'fluxerr', 'mag', 'magerr', 'flags', 'cat']
obj0 = np.lib.recfunctions.append_fields(obj0, fields, [np.zeros_like(obj0['x'], dtype=np.int if _ in ['flags', 'cat'] else np.double) for _ in fields], usemask=False)
obj0['ra'],obj0['dec'] = wcs.all_pix2world(obj0['x'], obj0['y'], 0)
obj0['flags'] = obj0['flag']
if verbose:
print(len(obj0['x']), 'objects detected on the frame')
## Filter out objects not coincident with catalogue positions
h = htm.HTM(10)
m = h.match(obj0['ra'], obj0['dec'], cat['ra'], cat['dec'], aper*pixscale)
nidx = np.isin(np.arange(len(obj0['ra'])), m[0], invert=True)
obj0 = obj0[nidx]
if verbose:
print(len(obj0['x']), 'are outside catalogue apertures')
# Catalogue stars
xc,yc = wcs.all_world2pix(cat['ra'], cat['dec'], 0)
obj = {'x':xc, 'y':yc, 'ra':cat['ra'], 'dec':cat['dec']}
obj['flags'] = np.zeros_like(xc, dtype=np.int)
obj['flags'][blended] |= FLAG_BLENDED
obj['cat'] = np.ones_like(xc, dtype=np.int)
for _ in ['mag', 'magerr', 'flux', 'fluxerr']:
obj[_] = np.zeros_like(xc)
# Merge detected objects
for _ in ['x', 'y', 'ra', 'dec', 'flags', 'mag', 'magerr', 'flux', 'fluxerr', 'cat']:
obj[_] = np.concatenate((obj[_], obj0[_]))
if verbose:
print(len(obj['x']), 'objects for photometry')
# Simple aperture photometry
obj['flux'],obj['fluxerr'],flag = sep.sum_circle(image1, obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann)
obj['flags'] |= flag
# Normalize flags
obj['flags'][obj['flags'] & sep.APER_TRUNC] |= FLAG_TRUNCATED
obj['flags'][obj['flags'] & sep.APER_ALLMASKED] |= FLAG_MASKED
obj['flags'] &= FLAG_NORMAL | FLAG_BLENDED | FLAG_TRUNCATED | FLAG_MASKED | FLAG_NO_BACKGROUND | FLAG_BAD_CALIBRATION
area,_,_ = sep.sum_circle(np.ones_like(image1), obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann)
# Simple local background estimation
bgflux,bgfluxerr,bgflag = sep.sum_circann(image1, obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0))
bgarea,_,_ = sep.sum_circann(np.ones_like(image1), obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0))
bgidx = np.isfinite(bgarea) & np.isfinite(area)
bgidx[bgidx] &= (bgarea[bgidx] > 10) & (area[bgidx] > 1)
obj['flux'][bgidx] -= bgflux[bgidx]*area[bgidx]/bgarea[bgidx]
obj['flags'][~bgidx] |= FLAG_NO_BACKGROUND # No local background
obj['deltabgflux'] = np.zeros_like(obj['x'])
obj['deltabgflux'][bgidx] = bgflux[bgidx]*area[bgidx]/bgarea[bgidx]
fidx = np.isfinite(obj['flux']) & np.isfinite(obj['fluxerr'])
fidx[fidx] &= (obj['flux'][fidx] > 0)
obj['mag'][fidx] = -2.5*np.log10(obj['flux'][fidx])
obj['magerr'][fidx] = 2.5/np.log(10)*obj['fluxerr'][fidx]/obj['flux'][fidx]
fidx[fidx] &= (obj['magerr'][fidx] > 0)
fidx[fidx] &= 1/obj['magerr'][fidx] > sn
for _ in obj.keys():
if hasattr(obj[_], '__len__'):
obj[_] = obj[_][fidx]
obj['aper'] = aper
if verbose:
print(len(obj['x']), 'objects with S/N >', sn)
if len(obj['x']) < 1000:
print('Only', len(obj['x']), 'objects on the frame, skipping', filename)
survey.save_objects(catname, None)
return
obj['fwhm'] = 2.0*sep.flux_radius(image1, obj['x'], obj['y'], 2.0*aper*np.ones_like(obj['x']), 0.5, mask=mask|dmask|fmask|mask_bg|mask_segm)[0]
#### Check FWHM of all objects and select only 'good' ones
idx = obj['flags'] == 0
idx &= obj['magerr'] < 1/20
fwhm0 = survey.fit_2d(obj['x'][idx], obj['y'][idx], obj['fwhm'][idx], obj['x'], obj['y'], weights=1/obj['magerr'][idx])
fwhm_idx = np.abs(obj['fwhm'] - fwhm0 - np.median((obj['fwhm'] - fwhm0)[idx])) < 3.0*mad_std((obj['fwhm'] - fwhm0)[idx])
obj['flags'][~fwhm_idx] |= FLAG_BLENDED
#### Catalogue matching
idx = obj['flags']
m = htm.HTM(10).match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], 1e-5)
fidx = np.in1d(np.arange(len(cat['ra'])), m[1]) # Stars that got successfully measured and not blended
cidx = (cat['good'] == 1) & (cat['var'] == 0)
cidx &= np.isfinite(cat['B']) & np.isfinite(cat['V']) # & np.isfinite(cat['lum'])
cidx[cidx] &= ((cat['B'] - cat['V'])[cidx] > -0.5) & ((cat['B'] - cat['V'])[cidx] < 2.0)
# cidx[cidx] &= (cat['lum'][cidx] > 0.3) & (cat['lum'][cidx] < 30)
if np.sum(cidx & fidx & (cat['multi_70'] == 0)) > 2000:
cidx &= (cat['multi_70'] == 0)
obj['cat_multi'] = 70
elif np.sum(cidx & fidx & (cat['multi_45'] == 0)) > 1000:
cidx &= (cat['multi_45'] == 0)
obj['cat_multi'] = 45
else:
cidx &= (cat['multi_30'] == 0)
obj['cat_multi'] = 30
if verbose:
print(np.sum(obj['flags'] == 0), 'objects without flags')
print('Amount of good stars:',
np.sum(cidx & fidx & (cat['multi_70'] == 0)),
np.sum(cidx & fidx & (cat['multi_45'] == 0)),
np.sum(cidx & fidx & (cat['multi_30'] == 0)))
print('Using %d arcsec avoidance radius' % obj['cat_multi'])
# We match with very small SR to only account for manually placed apertures
if verbose:
print('Trying full fit:', len(obj['x']), 'objects,', np.sum(cidx), 'stars')
match = Match(width=image.shape[1], height=image.shape[0])
prev_ngoodstars = len(obj['x'])
for iter in range(10):
if not match.match(obj=obj, cat=cat[cidx], sr=1e-5, filter_name='V', order=order, bg_order=bg_order, color_order=color_order, verbose=False) or match.ngoodstars < 500:
if verbose:
print(match.ngoodstars, 'good matches, matching failed for', filename)
survey.save_objects(catname, None)
return
if verbose:
print(match.ngoodstars, 'good matches, std =', match.std)
if match.ngoodstars == prev_ngoodstars:
if verbose:
print('Converged on iteration', iter)
break
prev_ngoodstars = match.ngoodstars
# Match good objects with stars
oidx = obj['flags'] == 0
oidx1,cidx1,dist1 = htm.HTM(10).match(obj['ra'][oidx], obj['dec'][oidx], cat['ra'][cidx], cat['dec'][cidx], 1e-5)
x = obj['x'][oidx][oidx1]
y = obj['y'][oidx][oidx1]
cbv = match.color_term[oidx][oidx1]
cbv2 = match.color_term2[oidx][oidx1]
cbv3 = match.color_term3[oidx][oidx1]
bv = (cat['B'] - cat['V'])[cidx][cidx1]
cmag = cat[match.cat_filter_name][cidx][cidx1]
mag = match.mag[oidx][oidx1] + bv*cbv + bv**2*cbv2 + bv**3*cbv3
magerr = np.hypot(obj['magerr'][oidx][oidx1], 0.02)
dmag = mag-cmag
ndmag = ((mag-cmag)/magerr)
idx = cmag < match.mag_limit[oidx][oidx1]
x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag = [_[idx] for _ in [x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag]]
# Match all objects with good objects
xy = np.array([x,y]).T
xy0 = np.array([obj['x'], obj['y']]).T
kd = cKDTree(xy)
dist,m = kd.query(xy0, 101)
dist = dist[:,1:]
m = m[:,1:]
vchi2 = mad_std(ndmag[m]**2, axis=1)
# Mark regions of too sparse or too noisy matches as bad
obj['flags'][vchi2 > 5] |= FLAG_BAD_CALIBRATION
# obj['flags'][vchi2 > np.median(vchi2) + 5.0*mad_std(vchi2)] |= FLAG_BAD_CALIBRATION
obj['flags'][dist[:,10] > np.median(dist[:,10]) + 10.0*mad_std(dist[:,10])] |= FLAG_BAD_CALIBRATION
match.good_idx = (obj['flags'] & FLAG_BAD_CALIBRATION) == 0
if verbose:
print(np.sum(match.good_idx), 'of', len(match.good_idx), 'stars are good')
#### Store objects to file
try:
os.makedirs(dirname)
except:
pass
obj['mag_limit'] = match.mag_limit
obj['color_term'] = match.color_term
obj['color_term2'] = match.color_term2
obj['color_term3'] = match.color_term3
obj['filename'] = filename
obj['night'] = night
obj['channel'] = channel
obj['filter'] = fname
obj['cat_filter'] = match.cat_filter_name
obj['time'] = time
obj['mag_id'] = match.mag_id
obj['good_idx'] = match.good_idx
obj['calib_mag'] = match.mag
obj['calib_magerr'] = match.magerr
obj['std'] = match.std
obj['nstars'] = match.ngoodstars
survey.save_objects(catname, obj, header=header)
def make_flats(night='all',
imtype='skyflat',
flattype='flat',
filter='Clear',
shutter=-1):
print "Making flats for night %s" % night
favor2 = Favor2()
limit = 0.015
for channel_id in [1, 2, 3, 4, 5, 6, 7, 8, 9]:
res = favor2.query(
'SELECT * FROM images WHERE type=%s AND channel_id=%s AND filter=get_filter_id(%s) AND (night=%s OR %s=\'all\') AND ((keywords->\'SHUTTER\')::int=%s OR %s<0) ORDER BY time ASC',
(imtype, channel_id, filter, night, night, shutter, shutter))
print "%d flats for channel %d in filter %s" % (len(res), channel_id,
filter)
flats = []
filenames = []
calib = None
for r in res:
filename = fix_remote_path(r['filename'],
channel_id=r['channel_id'])
flat, header = pyfits.getdata(filename,
-1), pyfits.getheader(filename, -1)
if calib is None:
calib = Calibrator(shutter=int(r['keywords']['SHUTTER']),
channel_id=r['channel_id'])
flat, header = calib.calibrate(flat, header)
idx = flat >= header['SATURATE']
if 'skyflat' in r['type']:
darkname = favor2.find_image(r['time'],
'dark',
channel_id=r['channel_id'])
darkname = fix_remote_path(darkname,
channel_id=r['channel_id'])
dark = pyfits.getdata(darkname, -1)
dark = calib.calibrate(dark)
flat = 1.0 * flat - dark
flat /= np.median(flat)
flat[flat == 0] = 0.1
flat[idx] = 0.1
flats.append(flat)
filenames.append(filename)
mflat = np.median(flats[:20], axis=0) # Initial estimation of the flat
mflat1 = None
N = 0
flats1 = []
for i, flat in enumerate(flats):
frac = flat / mflat
print filenames[i], np.mean(frac), np.std(frac)
if np.std(frac) < limit:
flats1.append(flat)
if mflat1 is None:
mflat1 = flat
N = 1
else:
mflat1 += flat
N += 1
print "%d good flats" % N
if N:
if N < 20:
mflat1 = np.median(flats1, axis=0)
else:
mflat1 /= N
r = res[0]
time = r['time']
header = pyfits.getheader(filenames[0], -1)
header['TYPE'] = flattype
if posixpath.exists(posixpath.join('AVG', r['night'])):
filename = posixpath.join(
'AVG', r['night'],
time.strftime('%Y%m%d_%H%M%S') + '_%d.%s.fits' %
(r['channel_id'], flattype))
else:
filename = posixpath.join(
'/MMT/%d' % r['channel_id'], 'AVG', r['night'],
time.strftime('%Y%m%d_%H%M%S') + '_%d.%s.fits' %
(r['channel_id'], flattype))
print filename
pyfits.writeto(filename, mflat1, header, clobber=True)
favor2.register_image(filename, imtype=flattype, time=time)
parser.add_option('-b', '--base', help='Base path for storing data', action='store', dest='base_path', default='TRIGGERS')
parser.add_option('-H', '--host', help='Host to connect', action='store', dest='host', default='68.169.57.253')
parser.add_option('-p', '--port', help='Port to connect', action='store', dest='port', type='int', default=8099)
parser.add_option('--scheduler-host', help='Hostname of scheduler', action='store', dest='scheduler_host', default='localhost')
parser.add_option('--scheduler-port', help='Port number of scheduler', action='store', dest='scheduler_port', type='int', default=5557)
parser.add_option('-f', '--fake', help='Initiate fake trigger', action='store_true', dest='is_fake', default=False)
parser.add_option('--ra', help='RA', action='store', dest='ra', type='float', default=0.0)
parser.add_option('--dec', help='Dec', action='store', dest='dec', type='float', default=0.0)
(options,args) = parser.parse_args()
scheduler_host = options.scheduler_host
scheduler_port = options.scheduler_port
base_path = options.base_path
favor2 = Favor2()
# TODO: make it configurable
gracedb_username='******'
gracedb_password='******'
if options.is_fake:
print "Initiating fake trigger at RA=%g Dec=%g" % (options.ra, options.dec)
favor2.query("INSERT INTO scheduler_targets (ra, dec, name, type, timetolive, priority) VALUES (%s, %s, %s, %s, %s, 10) RETURNING id", (options.ra, options.dec, 'test', 'Swift', 3600), simplify=True)
# We should also inform the scheduler that new high-priority target has been added
message_to_scheduler('reschedule')
else:
print "Listening for VOEvents at %s:%d" % (options.host, options.port)
print "Serving to scheduler at %s:%d" % (scheduler_host, scheduler_port)
def plot_LVC_map(id=0, size=800, favor2=None, horizon=True, file=None):
favor2 = favor2 or Favor2()
target = favor2.query("SELECT * FROM scheduler_targets WHERE id=%s", (id, ), simplify=True)
images = favor2.query("SELECT * FROM images WHERE keywords->'TARGET ID' = %s::text ORDER BY time", (target['id'],), simplify=False)
# Trigger base dir
base = posixpath.join('TRIGGERS', target['type'], str(target['external_id']))
files = glob.glob(posixpath.join(base, '*.fits.gz'))
files.sort(reverse=True)
skymap, header = hp.read_map(files[0], h=True, verbose=False)
header = dict(header)
date = header['DATE'][:19]
date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S')
# TODO: use time from file header
date = target['time_created']
fig = Figure(facecolor='white', dpi=72, figsize=(size/72, 0.6*size/72), tight_layout=True)
ax = fig.add_subplot(111)
# set up gnomonic map projection
map = Basemap(projection='hammer', lat_0 = 0, lon_0 = 0, resolution = None, celestial = True, ax = ax)
# draw the edge of the map projection region (the projection limb)
map.drawmapboundary()
degtoralabel = lambda deg : "$%d^h$" % int(deg/15)
degtodeclabel = lambda deg : "%+d$^\circ$" % deg
map.drawparallels(np.arange(-90, 90, 30), color='lightgray')#, fmt=degtodeclabel)
map.drawmeridians(np.arange(0, 360, 45), color='lightgray')
for h in [0,3,6,9,12,15,18,21]:
try:
x,y = map(h*15,0)
if abs(x) < 1e10 and abs(y) < 1e10:
ax.text(x, y, degtoralabel(h*15), color='black')
except:
pass
# Rebin the map
nside = hp.npix2nside(len(skymap))
ra=np.arange(0, 360, 1)
dec=np.arange(-90, 90, 1)
theta, phi = np.deg2rad(90 - dec), np.deg2rad(ra)
PHI, THETA = np.meshgrid(phi, theta)
grid_pix = hp.ang2pix(nside, THETA, PHI)
grid_map = skymap[grid_pix]
ra0, dec0 = np.meshgrid(ra, dec)
m = map.pcolormesh(ra0, dec0, grid_map, latlon=True, cmap='YlOrRd')
#map.colorbar(m)
for image in images:
filename = fix_remote_path(image['filename'], channel_id=image['channel_id'])
header = pyfits.getheader(filename, -1)
wcs = pywcs.WCS(header)
ra1, dec1 = wcs.all_pix2world([0, 0, header['NAXIS1'], header['NAXIS1'], 0], [0, header['NAXIS2'], header['NAXIS2'], 0, 0], 0)
try:
map.plot(ra1, dec1, '-', latlon=True, color='blue')
except:
pass
fig.suptitle("Trigger %s at %s" % (target['name'], date))
if images:
ax.set_title("%d images from %s to %s" % (len(images), images[0]['time'], images[-1]['time']))
if horizon:
for iter in [0,1]:
if iter == 0:
if images:
favor2.obs.date = images[0]['time']
else:
favor2.obs.date = date
color, alpha = 'black', 1
else:
favor2.obs.date = datetime.datetime.utcnow()
color, alpha = 'black', 0.3
ra_h, dec_h = [], []
for A in np.arange(0, 2*np.pi, 0.1):
radec = favor2.obs.radec_of(A, np.deg2rad(0))
ra_h.append(np.rad2deg(radec[0]))
dec_h.append(np.rad2deg(radec[1]))
map.plot(ra_h, dec_h, '--', latlon=True, color=color, alpha=alpha)
favor2.obs.date = date
# Sun
s = favor2.sun
s.compute(favor2.obs)
px, py = map(np.rad2deg(s.ra), np.rad2deg(s.dec))
if abs(px) < 1e10 and abs(py) < 1e10:
map.scatter(px, py, s=2000, c='lightgray', marker='o', alpha=0.7)
ax.text(px, py, "Sun", color='black', va='center', ha='center')
# Moon
m = favor2.moon
m.compute(favor2.obs)
px, py = map(np.rad2deg(m.ra), np.rad2deg(m.dec))
if abs(px) < 1e10 and abs(py) < 1e10:
map.scatter(px, py, s=1200, c='lightgray', marker='o', alpha=0.7)
ax.text(px, py, "Moon", color='black', va='center', ha='center')
# Return the image
if file:
canvas = FigureCanvas(fig)
canvas.print_png(file, bbox_inches='tight')
def process_file(filename, night=None, favor2=None, verbose=False):
if favor2 is None:
favor2 = Favor2()
header = fits.getheader(filename, -1)
if night is None:
time = parse_time(header['TIME'])
night = get_night(time)
if verbose:
print(night, header['TYPE'])
# Skip old master calibrations
if header['TYPE'] in ['bgflat', 'superflat', 'flat']:
return None
image = fits.getdata(filename, -1).astype(np.double)
# Frame dimensions
width, height = header['NAXIS1'], header['NAXIS2']
# image,header = crop_overscans(image, header, subtract=False)
if header['TYPE'] not in ['flat', 'masterflat', 'dark', 'masterdark']:
image -= header.get('BASELINE', 100.0)
# Clean up the header a bit
header.remove('HISTORY', remove_all=True, ignore_missing=True)
header.remove('COMMENT', remove_all=True, ignore_missing=True)
header.remove('', remove_all=True, ignore_missing=True)
for _ in header.keys():
if _ and _[0] == '_':
header.remove(_, remove_all=True, ignore_missing=True)
type = header.get('TYPE', 'unknown')
filter = header.get('FILTER', 'unknown')
if filter == 'Custom' or 'Pol' in filter:
print('Broken filter in', filename)
return None
if type not in [
'dark', 'masterdark', 'flat', 'skyflat', 'superflat', 'masterflat'
]:
wcs = WCS(header) if header.get('CTYPE1') else None
if not wcs or not wcs.sip:
print('No WCS information in', filename)
return None
ra, dec = wcs.all_pix2world([0, image.shape[1], 0.5 * image.shape[1]],
[0, image.shape[0], 0.5 * image.shape[0]],
0)
ra1, dec1 = wcs.all_pix2world([image.shape[1], 0], [0, image.shape[0]],
0)
radius = 0.5 * coords.sphdist(ra[0], dec[0], ra[1], dec[1])[0]
radius1 = 0.5 * coords.sphdist(ra1[0], dec1[0], ra1[1], dec1[1])[0]
# Sanity checking with some hard-coded values
if radius < 7.25 or radius > 7.45 or np.abs(1.0 -
radius / radius1) > 0.1:
print('Broken WCS information in', filename)
return None
ra0, dec0 = ra[2], dec[2]
# Frame footprint
ra, dec = wcs.all_pix2world([0, 0, image.shape[1], image.shape[1]],
[0, image.shape[0], image.shape[0], 0], 0)
footprint = "(" + ",".join(
["(%g,%g)" % (_, __) for _, __ in zip(ra, dec)]) + ")"
# Frame footprint at +10 pixels from the edge
ra, dec = wcs.all_pix2world(
[10, 10, image.shape[1] - 10, image.shape[1] - 10],
[10, image.shape[0] - 10, image.shape[0] - 10, 10], 0)
footprint10 = "(" + ",".join(
["(%g,%g)" % (_, __) for _, __ in zip(ra, dec)]) + ")"
else:
# Should we really discard WCS for non-object frames?
ra0, dec0, radius = None, None, None
footprint, footprint10 = None, None
time = parse_time(header['TIME'])
exposure = header.get('EXPOSURE')
shutter = header.get('SHUTTER')
channel = header.get('CHANNEL ID')
pos0 = header.get('MIRROR_POS0')
pos1 = header.get('MIRROR_POS1')
mean = np.mean(image)
keywords = dict(header)
favor2.query(
'INSERT INTO images (filename,night,time,channel,type,filter,exposure,shutter,pos0,pos1,ra,dec,radius,width,height,mean,footprint,footprint10,keywords) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (filename) DO NOTHING',
(filename, night, time, channel, type, filter, exposure, shutter, pos0,
pos1, ra0, dec0, radius, width, height, mean, footprint, footprint10,
keywords))
return {
'filename': filename,
'night': night,
'time': time,
'channel': channel,
'type': type,
'filter': filter,
'shutter': shutter,
'ra0': ra0,
'dec0': dec0,
'radius': radius,
'exposure': exposure,
'width': width,
'height': height,
'mean': mean,
'keywords': keywords
}
def calibrate_image(filename,
dark='',
flat='',
simple=True,
match=True,
clean=False,
maglim=None):
print "Performing photometric calibration of file %s" % filename
res = {}
image = pyfits.getdata(filename, -1)
header = pyfits.getheader(filename, -1)
f = Favor2()
if match or not has_wcs(filename):
print "No WCS info, blindmatching"
wcs = f.blind_match_file(filename)
else:
wcs = pywcs.WCS(header=header)
res['wcs'] = wcs
if not wcs:
print "WCS match failure"
return res
ra0, dec0, sr0 = get_frame_center(header=header, wcs=wcs)
res['ra0'], res['dec0'], res['sr0'] = ra0, dec0, sr0
# Saturated pixels, to be masked after the flatfielding
#isat = image == np.max(image)
isat = image >= 30000
if dark:
idark = pyfits.getdata(dark, -1)
image = 1.0 * image - idark
if flat:
iflat = pyfits.getdata(flat, -1)
iflat = 1.0 * iflat - idark
image *= np.mean(iflat) / iflat
if clean:
image = clean_image(image)
image[isat] = 65535
# Detect objects
obj = get_objects(image=image, wcs=wcs)
print "%d objects detected" % (len(obj['flux']))
res['filter'] = header['FILTER']
res['mag_calibrated'] = False
if len(obj['flux']) > 1:
# Get the catalogue
cat = f.get_stars(ra0, dec0, sr0, catalog='pickles')
zero = get_zero_point_simple(obj,
cat,
filter=res['filter'],
maglim=maglim)
if zero:
mag0 = zero['mag0']
if mag0:
res['mag_calibrated'] = True
else:
mag0 = 0
res['mag0'] = mag0
return res
def handle(self, *args, **options):
f = Favor2()
f.query('REFRESH MATERIALIZED VIEW CONCURRENTLY satellites_view')
from sextractor import Sextractor
from favor2 import Favor2
from esutil import coords, htm
def breakpoint():
try:
from IPython.core.debugger import Tracer
Tracer()()
except:
import pdb
pdb.set_trace()
f = Favor2()
def has_wcs(filename=None, header=None):
if not header:
header = pyfits.getheader(filename, -1)
w = pywcs.WCS(header=header)
return w.wcs.has_cd()
def get_frame_center(filename=None, header=None):
if not header:
header = pyfits.getheader(filename, -1)
def refreshViews():
f = Favor2()
res = f.query('REFRESH MATERIALIZED VIEW CONCURRENTLY satellites_view')
return res
def survey_dbg_image(request, id=0, size=900, cat='pickles'):
if request.method == 'GET':
if request.GET.has_key('cat'):
cat = request.GET.get('cat')
image_obj = Images.objects.get(id=id)
filename = fix_remote_path(image_obj.filename, image_obj.channel_id)
darkname = find_image(image_obj.time, 'dark', image_obj.channel_id)
darkname = fix_remote_path(darkname, image_obj.channel_id)
flatname = find_image(image_obj.time, 'flat', image_obj.channel_id)
flatname = fix_remote_path(flatname, image_obj.channel_id)
data = pyfits.getdata(filename, -1)
header = pyfits.getheader(filename, -1)
# Prepare
if posixpath.exists(darkname):
dark = pyfits.getdata(darkname, -1)
data -= dark
if posixpath.exists(flatname):
flat = pyfits.getdata(flatname, -1)
data *= np.mean(flat) / flat
# Calibrate
obj = survey.get_objects_from_file(filename, simple=False, filter=True)
wcs = pywcs.WCS(header)
ra0, dec0, sr0 = survey.get_frame_center(header=header, wcs=wcs)
print filename, '-', ra0, dec0, sr0
favor2 = Favor2()
if cat == 'apass':
cat = favor2.get_stars(ra0,
dec0,
sr0,
catalog='apass',
limit=600000,
extra=[
"g<14 and g>8.5",
"b<20 and v<20 and r<20 and i<20 and g<20",
"var = 0", "gerr>0 and rerr>0 and ierr>0"
])
else:
cat = favor2.get_stars(ra0,
dec0,
sr0,
catalog='pickles',
limit=200000,
extra=["var = 0", "rank=1"])
print "%d objects, %d stars" % (len(obj['ra']), len(cat['ra']))
survey.fix_distortion(obj, cat, header=header)
Y, YY, corr, oidx = survey.calibrate_objects(obj,
cat,
sr=10. / 3600,
retval='all')
# Plot
dpi = 72
figsize = (size / dpi, size * 4 * 0.6 / dpi)
fig = Figure(facecolor='white', figsize=figsize, tight_layout=True)
# 1
ax = fig.add_subplot(411)
limits = np.percentile(data, [0.5, 99.5])
i = ax.imshow(data,
origin='lower',
cmap='hot',
interpolation='nearest',
vmin=limits[0],
vmax=limits[1])
fig.colorbar(i)
ax.set_title(
"%d / %s / ch=%d at %s" %
(image_obj.id, image_obj.type, image_obj.channel_id, image_obj.time))
# 2
ax = fig.add_subplot(412)
gmag0, bins, binloc = griddata(obj['x'][oidx],
obj['y'][oidx],
Y,
binsize=100)
limits = np.percentile(gmag0[np.isfinite(gmag0)], [0.5, 95.5])
i = ax.imshow(gmag0,
origin='lower',
extent=[0, header['NAXIS1'], 0, header['NAXIS2']],
interpolation='nearest',
vmin=limits[0],
vmax=limits[1])
fig.colorbar(i)
ax.autoscale(False)
ax.plot(obj['x'][oidx], obj['y'][oidx], 'b.', alpha=0.3)
ax.set_title("Actual zero point")
# 3
ax = fig.add_subplot(413)
gmag0, bins, binloc = griddata(obj['x'][oidx],
obj['y'][oidx],
YY,
binsize=100)
#limits = np.percentile(gmag0[np.isfinite(gmag0)], [0.5, 95.5])
i = ax.imshow(gmag0,
origin='lower',
extent=[0, header['NAXIS1'], 0, header['NAXIS2']],
interpolation='nearest',
vmin=limits[0],
vmax=limits[1])
fig.colorbar(i)
ax.autoscale(False)
ax.plot(obj['x'][oidx], obj['y'][oidx], 'b.', alpha=0.3)
ax.set_title("Fitted zero point")
# 4
ax = fig.add_subplot(414)
gmag0, bins, binloc = griddata(obj['x'][oidx],
obj['y'][oidx],
Y - YY,
binsize=100)
limits = np.percentile(gmag0[np.isfinite(gmag0)], [0.5, 95.5])
i = ax.imshow(gmag0,
origin='lower',
extent=[0, header['NAXIS1'], 0, header['NAXIS2']],
interpolation='nearest',
vmin=limits[0],
vmax=limits[1])
fig.colorbar(i)
ax.autoscale(False)
ax.plot(obj['x'][oidx], obj['y'][oidx], 'b.', alpha=0.3)
ax.set_title("Zero point residuals. std=%g" % np.std(Y - YY))
canvas = FigureCanvas(fig)
response = HttpResponse(content_type='image/png')
canvas.print_png(response)
return response
def find_transients_in_frame(filename,
refine=True,
favor2=None,
workdir=None,
figure=None,
show=False,
verbose=False,
plot=True):
transients = []
favor2 = favor2 or Favor2()
ra0, dec0, sr0 = get_frame_center(filename=filename)
print "Frame %s at RA=%g Dec=%g with SR=%g" % (filename, ra0, dec0, sr0)
# Dir to hold all temporary files
basedir = workdir or tempfile.mkdtemp(prefix='survey')
filtered_name = posixpath.join(basedir, "filtered.fits")
print "Cleaning the frame"
cleaned_name = posixpath.join(basedir, "clean.fits")
filter_cosmics_in_file(filename, cleaned_name)
header = pyfits.getheader(cleaned_name, -1)
time = datetime.datetime.strptime(header['TIME'], '%d.%m.%Y %H:%M:%S.%f')
print "Pre-processing the frame"
os.system("nebuliser %s noconf %s 3 3 --takeout_sky" %
(cleaned_name, filtered_name))
if refine:
print "Refining the WCS"
matched_name = posixpath.join(basedir, "image.fits")
if not favor2.blind_match_file(filtered_name, outfile=matched_name):
print "Failure refining the WCS"
return transients
else:
matched_name = filtered_name
print "Extracting the objects"
obj = ss.get_objects(filename=matched_name,
options={
'DETECT_MINAREA': 3.0,
'DETECT_THRESH': 1.5,
'BACKPHOTO_TYPE': 'GLOBAL',
'DEBLEND_NTHRESH': 64,
'DEBLEND_MINCONT': 0.001,
'FILTER': 'Y',
'BACK_TYPE': 'MANUAL',
'BACK_VALUE': 0.0,
'BACK_SIZE': 64,
'BACK_FILTERSIZE': 3
},
fwhm=2,
aper_fwhm=2)
print "Querying the catalogues"
cat = favor2.get_stars(ra0, dec0, sr0, catalog='twomass', limit=1000000)
catn = favor2.get_stars(ra0, dec0, sr0, catalog='nomad', limit=1000000)
catt = favor2.get_stars(ra0, dec0, sr0, catalog='tycho2', limit=1000000)
cattb = favor2.get_stars(ra0,
dec0,
sr0,
catalog='tycho2',
limit=1000000,
extra='vt < 7')
print "%d objects, %d 2MASS stars, %d NOMAD stars, %d bright Tycho2 stars" % (
len(obj['ra']), len(cat['ra']), len(catn['ra']), len(cattb['ra']))
print "Matching the objects"
wcs = pywcs.WCS(header)
pixscale = wcs.pixel_scale_matrix.max(
) * 3600 # estimate of pixel scale, in arcsec/pix
size = 2.0 * obj['A_WORLD'] * pixscale
size = np.maximum(size, np.repeat(40.0, len(obj['ra'])))
h = htm.HTM(10)
m = h.match(obj['ra'],
obj['dec'],
cat['ra'],
cat['dec'],
size / 3600,
maxmatch=0)
mn = h.match(obj['ra'],
obj['dec'],
catn['ra'],
catn['dec'],
size / 3600,
maxmatch=0)
mt = h.match(obj['ra'],
obj['dec'],
catt['ra'],
catt['dec'],
size / 3600,
maxmatch=0)
if len(cattb['ra']):
mtb = h.match(obj['ra'],
obj['dec'],
cattb['ra'],
cattb['dec'], (size + 100.0) / 3600,
maxmatch=0)
uidx = np.setdiff1d(np.arange(len(obj['ra'])), m[0])
print "%d objects not matched with 2MASS" % len(uidx)
uidx = np.setdiff1d(uidx, mn[0])
print "%d objects not matched with NOMAD" % len(uidx)
uidx = np.setdiff1d(uidx, mt[0])
print "%d objects not matched with Tycho2" % len(uidx)
if len(cattb['ra']):
uidx = np.setdiff1d(uidx, mtb[0])
print "%d objects not matched with brightest Tycho2 stars" % len(uidx)
if len(uidx) < 100:
uidx = filter_indices(uidx, obj, time)
print "%d objects after line filtering and GSC 2.3.2 matching" % len(
uidx)
# Get the image and project the catalogues onto it
image = pyfits.getdata(filtered_name, -1)
cat_x, cat_y = wcs.all_world2pix(cat['ra'], cat['dec'], 0)
catn_x, catn_y = wcs.all_world2pix(catn['ra'], catn['dec'], 0)
catt_x, catt_y = wcs.all_world2pix(catt['ra'], catt['dec'], 0)
if len(cattb['ra']):
cattb_x, cattb_y = wcs.all_world2pix(cattb['ra'], cattb['dec'], 0)
if len(uidx) > 100:
print "Too many unmatched objects (%d), something is wrong!" % len(
uidx)
else:
# Let's iterate over the transients
for i in uidx:
transient = {
'x': float(obj['x'][i]),
'y': float(obj['y'][i]),
'flux': float(obj['FLUX_AUTO'][i]),
'flux_err': float(obj['FLUXERR_AUTO'][i]),
'flags': int(obj['FLAGS'][i]),
'a': float(obj['A_WORLD'][i]),
'b': float(obj['B_WORLD'][i]),
'theta': float(obj['THETA_WORLD'][i]),
'ra': float(obj['ra'][i]),
'dec': float(obj['dec'][i]),
'filename': filename,
'time': time,
'channel_id': header['CHANNEL ID']
}
transient['simbad'] = None
transient['mpc'] = None
try:
res = Simbad.query_region(
SkyCoord(obj['ra'][i], obj['dec'][i], unit='deg'),
"30 seconds")
if res is not None:
transient['simbad'] = ", ".join(
[r for r in res['MAIN_ID']])
except:
pass
try:
# Minor planets
mpc = get_skybot(obj['ra'][i],
obj['dec'][i],
30.0 / 3600,
time,
objFilter=110)
# Comets
if mpc is None:
mpc = get_skybot(obj['ra'][i],
obj['dec'][i],
300.0 / 3600,
time,
objFilter=1)
if mpc is not None:
transient['mpc'] = ", ".join([
"%s %s %s (Mv=%s)" % (mpc['class'][ii], mpc['num'][ii],
mpc['name'][ii], mpc['Mv'][ii])
for ii in xrange(len(mpc['name']))
])
except:
pass
if verbose:
#print transient
print "x = %g y = %g" % (obj['x'][i], obj['y'][i])
print "RA/Dec = %g %g = %s %s" % (
obj['ra'][i], obj['dec'][i],
str(ephem.hours(obj['ra'][i] * np.pi / 180)).replace(
":",
" "), str(ephem.degrees(
obj['dec'][i] * np.pi / 180)).replace(":", " "))
time0 = time.replace(hour=0, minute=0, second=0, microsecond=0)
print "Time = %s" % time
print "Time = %s + %g" % (time0,
(time - time0).total_seconds() / 24 /
3600)
if transient['simbad']:
print "SIMBAD: " + transient['simbad']
print
if plot:
icrop, x0, y0 = crop_image(image, obj['x'][i], obj['y'][i], 40)
limits = np.percentile(icrop, [0.5, 99.5])
if not figure:
size = 400
figure = Figure(facecolor='white',
dpi=72,
figsize=(size / 72, size / 72),
tight_layout=True)
figure.clear()
ax = figure.add_axes([0, 0, 1, 1])
ax.axis('off')
im = ax.imshow(icrop,
origin="bottom",
interpolation="nearest",
cmap="hot",
vmin=limits[0],
vmax=limits[1])
#figure.colorbar(im)
ax.autoscale(False)
ax.scatter(obj['x'] - x0,
obj['y'] - y0,
marker='o',
color='green',
s=400,
facecolors='None',
lw=3)
ax.scatter(cat_x - x0,
cat_y - y0,
marker='o',
color='blue',
s=100,
facecolors='None',
lw=2)
ax.scatter(catt_x - x0,
catt_y - y0,
marker='x',
s=100,
color='blue',
facecolors='None',
lw=2)
#ellipse = Ellipse(xy=(obj['x'][i]-x0, obj['y'][i]-y0), width=2.0*obj['A_WORLD'][i], height=2.0*obj['B_WORLD'][i], angle=obj['THETA_WORLD'][i], edgecolor='b', fc='None', axes=ax)
#ax.add_patch(ellipse)
if show:
figure.show(warn=False)
else:
canvas = FigureCanvas(figure)
sio = StringIO.StringIO()
canvas.print_png(sio, bbox_inches='tight')
transient[
'preview'] = "data:image/png;base64," + base64.b64encode(
sio.getvalue())
pass
else:
transient['preview'] = None
transients.append(transient)
# cleanup
if not workdir:
shutil.rmtree(basedir)
print "Done"
return transients
def process_cfg(cfg, dbname=None, dbhost=None, replace=False):
print(
"Processing channel %d / shutter %d / pos %d %d / filter %s / %s - %s: %d frames"
% (cfg['channel'], cfg['shutter'], cfg['pos0'], cfg['pos1'],
cfg['filter'], cfg['night1'], cfg['night2'], cfg['count']))
favor2 = Favor2(dbname=dbname, dbhost=dbhost)
res = favor2.query(
'select time,night,filename from images where channel=%s and shutter=%s and pos0=%s and pos1=%s and filter=%s and (type=\'survey\' or type=\'widefield\') and night>=%s and night<=%s order by time',
(cfg['channel'], cfg['shutter'], cfg['pos0'], cfg['pos1'],
cfg['filter'], cfg['night1'], cfg['night2']))
night0 = res[0]['night']
header0 = fits.getheader(res[0]['filename'], -1)
header0['type'] = 'masterflat'
header0['NFRAMES'] = cfg['count']
header0['NIGHT1'] = cfg['night1']
header0['NIGHT2'] = cfg['night2']
darkname = favor2.find_image(res[0]['time'],
type='masterdark',
shutter=cfg['shutter'],
channel=cfg['channel'])
dark = fits.getdata(darkname, -1)
basename = 'calibrations/masterflats/superflat_channel_%d_shutter_%d_pos_%d_%d_%s_%s.fits' % (
cfg['channel'], cfg['shutter'], cfg['pos0'], cfg['pos1'],
cfg['filter'], night0)
if posixpath.exists(basename) and not replace:
print('Skipping', basename)
return
random.shuffle(res)
images = []
coadd, counts = None, 0
for i, r in enumerate(res):
filename = r['filename']
image, header = fits.getdata(filename,
-1), fits.getheader(filename, -1)
image, header = calibrate(image, header, dark=dark)
# Mask stars in the image
bg = sep.Background(image)
mask = image > bg.back() + 2.0 * bg.rms()
image[mask] = np.nan
image /= np.nanmedian(image)
images.append(image)
if len(images) == 5:
flat = np.nanmedian(images, axis=0)
flat /= np.nanmedian(flat)
idx = np.isfinite(flat)
images = []
if coadd is None:
coadd = np.zeros_like(image)
counts = np.zeros_like(image, dtype=np.int)
coadd[idx] += flat[idx]
counts[idx] += 1
print(i, '/', len(res), basename)
# Make masterflat and interpolate the gaps
idx = counts > 5
masterflat = np.ones_like(coadd) * np.nan
masterflat[idx] = coadd[idx] / counts[idx]
idx |= masterflat < 0.1
bg = sep.Background(masterflat, mask=~idx)
masterflat[~idx] = bg.back()[~idx]
fits.writeto(basename, masterflat, header0, overwrite=True)
print(basename)
def realtime_object_change_state(id=0, state='particle'):
db = settings.DATABASES['favor2']
f = Favor2(dbname=db['NAME'], dbuser=db['USER'], dbpassword=db['PASSWORD'])
f.change_object_state(id, state)
def reportTargetObserved(id=0, favor2=None, lvc=False):
favor2 = favor2 or Favor2()
target = favor2.query("SELECT * FROM scheduler_targets WHERE id=%s",
(id, ),
simplify=True)
if target:
# Send an email
if target['type'] in ['Swift', 'Fermi', 'LVC']:
subject = 'scheduler: target %d: %s / %s observed' % (
target['id'], target['name'], target['type'])
body = subject
body += "\n"
body += "http://mmt.favor2.info/scheduler/%d" % (target['id'])
if target['type'] == 'LVC':
dirname = posixpath.join("TRIGGERS", "LVC",
str(target['external_id']))
imagename = posixpath.join(dirname, "map.png")
plot_LVC_map(target['id'], file=imagename)
send_email(body, subject=subject, attachments=[imagename])
else:
send_email(body, subject=subject)
# Submit EM footprint to LVC GraceDB
if target['type'] == 'LVC' and lvc:
base = posixpath.join('TRIGGERS', 'LVC',
str(target['external_id']))
files = glob.glob(posixpath.join(base, 'voevent_*.xml'))
files.sort()
with open(files[0]) as file:
# FIXME: all these should be extracted and stored to the DB as soon as trigger arrived!
root = xml.etree.cElementTree.fromstringlist(file.readlines())
grace_id = root.find(
"./What/Param[@name='GraceID']").attrib['value']
# TODO: make it configurable
gracedb_username = '******'
gracedb_password = '******'
group = 'GRA SAO RAS'
raList = []
decList = []
timeList = []
exposureList = []
raWidthList = []
decWidthList = []
images = favor2.query(
"SELECT * FROM images WHERE keywords->'TARGET ID' = %s::text ORDER BY time",
(id, ),
simplify=False)
for image in images:
raList.append(image['ra0'])
decList.append(image['dec0'])
timeList.append(image['time'].isoformat())
exposureList.append(float(image['keywords']['EXPOSURE']))
raWidthList.append(10)
decWidthList.append(10)
#print ra, dec, time, exposure
client = GraceDbBasic(username=gracedb_username,
password=gracedb_password)
r = client.writeEMObservation(grace_id, group, raList,
raWidthList, decList,
decWidthList, timeList,
exposureList)
if r.status == 201: # 201 means 'Created'
print 'Successfully reported the observation to GraceDB'
else:
print 'Error %d reporting the observation to GraceDB' % r.status
def logMessage(text='', id='news'):
f = Favor2()
f.log(text, id=id)
return True
