def diff_image(mjd, mjd2, filter_name='R'):
"""
Let's just load up a single image and difference it with the one taken before
"""
sm.setRaDecMjd(ra, dec, mjd)
frame = single_frame(mjd, filter_name=filter_name)
dist2moon = haversine(sm.azs, sm.alts, sm.moonAz, sm.moonAlt)
frame[np.where(dist2moon < np.radians(moonLimit))] = hp.UNSEEN
template_frame = single_frame(mjd2, filter_name=filter_name)
good = np.where((frame != hp.UNSEEN) & (template_frame != hp.UNSEEN)
& (sm.airmass >= 1.) & (sm.airmass <= am_limit)
& (~np.isnan(frame)) & (~np.isnan(template_frame)))
diff = np.zeros(frame.size, dtype=float) + hp.UNSEEN
diff[good] = frame[good] - template_frame[good]
return diff
def diff_image(mjd, mjd2, filter_name="R"):
"""
Let's just load up a single image and difference it with the one taken before
"""
sm.setRaDecMjd(ra, dec, mjd)
frame = single_frame(mjd, filter_name=filter_name)
dist2moon = haversine(sm.azs, sm.alts, sm.moonAz, sm.moonAlt)
frame[np.where(dist2moon < np.radians(moonLimit))] = hp.UNSEEN
template_frame = single_frame(mjd2, filter_name=filter_name)
good = np.where(
(frame != hp.UNSEEN)
& (template_frame != hp.UNSEEN)
& (sm.airmass >= 1.0)
& (sm.airmass <= am_limit)
& (~np.isnan(frame))
& (~np.isnan(template_frame))
)
diff = np.zeros(frame.size, dtype=float) + hp.UNSEEN
diff[good] = frame[good] - template_frame[good]
return diff
def fixBias(frame):
good = np.where((frame != hp.UNSEEN) & (frame > 0))
biasLevel = frame[good].min()
result = frame - biasLevel
result[np.where(frame == hp.UNSEEN)] = hp.UNSEEN
return result
# umjd = medDB(full_select='select DISTINCT(mjd) from medskybrightness;', dtypes=float)
skyMaps = np.load('sky_maps.npz')
umjd = skyMaps['umjd'].copy()
nstart = 5506 + 67 # partly cloudy frame
#nstart = 6447 # very cloudy frame
# nstart = 982 # clear, with moon
previous = single_frame(umjd[nstart - 1])
nside = hp.npix2nside(previous.size)
mjd = umjd[nstart]
site = Site('LSST')
dec, ra = hp.pix2ang(nside, np.arange(previous.size))
dec = np.pi / 2. - dec
lmst, last = calcLmstLast(mjd, site.longitude_rad)
lmst = lmst / 12. * 180.
alt, az = stupidFast_RaDec2AltAz(ra, dec, site.latitude_rad,
site.longitude_rad, mjd)
frame = single_frame(mjd, filter_name='R')
seen = np.where((frame != hp.UNSEEN) & (previous != hp.UNSEEN))
unseen = np.where((frame == hp.UNSEEN) | (previous == hp.UNSEEN))
def fixBias(frame):
good = np.where((frame != hp.UNSEEN) & (frame > 0))
biasLevel = frame[good].min()
result = frame - biasLevel
result[np.where(frame == hp.UNSEEN)] = hp.UNSEEN
return result
# umjd = medDB(full_select='select DISTINCT(mjd) from medskybrightness;', dtypes=float)
skyMaps = np.load('sky_maps.npz')
umjd = skyMaps['umjd'].copy()
nstart = 5506+67 # partly cloudy frame
#nstart = 6447 # very cloudy frame
# nstart = 982 # clear, with moon
previous = single_frame(umjd[nstart-1])
nside = hp.npix2nside(previous.size)
mjd = umjd[nstart]
site = Site('LSST')
dec, ra = hp.pix2ang(nside, np.arange(previous.size))
dec = np.pi/2. - dec
lmst, last = calcLmstLast(mjd, site.longitude_rad)
lmst = lmst/12.*180.
alt, az = stupidFast_RaDec2AltAz(ra, dec, site.latitude_rad, site.longitude_rad, mjd)
frame = single_frame(mjd, filter_name='R')
seen = np.where((frame != hp.UNSEEN) & (previous != hp.UNSEEN))
def gen_cloud_maps():
sm = sb.SkyModel(mags=True)
hpindx = np.arange(hp.nside2npix(32))
ra, dec = utils._hpid2RaDec(32, hpindx)
do_query = False
dbAddress = 'sqlite:///meddata.sqlite'
engine = sqla.create_engine(dbAddress)
connection = engine.raw_connection()
cursor = connection.cursor()
# Check all the mjd values in the db
if do_query:
query = 'select distinct(mjd) from medskybrightness;'
cursor.execute(query)
data = cursor.fetchall()
mjd_clean = [ack[0] for ack in data]
mjd_clean = np.array(mjd_clean)
# Let's go 57200 as a start
mjd_start = 57200
mjd_length = 40.
query = 'select distinct(mjd) from medskybrightness where mjd > %f and mjd < %f;' % (
mjd_start, mjd_start + mjd_length)
cursor.execute(query)
data = cursor.fetchall()
mjd_clean = np.array([ack[0] for ack in data])
cloud_maps = []
mjd_final = []
moon_limit = np.radians(30.)
am_limit = 2.5
maxI = mjd_clean[1:].size
for i, mjd in enumerate(mjd_clean[1:]):
if (mjd_clean[i] - mjd_clean[i - 1]) < (2. / 60. / 24.):
sf = single_frame(mjd_clean[i - 1])
sf2 = single_frame(mjd_clean[i])
if (sf.max() != hp.UNSEEN) & (sf2.max() != hp.UNSEEN):
diff = (sf - sf2) / sf
mask = np.where((sf == hp.UNSEEN) | (sf2 == hp.UNSEEN))
diff[mask] = hp.UNSEEN
if diff.max() != hp.UNSEEN:
cf, cloud_mask = cloudyness(diff, skyRMS_max=0.05)
sm.setRaDecMjd(ra, dec, mjd)
skymap = sm.returnMags()['r']
mask = np.where((skymap == hp.UNSEEN)
| np.isnan(skymap))[0]
cloud_mask[mask] = 2
moon_dist = utils.haversine(ra, dec, sm.moonRA, sm.moonDec)
mask = np.where((moon_dist < moon_limit)
| (sm.airmass < 1.)
| (sm.airmass > am_limit))
cloud_mask[mask] = 2
cloud_maps.append(cloud_mask)
# put in some masking near the moon
mjd_final.append(mjd)
progress = i / float(maxI) * 100
text = "\r progress = %.1f%%" % progress
sys.stdout.write(text)
sys.stdout.flush()
cloud_maps = np.array(cloud_maps)
map_key = {
'good': 0,
'masked': 2,
'bright_in_current': -1,
'bright_in_last': 1
}
np.savez('month_o_clouds.npz',
cloud_maps=cloud_maps,
mjds=mjd_final,
map_key=map_key)
biasLevel = frame[good].min()
result = frame - biasLevel
result[np.where(frame == hp.UNSEEN)] = hp.UNSEEN
return result
skyMaps = np.load('sky_maps.npz')
umjd = skyMaps['umjd'].copy()
# only do every few frames
nskip = 100
skyMedians = []
skyRMSs = []
previous = single_frame(umjd[0])
nside = hp.npix2nside(previous.size)
site = Site('LSST')
dec, ra = hp.pix2ang(nside, np.arange(previous.size))
dec = np.pi / 2. - dec
imax = 10000 # umjd.size
for i in np.arange(0, imax, nskip):
previous = single_frame(umjd[i - 1])
frame = single_frame(umjd[i])
mjd = umjd[i]
# lmst, last = calcLmstLast(mjd, site.longitude_rad)
# lmst = lmst/12.*180.
# kinda cloudy?
# kinda = np.where((cloudy_frac > .05) & (cloudy_frac < 0.1) & (sun_alts[1:] < np.radians(-12.)))[0]
# XXX--also make sure RMS is not hp.UNSEEN
fig = plt.figure(1)
clear_thesh = 0.02
cloudy_thresh = 0.05
kinda = np.where((cloudy_frac > clear_thesh) & (cloudy_frac < cloudy_thresh) & (sun_alts[1:] < np.radians(-12.)))[0]
clear = np.where((cloudy_frac < clear_thesh) & (sun_alts[1:] < np.radians(-12.)))[0]
cloudy = np.where((cloudy_frac >= cloudy_thresh) & (sun_alts[1:] < np.radians(-12.)))[0]
i = 501
f2 = single_frame(umjd[1:][kinda[i]])
f1 = single_frame(umjd[1:][kinda[i]-1])
unseen = np.where((f2 == hp.UNSEEN) | (f1 == hp.UNSEEN))
diff = (f2-f1)/f1
diff[unseen] = hp.UNSEEN
hp.mollview(diff, title='%.2f, Cloudy fraction %.2f' % (umjd[1:][kinda[i]], cloudy_frac[kinda[i]]),
min=-.1, max=.1, cmap=RdBu, fig=1, sub=(2, 2, 1))
f2 = single_frame(umjd[1:][clear[i]])
f1 = single_frame(umjd[1:][clear[i]-1])
unseen = np.where((f2 == hp.UNSEEN) | (f1 == hp.UNSEEN))
diff = (f2-f1)/f1
diff[unseen] = hp.UNSEEN
hp.mollview(diff, title='%.2f, Cloudy fraction %.2f' % (umjd[1:][clear[i]], cloudy_frac[clear[i]]),
min=-.1, max=.1, cmap=RdBu, fig=1, sub=(2, 2, 3))
biasLevel = frame[good].min()
result = frame - biasLevel
result[np.where(frame == hp.UNSEEN)] = hp.UNSEEN
return result
skyMaps = np.load('sky_maps.npz')
umjd = skyMaps['umjd'].copy()
# only do every few frames
nskip = 100
skyMedians = []
skyRMSs = []
previous = single_frame(umjd[0])
nside = hp.npix2nside(previous.size)
site = Site('LSST')
dec, ra = hp.pix2ang(nside, np.arange(previous.size))
dec = np.pi/2. - dec
imax = 10000 # umjd.size
for i in np.arange(0, imax, nskip):
previous = single_frame(umjd[i-1])
frame = single_frame(umjd[i])
mjd = umjd[i]
# lmst, last = calcLmstLast(mjd, site.longitude_rad)
# lmst = lmst/12.*180.
moon_alts = skyMaps['moonAlts'].copy()
RdBu = plt.get_cmap('RdBu')
RdBu.set_bad('gray')
RdBu.set_under('w')
nframes = umjd.size - 1
nstart = 1
# XXX
#nframes = 500
#nstart = 175100 # 192265 #1500# 700 #5506
print 'making %i frames' % nframes
previous = single_frame(umjd[nstart - 1])
nside = hp.npix2nside(previous.size)
site = Site('LSST')
dec, ra = hp.pix2ang(nside, np.arange(previous.size))
dec = np.pi / 2. - dec
median_map = np.load('sky_maps.npz')
median_filt = median_map['sky_maps']['median%s' % cannonFilter].copy()
median_map.close()
median_filt[np.isnan(median_filt)] = hp.UNSEEN
outlier_mag = 0.1
alt_limit = np.radians(24.) # Airmass of 2.485
fracs_out = []
med_diff_frame = []
rms_diff_frame = []
def gen_cloud_maps():
sm = sb.SkyModel(mags=True)
hpindx = np.arange(hp.nside2npix(32))
ra, dec = utils._hpid2RaDec(32, hpindx)
do_query = False
dbAddress = 'sqlite:///meddata.sqlite'
engine = sqla.create_engine(dbAddress)
connection = engine.raw_connection()
cursor = connection.cursor()
# Check all the mjd values in the db
if do_query:
query = 'select distinct(mjd) from medskybrightness;'
cursor.execute(query)
data = cursor.fetchall()
mjd_clean = [ack[0] for ack in data]
mjd_clean = np.array(mjd_clean)
# Let's go 57200 as a start
mjd_start = 57200
mjd_length = 40.
query = 'select distinct(mjd) from medskybrightness where mjd > %f and mjd < %f;' % (mjd_start,
mjd_start+mjd_length)
cursor.execute(query)
data = cursor.fetchall()
mjd_clean = np.array([ack[0] for ack in data])
cloud_maps = []
mjd_final = []
moon_limit = np.radians(30.)
am_limit = 2.5
maxI = mjd_clean[1:].size
for i, mjd in enumerate(mjd_clean[1:]):
if (mjd_clean[i]-mjd_clean[i-1]) < (2./60./24.):
sf = single_frame(mjd_clean[i-1])
sf2 = single_frame(mjd_clean[i])
if (sf.max() != hp.UNSEEN) & (sf2.max() != hp.UNSEEN):
diff = (sf-sf2)/sf
mask = np.where((sf == hp.UNSEEN) | (sf2 == hp.UNSEEN))
diff[mask] = hp.UNSEEN
if diff.max() != hp.UNSEEN:
cf, cloud_mask = cloudyness(diff, skyRMS_max=0.05)
sm.setRaDecMjd(ra, dec, mjd)
skymap = sm.returnMags()['r']
mask = np.where((skymap == hp.UNSEEN) | np.isnan(skymap))[0]
cloud_mask[mask] = 2
moon_dist = utils.haversine(ra, dec, sm.moonRA, sm.moonDec)
mask = np.where((moon_dist < moon_limit) | (sm.airmass < 1.) | (sm.airmass > am_limit))
cloud_mask[mask] = 2
cloud_maps.append(cloud_mask)
# put in some masking near the moon
mjd_final.append(mjd)
progress = i/float(maxI)*100
text = "\r progress = %.1f%%" % progress
sys.stdout.write(text)
sys.stdout.flush()
cloud_maps = np.array(cloud_maps)
map_key = {'good': 0, 'masked': 2, 'bright_in_current': -1, 'bright_in_last': 1}
np.savez('month_o_clouds.npz', cloud_maps=cloud_maps, mjds=mjd_final, map_key=map_key)
moon_alts = skyMaps['moonAlts'].copy()
RdBu = plt.get_cmap('RdBu')
RdBu.set_bad('gray')
RdBu.set_under('w')
nframes = umjd.size - 1
nstart = 1
# XXX
#nframes = 500
#nstart = 175100 # 192265 #1500# 700 #5506
print 'making %i frames' % nframes
previous = single_frame(umjd[nstart-1])
nside = hp.npix2nside(previous.size)
site = Site('LSST')
dec, ra = hp.pix2ang(nside, np.arange(previous.size))
dec = np.pi/2. - dec
median_map = np.load('sky_maps.npz')
median_filt = median_map['sky_maps']['median%s' % cannonFilter].copy()
median_map.close()
median_filt[np.isnan(median_filt)] = hp.UNSEEN
outlier_mag = 0.1
alt_limit = np.radians(24.) # Airmass of 2.485
fracs_out = []
med_diff_frame = []
rms_diff_frame = []
skyMaps = np.load('sky_maps.npz')
umjd = skyMaps['umjd'].copy()
good = np.where((np.floor(umjd) == night_of_interest)
& (skyMaps['sunAlts'] < np.radians(-12)))
umjd = umjd[good]
sun_alts = skyMaps['sunAlts'][good].copy()
moon_alts = skyMaps['moonAlts'][good].copy()
startnum = 300 + 20
mjd1 = umjd[startnum]
mjd2 = umjd[startnum + 1]
cm = CloudMotion()
cm.set_maps(single_frame(mjd1), single_frame(mjd2), mjd1, mjd2)
dt = mjd2 + 10. / 60. / 24.
newmask = cm.forecast(dt)
#diff = diff_hp(single_frame(umjd[startnum+1]), single_frame(umjd[startnum]))
#cloudyArea, cloudmask = cloudyness(diff)
# ah, cloudmask is in ra,dec. Need to rotate os that we're in alt-az!
#nside = hp.npix2nside(cloudmask.size)
#hpids = screen2hp(nside)
#screen = cloudmask[hpids]
#grid_x, grid_y, screen_grid = hp2screen(cloudmask, umjd[startnum])
#H, xe, ye = np.histogram2d(x,y, bins=1000, weights=z)
# XXX--also make sure RMS is not hp.UNSEEN
fig = plt.figure(1)
clear_thesh = 0.02
cloudy_thresh = 0.05
kinda = np.where((cloudy_frac > clear_thesh) & (cloudy_frac < cloudy_thresh)
& (sun_alts[1:] < np.radians(-12.)))[0]
clear = np.where((cloudy_frac < clear_thesh)
& (sun_alts[1:] < np.radians(-12.)))[0]
cloudy = np.where((cloudy_frac >= cloudy_thresh)
& (sun_alts[1:] < np.radians(-12.)))[0]
i = 501
f2 = single_frame(umjd[1:][kinda[i]])
f1 = single_frame(umjd[1:][kinda[i] - 1])
unseen = np.where((f2 == hp.UNSEEN) | (f1 == hp.UNSEEN))
diff = (f2 - f1) / f1
diff[unseen] = hp.UNSEEN
hp.mollview(diff,
title='%.2f, Cloudy fraction %.2f' %
(umjd[1:][kinda[i]], cloudy_frac[kinda[i]]),
min=-.1,
max=.1,
cmap=RdBu,
fig=1,
sub=(2, 2, 1))
f2 = single_frame(umjd[1:][clear[i]])
f1 = single_frame(umjd[1:][clear[i] - 1])
night_of_interest = 57413
skyMaps = np.load('sky_maps.npz')
umjd = skyMaps['umjd'].copy()
good = np.where((np.floor(umjd) == night_of_interest) & (skyMaps['sunAlts'] < np.radians(-12)))
umjd = umjd[good]
sun_alts = skyMaps['sunAlts'][good].copy()
moon_alts = skyMaps['moonAlts'][good].copy()
startnum = 300+20
mjd1 = umjd[startnum]
mjd2 = umjd[startnum+1]
cm = CloudMotion()
cm.set_maps(single_frame(mjd1), single_frame(mjd2), mjd1, mjd2)
dt = mjd2+10./60./24.
newmask = cm.forecast(dt)
#diff = diff_hp(single_frame(umjd[startnum+1]), single_frame(umjd[startnum]))
#cloudyArea, cloudmask = cloudyness(diff)
# ah, cloudmask is in ra,dec. Need to rotate os that we're in alt-az!
#nside = hp.npix2nside(cloudmask.size)
#hpids = screen2hp(nside)
#screen = cloudmask[hpids]
#grid_x, grid_y, screen_grid = hp2screen(cloudmask, umjd[startnum])
