def diff_image(mjd, mjd2, filter_name='R'): """ Let's just load up a single image and difference it with the one taken before """ sm.setRaDecMjd(ra, dec, mjd) frame = single_frame(mjd, filter_name=filter_name) dist2moon = haversine(sm.azs, sm.alts, sm.moonAz, sm.moonAlt) frame[np.where(dist2moon < np.radians(moonLimit))] = hp.UNSEEN template_frame = single_frame(mjd2, filter_name=filter_name) good = np.where((frame != hp.UNSEEN) & (template_frame != hp.UNSEEN) & (sm.airmass >= 1.) & (sm.airmass <= am_limit) & (~np.isnan(frame)) & (~np.isnan(template_frame))) diff = np.zeros(frame.size, dtype=float) + hp.UNSEEN diff[good] = frame[good] - template_frame[good] return diff
def diff_image(mjd, mjd2, filter_name="R"): """ Let's just load up a single image and difference it with the one taken before """ sm.setRaDecMjd(ra, dec, mjd) frame = single_frame(mjd, filter_name=filter_name) dist2moon = haversine(sm.azs, sm.alts, sm.moonAz, sm.moonAlt) frame[np.where(dist2moon < np.radians(moonLimit))] = hp.UNSEEN template_frame = single_frame(mjd2, filter_name=filter_name) good = np.where( (frame != hp.UNSEEN) & (template_frame != hp.UNSEEN) & (sm.airmass >= 1.0) & (sm.airmass <= am_limit) & (~np.isnan(frame)) & (~np.isnan(template_frame)) ) diff = np.zeros(frame.size, dtype=float) + hp.UNSEEN diff[good] = frame[good] - template_frame[good] return diff
def fixBias(frame): good = np.where((frame != hp.UNSEEN) & (frame > 0)) biasLevel = frame[good].min() result = frame - biasLevel result[np.where(frame == hp.UNSEEN)] = hp.UNSEEN return result # umjd = medDB(full_select='select DISTINCT(mjd) from medskybrightness;', dtypes=float) skyMaps = np.load('sky_maps.npz') umjd = skyMaps['umjd'].copy() nstart = 5506 + 67 # partly cloudy frame #nstart = 6447 # very cloudy frame # nstart = 982 # clear, with moon previous = single_frame(umjd[nstart - 1]) nside = hp.npix2nside(previous.size) mjd = umjd[nstart] site = Site('LSST') dec, ra = hp.pix2ang(nside, np.arange(previous.size)) dec = np.pi / 2. - dec lmst, last = calcLmstLast(mjd, site.longitude_rad) lmst = lmst / 12. * 180. alt, az = stupidFast_RaDec2AltAz(ra, dec, site.latitude_rad, site.longitude_rad, mjd) frame = single_frame(mjd, filter_name='R') seen = np.where((frame != hp.UNSEEN) & (previous != hp.UNSEEN)) unseen = np.where((frame == hp.UNSEEN) | (previous == hp.UNSEEN))
def fixBias(frame): good = np.where((frame != hp.UNSEEN) & (frame > 0)) biasLevel = frame[good].min() result = frame - biasLevel result[np.where(frame == hp.UNSEEN)] = hp.UNSEEN return result # umjd = medDB(full_select='select DISTINCT(mjd) from medskybrightness;', dtypes=float) skyMaps = np.load('sky_maps.npz') umjd = skyMaps['umjd'].copy() nstart = 5506+67 # partly cloudy frame #nstart = 6447 # very cloudy frame # nstart = 982 # clear, with moon previous = single_frame(umjd[nstart-1]) nside = hp.npix2nside(previous.size) mjd = umjd[nstart] site = Site('LSST') dec, ra = hp.pix2ang(nside, np.arange(previous.size)) dec = np.pi/2. - dec lmst, last = calcLmstLast(mjd, site.longitude_rad) lmst = lmst/12.*180. alt, az = stupidFast_RaDec2AltAz(ra, dec, site.latitude_rad, site.longitude_rad, mjd) frame = single_frame(mjd, filter_name='R') seen = np.where((frame != hp.UNSEEN) & (previous != hp.UNSEEN))
def gen_cloud_maps(): sm = sb.SkyModel(mags=True) hpindx = np.arange(hp.nside2npix(32)) ra, dec = utils._hpid2RaDec(32, hpindx) do_query = False dbAddress = 'sqlite:///meddata.sqlite' engine = sqla.create_engine(dbAddress) connection = engine.raw_connection() cursor = connection.cursor() # Check all the mjd values in the db if do_query: query = 'select distinct(mjd) from medskybrightness;' cursor.execute(query) data = cursor.fetchall() mjd_clean = [ack[0] for ack in data] mjd_clean = np.array(mjd_clean) # Let's go 57200 as a start mjd_start = 57200 mjd_length = 40. query = 'select distinct(mjd) from medskybrightness where mjd > %f and mjd < %f;' % ( mjd_start, mjd_start + mjd_length) cursor.execute(query) data = cursor.fetchall() mjd_clean = np.array([ack[0] for ack in data]) cloud_maps = [] mjd_final = [] moon_limit = np.radians(30.) am_limit = 2.5 maxI = mjd_clean[1:].size for i, mjd in enumerate(mjd_clean[1:]): if (mjd_clean[i] - mjd_clean[i - 1]) < (2. / 60. / 24.): sf = single_frame(mjd_clean[i - 1]) sf2 = single_frame(mjd_clean[i]) if (sf.max() != hp.UNSEEN) & (sf2.max() != hp.UNSEEN): diff = (sf - sf2) / sf mask = np.where((sf == hp.UNSEEN) | (sf2 == hp.UNSEEN)) diff[mask] = hp.UNSEEN if diff.max() != hp.UNSEEN: cf, cloud_mask = cloudyness(diff, skyRMS_max=0.05) sm.setRaDecMjd(ra, dec, mjd) skymap = sm.returnMags()['r'] mask = np.where((skymap == hp.UNSEEN) | np.isnan(skymap))[0] cloud_mask[mask] = 2 moon_dist = utils.haversine(ra, dec, sm.moonRA, sm.moonDec) mask = np.where((moon_dist < moon_limit) | (sm.airmass < 1.) | (sm.airmass > am_limit)) cloud_mask[mask] = 2 cloud_maps.append(cloud_mask) # put in some masking near the moon mjd_final.append(mjd) progress = i / float(maxI) * 100 text = "\r progress = %.1f%%" % progress sys.stdout.write(text) sys.stdout.flush() cloud_maps = np.array(cloud_maps) map_key = { 'good': 0, 'masked': 2, 'bright_in_current': -1, 'bright_in_last': 1 } np.savez('month_o_clouds.npz', cloud_maps=cloud_maps, mjds=mjd_final, map_key=map_key)
biasLevel = frame[good].min() result = frame - biasLevel result[np.where(frame == hp.UNSEEN)] = hp.UNSEEN return result skyMaps = np.load('sky_maps.npz') umjd = skyMaps['umjd'].copy() # only do every few frames nskip = 100 skyMedians = [] skyRMSs = [] previous = single_frame(umjd[0]) nside = hp.npix2nside(previous.size) site = Site('LSST') dec, ra = hp.pix2ang(nside, np.arange(previous.size)) dec = np.pi / 2. - dec imax = 10000 # umjd.size for i in np.arange(0, imax, nskip): previous = single_frame(umjd[i - 1]) frame = single_frame(umjd[i]) mjd = umjd[i] # lmst, last = calcLmstLast(mjd, site.longitude_rad) # lmst = lmst/12.*180.
# kinda cloudy? # kinda = np.where((cloudy_frac > .05) & (cloudy_frac < 0.1) & (sun_alts[1:] < np.radians(-12.)))[0] # XXX--also make sure RMS is not hp.UNSEEN fig = plt.figure(1) clear_thesh = 0.02 cloudy_thresh = 0.05 kinda = np.where((cloudy_frac > clear_thesh) & (cloudy_frac < cloudy_thresh) & (sun_alts[1:] < np.radians(-12.)))[0] clear = np.where((cloudy_frac < clear_thesh) & (sun_alts[1:] < np.radians(-12.)))[0] cloudy = np.where((cloudy_frac >= cloudy_thresh) & (sun_alts[1:] < np.radians(-12.)))[0] i = 501 f2 = single_frame(umjd[1:][kinda[i]]) f1 = single_frame(umjd[1:][kinda[i]-1]) unseen = np.where((f2 == hp.UNSEEN) | (f1 == hp.UNSEEN)) diff = (f2-f1)/f1 diff[unseen] = hp.UNSEEN hp.mollview(diff, title='%.2f, Cloudy fraction %.2f' % (umjd[1:][kinda[i]], cloudy_frac[kinda[i]]), min=-.1, max=.1, cmap=RdBu, fig=1, sub=(2, 2, 1)) f2 = single_frame(umjd[1:][clear[i]]) f1 = single_frame(umjd[1:][clear[i]-1]) unseen = np.where((f2 == hp.UNSEEN) | (f1 == hp.UNSEEN)) diff = (f2-f1)/f1 diff[unseen] = hp.UNSEEN hp.mollview(diff, title='%.2f, Cloudy fraction %.2f' % (umjd[1:][clear[i]], cloudy_frac[clear[i]]), min=-.1, max=.1, cmap=RdBu, fig=1, sub=(2, 2, 3))
biasLevel = frame[good].min() result = frame - biasLevel result[np.where(frame == hp.UNSEEN)] = hp.UNSEEN return result skyMaps = np.load('sky_maps.npz') umjd = skyMaps['umjd'].copy() # only do every few frames nskip = 100 skyMedians = [] skyRMSs = [] previous = single_frame(umjd[0]) nside = hp.npix2nside(previous.size) site = Site('LSST') dec, ra = hp.pix2ang(nside, np.arange(previous.size)) dec = np.pi/2. - dec imax = 10000 # umjd.size for i in np.arange(0, imax, nskip): previous = single_frame(umjd[i-1]) frame = single_frame(umjd[i]) mjd = umjd[i] # lmst, last = calcLmstLast(mjd, site.longitude_rad) # lmst = lmst/12.*180.
moon_alts = skyMaps['moonAlts'].copy() RdBu = plt.get_cmap('RdBu') RdBu.set_bad('gray') RdBu.set_under('w') nframes = umjd.size - 1 nstart = 1 # XXX #nframes = 500 #nstart = 175100 # 192265 #1500# 700 #5506 print 'making %i frames' % nframes previous = single_frame(umjd[nstart - 1]) nside = hp.npix2nside(previous.size) site = Site('LSST') dec, ra = hp.pix2ang(nside, np.arange(previous.size)) dec = np.pi / 2. - dec median_map = np.load('sky_maps.npz') median_filt = median_map['sky_maps']['median%s' % cannonFilter].copy() median_map.close() median_filt[np.isnan(median_filt)] = hp.UNSEEN outlier_mag = 0.1 alt_limit = np.radians(24.) # Airmass of 2.485 fracs_out = [] med_diff_frame = [] rms_diff_frame = []
def gen_cloud_maps(): sm = sb.SkyModel(mags=True) hpindx = np.arange(hp.nside2npix(32)) ra, dec = utils._hpid2RaDec(32, hpindx) do_query = False dbAddress = 'sqlite:///meddata.sqlite' engine = sqla.create_engine(dbAddress) connection = engine.raw_connection() cursor = connection.cursor() # Check all the mjd values in the db if do_query: query = 'select distinct(mjd) from medskybrightness;' cursor.execute(query) data = cursor.fetchall() mjd_clean = [ack[0] for ack in data] mjd_clean = np.array(mjd_clean) # Let's go 57200 as a start mjd_start = 57200 mjd_length = 40. query = 'select distinct(mjd) from medskybrightness where mjd > %f and mjd < %f;' % (mjd_start, mjd_start+mjd_length) cursor.execute(query) data = cursor.fetchall() mjd_clean = np.array([ack[0] for ack in data]) cloud_maps = [] mjd_final = [] moon_limit = np.radians(30.) am_limit = 2.5 maxI = mjd_clean[1:].size for i, mjd in enumerate(mjd_clean[1:]): if (mjd_clean[i]-mjd_clean[i-1]) < (2./60./24.): sf = single_frame(mjd_clean[i-1]) sf2 = single_frame(mjd_clean[i]) if (sf.max() != hp.UNSEEN) & (sf2.max() != hp.UNSEEN): diff = (sf-sf2)/sf mask = np.where((sf == hp.UNSEEN) | (sf2 == hp.UNSEEN)) diff[mask] = hp.UNSEEN if diff.max() != hp.UNSEEN: cf, cloud_mask = cloudyness(diff, skyRMS_max=0.05) sm.setRaDecMjd(ra, dec, mjd) skymap = sm.returnMags()['r'] mask = np.where((skymap == hp.UNSEEN) | np.isnan(skymap))[0] cloud_mask[mask] = 2 moon_dist = utils.haversine(ra, dec, sm.moonRA, sm.moonDec) mask = np.where((moon_dist < moon_limit) | (sm.airmass < 1.) | (sm.airmass > am_limit)) cloud_mask[mask] = 2 cloud_maps.append(cloud_mask) # put in some masking near the moon mjd_final.append(mjd) progress = i/float(maxI)*100 text = "\r progress = %.1f%%" % progress sys.stdout.write(text) sys.stdout.flush() cloud_maps = np.array(cloud_maps) map_key = {'good': 0, 'masked': 2, 'bright_in_current': -1, 'bright_in_last': 1} np.savez('month_o_clouds.npz', cloud_maps=cloud_maps, mjds=mjd_final, map_key=map_key)
moon_alts = skyMaps['moonAlts'].copy() RdBu = plt.get_cmap('RdBu') RdBu.set_bad('gray') RdBu.set_under('w') nframes = umjd.size - 1 nstart = 1 # XXX #nframes = 500 #nstart = 175100 # 192265 #1500# 700 #5506 print 'making %i frames' % nframes previous = single_frame(umjd[nstart-1]) nside = hp.npix2nside(previous.size) site = Site('LSST') dec, ra = hp.pix2ang(nside, np.arange(previous.size)) dec = np.pi/2. - dec median_map = np.load('sky_maps.npz') median_filt = median_map['sky_maps']['median%s' % cannonFilter].copy() median_map.close() median_filt[np.isnan(median_filt)] = hp.UNSEEN outlier_mag = 0.1 alt_limit = np.radians(24.) # Airmass of 2.485 fracs_out = [] med_diff_frame = [] rms_diff_frame = []
skyMaps = np.load('sky_maps.npz') umjd = skyMaps['umjd'].copy() good = np.where((np.floor(umjd) == night_of_interest) & (skyMaps['sunAlts'] < np.radians(-12))) umjd = umjd[good] sun_alts = skyMaps['sunAlts'][good].copy() moon_alts = skyMaps['moonAlts'][good].copy() startnum = 300 + 20 mjd1 = umjd[startnum] mjd2 = umjd[startnum + 1] cm = CloudMotion() cm.set_maps(single_frame(mjd1), single_frame(mjd2), mjd1, mjd2) dt = mjd2 + 10. / 60. / 24. newmask = cm.forecast(dt) #diff = diff_hp(single_frame(umjd[startnum+1]), single_frame(umjd[startnum])) #cloudyArea, cloudmask = cloudyness(diff) # ah, cloudmask is in ra,dec. Need to rotate os that we're in alt-az! #nside = hp.npix2nside(cloudmask.size) #hpids = screen2hp(nside) #screen = cloudmask[hpids] #grid_x, grid_y, screen_grid = hp2screen(cloudmask, umjd[startnum]) #H, xe, ye = np.histogram2d(x,y, bins=1000, weights=z)
# XXX--also make sure RMS is not hp.UNSEEN fig = plt.figure(1) clear_thesh = 0.02 cloudy_thresh = 0.05 kinda = np.where((cloudy_frac > clear_thesh) & (cloudy_frac < cloudy_thresh) & (sun_alts[1:] < np.radians(-12.)))[0] clear = np.where((cloudy_frac < clear_thesh) & (sun_alts[1:] < np.radians(-12.)))[0] cloudy = np.where((cloudy_frac >= cloudy_thresh) & (sun_alts[1:] < np.radians(-12.)))[0] i = 501 f2 = single_frame(umjd[1:][kinda[i]]) f1 = single_frame(umjd[1:][kinda[i] - 1]) unseen = np.where((f2 == hp.UNSEEN) | (f1 == hp.UNSEEN)) diff = (f2 - f1) / f1 diff[unseen] = hp.UNSEEN hp.mollview(diff, title='%.2f, Cloudy fraction %.2f' % (umjd[1:][kinda[i]], cloudy_frac[kinda[i]]), min=-.1, max=.1, cmap=RdBu, fig=1, sub=(2, 2, 1)) f2 = single_frame(umjd[1:][clear[i]]) f1 = single_frame(umjd[1:][clear[i] - 1])
night_of_interest = 57413 skyMaps = np.load('sky_maps.npz') umjd = skyMaps['umjd'].copy() good = np.where((np.floor(umjd) == night_of_interest) & (skyMaps['sunAlts'] < np.radians(-12))) umjd = umjd[good] sun_alts = skyMaps['sunAlts'][good].copy() moon_alts = skyMaps['moonAlts'][good].copy() startnum = 300+20 mjd1 = umjd[startnum] mjd2 = umjd[startnum+1] cm = CloudMotion() cm.set_maps(single_frame(mjd1), single_frame(mjd2), mjd1, mjd2) dt = mjd2+10./60./24. newmask = cm.forecast(dt) #diff = diff_hp(single_frame(umjd[startnum+1]), single_frame(umjd[startnum])) #cloudyArea, cloudmask = cloudyness(diff) # ah, cloudmask is in ra,dec. Need to rotate os that we're in alt-az! #nside = hp.npix2nside(cloudmask.size) #hpids = screen2hp(nside) #screen = cloudmask[hpids] #grid_x, grid_y, screen_grid = hp2screen(cloudmask, umjd[startnum])