def add_observation(self, observation, indx=None):
if observation['filter'][0] == self.filtername:
m5 = m5_flat_sed(observation['filter'][0], observation['skybrightness'][0],
observation['FWHMeff'][0], observation['exptime'][0],
observation['airmass'][0])
self.feature[indx] = 1.25 * np.log10(10.**(0.8*self.feature[indx]) + 10.**(0.8*m5))
def generate_percentiles(nbins=20):
"""
Make histograms of the 5-sigma limiting depths for each point and each filter.
"""
filters = ['u', 'g', 'r', 'i', 'z', 'y']
restore_file = 'healpix/59560_59772.npz'
disk_data = np.load(restore_file)
required_mjds = disk_data['header'][()]['required_mjds'].copy()
dict_of_lists = disk_data['dict_of_lists'][()].copy()
disk_data.close()
sb_file = 'healpix/59560_59772.npy'
sky_brightness = np.load(sb_file)
npix = sky_brightness['r'].shape[-1]
histograms = np.zeros((nbins, npix), dtype=list(zip(filters, [float] * 6)))
histogram_npts = np.zeros(npix, dtype=list(zip(filters, [int] * 6)))
# find the indices of all the evenly spaced mjd values
even_mjd_indx = np.in1d(dict_of_lists['mjds'], required_mjds)
# make an array to hold only the sky brightness values for evenly spaced mjds
sky_brightness_reduced = np.ones((even_mjd_indx.sum(), npix),
dtype=sky_brightness.dtype)
for key in dict_of_lists:
dict_of_lists[key] = dict_of_lists[key][even_mjd_indx]
for key in filters:
sky_brightness_reduced[key] = sky_brightness[key][even_mjd_indx, :]
del sky_brightness
for filtername in filters:
# convert surface brightness to m5
FWHMeff = LSSTdefaults().FWHMeff(filtername)
# increase as a function of airmass
airmass_correction = np.power(dict_of_lists['airmass'], 0.6)
FWHMeff *= airmass_correction
m5_arr = m5_flat_sed(filtername, sky_brightness_reduced[filtername],
FWHMeff, 30., dict_of_lists['airmass'])
for indx in np.arange(npix):
m5s = m5_arr[:, indx]
m5s = m5s[np.isfinite(m5s)]
m5s = np.sort(m5s)
percentile_points = np.round(np.linspace(0, m5s.size - 1, nbins))
if m5s.size > percentile_points.size:
histograms[filtername][:, indx] = m5s[percentile_points.astype(
int)]
histogram_npts[filtername][indx] = m5s.size
# make the histogram for this point in the sky
# histograms[filtername][:, indx] += np.histogram(m5s[np.isfinite(m5s)],
# bins=bins[filtername])[0]
np.savez('percentile_m5_maps.npz',
histograms=histograms,
histogram_npts=histogram_npts)
def observation_add_data(self, observation):
"""
Fill in the metadata for a completed observation
"""
current_time = Time(self.mjd, format='mjd')
observation['clouds'] = self.cloud_data(current_time)
observation['airmass'] = 1. / np.cos(np.pi / 2. - observation['alt'])
# Seeing
fwhm_500 = self.seeing_data(current_time)
seeing_dict = self.seeing_model(fwhm_500, observation['airmass'])
observation['FWHMeff'] = seeing_dict['fwhmEff'][self.seeing_indx_dict[
observation['filter'][0]]]
observation['FWHM_geometric'] = seeing_dict['fwhmGeom'][
self.seeing_indx_dict[observation['filter'][0]]]
observation['FWHM_500'] = fwhm_500
observation['night'] = self.night
observation['mjd'] = self.mjd
hpid = _raDec2Hpid(self.sky_model.nside, observation['RA'],
observation['dec'])
observation['skybrightness'] = self.sky_model.returnMags(
self.mjd, indx=[hpid], extrapolate=True)[observation['filter'][0]]
observation['fivesigmadepth'] = m5_flat_sed(
observation['filter'][0],
observation['skybrightness'],
observation['FWHMeff'],
observation['exptime'] / observation['nexp'],
observation['airmass'],
nexp=observation['nexp'])
lmst, last = calcLmstLast(self.mjd, self.site.longitude_rad)
observation['lmst'] = lmst
sun_moon_info = self.almanac.get_sun_moon_positions(self.mjd)
observation['sunAlt'] = sun_moon_info['sun_alt']
observation['sunAz'] = sun_moon_info['sun_az']
observation['sunRA'] = sun_moon_info['sun_RA']
observation['sunDec'] = sun_moon_info['sun_dec']
observation['moonAlt'] = sun_moon_info['moon_alt']
observation['moonAz'] = sun_moon_info['moon_az']
observation['moonRA'] = sun_moon_info['moon_RA']
observation['moonDec'] = sun_moon_info['moon_dec']
observation['moonDist'] = _angularSeparation(observation['RA'],
observation['dec'],
observation['moonRA'],
observation['moonDec'])
observation['solarElong'] = _angularSeparation(observation['RA'],
observation['dec'],
observation['sunRA'],
observation['sunDec'])
observation['moonPhase'] = sun_moon_info['moon_phase']
observation['ID'] = self.obsID_counter
self.obsID_counter += 1
return observation
def add_m5(df):
m5 = np.zeros(len(df))
for f in df['filter'].unique():
match = np.where(df['filter'] == f)
m5[match] = m5_flat_sed(f, df.sims_skybright.as_matrix()[match], df.FWHMeff.as_matrix()[match],
df.visitExpTime.as_matrix()[match], df.airmass.as_matrix()[match])
df['sims_m5'] = m5
return df
def calc_M5Depth(self):
self._M5Depth = {}
for filtername in self._skybrightness:
good = ~np.isnan(self._skybrightness[filtername])
self._M5Depth[filtername] = self.nan_map.copy()
self._M5Depth[filtername][good] = m5_flat_sed(filtername,
self._skybrightness[filtername][good],
self._FWHMeff[filtername][good],
self.exptime,
self._airmass[good])
def _run(self, simData):
filts = np.unique(simData[self.filterCol])
for filtername in filts:
infilt = np.where(simData[self.filterCol] == filtername)
simData['m5_simsUtils'][infilt] = m5_flat_sed(
filtername, simData[infilt][self.skybrightnessCol],
simData[infilt][self.seeingCol],
simData[infilt][self.exptimeCol],
simData[infilt][self.airmassCol])
return simData
def add_m5(df):
m5 = np.zeros(len(df))
for f in df['filter'].unique():
match = np.where(df['filter'] == f)
m5[match] = m5_flat_sed(f,
df.sims_skybright.as_matrix()[match],
df.FWHMeff.as_matrix()[match],
df.visitExpTime.as_matrix()[match],
df.airmass.as_matrix()[match])
df['sims_m5'] = m5
return df
def convert_cols(data):
cols_need = ['obsHistID', 'sessionID', 'propID', 'fieldID', 'fieldRA', 'fieldDec', 'filter',
'expDate', 'expMJD', 'night', 'visitTime', 'visitExpTime', 'finRank', 'finSeeing',
'transparency', 'airmass', 'vSkyBright', 'filtSkyBrightness', 'rotSkyPos', 'rotTelPos',
'lst', 'altitude', 'azimuth', 'dist2Moon', 'solarElong',
'moonRA', 'moonDec', 'moonAlt', 'moonAZ', 'moonPhase', 'sunAlt', 'sunAz', 'phaseAngle',
'rScatter', 'mieScatter', 'moonIllum', 'moonBright', 'darkBright', 'rawSeeing', 'wind',
'humidity', 'slewDist', 'slewTime', 'fiveSigmaDepth', 'ditheredRA', 'ditheredDec']
cols_have = []
for k in data:
cols_have.append(k)
cols_map = {'ditheredRA':'hexdithra', 'ditheredDec':'hexdithdec', 'visitTime':'expTime',
'vSkyBright':'VskyBright', 'filtSkyBrightness':'filtSky', 'finSeeing':'seeing',
'transparency':'xparency' }
cols_map_backup = {'ditheredRA':-99, 'ditheredDec':-99}
cols_default = {'visitExpTime':30.0, 'solarElong':-99, 'wind':-99, 'humidity':-99, 'moonAZ':-99}
cols_calc = {'fiveSigmaDepth':m5_flat_sed, 'night':calc_night}
print "Needed but not present or translated."
for c in cols_need:
if c not in cols_have and c not in cols_default and c not in cols_calc:
if c in cols_map:
if cols_map[c] not in cols_have:
if c not in cols_map_backup:
print c
else:
print c
print "Present, but not in needed list. "
for k in cols_have:
if k not in cols_need:
print k
nvisits = len(data)
data2 = {}
for c in cols_need:
if c in cols_have:
data2[c] = data[c]
elif c in cols_map:
if cols_map[c] not in data and c in cols_map_backup:
data2[c] = np.ones(nvisits,float) * cols_map_backup[c]
else:
data2[c] = data[cols_map[c]]
elif c in cols_default:
data2[c] = np.ones(nvisits, float) * cols_default[c]
elif c in cols_calc:
if c == 'fiveSigmaDepth':
data2[c] = np.zeros(nvisits, float)
for index, v in data.iterrows():
data2[c][index] = m5_flat_sed(v['filter'], v['filtSky'], v['seeing'],
v['expTime']-4.0, v['airmass'])
elif c == 'night':
data2[c] = calc_night(data['expMJD'], int(data['expMJD'].min()))
data2 = pd.DataFrame(data2)
return data2
def testm5(self):
filters = ['u', 'g', 'r', 'i', 'z', 'y']
kwargs = {}
# List all parameters to test, with better conditions first
kwargs['musky'] = [23., 22.]
kwargs['FWHMeff'] = [1., 1.5]
kwargs['expTime'] = [60., 30.]
kwargs['airmass'] = [1., 2.2]
kwargs['tauCloud'] = [0., 2.2]
k_default = {}
for key in kwargs:
k_default[key] = kwargs[key][0]
for filtername in filters:
m5_baseline = m5_flat_sed(filtername, **k_default)
for key in kwargs:
k_new = k_default.copy()
k_new[key] = kwargs[key][1]
m5_new = m5_flat_sed(filtername, **k_new)
assert(m5_new < m5_baseline)
def _run(self, simData, cols_present=False):
if cols_present:
# Column already present in data; assume it needs updating and recalculate.
return simData
filts = np.unique(simData[self.filterCol])
for filtername in filts:
infilt = np.where(simData[self.filterCol] == filtername)
simData['m5_simsUtils'][infilt] = m5_flat_sed(filtername,
simData[infilt][self.skybrightnessCol],
simData[infilt][self.seeingCol],
simData[infilt][self.exptimeCol],
simData[infilt][self.airmassCol])
return simData
def calc_target_m5s(alt=65., fiducial_seeing=0.9, exptime=20.):
"""
Use the skybrightness model to find some good target m5s
"""
import lsst.sims.skybrightness as sb
sm = sb.SkyModel(moon=False, twilight=False, mags=True)
sm.setRaDecMjd(np.array([0.]), np.array([alt]), 49353.177645, degrees=True, azAlt=True)
sky_mags = sm.returnMags()
airmass = 1./np.cos(np.pi/2.-np.radians(alt))
goal_m5 = {}
for filtername in sky_mags:
goal_m5[filtername] = m5_flat_sed(filtername, sky_mags[filtername], fiducial_seeing,
exptime, airmass)
return goal_m5
def update_conditions(self, conditions):
"""
Parameters
----------
conditions : dict
Keys should include airmass, sky_brightness, seeing.
"""
m5 = np.empty(conditions['skybrightness'][self.filtername].size)
m5.fill(hp.UNSEEN)
m5_mask = np.zeros(m5.size, dtype=bool)
m5_mask[np.where(conditions['skybrightness'][self.filtername] == hp.UNSEEN)] = True
good = np.where(conditions['skybrightness'][self.filtername] != hp.UNSEEN)
m5[good] = m5_flat_sed(self.filtername, conditions['skybrightness'][self.filtername][good],
conditions['FWHMeff_%s' % self.filtername][good],
self.expTime, conditions['airmass'][good])
self.feature = self.m5p.m5map2percentile(m5, filtername=self.filtername)
self.feature[m5_mask] = hp.UNSEEN
self.feature = hp.ud_grade(self.feature, nside_out=self.nside)
self.feature = ma.masked_values(self.feature, hp.UNSEEN)
def generate_ddf(ddf_name, nyears=10, space=2):
previous_ddf = generate_dd_surveys()
survey_names = np.array([survey.survey_name for survey in previous_ddf])
survey_indx = np.where(survey_names == ddf_name)[0].max()
ddf_ra = previous_ddf[survey_indx].ra * u.rad
ddf_dec = previous_ddf[survey_indx].dec * u.rad
site = Site('LSST')
location = EarthLocation(lat=site.latitude,
lon=site.longitude,
height=site.height)
mjd = np.arange(59853.5, 59853.5 + 365.25 * nyears, 20. / 60 / 24.)
times = Time(mjd, format='mjd', location=location)
airmass_limit = 2.5 # demand airmass lower than this
twilight_limit = -18. # Sun below this altitude in degrees
dist_to_moon_limit = 30. # minimum distance to keep from moon degrees
zenith_limit = 10. # Need to be this far away from zenith to start (20 min = 5 deg)
g_m5_limit = 23.5 # mags
season_gap = 20. # days. Count any gap longer than this as it's own season
season_length_limit = 80 # Days. Demand at least this many days in a season
# How long to keep attempting a DDF
expire_dict = {1: 36. / 24., 2: 0.5}
sun_coords = get_sun(times)
moon_coords = get_moon(times)
sched_downtime_data = ScheduledDowntimeData(Time(mjd[0], format='mjd'))
observatory_up = np.ones(mjd.size, dtype=bool)
for dt in sched_downtime_data():
indx = np.where((mjd >= dt['start'].mjd) & (mjd <= dt['end'].mjd))[0]
observatory_up[indx] = False
lst = times.sidereal_time('mean')
sun_altaz = sun_coords.transform_to(AltAz(location=location))
# generate a night label for each timestep
sun_rise = np.where((sun_altaz.alt[0:-1] < 0) & (sun_altaz.alt[1:] > 0))[0]
night = np.zeros(mjd.size, dtype=int)
night[sun_rise] = 1
night = np.cumsum(night) + 1 # 1-index for night
sun_down = np.where(sun_altaz.alt < twilight_limit * u.deg)[0]
ddf_coord = SkyCoord(ra=ddf_ra, dec=ddf_dec)
ddf_altaz = ddf_coord.transform_to(AltAz(location=location, obstime=times))
ddf_airmass = 1. / np.cos(np.radians(90. - ddf_altaz.az.deg))
zenith = AltAz(alt=90. * u.deg, az=0. * u.deg)
ddf_zenth_dist = zenith.separation(ddf_altaz)
nside = 32
ddf_indx = raDec2Hpid(nside, ddf_coord.ra.deg, ddf_coord.dec.deg)
sm = SkyModelPre()
g_sb = mjd * 0 + np.nan
indices = np.where((sun_altaz.alt < twilight_limit * u.deg)
& (ddf_airmass > airmass_limit))[0]
# In theory, one could reach into the sky brightness model and do a much faster interpolation
# There might be an airmass limit on the sky brightness.
for indx in sun_down:
g_sb[indx] = sm.returnMags(mjd[indx],
indx=[ddf_indx],
filters='g',
badval=np.nan)['g']
dist_to_moon = ddf_coord.separation(moon_coords)
seeing_model = SeeingModel()
ddf_approx_fwhmEff = seeing_model(0.7, ddf_airmass)
# I think this should pluck out the g-filter. Really should be labled
ddf_approx_fwhmEff = ddf_approx_fwhmEff['fwhmEff'][1].ravel()
ddf_m5 = m5_flat_sed('g',
g_sb,
ddf_approx_fwhmEff,
30.,
ddf_airmass,
nexp=1.)
# demand sun down past twilight, ddf is up, and observatory is open, and not too close to the moon
good = np.where((ddf_airmass < airmass_limit)
& (sun_altaz.alt < twilight_limit * u.deg)
& (ddf_airmass > 0) & (observatory_up == True)
& (dist_to_moon > dist_to_moon_limit * u.deg)
& (ddf_zenth_dist > zenith_limit * u.deg)
& (ddf_m5 > g_m5_limit))
potential_nights = np.unique(night[good])
night_gap = potential_nights[1:] - potential_nights[0:-1]
big_gap = np.where(night_gap > season_gap)[0] + 1
season = potential_nights * 0
season[big_gap] = 1
season = np.cumsum(season)
u_seasons = np.unique(season)
season_lengths = []
for se in u_seasons:
in_se = np.where(season == se)
season_lengths.append(
np.max(potential_nights[in_se]) - np.min(potential_nights[in_se]))
season_lengths = np.array(season_lengths)
good_seasons = u_seasons[np.where(season_lengths > season_length_limit)[0]]
gn = np.isin(season, good_seasons)
potential_nights = potential_nights[gn]
season = season[gn]
obs_attempts = []
for sea in np.unique(season):
night_indx = np.where(season == sea)
obs_attempts.append(
place_obs(potential_nights[night_indx], space=space))
obs_attempts = np.concatenate(obs_attempts)
mjd_observe = []
m5_approx = []
for indx in np.where(obs_attempts > 0)[0]:
in_night_indx = np.where(night == potential_nights[indx])[0]
best_depth_indx = np.min(
np.where(
ddf_m5[in_night_indx] == np.nanmax(ddf_m5[in_night_indx]))[0])
mjd_start = mjd[in_night_indx[best_depth_indx]]
m5_approx.append(ddf_m5[in_night_indx[best_depth_indx]])
mjd_end = mjd_start + expire_dict[obs_attempts[indx]]
mjd_observe.append((mjd_start, mjd_end))
result = np.zeros(len(mjd_observe),
dtype=[('mjd_start', '<f8'), ('mjd_end', '<f8'),
('label', '<U10')])
mjd_observe = np.array(mjd_observe)
result['mjd_start'] = mjd_observe[:, 0]
result['mjd_end'] = mjd_observe[:, 1]
result['label'] = ddf_name
return result #, ddf_ra, ddf_dec #, previous_ddf[survey_indx].observations, m5_approx
def create_output_table(cursor, database, hname, sessionID):
"""
Create summary table.
"""
# Expected summary table name is -
summarytable = 'summary_%s_%d' % (hname, sessionID)
# First check if summary table with expected name exists.
# Note that by using the connect_db method above, we are already using the correct database (OpsimDB).
sql = "show tables like '%s'" % (summarytable)
results = getDbData(cursor, sql)
if len(results) > 0:
# Summary table exists. Stop and ask user to handle this.
message = []
message.append(
"The summary table %s already exists in the MySQL %s database." %
(summarytable, database))
message.append(
"Please remove the table if you wish to rerun gen_output.py")
print os.linesep.join(message)
sys.exit(255)
# Otherwise, table does not exist and we're good to go.
print 'Creating summary table %s' % (summarytable)
sql = 'create table %s (obsHistID int(10) unsigned not null, sessionID int(10) unsigned not null, ' % (
summarytable)
sql += 'propID int(10), fieldID int(10) unsigned not null, fieldRA double, fieldDec double, '
sql += 'filter varchar(8), expDate int(10) unsigned, expMJD double, night int(10) unsigned, '
sql += 'visitTime double, visitExpTime double, finRank double, FWHMeff double, FWHMgeom double, transparency double, '
sql += 'airmass double, vSkyBright double, filtSkyBrightness double, rotSkyPos double, rotTelPos double, lst double, '
sql += 'altitude double, azimuth double, dist2Moon double, solarElong double, moonRA double, moonDec double, '
sql += 'moonAlt double, moonAZ double, moonPhase double, sunAlt double, sunAz double, phaseAngle double, '
sql += 'rScatter double, mieScatter double, moonIllum double, moonBright double, darkBright double, '
sql += 'rawSeeing double, wind double, humidity double, slewDist double, slewTime double, fiveSigmaDepth double);'
ret = getDbData(cursor, sql)
# Select data from ObsHistory table.
sql = 'select obsHistID, Session_sessionID as sessionID, Field_fieldID as fieldID, filter, expDate, expMJD, '
sql += 'night, visitTime, visitExpTime, finRank, finSeeing, transparency, airmass, vSkyBright, '
sql += 'filtSkyBright as filtSkyBrightness, rotSkyPos, lst, alt as altitude, az as azimuth, dist2Moon, '
sql += 'solarElong, moonRA, moonDec, moonAlt, moonAZ, moonPhase, sunAlt, sunAz, phaseAngle, rScatter, mieScatter, '
sql += 'moonIllum, moonBright, darkBright, rawSeeing, wind, humidity from ObsHistory '
sql += 'where Session_sessionID = %d;' % (sessionID)
ret = getDbData(cursor, sql)
# ctioHeight = 2215.;
# ctioLat = -30.16527778;
# ctioLon = 70.815;
# extC = .172;
# DEG2RAD = 0.0174532925;
# RAD2DEG = 57.2957795;
# For each observation, add the relevant additional information from other tables.
# Note that the summary table is not one-to-one with the ObsHistory table (observations used for multiple proposals
# are recorded multiple times into the summary table).
for k in range(len(ret)):
obsHistID = ret[k][0]
fieldID = ret[k][2]
sql = 'select fieldRA, fieldDec from Field where fieldID = %d' % fieldID
fld = getDbData(cursor, sql)
sql = 'select slewTime, slewDist, slewID from SlewHistory where ObsHistory_Session_sessionID = %d and ObsHistory_obsHistID = %d' % (
sessionID, obsHistID)
slw = getDbData(cursor, sql)
sql = 'select Proposal_propID as propID from ObsHistory_Proposal where ObsHistory_Session_sessionID = %d and ObsHistory_obsHistID = %d' % (
sessionID, obsHistID)
prp = getDbData(cursor, sql)
sql = 'select rotTelPos from SlewState where SlewHistory_slewID = %d' % slw[
0][2]
rtp = getDbData(cursor, sql)
for i in range(len(prp)):
MJD = float(ret[k][5])
alt_RAD = float(ret[k][17])
az_RAD = float(ret[k][18])
moonRA_RAD = float(ret[k][21])
moonDec_RAD = float(ret[k][22])
lst_RAD = float(ret[k][16])
moonha_RAD = lst_RAD - moonRA_RAD
malt_RAD = float(ret[k][23])
mphase = float(ret[k][25])
saz = float(ret[k][27])
salt = float(ret[k][26])
mphase = np.arccos((mphase / 50) - 1) * 180 / np.pi
# 5 sigma calculations
visitFilter = ret[k][3]
FWHMeff = float(ret[k][10])
# Calculation of FWHMgeom based on Bo & Zeljko's
# fit between apparent (FWHMgeom) and Neff (FWHMeff) values
FWHMgeom = 0.822 * FWHMeff + 0.052
airmass = float(ret[k][12])
filtsky = float(ret[k][14])
expTime = float(ret[k][8])
tauCloud = 0
m5 = m5_flat_sed(visitFilter, filtsky, FWHMeff, expTime, airmass,
tauCloud)
sql = 'insert into %s (obsHistID, sessionID, propID, fieldID, fieldRA, fieldDec, filter, ' % (
summarytable)
sql += 'expDate, expMJD, night, visitTime, visitExpTime, finRank, FWHMeff, FWHMgeom, transparency, airmass, vSkyBright, '
sql += 'filtSkyBrightness, rotSkyPos, rotTelPos, lst, altitude, azimuth, dist2Moon, solarElong, moonRA, moonDec, '
sql += 'moonAlt, moonAZ, moonPhase, sunAlt, sunAz, phaseAngle, rScatter, mieScatter, moonIllum, '
sql += 'moonBright, darkBright, rawSeeing, wind, humidity, slewDist, slewTime, fiveSigmaDepth) values '
sql += '(%d, %d, %d, %d, %f, %f, "%s", %d, %f, %d, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f)' % (
ret[k][0], ret[k][1], prp[i][0], ret[k][2],
np.radians(fld[0][0]), np.radians(
fld[0][1]), ret[k][3], ret[k][4], ret[k][5], ret[k][6],
ret[k][7], ret[k][8], ret[k][9], FWHMeff, FWHMgeom, ret[k][11],
ret[k][12], ret[k][13], ret[k][14], ret[k][15], rtp[1][0],
ret[k][16], ret[k][17], ret[k][18], ret[k][19], ret[k][20],
ret[k][21], ret[k][22], ret[k][23], ret[k][24], ret[k][25],
ret[k][26], ret[k][27], ret[k][28], ret[k][29], ret[k][30],
ret[k][31], ret[k][32], ret[k][33], ret[k][34], ret[k][35],
ret[k][36], slw[0][1], slw[0][0], m5)
insertDbData(cursor, sql)
def obs2sqlite(observations_in, location='LSST', outfile='observations.sqlite', slewtime_limit=5.,
full_sky=False, radians=True):
"""
Utility to take an array of observations and dump it to a sqlite file, filling in useful columns along the way.
observations_in: numpy array with at least columns of
ra : RA in degrees
dec : dec in degrees
mjd : MJD in day
filter : string with the filter name
exptime : the exposure time in seconds
slewtime_limit : float
Consider all slewtimes larger than this to be closed-dome time not part of a slew.
"""
# Set the location to be LSST
if location == 'LSST':
telescope = Site('LSST')
# Check that we have the columns we need
needed_cols = ['ra', 'dec', 'mjd', 'filter']
in_cols = observations_in.dtype.names
for col in needed_cols:
if needed_cols not in in_cols:
ValueError('%s column not found in observtion array' % col)
n_obs = observations_in.size
sm = None
# make sure they are in order by MJD
observations_in.sort(order='mjd')
# Take all the columns that are in the input and add any missing
names = ['filter', 'ra', 'dec', 'mjd', 'exptime', 'alt', 'az', 'skybrightness',
'seeing', 'night', 'slewtime', 'fivesigmadepth', 'airmass', 'sunAlt', 'moonAlt']
types = ['|S1']
types.extend([float]*(len(names)-1))
observations = np.zeros(n_obs, dtype=list(zip(names, types)))
# copy over the ones we have
for col in in_cols:
observations[col] = observations_in[col]
# convert output to be in degrees like expected
if radians:
observations['ra'] = np.degrees(observations['ra'])
observations['dec'] = np.degrees(observations['dec'])
if 'exptime' not in in_cols:
observations['exptime'] = 30.
# Fill in the slewtime. Note that filterchange time gets included in slewtimes
if 'slewtime' not in in_cols:
# Assume MJD is midpoint of exposures
mjd_sec = observations_in['mjd']*24.*3600.
observations['slewtime'][1:] = mjd_sec[1:]-mjd_sec[0:-1] - observations['exptime'][0:-1]*0.5 - observations['exptime'][1:]*0.5
closed = np.where(observations['slewtime'] > slewtime_limit*60.)
observations['slewtime'][closed] = 0.
# Let's just use the stupid-fast to get alt-az
if 'alt' not in in_cols:
alt, az = stupidFast_RaDec2AltAz(np.radians(observations['ra']), np.radians(observations['dec']),
telescope.latitude_rad, telescope.longitude_rad, observations['mjd'])
observations['alt'] = np.degrees(alt)
observations['az'] = np.degrees(az)
# Fill in the airmass
if 'airmass' not in in_cols:
observations['airmass'] = 1./np.cos(np.pi/2. - np.radians(observations['alt']))
# Fill in the seeing
if 'seeing' not in in_cols:
# XXX just fill in a dummy val
observations['seeing'] = 0.8
if 'night' not in in_cols:
m2n = mjd2night()
observations['night'] = m2n(observations['mjd'])
# Sky Brightness
if 'skybrightness' not in in_cols:
if full_sky:
sm = SkyModel(mags=True)
for i, obs in enumerate(observations):
sm.setRaDecMjd(obs['ra'], obs['dec'], obs['mjd'], degrees=True)
observations['skybrightness'][i] = sm.returnMags()[obs['filter']]
else:
# Let's try using the pre-computed sky brighntesses
sm = sb.SkyModelPre(preload=False)
full = sm.returnMags(observations['mjd'][0])
nside = hp.npix2nside(full['r'].size)
imax = float(np.size(observations))
for i, obs in enumerate(observations):
indx = raDec2Hpid(nside, obs['ra'], obs['dec'])
observations['skybrightness'][i] = sm.returnMags(obs['mjd'], indx=[indx])[obs['filter']]
sunMoon = sm.returnSunMoon(obs['mjd'])
observations['sunAlt'][i] = sunMoon['sunAlt']
observations['moonAlt'][i] = sunMoon['moonAlt']
progress = i/imax*100
text = "\rprogress = %.2f%%"%progress
sys.stdout.write(text)
sys.stdout.flush()
observations['sunAlt'] = np.degrees(observations['sunAlt'])
observations['moonAlt'] = np.degrees(observations['moonAlt'])
# 5-sigma depth
for fn in np.unique(observations['filter']):
good = np.where(observations['filter'] == fn)
observations['fivesigmadepth'][good] = m5_flat_sed(fn, observations['skybrightness'][good],
observations['seeing'][good],
observations['exptime'][good],
observations['airmass'][good])
conn = sqlite3.connect(outfile)
df = pd.DataFrame(observations)
df.to_sql('observations', conn)
def select_next_target(self):
if not self.isnight:
return self.nulltarget
# Telemetry stream
telemetry_stream = self.get_telemetry()
self.scheduler.update_conditions(telemetry_stream)
winner_target = self.scheduler.request_observation()
if winner_target is None:
self.log.debug(
'[mjd = %.3f]: No target generated by the scheduler...' %
telemetry_stream['mjd'])
self.scheduler.flush_queue()
self.last_winner_target = self.nulltarget.get_copy()
return self.last_winner_target
self.log.debug('winner target: %s' % winner_target)
self.scheduler_winner_target = winner_target
hpid = _raDec2Hpid(self.sky_nside, winner_target['RA'][0],
winner_target['dec'][0])
propid = winner_target['survey_id'][0]
filtername = winner_target['filter'][0]
indx = self.proposal_id_dict[propid][0]
target = self.generate_target(winner_target[0])
self.target_list[target.fieldid] = {filtername: target}
self.science_proposal_list[indx].survey_targets_dict[
target.fieldid] = {
filtername: target
}
target.time = self.time
target.ang_rad = self.observatoryModel.radec2altazpa(
self.observatoryModel.dateprofile, target.ra_rad,
target.dec_rad)[2]
self.observatoryModel.current_state.ang_rad = target.ang_rad
slewtime, slew_state = self.observatoryModel.get_slew_delay(target)
if slewtime > 0.:
self.scheduler_winner_target[
'mjd'] = telemetry_stream['mjd'] + slewtime / 60. / 60. / 24.
self.scheduler_winner_target['night'] = self.night
self.scheduler_winner_target['slewtime'] = slewtime
self.scheduler_winner_target['skybrightness'] = \
self.sky_brightness.returnMags(self.observatoryModel.dateprofile.mjd,
indx=[hpid],
extrapolate=True)[filtername]
self.scheduler_winner_target['FWHMeff'] = telemetry_stream[
'FWHMeff_%s' % filtername][hpid]
self.scheduler_winner_target['FWHM_geometric'] = \
telemetry_stream['FWHM_geometric_%s' % winner_target['filter'][0]][hpid]
self.scheduler_winner_target['airmass'] = telemetry_stream[
'airmass'][hpid]
self.scheduler_winner_target['fivesigmadepth'] = m5_flat_sed(
filtername, self.scheduler_winner_target['skybrightness'],
self.scheduler_winner_target['FWHMeff'],
self.scheduler_winner_target['exptime'],
self.scheduler_winner_target['airmass'])
self.scheduler_winner_target['alt'] = target.alt_rad
self.scheduler_winner_target['az'] = target.az_rad
self.scheduler_winner_target['rotSkyPos'] = target.ang_rad
self.scheduler_winner_target['clouds'] = self.cloud
self.scheduler_winner_target['sunAlt'] = telemetry_stream['sunAlt']
self.scheduler_winner_target['moonAlt'] = telemetry_stream[
'moonAlt']
target.slewtime = slewtime
target.airmass = telemetry_stream['airmass'][hpid]
target.sky_brightness = self.sky_brightness.returnMags(
self.observatoryModel.dateprofile.mjd,
indx=[hpid],
extrapolate=True)[filtername][0]
self.observatoryModel2.set_state(self.observatoryState)
self.observatoryState.ang_rad = target.ang_rad
self.observatoryModel2.observe(target)
target.seeing = self.seeing
target.cloud = self.cloud
ntime = self.observatoryModel2.current_state.time
if ntime < self.sunrise_timestamp:
# self.observatoryModel2.update_state(ntime)
if self.observatoryModel2.current_state.tracking:
target.time = self.time
if self.last_winner_target.targetid == target.targetid:
self.last_winner_target = self.nulltarget
self.targetid -= 1
else:
self.last_winner_target = target.get_copy()
else:
self.log.debug("select_next_target: target rejected %s" %
(str(target)))
self.log.debug("select_next_target: state rejected %s" %
str(self.observatoryModel2.current_state))
self.last_winner_target = self.nulltarget
self.targetid -= 1
else:
self.last_winner_target = self.nulltarget
self.targetid -= 1
else:
self.log.debug('Fail state: %i' % slew_state)
self.log.debug('Slewtime lower than zero! (slewtime = %f)' %
slewtime)
self.scheduler.flush_queue()
self.targetid -= 1
self.last_winner_target = self.nulltarget
self.log.debug(self.last_winner_target)
for propid in self.proposal_id_dict.keys():
self.science_proposal_list[self.proposal_id_dict[propid]
[0]].winners_list = []
self.science_proposal_list[indx].winners_list = [target.get_copy()]
return self.last_winner_target
def convert_cols(data):
cols_need = [
'obsHistID', 'sessionID', 'propID', 'fieldID', 'fieldRA', 'fieldDec',
'filter', 'expDate', 'expMJD', 'night', 'visitTime', 'visitExpTime',
'finRank', 'finSeeing', 'transparency', 'airmass', 'vSkyBright',
'filtSkyBrightness', 'rotSkyPos', 'rotTelPos', 'lst', 'altitude',
'azimuth', 'dist2Moon', 'solarElong', 'moonRA', 'moonDec', 'moonAlt',
'moonAZ', 'moonPhase', 'sunAlt', 'sunAz', 'phaseAngle', 'rScatter',
'mieScatter', 'moonIllum', 'moonBright', 'darkBright', 'rawSeeing',
'wind', 'humidity', 'slewDist', 'slewTime', 'fiveSigmaDepth',
'ditheredRA', 'ditheredDec'
]
cols_have = []
for k in data:
cols_have.append(k)
cols_map = {
'ditheredRA': 'hexdithra',
'ditheredDec': 'hexdithdec',
'visitTime': 'expTime',
'vSkyBright': 'VskyBright',
'filtSkyBrightness': 'filtSky',
'finSeeing': 'seeing',
'transparency': 'xparency'
}
cols_map_backup = {'ditheredRA': -99, 'ditheredDec': -99}
cols_default = {
'visitExpTime': 30.0,
'solarElong': -99,
'wind': -99,
'humidity': -99,
'moonAZ': -99
}
cols_calc = {'fiveSigmaDepth': m5_flat_sed, 'night': calc_night}
print "Needed but not present or translated."
for c in cols_need:
if c not in cols_have and c not in cols_default and c not in cols_calc:
if c in cols_map:
if cols_map[c] not in cols_have:
if c not in cols_map_backup:
print c
else:
print c
print "Present, but not in needed list. "
for k in cols_have:
if k not in cols_need:
print k
nvisits = len(data)
data2 = {}
for c in cols_need:
if c in cols_have:
data2[c] = data[c]
elif c in cols_map:
if cols_map[c] not in data and c in cols_map_backup:
data2[c] = np.ones(nvisits, float) * cols_map_backup[c]
else:
data2[c] = data[cols_map[c]]
elif c in cols_default:
data2[c] = np.ones(nvisits, float) * cols_default[c]
elif c in cols_calc:
if c == 'fiveSigmaDepth':
data2[c] = np.zeros(nvisits, float)
for index, v in data.iterrows():
data2[c][index] = m5_flat_sed(v['filter'], v['filtSky'],
v['seeing'],
v['expTime'] - 4.0,
v['airmass'])
elif c == 'night':
data2[c] = calc_night(data['expMJD'],
int(data['expMJD'].min()))
data2 = pd.DataFrame(data2)
return data2
def create_output_table(cursor, database, hname, sessionID):
"""
Create summary table.
"""
# Expected summary table name is -
summarytable = 'summary_%s_%d' %(hname, sessionID)
# First check if summary table with expected name exists.
# Note that by using the connect_db method above, we are already using the correct database (OpsimDB).
sql = "show tables like '%s'" %(summarytable)
results = getDbData(cursor, sql)
if len(results) > 0:
# Summary table exists. Stop and ask user to handle this.
message = []
message.append("The summary table %s already exists in the MySQL %s database." % (summarytable, database))
message.append("Please remove the table if you wish to rerun gen_output.py")
print os.linesep.join(message)
sys.exit(255)
# Otherwise, table does not exist and we're good to go.
print 'Creating summary table %s' %(summarytable)
sql = 'create table %s (obsHistID int(10) unsigned not null, sessionID int(10) unsigned not null, ' %(summarytable)
sql += 'propID int(10), fieldID int(10) unsigned not null, fieldRA double, fieldDec double, '
sql += 'filter varchar(8), expDate int(10) unsigned, expMJD double, night int(10) unsigned, '
sql += 'visitTime double, visitExpTime double, finRank double, FWHMeff double, FWHMgeom double, transparency double, '
sql += 'airmass double, vSkyBright double, filtSkyBrightness double, rotSkyPos double, rotTelPos double, lst double, '
sql += 'altitude double, azimuth double, dist2Moon double, solarElong double, moonRA double, moonDec double, '
sql += 'moonAlt double, moonAZ double, moonPhase double, sunAlt double, sunAz double, phaseAngle double, '
sql += 'rScatter double, mieScatter double, moonIllum double, moonBright double, darkBright double, '
sql += 'rawSeeing double, wind double, humidity double, slewDist double, slewTime double, fiveSigmaDepth double);'
ret = getDbData(cursor, sql)
# Select data from ObsHistory table.
sql = 'select obsHistID, Session_sessionID as sessionID, Field_fieldID as fieldID, filter, expDate, expMJD, '
sql += 'night, visitTime, visitExpTime, finRank, finSeeing, transparency, airmass, vSkyBright, '
sql += 'filtSkyBright as filtSkyBrightness, rotSkyPos, lst, alt as altitude, az as azimuth, dist2Moon, '
sql += 'solarElong, moonRA, moonDec, moonAlt, moonAZ, moonPhase, sunAlt, sunAz, phaseAngle, rScatter, mieScatter, '
sql += 'moonIllum, moonBright, darkBright, rawSeeing, wind, humidity from ObsHistory '
sql += 'where Session_sessionID = %d;' %(sessionID)
ret = getDbData(cursor, sql)
# ctioHeight = 2215.;
# ctioLat = -30.16527778;
# ctioLon = 70.815;
# extC = .172;
# DEG2RAD = 0.0174532925;
# RAD2DEG = 57.2957795;
# For each observation, add the relevant additional information from other tables.
# Note that the summary table is not one-to-one with the ObsHistory table (observations used for multiple proposals
# are recorded multiple times into the summary table).
for k in range(len(ret)):
obsHistID = ret[k][0]
fieldID = ret[k][2]
sql = 'select fieldRA, fieldDec from Field where fieldID = %d' % fieldID
fld = getDbData(cursor, sql)
sql = 'select slewTime, slewDist, slewID from SlewHistory where ObsHistory_Session_sessionID = %d and ObsHistory_obsHistID = %d' % (sessionID, obsHistID)
slw = getDbData(cursor, sql)
sql = 'select Proposal_propID as propID from ObsHistory_Proposal where ObsHistory_Session_sessionID = %d and ObsHistory_obsHistID = %d' % (sessionID, obsHistID)
prp = getDbData(cursor, sql)
sql = 'select rotTelPos from SlewState where SlewHistory_slewID = %d' % slw[0][2]
rtp = getDbData(cursor, sql)
for i in range(len(prp)):
MJD = float(ret[k][5]);
alt_RAD = float(ret[k][17]);
az_RAD = float(ret[k][18]);
moonRA_RAD = float(ret[k][21]);
moonDec_RAD = float(ret[k][22]);
lst_RAD = float(ret[k][16]);
moonha_RAD = lst_RAD - moonRA_RAD;
malt_RAD = float(ret[k][23]);
mphase = float(ret[k][25]);
saz = float(ret[k][27]);
salt = float(ret[k][26]);
mphase = np.arccos((mphase/50)-1)*180/np.pi;
# 5 sigma calculations
visitFilter = ret[k][3];
FWHMeff = float(ret[k][10]);
# Calculation of FWHMgeom based on Bo & Zeljko's
# fit between apparent (FWHMgeom) and Neff (FWHMeff) values
FWHMgeom = 0.822*FWHMeff + 0.052;
airmass = float(ret[k][12]);
filtsky = float(ret[k][14]);
expTime = float(ret[k][8]);
tauCloud = 0
m5 = m5_flat_sed(visitFilter, filtsky, FWHMeff, expTime, airmass, tauCloud)
sql = 'insert into %s (obsHistID, sessionID, propID, fieldID, fieldRA, fieldDec, filter, ' %(summarytable)
sql += 'expDate, expMJD, night, visitTime, visitExpTime, finRank, FWHMeff, FWHMgeom, transparency, airmass, vSkyBright, '
sql += 'filtSkyBrightness, rotSkyPos, rotTelPos, lst, altitude, azimuth, dist2Moon, solarElong, moonRA, moonDec, '
sql += 'moonAlt, moonAZ, moonPhase, sunAlt, sunAz, phaseAngle, rScatter, mieScatter, moonIllum, '
sql += 'moonBright, darkBright, rawSeeing, wind, humidity, slewDist, slewTime, fiveSigmaDepth) values '
sql += '(%d, %d, %d, %d, %f, %f, "%s", %d, %f, %d, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f)' % (ret[k][0], ret[k][1], prp[i][0], ret[k][2], np.radians(fld[0][0]), np.radians(fld[0][1]), ret[k][3], ret[k][4], ret[k][5], ret[k][6], ret[k][7], ret[k][8], ret[k][9], FWHMeff, FWHMgeom, ret[k][11], ret[k][12], ret[k][13], ret[k][14], ret[k][15], rtp[1][0], ret[k][16], ret[k][17], ret[k][18], ret[k][19], ret[k][20], ret[k][21], ret[k][22], ret[k][23], ret[k][24], ret[k][25], ret[k][26], ret[k][27], ret[k][28], ret[k][29], ret[k][30], ret[k][31], ret[k][32], ret[k][33], ret[k][34], ret[k][35], ret[k][36], slw[0][1], slw[0][0], m5)
insertDbData(cursor, sql)
def attempt_observe(self, observation_in, indx=None):
"""
Check an observation, if there is enough time, execute it and return it, otherwise, return None.
"""
# If we were in a parked position, assume no time lost to slew, settle, filter change
observation = observation_in.copy()
alt, az = _approx_RaDec2AltAz(np.array([observation['RA']]),
np.array([observation['dec']]),
self.obs.lat, self.obs.lon, self.mjd)
if self.ra is not None:
if self.filtername != observation['filter']:
ft = self.f_change_time
st = 0.
else:
ft = 0.
st = self.slew_time(alt, az)
else:
st = 0.
ft = 0.
# Assume we can slew while reading the last exposure (note that slewtime calc gives 2 as a minimum. So this
# will not fail for DD fields, etc.)
# So, filter change time, slew to target time, expose time, read time
rt = (observation['nexp']-1.)*self.readtime
shutter_time = self.shutter_time*observation['nexp']
total_time = (ft + st + observation['exptime'] + rt + shutter_time)*sec2days
check_result, jump_mjd = self.check_mjd(self.mjd + total_time)
if check_result:
# XXX--major decision here, should the status be updated after every observation? Or just assume
# airmass, seeing, and skybrightness do not change significantly?
if self.ra is None:
update_status = True
else:
update_status = False
# This should be the start of the exposure.
observation['mjd'] = self.mjd + (ft + st)*sec2days
self.set_mjd(self.mjd + (ft + st)*sec2days)
self.ra = observation['RA']
self.dec = observation['dec']
if update_status:
# What's the name for temp variables?
status = self.return_status()
observation['night'] = self.night
# XXX I REALLY HATE THIS! READTIME SHOULD NOT BE LUMPED IN WITH SLEWTIME!
# XXX--removing that so I may not be using the same convention as opsim.
observation['slewtime'] = ft+st
self.filtername = observation['filter'][0]
hpid = _raDec2Hpid(self.sky_nside, self.ra, self.dec)
observation['skybrightness'] = self.sky.returnMags(observation['mjd'], indx=[hpid],
extrapolate=True)[self.filtername]
observation['FWHMeff'] = self.status['FWHMeff_%s' % self.filtername][hpid]
observation['FWHM_geometric'] = self.status['FWHM_geometric_%s' % self.filtername][hpid]
observation['airmass'] = self.status['airmass'][hpid]
observation['fivesigmadepth'] = m5_flat_sed(observation['filter'][0],
observation['skybrightness'],
observation['FWHMeff'],
observation['exptime'],
observation['airmass'])
observation['alt'] = alt
observation['az'] = az
observation['clouds'] = self.status['clouds']
observation['sunAlt'] = self.status['sunAlt']
observation['moonAlt'] = self.status['moonAlt']
# We had advanced the slew and filter change, so subtract that off and add the total visit time.
self.set_mjd(self.mjd + total_time - (ft + st)*sec2days)
return observation
else:
self.mjd = jump_mjd
self.night = self.mjd2night(self.mjd)
self.ra = None
self.dec = None
self.status = None
self.filtername = None
return None
def obs2sqlite(observations_in, location='LSST', outfile='observations.sqlite', slewtime_limit=5.,
full_sky=False, radians=True):
"""
Utility to take an array of observations and dump it to a sqlite file, filling in useful columns along the way.
observations_in: numpy array with at least columns of
ra : RA in degrees
dec : dec in degrees
mjd : MJD in day
filter : string with the filter name
exptime : the exposure time in seconds
slewtime_limit : float
Consider all slewtimes larger than this to be closed-dome time not part of a slew.
"""
# Set the location to be LSST
if location == 'LSST':
telescope = Site('LSST')
# Check that we have the columns we need
needed_cols = ['ra', 'dec', 'mjd', 'filter']
in_cols = observations_in.dtype.names
for col in needed_cols:
if needed_cols not in in_cols:
ValueError('%s column not found in observtion array' % col)
n_obs = observations_in.size
sm = None
# make sure they are in order by MJD
observations_in.sort(order='mjd')
# Take all the columns that are in the input and add any missing
names = ['filter', 'ra', 'dec', 'mjd', 'exptime', 'alt', 'az', 'skybrightness',
'seeing', 'night', 'slewtime', 'fivesigmadepth', 'airmass', 'sunAlt', 'moonAlt']
types = ['|S1']
types.extend([float]*(len(names)-1))
observations = np.zeros(n_obs, dtype=list(zip(names, types)))
# copy over the ones we have
for col in in_cols:
observations[col] = observations_in[col]
# convert output to be in degrees like expected
if radians:
observations['ra'] = np.degrees(observations['ra'])
observations['dec'] = np.degrees(observations['dec'])
if 'exptime' not in in_cols:
observations['exptime'] = 30.
# Fill in the slewtime. Note that filterchange time gets included in slewtimes
if 'slewtime' not in in_cols:
# Assume MJD is midpoint of exposures
mjd_sec = observations_in['mjd']*24.*3600.
observations['slewtime'][1:] = mjd_sec[1:]-mjd_sec[0:-1] - observations['exptime'][0:-1]*0.5 - observations['exptime'][1:]*0.5
closed = np.where(observations['slewtime'] > slewtime_limit*60.)
observations['slewtime'][closed] = 0.
# Let's just use the stupid-fast to get alt-az
if 'alt' not in in_cols:
alt, az = _approx_RaDec2AltAz(np.radians(observations['ra']), np.radians(observations['dec']),
telescope.latitude_rad, telescope.longitude_rad, observations['mjd'])
observations['alt'] = np.degrees(alt)
observations['az'] = np.degrees(az)
# Fill in the airmass
if 'airmass' not in in_cols:
observations['airmass'] = 1./np.cos(np.pi/2. - np.radians(observations['alt']))
# Fill in the seeing
if 'seeing' not in in_cols:
# XXX just fill in a dummy val
observations['seeing'] = 0.8
if 'night' not in in_cols:
m2n = mjd2night()
observations['night'] = m2n(observations['mjd'])
# Sky Brightness
if 'skybrightness' not in in_cols:
if full_sky:
sm = SkyModel(mags=True)
for i, obs in enumerate(observations):
sm.setRaDecMjd(obs['ra'], obs['dec'], obs['mjd'], degrees=True)
observations['skybrightness'][i] = sm.returnMags()[obs['filter']]
else:
# Let's try using the pre-computed sky brighntesses
sm = sb.SkyModelPre(preload=False)
full = sm.returnMags(observations['mjd'][0])
nside = hp.npix2nside(full['r'].size)
imax = float(np.size(observations))
for i, obs in enumerate(observations):
indx = raDec2Hpid(nside, obs['ra'], obs['dec'])
observations['skybrightness'][i] = sm.returnMags(obs['mjd'], indx=[indx])[obs['filter']]
sunMoon = sm.returnSunMoon(obs['mjd'])
observations['sunAlt'][i] = sunMoon['sunAlt']
observations['moonAlt'][i] = sunMoon['moonAlt']
progress = i/imax*100
text = "\rprogress = %.2f%%"%progress
sys.stdout.write(text)
sys.stdout.flush()
observations['sunAlt'] = np.degrees(observations['sunAlt'])
observations['moonAlt'] = np.degrees(observations['moonAlt'])
# 5-sigma depth
for fn in np.unique(observations['filter']):
good = np.where(observations['filter'] == fn)
observations['fivesigmadepth'][good] = m5_flat_sed(fn, observations['skybrightness'][good],
observations['seeing'][good],
observations['exptime'][good],
observations['airmass'][good])
conn = sqlite3.connect(outfile)
df = pd.DataFrame(observations)
df.to_sql('observations', conn)
def run(self):
"""Run the simulation.
"""
self.log.info("Starting simulation")
self.conf_comm.run()
self.save_configuration()
self.save_proposal_information()
self.save_field_information()
self.log.debug("Duration = {}".format(self.duration))
for night in range(1, int(self.duration) + 1):
self.start_night(night)
while self.time_handler.current_timestamp < self.end_of_night:
self.comm_time.timestamp = self.time_handler.current_timestamp
self.log.log(
LoggingLevel.EXTENSIVE.value,
"Timestamp sent: {:.6f}".format(
self.time_handler.current_timestamp))
self.sal.put(self.comm_time)
observatory_state = self.seq.get_observatory_state(
self.time_handler.current_timestamp)
self.log.log(
LoggingLevel.EXTENSIVE.value,
"Observatory State: {}".format(
topic_strdict(observatory_state)))
self.sal.put(observatory_state)
self.cloud_model.set_topic(self.time_handler, self.cloud)
self.sal.put(self.cloud)
self.seeing_model.set_topic(self.time_handler, self.seeing)
self.sal.put(self.seeing)
self.get_target_from_scheduler()
observation, slew_info, exposure_info = self.seq.observe_target(
self.target, self.time_handler)
# Add a few more things to the observation
observation.night = night
elapsed_time = self.time_handler.time_since_given(
observation.observation_start_time)
observation.cloud = self.cloud_model.get_cloud(elapsed_time)
seeing_values = self.seeing_model.calculate_seeing(
elapsed_time, observation.filter, observation.airmass)
observation.seeing_fwhm_500 = seeing_values[0]
observation.seeing_fwhm_geom = seeing_values[1]
observation.seeing_fwhm_eff = seeing_values[2]
visit_exposure_time = sum([
observation.exposure_times[i]
for i in range(observation.num_exposures)
])
observation.five_sigma_depth = m5_flat_sed(
observation.filter, observation.sky_brightness,
observation.seeing_fwhm_eff, visit_exposure_time,
observation.airmass)
# Pass observation back to scheduler
self.log.log(LoggingLevel.EXTENSIVE.value, "tx: observation")
self.sal.put(observation)
# Wait for interested proposal information
lastconfigtime = time.time()
while self.wait_for_scheduler:
rcode = self.sal.manager.getNextSample_interestedProposal(
self.interested_proposal)
if rcode == 0 and self.interested_proposal.num_proposals >= 0:
self.log.log(LoggingLevel.EXTENSIVE.value,
"Received interested proposal.")
break
else:
tf = time.time()
if (tf - lastconfigtime) > 5.0:
self.log.log(
LoggingLevel.EXTENSIVE.value,
"Failed to receive interested proposal due to timeout."
)
break
if self.wait_for_scheduler and observation.targetId != -1:
self.db.append_data("target_history", self.target)
self.db.append_data("observation_history", observation)
self.gather_proposal_history("target", self.target)
self.gather_proposal_history("observation",
self.interested_proposal)
for slew_type, slew_data in slew_info.items():
self.log.log(
LoggingLevel.TRACE.value,
"{}, {}".format(slew_type, type(slew_data)))
if isinstance(slew_data, list):
for data in slew_data:
self.db.append_data(slew_type, data)
else:
self.db.append_data(slew_type, slew_data)
for exposure_type in exposure_info:
self.log.log(LoggingLevel.TRACE.value,
"Adding {} to DB".format(exposure_type))
self.log.log(
LoggingLevel.TRACE.value,
"Number of exposures being added: "
"{}".format(len(exposure_info[exposure_type])))
for exposure in exposure_info[exposure_type]:
self.db.append_data(exposure_type, exposure)
self.end_night()
self.start_day()
maxi = mjds.size
for i, mjd in enumerate(mjds):
progress = i/maxi*100
text = "\rprogress = %0.1f%%" % progress
sys.stdout.write(text)
sys.stdout.flush()
sm.setRaDecMjd(ras, decs, mjd, degrees=False)
if sm.sunAlt > sun_limit:
mags.append(sm.returnMags()['g']*0)
airmasses.append(sm.airmass*0)
else:
mags.append(sm.returnMags()['g'])
airmasses.append(sm.airmass)
sun_alts.append(sm.sunAlt)
mags = np.array(mags)
airmasses = np.array(airmasses)
result['sun_alt'] = sun_alts
for i, survey in enumerate(dds):
result[survey.survey_name+'_airmass'] = airmasses[:, i]
result[survey.survey_name+'_sky_g'] = mags[:, i]
# now to compute the expected seeing if the zenith is nominal
FWHMeff = seeing_model(nominal_seeing, airmasses[:, i])['fwhmEff'][seeing_indx, :]
result[survey.survey_name+'_m5_g'] = m5_flat_sed('g', mags[:, i], FWHMeff, 30., airmasses[:, i], nexp=1)
np.savez('ddf_grid.npz', ddf_grid=result)
