def run(self, dataSlice, slicePoint=None):
pm = np.array(self.simobj['PM_out'])
mag = np.array(self.simobj['MAG'])
obs = np.where(dataSlice[self.mjdCol] < min(dataSlice[self.mjdCol]) +
365 * self.surveyduration)
np.random.seed(5000)
mjd = dataSlice[self.mjdCol][obs]
flt = dataSlice[self.filterCol][obs]
if (self.f in flt):
snr = m52snr(mag[:, np.newaxis], dataSlice[self.m5Col][obs])
row, col = np.where(snr > self.snr_lim)
Times = np.sort(mjd)
dt = np.array(list(combinations(Times, 2)))
if np.size(dt) > 0:
DeltaTs = np.absolute(np.subtract(dt[:, 0], dt[:, 1]))
DeltaTs = np.unique(DeltaTs)
dt_pm = 0.05 * np.amin(
dataSlice[self.seeingCol]) / pm[np.unique(row)]
selection = np.where((dt_pm > min(DeltaTs))
& (dt_pm < max(DeltaTs)))
#precis = astrom_precision(dataSlice[self.seeingCol][obs], snr[row,:])
#sigmapm= sigma_slope(dataSlice[self.mjdCol][obs], precis)*365.25*1e3
objRate = 0.7 # how many go off per day
nObj = np.size(pm[selection])
m0s = mag[selection]
t = dataSlice[self.mjdCol][obs] - dataSlice[self.mjdCol].min()
detected = 0
# Loop though each generated transient and decide if it was detected ,
# This could be a more complicated piece of code, for example demanding ,
# A color measurement in a night. ,
durations = dt_pm[selection]
slopes = np.random.uniform(-3, 3, np.size(selection))
t0s = np.random.uniform(0, self.surveyduration, nObj)
lcs = self.lightCurve(t, t0s, m0s, durations, slopes)
good = m52snr(lcs, dataSlice[self.m5Col][obs]) > self.snr_lim
detectedTest = good.sum(axis=0)
detected = np.sum(detectedTest > 2)
#for i,t0 in enumerate(np.random.uniform(0,self.surveyduration,nObj)):
# duration =dt_pm[selection][i]
# slope = np.random.uniform(-3,3)
# lc = self.lightCurve(t, t0, m0s[i],duration, slope)
# good = m52snr(lc,dataSlice[self.m5Col][obs])> self.snr_lim
# detectTest = dataSlice[self.m5Col][obs] - lc
# if detectTest.max() > 0 and len(good)>2:
# detected += 1
# Return the fraction of transients detected ,
if float(nObj) == 0:
A = np.inf
else:
A = float(nObj)
res = float(detected) / A
#print('detected fraction:{}'.format(res))
return res
def run(self, dataSlice, slicePoint=None):
obs = np.where(dataSlice[self.mjdCol]<min(dataSlice[self.mjdCol])+365*self.surveyduration)
deltamag= np.arange(self.MagIterLim[0],self.MagIterLim[1],self.MagIterLim[2])
out = {}
for dm in deltamag:
if self.mode == 'distance':
pmnew= self.mu_sag /(10**(dm/5))
mag = self.mag_sag + dm
elif self.mode == 'density':
pmnew= self.mu_sag
mag = self.mag_sag + dm
else:
print('##### ERROR: the metric is not implemented for this mode.')
mjd = dataSlice[self.mjdCol][obs]
flt = dataSlice[self.filterCol][obs]
if ('g' in flt) and ('r' in flt):
# select objects above the limit magnitude threshold
snr = m52snr(mag[:, np.newaxis],dataSlice[self.m5Col][obs])
row, col =np.where(snr>self.snr_lim)
if self.gap_selection:
Times = np.sort(mjd)
dt = np.array(list(combinations(Times,2)))
DeltaTs = np.absolute(np.subtract(dt[:,0],dt[:,1]))
DeltaTs = np.unique(DeltaTs)
if np.size(DeltaTs)>0:
dt_pm = 0.05*np.amin(dataSlice[self.seeingCol])/pmnew[np.unique(row)]
selection = np.where((dt_pm>min(DeltaTs)) & (dt_pm<max(DeltaTs)))
else:
selection = np.unique(row)
precis = astrom_precision(dataSlice[self.seeingCol][obs], snr[row,:])
sigmapm= self.sigma_slope(dataSlice[self.mjdCol][obs], precis)
Hg = mag[selection]+5*np.log10(pmnew[selection])-10
sigmaHg = np.sqrt((mag[selection]/m52snr(mag[selection],np.median(dataSlice[self.m5Col])))**(2)+ (4.715*sigmapm[selection]/np.ceil(pmnew[selection]))**2)
sigmag = np.sqrt((mag[selection]/m52snr(mag[selection],np.median(dataSlice[self.m5Col])))**2+((mag[selection]-self.gr[selection])/m52snr((mag[selection]-self.gr[selection]),np.median(dataSlice[self.m5Col])))**2)
err_ellipse = np.pi*sigmaHg*sigmag
if self.dataout:
CI = np.array([np.nansum((([gr-gcol ])/sigmag)**2 + ((Hg-h)/sigmaHg)**2 <= 1)/np.size(pmnew[selection]) for (gcol,h) in zip(gr,Hg)])
out[dm] = {'CI':CI,'alpha':np.size(pmnew[selection])/np.size(pmnew) ,'err':err_ellipse,'Hg':Hg,'gr':gr, 'sigmaHg':sigmaHg,'sigmagr':sigmag}
else:
out[dm] = {'alpha':np.size(pmnew[selection])/np.size(pmnew) ,'err':err_ellipse}
else:
if self.dataout:
out[dm] = {'CI':0,'alpha':0,'err':0,'Hg':Hg,'gr':gr, 'sigmaHg':sigmaHg,'sigmagr':sigmag}
else:
out[dm] = {'alpha':0 ,'err':0}
if self.dataout:
return out
else:
if ('g' in flt) and ('r' in flt):
res = out[dm]['alpha']/np.nanmean(out[dm]['err'][np.isfinite(out[dm]['err'])])
return res
def run(self, dataslice, slicePoint=None):
filters = np.unique(dataslice['filter'])
precis = np.zeros(dataslice.size, dtype='float')
for f in filters:
observations = np.where(dataslice['filter'] == f)
if np.size(observations[0]) < 2:
precis[observations] = self.badval
else:
snr = mafUtils.m52snr(self.mags[f],
dataslice[self.m5Col][observations])
precis[observations] = mafUtils.astrom_precision(
dataslice[self.seeingCol][observations], snr)
precis[observations] = np.sqrt(precis[observations]**2 + self.atm_err**2)
good = np.where(precis != self.badval)
result = mafUtils.sigma_slope(dataslice['expMJD'][good], precis[good])
result = result*365.25*1e3 #convert to mas/yr
if (self.normalize) & (good[0].size > 0):
new_dates=dataslice['expMJD'][good]*0
nDates = new_dates.size
new_dates[nDates/2:] = self.baseline*365.25
result = (mafUtils.sigma_slope(new_dates, precis[good])*365.25*1e3)/result
# Observations that are very close together can still fail
if np.isnan(result):
result = self.badval
return result
def run(self, dataSlice, slicePoint=None):
snr = np.zeros(len(dataSlice), dtype='float')
for filt in self.filters:
inFilt = np.where(dataSlice[self.filterCol] == filt)
snr[inFilt] = mafUtils.m52snr(self.mags[filt],
dataSlice[self.m5Col][inFilt])
position_errors = np.sqrt(
mafUtils.astrom_precision(dataSlice[self.seeingCol], snr)**2 +
self.atm_err**2)
x_coord = np.tan(dataSlice['zenithDistance']) * np.sin(
dataSlice[self.PACol])
# Things should be the same for RA and dec.
# Now I want to compute the error if I interpolate/extrapolate to +/-1.
# function is of form, y=ax. a=y/x. da = dy/x.
# Only strictly true if we know the unshifted position. But this should be a reasonable approx.
slope_uncerts = position_errors / x_coord
total_slope_uncert = 1. / np.sqrt(np.sum(1. / slope_uncerts**2))
# So, this will be the uncertainty in the RA or Dec offset at x= +/- 1.
result = total_slope_uncert
return result
def run(self, dataslice, slicePoint=None):
filters = np.unique(dataslice['filter'])
filters = [str(f) for f in filters]
precis = np.zeros(dataslice.size, dtype='float')
for f in filters:
observations = np.where(dataslice['filter'] == f)
if np.size(observations[0]) < 2:
precis[observations] = self.badval
else:
snr = mafUtils.m52snr(self.mags[f],
dataslice[self.m5Col][observations])
precis[observations] = mafUtils.astrom_precision(
dataslice[self.seeingCol][observations], snr)
precis[observations] = np.sqrt(precis[observations]**2 +
self.atm_err**2)
good = np.where(precis != self.badval)
result = mafUtils.sigma_slope(dataslice[self.mjdCol][good],
precis[good])
result = result * 365.25 * 1e3 # Convert to mas/yr
if (self.normalize) & (good[0].size > 0):
new_dates = dataslice[self.mjdCol][good] * 0
nDates = new_dates.size
new_dates[nDates // 2:] = self.baseline * 365.25
result = (mafUtils.sigma_slope(new_dates, precis[good]) * 365.25 *
1e3) / result
# Observations that are very close together can still fail
if np.isnan(result):
result = self.badval
return result
def run(self, dataSlice, slicePoint=None):
# punt if we don't have enough points
if dataSlice.size < self.means.size + 3:
return self.badval
# Generate independt variable for light curve
t = np.empty(dataSlice.size,
dtype=zip(['time', 'filter'], [float, '|S1']))
t['time'] = dataSlice[self.mjdCol] - dataSlice[self.mjdCol].min()
t['filter'] = dataSlice[self.filterCol]
if 'distMod' in slicePoint.keys():
mags = self.means + slicePoint['distMod']
else:
mags = self.means
trueParams = np.append(
np.array([self.period, self.phase, self.amplitude]), mags)
trueLC = periodicStar(t, *trueParams)
fits = np.zeros((self.nMonte, trueParams.size), dtype=float)
for i in np.arange(self.nMonte):
snr = m52snr(trueLC, dataSlice[self.m5Col])
dmag = 2.5 * np.log10(1. + 1. / snr)
noise = np.random.randn(trueLC.size) * dmag
# Suppress warnings about failing on covariance
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# If it fails to converge, save values that should fail later
try:
parmVals, pcov = curve_fit(periodicStar,
t,
trueLC + noise,
p0=trueParams,
sigma=dmag)
except:
parmVals = trueParams * 0 - 666
fits[i, :] = parmVals
# Throw out any parameters if there are no observations in that filter
ufilters = np.unique(dataSlice[self.filterCol])
if ufilters.size < 9:
for key in self.filter2index.keys():
if key not in ufilters:
fits[:, self.filter2index[key]] = -np.inf
# Find the fraction of fits that meet the "well-fit" criteria
periodFracErr = (fits[:, 0] - trueParams[0]) / trueParams[0]
ampFracErr = (fits[:, 2] - trueParams[2]) / trueParams[2]
magErr = fits[:, 3:] - trueParams[3:]
nBands = np.zeros(magErr.shape, dtype=int)
nBands[np.where(magErr <= self.magTol)] = 1
nBands = np.sum(nBands, axis=1)
nRecovered = np.size(
np.where((periodFracErr <= self.periodTol)
& (ampFracErr <= self.ampTol)
& (nBands >= self.nBands))[0])
fracRecovered = float(nRecovered) / self.nMonte
return fracRecovered
def run(self, dataSlice, slicePoint=None):
"""
Calculate the detectability of a transient with the specified SED.
If self.output_data is True, then returns the full lightcurve for each
object instead of the total number of transients that are detected.
Parameters:
----------
dataSlice : numpy.array
Numpy structured array containing the data related to the visits
provided by the slicer.
slicePoint : dict, optional
Dictionary containing information about the slicepoint currently
active in the slicer.
Returns:
-------
float or dict
The fraction of transients that could be detected.
(if output_data is False) Otherwise, a dictionary
with arrays of 'transient_id', 'lcMag', 'detected', 'expMJD',
'SNR', 'filter', 'epoch'
"""
dataSlice = self.setup_run_metric_variables(dataSlice)
self.initialize_phase_loop_variables(dataSlice)
# Consider each different 'phase shift' separately.
# We then just have a series of lightcurves, taking place back-to-back.
for time_shift in self.time_phase_shifts:
self.setup_phase_shift_dependent_variables(time_shift, dataSlice)
# Generate the actual light curve magnitudes and SNR
self.make_lightcurve(self.observation_epoch,
dataSlice[self.filterCol])
self.light_curve_SNRs = m52snr(self.light_curve_mags,
dataSlice[self.m5Col])
# Check observations above the defined threshold for detection.
self.evaluate_SNR_thresholds(dataSlice)
# With useable observations computed, evaluate all detection criteria
self.evaluate_all_detection_criteria(dataSlice)
if self.output_data:
# Output all the light curves, regardless of detection threshhold,
# but indicate which were 'detected'.
# Only returns for one phase shift, not all.
return {
"transient_id": self.transient_id,
"expMJD": dataSlice[self.mjdCol],
"epoch": self.observation_epoch,
"filter": dataSlice[self.filterCol],
"lcMag": self.light_curve_mags,
"SNR": self.light_curve_SNRs,
"detected": self.transient_detected,
}
else:
return float(self.num_detected) / self.max_num_transients
def run(self, dataSlice, slicePoint=None):
"""
Calculate the detectability of a transient with the specified SED.
If self.output_data is True, then returns the full lightcurve for each
object instead of the total number of transients that are detected.
Parameters:
----------
dataSlice : numpy.array
Numpy structured array containing the data related to the visits
provided by the slicer.
slicePoint : dict, optional
Dictionary containing information about the slicepoint currently
active in the slicer.
Returns:
-------
float or dict
The fraction of transients that could be detected.
(if output_data is False) Otherwise, a dictionary
with arrays of 'transient_id', 'lcMag', 'detected', 'expMJD',
'SNR', 'filter', 'epoch'
"""
dataSlice = self.setup_run_metric_variables(dataSlice)
self.initialize_phase_loop_variables(dataSlice)
# Consider each different 'phase shift' separately.
# We then just have a series of lightcurves, taking place back-to-back.
for time_shift in self.time_phase_shifts:
self.setup_phase_shift_dependent_variables(time_shift, dataSlice)
# Generate the actual light curve magnitudes and SNR
self.make_lightcurve(self.observation_epoch, dataSlice[self.filterCol])
self.light_curve_SNRs = m52snr(self.light_curve_mags, dataSlice[self.m5Col])
# Check observations above the defined threshold for detection.
self.evaluate_SNR_thresholds(dataSlice)
# With useable observations computed, evaluate all detection criteria
self.evaluate_all_detection_criteria(dataSlice)
if self.output_data:
# Output all the light curves, regardless of detection threshhold,
# but indicate which were 'detected'.
# Only returns for one phase shift, not all.
return {
"transient_id": self.transient_id,
"expMJD": dataSlice[self.mjdCol],
"epoch": self.observation_epoch,
"filter": dataSlice[self.filterCol],
"lcMag": self.light_curve_mags,
"SNR": self.light_curve_SNRs,
"detected": self.transient_detected,
}
else:
return float(self.num_detected) / self.max_num_transients
def _calc_amp(self, dataSlice):
"""Fractional SNR on the amplitude, testing for a variety of possible phases
"""
phases = np.arange(0, np.pi, np.pi / 8.)
snr = m52snr(self.starMag, dataSlice[self.m5Col])
amp_snrs = np.sin(dataSlice[self.mjdCol] / self.period * 2 * np.pi +
phases[:, np.newaxis]) * snr
amp_snr = np.min(np.sqrt(np.sum(amp_snrs**2, axis=1)))
max_snr = np.sqrt(np.sum(snr**2))
return amp_snr / max_snr
def run(self, dataSlice, slicePoint=None):
# Bail if we don't have enough points
if dataSlice.size < self.means.size+3:
return self.badval
# Generate input for true light curve
t = np.empty(dataSlice.size, dtype=list(zip(['time','filter'],[float,'|S1'])))
t['time'] = dataSlice[self.mjdCol]-dataSlice[self.mjdCol].min()
t['filter'] = dataSlice[self.filterCol]
# If we are adding a distance modulus to the magnitudes
if 'distMod' in list(slicePoint.keys()):
mags = self.means + slicePoint['distMod']
else:
mags = self.means
trueParams = np.append(np.array([self.period, self.phase, self.amplitude]), mags)
trueLC = periodicStar(t, *trueParams)
# Array to hold the fit results
fits = np.zeros((self.nMonte,trueParams.size),dtype=float)
for i in np.arange(self.nMonte):
snr = m52snr(trueLC,dataSlice[self.m5Col])
dmag = 2.5*np.log10(1.+1./snr)
noise = np.random.randn(trueLC.size)*dmag
# Suppress warnings about failing on covariance
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# If it fails to converge, save values that should fail later
try:
parmVals, pcov = curve_fit(periodicStar, t, trueLC+noise, p0=trueParams, sigma=dmag)
except:
parmVals = trueParams*0-666
fits[i,:] = parmVals
# Throw out any magnitude fits if there are no observations in that filter
ufilters = np.unique(dataSlice[self.filterCol])
if ufilters.size < 9:
for key in list(self.filter2index.keys()):
if key not in ufilters:
fits[:,self.filter2index[key]] = -np.inf
# Find the fraction of fits that meet the "well-fit" criteria
periodFracErr = (fits[:,0]-trueParams[0])/trueParams[0]
ampFracErr = (fits[:,2]-trueParams[2])/trueParams[2]
magErr = fits[:,3:]-trueParams[3:]
nBands = np.zeros(magErr.shape,dtype=int)
nBands[np.where(magErr <= self.magTol)] = 1
nBands = np.sum(nBands, axis=1)
nRecovered = np.size(np.where( (periodFracErr <= self.periodTol) &
(ampFracErr <= self.ampTol) &
(nBands >= self.nBands) )[0])
fracRecovered = float(nRecovered)/self.nMonte
return fracRecovered
def run(self, dataslice, slicePoint=None):
filters = np.unique(dataslice[self.filterCol])
snr = np.zeros(len(dataslice), dtype='float')
# compute SNR for all observations
for filt in filters:
good = np.where(dataslice[self.filterCol] == filt)
snr[good] = mafUtils.m52snr(self.mags[filt], dataslice[self.m5Col][good])
position_errors = np.sqrt(mafUtils.astrom_precision(dataslice[self.seeingCol], snr)**2+self.atm_err**2)
sigma = self._final_sigma(position_errors,dataslice['ra_pi_amp'],dataslice['dec_pi_amp'] )
if self.normalize:
# Leave the dec parallax as zero since one can't have ra and dec maximized at the same time.
sigma = self._final_sigma(position_errors,dataslice['ra_pi_amp']*0+1.,dataslice['dec_pi_amp']*0 )/sigma
return sigma
def run(self, dataSlice, slicePoint=None):
# The idea here is that we calculate position errors (in RA and Dec) for all observations.
# Then we generate arrays of the parallax offsets (delta RA parallax = ra_pi_amp, etc)
# and the DCR offsets (delta RA DCR = ra_dcr_amp, etc), and just add them together into one
# RA (and Dec) offset. Then, we try to fit for how we combined these offsets, but while
# considering the astrometric noise. If we can figure out that we just added them together
# (i.e. the curve_fit result is [a=1, b=1] for the function _positions above)
# then we should be able to disentangle the parallax and DCR offsets when fitting 'for real'.
# compute SNR for all observations
snr = np.zeros(len(dataSlice), dtype='float')
for filt in self.filters:
inFilt = np.where(dataSlice[self.filterCol] == filt)
snr[inFilt] = mafUtils.m52snr(self.mags[filt],
dataSlice[self.m5Col][inFilt])
# Compute the centroiding uncertainties
# Temporary fix for FWHMeff to FWHMgeom calculation.
if self.seeingCol.endswith('Eff'):
seeing = dataSlice[self.seeingCol] * 0.822 + 0.052
else:
seeing = dataSlice[self.seeingCol]
position_errors = np.sqrt(
mafUtils.astrom_precision(seeing, snr)**2 + self.atm_err**2)
# Construct the vectors of RA/Dec offsets. xdata is the "input data". ydata is the "output".
xdata = np.empty((2, dataSlice.size * 2), dtype=float)
xdata[0, :] = np.concatenate(
(dataSlice['ra_pi_amp'], dataSlice['dec_pi_amp']))
xdata[1, :] = np.concatenate(
(dataSlice['ra_dcr_amp'], dataSlice['dec_dcr_amp']))
ydata = np.sum(xdata, axis=0)
# Use curve_fit to compute covariance between parallax and dcr amplitudes
# Set the initial guess slightly off from the correct [1,1] to make sure it iterates.
popt, pcov = curve_fit(self._positions,
xdata,
ydata,
p0=[1.1, 0.9],
sigma=np.concatenate(
(position_errors, position_errors)),
absolute_sigma=True)
# Catch if the fit failed to converge on the correct solution.
if np.max(np.abs(popt - np.array([1., 1.]))) > self.tol:
return self.badval
# Covariance between best fit parallax amplitude and DCR amplitude.
cov = pcov[1, 0]
# Convert covarience between parallax and DCR amplitudes to normalized correlation
perr = np.sqrt(np.diag(pcov))
correlation = cov / (perr[0] * perr[1])
result = correlation
# This can throw infs.
if np.isinf(result):
result = self.badval
return result
def _calc_phase(self, dataSlice):
"""1 is perfectly balanced phase coverage, 0 is no effective coverage.
"""
angles = dataSlice[self.mjdCol] % self.period
angles = angles / self.period * 2. * np.pi
x = np.cos(angles)
y = np.sin(angles)
snr = m52snr(self.starMag, dataSlice[self.m5Col])
x_ave = np.average(x, weights=snr)
y_ave = np.average(y, weights=snr)
vector_off = np.sqrt(x_ave**2 + y_ave**2)
return 1. - vector_off
def run(self, dataslice, slicePoint=None):
filters = np.unique(dataslice[self.filterCol])
if hasattr(filters[0], 'decode'):
filters = [str(f.decode('utf-8')) for f in filters]
snr = np.zeros(len(dataslice), dtype='float')
# compute SNR for all observations
for filt in filters:
good = np.where(dataslice[self.filterCol] == filt)
snr[good] = mafUtils.m52snr(self.mags[str(filt)], dataslice[self.m5Col][good])
position_errors = np.sqrt(mafUtils.astrom_precision(dataslice[self.seeingCol],
snr)**2+self.atm_err**2)
sigma = self._final_sigma(position_errors, dataslice['ra_pi_amp'], dataslice['dec_pi_amp'])
if self.normalize:
# Leave the dec parallax as zero since one can't have ra and dec maximized at the same time.
sigma = self._final_sigma(position_errors,
dataslice['ra_pi_amp']*0+1., dataslice['dec_pi_amp']*0)/sigma
return sigma
def run(self, dataSlice, slicePoint=None):
if np.size(dataSlice) < 2:
return self.badval
filters = np.unique(dataSlice[self.filterCol])
snr = np.zeros(len(dataSlice), dtype='float')
# compute SNR for all observations
for filt in filters:
good = np.where(dataSlice[self.filterCol] == filt)
snr[good] = mafUtils.m52snr(self.mags[filt], dataSlice[self.m5Col][good])
# Compute total parallax distance
pf = np.sqrt(dataSlice['ra_pi_amp']**2+dataSlice['dec_pi_amp']**2)
# Correlation between parallax factor and hour angle
aboveLimit = np.where(snr >= self.snrLimit)[0]
if np.size(aboveLimit) < 2:
return self.badval
rho,p = spearmanr(pf[aboveLimit], dataSlice[self.haCol][aboveLimit])
return rho
def run(self, dataSlice, slicePoint=None):
if np.size(dataSlice) < 2:
return self.badval
filters = np.unique(dataSlice[self.filterCol])
snr = np.zeros(len(dataSlice), dtype='float')
# compute SNR for all observations
for filt in filters:
good = np.where(dataSlice[self.filterCol] == filt)
snr[good] = mafUtils.m52snr(self.mags[filt],
dataSlice[self.m5Col][good])
# Compute total parallax distance
pf = np.sqrt(dataSlice['ra_pi_amp']**2 + dataSlice['dec_pi_amp']**2)
# Correlation between parallax factor and hour angle
aboveLimit = np.where(snr >= self.snrLimit)[0]
if np.size(aboveLimit) < 2:
return self.badval
rho, p = spearmanr(pf[aboveLimit], dataSlice[self.haCol][aboveLimit])
return rho
def run(self, dataSlice, slicePoint=None):
result = 0
n_pts = np.size(dataSlice[self.mjdCol])
n_filt = np.size(np.unique(dataSlice[self.filterCol]))
# If we had a correct model with phase, amplitude, period, mean_mags, then chi_squared/DoF would be ~1 with 3+n_filt free parameters.
# The mean is one free parameter
p1 = n_filt
p2 = 3. + n_filt
chi_sq_2 = 1. * (n_pts - p2)
u_filters = np.unique(dataSlice[self.filterCol])
if n_pts > p2:
for period, starMag, amplitude in zip(self.periods, self.starMags,
self.amplitudes):
chi_sq_1 = 0
mags = utils.stellarMags(self.SedTemplate, rmag=starMag)
for filtername in u_filters:
in_filt = np.where(
dataSlice[self.filterCol] == filtername)[0]
lc = amplitude * np.sin(
dataSlice[self.mjdCol][in_filt] *
(np.pi * 2) / period) + mags[filtername]
snr = m52snr(lc, dataSlice[self.m5Col][in_filt])
delta_m = 2.5 * np.log10(1. + 1. / snr)
weights = 1. / (delta_m**2)
weighted_mean = np.sum(weights * lc) / np.sum(weights)
chi_sq_1 += np.sum(((lc - weighted_mean)**2 / delta_m**2))
# Yes, I'm fitting magnitudes rather than flux. At least I feel kinda bad about it.
# F-test for nested models Regression problems: https://en.wikipedia.org/wiki/F-test
f_numerator = (chi_sq_1 - chi_sq_2) / (p2 - p1)
f_denom = 1. # This is just reduced chi-squared for the more complicated model, so should be 1.
f_val = f_numerator / f_denom
# Has DoF (p2-p1, n-p2)
# https://stackoverflow.com/questions/21494141/how-do-i-do-a-f-test-in-python/21503346
p_value = scipy.stats.f.sf(f_val, p2 - p1, n_pts - p2)
if np.isfinite(p_value):
if p_value < self.sig_level:
result += 1
return result
def run(self, dataSlice, slicePoint=None):
snr = np.zeros(len(dataSlice), dtype='float')
# compute SNR for all observations
for filt in self.filters:
inFilt = np.where(dataSlice[self.filterCol] == filt)
snr[inFilt] = mafUtils.m52snr(self.mags[filt],
dataSlice[self.m5Col][inFilt])
# Compute the centroiding uncertainties
position_errors = np.sqrt(
mafUtils.astrom_precision(dataSlice[self.seeingCol], snr)**2 +
self.atm_err**2)
# Construct the vectors
xdata = np.empty((2, dataSlice.size * 2), dtype=float)
xdata[0, :] = np.concatenate(
(dataSlice['ra_pi_amp'], dataSlice['dec_pi_amp']))
xdata[1, :] = np.concatenate(
(dataSlice['ra_dcr_amp'], dataSlice['dec_dcr_amp']))
ydata = np.sum(xdata, axis=0)
# Use curve_fit to compute covariance between parallax and dcr amplitudes
# Set the initial guess slightly off from the correct [1,1] to make sure it iterates.
popt, pcov = curve_fit(self._positions,
xdata,
ydata,
p0=[1.1, 0.9],
sigma=np.concatenate(
(position_errors, position_errors)),
absolute_sigma=True)
# Catch if the fit failed to converge on the correct solution.
if np.max(np.abs(popt - np.array([1., 1.]))) > self.tol:
return self.badval
# Covariance between best fit parallax amplitude and DCR amplitude.
cov = pcov[1, 0]
# Convert covarience between parallax and DCR amplitudes to normalized correlation
perr = np.sqrt(np.diag(pcov))
correlation = cov / (perr[0] * perr[1])
result = correlation
# This can throw infs.
if np.isinf(result):
result = self.badval
return result
def run(self, dataSlice, slicePoint=None):
if np.size(dataSlice) < 2:
return self.badval
filters = np.unique(dataSlice[self.filterCol])
snr = np.zeros(len(dataSlice), dtype='float')
# compute SNR for all observations
for filt in filters:
inFilt = np.where(dataSlice[self.filterCol] == filt)
snr[inFilt] = mafUtils.m52snr(self.mags[filt], dataSlice[self.m5Col][inFilt])
weights = self._computeWeights(dataSlice, snr)
aveR = self._weightedR(dataSlice['ra_pi_amp'], dataSlice['dec_pi_amp'], weights)
if self.thetaRange > 0:
thetaCheck = self._thetaCheck(dataSlice['ra_pi_amp'], dataSlice['dec_pi_amp'], snr)
else:
thetaCheck = 1.
result = aveR*thetaCheck
return result
def run(self, dataSlice, slicePoint=None):
if np.size(dataSlice) < 2:
return self.badval
filters = np.unique(dataSlice[self.filterCol])
filters = [str(f) for f in filters]
snr = np.zeros(len(dataSlice), dtype='float')
# compute SNR for all observations
for filt in filters:
inFilt = np.where(dataSlice[self.filterCol] == filt)
snr[inFilt] = mafUtils.m52snr(self.mags[str(filt)], dataSlice[self.m5Col][inFilt])
weights = self._computeWeights(dataSlice, snr)
aveR = self._weightedR(dataSlice['ra_pi_amp'], dataSlice['dec_pi_amp'], weights)
if self.thetaRange > 0:
thetaCheck = self._thetaCheck(dataSlice['ra_pi_amp'], dataSlice['dec_pi_amp'], snr)
else:
thetaCheck = 1.
result = aveR*thetaCheck
return result
def run(self, dataSlice, slicePoint=None):
# The idea here is that we calculate position errors (in RA and Dec) for all observations.
# Then we generate arrays of the parallax offsets (delta RA parallax = ra_pi_amp, etc)
# and the DCR offsets (delta RA DCR = ra_dcr_amp, etc), and just add them together into one
# RA (and Dec) offset. Then, we try to fit for how we combined these offsets, but while
# considering the astrometric noise. If we can figure out that we just added them together
# (i.e. the curve_fit result is [a=1, b=1] for the function _positions above)
# then we should be able to disentangle the parallax and DCR offsets when fitting 'for real'.
# compute SNR for all observations
snr = np.zeros(len(dataSlice), dtype='float')
for filt in self.filters:
inFilt = np.where(dataSlice[self.filterCol] == filt)
snr[inFilt] = mafUtils.m52snr(self.mags[filt], dataSlice[self.m5Col][inFilt])
# Compute the centroiding uncertainties
# Note that these centroiding uncertainties depend on the physical size of the PSF, thus
# we are using seeingFwhmGeom for these metrics, not seeingFwhmEff.
position_errors = np.sqrt(mafUtils.astrom_precision(dataSlice[self.seeingCol], snr)**2 +
self.atm_err**2)
# Construct the vectors of RA/Dec offsets. xdata is the "input data". ydata is the "output".
xdata = np.empty((2, dataSlice.size * 2), dtype=float)
xdata[0, :] = np.concatenate((dataSlice['ra_pi_amp'], dataSlice['dec_pi_amp']))
xdata[1, :] = np.concatenate((dataSlice['ra_dcr_amp'], dataSlice['dec_dcr_amp']))
ydata = np.sum(xdata, axis=0)
# Use curve_fit to compute covariance between parallax and dcr amplitudes
# Set the initial guess slightly off from the correct [1,1] to make sure it iterates.
popt, pcov = curve_fit(self._positions, xdata, ydata, p0=[1.1, 0.9],
sigma=np.concatenate((position_errors, position_errors)),
absolute_sigma=True)
# Catch if the fit failed to converge on the correct solution.
if np.max(np.abs(popt - np.array([1., 1.]))) > self.tol:
return self.badval
# Covariance between best fit parallax amplitude and DCR amplitude.
cov = pcov[1, 0]
# Convert covarience between parallax and DCR amplitudes to normalized correlation
perr = np.sqrt(np.diag(pcov))
correlation = cov/(perr[0]*perr[1])
result = correlation
# This can throw infs.
if np.isinf(result):
result = self.badval
return result
def run(self, dataSlice, slicePoint=None):
if self.detect == True and self.time_before_peak > 0:
raise Exception(
"When detect = True, time_before_peak must be zero")
# Generate the lightcurve for this object
# make t a kind of simple way
t = dataSlice[self.mjdCol] - np.min(dataSlice[self.nightCol])
t = t - t.min()
# Try for if a blending factor slice was added if not default to no blending factor
try:
amplitudes = microlensing_amplification(
t,
impact_parameter=slicePoint['impact_parameter'],
crossing_time=slicePoint['crossing_time'],
peak_time=slicePoint['peak_time'],
blending_factor=slicePoint['blending_factor'])
except:
amplitudes = microlensing_amplification(
t,
impact_parameter=slicePoint['impact_parameter'],
crossing_time=slicePoint['crossing_time'],
peak_time=slicePoint['peak_time'])
filters = np.unique(dataSlice[self.filterCol])
amplified_mags = amplitudes * 0
for filtername in filters:
infilt = np.where(dataSlice[self.filterCol] == filtername)[0]
amplified_mags[infilt] = self.mags[filtername] - 2.5 * np.log10(
amplitudes[infilt])
# The SNR of each point in the light curve
snr = m52snr(amplified_mags, dataSlice[self.m5Col])
# The magnitude uncertainties that go with amplified mags
mag_uncert = 2.5 * np.log10(1 + 1. / snr)
n_pre = []
n_post = []
for filtername in filters:
if self.metricCalc == 'detect':
if self.time_before_peak == 'optimal':
time_before_peak_optimal = info_peak_before_t0(
slicePoint['impact_parameter'],
slicePoint['crossing_time'])
# observations pre-peak and in the given filter
infilt = np.where((dataSlice[self.filterCol] == filtername)
& (t < (slicePoint['peak_time'] -
time_before_peak_optimal)))[0]
else:
# observations pre-peak and in the given filter
infilt = np.where((dataSlice[self.filterCol] == filtername)
& (t < (slicePoint['peak_time'] -
self.time_before_peak)))[0]
# observations post-peak and in the given filter
outfilt = np.where((dataSlice[self.filterCol] == filtername)
& (t > slicePoint['peak_time']))[0]
# Broadcast to calc the mag_i - mag_j
diffs = amplified_mags[infilt] - amplified_mags[
infilt][:, np.newaxis]
diffs_uncert = np.sqrt(mag_uncert[infilt]**2 +
mag_uncert[infilt][:, np.newaxis]**2)
diffs_post = amplified_mags[outfilt] - amplified_mags[
outfilt][:, np.newaxis]
diffs_post_uncert = np.sqrt(mag_uncert[outfilt]**2 +
mag_uncert[outfilt][:,
np.newaxis]**2)
# Calculating this as a catalog-level detection. In theory,
# we could have a high SNR template image, so there would be
# little to no additional uncertianty from the subtraction.
sigma_above = np.abs(diffs) / diffs_uncert
sigma_above_post = np.abs(diffs_post) / diffs_post_uncert
# divide by 2 because array has i,j and j,i
n_above = np.size(
np.where(sigma_above > self.detect_sigma)[0]) / 2
n_pre.append(n_above)
n_above_post = np.size(
np.where(sigma_above_post > self.detect_sigma)[0]) / 2
n_post.append(n_above_post)
elif self.metricCalc == 'Npts':
# observations pre-peak and in the given filter within 2tE
infilt = np.where((dataSlice[self.filterCol] == filtername)
& (t < (slicePoint['peak_time']))
& (t > (slicePoint['peak_time'] -
slicePoint['crossing_time'])))[0]
# observations post-peak and in the given filter within 2tE
outfilt = np.where((dataSlice[self.filterCol] == filtername)
& (t > (slicePoint['peak_time']))
& (t < (slicePoint['peak_time'] +
slicePoint['crossing_time'])))[0]
n_pre.append(len(infilt))
n_post.append(len(outfilt))
npts = np.sum(n_pre)
npts_post = np.sum(n_post)
if self.metricCalc == 'detect':
if self.detect == True:
if npts >= self.ptsNeeded and npts_post >= self.ptsNeeded:
return 1
else:
return 0
else:
if npts >= self.ptsNeeded:
return 1
else:
return 0
elif self.metricCalc == 'Npts':
return npts + npts_post
def run(self, dataSlice, slicePoint=None):
# Bail if we don't have enough points
if dataSlice.size < self.means.size+3:
return self.badval
# Generate input for true light curve
lightcurvelength = dataSlice.size
np.set_printoptions(suppress=True)
t = np.empty(lightcurvelength, dtype=list(zip(['time','filter'],[float,'|U1'])))
t['time'] = (dataSlice[self.mjdCol]-dataSlice[self.mjdCol].min())
t['filter'] = dataSlice[self.filterCol]
m5 = dataSlice[self.m5Col]
lightcurvelength_days = self.time_interval
# evaluate light curves piecewise in subruns
subruns = int(np.max(t['time']) / lightcurvelength_days)
print('number of subruns: ', subruns)
fracRecovered_list=[]
for subrun_idx in range(0,subruns):
good = ( (t['time']>=(lightcurvelength_days*(subrun_idx))) & (t['time']<= (lightcurvelength_days*(subrun_idx+1))) )
t_subrun = t[good]
m5_subrun = m5[good]
if(t_subrun['time'].size>0):
# If we are adding a distance modulus to the magnitudes
if 'distMod' in list(slicePoint.keys()):
mags = self.means + slicePoint['distMod']
else:
mags = self.means
#slightly different periods and amplitudes (+/- 10 %) to mimic true stars
#random phase offsets to mimic observation starting at random phase
true_period=random.uniform(0.9,1.1)*self.period
true_amplitude=random.uniform(0.9,1.1)*self.amplitude
if(np.isnan(self.phase)):
#a random phase (in days) should be assigned
true_phase=random.uniform(0,1)*self.period
else:
true_phase = self.phase
trueParams = np.append(np.array([true_period, true_phase, true_amplitude]), mags)
true_obj = periodicStar(t_subrun['filter'])
trueLC = true_obj(t_subrun['time'], *trueParams)
# Array to hold the fit results
fits = np.zeros((self.nMonte,trueParams.size),dtype=float)
for i in np.arange(self.nMonte):
snr = m52snr(trueLC,m5_subrun)
dmag = 2.5*np.log10(1.+1./snr)
noise = np.random.randn(trueLC.size)*dmag
# Suppress warnings about failing on covariance
fit_obj = periodicStar(t_subrun['filter'])
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# If it fails to converge, save values that should fail later
try:
parmVals, pcov = curve_fit(fit_obj, t_subrun['time'], trueLC+noise, p0=trueParams,
sigma=dmag)
except:
parmVals = trueParams*0+np.inf
fits[i,:] = parmVals
# Throw out any magnitude fits if there are no observations in that filter
ufilters = np.unique(dataSlice[self.filterCol])
if ufilters.size < 9:
for key in list(self.filter2index.keys()):
if key not in ufilters:
fits[:,self.filter2index[key]] = -np.inf
# Find the fraction of fits that meet the "well-fit" criteria
periodFracErr = np.abs((fits[:,0]-trueParams[0])/trueParams[0])
ampFracErr = np.abs((fits[:,2]-trueParams[2])/trueParams[2])
magErr = np.abs(fits[:,3:]-trueParams[3:])
nBands = np.zeros(magErr.shape,dtype=int)
nBands[np.where(magErr <= self.magTol)] = 1
nBands = np.sum(nBands, axis=1)
nRecovered = np.size(np.where( (periodFracErr <= self.periodTol) &
(ampFracErr <= self.ampTol) &
(nBands >= self.nBands) )[0])
fracRecovered = float(nRecovered)/self.nMonte
fracRecovered_list.append(fracRecovered)
fracRecovered = np.sum(fracRecovered_list)/(len(fracRecovered_list))
return fracRecovered
def run(self, dataSlice, slicePoint=None):
""""Calculate the detectability of a transient with the specified lightcurve.
If self.dataout is True, then returns the full lightcurve for each object instead of the total
number of transients that are detected.
Parameters
----------
dataSlice : numpy.array
Numpy structured array containing the data related to the visits provided by the slicer.
slicePoint : dict, optional
Dictionary containing information about the slicepoint currently active in the slicer.
Returns
-------
float or list of dicts
The total number of transients that could be detected. (if dataout is False)
A dictionary with arrays of 'lcNumber', 'lcMag', 'detected', 'time', 'detectThresh', 'filter'
"""
# Sort the entire dataSlice in order of time.
dataSlice.sort(order=self.mjdCol)
tSpan = (dataSlice[self.mjdCol].max() - dataSlice[self.mjdCol].min()
) # in days
lcv_template = self.lcv_template
transDuration = lcv_template['ph'].max() - lcv_template['ph'].min(
) # in days
# phase check
tshifts = np.arange(self.nPhaseCheck) * transDuration / float(
self.nPhaseCheck)
lcNumber = np.floor(
(dataSlice[self.mjdCol] - dataSlice[self.mjdCol].min()) /
transDuration)
ulcNumber = np.unique(lcNumber)
nTransMax = 0
nDetected = 0
dataout_dict_list = []
for tshift in tshifts:
#print('check tshift ', tshift)
lcEpoch = np.fmod(
dataSlice[self.mjdCol] - dataSlice[self.mjdCol].min() + tshift,
transDuration) + self.epochStart
# total number of transients possibly detected
nTransMax += np.ceil(tSpan / transDuration)
# generate the actual light curve
lcFilters = dataSlice[self.filterCol]
lcMags = self.make_lightCurve(lcEpoch, lcFilters)
lcSNR = utils.m52snr(lcMags, dataSlice[self.m5Col])
# Identify detections above SNR for each filter
lcAboveThresh = np.zeros(len(lcSNR), dtype=bool)
for f in np.unique(lcFilters):
filtermatch = np.where(dataSlice[self.filterCol] == f)
lcAboveThresh[filtermatch] = np.where(
lcSNR[filtermatch] >= self.detectSNR[f], True, False)
# check conditions for each light curve
lcDetect = np.ones(len(ulcNumber), dtype=bool)
lcDetectOut = np.ones(len(lcNumber), dtype=bool)
for i, lcN in enumerate(ulcNumber):
lcN_idx = np.where(lcNumber == lcN)
lcEpoch_i = lcEpoch[lcN_idx]
lcMags_i = lcMags[lcN_idx]
lcFilters_i = lcFilters[lcN_idx]
lcAboveThresh_i = lcAboveThresh[lcN_idx]
#check total number of observations for each band
for f in np.unique(lcFilters_i):
f_Idx = np.where(lcFilters_i == f)
if len(np.where(
lcAboveThresh_i[f_Idx])[0]) < self.nObsTotal[f]:
lcDetect[i] = False
lcDetectOut[lcN_idx] = False
# number of observations before peak
prePeakCheck = (lcEpoch_i <
self.peakEpoch - self.nearPeakT / 2)
prePeakIdx = np.where(prePeakCheck == True)
if len(np.where(
lcAboveThresh_i[prePeakIdx])[0]) < self.nObsPrePeak:
lcDetect[i] = False
lcDetectOut[lcN_idx] = False
# check number of observations near peak for each band
nearPeakCheck = (
lcEpoch_i >= self.peakEpoch - self.nearPeakT / 2) & (
lcEpoch_i <= self.peakEpoch + self.nearPeakT / 2)
nearPeakIdx = np.where(nearPeakCheck == True)
for f in np.unique(lcFilters_i):
nearPeakIdx_f = np.intersect1d(nearPeakIdx,
np.where(lcFilters_i == f))
if len(np.where(lcAboveThresh_i[nearPeakIdx_f])
[0]) < self.nObsNearPeak[f]:
lcDetect[i] = False
lcDetectOut[lcN_idx] = False
# check number of filters near peak
filtersNearPeakIdx = np.intersect1d(
nearPeakIdx,
np.where(lcAboveThresh_i)[0])
if len(np.unique(lcFilters_i[filtersNearPeakIdx])
) < self.nFiltersNearPeak:
lcDetect[i] = False
lcDetectOut[lcN_idx] = False
## check number of observations post peak
# postPeakCheck
postPeakCheck = (lcEpoch_i >= self.peakEpoch + self.nearPeakT /
2) & (lcEpoch_i <= self.peakEpoch +
self.nearPeakT / 2 + self.postPeakT)
postPeakIdx = np.where(postPeakCheck == True)
if len(np.where(
lcAboveThresh_i[postPeakIdx])[0]) < self.nObsPostPeak:
lcDetect[i] = False
lcDetectOut[lcN_idx] = False
# check number of filters post peak
filtersPostPeakIdx = np.intersect1d(
postPeakIdx,
np.where(lcAboveThresh_i)[0])
if len(np.unique(lcFilters_i[filtersPostPeakIdx])
) < self.nFiltersPostPeak:
lcDetect[i] = False
lcDetectOut[lcN_idx] = False
# return values
nDetected += len(np.where(lcDetect == True)[0])
prePeakCheck = (lcEpoch <= self.peakEpoch - self.nearPeakT / 2)
nearPeakCheck = (lcEpoch >=
(self.peakEpoch - self.nearPeakT / 2)) & (
lcEpoch <=
(self.peakEpoch + self.nearPeakT / 2))
postPeakCheck = (
lcEpoch >= (self.peakEpoch + self.nearPeakT / 2)) & (
lcEpoch <=
(self.peakEpoch + self.nearPeakT / 2 + self.postPeakT))
#print(nTransMax, nDetected, lcDetect)
dataout_dict_tshift = {
'tshift': np.repeat(tshift, len(lcEpoch)),
'expMJD': dataSlice[self.mjdCol],
'm5': dataSlice[self.m5Col],
'filters': dataSlice[self.filterCol],
'lcNumber': lcNumber,
'lcEpoch': lcEpoch,
'prePeakCheck': prePeakCheck,
'nearPeakCheck': nearPeakCheck,
'postPeakCheck': postPeakCheck,
'lcMags': lcMags,
'lcSNR': lcSNR,
'lcMagsStd': self.snr2std(lcSNR),
'lcAboveThresh': lcAboveThresh,
'detected': lcDetectOut
}
dataout_dict_list.append(dataout_dict_tshift)
if self.dataout:
return dataout_dict_list
else:
return float(nDetected / nTransMax) if nTransMax != 0 else 0.
def run(self, dataSlice, slicePoint=None):
""""Calculate the detectability of a transient with the specified lightcurve.
If self.dataout is True, then returns the full lightcurve for each object instead of the total
number of transients that are detected.
Parameters
----------
dataSlice : numpy.array
Numpy structured array containing the data related to the visits provided by the slicer.
slicePoint : dict, optional
Dictionary containing information about the slicepoint currently active in the slicer.
Returns
-------
float or list of dicts
The total number of transients that could be detected. (if dataout is False)
A dictionary with arrays of 'lcNumber', 'lcMag', 'detected', 'time', 'detectThresh', 'filter'
"""
# Sort the entire dataSlice in order of time.
dataSlice.sort(order=self.mjdCol)
survey_length = (dataSlice[self.mjdCol].max() -
dataSlice[self.mjdCol].min()) # in days
lcv_template = self.lcv_template
transDuration = lcv_template['ph'].max() - lcv_template['ph'].min(
) # in days
# how many event occured
nLc = np.random.poisson(self.eventRate * survey_length)
# generate nLc random start time of each light curve
t0 = np.random.randint(0,
int(survey_length) + 1,
nLc) + dataSlice[self.mjdCol].min()
# dict to store output info
lcDictList = []
nDetected = 0
# loop over each light curve
for i, t0_i in enumerate(t0):
# the index for ith light curve
lcIdx = (dataSlice[self.mjdCol] >=
t0_i) & (dataSlice[self.mjdCol] <= t0_i + transDuration)
lcMjd = dataSlice[self.mjdCol][lcIdx]
lcEpoch = lcMjd - t0_i + self.epochStart
lcFilters = dataSlice[self.filterCol][lcIdx]
# make light curve
lcMags = self.make_lightCurve(lcEpoch, lcFilters)
# get SNR
m5 = dataSlice[self.m5Col][lcIdx]
lcSNR = utils.m52snr(lcMags, m5)
# check SNR for each filter
lcAboveThresh = np.zeros(len(lcSNR), dtype=bool)
for f in np.unique(lcFilters):
filtermatch = np.where(lcFilters == f)
lcAboveThresh[filtermatch] = np.where(
lcSNR[filtermatch] >= self.detectSNR[f], True, False)
# ----------check all conditions-----------
# first assume lcDetect = True, if one condition fails, set to False
lcDetect = True
# check total number of observations for each band
for f in np.unique(lcFilters):
filtermatch = np.where(lcFilters == f)
if len(np.where(
lcAboveThresh[filtermatch])[0]) < self.nObsTotal[f]:
lcDetect = False
# number of observations before peak
prePeakCheck = (lcEpoch < self.peakEpoch - self.nearPeakT / 2)
prePeakIdx = np.where(prePeakCheck == True)
if len(np.where(lcAboveThresh[prePeakIdx])[0]) < self.nObsPrePeak:
lcDetect = False
# check number of observations near peak for each band
nearPeakCheck = (
lcEpoch >= self.peakEpoch - self.nearPeakT / 2) & (
lcEpoch <= self.peakEpoch + self.nearPeakT / 2)
nearPeakIdx = np.where(nearPeakCheck == True)
# near peak obs for each band
for f in np.unique(lcFilters):
nearPeakIdx_f = np.intersect1d(nearPeakIdx,
np.where(lcFilters == f))
if len(np.where(lcAboveThresh[nearPeakIdx_f])
[0]) < self.nObsNearPeak[f]:
lcDetect = False
# check number of filters near peak
filtersNearPeakIdx = np.intersect1d(nearPeakIdx,
np.where(lcAboveThresh)[0])
if len(np.unique(
lcFilters[filtersNearPeakIdx])) < self.nFiltersNearPeak:
lcDetect = False
## check number of observations post peak
# postPeakCheck
postPeakCheck = (lcEpoch >= self.peakEpoch + self.nearPeakT / 2
) & (lcEpoch <= self.peakEpoch +
self.nearPeakT / 2 + self.postPeakT)
postPeakIdx = np.where(postPeakCheck == True)
if len(np.where(
lcAboveThresh[postPeakIdx])[0]) < self.nObsPostPeak:
lcDetect = False
# check number of filters post peak
filtersPostPeakIdx = np.intersect1d(postPeakIdx,
np.where(lcAboveThresh)[0])
if len(np.unique(
lcFilters[filtersPostPeakIdx])) < self.nFiltersPostPeak:
lcDetect = False
# ----------------------
# values for output
if lcDetect == True:
nDetected += 1
lcDict = {
'lcN': i,
'lcMjd': lcMjd,
'lcEpoch': lcEpoch,
'lcFilters': lcFilters,
'lcMags': lcMags,
'm5': m5,
'lcSNR': lcSNR,
'lcMagsStd': self.snr2std(lcSNR),
'lcAboveThresh': lcAboveThresh,
'prePeakCheck': prePeakCheck,
'nearPeakCheck': nearPeakCheck,
'postPeakCheck': postPeakCheck,
'detected': lcDetect
}
lcDictList.append(lcDict)
if self.dataout:
return lcDictList
else:
#return float(nDetected / nTransMax) if nTransMax!=0 else 0.
return float(nDetected / nLc) if nLc != 0 else 0.
def run(self, dataSlice, slicePoint=None):
# Bail if we don't have enough points
if dataSlice.size < self.means.size + 3:
return self.badval
# Generate input for true light curve
t = np.empty(dataSlice.size, dtype=zip(["time", "filter"], [float, "|S1"]))
t["time"] = dataSlice[self.mjdCol] - dataSlice[self.mjdCol].min()
t["filter"] = dataSlice[self.filterCol]
# If we are adding a distance modulus to the magnitudes
if "distMod" in slicePoint.keys():
mags = self.means + slicePoint["distMod"]
else:
mags = self.means
trueParams = np.append(np.array([self.period, self.phase, self.amplitude]), mags)
trueLC = periodicStar(t, *trueParams)
# Array to hold the fit results
fits = np.zeros((self.nMonte, trueParams.size), dtype=float)
# generate phase array up-front
phaseMC = np.repeat(self.phase, self.nMonte)
if self.randomisePhase:
phaseMC = np.random.uniform(size=self.nMonte) * self.period
# Set up object to hold the luminosity function. Find the
# interpolation function before doing the monte carlo trials
# (since the fit to the LF is only dependent on the
# location). Also populate the seeing
PhotCrowd = confusion.CrowdingSigma(dataSlice, slicePoint)
PhotCrowd.getErrorFuncAndSeeing()
# We only have crowding information for r-band.
bCanCrowd = t["filter"] == "r" # can add more conditions here
gCanCrowd = np.where(bCanCrowd)
# WIC - be a bit stricter with outliers
if np.size(gCanCrowd) < 80:
return self.badval
# Loop through the Monte Carlo trials
for i in np.arange(self.nMonte):
# Copy the parameters, slot in the phase (randomized or not)
trialParams = np.copy(trueParams)
trialParams[1] = phaseMC[i]
# Generate the "clean" lightcurve for this trial (incl randomized phase)
trialLC = periodicStar(t, *trialParams)
# Estimate the photometric uncertainty.
snr = m52snr(trialLC, dataSlice[self.m5Col])
sigmPhot = 2.5 * np.log10(1.0 + 1.0 / snr)
# print "DEBUG:", np.size(sigmPhot), np.size(trialLC)
# Now for crowding uncertainty. Pass the current set of
# magnitudes to the Crowd object, estimate the sigmCrowd
# at each point. At this date (2016-01-03) I still think
# it's better to use the "true" magnitude rather than
# rather than perturbing by photometric noise first.
PhotCrowd.magSamples = np.copy(trialLC)
PhotCrowd.calcSigmaSeeingFromInterp()
sigmCrowd = np.copy(PhotCrowd.sigmaWithSeeing)
# Now apply both sources of error in succession. I am not
# convinced the quad sum is correct in this situation, but
# we have to start somewhere...
photLC = trialLC + np.random.randn(trialLC.size) * sigmPhot
if not self.ignoreCrowding:
bothLC = photLC + np.random.randn(photLC.size) * sigmCrowd
sigmBoth = np.sqrt(sigmPhot ** 2 + sigmCrowd ** 2)
else:
bothLC = np.copy(photLC)
sigmaBoth = np.copy(sigmaPhot)
if i < 1 and self.beVerbose:
ThisRA = slicePoint["ra"] * 180.0 / np.pi
ThisDE = slicePoint["dec"] * 180.0 / np.pi
medLC = np.median(bothLC[gCanCrowd])
medTr = np.median(trialLC[gCanCrowd])
medCr = np.median(sigmCrowd[gCanCrowd])
stdCr = np.std(sigmCrowd[gCanCrowd])
# Update - not convinced crowding metric is being
# correctly applied...
seeMed = np.median(PhotCrowd.vecSeeing[gCanCrowd])
medPh = np.median(sigmPhot[gCanCrowd])
stdTo = np.std(sigmBoth)
lcPho = np.std(photLC[gCanCrowd])
lcBot = np.std(bothLC[gCanCrowd])
print "INFO: %4i %.2f %.2f, %.3f %.3f, %.3f, %.4f %.4f ; %.4f; %.3f %.3f" % (
np.size(gCanCrowd),
ThisRA,
ThisDE,
medLC,
medTr,
seeMed,
medPh,
medCr,
stdCr,
lcPho,
lcBot,
)
# At this point we search for the periodic signal.
# WIC 2016-01-03 - I *think* this should work OK even
# though we only have a subset of datapoints that can be
# useful (i.e. at the right filter). Just limit the points
# fed to curve_fit to those for which we have crowding
# information.
### noise = np.random.randn(trueLC.size)*dmag
# Suppress warnings about failing on covariance
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# If it fails to converge, save values that should fail later
try:
parmVals, pcov = curve_fit(
periodicStar, t[gCanCrowd], bothLC[gCanCrowd], p0=trueParams, sigma=sigmBoth[gCanCrowd]
)
except:
parmVals = trueParams * 0 - 666
fits[i, :] = parmVals
# Throw out any magnitude fits if there are no observations in that filter
ufilters = np.unique(dataSlice[self.filterCol])
if ufilters.size < 9:
for key in self.filter2index.keys():
if key not in ufilters:
fits[:, self.filter2index[key]] = -np.inf
# Find the fraction of fits that meet the "well-fit" criteria
periodFracErr = (fits[:, 0] - trueParams[0]) / trueParams[0]
ampFracErr = (fits[:, 2] - trueParams[2]) / trueParams[2]
magErr = fits[:, 3:] - trueParams[3:]
nBands = np.zeros(magErr.shape, dtype=int)
nBands[np.where(magErr <= self.magTol)] = 1
nBands = np.sum(nBands, axis=1)
nRecovered = np.size(
np.where((periodFracErr <= self.periodTol) & (ampFracErr <= self.ampTol) & (nBands >= self.nBands))[0]
)
fracRecovered = float(nRecovered) / self.nMonte
return fracRecovered
def run(self, dataSlice, slicePoint=None):
np.random.seed(2500)
obs = np.where((dataSlice['filter'] == self.f) &
(dataSlice[self.mjdCol] < min(dataSlice[self.mjdCol]) +
365 * self.surveyduration))
d = np.array(self.simobj['d'])
M = np.array(self.simobj['MAG'])
fieldRA, fieldDec = np.mean(dataSlice['fieldRA']), np.mean(
dataSlice['fieldDec'])
z, R = d * np.sin(fieldRA), d * np.cos(fieldRA)
component = self.position_selection(R, z)
mjd = dataSlice[self.mjdCol][obs]
fwhm = dataSlice[self.seeingCol][obs]
V_galactic = np.vstack((self.U, self.V, self.W))
Pv = self.DF(V_galactic, component, R, z)
marg_P = np.nanmean(Pv / np.nansum(Pv, axis=0), axis=0)
marg_P /= np.nansum(marg_P)
vel_idx = np.random.choice(np.arange(0, len(V_galactic[0, :]),
1)[np.isfinite(marg_P)],
p=marg_P[np.isfinite(marg_P)],
size=3)
vT_unusual = V_galactic[0, vel_idx][2]
if self.prob_type == 'uniform':
p_vel_unusual = uniform(-100, 100)
v_unusual = p_vel_unusual.rvs(size=(3, np.size(d)))
vT = v_unusual[2, :]
else:
p_vel_un = pd.read_csv(self.prob_type)
vel_idx = np.random.choice(p_vel_un['vel'],
p=p_vel_un['fraction'] /
np.sum(p_vel_un['fraction']),
size=3)
vT_unusual = V_galactic[0, vel_idx][2]
#vel_unusual = V_galactic[0,vel_idx]
direction = np.random.choice((-1, 1))
mu = direction * vT / 4.75 / d
mu_unusual = direction * vT_unusual / 4.75 / d
if len(dataSlice[self.m5Col][obs]) > 2:
# select objects above the limit magnitude threshold
snr = m52snr(M[:, np.newaxis], dataSlice[self.m5Col][obs])
row, col = np.where(snr > self.snr_lim)
if self.gap_selection:
Times = np.sort(mjd)
dt = np.array(list(combinations(Times, 2)))
DeltaTs = np.absolute(np.subtract(dt[:, 0], dt[:, 1]))
DeltaTs = np.unique(DeltaTs)
if np.size(DeltaTs) > 0:
dt_pm = 0.05 * np.amin(
dataSlice[self.seeingCol][obs]) / muf[np.unique(row)]
selection = np.where((dt_pm > min(DeltaTs))
& (dt_pm < max(DeltaTs)))
else:
selection = np.unique(row)
precis = astrom_precision(dataSlice[self.seeingCol][obs],
snr[row, :])
sigmapm = self.sigma_slope(dataSlice[self.mjdCol][obs],
precis) * 365.25 * 1e3
if np.size(selection) > 0:
pa = np.random.uniform(0, 2 * np.pi,
len(mu_unusual[selection]))
pm_alpha, pm_delta = mu[selection] * np.sin(
pa), mu[selection] * np.cos(pa)
pm_un_alpha, pm_un_delta = mu_unusual[selection] * np.sin(
pa), mu_unusual[selection] * np.cos(pa)
#p_min,p_max,p_mean = self.percentiles[0],self.percentiles[1],self.percentiles[2]
mu = mu[selection] * 1e3
mu_unusual = mu_unusual[selection] * 1e3
variance_k = np.array([
np.std(mu[np.where(component[selection] == p)])
for p in ['H', 'D', 'B']
])
variance_mu = np.std(mu)
sigmaL = np.sqrt(
np.prod(variance_k, where=np.isfinite(variance_k))**2 +
variance_mu**2 + np.nanmedian(sigmapm)**2)
unusual = np.where((mu_unusual < np.mean(mu_unusual) -
self.sigma_threshold * sigmaL / 2)
| (mu_unusual > np.mean(mu_unusual) +
self.sigma_threshold * sigmaL / 2))
res = np.size(unusual) / np.size(selection)
if self.dataout:
dic = {
'detected':
res,
'pixID':
radec2pix(nside=16,
ra=np.radians(fieldRA),
dec=np.radians(fieldDec)),
'PM':
pd.DataFrame({
'pm_alpha': pm_alpha,
'pm_delta': pm_delta
}),
'PM_un':
pd.DataFrame({
'pm_alpha': pm_un_alpha,
'pm_delta': pm_un_delta
})
}
return dic
else:
return res
def run(self, dataSlice, slicePoint=None):
""""Calculate the detectability of a transient with the specified lightcurve.
If self.dataout is True, then returns the full lightcurve for each object instead of the total
number of transients that are detected.
Parameters
----------
dataSlice : numpy.array
Numpy structured array containing the data related to the visits provided by the slicer.
slicePoint : dict, optional
Dictionary containing information about the slicepoint currently active in the slicer.
Returns
-------
float or dict
The total number of transients that could be detected. (if dataout is False)
A dictionary with arrays of 'lcNumber', 'lcMag', 'detected', 'time', 'detectThresh', 'filter'
"""
# Sort the entire dataSlice in order of time.
dataSlice.sort(order=self.mjdCol)
# Check that surveyDuration is not larger than the time of observations we obtained.
# (if it is, then the nTransMax will not be accurate).
tSpan = (dataSlice[self.mjdCol].max() - dataSlice[self.mjdCol].min()) / 365.25
surveyDuration = np.max([tSpan, self.surveyDuration])
if self.surveyStart is None:
surveyStart = dataSlice[self.mjdCol].min()
else:
surveyStart = self.surveyStart
# Set up the starting times for each of the back-to-back sets of transients.
tshifts = np.arange(self.nPhaseCheck) * self.transDuration / float(self.nPhaseCheck)
# Total number of transient which have reached detection threshholds.
nDetected = 0
# Total number of transients which could possibly be detected,
# given survey duration and transient duration.
nTransMax = 0
# Set this, in case surveyStart was set to be much earlier than this data (so we start counting at 0).
lcNumberStart = -1 * np.floor((dataSlice[self.mjdCol].min() - surveyStart) / self.transDuration)
# Consider each different 'phase shift' separately.
# We then just have a series of lightcurves, taking place back-to-back.
for tshift in tshifts:
# Update the maximum possible transients that could have been observed during surveyDuration.
nTransMax += np.ceil(surveyDuration / (self.transDuration / 365.25))
# Calculate the time/epoch for each lightcurve.
lcEpoch = (dataSlice[self.mjdCol] - surveyStart + tshift) % self.transDuration
# Identify the observations which belong to each distinct light curve.
lcNumber = np.floor((dataSlice[self.mjdCol] - surveyStart) / self.transDuration) + lcNumberStart
lcNumberStart = lcNumber.max()
ulcNumber = np.unique(lcNumber)
lcLeft = np.searchsorted(lcNumber, ulcNumber, side='left')
lcRight = np.searchsorted(lcNumber, ulcNumber, side='right')
# Generate the actual light curve magnitudes and SNR
lcMags = self.make_lightCurve(lcEpoch, dataSlice[self.filterCol])
lcSNR = m52snr(lcMags, dataSlice[self.m5Col])
# Identify which detections rise above the required SNR threshhold, in each filter.
lcAboveThresh = np.zeros(len(lcSNR), dtype=bool)
for f in np.unique(dataSlice[self.filterCol]):
filtermatch = np.where(dataSlice[self.filterCol] == f)[0]
lcAboveThresh[filtermatch] = np.where(lcSNR[filtermatch] >= self.detectSNR[f],
True,
False)
# Track whether each individual light curve was detected.
# Start with the assumption that it is True, and if it fails criteria then becomes False.
lcDetect = np.ones(len(ulcNumber), dtype=bool)
# Loop through each lightcurve and check if it meets requirements.
for lcN, le, ri in zip(ulcNumber, lcLeft, lcRight):
# If there were no observations at all for this lightcurve:
if le == ri:
lcDetect[lcN] = False
# Skip the rest of this loop, go on to the next lightcurve.
continue
lcEpochAboveThresh = lcEpoch[le:ri][np.where(lcAboveThresh[le:ri])]
# If we did not get enough detections before preT, set lcDetect to False.
timesPreT = np.where(lcEpochAboveThresh < self.preT)[0]
if len(timesPreT) < self.nPreT:
lcDetect[lcN] = False
continue
# If we did not get detections over enough sections of the lightcurve, set lcDtect to False.
phaseSections = np.unique(np.floor(lcEpochAboveThresh / self.transDuration * self.nPerLC))
if len(phaseSections) < self.nPerLC:
lcDetect[lcN] = False
continue
# If we did not get detections in enough filters, set lcDetect to False.
lcFilters = dataSlice[le:ri][np.where(lcAboveThresh[le:ri])][self.filterCol]
if len(np.unique(lcFilters)) < self.nFilters:
lcDetect[lcN] = False
continue
# If we did not get detections in enough filters within required time, set lcDetect to False.
if (self.filterT is not None) and (self.nFilters > 1):
xr = np.searchsorted(lcEpochAboveThresh, lcEpochAboveThresh + self.filterT, 'right')
xr = np.where(xr < len(lcEpochAboveThresh) - 1, xr, len(lcEpochAboveThresh) - 1)
foundGood = False
for i, xri in enumerate(xr):
if len(np.unique(lcFilters[i:xri])) >= self.nFilters:
foundGood = True
break
if not foundGood:
lcDetect[lcN] = False
continue
# Done with current set of conditions.
# (more complicated conditions should go later in the loop, simpler ones earlier).
# Find the unique number of light curves that passed the required number of conditions
nDetected += len(np.where(lcDetect == True)[0])
if self.dataout:
# Output all the light curves, regardless of detection threshhold,
# but indicate which were 'detected'.
lcDetectOut = np.ones(len(dataSlice), dtype=bool)
for i, lcN in enumerate(lcNumber):
lcDetectOut[i] = lcDetect[lcN]
return {'lcNumber': lcNumber, 'expMJD': dataSlice[self.mjdCol], 'epoch': lcEpoch,
'filter': dataSlice[self.filterCol], 'lcMag': lcMags, 'SNR': lcSNR,
'detected': lcDetectOut}
else:
return float(nDetected) / nTransMax
def run(self, dataSlice, slicePoint=None):
# Bail if we don't have enough points
if dataSlice.size < self.means.size + 3:
return self.badval
# Generate input for true light curve
t = np.empty(dataSlice.size,
dtype=zip(['time', 'filter'], [float, '|S1']))
t['time'] = dataSlice[self.mjdCol] - dataSlice[self.mjdCol].min()
t['filter'] = dataSlice[self.filterCol]
# If we are adding a distance modulus to the magnitudes
if 'distMod' in slicePoint.keys():
mags = self.means + slicePoint['distMod']
else:
mags = self.means
trueParams = np.append(
np.array([self.period, self.phase, self.amplitude]), mags)
trueLC = periodicStar(t, *trueParams)
# Array to hold the fit results
fits = np.zeros((self.nMonte, trueParams.size), dtype=float)
# generate phase array up-front
phaseMC = np.repeat(self.phase, self.nMonte)
if self.randomisePhase:
phaseMC = np.random.uniform(size=self.nMonte) * self.period
# Set up object to hold the luminosity function. Find the
# interpolation function before doing the monte carlo trials
# (since the fit to the LF is only dependent on the
# location). Also populate the seeing
PhotCrowd = confusion.CrowdingSigma(dataSlice, slicePoint)
PhotCrowd.getErrorFuncAndSeeing()
# We only have crowding information for r-band.
bCanCrowd = t['filter'] == "r" # can add more conditions here
gCanCrowd = np.where(bCanCrowd)
# WIC - be a bit stricter with outliers
if np.size(gCanCrowd) < 80:
return self.badval
# Loop through the Monte Carlo trials
for i in np.arange(self.nMonte):
# Copy the parameters, slot in the phase (randomized or not)
trialParams = np.copy(trueParams)
trialParams[1] = phaseMC[i]
# Generate the "clean" lightcurve for this trial (incl randomized phase)
trialLC = periodicStar(t, *trialParams)
# Estimate the photometric uncertainty.
snr = m52snr(trialLC, dataSlice[self.m5Col])
sigmPhot = 2.5 * np.log10(1. + 1. / snr)
#print "DEBUG:", np.size(sigmPhot), np.size(trialLC)
# Now for crowding uncertainty. Pass the current set of
# magnitudes to the Crowd object, estimate the sigmCrowd
# at each point. At this date (2016-01-03) I still think
# it's better to use the "true" magnitude rather than
# rather than perturbing by photometric noise first.
PhotCrowd.magSamples = np.copy(trialLC)
PhotCrowd.calcSigmaSeeingFromInterp()
sigmCrowd = np.copy(PhotCrowd.sigmaWithSeeing)
# Now apply both sources of error in succession. I am not
# convinced the quad sum is correct in this situation, but
# we have to start somewhere...
photLC = trialLC + np.random.randn(trialLC.size) * sigmPhot
if not self.ignoreCrowding:
bothLC = photLC + np.random.randn(photLC.size) * sigmCrowd
sigmBoth = np.sqrt(sigmPhot**2 + sigmCrowd**2)
else:
bothLC = np.copy(photLC)
sigmaBoth = np.copy(sigmaPhot)
if i < 1 and self.beVerbose:
ThisRA = slicePoint['ra'] * 180. / np.pi
ThisDE = slicePoint['dec'] * 180. / np.pi
medLC = np.median(bothLC[gCanCrowd])
medTr = np.median(trialLC[gCanCrowd])
medCr = np.median(sigmCrowd[gCanCrowd])
stdCr = np.std(sigmCrowd[gCanCrowd])
# Update - not convinced crowding metric is being
# correctly applied...
seeMed = np.median(PhotCrowd.vecSeeing[gCanCrowd])
medPh = np.median(sigmPhot[gCanCrowd])
stdTo = np.std(sigmBoth)
lcPho = np.std(photLC[gCanCrowd])
lcBot = np.std(bothLC[gCanCrowd])
print "INFO: %4i %.2f %.2f, %.3f %.3f, %.3f, %.4f %.4f ; %.4f; %.3f %.3f" \
% (np.size(gCanCrowd), ThisRA, ThisDE, medLC, medTr, seeMed, medPh, medCr, stdCr, lcPho, lcBot)
# At this point we search for the periodic signal.
# WIC 2016-01-03 - I *think* this should work OK even
# though we only have a subset of datapoints that can be
# useful (i.e. at the right filter). Just limit the points
# fed to curve_fit to those for which we have crowding
# information.
### noise = np.random.randn(trueLC.size)*dmag
# Suppress warnings about failing on covariance
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# If it fails to converge, save values that should fail later
try:
parmVals, pcov = curve_fit(periodicStar, \
t[gCanCrowd], bothLC[gCanCrowd], \
p0=trueParams, \
sigma=sigmBoth[gCanCrowd])
except:
parmVals = trueParams * 0 - 666
fits[i, :] = parmVals
# Throw out any magnitude fits if there are no observations in that filter
ufilters = np.unique(dataSlice[self.filterCol])
if ufilters.size < 9:
for key in self.filter2index.keys():
if key not in ufilters:
fits[:, self.filter2index[key]] = -np.inf
# Find the fraction of fits that meet the "well-fit" criteria
periodFracErr = (fits[:, 0] - trueParams[0]) / trueParams[0]
ampFracErr = (fits[:, 2] - trueParams[2]) / trueParams[2]
magErr = fits[:, 3:] - trueParams[3:]
nBands = np.zeros(magErr.shape, dtype=int)
nBands[np.where(magErr <= self.magTol)] = 1
nBands = np.sum(nBands, axis=1)
nRecovered = np.size(
np.where((periodFracErr <= self.periodTol)
& (ampFracErr <= self.ampTol)
& (nBands >= self.nBands))[0])
fracRecovered = float(nRecovered) / self.nMonte
return fracRecovered
def run(self, dataSlice, slicePoint=None):
""""Calculate the detectability of a transient with the specified lightcurve.
If self.dataout is True, then returns the full lightcurve for each object instead of the total
number of transients that are detected.
Parameters
----------
dataSlice : numpy.array
Numpy structured array containing the data related to the visits provided by the slicer.
slicePoint : dict, optional
Dictionary containing information about the slicepoint currently active in the slicer.
Returns
-------
float or dict
The total number of transients that could be detected. (if dataout is False)
A dictionary with arrays of 'lcNumber', 'lcMag', 'detected', 'time', 'detectThresh', 'filter'
"""
# Sort the entire dataSlice in order of time.
dataSlice.sort(order=self.mjdCol)
# Check that surveyDuration is not larger than the time of observations we obtained.
# (if it is, then the nTransMax will not be accurate).
tSpan = (dataSlice[self.mjdCol].max() - dataSlice[self.mjdCol].min()) / 365.25
surveyDuration = np.max([tSpan, self.surveyDuration])
if self.surveyStart is None:
surveyStart = dataSlice[self.mjdCol].min()
else:
surveyStart = self.surveyStart
# Set up the starting times for each of the back-to-back sets of transients.
tshifts = np.arange(self.nPhaseCheck) * self.transDuration / float(self.nPhaseCheck)
# Total number of transient which have reached detection threshholds.
nDetected = 0
# Total number of transients which could possibly be detected,
# given survey duration and transient duration.
nTransMax = 0
# Set this, in case surveyStart was set to be much earlier than this data (so we start counting at 0).
lcNumberStart = -1 * np.floor((dataSlice[self.mjdCol].min() - surveyStart) / self.transDuration)
# Consider each different 'phase shift' separately.
# We then just have a series of lightcurves, taking place back-to-back.
for tshift in tshifts:
# Update the maximum possible transients that could have been observed during surveyDuration.
nTransMax += np.ceil(surveyDuration / (self.transDuration / 365.25))
# Calculate the time/epoch for each lightcurve.
lcEpoch = (dataSlice[self.mjdCol] - surveyStart + tshift) % self.transDuration
# Identify the observations which belong to each distinct light curve.
lcNumber = np.floor((dataSlice[self.mjdCol] - surveyStart) / self.transDuration) + lcNumberStart
lcNumberStart = lcNumber.max()
ulcNumber = np.unique(lcNumber)
lcLeft = np.searchsorted(lcNumber, ulcNumber, side='left')
lcRight = np.searchsorted(lcNumber, ulcNumber, side='right')
# Generate the actual light curve magnitudes and SNR
lcMags = self.make_lightCurve(lcEpoch, dataSlice[self.filterCol])
lcSNR = m52snr(lcMags, dataSlice[self.m5Col])
# Identify which detections rise above the required SNR threshhold, in each filter.
lcAboveThresh = np.zeros(len(lcSNR), dtype=bool)
for f in np.unique(dataSlice[self.filterCol]):
filtermatch = np.where(dataSlice[self.filterCol] == f)[0]
lcAboveThresh[filtermatch] = np.where(lcSNR[filtermatch] >= self.detectSNR[f],
True,
False)
# Track whether each individual light curve was detected.
# Start with the assumption that it is True, and if it fails criteria then becomes False.
lcDetect = np.ones(len(ulcNumber), dtype=bool)
# Loop through each lightcurve and check if it meets requirements.
for lcN, le, ri in zip(ulcNumber, lcLeft, lcRight):
# If there were no observations at all for this lightcurve:
if le == ri:
lcDetect[lcN] = False
# Skip the rest of this loop, go on to the next lightcurve.
continue
lcEpochAboveThresh = lcEpoch[le:ri][np.where(lcAboveThresh[le:ri])]
# If we did not get enough detections before preT, set lcDetect to False.
timesPreT = np.where(lcEpochAboveThresh < self.preT)[0]
if len(timesPreT) < self.nPreT:
lcDetect[lcN] = False
continue
# If we did not get detections over enough sections of the lightcurve, set lcDtect to False.
phaseSections = np.unique(np.floor(lcEpochAboveThresh / self.transDuration * self.nPerLC))
if len(phaseSections) < self.nPerLC:
lcDetect[lcN] = False
continue
# If we did not get detections in enough filters, set lcDetect to False.
lcFilters = dataSlice[le:ri][np.where(lcAboveThresh[le:ri])][self.filterCol]
if len(np.unique(lcFilters)) < self.nFilters:
lcDetect[lcN] = False
continue
# If we did not get detections in enough filters within required time, set lcDetect to False.
if (self.filterT is not None) and (self.nFilters > 1):
xr = np.searchsorted(lcEpochAboveThresh, lcEpochAboveThresh + self.filterT, 'right')
xr = np.where(xr < len(lcEpochAboveThresh) - 1, xr, len(lcEpochAboveThresh) - 1)
foundGood = False
for i, xri in enumerate(xr):
if len(np.unique(lcFilters[i:xri])) >= self.nFilters:
foundGood = True
break
if not foundGood:
lcDetect[lcN] = False
continue
# Done with current set of conditions.
# (more complicated conditions should go later in the loop, simpler ones earlier).
# Find the unique number of light curves that passed the required number of conditions
nDetected += len(np.where(lcDetect == True)[0])
if self.dataout:
# Output all the light curves, regardless of detection threshhold,
# but indicate which were 'detected'.
lcDetectOut = np.ones(len(dataSlice), dtype=bool)
for i, lcN in enumerate(lcNumber):
lcDetectOut[i] = lcDetect[lcN]
return {'lcNumber': lcNumber, 'expMJD': dataSlice[self.mjdCol], 'epoch': lcEpoch,
'filter': dataSlice[self.filterCol], 'lcMag': lcMags, 'SNR': lcSNR,
'detected': lcDetectOut}
else:
return float(nDetected) / nTransMax
