def test_F2f_remove_noise(MACHO_example, denoised_MACHO_by_FATS):
me, dMF = MACHO_example, denoised_MACHO_by_FATS
p_time, p_mag, p_error = preprocess.remove_noise(me.time, me.mag, me.error)
p_time2, p_mag2, p_error2 = preprocess.remove_noise(
me.time2, me.mag2, me.error2)
np.testing.assert_array_equal(p_time, dMF.time)
np.testing.assert_array_equal(p_time2, dMF.time2)
np.testing.assert_array_equal(p_mag, dMF.mag)
np.testing.assert_array_equal(p_mag2, dMF.mag2)
np.testing.assert_array_equal(p_error, dMF.error)
np.testing.assert_array_equal(p_error2, dMF.error2)
def preprocessLc(timeData, magData, errorData, removes, stdLimit, errorLimit):
"""Returns a cleaned version of an LC. LC may be deemed unfit for use, in
which case the reason for rejection is specified.
:returns processed lc as a tuple and failure reason (string)
"""
removedCounts = {DATA_BOGUS_REMOVED: 0, DATA_OUTLIER_REMOVED: 0}
if len(timeData) < SUFFICIENT_LC_DATA:
return None, INSUFFICIENT_DATA_REASON, removedCounts
# remove bogus data
tm, mag, err = lcFilterBogus(timeData, magData, errorData, removes=removes)
removedCounts[DATA_BOGUS_REMOVED] = len(timeData) - len(tm)
if len(tm) < SUFFICIENT_LC_DATA:
return None, BOGUS_DATA_REASON, removedCounts
# removes statistical outliers
_tm, _mag, _err = preprocess.remove_noise(tm,
mag,
err,
error_limit=errorLimit,
std_limit=stdLimit)
removedCounts[DATA_OUTLIER_REMOVED] = len(tm) - len(_tm)
if len(_tm) < SUFFICIENT_LC_DATA:
return None, OUTLIERS_REASON, removedCounts
return [_tm, _mag, _err], None, removedCounts
def sigma_clip(obs):
time = obs.pwp_stack_src_hjd.values
magnitude = obs.pwp_stack_src_mag3.values
error = obs.pwp_stack_src_mag_err3.values
sort = np.argsort(time)
time, magnitude, error = time[sort], magnitude[sort], error[sort]
time, magnitude, error = preprocess.remove_noise(time,
magnitude,
error,
std_limit=3)
return time, magnitude, error
def test_remove_noise_no_outlier():
random = np.random.RandomState(42)
time = np.arange(5)
mag = random.rand(5)
error = np.zeros(5)
ptime, pmag, perror = preprocess.remove_noise(time, mag, error)
assert len(ptime) == len(time)
assert len(pmag) == len(mag)
assert len(perror) == len(error)
np.testing.assert_array_equal(ptime, time)
np.testing.assert_array_equal(pmag, mag)
np.testing.assert_array_equal(perror, error)
def test_remove_noise():
random = np.random.RandomState(42)
time = np.arange(5)
mag = random.rand(5)
mag[-1] = np.mean(mag) + 100 * np.std(mag)
error = np.zeros(5)
error[-1] = 10
ptime, pmag, perror = preprocess.remove_noise(time, mag, error)
assert len(ptime) == len(time) - 1
assert len(pmag) == len(mag) - 1
assert len(perror) == len(error) - 1
np.testing.assert_array_equal(ptime, time[:-1])
np.testing.assert_array_equal(pmag, mag[:-1])
np.testing.assert_array_equal(perror, error[:-1])
def extract(sid, obs, old_feats):
time = obs.pwp_stack_src_hjd.values
magnitude = obs.pwp_stack_src_mag3.values
error = obs.pwp_stack_src_mag_err3.values
sort = np.argsort(time)
time, magnitude, error = time[sort], magnitude[sort], error[sort]
time, magnitude, error = preprocess.remove_noise(time,
magnitude,
error,
std_limit=3)
new_feats = dict(
zip(*fs.extract(time=time, magnitude=magnitude, error=error)))
new_feats["ppmb"] = old_feats["ppmb"] * old_feats["PeriodLS"] / new_feats[
"PeriodLS"]
new_feats["cnt"] = len(time)
feats = old_feats.copy()
feats.update(new_feats)
return feats
def tutorial():
plot = 0
lc = datasets.load_MACHO_example()
if plot:
plotLc(lc)
# 69 features with ALL_DATA_TYPES in 6.15s
# 64 features with time, magnitude, error in 6.14s
# 58 features with time, magnitude in 3.3s
# 22 features with magnitude in 0.02s
basicData = ["time", "magnitude", "error"]
basicData = ["time", "magnitude"]
basicData = ["magnitude"]
start = time.time()
# remove points beyond 5 stds of the mean
tm, mag, error = preprocess.remove_noise(**lc.bands.B)
tm2, mag2, error2 = preprocess.remove_noise(**lc.bands.R)
# N.B. skip for LSST work
aTime, aMag, aMag2, aError, aError2 = preprocess.align(tm, tm2, mag,
mag2, error, error2)
lc = [tm, mag, error, mag2, aTime, aMag, aMag2, aError, aError2]
# only calculate these features
# fs = feets.FeatureSpace(only=['Std', 'StetsonL'])
fs = FeatureSpace()
# fs = FeatureSpace(data=basicData)
features, values = fs.extract(*lc)
elapsed = time.time() - start
print("Computed %s features in %.02fs" % (len(features), elapsed))
if plot:
g = plt.figure(2)
plt.plot(lc[0], lc[1], "*-", alpha=0.6)
plt.xlabel("Time")
plt.ylabel("Magnitude")
plt.gca().invert_yaxis()
g.show()
input()
t = PrettyTable(["Feature", "Value"])
t.align = "l"
for i, feat in enumerate(features):
t.add_row([feat, values[i]])
if plot:
print(t)
fdict = dict(zip(features, values))
# Ploting the example lightcurve in phase
T = 2 * fdict["PeriodLS"]
new_b = np.mod(lc[0], T) / T
idx = np.argsort(2 * new_b)
plt.plot(new_b, lc[1], '*')
plt.xlabel("Phase")
plt.ylabel("Magnitude")
plt.gca().invert_yaxis()
plt.show()