def test_featurize_time_series_default_errors():
"""Test featurize wrapper function for time series w/ missing errors"""
n_channels = 3
t, m, _ = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
meta_features = {}
fset = featurize.featurize_time_series(t,
m,
None,
features_to_use,
meta_features,
scheduler=dask.get)
t = [[t, t[0:-5], t[0:-10]]]
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
fset = featurize.featurize_time_series(t,
m,
None,
features_to_use,
meta_features,
scheduler=dask.get)
t = t[0][0]
m = m[0][0]
fset = featurize.featurize_time_series(t,
m,
None,
features_to_use,
meta_features,
scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
def test_featurize_time_series_default_times():
"""Test featurize wrapper function for time series w/ missing times"""
n_channels = 3
_, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
meta_features = {}
fset = featurize.featurize_time_series(None,
m,
e,
features_to_use,
meta_features,
scheduler=get_sync)
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
e = [[e[0], e[1][0:-5], e[2][0:-10]]]
fset = featurize.featurize_time_series(None,
m,
e,
features_to_use,
meta_features,
scheduler=get_sync)
m = m[0][0]
e = e[0][0]
fset = featurize.featurize_time_series(None,
m,
e,
features_to_use,
meta_features,
scheduler=get_sync)
def test_ignore_exceptions():
import cesium.features.graphs
def raise_exc(x):
raise ValueError()
old_value = cesium.features.graphs.dask_feature_graph['mean']
try:
cesium.features.graphs.dask_feature_graph['mean'] = (raise_exc, 't')
t, m, e = sample_values()
features_to_use = ['mean']
with pytest.raises(ValueError):
fset = featurize.featurize_time_series(t,
m,
e,
features_to_use,
scheduler=dask.get,
raise_exceptions=True)
fset = featurize.featurize_time_series(t,
m,
e,
features_to_use,
scheduler=dask.get,
raise_exceptions=False)
assert np.isnan(fset.values).all()
finally:
cesium.features.graphs.dask_feature_graph['mean'] = old_value
def test_featurize_time_series_pandas_metafeatures():
"""Test featurize function for metafeatures passed as Series/DataFrames."""
t, m, e = sample_values()
features_to_use = ['amplitude', 'std_err']
meta_features = pd.Series({'meta1': 0.5})
fset = featurize.featurize_time_series(t,
m,
e,
features_to_use,
meta_features,
scheduler=dask.get)
npt.assert_allclose(fset['meta1'], 0.5)
n_series = 5
list_of_series = [sample_values() for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
meta_features = pd.DataFrame({
'meta1': [0.5] * n_series,
'meta2': [0.8] * n_series
})
fset = featurize.featurize_time_series(times,
values,
errors,
features_to_use,
meta_features,
scheduler=dask.get)
npt.assert_allclose(fset['meta1'], 0.5)
npt.assert_allclose(fset['meta2'], 0.8)
def post(self):
ts_data = json_decode(self.get_argument('ts_data'))
model_id = json_decode(self.get_argument('modelID'))
meta_feats = json_decode(self.get_argument('meta_features', 'null'))
impute_kwargs = json_decode(self.get_argument('impute_kwargs', '{}'))
model = Model.query.get(model_id)
model_data = joblib.load(model.file_uri)
if hasattr(model_data, 'best_estimator_'):
model_data = model_data.best_estimator_
features_to_use = model.featureset.features_list
fset = featurize.featurize_time_series(*ts_data,
features_to_use=features_to_use,
meta_features=meta_feats,
raise_exceptions=False)
fset = featurize.impute_featureset(fset, **impute_kwargs)
fset.index = fset.index.astype(str) # ensure JSON-encodable
data = {'preds': model_data.predict(fset)}
if hasattr(model_data, 'predict_proba'):
data['pred_probs'] = pd.DataFrame(model_data.predict_proba(fset),
index=fset.index,
columns=model_data.classes_)
else:
data['pred_probs'] = []
pred_info = Prediction.format_pred_data(fset, data)
return self.success(pred_info)
def get_freq_features(N,
train_series,
times_list,
flux_list,
train_metadata,
subsetting_pos=None):
if subsetting_pos is None:
subset_times_list = times_list
subset_flux_list = flux_list
else:
subset_times_list = [
v for i, v in enumerate(times_list) if i in set(subsetting_pos)
]
subset_flux_list = [
v for i, v in enumerate(flux_list) if i in set(subsetting_pos)
]
feats = featurize.featurize_time_series(
times=subset_times_list[:N],
values=subset_flux_list[:N],
features_to_use=[
'skew', 'percent_beyond_1_std',
'percent_difference_flux_percentile'
],
scheduler=None,
)
subset = train_series[train_series['object_id'].isin(
train_metadata['object_id'].iloc[subsetting_pos].iloc[:N])]
models = list(map(fit_multiband_freq, subset.groupby('object_id')))
feats['object_pos'] = subsetting_pos[:N]
feats['freq1_freq'] = [model.best_period for model in models]
return feats, models
def post(self):
ts_data = json_decode(self.get_argument('ts_data'))
model_id = json_decode(self.get_argument('modelID'))
meta_feats = json_decode(self.get_argument('meta_features', 'null'))
impute_kwargs = json_decode(self.get_argument('impute_kwargs', '{}'))
model = Model.query.get(model_id)
model_data = joblib.load(model.file_uri)
if hasattr(model_data, 'best_estimator_'):
model_data = model_data.best_estimator_
features_to_use = model.featureset.features_list
fset = featurize.featurize_time_series(*ts_data,
features_to_use=features_to_use,
meta_features=meta_feats,
raise_exceptions=False)
fset = featurize.impute_featureset(fset, **impute_kwargs)
fset.index = fset.index.astype(str) # ensure JSON-encodable
data = {'preds': model_data.predict(fset)}
if hasattr(model_data, 'predict_proba'):
data['pred_probs'] = pd.DataFrame(model_data.predict_proba(fset),
index=fset.index,
columns=model_data.classes_)
else:
data['pred_probs'] = []
pred_info = Prediction.format_pred_data(fset, data)
return self.success(pred_info)
def test_featurize_time_series_single():
"""Test featurize wrapper function for single time series"""
t, m, e = sample_values()
features_to_use = ['amplitude', 'std_err']
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features, scheduler=dask.get)
assert fset['amplitude'].values.dtype == np.float64
def test_featurize_time_series_single():
"""Test featurize wrapper function for single time series"""
t, m, e = sample_values()
features_to_use = ['amplitude', 'std_err']
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features, scheduler=dask.get)
assert fset['amplitude'].values.dtype == np.float64
def get_features(time_series):
times = [df['mjd'].tolist() for df in time_series]
values = [df['flux'].tolist() for df in time_series]
errors = [df['flux_err'].tolist() for df in time_series]
features = featurize_time_series(times,
values,
errors,
features_to_use=FEATURES)
return features.values
def test_featurize_time_series_single_multichannel():
"""Test featurize wrapper function for single multichannel time series"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features, scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
assert 'meta1' in fset.columns
def test_featurize_time_series_single_multichannel():
"""Test featurize wrapper function for single multichannel time series"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features, scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
assert 'meta1' in fset.columns
def test_featurize_time_series_default_errors():
"""Test featurize wrapper function for time series w/ missing errors"""
n_channels = 3
t, m, _ = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
meta_features = {}
fset = featurize.featurize_time_series(t, m, None, features_to_use,
meta_features, scheduler=dask.get)
t = [[t, t[0:-5], t[0:-10]]]
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
fset = featurize.featurize_time_series(t, m, None, features_to_use,
meta_features, scheduler=dask.get)
t = t[0][0]
m = m[0][0]
fset = featurize.featurize_time_series(t, m, None, features_to_use,
meta_features, scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
def test_ignore_exceptions():
import cesium.features.graphs
def raise_exc(x):
raise ValueError()
old_value = cesium.features.graphs.dask_feature_graph['mean']
try:
cesium.features.graphs.dask_feature_graph['mean'] = (raise_exc, 't')
t, m, e = sample_values()
features_to_use = ['mean']
with pytest.raises(ValueError):
fset = featurize.featurize_time_series(t, m, e, features_to_use,
scheduler=dask.get,
raise_exceptions=True)
fset = featurize.featurize_time_series(t, m, e, features_to_use,
scheduler=dask.get,
raise_exceptions=False)
assert np.isnan(fset.values).all()
finally:
cesium.features.graphs.dask_feature_graph['mean'] = old_value
def test_featurize_time_series_single():
"""Test featurize wrapper function for single time series"""
t, m, e = sample_values()
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.target.values, ['class1'])
def test_featurize_time_series_single():
"""Test featurize wrapper function for single time series"""
t, m, e = sample_time_series()
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features, use_celery=False)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.target.values, ['class1'])
def test_featurize_time_series_pandas_metafeatures():
"""Test featurize function for metafeatures passed as Series/DataFrames."""
t, m, e = sample_values()
features_to_use = ['amplitude', 'std_err']
meta_features = pd.Series({'meta1': 0.5})
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features, scheduler=dask.get)
npt.assert_allclose(fset['meta1'], 0.5)
n_series = 5
list_of_series = [sample_values() for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
meta_features = pd.DataFrame({'meta1': [0.5] * n_series,
'meta2': [0.8] * n_series})
fset = featurize.featurize_time_series(times, values, errors,
features_to_use,
meta_features, scheduler=dask.get)
npt.assert_allclose(fset['meta1'], 0.5)
npt.assert_allclose(fset['meta2'], 0.8)
def test_featurize_time_series_default_errors():
"""Test featurize wrapper function for time series w/ missing errors"""
n_channels = 3
t, m, _ = sample_time_series(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {}
fset = featurize.featurize_time_series(t, m, None, features_to_use, target,
meta_features, use_celery=False)
npt.assert_array_equal(fset.channel, np.arange(n_channels))
t = [[t, t[0:-5], t[0:-10]]]
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
fset = featurize.featurize_time_series(t, m, None, features_to_use, target,
meta_features, use_celery=False)
npt.assert_array_equal(fset.channel, np.arange(n_channels))
t = t[0][0]
m = m[0][0]
fset = featurize.featurize_time_series(t, m, None, features_to_use, target,
meta_features, use_celery=False)
npt.assert_array_equal(fset.channel, [0])
def test_featurize_time_series_multiple():
"""Test featurize wrapper function for multiple time series"""
n_series = 5
list_of_series = [sample_values() for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
meta_features = [{'meta1': 0.5}] * n_series
fset = featurize.featurize_time_series(times, values, errors,
features_to_use,
meta_features, scheduler=dask.get)
npt.assert_array_equal(sorted(fset.columns.get_level_values('feature')),
['amplitude', 'meta1', 'std_err'])
def test_featurize_time_series_multiple():
"""Test featurize wrapper function for multiple time series"""
n_series = 5
list_of_series = [sample_values() for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
meta_features = [{'meta1': 0.5}] * n_series
fset = featurize.featurize_time_series(times, values, errors,
features_to_use,
meta_features, scheduler=dask.get)
npt.assert_array_equal(sorted(fset.columns.get_level_values('feature')),
['amplitude', 'meta1', 'std_err'])
def test_featurize_time_series_no_targets():
t, m, e = sample_values()
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
targets=None,
meta_features=meta_features,
scheduler=get_sync)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
assert('target' not in fset)
def test_featurize_time_series_uneven_multichannel():
"""Test featurize wrapper function for uneven-length multichannel data"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
t = [[t, t[0:-5], t[0:-10]]]
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
e = [[e[0], e[1][0:-5], e[2][0:-10]]]
features_to_use = ['amplitude', 'std_err']
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features, scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
assert 'meta1' in fset.columns
def test_featurize_time_series_default_times():
"""Test featurize wrapper function for time series w/ missing times"""
n_channels = 3
_, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {}
fset = featurize.featurize_time_series(None, m, e, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(fset.channel, np.arange(n_channels))
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
e = [[e[0], e[1][0:-5], e[2][0:-10]]]
fset = featurize.featurize_time_series(None, m, e, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(fset.channel, np.arange(n_channels))
m = m[0][0]
e = e[0][0]
fset = featurize.featurize_time_series(None, m, e, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(fset.channel, [0])
def test_featurize_time_series_uneven_multichannel():
"""Test featurize wrapper function for uneven-length multichannel data"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
t = [[t, t[0:-5], t[0:-10]]]
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
e = [[e[0], e[1][0:-5], e[2][0:-10]]]
features_to_use = ['amplitude', 'std_err']
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features, scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
assert 'meta1' in fset.columns
def test_featurize_time_series_default_errors():
"""Test featurize wrapper function for time series w/ missing errors"""
n_channels = 3
t, m, _ = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {}
fset = featurize.featurize_time_series(t, m, None, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(fset.channel, np.arange(n_channels))
t = [[t, t[0:-5], t[0:-10]]]
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
fset = featurize.featurize_time_series(t, m, None, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(fset.channel, np.arange(n_channels))
t = t[0][0]
m = m[0][0]
fset = featurize.featurize_time_series(t, m, None, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(fset.channel, [0])
def test_featurize_time_series_multiple():
"""Test featurize wrapper function for multiple time series"""
n_series = 5
list_of_series = [sample_time_series() for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
targets = np.array(['class1'] * n_series)
meta_features = [{'meta1': 0.5}] * n_series
fset = featurize.featurize_time_series(times, values, errors,
features_to_use, targets,
meta_features, use_celery=False)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.target.values, ['class1'] * n_series)
def test_featurize_time_series_multiple_multichannel():
"""Test featurize wrapper function for multiple multichannel time series"""
n_series = 5
n_channels = 3
list_of_series = [sample_values(channels=n_channels)
for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(times, values, errors,
features_to_use,
meta_features, scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
assert 'meta1' in fset.columns
def test_featurize_time_series_custom_functions():
"""Test featurize wrapper function for time series w/ custom functions"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err', 'test_f']
meta_features = {'meta1': 0.5}
custom_functions = {'test_f': lambda t, m, e: np.pi}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features,
custom_functions=custom_functions,
scheduler=dask.get)
npt.assert_array_equal(fset['test_f', 0], np.pi)
assert ('amplitude', 0) in fset.columns
assert 'meta1' in fset.columns
def test_featurize_time_series_custom_functions():
"""Test featurize wrapper function for time series w/ custom functions"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err', 'test_f']
meta_features = {'meta1': 0.5}
custom_functions = {'test_f': lambda t, m, e: np.pi}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features,
custom_functions=custom_functions,
scheduler=dask.get)
npt.assert_array_equal(fset['test_f', 0], np.pi)
assert ('amplitude', 0) in fset.columns
assert 'meta1' in fset.columns
def test_featurize_time_series_multiple_multichannel():
"""Test featurize wrapper function for multiple multichannel time series"""
n_series = 5
n_channels = 3
list_of_series = [sample_values(channels=n_channels)
for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(times, values, errors,
features_to_use,
meta_features, scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
assert 'meta1' in fset.columns
def test_featurize_time_series_multiple():
"""Test featurize wrapper function for multiple time series"""
n_series = 5
list_of_series = [sample_values() for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
targets = np.array(['class1'] * n_series)
meta_features = [{'meta1': 0.5}] * n_series
fset = featurize.featurize_time_series(times, values, errors,
features_to_use, targets,
meta_features, scheduler=get_sync)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.target.values, ['class1'] * n_series)
def test_featurize_time_series_custom_dask_graph():
"""Test featurize wrapper function for time series w/ custom dask graph"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err', 'test_f', 'test_meta']
meta_features = {'meta1': 0.5}
custom_functions = {'test_f': (lambda x: x.min() - x.max(), 'amplitude'),
'test_meta': (lambda x: 2. * x, 'meta1')}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features,
custom_functions=custom_functions,
scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
assert ('test_f', 0) in fset.columns
assert ('test_meta', 0) in fset.columns
def test_featurize_time_series_single_multichannel():
"""Test featurize wrapper function for single multichannel time series"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, ['class1'])
def test_featurize_time_series_custom_dask_graph():
"""Test featurize wrapper function for time series w/ custom dask graph"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err', 'test_f', 'test_meta']
meta_features = {'meta1': 0.5}
custom_functions = {'test_f': (lambda x: x.min() - x.max(), 'amplitude'),
'test_meta': (lambda x: 2. * x, 'meta1')}
fset = featurize.featurize_time_series(t, m, e, features_to_use,
meta_features,
custom_functions=custom_functions,
scheduler=dask.get)
assert ('amplitude', 0) in fset.columns
assert ('test_f', 0) in fset.columns
assert ('test_meta', 0) in fset.columns
def test_featurize_time_series_single_multichannel():
"""Test featurize wrapper function for single multichannel time series"""
n_channels = 3
t, m, e = sample_time_series(channels=n_channels)
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features, use_celery=False)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, ['class1'])
def encode(self, values_data, times=None):
"""
Encode a column data into time series
:param values_data: a list of timeseries data eg: ['91.0 92.0 93.0 94.0', '92.0 93.0 94.0 95.0' ...]
:param times: (optional) a list of lists such that, len(times[i])=len(values_data[i]) for
all i in range(len(times))
:return: a torch.floatTensor
"""
features_to_use = self._features
ret = []
for i, values in enumerate(values_data):
if type(values) == type([]):
values = list(map(float, values))
else:
values = list(map(lambda x: float(x), values.split()))
if times is None:
times_row = np.array(
[float(i) for i in range(1,
len(values) + 1)])
else:
times_row = np.array(
list(map(lambda x: float(x),
times[i].split()))) # np.array(times[i])
with warnings.catch_warnings():
warnings.simplefilter("ignore")
row = featurize.featurize_time_series(
times=times_row,
values=np.array(values),
errors=None,
features_to_use=features_to_use)
vector_row = []
for col in features_to_use:
val = list(row[col][0])[0]
val1 = 0
if (val in ['nan', None, 'NaN', False]) \
or math.isnan(val) or math.isinf(val):
val = 0
val1 = 1
if col in FEATURES_WITH_DEFAULT_NONE:
vector_row += [val, val1] # val1 is 1 if its null
else:
vector_row += [val]
ret += [vector_row]
ret_tensor = self._pytorch_wrapper(ret)
return ret_tensor
def lcFreq(df_main):
df_main = df_main.sort_values('mjd').reset_index(drop=True)
groups = df_main.groupby('passband')
t_list = groups.apply(lambda gr: gr['mjd'].values).tolist()
flx_list = groups.apply(lambda gr: gr['flux'].values).tolist()
flxer_list = groups.apply(lambda gr: gr['flux_err'].values).tolist()
feats = featurize.featurize_time_series(times=t_list,
values=flx_list,
errors=flxer_list,
features_to_use=['freq1_freq'],
scheduler=None)
feats.columns = feats.columns.droplevel(1)
feats['freq1_freq'].mean()
feats['freq1_freq'].std()
return pd.Series([feats['freq1_freq'].mean(), feats['freq1_freq'].std()],
index=['freq', 'freq_std'])
def test_featurize_time_series_custom_script():
"""Test featurize wrapper function for time series w/ custom script path"""
n_channels = 3
t, m, e = sample_time_series(channels=n_channels)
features_to_use = ['amplitude', 'std_err', 'f']
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features,
custom_script_path=CUSTOM_SCRIPT,
use_celery=False)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'f', 'meta1', 'std_err'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, ['class1'])
def test_featurize_time_series_custom_dask_graph():
"""Test featurize wrapper function for time series w/ custom dask graph"""
n_channels = 3
t, m, e = sample_time_series(channels=n_channels)
features_to_use = ['amplitude', 'std_err', 'test_f']
target = 'class1'
meta_features = {'meta1': 0.5}
custom_functions = {'test_f': (lambda x: x.min() - x.max(), 'amplitude')}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features,
custom_functions=custom_functions,
use_celery=False)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err', 'test_f'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, ['class1'])
def test_featurize_time_series_uneven_multichannel():
"""Test featurize wrapper function for uneven-length multichannel data"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
t = [[t, t[0:-5], t[0:-10]]]
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
e = [[e[0], e[1][0:-5], e[2][0:-10]]]
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features, scheduler=get_sync)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, ['class1'])
def test_featurize_time_series_uneven_multichannel():
"""Test featurize wrapper function for uneven-length multichannel data"""
n_channels = 3
t, m, e = sample_time_series(channels=n_channels)
t = [[t, t[0:-5], t[0:-10]]]
m = [[m[0], m[1][0:-5], m[2][0:-10]]]
e = [[e[0], e[1][0:-5], e[2][0:-10]]]
features_to_use = ['amplitude', 'std_err']
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features, use_celery=False)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, ['class1'])
def test_featurize_time_series_custom_functions():
"""Test featurize wrapper function for time series w/ custom functions"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err', 'test_f']
target = 'class1'
meta_features = {'meta1': 0.5}
custom_functions = {'test_f': lambda t, m, e: np.pi}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features,
custom_functions=custom_functions,
scheduler=get_sync)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err', 'test_f'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.test_f.values, np.pi)
npt.assert_array_equal(fset.target.values, ['class1'])
def featurize_time_series(ts):
from cesium import featurize
features_to_use = ["amplitude",
"percent_beyond_1_std",
"maximum",
"max_slope",
"median",
"median_absolute_deviation",
"percent_close_to_median",
"minimum",
"skew",
"std",
"weighted_average"]
new = featurize.featurize_time_series(times=np.arange(0, np.shape(ts)[0]),
values=ts,
errors=None,
features_to_use=features_to_use)
return new.values
def test_featurize_time_series_multiple_multichannel():
"""Test featurize wrapper function for multiple multichannel time series"""
n_series = 5
n_channels = 3
list_of_series = [sample_values(channels=n_channels)
for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
targets = np.array(['class1', 'class1', 'class1', 'class2', 'class2'])
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(times, values, errors,
features_to_use, targets,
meta_features, scheduler=get_sync)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, targets)
def printFrequency(dataTable):
""" This function obtain the frequency using cesium, which implies the period.
Arguments:
dataTable (array-like) : this is the .tbl data file downloaded from Caltech
IRSA website
"""
# freq1_amplitude1: Get the amplitude of the jth harmonic of the ith frequency from a fitted Lomb-Scargle model.
features_to_use = ["freq1_freq", "amplitude", "freq1_amplitude1"]
fset_cesium = featurize.featurize_time_series(
times=dataTable["obsmjd"],
values=dataTable["mag_autocorr"],
errors=dataTable["magerr_auto"],
features_to_use=features_to_use)
print(fset_cesium)
def test_featurize_time_series_multiple_multichannel():
"""Test featurize wrapper function for multiple multichannel time series"""
n_series = 5
n_channels = 3
list_of_series = [sample_time_series(channels=n_channels)
for i in range(n_series)]
times, values, errors = [list(x) for x in zip(*list_of_series)]
features_to_use = ['amplitude', 'std_err']
targets = np.array(['class1', 'class1', 'class1', 'class2', 'class2'])
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(times, values, errors,
features_to_use, targets,
meta_features, use_celery=False)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, targets)
def test_featurize_time_series_custom_dask_graph():
"""Test featurize wrapper function for time series w/ custom dask graph"""
n_channels = 3
t, m, e = sample_values(channels=n_channels)
features_to_use = ['amplitude', 'std_err', 'test_f']
target = 'class1'
meta_features = {'meta1': 0.5}
custom_functions = {'test_f': (lambda x: x.min() - x.max(), 'amplitude'),
'test_meta': (lambda x: 2. * x, 'meta1')}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features,
custom_functions=custom_functions,
scheduler=get_sync)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err', 'test_f'])
npt.assert_array_equal(fset.channel, np.arange(n_channels))
npt.assert_array_equal(sorted(fset.amplitude.coords),
['channel', 'name', 'target'])
npt.assert_array_equal(fset.target.values, ['class1'])
def get_freq_features(N, subsetting_pos=None):
if subsetting_pos is None:
subset_times_list = times_list
subset_flux_list = flux_list
else:
subset_times_list = [
v for i, v in enumerate(times_list) if i in set(subsetting_pos)
]
subset_flux_list = [
v for i, v in enumerate(flux_list) if i in set(subsetting_pos)
]
feats = featurize.featurize_time_series(
times=subset_times_list[:N],
values=subset_flux_list[:N],
features_to_use=[
'freq1_freq', 'freq1_signif', 'freq1_amplitude1', 'skew',
'percent_beyond_1_std', 'percent_difference_flux_percentile'
],
scheduler=None)
feats['object_pos'] = subsetting_pos[:N]
return feats
def test_featurize_time_series_celery():
"""Test `featurize_time_series` with Celery.
The actual featurization work is being done by
`featurize_tools.featurize_single_time_series`, which is called by both the
Celery and non-Celery versions; thus, besides the above tests, we only need
to check that the Celery task is configured properly."""
t, m, e = sample_time_series()
features_to_use = ['amplitude', 'std_err', 'test_f']
# This ideally would be a dummy lambda function but celery can't do that
from cesium.science_features import lomb_scargle_fast as lsf
custom_functions = {'test_f': lsf.lomb_scargle_fast_period}
target = 'class1'
meta_features = {'meta1': 0.5}
fset = featurize.featurize_time_series(t, m, e, features_to_use, target,
meta_features,
custom_functions=custom_functions,
use_celery=True)
npt.assert_array_equal(sorted(fset.data_vars),
['amplitude', 'meta1', 'std_err', 'test_f'])
npt.assert_array_equal(fset.target.values, ['class1'])
def try_cesium(df):
from cesium import featurize
features_to_use = [
"amplitude",
"percent_beyond_1_std",
"maximum",
"max_slope",
"median",
"median_absolute_deviation",
"percent_close_to_median",
"minimum",
"skew",
"std",
"weighted_average",
]
fset_cesium = featurize.featurize_time_series(
times=eeg["times"],
values=eeg["measurements"],
errors=None,
features_to_use=features_to_use,
)
print(fset_cesium.head())
# By default, the time series will featurized in parallel using the
# ``dask.threaded`` scheduler; other approaches, including serial and
# distributed approaches, can be implemented by passing in other ``dask``
# schedulers as the ``get`` argument to ``featurize_time_series``.
#
# .. |cesium.featurize| replace:: ``cesium.featurize``
# .. _cesium.featurize: http://cesium-ml.org/docs/api/cesium.featurize.html
from cesium import featurize
features_to_use = [
"amplitude", "percent_beyond_1_std", "maximum", "max_slope", "median",
"median_absolute_deviation", "percent_close_to_median", "minimum", "skew",
"std", "weighted_average"
]
fset_cesium = featurize.featurize_time_series(times=eeg["times"],
values=eeg["measurements"],
errors=None,
features_to_use=features_to_use)
print(fset_cesium.head())
###############################################################################
# The output of ``featurize_time_series`` is a ``pandas.DataFrame`` which contains all
# the feature information needed to train a machine learning model: feature
# names are stored as column indices (as well as channel numbers, as we'll see
# later for multi-channel data), and the time series index/class label are
# stored as row indices.
###############################################################################
# Custom feature functions
# ~~~~~~~~~~~~~~~~~~~~~~~~
# Custom feature functions not built into ``cesium`` may be passed in using the
# ``custom_functions`` keyword, either as a dictionary ``{feature_name: function}``, or as a
# .. _cesium.featurize: http://cesium-ml.org/docs/api/cesium.featurize.html
from cesium import featurize
features_to_use = ["amplitude",
"percent_beyond_1_std",
"maximum",
"max_slope",
"median",
"median_absolute_deviation",
"percent_close_to_median",
"minimum",
"skew",
"std",
"weighted_average"]
fset_cesium = featurize.featurize_time_series(times=eeg["times"],
values=eeg["measurements"],
errors=None,
features_to_use=features_to_use)
print(fset_cesium.head())
###############################################################################
# The output of ``featurize_time_series`` is a ``pandas.DataFrame`` which contains all
# the feature information needed to train a machine learning model: feature
# names are stored as column indices (as well as channel numbers, as we'll see
# later for multi-channel data), and the time series index/class label are
# stored as row indices.
###############################################################################
# Custom feature functions
# ~~~~~~~~~~~~~~~~~~~~~~~~
# Custom feature functions not built into ``cesium`` may be passed in using the
# ``custom_functions`` keyword, either as a dictionary ``{feature_name: function}``, or as a
def get_cesium_features(self, recompute=False):
"""
Compute all relevant cesium features.
"""
cesium = getattr(self, 'cesium', None)
if cesium is not None:
if not recompute:
return cesium
cesium = {}
outlc = self.get_lc(recompute=recompute)
for i, pb in enumerate(outlc):
tlc = outlc.get(pb)
ttime, tFlux, tFluxErr, tFluxUnred, tFluxErrUnred, tFluxRenorm, tFluxErrRenorm, tphotflag, tzeropoint, tobsId = tlc
photmask = tphotflag >= constants.GOOD_PHOTFLAG
ttime = ttime[photmask]
tFluxRenorm = tFluxRenorm[photmask]
tFluxErrRenorm = tFluxErrRenorm[photmask]
features_general = ['flux_percentile_ratio_mid20',
'flux_percentile_ratio_mid50',
'flux_percentile_ratio_mid65',
'flux_percentile_ratio_mid80',
'max_slope',
'maximum',
'median',
'median_absolute_deviation',
'percent_amplitude',
'period_fast',
'qso_log_chi2_qsonu',
'qso_log_chi2nuNULL_chi2nu',
'fold2P_slope_90percentile',
'freq1_amplitude1',
'freq1_amplitude2',
'freq1_amplitude3',
'freq1_amplitude4',
'freq1_freq',
'freq1_lambda',
'freq1_rel_phase2',
'freq1_rel_phase3',
'freq1_rel_phase4',
'freq1_signif',
'freq2_amplitude1',
'freq2_amplitude2',
'freq2_amplitude3',
'freq2_amplitude4',
'freq2_rel_phase2',
'freq2_rel_phase3',
'freq2_rel_phase4',
'freq3_amplitude1',
'freq3_amplitude2',
'freq3_amplitude3',
'freq3_amplitude4',
'freq3_rel_phase2',
'freq3_rel_phase3',
'freq3_rel_phase4',
'freq_amplitude_ratio_21',
'freq_amplitude_ratio_31',
'freq_n_alias',
'freq_signif_ratio_21',
'freq_signif_ratio_31',
'freq_varrat',
'freq_y_offset',
'medperc90_2p_p',
'p2p_scatter_pfold_over_mad',
'p2p_ssqr_diff_over_var',
'scatter_res_raw']
if len(tFluxRenorm) <= 1: # if t Flux is empty
fset_cesium = pd.DataFrame({f: {'channel': {0: 0}} for f in features_general})
else:
fset_cesium = featurize.featurize_time_series(times=ttime,
values=tFluxRenorm,
errors=tFluxErrRenorm,
features_to_use=features_general)
cesium[pb] = fset_cesium
self.cesium = cesium
return cesium