def test_normalize_sensor_tags_not_ok():
with pytest.raises(SensorTagNormalizationError):
tag_list_as_list_of_strings_nonsense = [
NON_RESOLVABLE_TAG_NAME1,
NON_RESOLVABLE_TAG_NAME2,
]
normalize_sensor_tags(tag_list_as_list_of_strings_nonsense)
def test_load_series_dry_run(dates, ncs_reader):
valid_tag_list_no_asset = normalize_sensor_tags(["TRC-123", "TRC-321"])
for frame in ncs_reader.load_series(dates[0],
dates[1],
valid_tag_list_no_asset,
dry_run=True):
assert len(frame) == 0
def test_normalize_sensor_tags_ok(
good_input_tags, asset, default_asset, expected_output_tags
):
tag_list_as_list_of_sensor_tag = normalize_sensor_tags(
good_input_tags, asset, default_asset=default_asset
)
assert tag_list_as_list_of_sensor_tag == expected_output_tags
def test_with_conflicted_file_types(dates):
ncs_reader = NcsReader(AzureDLFileSystemMock(), remove_status_codes=[0])
valid_tag_list = normalize_sensor_tags(["TRC-324"])
series_gen = ncs_reader.load_series(dates[0], dates[1], valid_tag_list)
tags_series = [v for v in series_gen]
assert len(tags_series) == 1
trc_324_series = tags_series[0]
# Parquet file should be with 15 rows
assert len(trc_324_series) == 15
def test_parquet_files_lookup(dates):
ncs_reader = NcsReader(AzureDLFileSystemMock(), remove_status_codes=[0])
valid_tag_list = normalize_sensor_tags(["TRC-323"])
series_gen = ncs_reader.load_series(dates[0], dates[1], valid_tag_list)
tags_series = [v for v in series_gen]
assert len(tags_series) == 1
trc_323_series = tags_series[0]
assert trc_323_series.name == "TRC-323"
assert trc_323_series.dtype.name == "float64"
assert len(trc_323_series) == 20
def test_load_series_with_filter_bad_data(dates, remove_status_codes):
ncs_reader = NcsReader(AzureDLFileSystemMock(),
remove_status_codes=remove_status_codes)
valid_tag_list = normalize_sensor_tags(["TRC-322"])
series_gen = ncs_reader.load_series(dates[0], dates[1], valid_tag_list)
# Checks if the bad data from the files under tests/gordo/data_provider/data/datalake/TRC-322
# are filtered out. 20 rows exists, 5 of then have the value 0.
n_expected = 15 if remove_status_codes != [] else 20
assert all(len(series) == n_expected for series in series_gen)
def test_with_conflicted_file_types_with_preferable_csv(dates):
ncs_reader = NcsReader(AzureDLFileSystemMock(),
remove_status_codes=[0],
lookup_for=["csv"])
valid_tag_list = normalize_sensor_tags(["TRC-324"])
series_gen = ncs_reader.load_series(dates[0], dates[1], valid_tag_list)
tags_series = [v for v in series_gen]
assert len(tags_series) == 1
trc_324_series = tags_series[0]
# CSV file should be with 1 row
assert len(trc_324_series) == 1
def tags(self) -> typing.List[SensorTag]:
"""
The input tags for this model
Returns
-------
typing.List[SensorTag]
"""
return normalize_sensor_tags(
g.metadata["dataset"]["tag_list"],
asset=g.metadata["dataset"].get("asset"),
default_asset=g.metadata["dataset"].get("default_asset"),
)
def _get_default_dataset_config():
train_start_date = dateutil.parser.isoparse("2017-01-01T08:56:00+00:00")
train_end_date = dateutil.parser.isoparse("2017-01-01T10:01:00+00:00")
return {
"type":
"TimeSeriesDataset",
"train_start_date":
train_start_date,
"train_end_date":
train_end_date,
"tag_list":
normalize_sensor_tags(["TRC-FIQ -39-0706", "GRA-EM-23-0003ARV.PV"]),
"data_provider":
DataLakeProvider(),
}
def test_normalize_iroc_tags():
normalized_tags = normalize_sensor_tags(IROC_MANY_ASSETS_TAG_LIST)
assert normalized_tags == IROC_MANY_ASSETS_SENSOR_TAG_LIST
def test_can_handle_tag_unknow_prefix_raise(ncs_reader):
with pytest.raises(ValueError):
ncs_reader.can_handle_tag(normalize_sensor_tags(["XYZ-123"])[0])
def test_load_series_invalid_year(start_date, end_date, frame_len, ncs_reader):
valid_tag_list = normalize_sensor_tags(["TRC-123"])
frame = next(ncs_reader.load_series(start_date, end_date, valid_tag_list))
assert len(frame) == frame_len
def test_load_series_known_prefix(dates, ncs_reader):
valid_tag_list_no_asset = normalize_sensor_tags(["TRC-123", "TRC-321"])
for frame in ncs_reader.load_series(dates[0], dates[1],
valid_tag_list_no_asset):
assert len(frame) == 20
@pytest.fixture
def ncs_reader():
return NcsReader(AzureDLFileSystemMock())
@pytest.fixture
def dates():
return (
dateutil.parser.isoparse("2000-01-01T08:56:00+00:00"),
dateutil.parser.isoparse("2001-09-01T10:01:00+00:00"),
)
@pytest.mark.parametrize(
"tag_to_check",
[normalize_sensor_tags(["TRC-123"])[0],
SensorTag("XYZ-123", "1776-TROC")],
)
def test_can_handle_tag_ok(tag_to_check, ncs_reader):
assert ncs_reader.can_handle_tag(tag_to_check)
@pytest.mark.parametrize(
"tag_to_check",
[SensorTag("TRC-123", None),
SensorTag("XYZ-123", "123-XXX")])
def test_can_handle_tag_notok(tag_to_check, ncs_reader):
assert not ncs_reader.can_handle_tag(tag_to_check)
def test_can_handle_tag_unknow_prefix_raise(ncs_reader):
def get_machine_log_items(
machine: Machine) -> Tuple[List[Metric], List[Param]]:
"""
Create flat lists of MLflow logging entities from multilevel dictionary
For more information, see the mlflow docs:
https://www.mlflow.org/docs/latest/python_api/mlflow.tracking.html#mlflow.tracking.MlflowClient.log_batch
Parameters
----------
machine: Machine
Returns
-------
metrics: List[Metric]
List of MLFlow Metric objects to log.
params: List[Param]
List of MLFlow Param objects to log.
"""
metrics: List[Metric] = list()
build_metadata = machine.metadata.build_metadata
# Project/machine parameters
keys = ["project_name", "name"]
params = [Param(attr, getattr(machine, attr)) for attr in keys]
# Dataset parameters
dataset_keys = [
"train_start_date",
"train_end_date",
"resolution",
"row_filter",
"row_filter_buffer_size",
]
params.extend(
Param(k, str(getattr(machine.dataset, k))) for k in dataset_keys)
# Model parameters
model_keys = [
"model_creation_date", "model_builder_version", "model_offset"
]
params.extend(
Param(k, str(getattr(build_metadata.model, k))) for k in model_keys)
# Parse cross-validation split metadata
splits = build_metadata.model.cross_validation.splits
params.extend(Param(k, str(v)) for k, v in splits.items())
# Parse cross-validation metrics
tag_list = normalize_sensor_tags(machine.dataset.tag_list,
asset=machine.dataset.asset)
scores = build_metadata.model.cross_validation.scores
keys = sorted(list(scores.keys()))
subkeys = ["mean", "max", "min", "std"]
n_folds = len(scores[keys[0]]) - len(subkeys)
for k in keys:
# Skip per tag data, produces too many params for MLflow
if any([t.name in k for t in tag_list]):
continue
# Summary stats per metric
for sk in subkeys:
metrics.append(
Metric(f"{k}-{sk}", scores[k][f"fold-{sk}"], epoch_now(), 0))
# Append value for each fold with increasing steps
metrics.extend(
Metric(k, scores[k][f"fold-{i+1}"], epoch_now(), i)
for i in range(n_folds))
# Parse fit metrics
try:
meta_params = build_metadata.model.model_meta["history"]["params"]
except KeyError:
logger.debug(
"Key 'build-metadata.model.history.params' not found found in metadata."
)
else:
metrics.extend(
Metric(k, float(getattr(build_metadata.model, k)), epoch_now(), 0)
for k in ["model_training_duration_sec"])
for m in meta_params["metrics"]:
data = build_metadata.model.model_meta["history"][m]
metrics.extend(
Metric(m, float(x), timestamp=epoch_now(), step=i)
for i, x in enumerate(data))
params.extend(
Param(k, str(meta_params[k]))
for k in (p for p in meta_params if p != "metrics"))
return metrics, params
def __init__(
self,
train_start_date: Union[datetime, str],
train_end_date: Union[datetime, str],
tag_list: Sequence[Union[str, Dict, SensorTag]],
target_tag_list: Optional[Sequence[Union[str, Dict,
SensorTag]]] = None,
data_provider: Union[GordoBaseDataProvider, dict] = DataLakeProvider(),
resolution: Optional[str] = "10T",
row_filter: str = "",
aggregation_methods: Union[str, List[str], Callable] = "mean",
row_filter_buffer_size: int = 0,
asset: Optional[str] = None,
default_asset: Optional[str] = None,
n_samples_threshold: int = 0,
low_threshold=-1000,
high_threshold=50000,
interpolation_method: str = "linear_interpolation",
interpolation_limit: str = "8H",
filter_periods={},
):
"""
Creates a TimeSeriesDataset backed by a provided dataprovider.
A TimeSeriesDataset is a dataset backed by timeseries, but resampled,
aligned, and (optionally) filtered.
Parameters
----------
train_start_date: Union[datetime, str]
Earliest possible point in the dataset (inclusive)
train_end_date: Union[datetime, str]
Earliest possible point in the dataset (exclusive)
tag_list: Sequence[Union[str, Dict, sensor_tag.SensorTag]]
List of tags to include in the dataset. The elements can be strings,
dictionaries or SensorTag namedtuples.
target_tag_list: Sequence[List[Union[str, Dict, sensor_tag.SensorTag]]]
List of tags to set as the dataset y. These will be treated the same as
tag_list when fetching and pre-processing (resampling) but will be split
into the y return from ``.get_data()``
data_provider: Union[GordoBaseDataProvider, dict]
A dataprovider which can provide dataframes for tags from train_start_date to train_end_date
of which can also be a config definition from a data provider's ``.to_dict()`` method.
resolution: Optional[str]
The bucket size for grouping all incoming time data (e.g. "10T").
Available strings come from https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#dateoffset-objects
**Note**: If this parameter is ``None`` or ``False``, then _no_ aggregation/resampling is applied to the data.
row_filter: str
Filter on the rows. Only rows satisfying the filter will be in the dataset.
See :func:`gordo.machine.dataset.filter_rows.pandas_filter_rows` for
further documentation of the filter format.
aggregation_methods
Aggregation method(s) to use for the resampled buckets. If a single
resample method is provided then the resulting dataframe will have names
identical to the names of the series it got in. If several
aggregation-methods are provided then the resulting dataframe will
have a multi-level column index, with the series-name as the first level,
and the aggregation method as the second level.
See :py:func::`pandas.core.resample.Resampler#aggregate` for more
information on possible aggregation methods.
row_filter_buffer_size: int
Whatever elements are selected for removal based on the ``row_filter``, will also
have this amount of elements removed fore and aft.
Default is zero 0
asset: Optional[str]
Asset for which the tags are associated with.
default_asset: Optional[str]
Asset which will be used if `asset` is not provided and the tag is not
resolvable to a specific asset.
n_samples_threshold: int = 0
The threshold at which the generated DataFrame is considered to have too few rows of data.
interpolation_method: str
How should missing values be interpolated. Either forward fill (`ffill`) or by linear
interpolation (default, `linear_interpolation`).
interpolation_limit: str
Parameter sets how long from last valid data point values will be interpolated/forward filled.
Default is eight hours (`8H`).
If None, all missing values are interpolated/forward filled.
fiter_periods: dict
Performs a series of algorithms that drops noisy data is specified.
See `filter_periods` class for details.
"""
self.train_start_date = self._validate_dt(train_start_date)
self.train_end_date = self._validate_dt(train_end_date)
if self.train_start_date >= self.train_end_date:
raise ValueError(
f"train_end_date ({self.train_end_date}) must be after train_start_date ({self.train_start_date})"
)
self.tag_list = normalize_sensor_tags(list(tag_list), asset,
default_asset)
self.target_tag_list = (normalize_sensor_tags(list(target_tag_list),
asset, default_asset)
if target_tag_list else self.tag_list.copy())
self.resolution = resolution
self.data_provider = (data_provider
if not isinstance(data_provider, dict) else
GordoBaseDataProvider.from_dict(data_provider))
self.row_filter = row_filter
self.aggregation_methods = aggregation_methods
self.row_filter_buffer_size = row_filter_buffer_size
self.asset = asset
self.n_samples_threshold = n_samples_threshold
self.low_threshold = low_threshold
self.high_threshold = high_threshold
self.interpolation_method = interpolation_method
self.interpolation_limit = interpolation_limit
self.filter_periods = (FilterPeriods(granularity=self.resolution,
**filter_periods)
if filter_periods else None)
if not self.train_start_date.tzinfo or not self.train_end_date.tzinfo:
raise ValueError(
f"Timestamps ({self.train_start_date}, {self.train_end_date}) need to include timezone "
f"information")