import re
import sys
import six
import voluptuous
import voluptuous.humanize
from voluptuous import Schema, Optional, Any, All, Required, Length, Range, Msg, Match
Text = Any(six.text_type, six.binary_type)
id_regex = re.compile(r'^[a-z0-9-]+$')
feature_schema = Schema({
Match(id_regex): {
Required('title'): All(Text, Length(min=1)),
Required('description'): All(Text, Length(min=1)),
Required('bug-numbers'): All(Length(min=1), [All(int, Range(min=1))]),
Required('restart-required'): bool,
Required('type'):
'boolean', # In the future this may include other types
Optional('preference'): Text,
Optional('default-value'):
Any(bool, dict
), # the types of the keys here should match the value of `type`
Optional('is-public'): Any(bool, dict),
},
})
EXIT_OK = 0
EXIT_ERROR = 1
def check_mode(x):
if x in ["schedule", "none", "count_down"]:
return x
raise Invalid(f"invalid mode {x}")
def lb_dev_state(x):
if x in ["normal"]:
return x
raise Invalid(f"Invalid dev_state {x}")
TZ_SCHEMA = Schema(
{"zone_str": str, "dst_offset": int, "index": All(int, Range(min=0)), "tz_str": str}
)
CURRENT_CONSUMPTION_SCHEMA = Schema(
Any(
{
"voltage": Any(All(float, Range(min=0, max=300)), None),
"power": Any(Coerce(float, Range(min=0)), None),
"total": Any(Coerce(float, Range(min=0)), None),
"current": Any(All(float, Range(min=0)), None),
"voltage_mv": Any(
All(float, Range(min=0, max=300000)), int, None
), # TODO can this be int?
"power_mw": Any(Coerce(float, Range(min=0)), None),
"total_wh": Any(Coerce(float, Range(min=0)), None),
"current_ma": Any(
def max_num_segments(**kwargs):
return {
Required('max_num_segments'): All(Coerce(int), Range(min=1))
}
def test_range_excludes_nan():
s = Schema(Range(min=0, max=10))
assert_raises(MultipleInvalid, s, float('nan'))
def test_range_excludes_string():
s = Schema(Range(min=0, max=10))
assert_raises(MultipleInvalid, s, "abc")
def test_new_required_test():
schema = Schema({
'my_key': All(int, Range(1, 20)),
}, required=True)
assert_true(schema.required)
def test_range_outside():
s = Schema(Range(min=0, max=10))
assert_raises(MultipleInvalid, s, 12)
assert_raises(MultipleInvalid, s, -1)
from .util import dotdict
from .exceptions import ParametersError
from voluptuous import (All, Any, Coerce, Lower, Strip, Length, Range, Schema,
Required, PREVENT_EXTRA, MultipleInvalid)
from voluptuous.humanize import validate_with_humanized_errors
import os
intbool = Schema(lambda x: int(bool(x)))
twotupIntRange = Schema(All((All(int, Range(0, 999)), ), Length(min=2, max=2)))
def CTiterable(v):
if isinstance(v, int):
v = [v]
try:
iter(v)
except TypeError:
raise TypeError('Not an iterable object')
if not all([(isinstance(i, int) and i >= 0) for i in v]):
raise ValueError(
'All values in Channel/Time range must be integers >= 0')
return v
def smartbool(v):
if isinstance(v, bool):
return v
if isinstance(v, str):
if v.lower() in ['false', 'no', 'none', '', 'f']:
return False
def intRange(m, n):
return Schema(All(Coerce(int), Range(m, n)))
class TestSmartPlug(TestCase):
# these schemas should go to the mainlib as
# they can be useful when adding support for new features/devices
# as well as to check that faked devices are operating properly.
sysinfo_schema = Schema({
'active_mode': check_mode,
'alias': str,
'dev_name': str,
'deviceId': str,
'feature': str,
'fwId': str,
'hwId': str,
'hw_ver': str,
'icon_hash': str,
'latitude': All(float, Range(min=-90, max=90)),
'led_off': check_int_bool,
'longitude': All(float, Range(min=-180, max=180)),
'mac': check_mac,
'model': str,
'oemId': str,
'on_time': int,
'relay_state': int,
'rssi': All(int, Range(max=0)),
'sw_ver': str,
'type': str,
'updating': check_int_bool,
})
current_consumption_schema = Schema({
'voltage': All(float, Range(min=0, max=300)),
'power': All(float, Range(min=0)),
'total': All(float, Range(min=0)),
'current': All(float, Range(min=0)),
})
tz_schema = Schema({
'zone_str': str,
'dst_offset': int,
'index': All(int, Range(min=0)),
'tz_str': str,
})
def setUp(self):
self.plug = SmartPlug(PLUG_IP,
protocol=FakeTransportProtocol(sysinfo_hs110))
def tearDown(self):
self.plug = None
def test_initialize(self):
self.assertIsNotNone(self.plug.sys_info)
self.sysinfo_schema(self.plug.sys_info)
def test_initialize_invalid_connection(self):
plug = SmartPlug('127.0.0.1',
protocol=FakeTransportProtocol(sysinfo_hs110,
invalid=True))
with self.assertRaises(SmartPlugException):
plug.sys_info['model']
def test_query_helper(self):
with self.assertRaises(SmartPlugException):
self.plug._query_helper("test", "testcmd", {})
# TODO check for unwrapping?
@skipIf(SKIP_STATE_TESTS, "SKIP_STATE_TESTS is True, skipping")
def test_state(self):
def set_invalid(x):
self.plug.state = x
set_invalid_int = partial(set_invalid, 1234)
self.assertRaises(ValueError, set_invalid_int)
set_invalid_str = partial(set_invalid, "1234")
self.assertRaises(ValueError, set_invalid_str)
set_invalid_bool = partial(set_invalid, True)
self.assertRaises(ValueError, set_invalid_bool)
orig_state = self.plug.state
if orig_state == SmartPlug.SWITCH_STATE_OFF:
self.plug.state = "ON"
self.assertTrue(self.plug.state == SmartPlug.SWITCH_STATE_ON)
self.plug.state = "OFF"
self.assertTrue(self.plug.state == SmartPlug.SWITCH_STATE_OFF)
elif orig_state == SmartPlug.SWITCH_STATE_ON:
self.plug.state = "OFF"
self.assertTrue(self.plug.state == SmartPlug.SWITCH_STATE_OFF)
self.plug.state = "ON"
self.assertTrue(self.plug.state == SmartPlug.SWITCH_STATE_ON)
elif orig_state == SmartPlug.SWITCH_STATE_UNKNOWN:
self.fail("can't test for unknown state")
def test_get_sysinfo(self):
# initialize checks for this already, but just to be sure
self.sysinfo_schema(self.plug.get_sysinfo())
@skipIf(SKIP_STATE_TESTS, "SKIP_STATE_TESTS is True, skipping")
def test_turns_and_isses(self):
orig_state = self.plug.is_on
if orig_state:
self.plug.turn_off()
self.assertFalse(self.plug.is_on)
self.assertTrue(self.plug.is_off)
self.plug.turn_on()
self.assertTrue(self.plug.is_on)
else:
self.plug.turn_on()
self.assertFalse(self.plug.is_off)
self.assertTrue(self.plug.is_on)
self.plug.turn_off()
self.assertTrue(self.plug.is_off)
def test_has_emeter(self):
# a not so nice way for checking for emeter availability..
if "110" in self.plug.sys_info["model"]:
self.assertTrue(self.plug.has_emeter)
else:
self.assertFalse(self.plug.has_emeter)
def test_get_emeter_realtime(self):
self.current_consumption_schema((self.plug.get_emeter_realtime()))
def test_get_emeter_daily(self):
self.assertEqual(self.plug.get_emeter_daily(year=1900, month=1), {})
k, v = self.plug.get_emeter_daily().popitem()
self.assertTrue(isinstance(k, int))
self.assertTrue(isinstance(v, float))
def test_get_emeter_monthly(self):
self.assertEqual(self.plug.get_emeter_monthly(year=1900), {})
d = self.plug.get_emeter_monthly()
k, v = d.popitem()
self.assertTrue(isinstance(k, int))
self.assertTrue(isinstance(v, float))
@skip("not clearing your stats..")
def test_erase_emeter_stats(self):
self.fail()
def test_current_consumption(self):
x = self.plug.current_consumption()
self.assertTrue(isinstance(x, float))
self.assertTrue(x >= 0.0)
def test_identify(self):
ident = self.plug.identify()
self.assertTrue(isinstance(ident, tuple))
self.assertTrue(len(ident) == 3)
def test_alias(self):
test_alias = "TEST1234"
original = self.plug.alias
self.assertTrue(isinstance(original, str))
self.plug.alias = test_alias
self.assertEqual(self.plug.alias, test_alias)
self.plug.alias = original
self.assertEqual(self.plug.alias, original)
def test_led(self):
original = self.plug.led
self.plug.led = False
self.assertFalse(self.plug.led)
self.plug.led = True
self.assertTrue(self.plug.led)
self.plug.led = original
def test_icon(self):
self.assertEqual(set(self.plug.icon.keys()), {'icon', 'hash'})
def test_time(self):
self.assertTrue(isinstance(self.plug.time, datetime.datetime))
# TODO check setting?
def test_timezone(self):
self.tz_schema(self.plug.timezone)
def test_hw_info(self):
self.sysinfo_schema(self.plug.hw_info)
def test_on_since(self):
self.assertTrue(isinstance(self.plug.on_since, datetime.datetime))
def test_location(self):
self.sysinfo_schema(self.plug.location)
def test_location_i(self):
plug_i = SmartPlug(PLUG_IP,
protocol=FakeTransportProtocol(sysinfo_hs105))
self.sysinfo_schema(plug_i.location)
def test_rssi(self):
self.sysinfo_schema({'rssi': self.plug.rssi}) # wrapping for vol
def test_mac(self):
self.sysinfo_schema({'mac': self.plug.mac}) # wrapping for val
"""Recorder config."""
from voluptuous import All, Optional, Range, Schema
from .config_logging import SCHEMA as LOGGING_SCHEMA, LoggingConfig
SCHEMA = Schema({
Optional("lookback", default=5): All(int, Range(min=0)),
Optional("timeout", default=10): All(int, Range(min=0)),
Optional("retain", default=7): All(int, Range(min=1)),
Optional("folder", default="/recordings"): str,
Optional("extension", default="mp4"): str,
Optional("hwaccel_args", default=[]): [str],
Optional("codec", default="copy"): str,
Optional("audio_codec", default=""): str,
Optional("filter_args", default=[]): [str],
Optional("segments_folder", default="/segments"): str,
Optional("thumbnail", default={}): {
Optional("save_to_disk", default=False): bool,
Optional("send_to_mqtt", default=False): bool,
},
Optional("logging"): LOGGING_SCHEMA,
})
class Thumbnail:
"""Thumbnail config."""
def __init__(self, thumbnail):
self._save_to_disk = thumbnail["save_to_disk"]
self._send_to_mqtt = thumbnail["send_to_mqtt"]
@property
submission_schema = Schema({
Required("tid"):
check(("This does not look like a valid tid.", [str, Length(max=100)])),
Required("pid"):
check(("This does not look like a valid pid.", [str, Length(max=100)])),
Required("key"):
check(("This does not look like a valid key.", [str, Length(max=100)]))
})
problem_schema = Schema({
Required("name"):
check(("The problem's display name must be a string.", [str])),
Required("sanitized_name"):
check(("The problems's sanitized name must be a string.", [str])),
Required("score"):
check(("Score must be a positive integer.", [int, Range(min=0)])),
Required("author"):
check(("Author must be a string.", [str])),
Required("category"):
check(("Category must be a string.", [str])),
Required("instances"):
check(("The instances must be a list.", [list])),
Required("hints"):
check(("Hints must be a list.", [list])),
"description":
check(("The problem description must be a string.", [str])),
"version":
check(("A version must be a string.", [str])),
"tags":
check(("Tags must be described as a list.", [list])),
"organization":
from typing import Union
import flask
from voluptuous import All, Any, In, Length, Optional, Range, Schema
from core import db
from core.utils import require_permission, validate_data
from core.validators import BoolGET
from forums.models import Forum, ForumCategory, ForumThread
from . import bp
app = flask.current_app
VIEW_FORUM_SCHEMA = Schema({
'page': All(int, Range(min=0, max=2147483648)),
'limit': All(int, In((25, 50, 100))),
'include_dead': BoolGET,
})
@bp.route('/forums/<int:id>', methods=['GET'])
@require_permission('forums_view')
@validate_data(VIEW_FORUM_SCHEMA)
def view_forum(id: int,
page: int = 1,
limit: int = 50,
include_dead: bool = False) -> flask.Response:
"""
This endpoint allows users to view details about a forum and its threads.
class DonutModel(Model):
"""
Time-series VAE model, "Donut"
"""
TYPE = 'donut'
SCHEMA = Model.SCHEMA.extend({
Required('bucket_interval'):
schemas.TimeDelta(
min=0,
min_included=False,
),
Required('interval'):
schemas.TimeDelta(min=0, min_included=False),
Required('offset'):
schemas.TimeDelta(min=0),
Required('span'):
Any(None, "auto", All(int, Range(min=1))),
Optional('min_span'):
All(int, Range(min=1)),
Optional('max_span'):
All(int, Range(min=1)),
Optional('seasonality', default=DEFAULT_SEASONALITY):
schemas.seasonality,
Optional('forecast'):
Any(None, "auto", All(int, Range(min=1))),
Optional('grace_period', default=0):
schemas.TimeDelta(min=0, min_included=True),
'default_datasink':
schemas.key,
})
def __init__(self, settings, state=None):
global _hp_span_min, _hp_span_max
super().__init__(settings, state)
settings = self.validate(settings)
self.bucket_interval = parse_timedelta(
settings.get('bucket_interval')).total_seconds()
self.interval = parse_timedelta(
settings.get('interval')).total_seconds()
self.offset = parse_timedelta(settings.get('offset')).total_seconds()
self.span = settings.get('span')
self.means = None
self.stds = None
self.scores = None
self._keras_model = None
self._encoder_model = None
self._decoder_model = None
if self.span is None or self.span == "auto":
self.min_span = settings.get('min_span') or _hp_span_min
self.max_span = settings.get('max_span') or _hp_span_max
else:
self.min_span = self.span
self.max_span = self.span
self.grace_period = parse_timedelta(
settings['grace_period']).total_seconds()
self.current_eval = None
if len(self.features) > 1:
raise errors.LoudMLException(
"This model type supports one unique feature")
def enum_features(self, is_input=None, is_output=None):
j = 0
for i, feature in enumerate(self.features):
if feature.is_input == is_input or feature.is_output == is_output:
yield i, j, feature
j += 1
@property
def type(self):
return self.TYPE
@property
def W(self):
return self.span
def get_hp_span(self, label):
if (self.max_span - self.min_span) <= 0:
space = self.span
else:
space = self.min_span + hp.randint(label,
(self.max_span - self.min_span))
return space
def set_run_params(self, params=None):
"""
Set running parameters to make them persistent
"""
if params is None:
self._settings.pop('run', None)
else:
self._settings['run'] = params
def set_run_state(self, params=None):
"""
Set running forecast parameters to make them persistent
"""
if params is None:
self._state.pop('run', None)
else:
self._state['run'] = params
def get_run_state(self):
return self._state.get('run') or {}
def compute_nb_buckets(self, from_ts, to_ts):
"""
Compute the number of bucket between `from_ts` and `to_ts`
"""
return int((to_ts - from_ts) / self.bucket_interval) + 2
def apply_defaults(self, x):
"""
Apply default feature value to np array
"""
feature = self.features[0]
if feature.default == "previous":
previous = None
for j, value in enumerate(x):
if np.isnan(value):
x[j] = previous
else:
previous = x[j]
elif not np.isnan(feature.default):
x[np.isnan(x)] = feature.default
def scale_dataset(
self,
dataset,
):
"""
Scale dataset values
"""
out = _get_scores(
dataset,
_mean=self.means[0],
_std=self.stds[0],
)
return out
def unscale_dataset(
self,
dataset,
):
"""
Revert scaling dataset values
"""
out = _revert_scores(
dataset,
_mean=self.means[0],
_std=self.stds[0],
)
return out
def stat_dataset(self, dataset):
"""
Compute dataset sets and keep them as reference
"""
self.means = np.array([np.nanmean(dataset, axis=0)])
self.stds = np.array([np.nanstd(dataset, axis=0)])
self.stds[self.stds == 0] = 1.0
def set_auto_threshold(self):
"""
Compute best threshold values automatically
"""
# 68–95–99.7 three-sigma rule
self.min_threshold = 68
self.max_threshold = 99.7
def _set_xpu_config(self, num_cpus, num_gpus):
config = tf.ConfigProto(
allow_soft_placement=True,
device_count={
'CPU': num_cpus,
'GPU': num_gpus
},
)
config.gpu_options.allow_growth = True
# config.log_device_placement = True
# config.intra_op_parallelism_threads=num_cores
# config.inter_op_parallelism_threads=num_cores
sess = tf.Session(config=config)
K.set_session(sess)
def _train_on_dataset(
self,
dataset,
train_size=0.67,
batch_size=64,
num_epochs=100,
num_cpus=1,
num_gpus=0,
max_evals=None,
progress_cb=None,
abnormal=None,
):
if max_evals is None:
max_evals = self.settings.get('max_evals',
21) # latent_dim*intermediate_dim
self.current_eval = 0
self.stat_dataset(dataset)
dataset = self.scale_dataset(dataset)
def cross_val_model(params):
keras_model = None
# Destroys the current TF graph and creates a new one.
# Useful to avoid clutter from old models / layers.
K.clear_session()
self._set_xpu_config(num_cpus, num_gpus)
self.span = W = params.span
(X_miss, X_train), (X_miss_val, X_test) = self.train_test_split(
dataset,
train_size=train_size,
abnormal=abnormal,
)
if len(X_train) == 0:
raise errors.NoData("insufficient training data")
if len(X_test) == 0:
raise errors.NoData("insufficient validation data")
# expected input data shape: (batch_size, timesteps,)
# network parameters
input_shape = (W, )
intermediate_dim = params.intermediate_dim
latent_dim = params.latent_dim
# VAE model = encoder + decoder
# build encoder model
main_input = Input(shape=input_shape)
aux_input = Input(
shape=input_shape) # bool vector to flag missing data points
aux_output = Lambda(lambda x: x)(aux_input)
x = Dense(intermediate_dim,
kernel_regularizer=regularizers.l2(0.01),
activation='relu')(main_input)
z_mean = Dense(latent_dim, name='z_mean')(x)
z_log_var = Dense(latent_dim, name='z_log_var')(x)
# use reparameterization trick to push the sampling out as input
# note that "output_shape" isn't necessary with the TensorFlow backend
z = Lambda(sampling, output_shape=(latent_dim, ),
name='z')([z_mean, z_log_var])
# build decoder model
x = Dense(intermediate_dim,
kernel_regularizer=regularizers.l2(0.01),
activation='relu',
name='dense_1')(z)
main_output = Dense(W, activation='linear', name='dense_2')(x)
# instantiate Donut model
keras_model = _Model([main_input, aux_input],
[main_output, aux_output],
name='donut')
add_loss(keras_model, W)
optimizer_cls = None
if params.optimizer == 'adam':
optimizer_cls = tf.keras.optimizers.Adam()
keras_model.compile(optimizer=optimizer_cls, )
_stop = EarlyStopping(
monitor='val_loss',
patience=5,
verbose=_verbose,
mode='auto',
)
keras_model.fit_generator(
generator(X_train, X_miss, batch_size, keras_model),
epochs=num_epochs,
steps_per_epoch=len(X_train) / batch_size,
verbose=_verbose,
validation_data=([X_test, X_miss_val], None),
callbacks=[_stop],
workers=0, # https://github.com/keras-team/keras/issues/5511
)
# How well did it do?
score = keras_model.evaluate(
[X_test, X_miss_val],
batch_size=batch_size,
verbose=_verbose,
)
self.current_eval += 1
if progress_cb is not None:
progress_cb(self.current_eval, max_evals)
return score, keras_model
hyperparameters = HyperParameters()
# Parameter search space
def objective(args):
hyperparameters.assign(args)
try:
score, _ = cross_val_model(hyperparameters)
return {'loss': score, 'status': STATUS_OK}
except Exception as exn:
logging.warning("iteration failed: %s", exn)
return {'loss': None, 'status': STATUS_FAIL}
space = hp.choice('case', [{
'span':
self.get_hp_span('span'),
'latent_dim':
hp.choice('latent_dim', [3, 5, 8]),
'intermediate_dim':
hp.choice('i1', [21, 34, 55, 89, 144, 233, 377]),
'optimizer':
hp.choice('optimizer', ['adam']),
}])
# The Trials object will store details of each iteration
trials = Trials()
# Run the hyperparameter search using the tpe algorithm
try:
best = fmin(
objective,
space,
algo=tpe.suggest,
max_evals=max_evals,
trials=trials,
)
except ValueError:
raise errors.NoData(
"training failed, try to increase the time range")
# Get the values of the optimal parameters
best_params = space_eval(space, best)
score, self._keras_model = cross_val_model(
HyperParameters(best_params))
self.span = best_params['span']
return (best_params, score)
def _train_ckpt_on_dataset(
self,
dataset,
train_size=0.67,
batch_size=64,
num_epochs=100,
progress_cb=None,
abnormal=None,
):
self.current_eval = 0
self.stat_dataset(dataset)
dataset = self.scale_dataset(dataset)
(X_miss, X_train), (X_miss_val, X_test) = self.train_test_split(
dataset,
train_size=train_size,
)
_stop = EarlyStopping(
monitor='val_loss',
patience=5,
verbose=_verbose,
mode='auto',
)
self._keras_model.fit(
[X_train, X_miss],
epochs=num_epochs,
batch_size=batch_size,
verbose=_verbose,
validation_data=([X_test, X_miss_val], None),
callbacks=[_stop],
)
# How well did it do?
score = self._keras_model.evaluate(
[X_test, X_miss_val],
batch_size=batch_size,
verbose=_verbose,
)
return score
def compute_bucket_scores(self, y_true, y_pred, y_low, y_high):
"""
Compute scores and mean squared error
"""
feature = self.features[0]
diff = y_true - y_pred
ano_type = feature.anomaly_type
mu = (y_low + y_high) / 2.0
std = (y_high - mu) / 3.0
score = 2 * norm.cdf(abs(y_true - mu), loc=0, scale=std) - 1
# Required to handle the 'low' condition
if diff < 0:
score *= -1
if ano_type == 'low':
score = -min(score, 0)
elif ano_type == 'high':
score = max(score, 0)
else:
score = abs(score)
score = 100 * max(0, min(1, score))
mse = np.nanmean((diff**2), axis=None)
return score, mse
def compute_scores(self, observed, predicted, low, high):
"""
Compute timeseries scores and MSE
"""
nb_buckets = len(observed)
scores = np.empty((nb_buckets, ), dtype=float)
mses = np.empty((nb_buckets), dtype=float)
for i in range(nb_buckets):
scores[i], mses[i] = self.compute_bucket_scores(
observed[i],
predicted[i],
low[i],
high[i],
)
return scores, mses
def _format_dataset(self, x, accept_missing=True, abnormal=None):
"""
Format dataset for time-series training & inference
input:
[v0, v1, v2, v3, v4 ..., vn]
len: W
output:
missing = [0, 0, 1..., 0]
X = [
[v0, v1, v2], # span = W
[v1, v2, v3],
[v2, v3, v4],
...
[..., .., vn],
]
Buckets with missing values are flagged in the missing array.
"""
missing = []
data_x = []
for i in range(len(x) - self.W + 1):
j = i + self.W
if accept_missing or not np.isnan(x[i:j]).any():
# arxiv.org/abs/1802.03903
# set user defined abnormal data points to zero
if abnormal is None:
is_nan = np.isnan(x[i:j])
else:
is_nan = np.logical_or(
np.isnan(x[i:j]),
abnormal[i:j],
)
missing.append(is_nan)
_x = np.copy(x[i:j])
# set missing points to zero
_x[is_nan] = 0.0
data_x.append(_x)
return np.array(missing), np.array(data_x)
def train_test_split(self, dataset, abnormal=None, train_size=0.67):
"""
Splits data to training and testing parts
"""
ntrn = round(len(dataset) * train_size)
X_train_missing, X_train = self._format_dataset(dataset[0:ntrn],
abnormal=abnormal)
X_test_missing, X_test = self._format_dataset(dataset[ntrn:])
return (X_train_missing, X_train), (X_test_missing, X_test)
def train(
self,
datasource,
from_date,
to_date="now",
train_size=0.67,
batch_size=256,
num_epochs=100,
num_cpus=1,
num_gpus=0,
max_evals=None,
progress_cb=None,
incremental=False,
windows=[],
):
"""
Train model
"""
self.means, self.stds = None, None
self.scores = None
period = self.build_date_range(from_date, to_date)
logging.info(
"train(%s) range=%s train_size=%f batch_size=%d epochs=%d)",
self.name,
period,
train_size,
batch_size,
num_epochs,
)
# Prepare dataset
nb_buckets = self.compute_nb_buckets(period.from_ts, period.to_ts)
dataset = np.full((nb_buckets, ), np.nan, dtype=float)
abnormal = _format_windows(
period.from_ts,
period.to_ts,
self.bucket_interval,
windows,
)
# Fill dataset
data = datasource.get_times_data(self, period.from_ts, period.to_ts)
# FIXME: query abnormal points flagged
i = None
for i, (_, val, timeval) in enumerate(data):
dataset[i] = val
if i is None:
raise errors.NoData(
"no data found for time range {}".format(period))
self.apply_defaults(dataset)
nb_buckets_found = i + 1
if nb_buckets_found < nb_buckets:
dataset = np.resize(dataset, (nb_buckets_found, ))
logging.info("found %d time periods", nb_buckets_found)
if incremental:
best_params = self._state.get('best_params', dict())
# Destroys the current TF graph and creates a new one.
# Useful to avoid clutter from old models / layers.
self.load(num_cpus, num_gpus)
score = self._train_ckpt_on_dataset(
dataset,
train_size,
batch_size,
num_epochs,
progress_cb=progress_cb,
abnormal=abnormal,
)
else:
best_params, score = self._train_on_dataset(
dataset,
train_size,
batch_size,
num_epochs,
num_cpus,
num_gpus,
max_evals,
progress_cb=progress_cb,
abnormal=abnormal,
)
self.current_eval = None
for key, val in best_params.items():
if not isinstance(val, str) and \
not isinstance(val, int) and \
not isinstance(val, float):
best_params[key] = np.asscalar(val)
model_b64 = _serialize_keras_model(self._keras_model)
self._state = {
'h5py': model_b64,
'best_params': best_params,
'means': self.means.tolist(),
'stds': self.stds.tolist(),
'loss': score,
}
self.unload()
#prediction = self.predict(
# datasource,
# from_date,
# to_date,
# num_cpus=num_cpus,
# num_gpus=num_gpus,
#)
#prediction.stat()
return {
'loss': score,
}
def unload(self):
"""
Unload current model
"""
self._keras_model = None
self._encoder_model = None
self._decoder_model = None
K.clear_session()
def load(self, num_cpus, num_gpus):
"""
Load current model
"""
if not self.is_trained:
raise errors.ModelNotTrained()
if self._keras_model:
# Already loaded
return
K.clear_session()
self._set_xpu_config(num_cpus, num_gpus)
if self._state.get('h5py', None) is not None:
self._keras_model = _load_keras_model(self._state.get('h5py'))
# instantiate encoder model
self._encoder_model = _get_encoder(self._keras_model)
# instantiate decoder model
self._decoder_model = _get_decoder(self._keras_model)
else:
raise errors.ModelNotTrained()
if 'means' in self._state:
self.means = np.array(self._state['means'])
if 'stds' in self._state:
self.stds = np.array(self._state['stds'])
if 'scores' in self._state:
self.scores = np.array(self._state['scores'])
if self.min_threshold == 0 and self.max_threshold == 0:
self.set_auto_threshold()
logging.info(
"setting threshold range min=%f max=%f",
self.min_threshold,
self.max_threshold,
)
@property
def is_trained(self):
"""
Tells if model is trained
"""
return self._state is not None and ('weights' in self._state
or 'h5py' in self._state)
@property
def _span(self):
if self._state and 'span' in self._state['best_params']:
return self._state['best_params']['span']
else:
return self.span
@property
def _window(self):
return self._span
def predict(
self,
datasource,
from_date,
to_date,
num_cpus=1,
num_gpus=0,
):
global g_mcmc_count
global g_mc_count
global g_mc_batch_size
period = self.build_date_range(from_date, to_date)
# This is the number of buckets that the function MUST return
predict_len = int(
(period.to_ts - period.from_ts) / self.bucket_interval)
logging.info("predict(%s) range=%s", self.name, period)
self.load(num_cpus, num_gpus)
# Build history time range
# Extra data are required to predict first buckets
_window = self._window - 1
hist = DateRange(
period.from_ts - _window * self.bucket_interval,
period.to_ts,
)
# Prepare dataset
nb_buckets = int((hist.to_ts - hist.from_ts) / self.bucket_interval)
dataset = np.full((nb_buckets, ), np.nan, dtype=float)
X = []
# Fill dataset
logging.info("extracting data for range=%s", hist)
data = datasource.get_times_data(self, hist.from_ts, hist.to_ts)
# Only a subset of history will be used for computing the prediction
X_until = None # right bound for prediction
i = None
for i, (_, val, timeval) in enumerate(data):
dataset[i] = val
dt = make_datetime(timeval)
ts = dt.timestamp()
if ts < period.to_ts:
X.append(make_ts(timeval))
X_until = i + 1
if i is None:
raise errors.NoData("no data found for time range {}".format(hist))
self.apply_defaults(dataset)
nb_buckets_found = i + 1
if nb_buckets_found < nb_buckets:
dataset = np.resize(dataset, (nb_buckets_found, ))
logging.info("found %d time periods", nb_buckets_found)
real = np.copy(dataset)
norm_dataset = self.scale_dataset(dataset)
missing, X_test = self._format_dataset(norm_dataset[:X_until])
if len(X_test) == 0:
raise errors.LoudMLException("not enough data for prediction")
# force last col to missing
missing[:, -1] = True
logging.info("generating prediction")
x_ = X_test.copy()
# MCMC
for _ in range(g_mcmc_count):
z_mean, _, _ = self._encoder_model.predict(
[x_, missing], batch_size=g_mc_batch_size)
x_decoded = self._decoder_model.predict(z_mean,
batch_size=g_mc_batch_size)
x_[missing] = x_decoded[missing]
y = np.full((predict_len, ), np.nan, dtype=float)
y_low = np.full((predict_len, ), np.nan, dtype=float)
y_high = np.full((predict_len, ), np.nan, dtype=float)
no_missing_point = np.full((g_mc_count, self.W), False, dtype=bool)
for j, x in enumerate(x_):
y[j] = x[-1]
# MC integration
_, _, Z = self._encoder_model.predict(
[np.tile(x, [g_mc_count, 1]), no_missing_point],
batch_size=g_mc_batch_size,
)
x_decoded = self._decoder_model.predict(Z,
batch_size=g_mc_batch_size)
std = np.std(x_decoded[:, -1])
y_low[j] = x[-1] - 3 * std
y_high[j] = x[-1] + 3 * std
y = self.unscale_dataset(y)
y_low = self.unscale_dataset(y_low)
y_high = self.unscale_dataset(y_high)
# Build final result
timestamps = X[_window:]
shape = (predict_len, len(self.features))
observed = np.full(shape, np.nan, dtype=float)
observed = real[_window:]
self.apply_defaults(observed)
self.apply_defaults(y)
return TimeSeriesPrediction(
self,
timestamps=timestamps,
observed=observed,
predicted=y,
lower=y_low,
upper=y_high,
)
def generate_fake_prediction(self):
now_ts = datetime.datetime.now().timestamp()
timestamps = [
now_ts - 2 * self.bucket_interval,
now_ts - self.bucket_interval,
now_ts,
]
normal = [0.0] * len(self.features)
anomaly = [sys.float_info.max] * len(self.features)
return TimeSeriesPrediction(
self,
timestamps=timestamps,
observed=np.array([normal, anomaly, normal]),
predicted=np.array([normal, normal, normal]),
)
def forecast(
self,
datasource,
from_date,
to_date,
percent_interval=0.68,
percent_noise=0,
num_cpus=1,
num_gpus=0,
):
global g_mcmc_count
global g_mc_count
global g_mc_batch_size
period = self.build_date_range(from_date, to_date)
# This is the number of buckets that the function MUST return
forecast_len = int(
(period.to_ts - period.from_ts) / self.bucket_interval)
logging.info("forecast(%s) range=%s", self.name, period)
self.load(num_cpus, num_gpus)
# Build history time range
# Extra data are required to predict first buckets
_window = self._window - 1
hist = DateRange(
period.from_ts - _window * self.bucket_interval,
period.to_ts,
)
# Prepare dataset
nb_buckets = int((hist.to_ts - hist.from_ts) / self.bucket_interval)
dataset = np.full((nb_buckets, ), np.nan, dtype=float)
X = []
# Fill dataset
logging.info("extracting data for range=%s", hist)
data = datasource.get_times_data(self, hist.from_ts, hist.to_ts)
# Only a subset of history will be used for computing the prediction
X_until = None # right bound for prediction
i = None
for i, (_, val, timeval) in enumerate(data):
dataset[i] = val
dt = make_datetime(timeval)
ts = dt.timestamp()
if ts < period.to_ts:
X.append(make_ts(timeval))
X_until = i + 1
if i is None:
raise errors.NoData("no data found for time range {}".format(hist))
self.apply_defaults(dataset)
nb_buckets_found = i + 1
if nb_buckets_found < nb_buckets:
dataset = np.resize(dataset, (nb_buckets_found, ))
logging.info("found %d time periods", nb_buckets_found)
real = np.copy(dataset)
norm_dataset = self.scale_dataset(dataset)
_, X_test = self._format_dataset(norm_dataset[:X_until])
if len(X_test) == 0:
raise errors.LoudMLException("not enough data for prediction")
logging.info("generating prediction")
x_ = X_test.copy()
p = norm().ppf(1 - (1 - percent_interval) / 2)
missing = np.full((self._window, ), False, dtype=bool)
# force last col to missing
missing[-1] = True
y = np.full((forecast_len, ), np.nan, dtype=float)
y_low = np.full((forecast_len, ), np.nan, dtype=float)
y_high = np.full((forecast_len, ), np.nan, dtype=float)
x = x_[0]
noise = percent_noise * float(self.bucket_interval) / (24 * 3600)
for j, _ in enumerate(x_):
# MCMC
for _ in range(g_mcmc_count):
z_mean, _, _ = self._encoder_model.predict(
[np.array([x]), np.array([missing])],
batch_size=g_mc_batch_size,
)
x_decoded = self._decoder_model.predict(
z_mean, batch_size=g_mc_batch_size)
x[missing] = x_decoded[0][missing]
# uncertainty is modeled using a random uniform noise distribution
# that increases over time
expand = np.random.uniform(-noise * j, noise * j, len(x))
x *= 1 + expand
# MC integration
_, _, Z = self._encoder_model.predict(
[
np.tile(x, [g_mc_count, 1]),
np.tile(missing, [g_mc_count, 1])
],
batch_size=g_mc_batch_size,
)
x_decoded = self._decoder_model.predict(Z,
batch_size=g_mc_batch_size)
std = np.std(x_decoded[:, -1])
y_low[j] = x[-1] - p * std
y_high[j] = x[-1] + p * std
y[j] = x[-1]
x = np.roll(x, -1)
# set missing point to zero
x[-1] = 0
y = self.unscale_dataset(y)
y_low = self.unscale_dataset(y_low)
y_high = self.unscale_dataset(y_high)
# Build final result
timestamps = X[_window:]
shape = (forecast_len, len(self.features))
observed = np.full(shape, np.nan, dtype=float)
observed = real[_window:]
self.apply_defaults(observed)
self.apply_defaults(y)
return TimeSeriesPrediction(
self,
timestamps=timestamps,
observed=observed,
predicted=y,
lower=y_low,
upper=y_high,
)
def detect_anomalies(self, prediction, hooks=[]):
"""
Detect anomalies on observed data by comparing them to the values
predicted by the model
"""
prediction.stat()
stats = []
anomaly_indices = []
for i, ts in enumerate(prediction.timestamps):
last_anomaly_ts = self._state.get('last_anomaly_ts', 0)
in_grace_period = (ts - last_anomaly_ts) < self.grace_period
dt = ts_to_datetime(ts)
date_str = datetime_to_str(dt)
is_anomaly = False
anomalies = {}
predicted = prediction.predicted[i]
observed = prediction.observed[i]
score = prediction.scores[i]
mse = prediction.mses[i]
max_score = 0
feature = self.features[0]
max_score = max(max_score, score)
if (not in_grace_period) and score >= self.max_threshold:
anomalies[feature.name] = {
'type': 'low' if observed < predicted else 'high',
'score': score,
}
if len(anomalies):
is_anomaly = True
anomaly_indices.append(i)
anomaly = self._state.get('anomaly')
if anomaly is None:
if is_anomaly:
# This is a new anomaly
# TODO have a Model.logger to prefix all logs with model name
logging.warning(
"detected anomaly for model '%s' at %s (score = %.1f)",
self.name,
date_str,
max_score,
)
self._state['anomaly'] = {
'start_ts': ts,
'max_score': max_score,
}
for hook in hooks:
logging.debug("notifying '%s' hook", hook.name)
data = prediction.format_bucket_data(i)
try:
hook.on_anomaly_start(
dt=dt,
score=max_score,
predicted=data['predicted'],
observed=data['observed'],
anomalies=anomalies,
)
except Exception as exn:
# XXX: catch all the exception to avoid
# interruption
logging.exception(exn)
else:
if is_anomaly:
anomaly['max_score'] = max(anomaly['max_score'], max_score)
logging.warning(
"anomaly still in progress for model '%s' at %s (score = %.1f)",
self.name,
date_str,
max_score,
)
elif score < self.min_threshold:
logging.info(
"anomaly ended for model '%s' at %s (score = %.1f)",
self.name,
date_str,
max_score,
)
for hook in hooks:
logging.debug("notifying '%s' hook", hook.name)
hook.on_anomaly_end(dt, max_score)
self._state['anomaly'] = None
self._state['last_anomaly_ts'] = ts
stats.append({
'mse': nan_to_none(mse),
'score': max_score,
'anomaly': is_anomaly,
'anomalies': anomalies,
})
prediction.stats = stats
prediction.anomaly_indices = anomaly_indices
def predict2(
self,
datasource,
from_date,
to_date,
mse_rtol,
_state={},
num_cpus=1,
num_gpus=0,
):
return self.predict(
datasource,
from_date,
to_date,
num_cpus=num_cpus,
num_gpus=num_gpus,
)
def plot_results(
self,
datasource,
from_date,
to_date,
num_cpus=1,
num_gpus=0,
x_dim=-1,
y_dim=-1,
output=None,
):
"""
# Arguments:
models (tuple): encoder and decoder models
data (tuple): test data and label
model_name (string): which model is using this function
"""
global g_mc_batch_size
# Agg = Anti-grain geometry engine
# running inside a Docker image. No Xwindow
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
period = self.build_date_range(from_date, to_date)
logging.info("plot_results(%s) range=%s", self.name, period)
self.load(num_cpus, num_gpus)
_, latent_dim = self._encoder_model.outputs[0].get_shape()
# Build history time range
# Extra data are required to predict first buckets
_window = self._window - 1
hist = DateRange(
period.from_ts - _window * self.bucket_interval,
period.to_ts,
)
# Prepare dataset
nb_buckets = int((hist.to_ts - hist.from_ts) / self.bucket_interval)
dataset = np.full((nb_buckets, ), np.nan, dtype=float)
# Fill dataset
logging.info("extracting data for range=%s", hist)
data = datasource.get_times_data(self, hist.from_ts, hist.to_ts)
# Only a subset of history will be used for computing the prediction
X_until = None # right bound for prediction
i = None
for i, (_, val, timeval) in enumerate(data):
dataset[i] = val
dt = make_datetime(timeval)
ts = dt.timestamp()
if ts < period.to_ts:
X_until = i + 1
if i is None:
raise errors.NoData("no data found for time range {}".format(hist))
self.apply_defaults(dataset)
nb_buckets_found = i + 1
if nb_buckets_found < nb_buckets:
dataset = np.resize(dataset, (nb_buckets_found, ))
logging.info("found %d time periods", nb_buckets_found)
norm_dataset = self.scale_dataset(dataset)
X_miss_val, X_test = self._format_dataset(norm_dataset[:X_until])
if len(X_test) == 0:
raise errors.LoudMLException("not enough data for prediction")
# display a 2D plot of the digit classes in the latent space
z_mean, _, _ = self._encoder_model.predict([X_test, X_miss_val],
batch_size=g_mc_batch_size)
if x_dim < 0 or y_dim < 0:
mses = []
for (x, y) in itertools.combinations(range(0, latent_dim), 2):
_mean = np.mean(z_mean, axis=0)[[x, y]]
mse = ((z_mean[:, [x, y]] - _mean)**2).mean(axis=0)
mses.append([x, y, mse[0] + mse[1]])
mses = sorted(mses, key=lambda x: x[2])
x_dim = mses[0][0]
y_dim = mses[0][1]
excl = [x for x in range(latent_dim) if x != x_dim and x != y_dim]
plt.figure(figsize=(12, 10))
if latent_dim > 3:
ax = plt.axes(projection='3d')
ax.set_zticks([])
else:
ax = plt.axes()
# Hide grid lines
ax.grid(False)
# Hide axes ticks
ax.set_xticks([])
ax.set_yticks([])
if latent_dim > 3:
zc = np.array(
[[z_mean[i, excl[0]], z_mean[i, excl[1]], z_mean[i, excl[2]]]
for i, _ in enumerate(z_mean)])
# (x-min(x))/(max(x)-min(x)). RGBA values should be within 0-1 range
zc = (zc - np.min(zc, axis=0)) / (np.max(zc, axis=0) -
np.min(zc, axis=0))
if latent_dim > 5:
ax.set_zlabel("z[{}]".format(excl[3]))
ax.scatter(z_mean[:, x_dim],
z_mean[:, y_dim],
z_mean[:, excl[3]],
c=zc)
else:
zc[:, 0] = 0
ax.set_zlabel("z[{}]".format(excl[0]))
ax.scatter(z_mean[:, x_dim],
z_mean[:, y_dim],
z_mean[:, excl[0]],
c=zc)
else:
plt.scatter(z_mean[:, x_dim],
z_mean[:, y_dim],
c=z_mean[:, excl[0]])
plt.colorbar()
plt.xlabel("z[{}]".format(x_dim))
plt.ylabel("z[{}]".format(y_dim))
if output is None:
plt.show()
else:
plt.savefig(output)
from viseron.const import ENV_CUDA_SUPPORTED, ENV_OPENCL_SUPPORTED
from viseron.detector import SCHEMA, DetectorConfig
from viseron.detector.detected_object import DetectedObject
from .defaults import LABEL_PATH, MODEL_CONFIG, MODEL_PATH
SCHEMA = SCHEMA.extend({
Required("model_path", default=MODEL_PATH):
str,
Required("model_config", default=MODEL_CONFIG):
str,
Required("label_path", default=LABEL_PATH):
str,
Optional("suppression", default=0.4):
All(Any(0, 1, All(float, Range(min=0.0, max=1.0))), Coerce(float)),
})
LOGGER = logging.getLogger(__name__)
class ObjectDetection:
"""Performs object detection."""
def __init__(self, config):
self.nms = config.suppression
# Activate OpenCL
if cv2.ocl.haveOpenCL():
cv2.ocl.setUseOpenCL(True)
self.load_labels(config.label_path)
def job(extra_context_variables=[]):
context_variables = CONTEXT_VARIABLES + extra_context_variables
lava_lxc = {
Required("name"): str,
Required("distribution"): str,
Required("release"): str,
Optional("arch"): str,
Optional("mirror"): str,
Optional("persist"): bool,
Optional("security_mirror"): str,
Optional("template"): str,
Optional("timeout"): timeout(),
Optional("verbose"): bool,
}
return All(
{
Required("job_name"):
All(str, Length(min=1, max=200)),
Optional("device_type"):
All(str, Length(min=1, max=200)),
Required("timeouts"): {
Required("job"): timeout(),
Optional("action"): timeout(),
Optional("actions"): {
str: timeout()
},
Optional("connection"): timeout(),
Optional("connections"): {
str: timeout()
},
Optional("queue"): timeout(),
},
Required("visibility"):
Any("public", "personal", {"group": [str]}),
Optional("context"):
Schema({In(context_variables): Any(int, str, [int, str])},
extra=False),
Optional("metadata"): {
str: object
},
Optional("priority"):
Any("high", "medium", "low", Range(min=0, max=100)),
Optional("tags"): [str],
Optional("secrets"):
dict,
Optional("environment"):
dict,
Optional("protocols"): {
Optional("lava-lxc"):
Any(lava_lxc, {str: lava_lxc}),
Optional("lava-multinode"): {
Required("roles"): {
str:
Any(
{
Required("device_type"):
str,
Required("count"):
Range(min=0),
Optional("context"):
Schema(
{
In(context_variables):
Any(int, str, [int, str])
},
extra=False,
),
Optional("tags"): [str],
Optional("environment"):
dict,
Optional("essential"):
bool,
Optional("timeout"):
timeout(),
},
{
Required("connection"):
str,
Required("count"):
Range(min=0),
Required("expect_role"):
str,
Required("host_role"):
str,
Optional("essential"):
bool,
Optional("request"):
str,
Optional("tags"): [str],
Optional("timeout"):
timeout(),
Optional("context"):
Schema(
{
In(context_variables):
Any(int, str, [int, str])
},
extra=False,
),
},
)
},
Optional("timeout"): timeout(),
},
Optional("lava-vland"):
Any(
{str: {
str: {
Required("tags"): [str]
}
}},
{str: {
Required("tags"): [str]
}},
),
Optional("lava-xnbd"): {
Required("port"): Any("auto", int),
Optional("timeout"): timeout(),
},
},
Optional("notify"):
notify(),
Optional("reboot_to_fastboot"):
bool,
Required("actions"): [{
Any("boot", "command", "deploy", "test"):
dict
}],
},
extra_checks,
)
:statuscode 403: User does not have permission to view post
:statuscode 404: Post does not exist
"""
return flask.jsonify(
ForumPost.from_pk(
id,
_404=True,
include_dead=flask.g.user.has_permission(
'forums_posts_modify_advanced'),
))
CREATE_FORUM_POST_SCHEMA = Schema(
{
'contents': All(str, PostLength(max=256000)),
'thread_id': All(int, Range(min=0, max=2147483648)),
},
required=True,
)
@bp.route('/forums/posts', methods=['POST'])
@require_permission('forums_posts_create')
@validate_data(CREATE_FORUM_POST_SCHEMA)
def create_post(contents: str, thread_id: int) -> flask.Response:
"""
This is the endpoint for forum posting. The ``forums_posts_modify``
permission is required to access this endpoint.
.. :quickref: ForumPost; Create a forum post.
return schemas[""](data)
if parsed.scheme not in schemas:
raise Invalid(f"Unsupported URL type {parsed.scheme}://")
return schemas[parsed.scheme](data)
return validate
class RelPath(str):
pass
REMOTE_COMMON = {
"url": str,
"checksum_jobs": All(Coerce(int), Range(1)),
Optional("no_traverse"): Bool, # obsoleted
"verify": Bool,
}
LOCAL_COMMON = {
"type": supported_cache_type,
Optional("protected", default=False): Bool, # obsoleted
"shared": All(Lower, Choices("group")),
Optional("slow_link_warning", default=True): Bool,
}
HTTP_COMMON = {
"auth": All(Lower, Choices("basic", "digest", "custom")),
"custom_auth_header": str,
"user": str,
"password": str,
"ask_password": Bool,
def test_range_inside():
s = Schema(Range(min=0, max=10))
assert_equal(5, s(5))
for tag, settings in ops_settings['column_mappings'].items():
custom_dict[tag] = Required(list, msg='Must be a list')
return custom_dict
geom_schema = {
Optional('geojson'):
Required(object, msg='Must be a geojson object'),
Optional('bbox'):
Required(All(list, Length(min=2, max=2)),
msg='Must be length of {}'.format(2)),
Optional('buffer'):
Required(All(
Coerce(int),
Range(min=0, max=ops_settings['maximum_search_radius_for_points'])),
msg='Must be between 1 and {}'.format(
ops_settings['maximum_search_radius_for_points']))
}
filters_schema = {
Optional('category_group_ids'):
Required(All(categories_tools.category_group_ids,
Length(max=ops_settings['maximum_categories'])),
msg='Must be one of {} and have a maximum amount of {}'.format(
categories_tools.category_group_ids,
ops_settings['maximum_categories'])),
Optional('category_ids'):
Required(All(categories_tools.category_ids,
Length(max=ops_settings['maximum_categories'])),
msg='Must be one of {} and have a maximum amount of {}'.format(
def test_range_no_upper_limit():
s = Schema(Range(min=0))
assert_equal(123, s(123))
long = int
else:
_unicode = unicode
_unicode_or_printable_ascii = Any(unicode, Match(r'^[\x20-\x7E]*$'))
_any_string = Any(_unicode_or_printable_ascii, str)
_any_number = Any(float, int, long, Decimal)
_custom_input_key = All(_any_string, Match(r'^[a-z0-9_]{1,25}$'))
_custom_input_value = Any(
All(_any_string, Match(r'^[^\n]{1,255}\Z')),
All(
_any_number,
Range(min=-(1 << 53),
max=1 << 53,
min_included=False,
max_included=False)), bool)
_md5 = All(_any_string, Match(r'^[0-9A-Fa-f]{32}$'))
_country_code = All(_any_string, Match(r'^[A-Z]{2}$'))
_telephone_country_code = Any(All(_any_string, Match('^[0-9]{1,4}$')),
All(int, Range(min=1, max=9999)))
_subdivision_iso_code = All(_any_string, Match(r'^[0-9A-Z]{1,4}$'))
def _ip_address(s):
# ipaddress accepts numeric IPs, which we don't want.
if isinstance(s, (str, _unicode)) and not re.match(r'^\d+$', s):
def test_range_excludes_none():
s = Schema(Range(min=0, max=10))
assert_raises(MultipleInvalid, s, None)
class DataSource(metaclass=ABCMeta):
"""
Abstract class for Loud ML storage
"""
SCHEMA = Schema(
{
Required('name'):
All(schemas.key, Length(max=256)),
Required('type'):
All(schemas.key, Length(max=256)),
Optional('max_series_per_request', default=2000):
All(
int,
Range(min=1),
),
},
extra=ALLOW_EXTRA)
def __init__(self, cfg):
self._cfg = self.validate(cfg)
self._pending = []
self._last_commit = datetime.datetime.now()
@classmethod
def validate(cls, cfg):
"""Validate configuration against the schema"""
return schemas.validate(cls.SCHEMA, cfg)
@property
def cfg(self):
"""
Return data source configuration
"""
return self._cfg
@property
def name(self):
return self._cfg.get('name')
@property
def max_series_per_request(self):
return self._cfg['max_series_per_request']
def init(self, *args, **kwargs):
pass
def drop(self):
pass
def nb_pending(self):
return len(self._pending)
def clear_pending(self):
del self._pending[:]
def commit(self):
"""
Send data
"""
if self.nb_pending() > 0:
self.send_bulk(self._pending)
self.clear_pending()
self._last_commit = datetime.datetime.now()
def must_commit(self):
"""
Tell if pending data must be sent to the datasource
"""
nb_pending = self.nb_pending()
if nb_pending == 0:
return False
if nb_pending >= 1000:
return True
if (datetime.datetime.now() - self._last_commit).seconds >= 1:
return True
return False
def enqueue(self, req):
"""
Enqueue query to bulk buffer
"""
self._pending.append(req)
if self.must_commit():
self.commit()
@abstractmethod
def get_quadrant_data(
self,
model,
aggregation,
from_date=None,
to_date=None,
key=None,
):
"""Get quadrant aggregation data"""
@abstractmethod
def get_times_data(
self,
model,
from_date=None,
to_date=None,
):
"""Get numeric data"""
@abstractmethod
def insert_data(self, data):
"""
Insert entry into the index
"""
@abstractmethod
def insert_times_data(self, ts, data, tags=None, *args, **kwargs):
"""
Insert time-indexed entry
"""
@abstractmethod
def save_timeseries_prediction(self, prediction):
"""
Save time-series prediction to the datasource
"""
def insert_annotation(
self,
dt,
desc,
_type,
_id,
measurement='annotations',
tags=None,
):
"""
Insert annotation and return data points saved to the TSDB
"""
return None
def update_annotation(
self,
dt,
points,
):
"""
Update annotation in the TSDB
"""
return None
def get_top_abnormal_keys(
self,
model,
from_date,
to_date,
size=10,
):
raise NotImplemented()
def list_anomalies(
self,
from_date,
to_date,
tags=None,
):
return []
def test_range_excludes_unordered_object():
class MyObject(object):
pass
s = Schema(Range(min=0, max=10))
assert_raises(MultipleInvalid, s, MyObject())
Required('allow_latex'):
bool,
Required('leaderboard_show'):
int,
Required('assessments'): [
Schema({
Required('name'):
All(utf8_validator, In(VALID_ASSESSMENT_TYPES)),
Required('start', default=None):
Any(datetime_validator, None),
Required('due', default=None):
Any(datetime_validator, None),
'required':
bool,
'must_grade':
All(int, Range(min=0)),
'must_be_graded_by':
All(int, Range(min=0)),
'examples': [
Schema({
Required('answer'): [utf8_validator],
Required('options_selected'): [
Schema({
Required('criterion'): utf8_validator,
Required('option'): utf8_validator
})
]
})
],
'examples_xml':
utf8_validator,
def count(**kwargs):
# This setting is only used with the count filtertype and is required
return {Required('count'): All(Coerce(int), Range(min=1))}
from voluptuous import Schema, Required, All, Length, Range
schema = Schema({
Required('id'): All(str, Length(min=1)),
Required('login', default=5): All(int, Range(min=1, max=20)),
})
schema({'id': 'real_id', 'login': 10})
def number_of_shards(**kwargs):
return {
Required('number_of_shards'): All(Coerce(int), Range(min=1))
}
return hwaccel_args
if os.getenv(ENV_VAAPI_SUPPORTED) == "true":
return HWACCEL_VAAPI
return hwaccel_args
STREAM_SCEHMA = Schema({
Required("path"):
All(str, Length(min=1)),
Optional("width", default=None):
Any(int, None),
Optional("height", default=None):
Any(int, None),
Optional("fps", default=None):
Any(All(int, Range(min=1)), None),
Optional("input_args", default=CAMERA_INPUT_ARGS):
list,
Optional("hwaccel_args", default=CAMERA_HWACCEL_ARGS):
check_for_hwaccels,
Optional("codec", default=""):
str,
Optional("rtsp_transport", default="tcp"):
Any("tcp", "udp", "udp_multicast", "http"),
Optional("filter_args", default=[]):
list,
})
CAMERA_SCHEMA = STREAM_SCEHMA.extend(
{
Required("name"):