class Validator:
from voluptuous import Schema, Required, Optional, Range, All, Or, And
SENSOR_SCHEMA = Schema({
Required("entity"): str,
Required("op"): And(str, Op.from_str),
Required("value"): Or(float, int)
})
SCHEMA = Schema(
{
Optional("for", default="5m"):
utils.parse_duration_literal, # Lights on for x seconds
Required("lights"):
Or([str], lambda v: [v] if isinstance(v, str) else []),
Required("motion"):
Or([str], lambda v: [v] if isinstance(v, str) else []),
Optional("sensor", default=[]):
Or([SENSOR_SCHEMA], lambda v: [Validator.SENSOR_SCHEMA(v)])
},
extra=True)
@classmethod
def validate(cls, dct):
return cls.SCHEMA(dct)
def eval_meta(meta):
with_meta = Namespace(
with_byoc,
'from byoc.meta import *',
MetaWrapper=MetaWrapper,
)
dict_schema = {
'type': with_meta.eval,
Optional('location', default=None): Or(None, str),
str: with_py.eval,
}
schema = Schema(Or(dict_schema, with_meta.eval))
meta = schema(meta)
if isinstance(meta, MetaWrapper):
return meta
elif isinstance(meta, dict):
return MetaWrapper(**meta)
else:
return MetaWrapper(meta)
class RequestProcessor(object):
HANDLER_CLASSES = {
'new_chart': NewChartRequestHandler,
'new_live_data': NewLiveDataRequestHandler,
}
REQUEST_SCHEMA = Schema({
Required('request_id'): Or(int, None),
Required('type'): str,
Required('params'): dict,
})
def __init__(self):
self._requests = dict()
self._lock = threading.Lock()
def _cleanup(self):
self._lock.acquire()
try:
requests_to_be_removed = [key for key in self._requests if self._requests[key].to_be_removed]
for r in requests_to_be_removed:
del self._requests[r]
finally:
self._lock.release()
def run_requests(self) -> Iterable[Dict[str, Any]]:
self._cleanup()
for key in self._requests:
response = self._requests[key]()
if response:
yield response
def process_new_request(self, payload: dict):
self._lock.acquire()
self.REQUEST_SCHEMA(payload)
request_id = payload['request_id']
request_type = payload['type']
params = payload['params']
try:
if request_type == 'cancel_request':
request_id = params.get('request_id')
if request_id in self._requests:
self._requests[request_id].remove()
else:
raise ValueError('request_id {} not found'.format(request_id))
elif request_type in self.HANDLER_CLASSES:
handler = self.HANDLER_CLASSES[request_type](request_id, params)
self._requests[request_id] = handler
else:
raise ValueError('Request type {} not found'.format(request_type))
finally:
self._lock.release()
def eval_layer(layer):
schema = Schema(
Or(
str, {
'values': with_py.eval,
'location': str,
Optional('root_key'): with_py.eval,
Optional('schema'): with_py.eval,
Optional('linenos'): with_py.eval,
}))
layer = schema(layer)
layer = with_byoc.eval(layer) if isinstance(layer, str) else \
byoc.DictLayer(**layer)
return LayerWrapper(layer)
class ArgumentSchemas:
MASS_UNITS = frozenset({'g', 'kg', 'lb', 'oz',})
VOLUME_UNITS = frozenset({'L', 'mL', 'cup', 'floz', 'Tbsp', 'tsp', 'qt', 'gal'})
TIME_UNITS = frozenset({'sec', 'min', 'hr', 'day', 'week'})
LENGTH_UNITS = frozenset({'cm', 'm', 'mm', 'in', 'ft',})
STRING = Schema(All(str, Length(min=1)))
POS_INT = Schema(All(int, Range(min=1)))
NON_NEG_INT = Schema(All(int, Range(min=0)))
VARIANT_TAG_SET = Schema(All(Unique(), Length(min=1), [NON_NEG_INT]))
FRACTION = Schema({
Required('keep'): POS_INT,
Required('pass'): POS_INT,
})
QUANTITY_BASE = Schema({
Required('base'): POS_INT,
Required('range', default=0): NON_NEG_INT,
Required('base_den', default=1): POS_INT,
Required('range_den', default=1): POS_INT,
Required('less_ok', default=True): bool,
Required('more_ok', default=True): bool,
})
QUANTITY_MASS = QUANTITY_BASE.extend({
Required('units'): All(str, In(MASS_UNITS)),
})
QUANTITY_VOLUME = QUANTITY_BASE.extend({
Required('units'): All(str, In(VOLUME_UNITS)),
})
QUANTITY_COUNT = QUANTITY_BASE.extend({
Required('units'): STRING,
})
QUANTITY_TIME = QUANTITY_BASE.extend({
Required('units'): All(str, In(TIME_UNITS)),
})
NONE = Schema(None)
STRING_OR_NONE = Schema(Or(STRING, NONE))
class SeriesAttributeHandler(Handler):
def __init__(self):
self.session = Session()
def get(self, entity_type_id, ident=None):
entity_type = get_one(self.session, EntityType, id=entity_type_id)
if ident is None:
return [
series.to_dict() for series in get_all(
self.session, SeriesAttribute, entity_type=entity_type)
]
else:
return get_one(self.session,
SeriesAttribute,
entity_type=entity_type,
id=ident).to_dict()
@requires_validation(assert_attribute_does_not_exist(SeriesAttribute),
with_route_params=True)
@requires_validation(
Schema({
Required('name'): non_empty_string,
'type': Or('real', 'enum'),
'refresh_time': Or(int, None),
'is_favourite': bool,
}))
def post(self, entity_type_id):
data = self.request.data
entity_type = get_one(self.session, EntityType, id=entity_type_id)
series = SeriesAttribute(entity_type=entity_type,
name=data['name'],
type=data.get('type', 'real'),
refresh_time=data.get('refresh_time'),
is_favourite=data.get('is_favourite', False))
self.session.add(series)
self.session.commit()
update_last_data_modification_ts(self.session)
return {'success': True, 'ID': series.id}
@requires_validation(
Schema({
'refresh_time': Or(int, None),
'is_favourite': bool,
}))
def put(self, entity_type_id, ident):
data = self.request.data
entity_type = get_one(self.session, EntityType, id=entity_type_id)
series = get_one(self.session,
SeriesAttribute,
entity_type=entity_type,
id=ident)
if 'refresh_time' in data:
series.refresh_time = data['refresh_time']
if 'is_favourite' in data:
series.is_favourite = data['is_favourite']
self.session.commit()
update_last_data_modification_ts(self.session)
return {'success': True, 'ID': series.id}
def delete(self, entity_type_id, ident):
now = time.time()
entity_type = get_one(self.session, EntityType,
id=entity_type_id) # check if route is correct
series = get_one(self.session,
SeriesAttribute,
entity_type=entity_type,
id=ident)
series.delete_ts = now
for alert in series.alerts:
alert.delete_ts = now
self.session.commit()
update_last_data_modification_ts(self.session)
return {'success': True}
class AlertHandler(Handler):
def __init__(self):
self.session = Session()
def get(self, ident=None):
if ident is None:
return [alert.to_dict() for alert in get_all(self.session, Alert)]
else:
return get_one(self.session, Alert, id=ident).to_dict()
@requires_validation(
Schema({
Required('entity_id'):
int,
Required('series_id'):
int,
Required('alert_predicate_type'):
Or('data_delay', 'value_too_low', 'value_too_high'),
Required('value'):
float,
Required('is_enabled'):
bool,
Required('alert_recipient_email'):
Email,
}))
def post(self):
data = self.request.data
entity = get_one(self.session, Entity, id=data['entity_id'])
series = get_one(self.session, SeriesAttribute, id=data['series_id'])
alert = Alert(entity=entity,
series=series,
alert_predicate_type=data['alert_predicate_type'],
value=data['value'],
is_enabled=data['is_enabled'],
alert_recipient_email=data['alert_recipient_email'])
self.session.add(alert)
self.session.commit()
return {'success': True, 'ID': alert.id}
@requires_validation(
Schema({
'entity_id':
int,
'series_id':
int,
'alert_predicate_type':
Or('data_delay', 'value_too_low', 'value_too_high'),
'value':
float,
'is_enabled':
bool,
'alert_recipient_email':
Email,
}))
def put(self, ident):
data = self.request.data
alert = get_one(self.session, Alert, id=ident)
if 'entity_id' in data:
entity = get_one(self.session, Entity, id=data['entity_id'])
alert.entity_id_fk = entity.id
if 'series_id' in data:
series = get_one(self.session,
SeriesAttribute,
id=data['series_id'])
alert.series_id_fk = series.id
if 'alert_predicate_type' in data:
alert.alert_predicate_type = data['alert_predicate_type']
if 'value' in data:
alert.value = data['value']
if 'is_enabled' in data:
alert.is_enabled = data['is_enabled']
if 'alert_recipient_email' in data:
alert.alert_recipient_email = data['alert_recipient_email']
# reset last check status
alert.last_check_status = None
self.session.commit()
return {
'success': True,
'ID': alert.id,
}
def delete(self, ident):
alert = get_one(self.session, Alert, id=ident)
now = time.time()
alert.delete_ts = now
self.session.commit()
update_last_data_modification_ts(self.session)
return {'success': True}
from voluptuous import Or, Schema, Email, Optional, ALLOW_EXTRA, Length, And
job_schema = Schema(
{
"id": Or(str, int),
"email": Email(),
Optional("name"): str
},
required=True,
extra=ALLOW_EXTRA,
)
job_request_schema = Schema(
{
"jobs": And([job_schema], Length(min=1)),
"template": str,
"subject": str
},
required=True,
)
settings_schema = Schema({
"auth": {
"user": str,
"password": str
},
"email_from": {
"name": str,
"email": Email()
},
"headers": dict,
def empty_ok(x):
return Or(x, And('', lambda y: type(x)()))
class ServiceConfiguration(AbstractDict):
"""
Service configuration object
"""
VALIDATION_SCHEMA = Schema(
{
# Application
Optional("path"):
str,
Optional("startup_dir"):
str,
Optional("arguments"):
Or(str, list),
# Details
Optional("display_name"):
str,
Optional("description"):
str,
Optional("startup"):
Coerce(StartupType.coerce),
# Log on
Optional("user_account"):
Or(
str,
Schema({
Required("username"): str,
Required("password"): str
},
extra=REMOVE_EXTRA)),
Optional("type"):
Coerce(ServiceType.coerce),
# Dependencies
Optional("dependencies"):
Or(str, list),
# Process
Optional("process_priority"):
Coerce(PriorityLevel.coerce),
Optional("console_window"):
bool,
Optional("cpu_affinity"):
Or("All", int),
# Shutdown
Optional("terminate_process"):
bool,
Optional("stop_console"):
int,
Optional("stop_window"):
int,
Optional("stop_threads"):
int,
# Exit action
Optional("restart_throttling"):
int,
Optional("restart_delay"):
int,
Optional("action_on_exit"):
Schema({Required("Default"): Coerce(ExitAction.coerce)},
extra=ALLOW_EXTRA),
# I/O
Optional("stdout"):
str,
Optional("stderr"):
str,
# File rotation
Optional("rotate_files"):
bool,
Optional("rotate_online"):
bool,
Optional("stdout_creation_disposition"):
int,
Optional("stderr_creation_disposition"):
int,
Optional("rotation_time"):
int,
Optional("rotation_size"):
int,
# Environment
Optional("env"):
Or(str, dict)
},
extra=REMOVE_EXTRA)
def __init__(self, *args, **kwargs):
"""
Service configuration class constructor
"""
super(ServiceConfiguration, self).__init__()
self.dictionary = validate_with_humanized_errors(
kwargs, self.VALIDATION_SCHEMA)
class EntityHandler(Handler):
def __init__(self):
self.session = Session()
def get(self, ident=None):
if ident is None:
return [
entity.to_dict(deep=False)
for entity in get_all(self.session, Entity)
]
else:
return get_one(self.session, Entity, id=ident).to_dict(deep=False)
@staticmethod
def _assert_got_all_needed_tag_and_meta_ids(entity_type, tag_ids,
meta_ids):
expected_tag_ids = sorted(tag.id for tag in entity_type.tags
if tag.delete_ts is None)
expected_meta_ids = sorted(meta.id for meta in entity_type.meta
if meta.delete_ts is None)
if tag_ids != expected_tag_ids:
raise HTTP_400('Expected tag IDs {}, got {}'.format(
expected_tag_ids, tag_ids))
if meta_ids != expected_meta_ids:
raise HTTP_400('Expected meta IDs {}, got {}'.format(
expected_meta_ids, meta_ids))
@requires_validation(
Schema(
{
Required('parent_id'): Or(int, None),
Required('entity_type_id'): int,
},
extra=ALLOW_EXTRA))
def post(self):
data = self.request.data
entity_type = get_one(self.session,
EntityType,
id=data['entity_type_id'])
# check if we got all tags and meta
tag_ids = sorted(
int(key.split('_')[1]) for key in data if 'tag_' in key)
meta_ids = sorted(
int(key.split('_')[1]) for key in data if 'meta_' in key)
self._assert_got_all_needed_tag_and_meta_ids(entity_type, tag_ids,
meta_ids)
entity = Entity(
entity_type=entity_type,
parent=None if data['parent_id'] is None else get_one(
self.session, Entity, id=data['parent_id']),
)
self.session.add(entity)
# add tags and meta
for key in data:
if 'tag_' in key:
self.session.add(
EntityTag(
entity=entity,
attribute=get_one(self.session,
TagAttribute,
id=int(key.split('_')[1])),
value=data[key],
))
elif 'meta_' in key:
self.session.add(
EntityMeta(
entity=entity,
attribute=get_one(self.session,
MetaAttribute,
id=int(key.split('_')[1])),
value=data[key],
))
self.session.commit()
update_last_data_modification_ts(self.session)
return {
'success': True,
'ID': entity.id,
}
def put(self, ident):
data = self.request.data
entity = get_one(self.session, Entity,
id=ident) # to ensure that the entity exists
if 'parent_id' in data:
get_one(self.session, Entity, id=data['parent_id'])
entity.parent_id_fk = data['parent_id']
# add tags and meta
for key in data:
if 'tag_' in key:
tag = get_one(self.session,
EntityTag,
entity=entity,
tag_id_fk=key.split('_')[1])
tag.value = data[key]
elif 'meta_' in key:
meta = get_one(self.session,
EntityMeta,
entity=entity,
meta_id_fk=key.split('_')[1])
meta.value = data[key]
self.session.commit()
update_last_data_modification_ts(self.session)
return {
'success': True,
'ID': entity.id,
}
def delete(self, ident):
entity = get_one(self.session, Entity, id=ident)
now = time.time()
entity.delete_ts = now
for tag in entity.tags:
tag.delete_ts = now
for meta in entity.meta:
meta.delete_ts = now
for alert in entity.alerts:
alert.delete_ts = now
for child in entity.children:
child.parent = entity.parent
self.session.commit()
update_last_data_modification_ts(self.session)
return {'success': True}
class NewChartRequestHandler(AbstractRequestHandler):
SCHEMA = Schema({
Required('node_id'): int,
Required('begin_ts'): int,
Required('requested_data'): [dict],
Required('end_ts'): int,
Required('update_data'): bool,
Required('aggregation_length'): int,
Required('aggregation_type'): Or('mean', 'max', 'min'),
})
def __init__(self, request_id: int, payload: dict):
super().__init__(request_id, payload)
session = Session()
self._entity = get_one(session,
Entity,
id=self._raw_payload.node_id,
exception_cls=ValueError)
self._requested_data = [
create_measurement_handler(self._raw_payload.node_id,
self._raw_payload.aggregation_length,
self._raw_payload.aggregation_type,
data)
for data in self._raw_payload.requested_data
]
self._run_assertions()
@property
def request_type(self) -> str:
return 'new_chart'
def _validate_payload(self, payload: dict):
self.SCHEMA(payload)
def _run_assertions(self):
measurements_for_entity = [
s.id for s in self._entity.entity_type.series
]
measurements_in_data = set()
for handler in self._requested_data:
measurements_in_data |= handler.measurements()
for measurement in measurements_in_data:
if measurement.id not in measurements_for_entity:
raise ValueError(
'Requested entity does not support requested measurement')
def _handle_request(self) -> Optional[Dict[str, Any]]:
begin_ts = self._raw_payload.begin_ts
end_ts = self._raw_payload.end_ts
# update begin_ts and end_ts to new values
now = int(time.time())
self._raw_payload.begin_ts = begin_ts + (now - end_ts)
self._raw_payload.end_ts = now
results = dict()
for i in range(len(self._requested_data)):
for ts, val in self._requested_data[i].evaluate(begin_ts, end_ts):
if ts in results:
results[ts][i] = val
else:
results[ts] = dict({i: val})
if not self._raw_payload.update_data:
self.remove()
return {
'plot_data':
sorted([
[ts] +
[results[ts].get(i) for i in range(len(self._requested_data))]
for ts in results
])
}
# main API to store things to db
import pickle
from collections import namedtuple
from utils.logdb import LogDB
import zlib
from voluptuous import Schema
from voluptuous import Or
import numpy as np
# name of the event, data should be python seriazible dict
num = Or(float, int)
evttypes = {
'LoggerInfo':
Schema({
'evtname': 'LoggerInfo',
'version': int,
}, required=True),
'SimpleTest':
Schema(
{
'evtname': 'SimpleTest',
'glsteps': int,
'avgscore': float,
'avglength': float,
'avgentropy': float,
'stdscore': float,
'tpassed': float
},
def __init__(self):
self.global_config_schema = Schema({
Optional("sumo_http_url"):
Url(),
Required("run_interval_seconds", default=60):
All(int, Range(min=1)),
Required("target_threads", default=10):
All(int, Range(min=1, max=50)),
Required("batch_size", default=1000):
All(int, Range(min=1)),
Required("retries", default=5):
All(int, Range(min=1, max=20)),
Required("backoff_factor", default=0.2):
All(float, Range(min=0)),
"source_category":
str,
"source_host":
str,
"source_name":
str,
"dimensions":
str,
"metadata":
str,
})
self.target_source_config = Schema(
Or(
{Required("url"): Url()},
{
Required("service"): str,
Required("namespace"): str
},
))
url_schema = Schema(
Or(
Required("url"),
Url(),
{
Required("service"): str,
Required("namespace"): str,
Required("path", default="/metrics"): str,
Required("protocol", default="http"): str,
},
))
self.target_config_schema = self.global_config_schema.extend({
Required("url", default={}):
url_schema,
Required("name"):
str,
Required("exclude_metrics", default=[]):
list([str]),
Required("include_metrics", default=[]):
list([str]),
Required("exclude_labels", default={}):
Schema({}, extra=ALLOW_EXTRA),
Required("include_labels", default={}):
Schema({}, extra=ALLOW_EXTRA),
Required("strip_labels", default=[]):
list([str]),
Required("should_callback", default=True):
bool,
"token_file_path":
IsFile(),
"verify":
Any(Boolean(), str),
# repeat keys from global to remove default values
"sumo_http_url":
Url(),
"run_interval_seconds":
All(int, Range(min=1)),
"target_threads":
All(int, Range(min=1, max=50)),
"batch_size":
All(int, Range(min=1)),
"retries":
All(int, Range(min=1, max=20)),
"backoff_factor":
All(float, Range(min=0)),
})
self.config_schema = Schema(
All(
{
Required("global", default={}):
self.global_config_schema,
Required("targets"):
All(Length(min=1, max=256), [self.target_config_schema]),
},
self.check_url,
))
def argsSchema(args):
"""Convert the argument values from text to the correct type"""
schema = Schema({
'<charslist>': str, # FIXME: Use IsFile to check if file exists
'<dictionary>': Or(None, str),
'--lesson-number': Or(None, Coerce(int)),
'--output': Or(None, str),
'--word-wrap': Or(None, Coerce(int)),
'--characters-per-lesson': Or(None, Coerce(int)),
'--min-word-length': Or(None, Coerce(int)),
'--max-word-length': Or(None, Coerce(int)),
'--symbols-density': Or(None, Coerce(float)),
'--previous-symbols-fraction': Or(None, Coerce(float)),
'--numbers-density': Or(None, Coerce(float)),
'--max-number-length': Or(None, Coerce(int)),
'--max-letters-combination-length': Or(None, Coerce(int)),
'--lesson-title-prefix': Or(None, str),
'--crop-dict': Or(None, Coerce(int)),
str: Boolean() # Treat all other arguments as bool
})
try:
args = schema(args)
except error.MultipleInvalid as ex:
print("\n".join([e.msg for e in ex.errors]))
return args
class Validator:
from voluptuous import Schema, Required, Optional, Range, All, Or
def str2time(candidate):
def _eval(c):
if not c:
return 0
return int(c)
hour, _, minute = candidate.partition(":")
if not hour and not minute:
raise ValueError(
"Could not extract whether hour nor minute from schedule")
return datetime.time(_eval(hour), _eval(minute), 0)
SETPOINT_SCHEMA = Schema(
All(Or(float, int), Range(min=MIN_TEMP, max=MAX_TEMP)))
HEATING_SCHEMA = Schema({
Required("setpoint"):
SETPOINT_SCHEMA,
Optional("schedule", default=[]): [{
Required("start"):
str2time,
Required("end"):
str2time,
Required("setpoint"):
SETPOINT_SCHEMA,
Optional("weekdays"):
str,
Optional("constraints"):
Or([str], lambda v: [v] if isinstance(v, str) else [])
}]
})
THERMOSTAT_SCHEMA = Schema(
Or(
str, {
Required("entity"): str,
Optional("offset", default=0): int,
Optional("force", default=None): Or(None, bool)
}))
ROOM_SETPOINT_SENSOR_SCHEMA = Schema({
Optional("name", default=None):
Or(None, str),
Optional("attributes", default={}): {
str: object
}
})
SCHEMA = Schema(
{
Optional("check_interval", default=0):
utils.parse_duration_literal,
Required("mode"): {
Required("entity"): str,
Optional("map", default={}): {
str: str
},
Optional("init_options", default=False): bool
},
Required("rooms"): {
str: {
Required("thermostats"): [THERMOSTAT_SCHEMA],
Optional("setpoint_sensor"):
Or(ROOM_SETPOINT_SENSOR_SCHEMA, None),
Required("comfort"):
HEATING_SCHEMA,
Required("energy"):
HEATING_SCHEMA,
Required("frost"):
HEATING_SCHEMA
}
}
},
extra=True)
@classmethod
def validate_config(cls, config):
return cls.SCHEMA(config)