def run(self, job_id, nice, func_name, config, *args, **kwargs):
"""
Run requested task and return the result
"""
self._msg_queue.put({
'type': 'job_state',
'job_id': job_id,
'state': 'running',
})
logging.info("job[%s] starting, nice=%d", job_id, nice)
self.job_id = job_id
self.config = config
self.storage = FileStorage(config.storage['path'])
curnice = os.nice(0)
os.nice(int(nice) - curnice)
try:
res = getattr(self, func_name)(*args, **kwargs)
except errors.LoudMLException as exn:
raise exn
except Exception as exn:
logging.exception(exn)
raise exn
finally:
self.job_id = None
self.config = None
self.storage = None
return res
def __init__(self, config_path, msg_queue):
self.config = loudml.config.load_config(config_path)
self.storage = FileStorage(self.config.storage['path'])
self._msg_queue = msg_queue
self.job_id = None
signal.signal(signal.SIGINT, signal.SIG_IGN)
class Worker:
"""
Loud ML worker
"""
def __init__(self, config_path, msg_queue):
self.config = loudml.config.load_config(config_path)
self.storage = FileStorage(self.config.storage['path'])
self._msg_queue = msg_queue
self.job_id = None
signal.signal(signal.SIGINT, signal.SIG_IGN)
def run(self, job_id, nice, func_name, *args, **kwargs):
"""
Run requested task and return the result
"""
self._msg_queue.put({
'type': 'job_state',
'job_id': job_id,
'state': 'running',
})
logging.info("job[%s] starting, nice=%d", job_id, nice)
self.job_id = job_id
curnice = os.nice(0)
os.nice(int(nice) - curnice)
try:
res = getattr(self, func_name)(*args, **kwargs)
except errors.LoudMLException as exn:
raise exn
except Exception as exn:
logging.exception(exn)
raise exn
finally:
self.job_id = None
return res
def train(self, model_name, datasource=None, **kwargs):
"""
Train model
"""
model = self.storage.load_model(model_name)
src_name = datasource or model.default_datasource
src_settings = self.config.get_datasource(src_name)
source = loudml.datasource.load_datasource(src_settings)
def progress_cb(current_eval, max_evals):
self._msg_queue.put({
'type': 'job_state',
'job_id': self.job_id,
'state': 'running',
'progress': {
'eval': current_eval,
'max_evals': max_evals,
},
})
windows = source.list_anomalies(
kwargs['from_date'],
kwargs['to_date'],
tags={'model': model_name},
)
model.train(source,
batch_size=self.config.training['batch_size'],
num_epochs=self.config.training['epochs'],
num_cpus=self.config.training['num_cpus'],
num_gpus=self.config.training['num_gpus'],
progress_cb=progress_cb,
windows=windows,
**kwargs)
self.storage.save_model(model)
def _save_timeseries_prediction(
self,
model,
prediction,
source,
datasink=None,
):
if datasink is None:
datasink = model.default_datasink
if datasink is None or datasink == source.name:
sink = source
else:
try:
sink_settings = self.config.get_datasource(datasink)
sink = loudml.datasource.load_datasource(sink_settings)
except errors.LoudMLException as exn:
logging.error("cannot load data sink: %s", str(exn))
return
sink.save_timeseries_prediction(prediction, model)
def predict(self,
model_name,
save_run_state=True,
save_prediction=False,
detect_anomalies=False,
datasink=None,
**kwargs):
"""
Ask model for a prediction
"""
model = self.storage.load_model(model_name)
src_settings = self.config.get_datasource(model.default_datasource)
source = loudml.datasource.load_datasource(src_settings)
if model.type in ['timeseries', 'donut']:
mse_rtol = self.config.server['mse_rtol']
_state = model.get_run_state()
if detect_anomalies:
prediction = model.predict2(
source,
mse_rtol=mse_rtol,
_state=_state,
num_cpus=self.config.inference['num_cpus'],
num_gpus=self.config.inference['num_gpus'],
**kwargs)
else:
prediction = model.predict(
source,
num_cpus=self.config.inference['num_cpus'],
num_gpus=self.config.inference['num_gpus'],
**kwargs)
logging.info("job[%s] predicted values for %d time buckets",
self.job_id, len(prediction.timestamps))
if detect_anomalies:
hooks = self.storage.load_model_hooks(
model.settings,
source,
)
model.detect_anomalies(prediction, hooks)
if save_run_state:
model.set_run_state(_state)
self.storage.save_state(model)
if save_prediction:
self._save_timeseries_prediction(
model,
prediction,
source,
datasink,
)
fmt = kwargs.get('format', 'series')
if fmt == 'buckets':
return prediction.format_buckets()
elif fmt == 'series':
return prediction.format_series()
else:
raise errors.Invalid('unknown requested format')
else:
logging.info("job[%s] prediction done", self.job_id)
def forecast(self,
model_name,
save_prediction=False,
datasink=None,
**kwargs):
"""
Ask model for a forecast
"""
model = self.storage.load_model(model_name)
src_settings = self.config.get_datasource(model.default_datasource)
source = loudml.datasource.load_datasource(src_settings)
constraint = kwargs.pop('constraint', None)
forecast = model.forecast(source,
num_cpus=self.config.inference['num_cpus'],
num_gpus=self.config.inference['num_gpus'],
**kwargs)
if model.type in ['timeseries', 'donut']:
logging.info("job[%s] forecasted values for %d time buckets",
self.job_id, len(forecast.timestamps))
if constraint:
model.test_constraint(
forecast,
constraint['feature'],
constraint['type'],
constraint['threshold'],
)
if save_prediction:
self._save_timeseries_prediction(
model,
forecast,
source,
datasink,
)
return forecast.format_series()
else:
logging.info("job[%s] forecast done", self.job_id)
def load(self, from_date=None, datasource=None, **kwargs):
"""
Load public data set
"""
src_settings = self.config.get_datasource(datasource)
source = loudml.datasource.load_datasource(src_settings)
load_nab(source, from_date)
logging.info("data loaded")
"""
def test_create_and_list(self):
with tempfile.TemporaryDirectory() as tmp:
storage = FileStorage(tmp)
# Create
model = DonutModel(
dict(
name='test-1',
offset=30,
span=300,
bucket_interval=3,
interval=60,
features=FEATURES,
max_threshold=70,
min_threshold=60,
))
self.assertEqual(model.type, 'donut')
storage.create_model(model)
self.assertTrue(storage.model_exists(model.name))
# Create
model = DonutModel(
dict(
name='test-2',
offset=56,
span=200,
bucket_interval=20,
interval=120,
features=FEATURES,
max_threshold=70,
min_threshold=60,
))
storage.create_model(model)
# List
self.assertEqual(storage.list_models(), ["test-1", "test-2"])
# Delete
storage.delete_model("test-1")
self.assertFalse(storage.model_exists("test-1"))
self.assertEqual(storage.list_models(), ["test-2"])
with self.assertRaises(errors.ModelNotFound):
storage.get_model_data("test-1")
# Rebuild
model = storage.load_model("test-2")
self.assertEqual(model.type, 'donut')
self.assertEqual(model.name, 'test-2')
self.assertEqual(model.offset, 56)
class Worker:
"""
Loud ML worker
"""
def __init__(self, msg_queue):
self.storage = None
self._msg_queue = msg_queue
self.job_id = None
signal.signal(signal.SIGINT, signal.SIG_IGN)
def run(self, job_id, nice, func_name, config, *args, **kwargs):
"""
Run requested task and return the result
"""
self._msg_queue.put({
'type': 'job_state',
'job_id': job_id,
'state': 'running',
})
logging.info("job[%s] starting, nice=%d", job_id, nice)
self.job_id = job_id
self.config = config
self.storage = FileStorage(config.storage['path'])
curnice = os.nice(0)
os.nice(int(nice) - curnice)
try:
res = getattr(self, func_name)(*args, **kwargs)
except errors.LoudMLException as exn:
raise exn
except Exception as exn:
logging.exception(exn)
raise exn
finally:
self.job_id = None
self.config = None
self.storage = None
return res
def train(self, model_name, bucket=None, **kwargs):
"""
Train model
"""
model = self.storage.load_model(model_name)
bucket_name = bucket or model.default_bucket
bucket_settings = self.config.get_bucket(bucket_name)
bucket = loudml.bucket.load_bucket(bucket_settings)
def progress_cb(current_eval, max_evals):
self._msg_queue.put({
'type': 'job_state',
'job_id': self.job_id,
'state': 'running',
'progress': {
'eval': current_eval,
'max_evals': max_evals,
},
})
windows = bucket.list_anomalies(
kwargs['from_date'],
kwargs['to_date'],
tags={'model': model_name},
)
num_epochs = kwargs.pop('num_epochs', self.config.training['epochs'])
model.train(
bucket,
batch_size=self.config.training['batch_size'],
num_epochs=num_epochs,
num_cpus=self.config.training['num_cpus'],
num_gpus=self.config.training['num_gpus'],
progress_cb=progress_cb,
windows=windows,
**kwargs
)
self.storage.save_model(model)
def _save_timeseries_prediction(
self,
model,
prediction,
input_bucket,
output_bucket=None,
):
if output_bucket is None:
output_bucket = model.default_bucket
if output_bucket is None or output_bucket == input_bucket.name:
bucket = input_bucket
else:
try:
bucket_settings = self.config.get_bucket(
output_bucket
)
bucket = loudml.bucket.load_bucket(bucket_settings)
except errors.LoudMLException as exn:
logging.error("cannot load bucket: %s", str(exn))
return
bucket.init(data_schema=prediction.get_schema())
bucket.save_timeseries_prediction(prediction, tags=model.get_tags())
def read_from_bucket(
self,
bucket_name,
from_date,
to_date,
bucket_interval,
features,
):
"""
Run query in the bucket TSDB and return data
"""
bucket_settings = self.config.get_bucket(bucket_name)
bucket = loudml.bucket.load_bucket(bucket_settings)
data = bucket.get_times_data(
bucket_interval=bucket_interval,
features=features,
from_date=from_date,
to_date=to_date,
)
timestamps = []
obs = {
feature.name: []
for feature in features
}
for (_, values, timeval) in data:
timestamps.append(make_ts(timeval))
for (feature, val) in zip(features, values):
obs[feature.name].append(float(val))
return {
'timestamps': timestamps,
'observed': obs,
}
def write_to_bucket(
self,
bucket_name,
points,
**kwargs
):
"""
Writes data points to the bucket TSDB
"""
bucket_settings = self.config.get_bucket(bucket_name)
bucket = loudml.bucket.load_bucket(bucket_settings)
fields = [
list(point.keys())
for point in points
]
flat_fields = [
field for sub_fields in fields for field in sub_fields
]
fields = set(flat_fields) - set(['timestamp', 'tags'])
tags = [
list(point['tags'].keys())
for point in points
if 'tags' in point
]
flat_tags = [
tag for sub_tags in tags for tag in sub_tags
]
tags = set(flat_tags)
data_schema = {}
data_schema.update({
tag: {"type": "keyword"}
for tag in tags
})
data_schema.update({
field: {"type": "float"}
for field in fields
})
bucket.init(data_schema=data_schema)
for point in points:
ts = make_ts(point.pop('timestamp'))
tags = point.pop('tags', None)
bucket.insert_times_data(
ts=ts,
data=point,
tags=tags,
**kwargs
)
bucket.commit()
def predict(
self,
model_name,
save_run_state=True,
save_prediction=False,
detect_anomalies=False,
output_bucket=None,
**kwargs
):
"""
Ask model for a prediction
"""
model = self.storage.load_model(model_name)
bucket_settings = self.config.get_bucket(model.default_bucket)
bucket = loudml.bucket.load_bucket(bucket_settings)
if model.type in ['timeseries', 'donut']:
_state = model.get_run_state()
if detect_anomalies:
prediction = model.predict2(
bucket,
_state=_state,
num_cpus=self.config.inference['num_cpus'],
num_gpus=self.config.inference['num_gpus'],
**kwargs
)
else:
prediction = model.predict(
bucket,
num_cpus=self.config.inference['num_cpus'],
num_gpus=self.config.inference['num_gpus'],
**kwargs
)
logging.info("job[%s] predicted values for %d time buckets",
self.job_id, len(prediction.timestamps))
if detect_anomalies:
hooks = self.storage.load_model_hooks(
model.settings,
bucket,
)
model.detect_anomalies(prediction, hooks)
if save_run_state:
model.set_run_state(_state)
self.storage.save_state(model)
if save_prediction:
self._save_timeseries_prediction(
model,
prediction,
bucket,
output_bucket,
)
fmt = kwargs.get('format', 'series')
if fmt == 'buckets':
return prediction.format_buckets()
elif fmt == 'series':
return prediction.format_series()
else:
raise errors.Invalid('unknown requested format')
else:
logging.info("job[%s] prediction done", self.job_id)
def forecast(
self,
model_name,
save_prediction=False,
output_bucket=None,
**kwargs
):
"""
Ask model for a forecast
"""
model = self.storage.load_model(model_name)
bucket_settings = self.config.get_bucket(model.default_bucket)
bucket = loudml.bucket.load_bucket(bucket_settings)
constraint = kwargs.pop('constraint', None)
forecast = model.forecast(
bucket,
num_cpus=self.config.inference['num_cpus'],
num_gpus=self.config.inference['num_gpus'],
**kwargs
)
if model.type in ['timeseries', 'donut']:
logging.info("job[%s] forecasted values for %d time buckets",
self.job_id, len(forecast.timestamps))
if constraint:
model.test_constraint(
forecast,
constraint['feature'],
constraint['type'],
constraint['threshold'],
)
if save_prediction:
self._save_timeseries_prediction(
model,
forecast,
bucket,
output_bucket,
)
return forecast.format_series()
else:
logging.info("job[%s] forecast done", self.job_id)
"""