def __init__(self, use_saved_model, checkpoint_path, likelihood_path):
self.use_saved_model = use_saved_model
if use_saved_model:
self.model = ModelFactory.loadFromCheckpoint(checkpoint_path)
self.model.enableInference({'predictedField': 'cpu'})
self.model.enableInference({'predictedField': 'memory'})
with open(likelihood_path, "rb") as f:
self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood(
).readFromFile(f)
else:
self.model = ModelFactory.create(model_params.MODEL_PARAMS)
self.model.enableInference({'predictedField': 'cpu'})
self.model.enableInference({'predictedField': 'memory'})
self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood()
def testCheckpoint(self):
tmpDir = tempfile.mkdtemp()
model = ModelFactory.create(MODEL_PARAMS)
model.enableInference({'predictedField': 'consumption'})
headers = ['timestamp', 'consumption']
# Now do a bunch of small load/train/save batches
for _ in range(20):
for _ in range(2):
record = [
datetime.datetime(2013, 12, 12),
numpy.random.uniform(100)
]
modelInput = dict(list(zip(headers, record)))
model.run(modelInput)
# Save and load a checkpoint after each batch. Clean up.
tmpBundleName = os.path.join(tmpDir, "test_checkpoint")
self.assertIs(model.save(tmpBundleName), None,
"Save command failed.")
model = ModelFactory.loadFromCheckpoint(tmpBundleName)
shutil.rmtree(tmpBundleName)
def resurrect_model(saved_model):
return ModelFactory.loadFromCheckpoint(saved_model)
def _runExperimentImpl(options, model=None):
"""Creates and runs the experiment
Args:
options: namedtuple ParseCommandLineOptionsResult
model: For testing: may pass in an existing OPF Model instance
to use instead of creating a new one.
Returns: reference to OPFExperiment instance that was constructed (this
is provided to aid with debugging) or None, if none was
created.
"""
json_helpers.validate(options.privateOptions,
schemaDict=g_parsedPrivateCommandLineOptionsSchema)
# Load the experiment's description.py module
experimentDir = options.experimentDir
descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(
experimentDir)
expIface = helpers.getExperimentDescriptionInterfaceFromModule(
descriptionPyModule)
# Handle "list checkpoints" request
if options.privateOptions['listAvailableCheckpoints']:
_printAvailableCheckpoints(experimentDir)
return None
# Load experiment tasks
experimentTasks = expIface.getModelControl().get('tasks', [])
# If the tasks list is empty, and this is a nupic environment description
# file being run from the OPF, convert it to a simple OPF description file.
if (len(experimentTasks) == 0 and
expIface.getModelControl()['environment'] == OpfEnvironment.Nupic):
expIface.convertNupicEnvToOPF()
experimentTasks = expIface.getModelControl().get('tasks', [])
# Ensures all the source locations are either absolute paths or relative to
# the nupic.datafiles package_data location.
expIface.normalizeStreamSources()
# Extract option
newSerialization = options.privateOptions['newSerialization']
# Handle listTasks
if options.privateOptions['listTasks']:
print "Available tasks:"
for label in [t['taskLabel'] for t in experimentTasks]:
print "\t", label
return None
# Construct the experiment instance
if options.privateOptions['runCheckpointName']:
assert model is None
checkpointName = options.privateOptions['runCheckpointName']
model = ModelFactory.loadFromCheckpoint(
savedModelDir=_getModelCheckpointDir(experimentDir, checkpointName),
newSerialization=newSerialization)
elif model is not None:
print "Skipping creation of OPFExperiment instance: caller provided his own"
else:
modelDescription = expIface.getModelDescription()
model = ModelFactory.create(modelDescription)
# Handle "create model" request
if options.privateOptions['createCheckpointName']:
checkpointName = options.privateOptions['createCheckpointName']
_saveModel(model=model,
experimentDir=experimentDir,
checkpointLabel=checkpointName,
newSerialization=newSerialization)
return model
# Build the task list
# Default task execution index list is in the natural list order of the tasks
taskIndexList = range(len(experimentTasks))
customTaskExecutionLabelsList = options.privateOptions['taskLabels']
if customTaskExecutionLabelsList:
taskLabelsList = [t['taskLabel'] for t in experimentTasks]
taskLabelsSet = set(taskLabelsList)
customTaskExecutionLabelsSet = set(customTaskExecutionLabelsList)
assert customTaskExecutionLabelsSet.issubset(taskLabelsSet), \
("Some custom-provided task execution labels don't correspond "
"to actual task labels: mismatched labels: %r; actual task "
"labels: %r.") % (customTaskExecutionLabelsSet - taskLabelsSet,
customTaskExecutionLabelsList)
taskIndexList = [taskLabelsList.index(label) for label in
customTaskExecutionLabelsList]
print "#### Executing custom task list: %r" % [taskLabelsList[i] for
i in taskIndexList]
# Run all experiment tasks
for taskIndex in taskIndexList:
task = experimentTasks[taskIndex]
# Create a task runner and run it!
taskRunner = _TaskRunner(model=model,
task=task,
cmdOptions=options)
taskRunner.run()
del taskRunner
if options.privateOptions['checkpointModel']:
_saveModel(model=model,
experimentDir=experimentDir,
checkpointLabel=task['taskLabel'],
newSerialization=newSerialization)
return model
def _runExperimentImpl(options, model=None):
"""Creates and runs the experiment
Args:
options: namedtuple ParseCommandLineOptionsResult
model: For testing: may pass in an existing OPF Model instance
to use instead of creating a new one.
Returns: reference to OPFExperiment instance that was constructed (this
is provided to aid with debugging) or None, if none was
created.
"""
jsonhelpers.validate(options.privateOptions,
schemaDict=g_parsedPrivateCommandLineOptionsSchema)
# Load the experiment's description.py module
experimentDir = options.experimentDir
descriptionPyModule = opf_helpers.loadExperimentDescriptionScriptFromDir(
experimentDir)
expIface = opf_helpers.getExperimentDescriptionInterfaceFromModule(
descriptionPyModule)
# Handle "list checkpoints" request
if options.privateOptions['listAvailableCheckpoints']:
_printAvailableCheckpoints(experimentDir)
return None
# Load experiment tasks
experimentTasks = expIface.getModelControl().get('tasks', [])
# If the tasks list is empty, and this is a nupic environment description
# file being run from the OPF, convert it to a simple OPF description file.
if (len(experimentTasks) == 0 and expIface.getModelControl()['environment']
== OpfEnvironment.Nupic):
expIface.convertNupicEnvToOPF()
experimentTasks = expIface.getModelControl().get('tasks', [])
# Ensures all the source locations are either absolute paths or relative to
# the nupic.datafiles package_data location.
expIface.normalizeStreamSources()
# Extract option
newSerialization = options.privateOptions['newSerialization']
# Handle listTasks
if options.privateOptions['listTasks']:
print "Available tasks:"
for label in [t['taskLabel'] for t in experimentTasks]:
print "\t", label
return None
# Construct the experiment instance
if options.privateOptions['runCheckpointName']:
assert model is None
checkpointName = options.privateOptions['runCheckpointName']
model = ModelFactory.loadFromCheckpoint(
savedModelDir=_getModelCheckpointDir(experimentDir,
checkpointName),
newSerialization=newSerialization)
elif model is not None:
print "Skipping creation of OPFExperiment instance: caller provided his own"
else:
modelDescription = expIface.getModelDescription()
model = ModelFactory.create(modelDescription)
# Handle "create model" request
if options.privateOptions['createCheckpointName']:
checkpointName = options.privateOptions['createCheckpointName']
_saveModel(model=model,
experimentDir=experimentDir,
checkpointLabel=checkpointName,
newSerialization=newSerialization)
return model
# Build the task list
# Default task execution index list is in the natural list order of the tasks
taskIndexList = range(len(experimentTasks))
customTaskExecutionLabelsList = options.privateOptions['taskLabels']
if customTaskExecutionLabelsList:
taskLabelsList = [t['taskLabel'] for t in experimentTasks]
taskLabelsSet = set(taskLabelsList)
customTaskExecutionLabelsSet = set(customTaskExecutionLabelsList)
assert customTaskExecutionLabelsSet.issubset(taskLabelsSet), \
("Some custom-provided task execution labels don't correspond "
"to actual task labels: mismatched labels: %r; actual task "
"labels: %r.") % (customTaskExecutionLabelsSet - taskLabelsSet,
customTaskExecutionLabelsList)
taskIndexList = [
taskLabelsList.index(label)
for label in customTaskExecutionLabelsList
]
print "#### Executing custom task list: %r" % [
taskLabelsList[i] for i in taskIndexList
]
# Run all experiment tasks
for taskIndex in taskIndexList:
task = experimentTasks[taskIndex]
# Create a task runner and run it!
taskRunner = _TaskRunner(model=model, task=task, cmdOptions=options)
taskRunner.run()
del taskRunner
if options.privateOptions['checkpointModel']:
_saveModel(model=model,
experimentDir=experimentDir,
checkpointLabel=task['taskLabel'],
newSerialization=newSerialization)
return model
def resurrect_model(saved_model): return ModelFactory.loadFromCheckpoint(saved_model)
def __init__(self, use_saved_model, checkpoint_path, likelihood_path):
self.use_saved_model = use_saved_model
if use_saved_model:
self.salt_acc_model = ModelFactory.loadFromCheckpoint(
checkpoint_path + '_SALT_ACC')
self.salt_qua_model = ModelFactory.loadFromCheckpoint(
checkpoint_path + '_SALT_QUA')
self.pepa_acc_model = ModelFactory.loadFromCheckpoint(
checkpoint_path + '_PEPA_ACC')
self.pepa_qua_model = ModelFactory.loadFromCheckpoint(
checkpoint_path + '_PEPA_QUA')
self.salt_acc_model.enableInference({'predictedField': 'x'})
self.salt_acc_model.enableInference({'predictedField': 'y'})
self.salt_acc_model.enableInference({'predictedField': 'z'})
self.salt_qua_model.enableInference({'predictedField': 'w'})
self.salt_qua_model.enableInference({'predictedField': 'x'})
self.salt_qua_model.enableInference({'predictedField': 'y'})
self.salt_qua_model.enableInference({'predictedField': 'z'})
self.pepa_acc_model.enableInference({'predictedField': 'x'})
self.pepa_acc_model.enableInference({'predictedField': 'y'})
self.pepa_acc_model.enableInference({'predictedField': 'z'})
self.pepa_qua_model.enableInference({'predictedField': 'w'})
self.pepa_qua_model.enableInference({'predictedField': 'x'})
self.pepa_qua_model.enableInference({'predictedField': 'y'})
self.pepa_qua_model.enableInference({'predictedField': 'z'})
with open(likelihood_path, "rb") as f:
self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood(
).readFromFile(f)
else:
self.salt_acc_model = ModelFactory.create(
model_params_salt_acc.MODEL_PARAMS)
self.salt_qua_model = ModelFactory.create(
model_params_salt_qua.MODEL_PARAMS)
self.pepa_acc_model = ModelFactory.create(
model_params_pepa_acc.MODEL_PARAMS)
self.pepa_qua_model = ModelFactory.create(
model_params_pepa_qua.MODEL_PARAMS)
self.salt_acc_model.enableInference({'predictedField': 'x'})
self.salt_acc_model.enableInference({'predictedField': 'y'})
self.salt_acc_model.enableInference({'predictedField': 'z'})
self.salt_qua_model.enableInference({'predictedField': 'w'})
self.salt_qua_model.enableInference({'predictedField': 'x'})
self.salt_qua_model.enableInference({'predictedField': 'y'})
self.salt_qua_model.enableInference({'predictedField': 'z'})
self.pepa_acc_model.enableInference({'predictedField': 'x'})
self.pepa_acc_model.enableInference({'predictedField': 'y'})
self.pepa_acc_model.enableInference({'predictedField': 'z'})
self.pepa_qua_model.enableInference({'predictedField': 'w'})
self.pepa_qua_model.enableInference({'predictedField': 'x'})
self.pepa_qua_model.enableInference({'predictedField': 'y'})
self.pepa_qua_model.enableInference({'predictedField': 'z'})
self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood()
def __get_model(self, metric, model_type,
models_number_below_configured_limit):
model_prefix = "_" + model_type
result_model = None
model_fqdn = model_prefix + "." + metric["metric_name"]
if not self.__loaded_models.model_exists(model_fqdn):
self.__create_model_thread_lock.acquire()
try:
if not self.__loaded_models.model_exists(
model_fqdn) and os.path.isdir(
self.__model_storage_manager.get_save_path(
model_fqdn)):
if models_number_below_configured_limit:
try:
self.__loaded_models.add_model_for_metric(
model_fqdn,
NupicModelFactory.loadFromCheckpoint(
self.__model_storage_manager.get_save_path(
model_fqdn)))
self._logger.debug(
"__get_model", "LOADED " + model_type.upper() +
" MODEL FROM DISK")
except Exception as ex:
self.__loaded_models.add_model_for_metric(
model_fqdn,
self.__create_model(
self.get_model_params_from_metric_name(
metric["metric_family"], model_type),
self._config_mgr.get("prediction_steps")))
self._logger.warn(
"__get_model",
"Failed to create a " + model_type +
" model from disk",
exception_message=str(ex.message),
exception_type=str(type(ex).__name__))
if not self.__loaded_models.model_exists(
model_fqdn) and not os.path.isdir(
self.__model_storage_manager.get_save_path(
model_fqdn)):
if models_number_below_configured_limit:
model_params = self.get_model_params_from_metric_name(
metric["metric_family"], model_type)
prediction_steps = self._config_mgr.get(
"prediction_steps")
model_to_add = self.__create_model(
model_params, prediction_steps)
self.__loaded_models.add_model_for_metric(
model_fqdn, model_to_add)
self._logger.debug("__get_model",
model_type.capitalize() +
" model created from params",
metric=str(metric["metric_name"]))
finally:
self.__create_model_thread_lock.release()
if self.__loaded_models.model_exists(model_fqdn):
result_model = self.__loaded_models.get_model(model_fqdn)
self._logger.debug("__get_model",
model_type.capitalize() +
" model loaded from cache",
metric=str(metric["metric_name"]))
return result_model
