def testCloneModelWithDefinitionOnly(self):
checkpointMgr = ModelCheckpointMgr()
modelID = uuid.uuid1().hex
destModelID = uuid.uuid1().hex
# Create the source model with meta-info only (no checkpoint)
modelDef = {'a': 1, 'b': 2, 'c': 3}
checkpointMgr.define(modelID, modelDef)
# Clone the source model
checkpointMgr.clone(modelID, destModelID)
# Verify that the destination model's definition is the same as the
# source model's
destModelDef = checkpointMgr.loadModelDefinition(destModelID)
self.assertDictEqual(destModelDef, modelDef)
# Calling load when the model checkpoint doesn't exist should raise an
# exception
with self.assertRaises(ModelNotFound):
checkpointMgr.load(destModelID)
# Calling clone when the destination model archive already exists should
# raise an exception
with self.assertRaises(ModelAlreadyExists):
checkpointMgr.clone(modelID, destModelID)
def testCloneModelWithCheckpoint(self):
checkpointMgr = ModelCheckpointMgr()
modelID = uuid.uuid1().hex
destModelID = uuid.uuid1().hex
# Create the source model with meta-info only (no checkpoint)
modelDef = {'a': 1, 'b': 2, 'c': 3}
checkpointMgr.define(modelID, modelDef)
# Create a model that we can clone
model1 = ModelFactory.create(self._getModelParams("variant1"))
checkpointMgr.save(modelID, model1, attributes="attributes1")
# Clone the source model
checkpointMgr.clone(modelID, destModelID)
# Discard the source model checkpoint
checkpointMgr.remove(modelID)
# Verify that the destination model's definition is the same as the
# source model's
destModelDef = checkpointMgr.loadModelDefinition(destModelID)
self.assertDictEqual(destModelDef, modelDef)
# Verify that the destination model's attributes match the source's
attributes = checkpointMgr.loadCheckpointAttributes(destModelID)
self.assertEqual(attributes, "attributes1")
# Attempt to load the cloned model from checkpoint
model = checkpointMgr.load(destModelID)
self.assertEqual(str(model.getFieldInfo()), str(model1.getFieldInfo()))
def testRemoveAll(self):
"""
Test removeAll
"""
checkpointMgr = ModelCheckpointMgr()
# Should be empty at first
ids = checkpointMgr.getModelIDs()
self.assertSequenceEqual(ids, [])
# Create some checkpoints using meta info
expModelIDs = [uuid.uuid1().hex, uuid.uuid1().hex]
expModelIDs.sort()
for modelID in expModelIDs:
checkpointMgr.define(modelID, definition={'a':1})
ids = checkpointMgr.getModelIDs()
self.assertItemsEqual(ids, expModelIDs)
# Delete checkpoint store
ModelCheckpointMgr.removeAll()
ids = checkpointMgr.getModelIDs()
self.assertSequenceEqual(ids, [])
def testStorageDirOverrideViaEnvironmentVariable(self):
with ModelCheckpointStoragePatch() as storagePatch:
checkpointMgr = ModelCheckpointMgr()
tempModelCheckpointDir = storagePatch.tempModelCheckpointDir
modelEntryDir = checkpointMgr._getModelDir(modelID="abc", mustExist=False)
self.assertIn(tempModelCheckpointDir, modelEntryDir)
def testUpdateCheckpointAttributesNoModelEntry(self):
""" When a model entry doesn't exist, calling updateCheckpointAttributes
should raise ModelNotFound
"""
checkpointMgr = ModelCheckpointMgr()
modelID = uuid.uuid1().hex
# Calling updateCheckpointAttributes when the model entry doesn't exist
# should raise an exception
with self.assertRaises(ModelNotFound):
checkpointMgr.updateCheckpointAttributes(modelID, "attributes")
def testUpdateCheckpointAttributesWithNoModeCheckpointException(self):
""" When a model entry exists, but a checkpoint hasn't been saved yet,
calling updateCheckpointAttributes should raise ModelNotFound
"""
checkpointMgr = ModelCheckpointMgr()
modelID = uuid.uuid1().hex
# Save its definition
checkpointMgr.define(modelID, definition=dict(a=1, b=2))
with self.assertRaises(ModelNotFound):
checkpointMgr.updateCheckpointAttributes(modelID, "attributes")
def testCloneModelFromNonExistentSourceRaisesModelNotFound(self):
checkpointMgr = ModelCheckpointMgr()
# Create the source model with meta-info only
modelID = uuid.uuid1().hex
destModelID = uuid.uuid1().hex
# Calling clone when the source model archive doesn't exist should
# raise an exception
with self.assertRaises(ModelNotFound):
checkpointMgr.clone(modelID, destModelID)
# Let's try it again to make sure that the first attempt did not create
# unwanted side-effect
with self.assertRaises(ModelNotFound):
checkpointMgr.clone(modelID, destModelID)
def testDefineModel(self):
""" Test support for defining a model """
checkpointMgr = ModelCheckpointMgr()
modelID = uuid.uuid1().hex
modelDefinition = {'a': 1, 'b': 2, 'c':3}
# Calling loadModelDefinition when the model doesn't exist should raise
# ModelNotFound
self.assertRaises(ModelNotFound, checkpointMgr.loadModelDefinition, modelID)
# Define the model
checkpointMgr.define(modelID, definition=modelDefinition)
# Load model definition and verify integrity of model definition object
retrievedModelDefinition = checkpointMgr.loadModelDefinition(modelID)
self.assertDictEqual(retrievedModelDefinition, modelDefinition)
def testRemoveAndGetModelIDs(self):
"""
Test getModelIDs and remove methods
"""
checkpointMgr = ModelCheckpointMgr()
# Should be empty at first
ids = checkpointMgr.getModelIDs()
self.assertListEqual(ids, [])
# Create some checkpoints using meta info
expModelIDs = [uuid.uuid1().hex, uuid.uuid1().hex]
expModelIDs.sort()
for modelID in expModelIDs:
checkpointMgr.define(modelID, definition={'a':1})
ids = checkpointMgr.getModelIDs()
ids.sort()
self.assertListEqual(ids, expModelIDs)
# Delete one of them
checkpointMgr.remove(expModelIDs[0])
expModelIDs.remove(expModelIDs[0])
ids = checkpointMgr.getModelIDs()
ids.sort()
self.assertListEqual(ids, expModelIDs)
# Delete all of them
for modelID in expModelIDs:
checkpointMgr.remove(modelID)
ids = checkpointMgr.getModelIDs()
self.assertListEqual(ids, [])
# If we try and delete a non-existing model, should get an exception
self.assertRaises(ModelNotFound, checkpointMgr.remove, "IDx")
def __init__(self, modelID):
"""
:param modelID: model ID; string
"""
self._modelID = modelID
# The model object from OPF ModelFactory; set up by the loadModel() method
self._model = None
# The _InputRowEncoder object; set up by the loadModel() method
self._inputRowEncoder = None
self._checkpointMgr = ModelCheckpointMgr()
# True if model was loaded from existing checkpoint or after a full
# checkpoint is made; False initially if model was created from
# params. This informs us whether an incremental checkpoint is possible.
self._hasCheckpoint = False
self._modelCheckpointBatchIDSetCache = None
# Input data samples that have accumulated since last full checkpoint
self._inputSamplesSinceLastFullCheckpointCache = None
def testUpdateCheckpointAttributes(self):
""" Test updateCheckpointAttributes """
checkpointMgr = ModelCheckpointMgr()
modelID = uuid.uuid1().hex
# Create a model that we can save
originalModel = ModelFactory.create(self._getModelParams("variant1"))
# Save its definition
checkpointMgr.define(modelID, definition=dict(a=1, b=2))
# Save the checkpoint
checkpointMgr.save(modelID, originalModel, attributes="attributes1")
self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID),
"attributes1")
# Update model checkpoint attribes
newAttributes = dict(this=[1, 2, 3], that="abc")
checkpointMgr.updateCheckpointAttributes(modelID, newAttributes)
self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID),
newAttributes)
def testModelCheckpointSaveAndLoadSupport(self):
""" Test saving and loading models """
checkpointMgr = ModelCheckpointMgr()
modelID = uuid.uuid1().hex
# Calling load when the model doesn't exist should raise an
# exception
self.assertRaises(ModelNotFound, checkpointMgr.load, modelID)
# Create a model that we can save
originalModel = ModelFactory.create(self._getModelParams("variant1"))
# Save it
checkpointMgr.define(modelID, definition=dict(a=1, b=2))
# Attempting to load model that hasn't been checkpointed should raise
# ModelNotFound
self.assertRaises(ModelNotFound, checkpointMgr.load, modelID)
# Save the checkpoint
checkpointMgr.save(modelID, originalModel, attributes="attributes1")
# Load the model from the saved checkpoint
loadedModel = checkpointMgr.load(modelID)
self.assertEqual(str(loadedModel.getFieldInfo()),
str(originalModel.getFieldInfo()))
del loadedModel
del originalModel
self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID),
"attributes1")
# Make sure we can replace an existing model
model2 = ModelFactory.create(self._getModelParams("variant2"))
checkpointMgr.save(modelID, model2, attributes="attributes2")
model = checkpointMgr.load(modelID)
self.assertEqual(str(model.getFieldInfo()), str(model2.getFieldInfo()))
self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID),
"attributes2")
model3 = ModelFactory.create(self._getModelParams("variant3"))
checkpointMgr.save(modelID, model3, attributes="attributes3")
model = checkpointMgr.load(modelID)
self.assertEqual(str(model.getFieldInfo()), str(model3.getFieldInfo()))
self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID),
"attributes3")
# Simulate a failure during checkpointing and make sure it doesn't mess
# up our already existing checkpoint
try:
checkpointMgr.save(modelID,
"InvalidModel",
attributes="attributes4")
except AttributeError:
pass
model = checkpointMgr.load(modelID)
self.assertEqual(str(model.getFieldInfo()), str(model3.getFieldInfo()))
self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID),
"attributes3")
