def testCloneModelWithDefinitionOnly(self): checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex destModelID = uuid.uuid1().hex # Create the source model with meta-info only (no checkpoint) modelDef = {'a': 1, 'b': 2, 'c': 3} checkpointMgr.define(modelID, modelDef) # Clone the source model checkpointMgr.clone(modelID, destModelID) # Verify that the destination model's definition is the same as the # source model's destModelDef = checkpointMgr.loadModelDefinition(destModelID) self.assertDictEqual(destModelDef, modelDef) # Calling load when the model checkpoint doesn't exist should raise an # exception with self.assertRaises(ModelNotFound): checkpointMgr.load(destModelID) # Calling clone when the destination model archive already exists should # raise an exception with self.assertRaises(ModelAlreadyExists): checkpointMgr.clone(modelID, destModelID)
def testCloneModelWithCheckpoint(self): checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex destModelID = uuid.uuid1().hex # Create the source model with meta-info only (no checkpoint) modelDef = {'a': 1, 'b': 2, 'c': 3} checkpointMgr.define(modelID, modelDef) # Create a model that we can clone model1 = ModelFactory.create(self._getModelParams("variant1")) checkpointMgr.save(modelID, model1, attributes="attributes1") # Clone the source model checkpointMgr.clone(modelID, destModelID) # Discard the source model checkpoint checkpointMgr.remove(modelID) # Verify that the destination model's definition is the same as the # source model's destModelDef = checkpointMgr.loadModelDefinition(destModelID) self.assertDictEqual(destModelDef, modelDef) # Verify that the destination model's attributes match the source's attributes = checkpointMgr.loadCheckpointAttributes(destModelID) self.assertEqual(attributes, "attributes1") # Attempt to load the cloned model from checkpoint model = checkpointMgr.load(destModelID) self.assertEqual(str(model.getFieldInfo()), str(model1.getFieldInfo()))
def testRemoveAll(self): """ Test removeAll """ checkpointMgr = ModelCheckpointMgr() # Should be empty at first ids = checkpointMgr.getModelIDs() self.assertSequenceEqual(ids, []) # Create some checkpoints using meta info expModelIDs = [uuid.uuid1().hex, uuid.uuid1().hex] expModelIDs.sort() for modelID in expModelIDs: checkpointMgr.define(modelID, definition={'a':1}) ids = checkpointMgr.getModelIDs() self.assertItemsEqual(ids, expModelIDs) # Delete checkpoint store ModelCheckpointMgr.removeAll() ids = checkpointMgr.getModelIDs() self.assertSequenceEqual(ids, [])
def testStorageDirOverrideViaEnvironmentVariable(self): with ModelCheckpointStoragePatch() as storagePatch: checkpointMgr = ModelCheckpointMgr() tempModelCheckpointDir = storagePatch.tempModelCheckpointDir modelEntryDir = checkpointMgr._getModelDir(modelID="abc", mustExist=False) self.assertIn(tempModelCheckpointDir, modelEntryDir)
def testUpdateCheckpointAttributesNoModelEntry(self): """ When a model entry doesn't exist, calling updateCheckpointAttributes should raise ModelNotFound """ checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex # Calling updateCheckpointAttributes when the model entry doesn't exist # should raise an exception with self.assertRaises(ModelNotFound): checkpointMgr.updateCheckpointAttributes(modelID, "attributes")
def testUpdateCheckpointAttributesWithNoModeCheckpointException(self): """ When a model entry exists, but a checkpoint hasn't been saved yet, calling updateCheckpointAttributes should raise ModelNotFound """ checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex # Save its definition checkpointMgr.define(modelID, definition=dict(a=1, b=2)) with self.assertRaises(ModelNotFound): checkpointMgr.updateCheckpointAttributes(modelID, "attributes")
def testCloneModelFromNonExistentSourceRaisesModelNotFound(self): checkpointMgr = ModelCheckpointMgr() # Create the source model with meta-info only modelID = uuid.uuid1().hex destModelID = uuid.uuid1().hex # Calling clone when the source model archive doesn't exist should # raise an exception with self.assertRaises(ModelNotFound): checkpointMgr.clone(modelID, destModelID) # Let's try it again to make sure that the first attempt did not create # unwanted side-effect with self.assertRaises(ModelNotFound): checkpointMgr.clone(modelID, destModelID)
def testDefineModel(self): """ Test support for defining a model """ checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex modelDefinition = {'a': 1, 'b': 2, 'c':3} # Calling loadModelDefinition when the model doesn't exist should raise # ModelNotFound self.assertRaises(ModelNotFound, checkpointMgr.loadModelDefinition, modelID) # Define the model checkpointMgr.define(modelID, definition=modelDefinition) # Load model definition and verify integrity of model definition object retrievedModelDefinition = checkpointMgr.loadModelDefinition(modelID) self.assertDictEqual(retrievedModelDefinition, modelDefinition)
def testRemoveAndGetModelIDs(self): """ Test getModelIDs and remove methods """ checkpointMgr = ModelCheckpointMgr() # Should be empty at first ids = checkpointMgr.getModelIDs() self.assertListEqual(ids, []) # Create some checkpoints using meta info expModelIDs = [uuid.uuid1().hex, uuid.uuid1().hex] expModelIDs.sort() for modelID in expModelIDs: checkpointMgr.define(modelID, definition={'a':1}) ids = checkpointMgr.getModelIDs() ids.sort() self.assertListEqual(ids, expModelIDs) # Delete one of them checkpointMgr.remove(expModelIDs[0]) expModelIDs.remove(expModelIDs[0]) ids = checkpointMgr.getModelIDs() ids.sort() self.assertListEqual(ids, expModelIDs) # Delete all of them for modelID in expModelIDs: checkpointMgr.remove(modelID) ids = checkpointMgr.getModelIDs() self.assertListEqual(ids, []) # If we try and delete a non-existing model, should get an exception self.assertRaises(ModelNotFound, checkpointMgr.remove, "IDx")
def __init__(self, modelID): """ :param modelID: model ID; string """ self._modelID = modelID # The model object from OPF ModelFactory; set up by the loadModel() method self._model = None # The _InputRowEncoder object; set up by the loadModel() method self._inputRowEncoder = None self._checkpointMgr = ModelCheckpointMgr() # True if model was loaded from existing checkpoint or after a full # checkpoint is made; False initially if model was created from # params. This informs us whether an incremental checkpoint is possible. self._hasCheckpoint = False self._modelCheckpointBatchIDSetCache = None # Input data samples that have accumulated since last full checkpoint self._inputSamplesSinceLastFullCheckpointCache = None
def testUpdateCheckpointAttributes(self): """ Test updateCheckpointAttributes """ checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex # Create a model that we can save originalModel = ModelFactory.create(self._getModelParams("variant1")) # Save its definition checkpointMgr.define(modelID, definition=dict(a=1, b=2)) # Save the checkpoint checkpointMgr.save(modelID, originalModel, attributes="attributes1") self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID), "attributes1") # Update model checkpoint attribes newAttributes = dict(this=[1, 2, 3], that="abc") checkpointMgr.updateCheckpointAttributes(modelID, newAttributes) self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID), newAttributes)
def testModelCheckpointSaveAndLoadSupport(self): """ Test saving and loading models """ checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex # Calling load when the model doesn't exist should raise an # exception self.assertRaises(ModelNotFound, checkpointMgr.load, modelID) # Create a model that we can save originalModel = ModelFactory.create(self._getModelParams("variant1")) # Save it checkpointMgr.define(modelID, definition=dict(a=1, b=2)) # Attempting to load model that hasn't been checkpointed should raise # ModelNotFound self.assertRaises(ModelNotFound, checkpointMgr.load, modelID) # Save the checkpoint checkpointMgr.save(modelID, originalModel, attributes="attributes1") # Load the model from the saved checkpoint loadedModel = checkpointMgr.load(modelID) self.assertEqual(str(loadedModel.getFieldInfo()), str(originalModel.getFieldInfo())) del loadedModel del originalModel self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID), "attributes1") # Make sure we can replace an existing model model2 = ModelFactory.create(self._getModelParams("variant2")) checkpointMgr.save(modelID, model2, attributes="attributes2") model = checkpointMgr.load(modelID) self.assertEqual(str(model.getFieldInfo()), str(model2.getFieldInfo())) self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID), "attributes2") model3 = ModelFactory.create(self._getModelParams("variant3")) checkpointMgr.save(modelID, model3, attributes="attributes3") model = checkpointMgr.load(modelID) self.assertEqual(str(model.getFieldInfo()), str(model3.getFieldInfo())) self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID), "attributes3") # Simulate a failure during checkpointing and make sure it doesn't mess # up our already existing checkpoint try: checkpointMgr.save(modelID, "InvalidModel", attributes="attributes4") except AttributeError: pass model = checkpointMgr.load(modelID) self.assertEqual(str(model.getFieldInfo()), str(model3.getFieldInfo())) self.assertEqual(checkpointMgr.loadCheckpointAttributes(modelID), "attributes3")