def testComputeRecall(self): trueValues = [] learnedValues = [] self.assertEqual(1.0, ModelEvaluation.recall(trueValues, learnedValues)) trueValues = [1] learnedValues = [] self.assertEqual(0.0, ModelEvaluation.recall(trueValues, learnedValues)) trueValues = [] learnedValues = [1] self.assertEqual(1.0, ModelEvaluation.recall(trueValues, learnedValues)) trueValues = [1, 2] learnedValues = [2] self.assertEqual(0.5, ModelEvaluation.recall(trueValues, learnedValues)) trueValues = [1, 2] learnedValues = [2, 3] self.assertEqual(0.5, ModelEvaluation.recall(trueValues, learnedValues)) trueValues = [1, 2] learnedValues = [2, 3, 4] self.assertEqual(0.5, ModelEvaluation.recall(trueValues, learnedValues)) trueValues = [1, 2, 3] learnedValues = [2, 3, 4] self.assertEqual(2 / 3, ModelEvaluation.recall(trueValues, learnedValues))
def testComputePrecision(self): trueValues = [] learnedValues = [] self.assertEqual(1.0, ModelEvaluation.precision(trueValues, learnedValues)) trueValues = [1] learnedValues = [] self.assertEqual(1.0, ModelEvaluation.precision(trueValues, learnedValues)) trueValues = [] learnedValues = [1] self.assertEqual(0.0, ModelEvaluation.precision(trueValues, learnedValues)) trueValues = [1, 2] learnedValues = [2] self.assertEqual(1.0, ModelEvaluation.precision(trueValues, learnedValues)) trueValues = [1, 2] learnedValues = [2, 3] self.assertEqual(0.5, ModelEvaluation.precision(trueValues, learnedValues)) trueValues = [1, 2] learnedValues = [2, 3, 4] self.assertEqual(1 / 3, ModelEvaluation.precision(trueValues, learnedValues))
def testSkeletonPrecision(self): schema = Schema() schema.addEntity('A') schema.addAttribute('A', 'X1') schema.addAttribute('A', 'X2') schema.addAttribute('A', 'X3') schema.addAttribute('A', 'X4') dependencies = [] model = Model(schema, dependencies) learnedDependencies = [] self.assertEqual( 1.0, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) learnedDependencies = ['[A].X1 -> [A].X2'] self.assertEqual( 0.0, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) # true model has one dependency dependencies = ['[A].X1 -> [A].X2'] model = Model(schema, dependencies) learnedDependencies = [] self.assertEqual( 1.0, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) learnedDependencies = ['[A].X1 -> [A].X2'] self.assertEqual( 1.0, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) learnedDependencies = ['[A].X1 -> [A].X2', '[A].X2 -> [A].X1'] self.assertEqual( 1.0, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) # true model has two dependencies dependencies = ['[A].X1 -> [A].X2', '[A].X3 -> [A].X2'] model = Model(schema, dependencies) learnedDependencies = ['[A].X3 -> [A].X2'] self.assertEqual( 1.0, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) learnedDependencies = ['[A].X3 -> [A].X2', '[A].X2 -> [A].X3'] self.assertEqual( 1.0, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) learnedDependencies = [ '[A].X3 -> [A].X2', '[A].X2 -> [A].X3', '[A].X2 -> [A].X4' ] self.assertEqual( 0.5, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) learnedDependencies = [ '[A].X3 -> [A].X2', '[A].X2 -> [A].X3', '[A].X2 -> [A].X4', '[A].X4 -> [A].X2' ] self.assertEqual( 0.5, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) learnedDependencies = [ '[A].X3 -> [A].X2', '[A].X2 -> [A].X3', '[A].X2 -> [A].X4', '[A].X4 -> [A].X2', '[A].X1 -> [A].X4' ] self.assertEqual( 1 / 3, ModelEvaluation.skeletonPrecision(model, learnedDependencies)) learnedDependencies = [ '[A].X3 -> [A].X2', '[A].X2 -> [A].X3', '[A].X2 -> [A].X4', '[A].X4 -> [A].X2', '[A].X1 -> [A].X4', '[A].X4 -> [A].X1' ] self.assertEqual( 1 / 3, ModelEvaluation.skeletonPrecision(model, learnedDependencies))
except Exception: continue # Some RCD algorithm takes too much time. # This limits generated models to be 'easy' if len(RelationalSpace.getRelationalDependencies(schema, hopThreshold)) > 100: continue # This oracle uses AGGs. oracle = Oracle(model, 2 * hopThreshold) # Since CI-tests are cached, comparing time spent on RCD and RCDL directly should be avoided. rcdl = RCDLight(schema, oracle, hopThreshold) rcdl.identifyUndirectedDependencies() rcdl.orientDependencies() rcd = RCD(schema, oracle, hopThreshold, depth=rcdDepth) rcd.identifyUndirectedDependencies() rcd.orientDependencies() assert ModelEvaluation.orientedPrecision(model, rcdl.orientedDependencies) == 1.0 assert ModelEvaluation.skeletonPrecision(model, rcdl.undirectedDependencies) == 1.0 assert ModelEvaluation.skeletonRecall(model, rcdl.undirectedDependencies) == 1.0 rcdl_ori_recall = ModelEvaluation.orientedRecall(model, rcdl.orientedDependencies) rcd_ori_recall = ModelEvaluation.orientedRecall(model, rcd.orientedDependencies) assert rcdl_ori_recall >= rcd_ori_recall print('.', end='', flush=True) if rcdl_ori_recall > rcd_ori_recall: print('\nRCDL beats RCD:', rcdl_ori_recall, '>', rcd_ori_recall)
from causality.model.RelationalDependency import RelationalVariable logger = logging.getLogger(__name__) logging.basicConfig(level=logging.ERROR) # Parameters schema, model = shlee.RCDLight.incompleteness_example() logger.info('Model: %s', model.dependencies) hopThreshold = max(len(d.relVar1.path) + 1 for d in model.dependencies) oracle = Oracle(model, 3 * hopThreshold) rcd = RCD(schema, oracle, hopThreshold, depth=2) rcd.identifyUndirectedDependencies() rcd.orientDependencies() print('Skeleton precision:', ModelEvaluation.skeletonPrecision(model, rcd.undirectedDependencies)) print('Skeleton recall:', ModelEvaluation.skeletonRecall(model, rcd.undirectedDependencies)) precision = ModelEvaluation.orientedPrecision(model, rcd.orientedDependencies) print('Oriented precision:', precision) print('Oriented recall:', ModelEvaluation.orientedRecall(model, rcd.orientedDependencies)) rcdl = shlee.RCDLight.RCDLight(schema, oracle, hopThreshold) rcdl.identifyUndirectedDependencies() rcdl.orientDependencies() print('Skeleton precision:', ModelEvaluation.skeletonPrecision(model, rcdl.undirectedDependencies)) print('Skeleton recall:', ModelEvaluation.skeletonRecall(model, rcdl.undirectedDependencies)) precision = ModelEvaluation.orientedPrecision(model, rcdl.orientedDependencies) print('Oriented precision:', precision) print('Oriented recall:', ModelEvaluation.orientedRecall(model, rcdl.orientedDependencies)) assert ModelEvaluation.orientedRecall(model, rcdl.orientedDependencies) == \
logger = logging.getLogger(__name__) logging.basicConfig(level=logging.ERROR) # Parameters schema, model = shlee.RCDLight.incompleteness_example() logger.info('Model: %s', model.dependencies) hopThreshold = max(len(d.relVar1.path) + 1 for d in model.dependencies) oracle = Oracle(model, 3 * hopThreshold) rcd = RCD(schema, oracle, hopThreshold, depth=2) rcd.identifyUndirectedDependencies() rcd.orientDependencies() print('Skeleton precision:', ModelEvaluation.skeletonPrecision(model, rcd.undirectedDependencies)) print('Skeleton recall:', ModelEvaluation.skeletonRecall(model, rcd.undirectedDependencies)) precision = ModelEvaluation.orientedPrecision(model, rcd.orientedDependencies) print('Oriented precision:', precision) print('Oriented recall:', ModelEvaluation.orientedRecall(model, rcd.orientedDependencies)) rcdl = shlee.RCDLight.RCDLight(schema, oracle, hopThreshold) rcdl.identifyUndirectedDependencies() rcdl.orientDependencies() print('Skeleton precision:', ModelEvaluation.skeletonPrecision(model, rcdl.undirectedDependencies)) print('Skeleton recall:', ModelEvaluation.skeletonRecall(model, rcdl.undirectedDependencies)) precision = ModelEvaluation.orientedPrecision(model, rcdl.orientedDependencies)
numDependencies = 10 hopThreshold = 4 maxNumParents = rcdDepth = 3 # Parameters schema = SchemaGenerator.generateSchema(numEntities, numRelationships, allowCycles=False, oneRelationshipPerPair=True) logger.info(schema) model = ModelGenerator.generateModel(schema, hopThreshold, numDependencies, maxNumParents=maxNumParents) logger.info('Model: %s', model.dependencies) oracle = Oracle(model, 2 * hopThreshold) # Run RCD algorithm and collect statistics on learned model rcd = RCD(schema, oracle, hopThreshold, depth=rcdDepth) rcd.identifyUndirectedDependencies() rcd.orientDependencies() logger.info( 'Skeleton precision: %s', ModelEvaluation.skeletonPrecision(model, rcd.undirectedDependencies)) logger.info('Skeleton recall: %s', ModelEvaluation.skeletonRecall(model, rcd.undirectedDependencies)) logger.info('Oriented precision: %s', ModelEvaluation.orientedPrecision(model, rcd.orientedDependencies)) logger.info('Oriented recall: %s', ModelEvaluation.orientedRecall(model, rcd.orientedDependencies))