def learn3(self, k, intBaseFeat, boolBaseFeat, baseFeatureValues, exclude,
call):
#on the empty set of data points, return true
if len(baseFeatureValues) == 0:
print("called learn3 with 0 feature vectors")
logger.info("called learn3 with 0 feature vectors")
return PrecisFormula(BoolVal(False))
#rename splitIntoBoolAndIntFeatureVectors
(intBaseFeatVectors,
boolBaseFeatVectors) = Featurizer.getBoolAndIntFeatureVectors(
intBaseFeat, boolBaseFeat, baseFeatureValues)
derivFeats = self.synthesizeUniqueFeatures(intBaseFeat, boolBaseFeat,
baseFeatureValues, exclude)
derivFeatVectors: List[
FeatureVector] = Featurizer.generateDerivedFeatureVectors(
derivFeats, intBaseFeat + boolBaseFeat, baseFeatureValues)
#assert(len(baseFeatureValues) == len(derivFeatVectors))
boolFvs = Featurizer.mergeFeatureVectors(boolBaseFeatVectors,
derivFeatVectors)
houdini = Houdini()
(allTrueFormula,
indicesAllwaysTrue) = houdini.learn2(boolBaseFeat + derivFeats,
boolFvs, call)
logger.info("Houdini AlwaysTrue for k=" + str(k) + " : " +
allTrueFormula.toInfix() + "\n")
if k == 0:
return allTrueFormula
else:
#removing features returned by houdini and their corresponding feature vector entries.
(remainingBaseBoolFeat, remainingDerivBoolFeat, featuresRemoved) = \
self.removeFeatureFromFeaturelist(boolBaseFeat, derivFeats, indicesAllwaysTrue)
(reaminingEntriesBaseBoolFv, reaminingEntriesDerivBoolFv) = \
self.removeFeatureEntryInFeatureVectors(boolBaseFeatVectors, derivFeatVectors, indicesAllwaysTrue)
# features that are true on parent node should not be passed down to children;(they are redundantly also true in child nodes)
exclude = exclude + featuresRemoved
lookAhead = len(intBaseFeatVectors[0])
######################################
#bug: chooseFeatureImplication does not update reamining bool features or feature vectors. Idx is with respect to updates
(f,idx, posBaseFv, negBaseFv, remainingBaseBoolFeat, remainingDerivBoolFeat ) = \
self.chooseFeatureImplication(allTrueFormula,intBaseFeat,remainingBaseBoolFeat , remainingDerivBoolFeat, \
Featurizer.mergeFeatureVectors(intBaseFeatVectors,reaminingEntriesBaseBoolFv) , reaminingEntriesDerivBoolFv, lookAhead, call )
######################################
if idx < 0:
print("Predicate: " + call + " for k = " + str(k) + " : None")
logger.info("Predicate: " + call + " for k = " + str(k) +
" : None" + "\n")
return allTrueFormula
#TODO: choose should return boolBasePosFv and intBasePosFv ...
#(f,idx, posBaseFv, negBaseFv) = \
# self.chooseFeature2(remainingBaseBoolFeat + remainingDerivBoolFeat, \
# Featurizer.mergeFeatureVectors(intBaseFeatVectors,reaminingEntriesBaseBoolFv), reaminingEntriesDerivBoolFv, call, lookAhead)
logger.info("Predicate: " + call + " for k = " + str(k) + " : " +
str(f) + "\n")
print("Predicate chosen at " + call + " : " + str(f))
#featureSplitRemoved == f
(newBoolBaseFeat, newDeriveBaseFeat, featureSplitRemoved) = \
self.removeFeatureFromFeaturelist(remainingBaseBoolFeat, remainingDerivBoolFeat, [idx])
# if predicate to split on is in derivedFeatures, then add to exclude list;
if len(remainingBaseBoolFeat) == len(newBoolBaseFeat):
exclude = exclude + (f, )
else:
# if predicate to split is in baseFeatures, the update posBaseFv and negBaseFv feature vectors
posBaseFv = self.removeFeatureEntryInBaseFv(
posBaseFv, [idx + lookAhead])
negBaseFv = self.removeFeatureEntryInBaseFv(
negBaseFv, [idx + lookAhead])
posPost = self.learn3( k-1,\
intBaseFeat, newBoolBaseFeat, posBaseFv, exclude, call + " Left") #recursive call
logger.info(call + " Left: " + " for k = " + str(k) + " : " +
posPost.toInfix())
print(call + " Left: " + " for k = " + str(k) + " : " +
posPost.toInfix())
negPost = self.learn3( k-1,\
intBaseFeat, newBoolBaseFeat, negBaseFv, exclude, call +" Right") #recursive call
logger.info(call + " Right: " + " for k = " + str(k) + " : " +
negPost.toInfix())
print(call + " Right: " + " for k = " + str(k) + " : " +
negPost.toInfix())
disjunctivePost = And(
allTrueFormula.formulaZ3,
Or(And(posPost.formulaZ3, f.varZ3),
And(negPost.formulaZ3, Not(f.varZ3))))
precisPost = PrecisFormula(disjunctivePost)
return precisPost
def chooseFeatureImplication(self, alwaysTrueFormula, intBaseFeatures, baseBoolFeatures, \
derivBoolFeatures, baseFv, derivFv, lookAhead, call ):
houdini = Houdini()
fvPos = list()
fvPosDeriv = list()
fvNeg = list()
fvNegDeriv = list()
irrelevantFeatures = ()
irrelevantIndices = []
boolFeatures = baseBoolFeatures + derivBoolFeatures
for idx in range(0, len(boolFeatures)):
#region pruneFunction
feature = boolFeatures[idx]
if is_int(feature.varZ3):
assert (False)
(fvPos, fvPosDeriv, fvNeg,
fvNegDeriv) = self.splitSamplesImplication(
feature, idx + lookAhead, baseFv, derivFv)
#if len(fvPos) == 0 or len(fvNeg) == 0:
#irrelevantIndices.append(idx)
#continue
(posIntBaseFv,
posBoolBaseFv) = Featurizer.getBoolAndIntFeatureVectors(
intBaseFeatures, baseBoolFeatures, fvPos)
(negIntBaseFv,
negBoolBaseFv) = Featurizer.getBoolAndIntFeatureVectors(
intBaseFeatures, baseBoolFeatures, fvNeg)
posFvs = Featurizer.mergeFeatureVectors(posBoolBaseFv, fvPosDeriv)
negFvs = Featurizer.mergeFeatureVectors(negBoolBaseFv, fvNegDeriv)
(posAllTrueFormula, posIndicesAllwaysTrue) = houdini.learn2(
boolFeatures, posFvs, call +
" from implication check-- split from pred " + str(feature))
(negAllTrueFormula, negIndicesAllwaysTrue) = houdini.learn2(
boolFeatures, negFvs, call +
" from implication check-- split from pred " + str(feature))
if len(fvPos) != 0 and len(fvNeg) != 0:
logger.info(call + " implication check-- split pred: " +
str(feature))
logger.info(call + " implication check-- featurePos: " +
str(posAllTrueFormula.toInfix()))
logger.info(call + " implication check-- featureNeg: " +
str(negAllTrueFormula.toInfix()) + "\n")
#disjunct z3 type
disjunct = Or(And(posAllTrueFormula.formulaZ3, feature.varZ3),
And(negAllTrueFormula.formulaZ3, Not(feature.varZ3)))
implication = Implies(alwaysTrueFormula.formulaZ3, disjunct)
solver = Solver()
# check (not (postK0 => postK1)) is unsat
solver.add(Not(implication))
check = solver.check()
#splitting on `feature does not` add new information: alwaysTrueFormula -> (OR(f and posSplit, ~f and negSplit)) is valid
if str(check) == 'unsat':
#collect irrelevant features and indices to remove
irrelevantFeatures = irrelevantFeatures + (feature, )
irrelevantIndices.append(idx)
#splitting adds new information
elif str(check) == 'sat':
pass
else:
# solver does not know
assert (False)
#endregion
copyBaseIntFeat = tuple(intBaseFeatures)
copyBaseBoolFeat = tuple(baseBoolFeatures)
copyDerivFeat = tuple(derivBoolFeatures)
#(remainingBaseBoolFeat, remainingDerivBoolFeat, featuresRemoved) = \
# self.removeFeatureFromFeaturelist(boolBaseFeat, derivFeats, indicesAllwaysTrue)
(intBaseFv, boolBaseFv) = Featurizer.getBoolAndIntFeatureVectors(
copyBaseIntFeat, copyBaseBoolFeat, baseFv)
(copyRemainingBaseBoolFeat, copyRemainingDerivBoolFeat, featuresRemoved) = \
self.removeFeatureFromFeaturelist(copyBaseBoolFeat, copyDerivFeat, irrelevantIndices)
#boolFvs = Featurizer.mergeFeatureVectors(boolBaseFv, derivFv)
(copyReaminingEntriesBaseBoolFv, reaminingEntriesDerivBoolFv) = \
self.removeFeatureEntryInFeatureVectors(boolBaseFv, derivFv, irrelevantIndices)
#Debug Check
if (len(copyRemainingBaseBoolFeat) +
len(copyRemainingDerivBoolFeat)) == 0:
return (None, -1, None, None, None, None)
skipAhead = len(intBaseFv[0])
newBaseFv = Featurizer.mergeFeatureVectors(
intBaseFv, copyReaminingEntriesBaseBoolFv)
(f, idx, posBaseFv, negBaseFv) = self.chooseFeature2(
copyRemainingBaseBoolFeat + copyRemainingDerivBoolFeat, newBaseFv,
reaminingEntriesDerivBoolFv, call, skipAhead)
#print(irrelevantIndices)
#intBaseFeatures = copyBaseIntFeat
#baseBoolFeatures = copyRemainingBaseBoolFeat
#erivBoolFeatures = copyDerivFeat
#baseFv = newBaseFv
#derivFv = reaminingEntriesDerivBoolFv
return (f, idx, posBaseFv, negBaseFv, copyRemainingBaseBoolFeat,
copyRemainingDerivBoolFeat)