def testUnicode(self):
testDocUni1 = [
"\u0395\u0396\u0397\u0398\u0399",
"\u0400\u0401\u0402\u0403\u0404",
"\u0405\u0406\u0407\u0408\u0409"]
testDocUni2 = [
"\u0395\u0396\u0397\u0398\u0399\u0410",
"\u0400\u0401\u0402\u0403\u0404\u0410",
"\u0405\u0406\u0407\u0408\u0409\u0410"]
params = SimHashDocumentEncoderParameters()
params.size = 400
params.sparsity = 0.33
# unicode 'tokenSimilarity' ON
params.tokenSimilarity = True
encoder1 = SimHashDocumentEncoder(params)
output1 = SDR(params.size)
output2 = SDR(params.size)
encoder1.encode(testDocUni1, output1)
encoder1.encode(testDocUni2, output2)
assert(output1.getOverlap(output2) > 65)
# unicode 'tokenSimilarity' OFF
params.tokenSimilarity = False
encoder2 = SimHashDocumentEncoder(params)
output1.zero()
output2.zero()
encoder2.encode(testDocUni1, output1)
encoder2.encode(testDocUni2, output2)
assert(output1.getOverlap(output2) < 65)
def testTokenSimilarity(self):
params = SimHashDocumentEncoderParameters()
params.size = 400
params.sparsity = 0.33
params.caseSensitivity = True
# tokenSimilarity ON
params.tokenSimilarity = True
encoder1 = SimHashDocumentEncoder(params)
output1 = SDR(params.size)
output2 = SDR(params.size)
output3 = SDR(params.size)
output4 = SDR(params.size)
encoder1.encode(testDoc1, output1)
encoder1.encode(testDoc2, output2)
encoder1.encode(testDoc3, output3)
encoder1.encode(testDoc4, output4)
assert(output3.getOverlap(output4) > output2.getOverlap(output3))
assert(output2.getOverlap(output3) > output1.getOverlap(output3))
assert(output1.getOverlap(output3) > output1.getOverlap(output4))
# tokenSimilarity OFF
params.tokenSimilarity = False
encoder2 = SimHashDocumentEncoder(params)
output1.zero()
output2.zero()
output3.zero()
output4.zero()
encoder2.encode(testDoc1, output1)
encoder2.encode(testDoc2, output2)
encoder2.encode(testDoc3, output3)
encoder2.encode(testDoc4, output4)
assert(output1.getOverlap(output2) > output2.getOverlap(output3))
assert(output2.getOverlap(output3) > output3.getOverlap(output4))
assert(output3.getOverlap(output4) > output1.getOverlap(output3))
def testTokenWeightMap(self):
weights = {
"aaa": 4, "bbb": 2, "ccc": 2, "ddd": 4, "eee": 2, "fff": 2, "sss": 1}
doc1 = ["aaa", "bbb", "ccc", "ddd", "sss"]
doc2 = ["eee", "bbb", "ccc", "fff", "sss"]
doc3 = ["aaa", "eee", "fff", "ddd"]
params = SimHashDocumentEncoderParameters()
params.size = 400
params.sparsity = 0.33
params.tokenSimilarity = False
params.encodeOrphans = False
params.vocabulary = weights
encoder = SimHashDocumentEncoder(params)
output1 = encoder.encode(doc1)
output2 = encoder.encode(doc2)
output3 = encoder.encode(doc3)
assert(output1.getOverlap(output3) > output1.getOverlap(output2))
assert(output1.getOverlap(output2) > output2.getOverlap(output3))
def testFrequency(self):
tokens = "a a a b b c d d d d e e f" # min 1 max 4
charTokens = "abbbbbbcccdefg aaaaaabccchijk aaabcccccclmno"
# Test token frequency floor/ceiling
params = SimHashDocumentEncoderParameters()
params.size = 400
params.sparsity = 0.33
encoder1 = SimHashDocumentEncoder(params)
output1 = encoder1.encode(tokens)
params.frequencyFloor = 1
encoder2 = SimHashDocumentEncoder(params)
output2 = encoder2.encode(tokens)
params.frequencyFloor = 0
params.frequencyCeiling = 4
encoder3 = SimHashDocumentEncoder(params)
output3 = encoder3.encode(tokens)
assert(output1 != output2)
assert(output1 != output3)
assert(output2 != output3)
# Test character frequency ceiling (only)
params4 = SimHashDocumentEncoderParameters()
params4.size = 400
params4.sparsity = 0.33
params4.tokenSimilarity = True
encoder4 = SimHashDocumentEncoder(params4)
output4 = encoder4.encode(charTokens)
params4.frequencyCeiling = 3
encoder5 = SimHashDocumentEncoder(params4)
output5 = encoder5.encode(charTokens)
assert(output4 != output5)
def testStatistics(self):
# 100 random simple English words run mass encoding stats against
testCorpus = [
"find", "any", "new", "work", "part", "take", "get", "place",
"made", "live", "where", "after", "back", "little", "only",
"round", "man", "year", "came", "show", "every", "good", "me",
"give", "our", "under", "name", "very", "through", "just", "form",
"sentence", "great", "think", "say", "help", "low", "line",
"differ", "turn", "cause", "much", "mean", "before", "move",
"right", "boy", "old", "too", "same", "tell", "does", "set",
"three", "want", "air", "well", "also", "play", "small", "end",
"put", "home", "read", "hand", "port", "large", "spell", "add",
"even", "land", "here", "must", "big", "high", "such", "follow",
"act", "why", "ask", "men", "change", "went", "light", "kind",
"off", "need", "house", "picture", "try", "us", "again", "animal",
"point", "mother", "world", "near", "build", "self", "earth"]
num_samples = 1000 # number of documents to run
num_tokens = 10 # tokens per document
# Case 1 = tokenSimilarity OFF
params1 = SimHashDocumentEncoderParameters()
params1.size = 400
params1.sparsity = 0.33
params1.tokenSimilarity = False
encoder1 = SimHashDocumentEncoder(params1)
# Case 2 = tokenSimilarity ON
params2 = params1
params2.tokenSimilarity = True
encoder2 = SimHashDocumentEncoder(params2)
sdrs1 = []
sdrs2 = []
for _ in range(num_samples):
document = []
for _ in range(num_tokens - 1):
token = testCorpus[random.randint(0, len(testCorpus) - 1)]
document.append(token)
sdrs1.append(encoder1.encode(document))
sdrs2.append(encoder2.encode(document))
report1 = Metrics([encoder1.size], len(sdrs1) + 1)
report2 = Metrics([encoder2.size], len(sdrs2) + 1)
for sdr in sdrs1:
report1.addData(sdr)
for sdr in sdrs2:
report2.addData(sdr)
# Assertions for Case 1 = tokenSimilarity OFF
assert(report1.activationFrequency.entropy() > 0.87)
assert(report1.activationFrequency.min() > 0.01)
assert(report1.activationFrequency.max() < 0.99)
assert(report1.activationFrequency.mean() > params1.sparsity - 0.005)
assert(report1.activationFrequency.mean() < params1.sparsity + 0.005)
assert(report1.overlap.min() > 0.21)
assert(report1.overlap.max() > 0.53)
assert(report1.overlap.mean() > 0.38)
assert(report1.sparsity.min() > params1.sparsity - 0.01)
assert(report1.sparsity.max() < params1.sparsity + 0.01)
assert(report1.sparsity.mean() > params1.sparsity - 0.005)
assert(report1.sparsity.mean() < params1.sparsity + 0.005)
# Assertions for Case 2 = tokenSimilarity ON
assert(report2.activationFrequency.entropy() > 0.59)
assert(report2.activationFrequency.min() >= 0)
assert(report2.activationFrequency.max() <= 1)
assert(report2.activationFrequency.mean() > params2.sparsity - 0.005)
assert(report2.activationFrequency.mean() < params2.sparsity + 0.005)
assert(report2.overlap.min() > 0.38)
assert(report2.overlap.max() > 0.78)
assert(report2.overlap.mean() > 0.61)
assert(report2.sparsity.min() > params2.sparsity - 0.01)
assert(report2.sparsity.max() < params2.sparsity + 0.01)
assert(report2.sparsity.mean() > params2.sparsity - 0.005)
assert(report2.sparsity.mean() < params2.sparsity + 0.005)