def test_dbm_export(self):
# Create a dbm classifier to export.
bayes = DBDictClassifier(TEMP_DBM_NAME)
# Stuff some messages in it so it's not empty.
bayes.learn(tokenize(spam1), True)
bayes.learn(tokenize(good1), False)
# Save & Close.
bayes.store()
bayes.close()
# Export.
sb_dbexpimp.runExport(TEMP_DBM_NAME, "dbm", TEMP_CSV_NAME)
# Reopen the original.
bayes = open_storage(TEMP_DBM_NAME, "dbm")
# Verify that the CSV holds all the original data (and, by using
# the CSV module to open it, that it is valid CSV data).
fp = open(TEMP_CSV_NAME, "rb")
reader = sb_dbexpimp.csv.reader(fp)
(nham, nspam) = reader.next()
self.assertEqual(int(nham), bayes.nham)
self.assertEqual(int(nspam), bayes.nspam)
for (word, hamcount, spamcount) in reader:
word = sb_dbexpimp.uunquote(word)
self.assert_(word in bayes._wordinfokeys())
wi = bayes._wordinfoget(word)
self.assertEqual(int(hamcount), wi.hamcount)
self.assertEqual(int(spamcount), wi.spamcount)
def test_merge_to_dbm(self):
# Create a dbm classifier to merge with.
bayes = DBDictClassifier(TEMP_DBM_NAME)
# Stuff some messages in it so it's not empty.
bayes.learn(tokenize(spam1), True)
bayes.learn(tokenize(good1), False)
# Save data to check against.
original_nham = bayes.nham
original_nspam = bayes.nspam
original_data = {}
for key in bayes._wordinfokeys():
original_data[key] = bayes._wordinfoget(key)
# Save & Close.
bayes.store()
bayes.close()
# Create a CSV file to import.
nham, nspam = 3,4
temp = open(TEMP_CSV_NAME, "wb")
temp.write("%d,%d\n" % (nham, nspam))
csv_data = {"this":(2,1), "is":(0,1), "a":(3,4), 'test':(1,1),
"of":(1,0), "the":(1,2), "import":(3,1)}
for word, (ham, spam) in csv_data.items():
temp.write("%s,%s,%s\n" % (word, ham, spam))
temp.close()
sb_dbexpimp.runImport(TEMP_DBM_NAME, "dbm", False, TEMP_CSV_NAME)
# Open the converted file and verify that it has all the data from
# the CSV file (and by opening it, that it is a valid dbm file),
# and the data from the original dbm database.
bayes2 = open_storage(TEMP_DBM_NAME, "dbm")
self.assertEqual(bayes2.nham, nham + original_nham)
self.assertEqual(bayes2.nspam, nspam + original_nspam)
words = original_data.keys()[:]
words.extend(csv_data.keys())
for word in words:
word = sb_dbexpimp.uquote(word)
self.assert_(word in bayes2._wordinfokeys())
h, s = csv_data.get(word, (0,0))
wi = original_data.get(word, None)
if wi:
h += wi.hamcount
s += wi.spamcount
wi2 = bayes2._wordinfoget(word)
self.assertEqual(h, wi2.hamcount)
self.assertEqual(s, wi2.spamcount)
def test_dbm_export(self):
# Create a dbm classifier to export.
bayes = DBDictClassifier(TEMP_DBM_NAME)
# Stuff some messages in it so it's not empty.
bayes.learn(tokenize(spam1), True)
bayes.learn(tokenize(good1), False)
# Save & Close.
bayes.store()
bayes.close()
# Export.
sb_dbexpimp.runExport(TEMP_DBM_NAME, "dbm", TEMP_CSV_NAME)
# Reopen the original.
bayes = open_storage(TEMP_DBM_NAME, "dbm")
# Verify that the CSV holds all the original data (and, by using
# the CSV module to open it, that it is valid CSV data).
fp = open(TEMP_CSV_NAME, "rb")
reader = sb_dbexpimp.csv.reader(fp)
(nham, nspam) = reader.next()
self.assertEqual(int(nham), bayes.nham)
self.assertEqual(int(nspam), bayes.nspam)
for (word, hamcount, spamcount) in reader:
word = sb_dbexpimp.uunquote(word)
self.assert_(word in bayes._wordinfokeys())
wi = bayes._wordinfoget(word)
self.assertEqual(int(hamcount), wi.hamcount)
self.assertEqual(int(spamcount), wi.spamcount)
def test_merge_to_dbm(self):
# Create a dbm classifier to merge with.
bayes = DBDictClassifier(TEMP_DBM_NAME)
# Stuff some messages in it so it's not empty.
bayes.learn(tokenize(spam1), True)
bayes.learn(tokenize(good1), False)
# Save data to check against.
original_nham = bayes.nham
original_nspam = bayes.nspam
original_data = {}
for key in bayes._wordinfokeys():
original_data[key] = bayes._wordinfoget(key)
# Save & Close.
bayes.store()
bayes.close()
# Create a CSV file to import.
nham, nspam = 3, 4
temp = open(TEMP_CSV_NAME, "wb")
temp.write("%d,%d\n" % (nham, nspam))
csv_data = {
"this": (2, 1),
"is": (0, 1),
"a": (3, 4),
'test': (1, 1),
"of": (1, 0),
"the": (1, 2),
"import": (3, 1)
}
for word, (ham, spam) in csv_data.items():
temp.write("%s,%s,%s\n" % (word, ham, spam))
temp.close()
sb_dbexpimp.runImport(TEMP_DBM_NAME, "dbm", False, TEMP_CSV_NAME)
# Open the converted file and verify that it has all the data from
# the CSV file (and by opening it, that it is a valid dbm file),
# and the data from the original dbm database.
bayes2 = open_storage(TEMP_DBM_NAME, "dbm")
self.assertEqual(bayes2.nham, nham + original_nham)
self.assertEqual(bayes2.nspam, nspam + original_nspam)
words = original_data.keys()[:]
words.extend(csv_data.keys())
for word in words:
word = sb_dbexpimp.uquote(word)
self.assert_(word in bayes2._wordinfokeys())
h, s = csv_data.get(word, (0, 0))
wi = original_data.get(word, None)
if wi:
h += wi.hamcount
s += wi.spamcount
wi2 = bayes2._wordinfoget(word)
self.assertEqual(h, wi2.hamcount)
self.assertEqual(s, wi2.spamcount)
def test_dbm_export(self):
bayes = DBDictClassifier(TEMP_DBM_NAME)
bayes.learn(tokenize(spam1), True)
bayes.learn(tokenize(good1), False)
bayes.store()
bayes.close()
sb_dbexpimp.runExport(TEMP_DBM_NAME, "dbm", TEMP_CSV_NAME)
bayes = open_storage(TEMP_DBM_NAME, "dbm")
fp = open(TEMP_CSV_NAME, "rb")
reader = sb_dbexpimp.csv.reader(fp)
(nham, nspam) = reader.next()
self.assertEqual(int(nham), bayes.nham)
self.assertEqual(int(nspam), bayes.nspam)
for (word, hamcount, spamcount) in reader:
word = sb_dbexpimp.uunquote(word)
self.assert_(word in bayes._wordinfokeys())
wi = bayes._wordinfoget(word)
self.assertEqual(int(hamcount), wi.hamcount)
self.assertEqual(int(spamcount), wi.spamcount)
def test_merge_to_dbm(self):
bayes = DBDictClassifier(TEMP_DBM_NAME)
bayes.learn(tokenize(spam1), True)
bayes.learn(tokenize(good1), False)
original_nham = bayes.nham
original_nspam = bayes.nspam
original_data = {}
for key in bayes._wordinfokeys():
original_data[key] = bayes._wordinfoget(key)
bayes.store()
bayes.close()
nham, nspam = 3,4
temp = open(TEMP_CSV_NAME, "wb")
temp.write("%d,%d\n" % (nham, nspam))
csv_data = {"this":(2,1), "is":(0,1), "a":(3,4), 'test':(1,1),
"of":(1,0), "the":(1,2), "import":(3,1)}
for word, (ham, spam) in csv_data.items():
temp.write("%s,%s,%s\n" % (word, ham, spam))
temp.close()
sb_dbexpimp.runImport(TEMP_DBM_NAME, "dbm", False, TEMP_CSV_NAME)
bayes2 = open_storage(TEMP_DBM_NAME, "dbm")
self.assertEqual(bayes2.nham, nham + original_nham)
self.assertEqual(bayes2.nspam, nspam + original_nspam)
words = original_data.keys()[:]
words.extend(csv_data.keys())
for word in words:
word = sb_dbexpimp.uquote(word)
self.assert_(word in bayes2._wordinfokeys())
h, s = csv_data.get(word, (0,0))
wi = original_data.get(word, None)
if wi:
h += wi.hamcount
s += wi.spamcount
wi2 = bayes2._wordinfoget(word)
self.assertEqual(h, wi2.hamcount)
self.assertEqual(s, wi2.spamcount)