class TestStatsWithMondoDB(unittest.TestCase):
def setUp(self):
self.db = MongoDB(col = 'stats_test')
self.stats = Statistics(self.db)
mock = MockDatetime()
self.expected_date = mock.now()
self.stats._datetime = mock
def test_save_stat(self):
json_doc = {'url':"url", 'datetime': "2014-01-01 14:10:00" }
self.stats.register_access_now("url")
result = self.db.find_one_by('url', "url")
self.assertEqual(result['url'], "url")
self.assertEqual(result['datetime'], str(self.expected_date))
self.db.drop('stats_test')
def test_all_visits(self):
self.stats.register_access_now("url")
visits = self.stats.all_visits()
self.assertEqual(len(visits), 1)
class TestCollection(unittest.TestCase):
def setUp(self):
self.db = MongoDB()
self.col = self.db.collection("stats_borrame")
def tearDown(self):
self.db.drop("stats_borrame")
def test_find_one_in_column(self):
stat = {u'stats': [{u'date': u'2014-3-22', u'blogs': u'0', u'threads': u'0', u'msgs': u'0'}], u'name': u'HootBoardGame'}
self.col.save(stat)
self.assertIsNotNone(self.col.find_one('name', 'HootBoardGame'))
def test_when_no_fin_returns_null(self):
self.assertIsNone(self.col.find_one('name', 'NoExiste'))
def test_update_stat(self):
stat = {u'stats': [{u'date': u'2014-3-22', u'blogs': u'0', u'threads': u'0', u'msgs': u'0'}], u'name': u'HootBoardGame'}
self.col.save(stat)
stat['stats'].append({u'date': u'2014-3-22', u'blogs': u'0', u'threads': u'0', u'msgs': u'0'})
self.col.remove('name', 'HootBoardGame')
self.col.save(stat)
new_stats = self.col.find_one('name', 'HootBoardGame')
self.assertEqual(len(new_stats['stats']), 2)
def test_find_one_when_col_is_empty(self):
result = self.col.find_one('no', 'exist')
self.assertIsNone(result)
class TestPorcessWithMongo(unittest.TestCase):
""" this test suites verify tje coordination between two collections and the
detection of an unmodified thead.
"""
def setUp(self):
self.obj_thread = ThreadModel(Reports.get_asylum_thread())
self.obj_thread.set_answers(59)
self.mock_listener = mock()
self.db = MongoDB()
self.db.query("labsk_merge")
def test_search_thread(self):
db = MongoDB()
db.query("labsk_merge")
tmp_col = "labsk_temp_temp"
db.insert(tmp_col)
process = ProcessThread()
process.database = db
result = process._search_thread(self.obj_thread)
self.assertIsNotNone(result)
db.drop(tmp_col)
def _create_process(self):
process = ProcessThread()
process.database = self.db
process.listener = self.mock_listener
return process
def test_skip_existing_thread_with_one_msg_and_0_answers(self):
link = 'http://labsk.net/index.php?topic=127734.0'
newT = ThreadModel({'link':link, 'msgs': [{}], 'answers':'0'})
process = self._create_process()
self.assertIsNotNone(process._search_thread(newT))
process._evaluate_thread(newT)
verify(self.mock_listener).skippingUnmodifiedThread(any(), newT)
def test_search_thread(self):
db = MongoDB()
db.query("labsk_merge")
tmp_col = "labsk_temp_temp"
db.insert(tmp_col)
process = ProcessThread()
process.database = db
result = process._search_thread(self.obj_thread)
self.assertIsNotNone(result)
db.drop(tmp_col)
def setUp(self):
self.db = MongoDB(col = 'stats_test')
self.stats = Statistics(self.db)
mock = MockDatetime()
self.expected_date = mock.now()
self.stats._datetime = mock
def setUp(self):
self.obj_thread = ThreadModel(Reports.get_asylum_thread())
self.obj_thread.set_answers(59)
self.mock_listener = mock()
self.db = MongoDB()
self.db.query("labsk_merge")
# Override
def enteringThread(self, obj_thread):
print "Thread ", obj_thread.title, ", ", obj_thread.date, " | ", obj_thread.link
self.thread += 1
# Override
def skippingUnmodifiedThread(self, old, new):
""" Old thread seems to be the same one than the new thread """
self.thread -= 1
print "Skipping ", new.title, ", ", new.date, " | ", new.link
def __str__(self):
return ". Threads: " + str(self.thread) \
db = MongoDB()
db.query("labsk_merge")
db.insert("labsk_" + str(datetime.now()))
starttime = datetime.now()
listener = StdListener()
threads = ProcessThreads(db, MsgPageFactory())
threads.setListener(listener)
threads.setPageLimit(1)
threads.setMsgPageLimit(250) # Nunca bajes este valor o perderas mensajes, al menos mantenlo igual
threads.scrapListOfURL(labsk_urls)
delta = datetime.now() - starttime
print "----------------------------------------------"
def newURL(self, url):
""" url is a web.URL object """
print "----------------------------------------------"
print "--- ", url.desc, " ", url.url
self.urls += 1
def skippingUnmodifiedThread(self, old, new):
""" Old thread seems to be the same one than the new thread """
self.skiped += 1
print old.answers(), "==", new.answers(), ". Skipping ", old.title(), " / ", old.link()
def __str__(self):
return "Urls: " + str(self.urls) + ". Threads: " + str(self.thread) + ". Messages readed: " + str(listener.msgs) + ". Threads skipped: " + str(self.skiped)
db = MongoDB()
db.query("labsk_merge")
db.insert("labsk_" + str(datetime.now()))
starttime = datetime.now()
listener = StdListener()
threads = ProcessThreads(db, MsgPageFactory())
threads.setListener(listener)
threads.setPageLimit(1)
threads.setMsgPageLimit(80) # Nunca bajes este valor o perderas mensajes, al menos mantenlo igual
threads.scrapListOfURL(labsk_urls)
delta = datetime.now() - starttime
def setUp(self):
self.db = MongoDB()
self.col = self.db.collection("stats_borrame")
def setUp(self):
self.db = MongoDB()
self.testcol = self.db.collection("labsk_test")
def test_database_contains(self):
self.db = MongoDB(col = "labsk_merge")
result = self.db.find_one_by('link', "http://labsk.net/index.php?topic=129533.0")
self.assertEqual("http://labsk.net/index.php?topic=129533.0", result['link'])
def test_when_query_and_insert_columns_are_different_insertion_and_query_uses_diferent_cols(self):
db = MongoDB()
db.query("labsk_test")
db.insert("to_delete")
db.saveThread({'id':'1'})
self.assertIsNone(db.find_one_by('id', '1'))
db.query("to_delete")
self.assertIsNotNone(db.find_one_by('id', '1'))
db.drop("to_delete")
class TestMongoDB(unittest.TestCase):
def setUp(self):
self.db = MongoDB()
self.testcol = self.db.collection("labsk_test")
def test_merge_with_empty_collection_all_docs_merged(self):
db = self.db
db.query("empty_new_col")
db.insert("labsk_test")
merge_result = db.merge_insert_wih_query('link')
self.assertEqual(60, db.len("empty_new_col"))
self.assertEqual(60, db.len("labsk_test"))
db.drop("empty_new_col")
def test_merge_returns_new_and_update_threads(self):
db = self.db
db.query("empty_new_col")
db.insert("labsk_test")
merge_result = db.merge_insert_wih_query('link')
self.assertEqual(60, merge_result.new_threads)
self.assertEqual(0, merge_result.updated_threads)
db.drop("empty_new_col")
def test_merge_drops_insert_column_and_avoid_its_use(self):
db = self.db
db.copy("labsk_test", "query_delete")
db.copy("labsk_test", "insert_delete")
db.query("query_delete")
db.insert("insert_delete")
merge_result = db.merge('link')
self.assertIsNone(db.insert_col)
self.assertEqual(0, db.len("insert_delete"))
self.assertEqual(60, db.len("query_delete"))
db.drop("query_delete")
def test_find_all_by(self):
results = self.db.find_one_by('link', 'http://labsk.net/index.php?topic=119300.0')
self.assertEqual(results['link'], 'http://labsk.net/index.php?topic=119300.0')
self.assertEqual(results['title'], u'Fief - Feudo Ediciones MasQueOca editar\xe1 Fief en Espa\xf1ol')
def test_find_all_by_not_found(self):
results = self.db.find_one_by('link', 'no no')
self.assertIsNone(results)
def test_by_default_query_and_insert_columns_are_the_column_in_constructor(self):
db = MongoDB(col = "labsk_test")
self.assertIs(db.col, db.query_col)
self.assertIs(db.col, db.insert_col)
def test_when_query_and_insert_columns_are_different_insertion_and_query_uses_diferent_cols(self):
db = MongoDB()
db.query("labsk_test")
db.insert("to_delete")
db.saveThread({'id':'1'})
self.assertIsNone(db.find_one_by('id', '1'))
db.query("to_delete")
self.assertIsNotNone(db.find_one_by('id', '1'))
db.drop("to_delete")
def test_database_contains(self):
self.db = MongoDB(col = "labsk_merge")
result = self.db.find_one_by('link', "http://labsk.net/index.php?topic=129533.0")
self.assertEqual("http://labsk.net/index.php?topic=129533.0", result['link'])
