def test_queries(self):
self.assertEqual(self.db.queries_sent(), 0)
self.assertEqual(self.db.queries_completed(), 0)
self.db.store_msg(
MonitorMsg(MonitorMsg.QUERY_SENT, -1, MonitorMsg.OK, 1, None))
self.db.store_msg(
MonitorMsg(MonitorMsg.QUERY_COMPLETED, -1, MonitorMsg.OK, 1, None))
self.assertEqual(self.db.queries_sent(), 1)
self.assertEqual(self.db.queries_completed(), 1)
def test_day_requests(self): self.assertEqual(self.db.yandex_search_requests(True), 0) msg = MonitorMsg(MonitorMsg.HTTP_SEARCH, Consts.Providers.YANDEX, MonitorMsg.OK, 1, None) self.db.store_msg(msg) one_day = datetime.datetime.utcnow() - datetime.timedelta(hours = 23, minutes = 59, seconds = 59) two_days = datetime.datetime.utcnow() - datetime.timedelta(days = 2) msg.timestamp = one_day self.db.store_msg(msg) msg.timestamp = two_days self.db.store_msg(msg) self.assertEqual(self.db.yandex_search_requests(True ,MonDB.SCOPE_DAY), 2)
def test_day_requests(self):
self.assertEqual(self.db.yandex_search_requests(True), 0)
msg = MonitorMsg(MonitorMsg.HTTP_SEARCH, Consts.Providers.YANDEX,
MonitorMsg.OK, 1, None)
self.db.store_msg(msg)
one_day = datetime.datetime.utcnow() - datetime.timedelta(
hours=23, minutes=59, seconds=59)
two_days = datetime.datetime.utcnow() - datetime.timedelta(days=2)
msg.timestamp = one_day
self.db.store_msg(msg)
msg.timestamp = two_days
self.db.store_msg(msg)
self.assertEqual(self.db.yandex_search_requests(True, MonDB.SCOPE_DAY),
2)
def test_posts(self):
self.assertEqual(self.db.posts_collected(), 0)
self.assertEqual(self.db.posts_dublicate_detected(), 0)
self.assertEqual(self.db.posts_update_detected(), 0)
self.assertEqual(self.db.posts_new_link_detected(), 0)
self.assertEqual(self.db.posts_persisted(), 0)
self.db.store_msg(
MonitorMsg(MonitorMsg.POST_COLLECTED, -1, MonitorMsg.OK, 1, None))
self.db.store_msg(
MonitorMsg(MonitorMsg.POST_DUBLICATE_DETECTED, -1, MonitorMsg.OK,
1, None))
self.db.store_msg(
MonitorMsg(MonitorMsg.POST_UPDATE_DETECTED, -1, MonitorMsg.OK, 1,
None))
self.db.store_msg(
MonitorMsg(MonitorMsg.POST_NEW_LINK_DETECTED, -1, MonitorMsg.OK, 1,
None))
self.db.store_msg(
MonitorMsg(MonitorMsg.POST_SPAM_DETECTED, -1, MonitorMsg.OK, 1,
None))
self.db.store_msg(
MonitorMsg(MonitorMsg.POST_PERSISTED, -1, MonitorMsg.OK, 1, None))
self.assertEqual(self.db.posts_collected(), 1)
self.assertEqual(self.db.posts_dublicate_detected(), 1)
self.assertEqual(self.db.posts_update_detected(), 1)
self.assertEqual(self.db.posts_new_link_detected(), 1)
self.assertEqual(self.db.posts_persisted(), 1)
def test_http_requests(self):
self.assertEqual(self.db.yandex_search_requests(True), 0)
self.assertEqual(self.db.yandex_content_requests(True), 0)
self.assertEqual(self.db.lj_content_requests(True), 0)
self.assertEqual(self.db.vk_content_requests(True), 0)
self.db.store_msg(
MonitorMsg(MonitorMsg.HTTP_SEARCH, Consts.Providers.YANDEX,
MonitorMsg.OK, 1, None))
self.db.store_msg(
MonitorMsg(MonitorMsg.HTTP_CONTENT, Consts.Providers.YANDEX,
MonitorMsg.OK, None, 'http://test'))
self.db.store_msg(
MonitorMsg(MonitorMsg.HTTP_CONTENT, Consts.Providers.VK,
MonitorMsg.OK, None, 'http://test'))
self.db.store_msg(
MonitorMsg(MonitorMsg.HTTP_CONTENT, Consts.Providers.LJ,
MonitorMsg.OK, None, 'http://test'))
self.assertEqual(self.db.yandex_search_requests(True), 1)
self.assertEqual(self.db.yandex_content_requests(True), 1)
self.assertEqual(self.db.lj_content_requests(True), 1)
self.assertEqual(self.db.vk_content_requests(True), 1)
def test_http_requests_err(self):
self.assertEqual(self.db.yandex_search_requests(False), 0)
self.assertEqual(self.db.yandex_content_requests(False), 0)
self.assertEqual(self.db.lj_content_requests(False), 0)
self.assertEqual(self.db.vk_content_requests(False), 0)
self.db.store_msg(
MonitorMsg(MonitorMsg.HTTP_SEARCH, Consts.Providers.YANDEX,
MonitorMsg.ERROR, 1, 'error'))
self.db.store_msg(
MonitorMsg(MonitorMsg.HTTP_CONTENT, Consts.Providers.YANDEX,
MonitorMsg.ERROR, None, 'error'))
self.db.store_msg(
MonitorMsg(MonitorMsg.HTTP_CONTENT, Consts.Providers.VK,
MonitorMsg.ERROR, None, 'error'))
self.db.store_msg(
MonitorMsg(MonitorMsg.HTTP_CONTENT, Consts.Providers.LJ,
MonitorMsg.ERROR, None, 'error'))
self.assertEqual(self.db.yandex_search_requests(False), 1)
self.assertEqual(self.db.yandex_content_requests(False), 1)
self.assertEqual(self.db.lj_content_requests(False), 1)
self.assertEqual(self.db.vk_content_requests(False), 1)
def post_persisted(self, post_id, link):
self.queue.put(
MonitorMsg(MonitorMsg.POST_PERSISTED, -1, MonitorMsg.OK, post_id,
link))
def query_error(self, query_id):
self.queue.put(
MonitorMsg(MonitorMsg.QUERY_COMPLETED, -1, MonitorMsg.ERROR,
query_id, None))
def post_new_link_detected(self, link):
self.queue.put(
MonitorMsg(MonitorMsg.POST_NEW_LINK_DETECTED, -1, MonitorMsg.OK,
None, link))
def post_spam_detected(self, link):
self.queue.put(
MonitorMsg(MonitorMsg.POST_SPAM_DETECTED, -1, MonitorMsg.OK, None,
link))
def post_dublicate_detected(self, link):
self.queue.put(
MonitorMsg(MonitorMsg.POST_DUBLICATE_DETECTED, -1, MonitorMsg.OK,
None, link))
def search_http_error(self, provider, query_id, code, url):
self.queue.put(
MonitorMsg(MonitorMsg.HTTP_SEARCH, provider, MonitorMsg.ERROR,
query_id, "%s, %s" %
(code, url))) # now only Yandex search is supported
def content_exception(self, provider, link, e):
self.queue.put(
MonitorMsg(MonitorMsg.HTTP_CONTENT, provider, MonitorMsg.ERROR,
None, str(e)))
def query_sent(self, query_id):
self.queue.put(
MonitorMsg(MonitorMsg.QUERY_SENT, -1, MonitorMsg.OK, query_id,
None))
def content_http_error(self, provider, code, url):
self.queue.put(
MonitorMsg(MonitorMsg.HTTP_CONTENT, provider, MonitorMsg.ERROR,
None, "%s, %s" % (code, url)))
def content_http_request(self, provider, link):
self.queue.put(
MonitorMsg(MonitorMsg.HTTP_CONTENT, provider, MonitorMsg.OK, None,
link))
def search_exception(self, provider, query_id, e):
self.queue.put(
MonitorMsg(MonitorMsg.HTTP_SEARCH, provider, MonitorMsg.ERROR,
query_id, str(e)))
def query_completed(self, query_id):
self.queue.put(
MonitorMsg(MonitorMsg.QUERY_COMPLETED, -1, MonitorMsg.OK, query_id,
None))
def post_update_detected(self, link):
self.queue.put(
MonitorMsg(MonitorMsg.POST_UPDATE_DETECTED, -1, MonitorMsg.OK,
None, link))
def post_collected(self, link):
self.queue.put(
MonitorMsg(MonitorMsg.POST_COLLECTED, -1, MonitorMsg.OK, None,
link))
def search_http_request(self, provider, query_id):
self.queue.put(
MonitorMsg(MonitorMsg.HTTP_SEARCH, provider, MonitorMsg.OK,
query_id, None)) # now only Yandex search is supported