def test_updating_lists_works(self):
uris = [
("http://localhost", "etag", int(time.time() * 1000),
int(time.time() * 1000), 1),
]
q = SQLiteSingleHostUriQueue(":memory:")
q.add_uris(uris)
uris = [
("http://localhost", "etag", int(time.time() * 1000),
int(time.time() * 1000), 2),
]
q.update_uris(uris)
cursor = q._connection.execute("SELECT * FROM queue")
uri_res = cursor.fetchone()
(url, etag, mod_date, next_date, prio) = uris[0]
(url_res, etag_res, mod_date_res, next_date_res, prio_res) = uri_res
self.assertEqual(url, url_res)
self.assertEqual(etag, etag_res)
self.assertEqual(mod_date, mod_date_res)
self.assertEqual(prio, prio_res)
self.assertEqual(next_date, next_date_res)
def test_removing_lists_works(self):
uris = [("http://localhost", "etag", int(time.time()*1000),
int(time.time() * 1000), 1),
("http://fogeignhost", "ETAG", int(time.time()*1000),
int(time.time() * 1000), 2),
]
q = SQLiteSingleHostUriQueue(":memory:")
q.add_uris(uris)
q.remove_uris(uris)
cursor = q._connection.execute("SELECT * FROM queue")
self.assertTrue(None is cursor.fetchone())
def test_queue_head_works(self):
uris = [
("http://localhost", "etag", int(time.time() * 1000),
int(time.time() * 1000), 1),
("http://fogeignhost", "ETAG", int(time.time() * 1000),
int(time.time() * 1001), 2),
]
q = SQLiteSingleHostUriQueue(":memory:")
q.add_uris(uris)
(url1, etag1, mod_date1, next_date1, prio1) = uris[0]
(url2, etag2, mod_date2, next_date2, prio2) = uris[1]
for uri_res in q.queue_head(n=1, offset=0):
(url_res, etag_res, mod_date_res, next_date_res,
prio_res) = uri_res
self.assertEqual(url1, url_res)
self.assertEqual(etag1, etag_res)
self.assertEqual(mod_date1, mod_date_res)
self.assertEqual(prio1, prio_res)
self.assertEqual(next_date1, next_date_res)
for uri_res in q.queue_head(n=1, offset=1):
(url_res, etag_res, mod_date_res, next_date_res,
prio_res) = uri_res
self.assertEqual(url2, url_res)
self.assertEqual(etag2, etag_res)
self.assertEqual(mod_date2, mod_date_res)
self.assertEqual(prio2, prio_res)
self.assertEqual(next_date2, next_date_res)
uris.append(("http://localhost/1", "eTag", int(time.time() * 1000),
int(time.time() * 1002), 1))
(url3, etag3, mod_date3, next_date3, prio3) = uris[2]
q.add_uri(uris[2])
q.ignore_uri("http://localhost", 404)
for uri_res in q.queue_head(n=1, offset=1):
(url_res, etag_res, mod_date_res, next_date_res,
prio_res) = uri_res
self.assertEqual(url3, url_res)
self.assertEqual(etag3, etag_res)
self.assertEqual(mod_date3, mod_date_res)
self.assertEqual(prio3, prio_res)
self.assertEqual(next_date3, next_date_res)
def test_removing_lists_works(self):
uris = [
("http://localhost", "etag", int(time.time() * 1000),
int(time.time() * 1000), 1),
("http://fogeignhost", "ETAG", int(time.time() * 1000),
int(time.time() * 1000), 2),
]
q = SQLiteSingleHostUriQueue(":memory:")
q.add_uris(uris)
q.remove_uris(uris)
cursor = q._connection.execute("SELECT * FROM queue")
self.assertTrue(None is cursor.fetchone())
def test_adding_lists_works(self):
uris = [("http://localhost", "etag", int(time.time()*1000),
int(time.time() * 1010), 1),
]
q = SQLiteSingleHostUriQueue(":memory:")
q.add_uris(uris)
cursor = q._connection.execute("SELECT * FROM queue")
uri_res = cursor.fetchone()
(url, etag, mod_date, next_date, prio) = uris[0]
(url_res, etag_res, mod_date_res, next_date_res, prio_res) = uri_res
self.assertEqual(url, url_res)
self.assertEqual(etag, etag_res)
self.assertEqual(mod_date, mod_date_res)
self.assertEqual(prio, prio_res)
self.assertEqual(next_date, next_date_res)
def test_queue_head_works(self):
uris = [("http://localhost", "etag", int(time.time()*1000),
int(time.time() * 1000), 1),
("http://fogeignhost", "ETAG", int(time.time()*1000),
int(time.time() * 1001), 2),
]
q = SQLiteSingleHostUriQueue(":memory:")
q.add_uris(uris)
(url1, etag1, mod_date1, next_date1, prio1) = uris[0]
(url2, etag2, mod_date2, next_date2, prio2) = uris[1]
for uri_res in q.queue_head(n=1, offset=0):
(url_res, etag_res, mod_date_res, next_date_res, prio_res) = uri_res
self.assertEqual(url1, url_res)
self.assertEqual(etag1, etag_res)
self.assertEqual(mod_date1, mod_date_res)
self.assertEqual(prio1, prio_res)
self.assertEqual(next_date1, next_date_res)
for uri_res in q.queue_head(n=1, offset=1):
(url_res, etag_res, mod_date_res, next_date_res, prio_res) = uri_res
self.assertEqual(url2, url_res)
self.assertEqual(etag2, etag_res)
self.assertEqual(mod_date2, mod_date_res)
self.assertEqual(prio2, prio_res)
self.assertEqual(next_date2, next_date_res)
uris.append(("http://localhost/1", "eTag", int(time.time()*1000),
int(time.time()*1002), 1))
(url3, etag3, mod_date3, next_date3, prio3) = uris[2]
q.add_uri(uris[2])
q.ignore_uri("http://localhost", 404)
for uri_res in q.queue_head(n=1, offset=1):
(url_res, etag_res, mod_date_res, next_date_res, prio_res) = uri_res
self.assertEqual(url3, url_res)
self.assertEqual(etag3, etag_res)
self.assertEqual(mod_date3, mod_date_res)
self.assertEqual(prio3, prio_res)
self.assertEqual(next_date3, next_date_res)
def test_iterating_over_all_uris_works(self):
uris = [("http://localhost", "etag", int(time.time()*1000),
int(time.time() * 1000), 1),
("http://foreignhost", "ETAG", int(time.time()*1000),
int(time.time() * 1000), 2),
]
urls = ["http://localhost", "http://foreignhost"]
q = SQLiteSingleHostUriQueue(":memory:")
q.add_uris(uris)
uri = q.get_uri("http://foreignhost")
self.assertEqual(uris[1], uri)
self.assertRaises(UriNotFound, q.get_uri, "http://gibtsnuesch")
for url in q.all_uris():
self.assertTrue(url in urls)
def test_iterating_over_all_uris_works(self):
uris = [
("http://localhost", "etag", int(time.time() * 1000),
int(time.time() * 1000), 1),
("http://foreignhost", "ETAG", int(time.time() * 1000),
int(time.time() * 1000), 2),
]
urls = ["http://localhost", "http://foreignhost"]
q = SQLiteSingleHostUriQueue(":memory:")
q.add_uris(uris)
uri = q.get_uri("http://foreignhost")
self.assertEqual(uris[1], uri)
self.assertRaises(UriNotFound, q.get_uri, "http://gibtsnuesch")
for url in q.all_uris():
self.assertTrue(url in urls)
