def test_get_raw(self): q = SQLiteQueue(path=self.path) q.put("val1") item = q.get(raw=True) # item should get val2 self.assertEqual(True, "pqid" in item) self.assertEqual(item.get("data"), 'val1')
def test_task_done_with_restart(self): """Test that items are not deleted before task_done.""" q = self.queue_class(path=self.path, auto_commit=False) for i in range(1, 11): q.put(i) self.assertEqual(1, q.get()) self.assertEqual(2, q.get()) # size is correct before task_done self.assertEqual(8, q.qsize()) q.task_done() # make sure the size still correct self.assertEqual(8, q.qsize()) self.assertEqual(3, q.get()) # without task done del q q = SQLiteQueue(path=self.path, auto_commit=False) # After restart, the qsize and head item are the same self.assertEqual(8, q.qsize()) # After restart, the queue still works self.assertEqual(3, q.get()) self.assertEqual(7, q.qsize())
def benchmark_sqlite_write_10000(self): """Benchmark sqlite queue by writing <BENCHMARK_COUNT> items.""" self.path = tempfile.mkdtemp('b_sql_10000') q = SQLiteQueue(self.path, auto_commit=True) for i in range(BENCHMARK_COUNT): q.put('bench%d' % i)
def test_multiple_consumers(self): """Test sqlqueue can be used by multiple consumers.""" queue = SQLiteQueue(path=self.path, multithreading=True, auto_commit=self.auto_commit) def producer(): for x in range(1000): queue.put('var%d' % x) task_done_if_required(queue) def consumer(): for _ in range(100): data = queue.get(block=True) self.assertTrue('var' in data) p = Thread(target=producer) p.start() consumers = [] for _ in range(10): t = Thread(target=consumer) t.start() consumers.append(t) for t in consumers: t.join() self.assertEqual(0, queue.qsize())
def test_multi_threaded_parallel(self): """Create consumer and producer threads, check parallelism""" # self.skipTest("Not supported multi-thread.") m_queue = SQLiteQueue(path=self.path, multithreading=True, auto_commit=self.auto_commit) def producer(): for i in range(1000): m_queue.put('var%d' % i) task_done_if_required(m_queue) def consumer(): for i in range(1000): x = m_queue.get(block=True) self.assertEqual('var%d' % i, x) c = Thread(target=consumer) c.start() p = Thread(target=producer) p.start() p.join() c.join() self.assertEqual(0, m_queue.size) self.assertEqual(0, len(m_queue)) self.assertIsNone(m_queue.get(block=False))
def test_raise_empty(self): q = SQLiteQueue(self.path, auto_commit=self.auto_commit) q.put('first') task_done_if_required(q) d = q.get() self.assertEqual('first', d) self.assertRaises(Empty, q.get, block=True)
def benchmark_sqlite_write(self): """Writing <BENCHMARK_COUNT> items.""" self.path = tempfile.mkdtemp('b_sql_10000') q = SQLiteQueue(self.path, auto_commit=False) for i in range(BENCHMARK_COUNT): q.put('bench%d' % i) assert q.qsize() == BENCHMARK_COUNT
def test_open_close_single(self): """Write 1 item, close, reopen checking if same item is there""" q = self.queue_class(self.path, auto_commit=self.auto_commit) q.put(b'var1') del q q = SQLiteQueue(self.path) self.assertEqual(1, q.qsize()) self.assertEqual(b'var1', q.get())
def test_raise_empty(self): q = SQLiteQueue(self.path, auto_commit=self.auto_commit) q.put('first') d = q.get() self.assertEqual('first', d) self.assertRaises(Empty, q.get, block=False) # assert with timeout self.assertRaises(Empty, q.get, block=True, timeout=1.0) # assert with negative timeout self.assertRaises(ValueError, q.get, block=True, timeout=-1.0)
def test_empty(self): q = SQLiteQueue(self.path, auto_commit=self.auto_commit) self.assertEqual(q.empty(), True) q.put('first') self.assertEqual(q.empty(), False) q.get() self.assertEqual(q.empty(), True)
def test_json_serializer(self): q = SQLiteQueue( path=self.path, serializer=serializers_json) x = dict( a=1, b=2, c=dict( d=list(range(5)), e=[1] )) q.put(x) self.assertEquals(q.get(), x)
def benchmark_sqlite_read_write_false(self): """Writing and reading <BENCHMARK_COUNT> items(1 task_done).""" self.path = tempfile.mkdtemp('b_sql_10000') q = SQLiteQueue(self.path, auto_commit=False) for i in range(BENCHMARK_COUNT): q.put('bench%d' % i) for i in range(BENCHMARK_COUNT): q.get() q.task_done() assert q.qsize() == 0
def test_multi_threaded_multi_producer(self): """Test sqlqueue can be used by multiple producers.""" queue = SQLiteQueue(path=self.path, multithreading=True, auto_commit=self.auto_commit) def producer(seq): for i in range(10): queue.put('var%d' % (i + (seq * 10))) task_done_if_required(queue) def consumer(): for i in range(100): data = queue.get(block=True) self.assertTrue('var' in data) c = Thread(target=consumer) c.start() producers = [] for seq in range(10): t = Thread(target=producer, args=(seq, )) t.start() producers.append(t) for t in producers: t.join() c.join()
def get_followed_queue(self): while True: try: return SQLiteQueue(self.directory + '/data/followed_users') except OSError as e: if e.errno != os.errno.EEXIST: raise sleep(1)
def test_get_id(self): q = SQLiteQueue(path=self.path) q.put("val1") val2_id = q.put("val2") q.put("val3") item = q.get(id=val2_id) # item id should be 2 self.assertEqual(val2_id, 2) # item should get val2 self.assertEqual(item, 'val2')
def benchmark_sqlite_wr_10000(self): """Benchmark sqlite queue by writing and reading <BENCHMARK_COUNT> items.""" self.path = tempfile.mkdtemp('b_sql_10000') q = SQLiteQueue(self.path, auto_commit=False) for i in range(BENCHMARK_COUNT): q.put('bench%d' % i) q.task_done() for i in range(BENCHMARK_COUNT): q.get()
def __init__(self, spec=None, spider=None, spiders=None, start_jobs=None, queue_path=None, threads=25, buffer_size=DEFAULT_IMAP_BUFFER_SIZE, throttle=DEFAULT_THROTTLE, join=True, daemonic=False): # NOTE: crawling could work depth-first but: # buffer_size should be 0 (requires to fix quenouille issue #1) # Params self.start_jobs = start_jobs self.queue_path = queue_path self.threads = threads self.buffer_size = buffer_size self.throttle = throttle self.join = join self.daemonic = daemonic self.using_persistent_queue = queue_path is not None self.pool = create_pool(threads=threads) self.state = CrawlerState() self.started = False # Memory queue if not self.using_persistent_queue: queue = Queue() # Persistent queue else: queue = SQLiteQueue(queue_path, multithreading=True, auto_commit=False) # Creating spiders if spec is not None: if not isinstance(spec, dict): spec = load_definition(spec) if 'spiders' in spec: spiders = {name: DefinitionSpider(s, name=name) for name, s in spec['spiders'].items()} self.single_spider = False else: spiders = {'default': DefinitionSpider(spec)} self.single_spider = True elif spider is not None: spiders = {'default': spider} elif spiders is None: raise TypeError('minet.Crawler: expecting either `spec`, `spider` or `spiders`.') # Solving function spiders for name, s in spiders.items(): if callable(s) and not isinstance(s, Spider): spiders[name] = FunctionSpider(s, name) self.queue = queue self.spiders = spiders
def test_queue(self): q = SQLiteQueue(path=self.path) q.put("val1") q.put("val2") q.put("val3") # queue should get the three items d = q.queue() self.assertEqual(len(d), 3) self.assertEqual(d[1].get("data"), "val2")
def open(self, **kwargs) -> "PersistQueueConnection": if self.queue_engine == "sqlite": queue = SQLiteQueue(self.queue_path, auto_commit=False) elif self.queue_engine == "file": queue = Queue(self.queue_path) else: raise ValueError("bad queue engine value") self._queue = queue return self
def test_multiple_consumers(self): """Test sqlqueue can be used by multiple consumers.""" queue = SQLiteQueue(path=self.path, multithreading=True, auto_commit=self.auto_commit) def producer(): for x in range(1000): queue.put('var%d' % x) counter = [] # Set all to 0 for _ in range(1000): counter.append(0) def consumer(index): for i in range(200): data = queue.get(block=True) self.assertTrue('var' in data) counter[index * 200 + i] = data p = Thread(target=producer) p.start() consumers = [] for index in range(5): t = Thread(target=consumer, args=(index, )) t.start() consumers.append(t) p.join() for t in consumers: t.join() self.assertEqual(0, queue.qsize()) for x in range(1000): self.assertNotEqual(0, counter[x], "not 0 for counter's index %s" % x) self.assertEqual(len(set(counter)), len(counter))
def test_open_close_1000(self): """Write 1000 items, close, reopen checking if all items are there""" q = self.queue_class(self.path, auto_commit=self.auto_commit) for i in range(1000): q.put('var%d' % i) self.assertEqual(1000, q.qsize()) del q q = SQLiteQueue(self.path) self.assertEqual(1000, q.qsize()) for i in range(1000): data = q.get() self.assertEqual('var%d' % i, data) # assert adding another one still works q.put('foobar') data = q.get() q.shrink_disk_usage() self.assertEqual('foobar', data)
async def send_event( body: Any = Body(...), event_queue: persistqueue.SQLiteQueue = Depends(_get_queue_session), ): """ Endpoint for slack events API verification and event ingestion. """ if "token" not in body: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Request body missing required fields.", ) if _SLACK_VERIFICATION_TOKEN is None: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Missing verification token. Unable to accept new event.", ) if not secrets.compare_digest(body["token"], _SLACK_VERIFICATION_TOKEN): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid token.", ) if "challenge" in body: # slack event endpoint verification step. return JSONResponse( status_code=status.HTTP_200_OK, content=dict(challenge=body["challenge"], ), ) log.info("Adding payload to slack event queue.") log.debug("payload value: %s", body) event_queue.put(body) return JSONResponse(status_code=status.HTTP_200_OK, content=dict(status="Event saved."))
def test_random_read_write(self): """Test random read/write""" q = SQLiteQueue(self.path) n = 0 for i in range(1000): if random.random() < 0.5: if n > 0: q.get() n -= 1 else: self.assertEqual(None, q.get()) else: q.put('var%d' % random.getrandbits(16)) n += 1
def test_random_read_write(self): """Test random read/write""" q = SQLiteQueue(self.path, auto_commit=self.auto_commit) n = 0 for i in range(1000): if random.random() < 0.5: if n > 0: q.get() n -= 1 else: self.assertEqual(None, q.get()) else: q.put('var%d' % random.getrandbits(16)) task_done_if_required(q) n += 1
def get_fifo_event( username: str = Depends(get_verified_username), event_queue: persistqueue.SQLiteQueue = Depends(_get_queue_session), ): """ ... """ if event_queue.size == 0: return JSONResponse( status_code=status.HTTP_204_NO_CONTENT, content="No available events present.", ) log.info(f"Sending FIFO slack event to user {username}") return dict( event=event_queue.get(), events_left_in_queue=event_queue.size, )
def test_random_read_write(self): """Test random read/write""" q = SQLiteQueue(self.path, auto_commit=self.auto_commit) n = 0 for i in range(1000): if random.random() < 0.5: if n > 0: q.get() n -= 1 else: self.assertRaises(Empty, q.get, block=False) else: q.put('var%d' % random.getrandbits(16)) n += 1
def test_protocol_2(self): q = SQLiteQueue(path=self.path) self.assertEqual(q.protocol, None)
def test_put_0(self): q = SQLiteQueue(path=self.path) q.put(0) d = q.get(block=False) self.assertIsNotNone(d)
def test_protocol_1(self): shutil.rmtree(self.path, ignore_errors=True) q = SQLiteQueue(path=self.path) self.assertEqual(q.protocol, 2 if sys.version_info[0] == 2 else 4)
def test_protocol_2(self): q = SQLiteQueue(path=self.path) self.assertEqual(q.protocol, 2 if sys.version_info[0] == 2 else 4)