def test_get_raw(self):
q = SQLiteQueue(path=self.path)
q.put("val1")
item = q.get(raw=True)
# item should get val2
self.assertEqual(True, "pqid" in item)
self.assertEqual(item.get("data"), 'val1')
def test_task_done_with_restart(self):
"""Test that items are not deleted before task_done."""
q = self.queue_class(path=self.path, auto_commit=False)
for i in range(1, 11):
q.put(i)
self.assertEqual(1, q.get())
self.assertEqual(2, q.get())
# size is correct before task_done
self.assertEqual(8, q.qsize())
q.task_done()
# make sure the size still correct
self.assertEqual(8, q.qsize())
self.assertEqual(3, q.get())
# without task done
del q
q = SQLiteQueue(path=self.path, auto_commit=False)
# After restart, the qsize and head item are the same
self.assertEqual(8, q.qsize())
# After restart, the queue still works
self.assertEqual(3, q.get())
self.assertEqual(7, q.qsize())
def benchmark_sqlite_write_10000(self):
"""Benchmark sqlite queue by writing <BENCHMARK_COUNT> items."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=True)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
def test_multiple_consumers(self):
"""Test sqlqueue can be used by multiple consumers."""
queue = SQLiteQueue(path=self.path,
multithreading=True,
auto_commit=self.auto_commit)
def producer():
for x in range(1000):
queue.put('var%d' % x)
task_done_if_required(queue)
def consumer():
for _ in range(100):
data = queue.get(block=True)
self.assertTrue('var' in data)
p = Thread(target=producer)
p.start()
consumers = []
for _ in range(10):
t = Thread(target=consumer)
t.start()
consumers.append(t)
for t in consumers:
t.join()
self.assertEqual(0, queue.qsize())
def test_multi_threaded_parallel(self):
"""Create consumer and producer threads, check parallelism"""
# self.skipTest("Not supported multi-thread.")
m_queue = SQLiteQueue(path=self.path,
multithreading=True,
auto_commit=self.auto_commit)
def producer():
for i in range(1000):
m_queue.put('var%d' % i)
task_done_if_required(m_queue)
def consumer():
for i in range(1000):
x = m_queue.get(block=True)
self.assertEqual('var%d' % i, x)
c = Thread(target=consumer)
c.start()
p = Thread(target=producer)
p.start()
p.join()
c.join()
self.assertEqual(0, m_queue.size)
self.assertEqual(0, len(m_queue))
self.assertIsNone(m_queue.get(block=False))
def test_raise_empty(self):
q = SQLiteQueue(self.path, auto_commit=self.auto_commit)
q.put('first')
task_done_if_required(q)
d = q.get()
self.assertEqual('first', d)
self.assertRaises(Empty, q.get, block=True)
def benchmark_sqlite_write(self):
"""Writing <BENCHMARK_COUNT> items."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=False)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
assert q.qsize() == BENCHMARK_COUNT
def test_open_close_single(self):
"""Write 1 item, close, reopen checking if same item is there"""
q = self.queue_class(self.path, auto_commit=self.auto_commit)
q.put(b'var1')
del q
q = SQLiteQueue(self.path)
self.assertEqual(1, q.qsize())
self.assertEqual(b'var1', q.get())
def test_raise_empty(self):
q = SQLiteQueue(self.path, auto_commit=self.auto_commit)
q.put('first')
d = q.get()
self.assertEqual('first', d)
self.assertRaises(Empty, q.get, block=False)
# assert with timeout
self.assertRaises(Empty, q.get, block=True, timeout=1.0)
# assert with negative timeout
self.assertRaises(ValueError, q.get, block=True, timeout=-1.0)
def test_empty(self):
q = SQLiteQueue(self.path, auto_commit=self.auto_commit)
self.assertEqual(q.empty(), True)
q.put('first')
self.assertEqual(q.empty(), False)
q.get()
self.assertEqual(q.empty(), True)
def test_json_serializer(self):
q = SQLiteQueue(
path=self.path,
serializer=serializers_json)
x = dict(
a=1,
b=2,
c=dict(
d=list(range(5)),
e=[1]
))
q.put(x)
self.assertEquals(q.get(), x)
def benchmark_sqlite_read_write_false(self):
"""Writing and reading <BENCHMARK_COUNT> items(1 task_done)."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=False)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
for i in range(BENCHMARK_COUNT):
q.get()
q.task_done()
assert q.qsize() == 0
def test_multi_threaded_multi_producer(self):
"""Test sqlqueue can be used by multiple producers."""
queue = SQLiteQueue(path=self.path,
multithreading=True,
auto_commit=self.auto_commit)
def producer(seq):
for i in range(10):
queue.put('var%d' % (i + (seq * 10)))
task_done_if_required(queue)
def consumer():
for i in range(100):
data = queue.get(block=True)
self.assertTrue('var' in data)
c = Thread(target=consumer)
c.start()
producers = []
for seq in range(10):
t = Thread(target=producer, args=(seq, ))
t.start()
producers.append(t)
for t in producers:
t.join()
c.join()
def get_followed_queue(self):
while True:
try:
return SQLiteQueue(self.directory + '/data/followed_users')
except OSError as e:
if e.errno != os.errno.EEXIST:
raise
sleep(1)
def test_get_id(self):
q = SQLiteQueue(path=self.path)
q.put("val1")
val2_id = q.put("val2")
q.put("val3")
item = q.get(id=val2_id)
# item id should be 2
self.assertEqual(val2_id, 2)
# item should get val2
self.assertEqual(item, 'val2')
def benchmark_sqlite_wr_10000(self):
"""Benchmark sqlite queue by writing and reading <BENCHMARK_COUNT> items."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=False)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
q.task_done()
for i in range(BENCHMARK_COUNT):
q.get()
def __init__(self, spec=None, spider=None, spiders=None, start_jobs=None,
queue_path=None, threads=25,
buffer_size=DEFAULT_IMAP_BUFFER_SIZE, throttle=DEFAULT_THROTTLE,
join=True, daemonic=False):
# NOTE: crawling could work depth-first but:
# buffer_size should be 0 (requires to fix quenouille issue #1)
# Params
self.start_jobs = start_jobs
self.queue_path = queue_path
self.threads = threads
self.buffer_size = buffer_size
self.throttle = throttle
self.join = join
self.daemonic = daemonic
self.using_persistent_queue = queue_path is not None
self.pool = create_pool(threads=threads)
self.state = CrawlerState()
self.started = False
# Memory queue
if not self.using_persistent_queue:
queue = Queue()
# Persistent queue
else:
queue = SQLiteQueue(queue_path, multithreading=True, auto_commit=False)
# Creating spiders
if spec is not None:
if not isinstance(spec, dict):
spec = load_definition(spec)
if 'spiders' in spec:
spiders = {name: DefinitionSpider(s, name=name) for name, s in spec['spiders'].items()}
self.single_spider = False
else:
spiders = {'default': DefinitionSpider(spec)}
self.single_spider = True
elif spider is not None:
spiders = {'default': spider}
elif spiders is None:
raise TypeError('minet.Crawler: expecting either `spec`, `spider` or `spiders`.')
# Solving function spiders
for name, s in spiders.items():
if callable(s) and not isinstance(s, Spider):
spiders[name] = FunctionSpider(s, name)
self.queue = queue
self.spiders = spiders
def test_queue(self):
q = SQLiteQueue(path=self.path)
q.put("val1")
q.put("val2")
q.put("val3")
# queue should get the three items
d = q.queue()
self.assertEqual(len(d), 3)
self.assertEqual(d[1].get("data"), "val2")
def open(self, **kwargs) -> "PersistQueueConnection":
if self.queue_engine == "sqlite":
queue = SQLiteQueue(self.queue_path, auto_commit=False)
elif self.queue_engine == "file":
queue = Queue(self.queue_path)
else:
raise ValueError("bad queue engine value")
self._queue = queue
return self
def test_multiple_consumers(self):
"""Test sqlqueue can be used by multiple consumers."""
queue = SQLiteQueue(path=self.path,
multithreading=True,
auto_commit=self.auto_commit)
def producer():
for x in range(1000):
queue.put('var%d' % x)
counter = []
# Set all to 0
for _ in range(1000):
counter.append(0)
def consumer(index):
for i in range(200):
data = queue.get(block=True)
self.assertTrue('var' in data)
counter[index * 200 + i] = data
p = Thread(target=producer)
p.start()
consumers = []
for index in range(5):
t = Thread(target=consumer, args=(index, ))
t.start()
consumers.append(t)
p.join()
for t in consumers:
t.join()
self.assertEqual(0, queue.qsize())
for x in range(1000):
self.assertNotEqual(0, counter[x],
"not 0 for counter's index %s" % x)
self.assertEqual(len(set(counter)), len(counter))
def test_open_close_1000(self):
"""Write 1000 items, close, reopen checking if all items are there"""
q = self.queue_class(self.path, auto_commit=self.auto_commit)
for i in range(1000):
q.put('var%d' % i)
self.assertEqual(1000, q.qsize())
del q
q = SQLiteQueue(self.path)
self.assertEqual(1000, q.qsize())
for i in range(1000):
data = q.get()
self.assertEqual('var%d' % i, data)
# assert adding another one still works
q.put('foobar')
data = q.get()
q.shrink_disk_usage()
self.assertEqual('foobar', data)
async def send_event(
body: Any = Body(...),
event_queue: persistqueue.SQLiteQueue = Depends(_get_queue_session),
):
"""
Endpoint for slack events API verification and event ingestion.
"""
if "token" not in body:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="Request body missing required fields.",
)
if _SLACK_VERIFICATION_TOKEN is None:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Missing verification token. Unable to accept new event.",
)
if not secrets.compare_digest(body["token"], _SLACK_VERIFICATION_TOKEN):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid token.",
)
if "challenge" in body:
# slack event endpoint verification step.
return JSONResponse(
status_code=status.HTTP_200_OK,
content=dict(challenge=body["challenge"], ),
)
log.info("Adding payload to slack event queue.")
log.debug("payload value: %s", body)
event_queue.put(body)
return JSONResponse(status_code=status.HTTP_200_OK,
content=dict(status="Event saved."))
def test_random_read_write(self):
"""Test random read/write"""
q = SQLiteQueue(self.path)
n = 0
for i in range(1000):
if random.random() < 0.5:
if n > 0:
q.get()
n -= 1
else:
self.assertEqual(None, q.get())
else:
q.put('var%d' % random.getrandbits(16))
n += 1
def test_random_read_write(self):
"""Test random read/write"""
q = SQLiteQueue(self.path, auto_commit=self.auto_commit)
n = 0
for i in range(1000):
if random.random() < 0.5:
if n > 0:
q.get()
n -= 1
else:
self.assertEqual(None, q.get())
else:
q.put('var%d' % random.getrandbits(16))
task_done_if_required(q)
n += 1
def get_fifo_event(
username: str = Depends(get_verified_username),
event_queue: persistqueue.SQLiteQueue = Depends(_get_queue_session),
):
"""
...
"""
if event_queue.size == 0:
return JSONResponse(
status_code=status.HTTP_204_NO_CONTENT,
content="No available events present.",
)
log.info(f"Sending FIFO slack event to user {username}")
return dict(
event=event_queue.get(),
events_left_in_queue=event_queue.size,
)
def test_random_read_write(self):
"""Test random read/write"""
q = SQLiteQueue(self.path, auto_commit=self.auto_commit)
n = 0
for i in range(1000):
if random.random() < 0.5:
if n > 0:
q.get()
n -= 1
else:
self.assertRaises(Empty, q.get, block=False)
else:
q.put('var%d' % random.getrandbits(16))
n += 1
def test_protocol_2(self):
q = SQLiteQueue(path=self.path)
self.assertEqual(q.protocol, None)
def test_put_0(self):
q = SQLiteQueue(path=self.path)
q.put(0)
d = q.get(block=False)
self.assertIsNotNone(d)
def test_protocol_1(self):
shutil.rmtree(self.path, ignore_errors=True)
q = SQLiteQueue(path=self.path)
self.assertEqual(q.protocol, 2 if sys.version_info[0] == 2 else 4)
def test_protocol_2(self):
q = SQLiteQueue(path=self.path)
self.assertEqual(q.protocol, 2 if sys.version_info[0] == 2 else 4)
