def test_enqueue_job(producer, logger):
producer_cls, producer_inst = producer
queue = Queue(hosts='host:7000', topic='foo', timeout=300)
old_job = Job(
id='2938401',
timestamp=int(time.time()),
topic='bar',
func=failure_func,
args=[1, 2],
kwargs={'a': 3},
timeout=100,
)
new_job = queue.enqueue(old_job)
assert isinstance(new_job, Job)
assert isinstance(new_job.id, str)
assert isinstance(new_job.timestamp, int)
assert old_job.id != new_job.id
assert old_job.timestamp <= new_job.timestamp
assert new_job.topic == 'foo'
assert new_job.func == failure_func
assert new_job.args == [1, 2]
assert new_job.kwargs == {'a': 3}
assert new_job.timeout == 300
producer_inst.send.assert_called_with('foo', dill.dumps(new_job))
logger.info.assert_called_once_with('Enqueued: {}'.format(new_job))
def test_invalid_call(producer, logger):
producer_cls, producer_inst = producer
queue = Queue(hosts='host:7000', topic='foo', timeout=300)
for bad_func in [None, 1, {1, 2}, [1, 2, 3]]:
with pytest.raises(ValueError) as e:
queue.enqueue(bad_func, 1, 2, a=3)
assert str(e.value) == '{} is not a callable'.format(bad_func)
assert not producer_inst.send.called
assert not logger.info.called
def test_init(producer, logger):
producer_cls, producer_inst = producer
queue = Queue(
hosts='host:7000,host:8000',
topic='foo',
timeout=1000,
compression='gzip',
acks=0,
retries=5,
job_size=10000000,
cafile='/test/files/cafile',
certfile='/test/files/certfile',
keyfile='/test/files/keyfile',
crlfile='/test/files/crlfile'
)
producer_cls.assert_called_with(
bootstrap_servers='host:7000,host:8000',
compression_type='gzip',
acks=0,
retries=5,
max_request_size=10000000,
buffer_memory=33554432,
ssl_cafile='/test/files/cafile',
ssl_certfile='/test/files/certfile',
ssl_keyfile='/test/files/keyfile',
ssl_crlfile='/test/files/crlfile',
)
assert repr(queue) == 'Queue(topic=foo)'
assert queue.hosts == ['host:7000', 'host:8000']
assert queue.timeout == 1000
assert queue.topic == 'foo'
assert queue.producer == producer_inst
assert not logger.info.called
def test_enqueue_call(producer, logger):
producer_cls, producer_inst = producer
queue = Queue(hosts='host:7000', topic='foo', timeout=300)
job = queue.enqueue(success_func, 1, 2, c=[3, 4, 5])
assert isinstance(job, Job)
assert isinstance(job.id, str)
assert isinstance(job.timestamp, int)
assert job.topic == 'foo'
assert job.func == success_func
assert job.args == (1, 2)
assert job.kwargs == {'c': [3, 4, 5]}
assert job.timeout == 300
producer_inst.send.assert_called_with('foo', dill.dumps(job))
logger.info.assert_called_once_with('Enqueued: {}'.format(job))
def test_job_decorator():
queue = Queue(hosts='host:7000', topic='foo')
@queue.job
def test_function(a, b, c=None):
return a, b, c
assert hasattr(test_function, 'delay')
with pytest.raises(Exception) as e:
test_function.delay(1, 2, 3, 4)
assert "Can't pickle" in str(e.value)
def _fail_record(self, record):
"""De-serialize the message and put the job in fail queue
to be handled by fail queue worker.
:param record: Record fetched from the Kafka topic.
:type record: kafka.consumer.fetcher.ConsumerRecord
"""
rec = rec_repr(record)
try:
job = dill.loads(record.value)
except:
self._logger.warning('{} unloadable. Cannot fail ...'.format(rec))
else:
Queue(hosts=self._hosts,
topic=self._get_fail_topic(),
timeout=self._connect_timeout,
acks=-1,
retries=5,
job_size=self._job_size,
cafile=self._cafile,
certfile=self._certfile,
keyfile=self._keyfile,
crlfile=self._crlfile).enqueue(job)
def test_queue_initialization_with_bad_args(producer):
with pytest.raises(AssertionError) as e:
Queue(topic=True, producer=producer)
assert str(e.value) == "topic must be a str"
with pytest.raises(AssertionError) as e:
Queue(topic="topic", producer="bar")
assert str(e.value) == "bad producer instance"
with pytest.raises(AssertionError) as e:
Queue(topic="topic", producer=producer, serializer="baz")
assert str(e.value) == "serializer must be a callable"
with pytest.raises(AssertionError) as e:
Queue(topic="topic", producer=producer, timeout="bar")
assert str(e.value) == "timeout must be an int or float"
with pytest.raises(AssertionError) as e:
Queue(topic="topic", producer=producer, timeout=-1)
assert str(e.value) == "timeout must be 0 or greater"
with pytest.raises(AssertionError) as e:
Queue(topic="topic", producer=producer, logger=1)
assert str(e.value) == "bad logger instance"
def test_queue_initialization_with_bad_args(producer):
with pytest.raises(AssertionError) as e:
Queue(topic=True, producer=producer)
assert str(e.value) == 'topic must be a str'
with pytest.raises(AssertionError) as e:
Queue(topic='topic', producer='bar')
assert str(e.value) == 'bad producer instance'
with pytest.raises(AssertionError) as e:
Queue(topic='topic', producer=producer, serializer='baz')
assert str(e.value) == 'serializer must be a callable'
with pytest.raises(AssertionError) as e:
Queue(topic='topic', producer=producer, timeout='bar')
assert str(e.value) == 'timeout must be an int or float'
with pytest.raises(AssertionError) as e:
Queue(topic='topic', producer=producer, timeout=-1)
assert str(e.value) == 'timeout must be 0 or greater'
with pytest.raises(AssertionError) as e:
Queue(topic='topic', producer=producer, logger=1)
assert str(e.value) == 'bad logger instance'
def test_flush(producer):
producer_cls, producer_inst = producer
queue = Queue(hosts='host:7000', topic='foo')
queue.flush()
producer_inst.flush.assert_called_once()
from kafka import KafkaProducer
from kq import Queue
def add(a, b):
return a + b
# Set up a Kafka producer.
producer = KafkaProducer(bootstrap_servers="127.0.0.1:9092")
# Set up a queue.
queue = Queue(topic="topic", producer=producer)
# Enqueue a function call.
job = queue.enqueue(add, 1, 2)
def queue(topic, producer):
return Queue(topic, producer)
import requests
from time import sleep
from kafka import KafkaProducer
from kq import Queue
# Set up a Kafka producer.
producer = KafkaProducer(bootstrap_servers='34.67.1.98:9092')
# Set up a queue.
queue = Queue(topic='topic_queue', producer=producer)
# Enqueue a function call.
job = queue.enqueue(requests.get, 'https://www.google.com')
sleep(.1)
print("success")