def test_enqueue_job(producer, logger): producer_cls, producer_inst = producer queue = Queue(hosts='host:7000', topic='foo', timeout=300) old_job = Job( id='2938401', timestamp=int(time.time()), topic='bar', func=failure_func, args=[1, 2], kwargs={'a': 3}, timeout=100, ) new_job = queue.enqueue(old_job) assert isinstance(new_job, Job) assert isinstance(new_job.id, str) assert isinstance(new_job.timestamp, int) assert old_job.id != new_job.id assert old_job.timestamp <= new_job.timestamp assert new_job.topic == 'foo' assert new_job.func == failure_func assert new_job.args == [1, 2] assert new_job.kwargs == {'a': 3} assert new_job.timeout == 300 producer_inst.send.assert_called_with('foo', dill.dumps(new_job)) logger.info.assert_called_once_with('Enqueued: {}'.format(new_job))
def test_invalid_call(producer, logger): producer_cls, producer_inst = producer queue = Queue(hosts='host:7000', topic='foo', timeout=300) for bad_func in [None, 1, {1, 2}, [1, 2, 3]]: with pytest.raises(ValueError) as e: queue.enqueue(bad_func, 1, 2, a=3) assert str(e.value) == '{} is not a callable'.format(bad_func) assert not producer_inst.send.called assert not logger.info.called
def test_init(producer, logger): producer_cls, producer_inst = producer queue = Queue( hosts='host:7000,host:8000', topic='foo', timeout=1000, compression='gzip', acks=0, retries=5, job_size=10000000, cafile='/test/files/cafile', certfile='/test/files/certfile', keyfile='/test/files/keyfile', crlfile='/test/files/crlfile' ) producer_cls.assert_called_with( bootstrap_servers='host:7000,host:8000', compression_type='gzip', acks=0, retries=5, max_request_size=10000000, buffer_memory=33554432, ssl_cafile='/test/files/cafile', ssl_certfile='/test/files/certfile', ssl_keyfile='/test/files/keyfile', ssl_crlfile='/test/files/crlfile', ) assert repr(queue) == 'Queue(topic=foo)' assert queue.hosts == ['host:7000', 'host:8000'] assert queue.timeout == 1000 assert queue.topic == 'foo' assert queue.producer == producer_inst assert not logger.info.called
def test_enqueue_call(producer, logger): producer_cls, producer_inst = producer queue = Queue(hosts='host:7000', topic='foo', timeout=300) job = queue.enqueue(success_func, 1, 2, c=[3, 4, 5]) assert isinstance(job, Job) assert isinstance(job.id, str) assert isinstance(job.timestamp, int) assert job.topic == 'foo' assert job.func == success_func assert job.args == (1, 2) assert job.kwargs == {'c': [3, 4, 5]} assert job.timeout == 300 producer_inst.send.assert_called_with('foo', dill.dumps(job)) logger.info.assert_called_once_with('Enqueued: {}'.format(job))
def test_job_decorator(): queue = Queue(hosts='host:7000', topic='foo') @queue.job def test_function(a, b, c=None): return a, b, c assert hasattr(test_function, 'delay') with pytest.raises(Exception) as e: test_function.delay(1, 2, 3, 4) assert "Can't pickle" in str(e.value)
def _fail_record(self, record): """De-serialize the message and put the job in fail queue to be handled by fail queue worker. :param record: Record fetched from the Kafka topic. :type record: kafka.consumer.fetcher.ConsumerRecord """ rec = rec_repr(record) try: job = dill.loads(record.value) except: self._logger.warning('{} unloadable. Cannot fail ...'.format(rec)) else: Queue(hosts=self._hosts, topic=self._get_fail_topic(), timeout=self._connect_timeout, acks=-1, retries=5, job_size=self._job_size, cafile=self._cafile, certfile=self._certfile, keyfile=self._keyfile, crlfile=self._crlfile).enqueue(job)
def test_queue_initialization_with_bad_args(producer): with pytest.raises(AssertionError) as e: Queue(topic=True, producer=producer) assert str(e.value) == "topic must be a str" with pytest.raises(AssertionError) as e: Queue(topic="topic", producer="bar") assert str(e.value) == "bad producer instance" with pytest.raises(AssertionError) as e: Queue(topic="topic", producer=producer, serializer="baz") assert str(e.value) == "serializer must be a callable" with pytest.raises(AssertionError) as e: Queue(topic="topic", producer=producer, timeout="bar") assert str(e.value) == "timeout must be an int or float" with pytest.raises(AssertionError) as e: Queue(topic="topic", producer=producer, timeout=-1) assert str(e.value) == "timeout must be 0 or greater" with pytest.raises(AssertionError) as e: Queue(topic="topic", producer=producer, logger=1) assert str(e.value) == "bad logger instance"
def test_queue_initialization_with_bad_args(producer): with pytest.raises(AssertionError) as e: Queue(topic=True, producer=producer) assert str(e.value) == 'topic must be a str' with pytest.raises(AssertionError) as e: Queue(topic='topic', producer='bar') assert str(e.value) == 'bad producer instance' with pytest.raises(AssertionError) as e: Queue(topic='topic', producer=producer, serializer='baz') assert str(e.value) == 'serializer must be a callable' with pytest.raises(AssertionError) as e: Queue(topic='topic', producer=producer, timeout='bar') assert str(e.value) == 'timeout must be an int or float' with pytest.raises(AssertionError) as e: Queue(topic='topic', producer=producer, timeout=-1) assert str(e.value) == 'timeout must be 0 or greater' with pytest.raises(AssertionError) as e: Queue(topic='topic', producer=producer, logger=1) assert str(e.value) == 'bad logger instance'
def test_flush(producer): producer_cls, producer_inst = producer queue = Queue(hosts='host:7000', topic='foo') queue.flush() producer_inst.flush.assert_called_once()
from kafka import KafkaProducer from kq import Queue def add(a, b): return a + b # Set up a Kafka producer. producer = KafkaProducer(bootstrap_servers="127.0.0.1:9092") # Set up a queue. queue = Queue(topic="topic", producer=producer) # Enqueue a function call. job = queue.enqueue(add, 1, 2)
def queue(topic, producer): return Queue(topic, producer)
import requests from time import sleep from kafka import KafkaProducer from kq import Queue # Set up a Kafka producer. producer = KafkaProducer(bootstrap_servers='34.67.1.98:9092') # Set up a queue. queue = Queue(topic='topic_queue', producer=producer) # Enqueue a function call. job = queue.enqueue(requests.get, 'https://www.google.com') sleep(.1) print("success")