def test_valid_nonempty_iterable_of_strings(self):
class SampleIterator(object):
def __init__(self, item):
self.test_list = list()
self.test_list.append(item)
def __iter__(self):
return iter(self.test_list)
def __len__(self):
return len(self.test_list)
self.assertTrue(
PanoptesValidators.valid_nonempty_iterable_of_strings(list(u'x')))
self.assertTrue(
PanoptesValidators.valid_nonempty_iterable_of_strings(set(u'x')))
self.assertTrue(
PanoptesValidators.valid_nonempty_iterable_of_strings(
SampleIterator(u'x')))
self.assertFalse(
PanoptesValidators.valid_nonempty_iterable_of_strings(u'x'))
self.assertFalse(
PanoptesValidators.valid_nonempty_iterable_of_strings([]))
self.assertFalse(
PanoptesValidators.valid_nonempty_iterable_of_strings([u'']))
self.assertFalse(
PanoptesValidators.valid_nonempty_iterable_of_strings([u'x', u'']))
self.assertFalse(
PanoptesValidators.valid_nonempty_iterable_of_strings(
[u'x', set(u'x')]))
def __init__(self,
panoptes_context,
consumer_type,
topics,
client_id,
group,
keys,
poll_timeout,
callback,
validate=False,
session_timeout=60,
max_poll_records=500,
max_partition_fetch_bytes=1048576):
"""
This class implements a helper/wrapper for writing Panoptes Consumers
The consumer object produced by this class joins the group provided in the arguments, subscribes to the relevant
topics (which are combination of the site name and consumer type), filters records by the provided keys, checks
validity of the records by converting them to JSON and validating against consumer type specific schemas and
then calls the callback function provided for each record.
The consumer object also takes care of advancing the committed marker for each topic and partition - if the
callback fails or returns false, the marker is not advanced.
Args:
panoptes_context (PanoptesContext): The PanoptesContext to be used by the consumer
consumer_type (int): A valid consumer type
topics (list): A valid consumer type
client_id (str): A non-empty string that uniquely identifies the consumer instance - this is used for \
logging by and group administration by Kafka
group (str): The Kafka group this consumer should be a part of
keys (list, None): A comma separated list of keys the consumer should filter against
poll_timeout (int): A non-negative integer which is the interval (in seconds) the consumer should sleep if \
no records are available on the Kafka bus
callback (callable): A callable to which each processed, validated consumer records (deserialized JSON \
object) should be passed. The callable should return false if it cannot process the record due to
temporary \
issues - it would be redelivered to the callback in that case
validate (bool): Whether each incoming record should be validated. Defaults to False
session_timeout (int): A non-negative integer which is the interval (in seconds) after which the Kafka \
Group Management system should consider the client disconnected
max_poll_records (int): The maximum number of records to fetch in one poll cycle
Returns:
None
"""
assert isinstance(
panoptes_context, PanoptesContext
), u'panoptes_context must be an instance of PanoptesContext'
assert consumer_type in CONSUMER_TYPE_NAMES, u'consumer_type must be an valid attribute from ' \
u'PanoptesConsumerTypes'
assert PanoptesValidators.valid_nonempty_iterable_of_strings(
topics), u''
assert PanoptesValidators.valid_nonempty_string(
client_id), u'client_id must be a non-empty string'
assert keys is None or PanoptesValidators.valid_nonempty_iterable_of_strings(keys), \
u'keys must be None or a list of non-empty strings'
assert PanoptesValidators.valid_positive_integer(
poll_timeout), u'poll_timeout must be an integer'
assert hasattr(callback, u'__call__'), u'callback must be a callable'
assert isinstance(validate, bool), u'validate must be a boolean'
assert PanoptesValidators.valid_nonzero_integer(session_timeout), u'session_timeout must be an integer '\
u'greater than zero'
assert PanoptesValidators.valid_nonzero_integer(max_poll_records), u'max_poll_records must be an integer '\
u'greater than zero'
self._panoptes_context = panoptes_context
self._consumer_type = consumer_type
self._topics = topics
self._client_id = client_id
self._keys = keys
self._group = group
self._poll_timeout = poll_timeout * 1000
self._session_timeout = session_timeout * 1000
self._request_timeout = self._session_timeout * 3
self._heartbeat_interval = old_div(self._session_timeout, 3)
self._max_poll_records = max_poll_records
self._max_partition_fetch_bytes = max_partition_fetch_bytes
self._callback = callback
self._validate = validate
self._consumer = None
self._last_polled = 0
self._asked_to_stop = False