def test_valid_nonempty_iterable_of_strings(self): class SampleIterator(object): def __init__(self, item): self.test_list = list() self.test_list.append(item) def __iter__(self): return iter(self.test_list) def __len__(self): return len(self.test_list) self.assertTrue( PanoptesValidators.valid_nonempty_iterable_of_strings(list(u'x'))) self.assertTrue( PanoptesValidators.valid_nonempty_iterable_of_strings(set(u'x'))) self.assertTrue( PanoptesValidators.valid_nonempty_iterable_of_strings( SampleIterator(u'x'))) self.assertFalse( PanoptesValidators.valid_nonempty_iterable_of_strings(u'x')) self.assertFalse( PanoptesValidators.valid_nonempty_iterable_of_strings([])) self.assertFalse( PanoptesValidators.valid_nonempty_iterable_of_strings([u''])) self.assertFalse( PanoptesValidators.valid_nonempty_iterable_of_strings([u'x', u''])) self.assertFalse( PanoptesValidators.valid_nonempty_iterable_of_strings( [u'x', set(u'x')]))
def __init__(self, panoptes_context, consumer_type, topics, client_id, group, keys, poll_timeout, callback, validate=False, session_timeout=60, max_poll_records=500, max_partition_fetch_bytes=1048576): """ This class implements a helper/wrapper for writing Panoptes Consumers The consumer object produced by this class joins the group provided in the arguments, subscribes to the relevant topics (which are combination of the site name and consumer type), filters records by the provided keys, checks validity of the records by converting them to JSON and validating against consumer type specific schemas and then calls the callback function provided for each record. The consumer object also takes care of advancing the committed marker for each topic and partition - if the callback fails or returns false, the marker is not advanced. Args: panoptes_context (PanoptesContext): The PanoptesContext to be used by the consumer consumer_type (int): A valid consumer type topics (list): A valid consumer type client_id (str): A non-empty string that uniquely identifies the consumer instance - this is used for \ logging by and group administration by Kafka group (str): The Kafka group this consumer should be a part of keys (list, None): A comma separated list of keys the consumer should filter against poll_timeout (int): A non-negative integer which is the interval (in seconds) the consumer should sleep if \ no records are available on the Kafka bus callback (callable): A callable to which each processed, validated consumer records (deserialized JSON \ object) should be passed. The callable should return false if it cannot process the record due to temporary \ issues - it would be redelivered to the callback in that case validate (bool): Whether each incoming record should be validated. Defaults to False session_timeout (int): A non-negative integer which is the interval (in seconds) after which the Kafka \ Group Management system should consider the client disconnected max_poll_records (int): The maximum number of records to fetch in one poll cycle Returns: None """ assert isinstance( panoptes_context, PanoptesContext ), u'panoptes_context must be an instance of PanoptesContext' assert consumer_type in CONSUMER_TYPE_NAMES, u'consumer_type must be an valid attribute from ' \ u'PanoptesConsumerTypes' assert PanoptesValidators.valid_nonempty_iterable_of_strings( topics), u'' assert PanoptesValidators.valid_nonempty_string( client_id), u'client_id must be a non-empty string' assert keys is None or PanoptesValidators.valid_nonempty_iterable_of_strings(keys), \ u'keys must be None or a list of non-empty strings' assert PanoptesValidators.valid_positive_integer( poll_timeout), u'poll_timeout must be an integer' assert hasattr(callback, u'__call__'), u'callback must be a callable' assert isinstance(validate, bool), u'validate must be a boolean' assert PanoptesValidators.valid_nonzero_integer(session_timeout), u'session_timeout must be an integer '\ u'greater than zero' assert PanoptesValidators.valid_nonzero_integer(max_poll_records), u'max_poll_records must be an integer '\ u'greater than zero' self._panoptes_context = panoptes_context self._consumer_type = consumer_type self._topics = topics self._client_id = client_id self._keys = keys self._group = group self._poll_timeout = poll_timeout * 1000 self._session_timeout = session_timeout * 1000 self._request_timeout = self._session_timeout * 3 self._heartbeat_interval = old_div(self._session_timeout, 3) self._max_poll_records = max_poll_records self._max_partition_fetch_bytes = max_partition_fetch_bytes self._callback = callback self._validate = validate self._consumer = None self._last_polled = 0 self._asked_to_stop = False