def get_producer_settings(self):
broker_url = self.get_broker_url()
logger.debug('connecting to kafka with url: ' + broker_url)
producer_name = self.get_producer_name()
logger.debug('using group id: ' + producer_name)
default_settings = {
'api.version.request': True,
'broker.version.fallback': '0.9.0',
'client.id': 'JanglProducer',
'bootstrap.servers': broker_url,
'group.id': producer_name,
'default.topic.config': {
'request.required.acks': 1,
'request.timeout.ms': 5000,
'message.timeout.ms': 60000,
'produce.offset.report': False,
},
'queue.buffering.max.messages': 100000,
'queue.buffering.max.ms': 1000,
'message.send.max.retries': 2,
'retry.backoff.ms': 100,
'compression.codec': 'none',
'batch.num.messages': 1000,
'delivery.report.only.error': False,
}
return generate_client_settings(default_settings,
self.producer_settings)
def on_revoke(self, consumer, partitions):
logger.debug('partitions revoked: {}'.format(partitions))
try:
consumer.commit(asynchronous=False)
except KafkaException:
pass
consumer.unassign()
def get_producer_settings(self):
broker_url = self.get_broker_url()
logger.debug('connecting to kafka with url: ' + broker_url)
producer_name = self.get_producer_name()
logger.debug('using group id: ' + producer_name)
default_settings = {
'api.version.request': True,
'broker.version.fallback': '0.9.0',
'client.id': 'JanglProducer',
'bootstrap.servers': broker_url,
'group.id': producer_name,
'default.topic.config': {
'request.required.acks': 1,
'request.timeout.ms': 5000,
'message.timeout.ms': 60000,
'produce.offset.report': False,
},
'queue.buffering.max.messages': 100000,
'queue.buffering.max.ms': 1000,
'message.send.max.retries': 2,
'retry.backoff.ms': 100,
'compression.codec': 'none',
'batch.num.messages': 1000,
'delivery.report.only.error': False,
}
return generate_client_settings(default_settings, self.producer_settings)
def on_revoke(self, consumer, partitions):
logger.debug('partitions revoked: {}'.format(partitions))
try:
consumer.commit(async=False)
except KafkaException:
pass
consumer.unassign()
def get_latest(self):
self.schema_id, self.schema_avro, self.schema_version = self.schema_client.get_latest_schema(self.subject)
if self.schema_id is None:
self.register_schema()
else:
logger.info('latest schema for "{0}" - id: {1} - version: {2}'
.format(self.subject, self.schema_id, self.schema_version))
logger.debug(self.schema_avro)
return self.schema_id
def register_schema(self, new_schema=None):
if new_schema is None:
new_schema = self.local_schema
self.schema_id = self.schema_client.register(self.subject, new_schema)
self.schema_avro = self.schema_client.get_by_id(self.schema_id)
self.schema_version = self.schema_client.get_version(self.subject, self.schema_avro)
logger.info('new schema for "{0}" - id: {1} - version: {2}'
.format(self.subject, self.schema_id, self.schema_version))
logger.debug(self.schema_avro)
def register_schema(self, new_schema=None):
if new_schema is None:
new_schema = self.local_schema
self.schema_id = self.schema_client.register(self.subject, new_schema)
self.schema_avro = self.schema_client.get_by_id(self.schema_id)
self.schema_version = self.schema_client.get_version(
self.subject, self.schema_avro)
logger.info('new schema for "{0}" - id: {1} - version: {2}'.format(
self.subject, self.schema_id, self.schema_version))
logger.debug(self.schema_avro)
def get_latest(self):
self.schema_id, self.schema_avro, self.schema_version = self.schema_client.get_latest_schema(
self.subject)
if self.schema_id is None:
self.register_schema()
else:
logger.info(
'latest schema for "{0}" - id: {1} - version: {2}'.format(
self.subject, self.schema_id, self.schema_version))
logger.debug(self.schema_avro)
return self.schema_id
def generate_client_settings(initial_settings, user_settings):
settings = initial_settings.copy()
for key, val in six.iteritems(user_settings):
if val is None:
continue
logger.debug('using kafka setting {}: {}'.format(key, val))
if key.startswith('topic.'):
settings['default.topic.config'][key[6:]] = val
else:
settings[key] = val
return settings
def generate_client_settings(initial_settings, user_settings):
settings = initial_settings.copy()
for key, val in user_settings.iteritems():
if val is None:
continue
logger.debug('using kafka setting {}: {}'.format(key, val))
if key.startswith('topic.'):
settings['default.topic.config'][key[6:]] = val
else:
settings[key] = val
return settings
def get_consumer_settings(self):
broker_url = self.get_broker_url()
logger.debug('connecting to kafka: ' + broker_url)
consumer_name = self.get_consumer_name()
logger.debug('using group id: ' + consumer_name)
initial_settings = {
'api.version.request': True,
'broker.version.fallback': '0.9.0',
'client.id': 'JanglConsumer',
'bootstrap.servers': broker_url,
'group.id': consumer_name,
'default.topic.config': {'auto.offset.reset': 'earliest'},
'enable.auto.commit': False,
'on_commit': self.on_commit,
'session.timeout.ms': 10000,
'heartbeat.interval.ms': 1000,
}
return generate_client_settings(initial_settings, self.consumer_settings)
def send_message(self, *args, **kwargs):
""" Send a message to kafka
If the topic has a key, this method accepts the following signatures:
- self.send_message(key, message, **kwargs)
- self.send_message((key, message), **kwargs)
- self.send_message(key='', message='', **kwargs)
If the topic does not have a key, this method accepts:
- self.send_message(message, **kwargs)
Accepts the following kwargs:
- _async: If _async is False, producer will send message immediately
- partition: The partition id to produce to
- callback: Delivery callback with signature on_delivery(err,msg)
"""
_async = kwargs.pop('_async', self._async)
if self.has_key:
if len(args) == 0:
key = kwargs.pop('key')
message = kwargs.pop('message')
elif len(args) == 1: # [(key, message)]
key, message = args[0]
elif len(args) == 2: # [key, message]
key, message = args
else:
raise ValueError('Invalid message format')
logger.info('### Sending kafka message ###\n'
'key: {}\n'
'message: {}'.format(key, message))
encoded_key = self.key_schema.encode_message(key)
encoded_message = self.value_schema.encode_message(message)
logger.debug('key encoded w/ schema #{}: {}'.format(
self.key_schema.schema_id, encoded_key))
logger.debug('message encoded w/ schema #{}: {}'.format(
self.value_schema.schema_id, encoded_message))
self._produce(encoded_message, key=encoded_key, **kwargs)
elif len(args) == 1:
message = args[0]
logger.info('### Sending kafka message ###\n'
'message: {}'.format(message))
encoded_message = self.value_schema.encode_message(message)
logger.debug('message encoded w/ schema #{}: {}'.format(
self.value_schema.schema_id, encoded_message))
self._produce(encoded_message, **kwargs)
else:
raise ValueError('Invalid message format')
if not _async:
self._flush()
def get_consumer_settings(self):
broker_url = self.get_broker_url()
logger.debug('connecting to kafka: ' + broker_url)
consumer_name = self.get_consumer_name()
logger.debug('using group id: ' + consumer_name)
initial_settings = {
'api.version.request': True,
'broker.version.fallback': '0.9.0',
'client.id': 'JanglConsumer',
'bootstrap.servers': broker_url,
'group.id': consumer_name,
'default.topic.config': {
'auto.offset.reset': 'earliest'
},
'enable.auto.commit': False,
'on_commit': self.on_commit,
'session.timeout.ms': 10000,
'heartbeat.interval.ms': 1000,
}
return generate_client_settings(initial_settings,
self.consumer_settings)
def get_message_serializer(self):
schema_registry_url = self.get_schema_registry_url()
logger.debug('loading schema registry: ' + schema_registry_url)
schema_client = CachedSchemaRegistryClient(url=schema_registry_url)
return MessageSerializer(schema_client)
message = kwargs.pop('message')
elif len(args) == 1: # [(key, message)]
key, message = args[0]
elif len(args) == 2: # [key, message]
key, message = args
else:
raise ValueError('Invalid message format')
logger.info('### Sending kafka message ###\n'
'key: {}\n'
'message: {}'.format(key, message))
encoded_key = self.key_schema.encode_message(key)
encoded_message = self.value_schema.encode_message(message)
logger.debug('key encoded w/ schema #{}: {}'.format(self.key_schema.schema_id, encoded_key))
logger.debug('message encoded w/ schema #{}: {}'.format(self.value_schema.schema_id, encoded_message))
self._produce(encoded_message, key=encoded_key, **kwargs)
elif len(args) == 1:
message = args[0]
logger.info('### Sending kafka message ###\n'
'message: {}'.format(message))
encoded_message = self.value_schema.encode_message(message)
logger.debug('message encoded w/ schema #{}: {}'.format(self.value_schema.schema_id, encoded_message))
self._produce(encoded_message, **kwargs)
else:
def on_assign(self, consumer, partitions):
logger.debug('partitions assigned: {}'.format(partitions))
consumer.assign(partitions)
def on_assign(self, consumer, partitions):
logger.debug('partitions assigned: {}'.format(partitions))
consumer.assign(partitions)
def set_topic(self):
topic_name = self.get_topic_name()
logger.debug('set kafka topic: ' + topic_name)
self.consumer.subscribe([topic_name], on_assign=self.on_assign, on_revoke=self.on_revoke)
def get_message_serializer(self):
schema_registry_url = self.get_schema_registry_url()
logger.debug('loading schema registry: ' + schema_registry_url)
schema_client = CachedSchemaRegistryClient(url=schema_registry_url)
return MessageSerializer(schema_client)
def on_commit(self, err, partitions):
if err is None:
logger.debug('commit done: {}'.format(partitions))
else:
logger.error('commit error: {} - {}'.format(err, partitions))
def on_commit(self, err, partitions):
if err is None:
logger.debug('commit done: {}'.format(partitions))
else:
logger.error('commit error: {} - {}'.format(err, partitions))
def set_topic(self):
topic_name = self.get_topic_name()
logger.debug('set kafka topic: ' + topic_name)
self.consumer.subscribe([topic_name],
on_assign=self.on_assign,
on_revoke=self.on_revoke)