def instance_manager_requires_kafka(loaded_instance_manager):
def _format(_type, b):
return (b['id'], _type, b)
# things that require kafka
# kafka_message XF
messages_topic_name = str(uuid4())
xf_kafka_msg = deepcopy(examples.XF_KAFKA_MESSAGE)
xf_kafka_msg['topic'] = messages_topic_name
# error reporting pipeline
reports_topic_name = str(uuid4())
reporting_pipeline = deepcopy(examples.PIPELINE_KAFKA_LOGS)
reporting_pipeline['error_handling']['error_topic'] = reports_topic_name
for i in [('kafkamessage', xf_kafka_msg),
('pipeline', reporting_pipeline)]:
_id, _type, body = _format(*i)
loaded_instance_manager.update(_id, _type, TENANT, body)
yield loaded_instance_manager
# delete topics
kafka_security = config.get_kafka_admin_config()
kadmin = get_admin_client(kafka_security)
for topic in (messages_topic_name, reports_topic_name):
delete_topic(kadmin, f'{TENANT}.{topic}')
def create_remote_kafka_assets(request, sample_generator, *args):
# @mark annotation does not work with autouse=True.
if 'integration' not in request.config.invocation_params.args:
LOG.debug(f'NOT creating Kafka Assets')
# return
LOG.debug(f'Creating Kafka Assets')
kafka_security = config.get_kafka_admin_config()
kadmin = get_admin_client(kafka_security)
new_topic = f'{TENANT}.{TEST_TOPIC}'
create_topic(kadmin, new_topic)
GENERATED_SAMPLES[new_topic] = []
producer = get_producer(kafka_security)
schema = parse(json.dumps(ANNOTATED_SCHEMA))
for subset in sample_generator(max=100, chunk=10):
GENERATED_SAMPLES[new_topic].extend(subset)
produce(subset, schema, new_topic, producer)
yield None # end of work before clean-up
LOG.debug(f'deleting topic: {new_topic}')
delete_topic(kadmin, new_topic)
def check_or_create_topic(self):
kadmin = get_admin_client(ADMIN_CONFIG)
info = get_broker_info(kadmin)
if self.topic not in info.get('topics'):
create_topic(kadmin, self.topic)
from spavro.io import DatumReader
from aet.logger import get_logger
from aet.kafka_utils import (create_topic, get_producer, get_admin_client,
get_broker_info, produce)
from .config import (get_function_config, get_kafka_admin_config)
from . import fb_utils
CONF = get_function_config()
MAX_KAFKA_MESSAGE_SIZE = int(
CONF.get('MAX_KAFKA_MESSAGE_SIZE',
100_000)) # keep things reasonably sized, MAX is 2mb
KAFKA_SECURITY = get_kafka_admin_config()
KADMIN = get_admin_client(KAFKA_SECURITY)
PRODUCER = get_producer(KAFKA_SECURITY)
_logger = get_logger('KAFKA')
def publish(objs: List[Tuple[str, Any]],
schema: Dict,
_type: str,
rtdb=None,
max_size=MAX_KAFKA_MESSAGE_SIZE):
_prepare_kafka(_type)
# have to split out _publish because it can be called on failure and
# we only want to try to create the topic once
res = _publish_kafka(
[i for (_id, i) in objs
], # strip out _ids, must be in the doc at this point