def __init__(self): if TEST: self.producer = TestKafkaProducer() elif not IS_HEROKU: self.producer = KP(bootstrap_servers=KAFKA_HOSTS) else: self.producer = kafka_helper.get_kafka_producer( value_serializer=lambda d: d)
def __init__(self): # Instantiating HelperVariable self.hv = HelperVariable() # Setting up Kafka-Brokers # self.producer = KafkaProducer(bootstrap_servers = '10.0.0.15:31000,10.0.0.13:31000,10.0.0.14:31000,10.0.0.16:31000,10.0.0.16:31001') # self.producer = KafkaProducer(bootstrap_servers = self.hv.KAFKA_SCHEDULER) self.producer = kafka_helper.get_kafka_producer() print(self.producer)
def forward_gitlab(): kafka_topic = "{}".format(os.environ['KAFKA_TOPIC']) debug = os.environ.get('DEBUG', False) app.logger.debug('received a message ') TOKEN = os.environ.get("GITLAB_TOKEN") if TOKEN is None or request.headers.get('X-Gitlab-Token') != TOKEN: abort(401) app.logger.debug('This message is INVALID') return app.logger.debug('This is a valid message') content = request.get_json(silent=True) js = json.dumps(content) producer = get_kafka_producer() producer.produce(kafka_topic, js, callback=_delivery_callback) producer.flush() return "OK"
def __init__(self, socketio, soft): Thread.__init__(self) self.event = Event() self.sock = socketio self.soft = soft print('\nKafka on Heroku Checks:') if os.environ.get('KAFKA_URL'): print('\nRunning on Heroku') self.producer = kafka_helper.get_kafka_producer() print(kafka_helper.get_kafka_ssl_context()) print(kafka_helper.get_kafka_brokers()) else: print('\nRunning Local - so not using kafka_helper') # We're not on Heroku, try local running Kafka: from kafka import KafkaProducer producer = KafkaProducer( bootstrap_servers=['localhost:9092'], value_serializer=lambda v: json.dumps(v).encode('utf-8')) self.producer = producer
def testKafkaHelperSND(): import kafka_helper producer = kafka_helper.get_kafka_producer() producer.send('ple2', key='my key', value={'k': 'v'})
""" Simple Flask app to receive Salesforce data and write it to Kafka. """ import kafka_helper import os from flask import Flask, abort, request app = Flask(__name__) app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY', 'this_should_be_configured') CHATTER_KAFKA_TOPIC = 'chatter' #salfrs_kafka_snowflake PRODUCER = kafka_helper.get_kafka_producer() @app.route('/sf-data', methods=['POST']) def sf_data(): """ Receives serialized sObject data and writes selected fields to Kafka. Intended to be called from an asynchronous Apex. """ if not verify_secret_key(request): abort(401) if request.method == 'POST': filtered_sobject_list = filter_sobject_list(request.get_json() or []) for sobject in filtered_sobject_list: write_to_kafka(key=str(sobject['ParentId']), value=sobject) return str(filtered_sobject_list)