def __init__(self):
if TEST:
self.producer = TestKafkaProducer()
elif not IS_HEROKU:
self.producer = KP(bootstrap_servers=KAFKA_HOSTS)
else:
self.producer = kafka_helper.get_kafka_producer(
value_serializer=lambda d: d)
def __init__(self):
# Instantiating HelperVariable
self.hv = HelperVariable()
# Setting up Kafka-Brokers
# self.producer = KafkaProducer(bootstrap_servers = '10.0.0.15:31000,10.0.0.13:31000,10.0.0.14:31000,10.0.0.16:31000,10.0.0.16:31001')
# self.producer = KafkaProducer(bootstrap_servers = self.hv.KAFKA_SCHEDULER)
self.producer = kafka_helper.get_kafka_producer()
print(self.producer)
def forward_gitlab():
kafka_topic = "{}".format(os.environ['KAFKA_TOPIC'])
debug = os.environ.get('DEBUG', False)
app.logger.debug('received a message ')
TOKEN = os.environ.get("GITLAB_TOKEN")
if TOKEN is None or request.headers.get('X-Gitlab-Token') != TOKEN:
abort(401)
app.logger.debug('This message is INVALID')
return
app.logger.debug('This is a valid message')
content = request.get_json(silent=True)
js = json.dumps(content)
producer = get_kafka_producer()
producer.produce(kafka_topic, js, callback=_delivery_callback)
producer.flush()
return "OK"
def __init__(self, socketio, soft):
Thread.__init__(self)
self.event = Event()
self.sock = socketio
self.soft = soft
print('\nKafka on Heroku Checks:')
if os.environ.get('KAFKA_URL'):
print('\nRunning on Heroku')
self.producer = kafka_helper.get_kafka_producer()
print(kafka_helper.get_kafka_ssl_context())
print(kafka_helper.get_kafka_brokers())
else:
print('\nRunning Local - so not using kafka_helper')
# We're not on Heroku, try local running Kafka:
from kafka import KafkaProducer
producer = KafkaProducer(
bootstrap_servers=['localhost:9092'],
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
self.producer = producer
def testKafkaHelperSND():
import kafka_helper
producer = kafka_helper.get_kafka_producer()
producer.send('ple2', key='my key', value={'k': 'v'})
"""
Simple Flask app to receive Salesforce data and write it to Kafka.
"""
import kafka_helper
import os
from flask import Flask, abort, request
app = Flask(__name__)
app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY',
'this_should_be_configured')
CHATTER_KAFKA_TOPIC = 'chatter' #salfrs_kafka_snowflake
PRODUCER = kafka_helper.get_kafka_producer()
@app.route('/sf-data', methods=['POST'])
def sf_data():
"""
Receives serialized sObject data and writes selected fields to Kafka.
Intended to be called from an asynchronous Apex.
"""
if not verify_secret_key(request):
abort(401)
if request.method == 'POST':
filtered_sobject_list = filter_sobject_list(request.get_json() or [])
for sobject in filtered_sobject_list:
write_to_kafka(key=str(sobject['ParentId']), value=sobject)
return str(filtered_sobject_list)
