def test_kafka_consumer(self): """ Test that the HerokuKafkaConsumer does not create any errors and is an instance of KafkaConsumer (python_kafka) """ consumer = HerokuKafkaConsumer(TOPIC1, **kafka_params) assert isinstance(consumer, KafkaConsumer) consumer.close()
def test_kafka_consumer_multiple_topic(self): """ Test that the consumer works with multiple topics and prefixes them correctly """ consumer = HerokuKafkaConsumer(TOPIC1, TOPIC2, **kafka_params) assert consumer.subscription() == { TOPIC1_WITH_PREFIX, TOPIC2_WITH_PREFIX } consumer.close()
def test_kafka_consumer_single_topic(self): """ Test that consumer works with a single topic and prefixes it correctly """ consumer = HerokuKafkaConsumer(TOPIC1, **kafka_params) assert consumer.subscription() == { TOPIC1_WITH_PREFIX, } consumer.close()
def receiveFromKafka(mode): consumer = HerokuKafkaConsumer( #KAKFA_TOPIC, # Optional: You don't need to pass any topic at all url=KAFKA_URL, # Url string provided by heroku ssl_cert=KAFKA_CLIENT_CERT, # Client cert string ssl_key=KAFKA_CLIENT_CERT_KEY, # Client cert key string ssl_ca=KAFKA_TRUSTED_CERT, # Client trusted cert string prefix=KAFKA_PREFIX, # Prefix provided by heroku, auto_offset_reset="smallest", max_poll_records=10, enable_auto_commit=True, auto_commit_interval_ms=10, #group_id=KAFKA_GROUP_ID, api_version=(0, 9)) """ To subscribe to topic(s) after creating a consumer pass in a list of topics without the KAFKA_PREFIX. """ partition = 1 tp = TopicPartition(KAFKA_PREFIX + KAFKA_TOPIC_READ, partition) if (mode == "subscribe"): consumer.subscribe(topics=(KAFKA_TOPIC_READ)) elif (mode == "assign"): consumer.assign([tp]) # display list of partition assignerd assignments = consumer.assignment() for assignment in assignments: logger.debug(assignment) partitions = consumer.partitions_for_topic(KAFKA_PREFIX + KAFKA_TOPIC_READ) if (partitions): for partition in partitions: logger.debug("Partition=" + str(partition)) topics = consumer.topics() if (topics): for topic in topics: logger.debug("Topic:" + topic) #exit(1) logger.debug('waiting ..') """ .assign requires a full topic name with prefix """ """ Listening to events it is exactly the same as in kafka_python. Read the documention linked below for more info! """ i = 0 for message in consumer: #logger.debug (' %s : %s ' % (i, message)) logger.debug("%i %s:%d:%d: key=%s value=%s" % (i, message.topic, message.partition, message.offset, message.key, message.value)) #consumer.commit() i += 1
def test_kafka_consumer_subscribe(self): """ Test that the consumer works with subscribe and prefixes topics correctly """ consumer = HerokuKafkaConsumer(**kafka_params) consumer.subscribe(topics=(TOPIC1, TOPIC2)) assert consumer.subscription() == { TOPIC1_WITH_PREFIX, TOPIC2_WITH_PREFIX } consumer.close()
def receiveFromKafka_EDF(mode): consumer = HerokuKafkaConsumer( #KAKFA_TOPIC, # Optional: You don't need to pass any topic at all url=KAFKA_URL, # Url string provided by heroku ssl_cert=KAFKA_CLIENT_CERT, # Client cert string ssl_key=KAFKA_CLIENT_CERT_KEY, # Client cert key string ssl_ca=KAFKA_TRUSTED_CERT, # Client trusted cert string prefix=KAFKA_PREFIX, # Prefix provided by heroku, auto_offset_reset="smallest", max_poll_records=10, enable_auto_commit=True, auto_commit_interval_ms=100, group_id=KAFKA_GROUP_ID, api_version=(0, 9)) """ To subscribe to topic(s) after creating a consumer pass in a list of topics without the KAFKA_PREFIX. """ partition = 1 tp = TopicPartition(KAFKA_PREFIX + KAFKA_TOPIC_READ, partition) if (mode == "subscribe"): consumer.subscribe(topics=(KAFKA_TOPIC_READ)) elif (mode == "assign"): consumer.assign([tp]) # display list of partition assignerd assignments = consumer.assignment() for assignment in assignments: logger.debug(assignment) partitions = consumer.partitions_for_topic(KAFKA_PREFIX + KAFKA_TOPIC_READ) if (partitions): for partition in partitions: logger.debug("Partition=" + str(partition)) topics = consumer.topics() if (topics): for topic in topics: logger.debug("Topic:" + topic) logger.debug('waiting ..') """ .assign requires a full topic name with prefix """ """ Listening to events it is exactly the same as in kafka_python. Read the documention linked below for more info! """ i = 0 for message in consumer: logger.debug("\n ######################################") #logger.debug (' %s : %s ' % (i, message)) logger.debug("%i %s:%d:%d: key=%s value=%s" % (i, message.topic, message.partition, message.offset, message.key, message.value)) i += 1 try: dictValue = ujson.loads(message.value) logger.debug(dictValue) sfid = dictValue['data']['payload']['IdObject__c'] typeEvent = dictValue['data']['payload']['TypeEvent__c'] if (typeEvent == 'EJ'): # generate sql request to retrieve all data about the account objecft using sfid data = postgres.__getAccountBySfId(sfid) dumped = ujson.dumps(data) sendToKafka_EDF(dumped) # now sends the data to another topic consumer.commit() except Exception as e: import traceback traceback.print_exc() logger.debug("merci Mohammed") #consumer.commit() #receiveFromKafka("subscribe") #import ujson #data = ujson.dumps({'key':'value'}) #print(data) #sendToKafka_HardCoded(data) #testKafkaHelperRCV()
def receiveFromKafka(mode, topic_override=None): TOPIC = KAFKA_TOPIC_READ if (topic_override != None): TOPIC = topic_override logger.info("Will use topic = {}".format(TOPIC)) consumer = HerokuKafkaConsumer( #KAKFA_TOPIC, # Optional: You don't need to pass any topic at all url= KAFKA_URL, # Url string provided by heroku ssl_cert= KAFKA_CLIENT_CERT, # Client cert string ssl_key= KAFKA_CLIENT_CERT_KEY, # Client cert key string ssl_ca= KAFKA_TRUSTED_CERT, # Client trusted cert string prefix= KAFKA_PREFIX, # Prefix provided by heroku, auto_offset_reset="smallest", max_poll_records=100, enable_auto_commit=True, auto_commit_interval_ms=100, group_id=KAFKA_GROUP_ID, api_version = (0,9) ) """ To subscribe to topic(s) after creating a consumer pass in a list of topics without the KAFKA_PREFIX. """ partition=1 tp = TopicPartition(KAFKA_PREFIX + TOPIC, partition) if (mode == "subscribe"): consumer.subscribe(topics=(TOPIC)) elif (mode == "assign"): consumer.assign([tp]) # display list of partition assignerd assignments = consumer.assignment() for assignment in assignments: logger.debug(assignment) partitions=consumer.partitions_for_topic(KAFKA_PREFIX + TOPIC) if (partitions): for partition in partitions: logger.debug("Partition="+str(partition)) topics=consumer.topics() if (topics): for topic in topics: logger.debug("Topic:"+topic) #exit(1) logger.debug('waiting ..') """ .assign requires a full topic name with prefix """ """ Listening to events it is exactly the same as in kafka_python. Read the documention linked below for more info! """ i=0 for message in consumer: try: logger.debug ("%i %s:%d:%d: key=%s value=%s" % (i, message.topic, message.partition, message.offset, message.key, message.value)) dictValue = ujson.loads(message.value) logger.debug(dictValue) # check value in the field Action Type if ("payload" in dictValue): if (dictValue['payload']['Action_Type__c'] == 'PushNotification'): logger.info("about to send a BROWSER NOTIFICATION using PUSHER") message = dictValue['payload']['message__c'] userid = dictValue['payload']['userid__c'], notification.sendNotification(userid, message) """ if ('channel' in dictValue): # means it's coming from a Platform EVENT via kafka if ('host_accept_guest__e' in dictValue['channel'].lower()): logger.info("about to send a SMS using BLOWER") message = "Dear {} {} , {} {} is aware of your arrival and will be here shortly".format( dictValue['data']['payload']['Guest_Firstname__c'], dictValue['data']['payload']['Guest_Lastname__c'], dictValue['data']['payload']['Host_Firstname__c'], dictValue['data']['payload']['Host_Lastname__c'], ) blower.sendMessage(message, dictValue['data']['payload']['Guest_Phone_Number__c']) elif ('send_smss__e' in dictValue['channel'].lower()): logger.info("about to send a SMS using BLOWER") message = dictValue['data']['payload']['message__c'] phone_Number = dictValue['data']['payload']['phone_Number__c'], blower.sendMessage(message, phone_Number) #{'schema': 'h7kPS4B7NEsigjlW7748lg', # 'payload': { # 'CreatedById': '0051t000002FB13AAG', # 'message__c': 'Hello ! ', # 'Action_Type__c': 'PushNotification', # 'CreatedDate': '2020-06-16T15:52:45.535Z', # 'userid__c': 'dac11bb3-148e-4b27-a6f2-caf0af09fb0a'}, 'event': {'replayId': 14570697}}} elif ('push_notification__e' in dictValue['channel'].lower()): logger.info("about to send a BROWSER NOTIFICATION using PUSHER") message = dictValue['data']['payload']['message__c'] userid = dictValue['data']['payload']['userid__c'], notification.sendNotification(userid, message) """ consumer.commit() except Exception as e : import traceback traceback.print_exc() consumer.commit() i += 1
producer = HerokuKafkaProducer( url=KAFKA_URL, # Url string provided by heroku ssl_cert=KAFKA_CLIENT_CERT, # Client cert string ssl_key=KAFKA_CLIENT_CERT_KEY, # Client cert key string ssl_ca=KAFKA_TRUSTED_CERT, # Client trusted cert string prefix=KAFKA_PREFIX # Prefix provided by heroku, #,partitioner="0" ) consumer = HerokuKafkaConsumer( #KAKFA_TOPIC, # Optional: You don't need to pass any topic at all url=KAFKA_URL, # Url string provided by heroku ssl_cert=KAFKA_CLIENT_CERT, # Client cert string ssl_key=KAFKA_CLIENT_CERT_KEY, # Client cert key string ssl_ca=KAFKA_TRUSTED_CERT, # Client trusted cert string prefix=KAFKA_PREFIX, # Prefix provided by heroku, auto_offset_reset="smallest", max_poll_records=500, enable_auto_commit=True, auto_commit_interval_ms=10, #group_id=KAFKA_GROUP_ID, api_version=(0, 9)) def sendToKafka(data): """ The .send method will automatically prefix your topic with the KAFKA_PREFIX NOTE: If the message doesn't seem to be sending try `producer.flush()` to force send. """ producer.send(KAKFA_TOPIC, data, partition=0)