def setUp(self):
self.kafka_monitor = KafkaMonitor("localsettings.py")
new_settings = self.kafka_monitor.wrapper.load("localsettings.py")
new_settings['KAFKA_INCOMING_TOPIC'] = "demo.incoming_test"
new_settings['KAFKA_CONSUMER_TIMEOUT'] = 5000
new_settings['STATS_TOTAL'] = False
new_settings['STATS_PLUGINS'] = False
new_settings['PLUGINS'] = {
'plugins.scraper_handler.ScraperHandler': None,
'plugins.action_handler.ActionHandler': None,
'tests.online.CustomHandler': 100,
}
self.kafka_monitor.wrapper.load = MagicMock(return_value=new_settings)
self.kafka_monitor.setup()
@MethodTimer.timeout(10, False)
def timer():
self.kafka_monitor._setup_kafka()
return True
retval = timer()
if not retval:
self.fail("Unable to connect to Kafka")
self.kafka_monitor._load_plugins()
self.kafka_monitor._setup_stats()
self.redis_conn = redis.Redis(
host=self.kafka_monitor.settings['REDIS_HOST'],
port=self.kafka_monitor.settings['REDIS_PORT'],
db=self.kafka_monitor.settings['REDIS_DB'])
def setUp(self):
self.kafka_monitor = KafkaMonitor("localsettings.py")
new_settings = self.kafka_monitor.wrapper.load("localsettings.py")
new_settings['KAFKA_INCOMING_TOPIC'] = "demo.incoming_test"
new_settings['STATS_TOTAL'] = False
new_settings['STATS_PLUGINS'] = False
new_settings['PLUGINS'] = {
'plugins.scraper_handler.ScraperHandler': None,
'plugins.action_handler.ActionHandler': None,
'tests.tests_online.CustomHandler': 100,
}
self.kafka_monitor.wrapper.load = MagicMock(return_value=new_settings)
self.kafka_monitor.setup()
self.kafka_monitor._setup_kafka()
self.kafka_monitor._load_plugins()
self.kafka_monitor._setup_stats()
self.kafka_monitor.logger = MagicMock()
self.redis_conn = redis.Redis(
host=self.kafka_monitor.settings['REDIS_HOST'],
port=self.kafka_monitor.settings['REDIS_PORT'])
def main():
kafka_monitor = KafkaMonitor(sys.argv[1:])
kafka_monitor.run()
def setUp(self):
self.kafka_monitor = KafkaMonitor("settings.py", True)
self.kafka_monitor.settings = self.kafka_monitor.wrapper.load(
"settings.py")
self.kafka_monitor.logger = MagicMock()
#!/usr/bin/python
from kafka.client import KafkaClient
from kafka.producer import SimpleProducer
import json
import importlib
from kafka_monitor import KafkaMonitor
__KafkaMonitor = KafkaMonitor(settings_name='localsettings.py')
__KafkaMonitor.setup()
def _feed(settings_file, json_item):
settings = importlib.import_module(settings_file[:-3])
kafka_conn = KafkaClient(settings.KAFKA_HOSTS)
topic = settings.KAFKA_INCOMING_TOPIC
producer = SimpleProducer(kafka_conn)
print "=> feeding JSON request into {0}...".format(topic)
print json.dumps(json_item, indent=4)
kafka_conn.ensure_topic_exists(topic)
producer.send_messages(topic, json.dumps(json_item))
print "=> done feeding request."
def feed(settings_file, json_req):
print('json_req==', json_req)
try:
parsed = json.loads(json_req)
except ValueError:
