class TestKafkaMonitor(TestCase):
def setUp(self):
self.kafka_monitor = KafkaMonitor("localsettings.py")
new_settings = self.kafka_monitor.wrapper.load("localsettings.py")
new_settings['KAFKA_INCOMING_TOPIC'] = "demo.incoming_test"
new_settings['KAFKA_CONSUMER_TIMEOUT'] = 5000
new_settings['STATS_TOTAL'] = False
new_settings['STATS_PLUGINS'] = False
new_settings['PLUGINS'] = {
'plugins.scraper_handler.ScraperHandler': None,
'plugins.action_handler.ActionHandler': None,
'tests.online.CustomHandler': 100,
}
self.kafka_monitor.wrapper.load = MagicMock(return_value=new_settings)
self.kafka_monitor.setup()
@MethodTimer.timeout(10, False)
def timer():
self.kafka_monitor._setup_kafka()
return True
retval = timer()
if not retval:
self.fail("Unable to connect to Kafka")
self.kafka_monitor._load_plugins()
self.kafka_monitor._setup_stats()
self.redis_conn = redis.Redis(
host=self.kafka_monitor.settings['REDIS_HOST'],
port=self.kafka_monitor.settings['REDIS_PORT'],
db=self.kafka_monitor.settings['REDIS_DB'],
password=self.kafka_monitor.settings['REDIS_PASSWORD'],
decode_responses=True)
def test_feed(self):
json_req = "{\"uuid\":\"mytestid\"," \
"\"appid\":\"testapp\",\"action\":\"info\",\"spiderid\":\"link\"}"
parsed = json.loads(json_req)
# ensure the group id is present so we pick up the 1st message
self.kafka_monitor._process_messages()
self.kafka_monitor.feed(parsed)
def test_run(self):
self.kafka_monitor._process_messages()
self.assertTrue(self.redis_conn.exists("cluster:test"))
value = self.redis_conn.get("cluster:test")
self.assertEqual(value, "mytestid")
def tearDown(self):
self.redis_conn.delete("cluster:test")
self.kafka_monitor.close()
class TestKafkaMonitor(TestCase):
def setUp(self):
self.kafka_monitor = KafkaMonitor("localsettings.py")
new_settings = self.kafka_monitor.wrapper.load("localsettings.py")
new_settings['KAFKA_INCOMING_TOPIC'] = "demo.incoming_test"
new_settings['STATS_TOTAL'] = False
new_settings['STATS_PLUGINS'] = False
new_settings['PLUGINS'] = {
'plugins.scraper_handler.ScraperHandler': None,
'plugins.action_handler.ActionHandler': None,
'tests.tests_online.CustomHandler': 100,
}
self.kafka_monitor.wrapper.load = MagicMock(return_value=new_settings)
self.kafka_monitor.setup()
self.kafka_monitor._setup_kafka()
self.kafka_monitor._load_plugins()
self.kafka_monitor._setup_stats()
self.kafka_monitor.logger = MagicMock()
print self.kafka_monitor.settings['KAFKA_HOSTS']
print self.kafka_monitor.settings['REDIS_HOST']
print self.kafka_monitor.settings['REDIS_PORT']
self.redis_conn = redis.Redis(
host=self.kafka_monitor.settings['REDIS_HOST'],
port=self.kafka_monitor.settings['REDIS_PORT'])
def test_feed(self):
json_req = "{\"uuid\":\"mytestid\"," \
"\"appid\":\"testapp\",\"action\":\"info\",\"spiderid\":\"link\"}"
parsed = json.loads(json_req)
self.kafka_monitor.feed(parsed)
def test_run(self):
self.kafka_monitor._process_messages()
self.assertTrue(self.redis_conn.exists("cluster:test"))
value = self.redis_conn.get("cluster:test")
self.assertEqual(value, "mytestid")
def tearDown(self):
self.redis_conn.delete("cluster:test")
class TestKafkaMonitor(TestCase):
def setUp(self):
self.kafka_monitor = KafkaMonitor("localsettings.py")
new_settings = self.kafka_monitor.wrapper.load("localsettings.py")
new_settings['KAFKA_INCOMING_TOPIC'] = "demo.incoming_test"
new_settings['STATS_TOTAL'] = False
new_settings['STATS_PLUGINS'] = False
new_settings['PLUGINS'] = {
'plugins.scraper_handler.ScraperHandler': None,
'plugins.action_handler.ActionHandler': None,
'tests.tests_online.CustomHandler': 100,
}
self.kafka_monitor.wrapper.load = MagicMock(return_value=new_settings)
self.kafka_monitor.setup()
self.kafka_monitor._setup_kafka()
self.kafka_monitor._load_plugins()
self.kafka_monitor._setup_stats()
self.kafka_monitor.logger = MagicMock()
self.redis_conn = redis.Redis(
host=self.kafka_monitor.settings['REDIS_HOST'],
port=self.kafka_monitor.settings['REDIS_PORT'])
def test_feed(self):
json_req = "{\"uuid\":\"mytestid\"," \
"\"appid\":\"testapp\",\"action\":\"info\",\"spiderid\":\"link\"}"
parsed = json.loads(json_req)
self.kafka_monitor.feed(parsed)
def test_run(self):
self.kafka_monitor._process_messages()
self.assertTrue(self.redis_conn.exists("cluster:test"))
value = self.redis_conn.get("cluster:test")
self.assertEqual(value, "mytestid")
def tearDown(self):
self.redis_conn.delete("cluster:test")
#!/usr/bin/python
from kafka.client import KafkaClient
from kafka.producer import SimpleProducer
import json
import importlib
from kafka_monitor import KafkaMonitor
__KafkaMonitor = KafkaMonitor(settings_name='localsettings.py')
__KafkaMonitor.setup()
def _feed(settings_file, json_item):
settings = importlib.import_module(settings_file[:-3])
kafka_conn = KafkaClient(settings.KAFKA_HOSTS)
topic = settings.KAFKA_INCOMING_TOPIC
producer = SimpleProducer(kafka_conn)
print "=> feeding JSON request into {0}...".format(topic)
print json.dumps(json_item, indent=4)
kafka_conn.ensure_topic_exists(topic)
producer.send_messages(topic, json.dumps(json_item))
print "=> done feeding request."
def feed(settings_file, json_req):
print('json_req==', json_req)
try:
parsed = json.loads(json_req)
except ValueError:
