class ConsumerExample(object):
def __init__(self, consul_endpoint, topic="voltha.heartbeat", runtime=60):
self.topic = topic
self.runtime = runtime
self.kafka_endpoint = get_endpoint_from_consul(consul_endpoint,
'kafka')
self._client = KafkaClient(self.kafka_endpoint)
self._consumer_list = [] # List of consumers
# List of deferred returned from consumers' start() methods
self._consumer_d_list = []
@inlineCallbacks
def start(self):
partitions = []
try:
while not partitions:
yield self._client.load_metadata_for_topics(self.topic)
e = self._client.metadata_error_for_topic(self.topic)
if e:
log.warning('no-metadata-for-topic', error=e,
topic=self.topic)
else:
partitions = self._client.topic_partitions[self.topic]
except KafkaUnavailableError:
log.error("unable-to-communicate-with-Kafka-brokers")
self.stop()
def _note_consumer_stopped(result, consumer):
log.info('consumer-stopped', consumer=consumer,
result=result)
for partition in partitions:
c = Consumer(self._client, self.topic, partition,
self.msg_processor)
self._consumer_list.append(c)
# log.info('consumer-started', topic=self.topic, partition=partition)
d = c.start(OFFSET_LATEST)
d.addBoth(_note_consumer_stopped, c)
self._consumer_d_list.append(d)
# Stop ourselves after we've run the allotted time
reactor.callLater(self.runtime, self.stop)
def stop(self):
log.info("\n")
log.info('end-of-execution-stopping-consumers')
# Ask each of our consumers to stop. When a consumer fully stops, it
# fires the deferred returned from its start() method. We saved all
# those deferreds away (above, in start()) in self._consumer_d_list,
# so now we'll use a DeferredList to wait for all of them...
for consumer in self._consumer_list:
consumer.stop()
dl = DeferredList(self._consumer_d_list)
# Once the consumers are all stopped, then close our client
def _stop_client(result):
if isinstance(result, Failure):
log.error('error', result=result)
else:
log.info('all-consumers-stopped', client=self._client)
self._client.close()
return result
dl.addBoth(_stop_client)
# And once the client is shutdown, stop the reactor
def _stop_reactor(result):
reactor.stop()
return result
dl.addBoth(_stop_reactor)
def msg_processor(self, consumer, msglist):
for msg in msglist:
log.info(msg)
class DashDaemon(object):
def __init__(self, consul_endpoint, kafka_endpoint, grafana_url, topic="voltha.heartbeat"):
#logging.basicConfig(
# format='%(asctime)s:%(name)s:' +
# '%(levelname)s:%(process)d:%(message)s',
# level=logging.INFO
#)
self.dash_meta = {}
self.timer_resolution = 10
self.timer_duration = 600
self.topic = topic
self.dash_template = DashTemplate(grafana_url)
self.grafana_url = grafana_url
self.kafka_endpoint = kafka_endpoint
self.consul_endpoint = consul_endpoint
if kafka_endpoint.startswith('@'):
retrys = 10
while True:
try:
self.kafka_endpoint = get_endpoint_from_consul(
self.consul_endpoint, kafka_endpoint[1:])
break
except:
log.error("unable-to-communicate-with-consul")
self.stop()
retrys -= 1
if retrys == 0:
log.error("unable-to-communicate-with-consul")
self.stop()
time.sleep(10)
self.on_start_callback = None
self._client = KafkaClient(self.kafka_endpoint)
self._consumer_list = [] # List of consumers
# List of deferred returned from consumers' start() methods
self._consumer_d_list = []
def set_on_start_callback(self, on_start_callback):
# This function is currently unused, future requirements.
self.on_start_callback = on_start_callback
return self
@inlineCallbacks
def start(self):
partitions = []
try:
while not partitions:
yield self._client.load_metadata_for_topics(self.topic)
#self._client.load_metadata_for_topics(self.topic)
e = self._client.metadata_error_for_topic(self.topic)
if e:
log.warning('no-metadata-for-topic', error=e,
topic=self.topic)
else:
partitions = self._client.topic_partitions[self.topic]
break
time.sleep(20)
except KafkaUnavailableError:
log.error("unable-to-communicate-with-Kafka-brokers")
self.stop()
def _note_consumer_stopped(result, consumer):
log.info('consumer-stopped', consumer=consumer,
result=result)
for partition in partitions:
c = Consumer(self._client, self.topic, partition,
self.msg_processor)
self._consumer_list.append(c)
log.info('consumer-started', topic=self.topic, partition=partition)
d = c.start(OFFSET_LATEST)
d.addBoth(_note_consumer_stopped, c)
self._consumer_d_list.append(d)
# Now read the list of existing dashboards from Grafana and create the
# dictionary of dashboard timers. If we've crashed there will be
# dashboards there. Just add them and if they're no longer valid
# they'll be deleted. If they are valid then they'll persist.
#print("Starting main loop")
try:
retrys = 10
while True:
r = requests.get(self.grafana_url + "/datasources")
if r.status_code == requests.codes.ok:
break
else:
retrys -= 1
if retrys == 0:
log.error("unable-to-communicate-with-grafana")
self.stop()
time.sleep(10)
j = r.json()
data_source = False
for i in j:
if i["name"] == "Voltha Stats":
data_source = True
break
if not data_source:
r = requests.post(self.grafana_url + "/datasources",
data = {"name":"Voltha Stats","type":"graphite",
"access":"proxy","url":"http://localhost:81"})
log.info('data-source-added',status=r.status_code, text=r.text)
retrys = 10
while True:
r = requests.get(self.grafana_url + "/search?")
if r.status_code == requests.codes.ok:
break
else:
retrys -= 1
if retrys == 0:
log.error("unable-to-communicate-with-grafana")
self.stop()
time.sleep(10)
j = r.json()
for i in j:
# Look for dashboards that have a title of *olt.[[:hexidgit:]].
# These will be the ones of interest. Others should just be left
# alone.
#print(i['title'])
match = re.search(r'(.*olt)\.([0-9a-zA-Z]+)',i['title'])
if match and match.lastindex > 0:
#print(match.group(1), match.group(2))
self.dash_meta[match.group(2)] = {}
self.dash_meta[match.group(2)]['timer'] = self.timer_duration # 10 min
self.dash_meta[match.group(2)]['device'] = match.group(1)
self.dash_meta[match.group(2)]['created'] = False
self.dash_meta[match.group(2)]['ports'] = {}
# TODO: We should really capture all of the chart data
# including the rows, panels, and data points being logged.
# This is good enough for now though to determine if
# there's already a dashboard for a given device.
def countdown_processor():
# Called every X (timer_resolution) seconds to count down each of the
# dash timers. If a timer reaches 0 the corresponding
# dashboard is removed.
#log.info("Counting down.")
try:
for dashboard in self.dash_meta.keys():
#print("Counting down %s." %dashboard)
# Issue a log if the counter decrement is somewhat relevant
if(self.dash_meta[dashboard]['timer'] % 100 == 0 and \
self.dash_meta[dashboard]['timer'] != self.timer_duration):
log.info("counting-down",dashboard=dashboard,
timer=self.dash_meta[dashboard]['timer'])
self.dash_meta[dashboard]['timer'] -= self.timer_resolution
if self.dash_meta[dashboard]['timer'] <= 0:
# Delete the dashboard here
log.info("FIXME:-Should-delete-the-dashboard-here",
dashboard=dashboard)
pass
except:
e = sys.exc_info()
log.error("error", error=e)
# Start the dashboard countdown processor
log.info("starting-countdown-processor")
lc = LoopingCall(countdown_processor)
lc.start(self.timer_resolution)
@inlineCallbacks
def template_checker():
try:
# Called every so often (timer_resolution seconds because it's
# convenient) to check if a template dashboard has been defined
# in Grafana. If it has been, replace the built in template
# with the one provided
r = requests.get(self.grafana_url + "/search?query=template")
db = r.json()
if len(db) == 1:
# Apply the template
yield self.dash_template.apply_template(db[0])
elif len(db) != 0:
# This is an error, log it.
log.warning("More-than-one-template-provided-ignoring")
except:
e = sys.exc_info()
log.error("error", error=e)
log.info("starting-template-checker")
lc = LoopingCall(template_checker)
lc.start(self.timer_resolution)
except:
e = sys.exc_info()
log.error("error", error=e)
def stop(self):
log.info("\n")
log.info('end-of-execution-stopping-consumers')
# Ask each of our consumers to stop. When a consumer fully stops, it
# fires the deferred returned from its start() method. We saved all
# those deferreds away (above, in start()) in self._consumer_d_list,
# so now we'll use a DeferredList to wait for all of them...
for consumer in self._consumer_list:
consumer.stop()
dl = DeferredList(self._consumer_d_list)
# Once the consumers are all stopped, then close our client
def _stop_client(result):
if isinstance(result, Failure):
log.error('error', result=result)
else:
log.info('all-consumers-stopped', client=self._client)
self._client.close()
return result
dl.addBoth(_stop_client)
# And once the client is shutdown, stop the reactor
def _stop_reactor(result):
reactor.stop()
return result
dl.addBoth(_stop_reactor)
def check_for_dashboard(self, msg):
need_dash = {}
done = {}
# Extract the ids for all olt(s) in the message and do one of 2
# things. If it exists, reset the meta_data timer for the dashboard and
# if it doesn't exist add it to the array of needed dashboards.
metrics = json.loads(getattr(msg.message,'value'))['prefixes']
for key in metrics.keys():
match = re.search(r'voltha\.(.*olt)\.([0-9a-zA-Z]+)\.(.*)',key)
if match and match.lastindex > 1:
if match.group(2) in self.dash_meta and match.group(2) not in done:
# Update the delete countdown timer
self.dash_meta[match.group(2)]['timer'] = self.timer_duration
done[match.group(2)] = True
# Issue a log if the reset if somewhat relevant.
if self.dash_meta[match.group(2)]['timer'] < \
self.timer_duration - self.timer_resolution:
log.info("reset-timer",device=match.group(2))
#print("reset timer for: %s" %match.group(2))
else:
# No dahsboard exists,
need_dash[key] = metrics[key]
return need_dash
def create_dashboards(self, createList):
dataIds = "ABCDEFGHIJKLMNOP"
for dash in createList:
#log.info("creating a dashboard for: %s" % self.dash_meta[dash])
# Create one row per "interface"
# Create one panel per metric type for the time being it's one
# panel for byte stats and one panel for packet stats.
newDash = json.loads(self.dash_template.dashBoard)
newDash['dashboard']['title'] = self.dash_meta[dash]['device'] + \
'.' + dash
# The port is the main grouping attribute
for port in self.dash_meta[dash]['ports']:
# Add in the rows for the port specified by the template
for row in self.dash_template.rows:
r = json.loads(self.dash_template.dashRow)
r['title'] = re.sub(r'%port%',port, row['title'])
p = {}
# Add the panels to the row per the template
panelId = 1
for panel in self.dash_template.panels:
p = json.loads(self.dash_template.dashPanel)
p['id'] = panelId
panelId += 1
p['title'] = re.sub(r'%port%', port.upper(), panel['title'])
t = {}
dataId = 0
# Add the targets to the panel
for dpoint in sorted(self.dash_meta[dash]['ports'][port]):
if dpoint in panel:
t['refId'] = dataIds[dataId]
db = re.sub(r'%port%',port,panel[dpoint])
db = re.sub(r'%device%',
self.dash_meta[dash]['device'],db)
db = re.sub(r'%deviceId%', dash,db)
t['target'] = db
p['targets'].append(t.copy())
dataId += 1
r['panels'].append(p.copy())
newDash['dashboard']['rows'].append(r.copy())
#print("NEW DASHBOARD: ",json.dumps(newDash))
#print(r.json())
r = \
requests.post(self.grafana_url + "/dashboards/db",
json=newDash)
self.dash_meta[dash]['slug'] = r.json()['slug']
self.dash_meta[dash]['created'] = True
log.info("created-dashboard", slug=self.dash_meta[dash]['slug'])
def msg_processor(self, consumer, msglist):
try:
createList = []
for msg in msglist:
# Reset the timer for existing dashboards and get back a dict
# of of dashboards to create if any.
need_dash = self.check_for_dashboard(msg)
# Now populate the meta data for all missing dashboards
for key in need_dash.keys():
match = re.search(r'voltha\.(.*olt)\.([0-9a-zA-Z]+)\.(.*)',key)
if match and match.lastindex > 2:
if match.group(2) in self.dash_meta:
# The entry will have been created when the first
# port in the record was encountered so just
# populate the metrics and port info.
# TODO: The keys below are the names of the metrics
# that are in the Kafka record. This auto-discovery
# is fine if all that's needed are raw metrics. If
# metrics are "cooked" by a downstream process and
# subsequently fed to graphite/carbon without being
# re-posted to Kafka, discovery becomes impossible.
# In those cases and in cases where finer grain
# control of what's displayed is required, a config
# file would be necessary.
self.dash_meta[match.group(2)]['ports'][match.group(3)] = \
need_dash[key]['metrics'].keys()
else:
# Not there, create a meta-data record for the
# device and add this port.
#print("Adding meta data for", match.group(1),
# match.group(2))
createList.append(match.group(2))
self.dash_meta[match.group(2)] = {}
self.dash_meta[match.group(2)]['timer'] = 600
self.dash_meta[match.group(2)]['device'] = match.group(1)
self.dash_meta[match.group(2)]['created'] = False
self.dash_meta[match.group(2)]['ports'] = {}
#print("Adding port", match.group(3), "to", match.group(1),
# match.group(2))
self.dash_meta[match.group(2)]['ports'][match.group(3)] = \
need_dash[key]['metrics'].keys()
# Now go ahead and create the dashboards using the meta data that
# wwas just populated for them.
if len(createList) != 0: # Create any missing dashboards.
self.create_dashboards(createList)
except:
e = sys.exc_info()
log.error("error", error=e)
class DashDaemon(object):
def __init__(self, consul_endpoint, grafana_url, topic="voltha.heartbeat"):
#logging.basicConfig(
# format='%(asctime)s:%(name)s:' +
# '%(levelname)s:%(process)d:%(message)s',
# level=logging.INFO
#)
self.dash_meta = {}
self.timer_resolution = 10
self.timer_duration = 600
self.topic = topic
self.dash_template = DashTemplate(grafana_url)
self.grafana_url = grafana_url
self.kafka_endpoint = get_endpoint_from_consul(consul_endpoint,
'kafka')
# print('kafka endpoint: ', self.kafka_endpoint)
self.on_start_callback = None
self._client = KafkaClient(self.kafka_endpoint)
self._consumer_list = [] # List of consumers
# List of deferred returned from consumers' start() methods
self._consumer_d_list = []
def set_on_start_callback(self, on_start_callback):
# This function is currently unused, future requirements.
self.on_start_callback = on_start_callback
return self
@inlineCallbacks
def start(self):
partitions = []
try:
while not partitions:
yield self._client.load_metadata_for_topics(self.topic)
e = self._client.metadata_error_for_topic(self.topic)
if e:
log.warning('no-metadata-for-topic',
error=e,
topic=self.topic)
else:
partitions = self._client.topic_partitions[self.topic]
except KafkaUnavailableError:
log.error("unable-to-communicate-with-Kafka-brokers")
self.stop()
def _note_consumer_stopped(result, consumer):
log.info('consumer-stopped', consumer=consumer, result=result)
for partition in partitions:
c = Consumer(self._client, self.topic, partition,
self.msg_processor)
self._consumer_list.append(c)
log.info('consumer-started', topic=self.topic, partition=partition)
d = c.start(OFFSET_LATEST)
d.addBoth(_note_consumer_stopped, c)
self._consumer_d_list.append(d)
# Now read the list of existing dashboards from Grafana and create the
# dictionary of dashboard timers. If we've crashed there will be
# dashboards there. Just add them and if they're no longer valid
# they'll be deleted. If they are valid then they'll persist.
#print("Starting main loop")
try:
r = requests.get(self.grafana_url + "/search?")
j = r.json()
for i in j:
# Look for dashboards that have a title of *olt.[[:hexidgit:]].
# These will be the ones of interest. Others should just be left
# alone.
#print(i['title'])
match = re.search(r'(.*olt)\.([0-9a-zA-Z]+)', i['title'])
if match and match.lastindex > 0:
#print(match.group(1), match.group(2))
self.dash_meta[match.group(2)] = {}
self.dash_meta[match.group(
2)]['timer'] = self.timer_duration # 10 min
self.dash_meta[match.group(2)]['device'] = match.group(1)
self.dash_meta[match.group(2)]['created'] = False
self.dash_meta[match.group(2)]['ports'] = {}
# TODO: We should really capture all of the chart data
# including the rows, panels, and data points being logged.
# This is good enough for now though to determine if
# there's already a dashboard for a given device.
def countdown_processor():
# Called every X (timer_resolution) seconds to count down each of the
# dash timers. If a timer reaches 0 the corresponding
# dashboard is removed.
#log.info("Counting down.")
try:
for dashboard in self.dash_meta.keys():
#print("Counting down %s." %dashboard)
# Issue a log if the counter decrement is somewhat relevant
if(self.dash_meta[dashboard]['timer'] % 100 == 0 and \
self.dash_meta[dashboard]['timer'] != self.timer_duration):
log.info("counting-down",
dashboard=dashboard,
timer=self.dash_meta[dashboard]['timer'])
self.dash_meta[dashboard][
'timer'] -= self.timer_resolution
if self.dash_meta[dashboard]['timer'] <= 0:
# Delete the dashboard here
log.info("FIXME:-Should-delete-the-dashboard-here",
dashboard=dashboard)
pass
except:
e = sys.exc_info()
log.error("error", error=e)
# Start the dashboard countdown processor
log.info("starting-countdown-processor")
lc = LoopingCall(countdown_processor)
lc.start(self.timer_resolution)
@inlineCallbacks
def template_checker():
try:
# Called every so often (timer_resolution seconds because it's
# convenient) to check if a template dashboard has been defined
# in Grafana. If it has been, replace the built in template
# with the one provided
r = requests.get(self.grafana_url +
"/search?query=template")
db = r.json()
if len(db) == 1:
# Apply the template
yield self.dash_template.apply_template(db[0])
elif len(db) != 0:
# This is an error, log it.
log.warning("More-than-one-template-provided-ignoring")
except:
e = sys.exc_info()
log.error("error", error=e)
log.info("starting-template-checker")
lc = LoopingCall(template_checker)
lc.start(self.timer_resolution)
except:
e = sys.exc_info()
log.error("error", error=e)
def stop(self):
log.info("\n")
log.info('end-of-execution-stopping-consumers')
# Ask each of our consumers to stop. When a consumer fully stops, it
# fires the deferred returned from its start() method. We saved all
# those deferreds away (above, in start()) in self._consumer_d_list,
# so now we'll use a DeferredList to wait for all of them...
for consumer in self._consumer_list:
consumer.stop()
dl = DeferredList(self._consumer_d_list)
# Once the consumers are all stopped, then close our client
def _stop_client(result):
if isinstance(result, Failure):
log.error('error', result=result)
else:
log.info('all-consumers-stopped', client=self._client)
self._client.close()
return result
dl.addBoth(_stop_client)
# And once the client is shutdown, stop the reactor
def _stop_reactor(result):
reactor.stop()
return result
dl.addBoth(_stop_reactor)
def check_for_dashboard(self, msg):
need_dash = {}
done = {}
# Extract the ids for all olt(s) in the message and do one of 2
# things. If it exists, reset the meta_data timer for the dashboard and
# if it doesn't exist add it to the array of needed dashboards.
metrics = json.loads(getattr(msg.message, 'value'))['prefixes']
for key in metrics.keys():
match = re.search(r'voltha\.(.*olt)\.([0-9a-zA-Z]+)\.(.*)', key)
if match and match.lastindex > 1:
if match.group(2) in self.dash_meta and match.group(
2) not in done:
# Update the delete countdown timer
self.dash_meta[match.group(
2)]['timer'] = self.timer_duration
done[match.group(2)] = True
# Issue a log if the reset if somewhat relevant.
if self.dash_meta[match.group(2)]['timer'] < \
self.timer_duration - self.timer_resolution:
log.info("reset-timer", device=match.group(2))
#print("reset timer for: %s" %match.group(2))
else:
# No dahsboard exists,
need_dash[key] = metrics[key]
return need_dash
def create_dashboards(self, createList):
dataIds = "ABCDEFGHIJKLMNOP"
for dash in createList:
#log.info("creating a dashboard for: %s" % self.dash_meta[dash])
# Create one row per "interface"
# Create one panel per metric type for the time being it's one
# panel for byte stats and one panel for packet stats.
newDash = json.loads(self.dash_template.dashBoard)
newDash['dashboard']['title'] = self.dash_meta[dash]['device'] + \
'.' + dash
# The port is the main grouping attribute
for port in self.dash_meta[dash]['ports']:
# Add in the rows for the port specified by the template
for row in self.dash_template.rows:
r = json.loads(self.dash_template.dashRow)
r['title'] = re.sub(r'%port%', port, row['title'])
p = {}
# Add the panels to the row per the template
panelId = 1
for panel in self.dash_template.panels:
p = json.loads(self.dash_template.dashPanel)
p['id'] = panelId
panelId += 1
p['title'] = re.sub(r'%port%', port.upper(),
panel['title'])
t = {}
dataId = 0
# Add the targets to the panel
for dpoint in sorted(
self.dash_meta[dash]['ports'][port]):
if dpoint in panel:
t['refId'] = dataIds[dataId]
db = re.sub(r'%port%', port, panel[dpoint])
db = re.sub(r'%device%',
self.dash_meta[dash]['device'], db)
db = re.sub(r'%deviceId%', dash, db)
t['target'] = db
p['targets'].append(t.copy())
dataId += 1
r['panels'].append(p.copy())
newDash['dashboard']['rows'].append(r.copy())
#print("NEW DASHBOARD: ",json.dumps(newDash))
#print(r.json())
r = \
requests.post(self.grafana_url + "/dashboards/db",
json=newDash)
self.dash_meta[dash]['slug'] = r.json()['slug']
self.dash_meta[dash]['created'] = True
log.info("created-dashboard", slug=self.dash_meta[dash]['slug'])
def msg_processor(self, consumer, msglist):
try:
createList = []
for msg in msglist:
# Reset the timer for existing dashboards and get back a dict
# of of dashboards to create if any.
need_dash = self.check_for_dashboard(msg)
# Now populate the meta data for all missing dashboards
for key in need_dash.keys():
match = re.search(r'voltha\.(.*olt)\.([0-9a-zA-Z]+)\.(.*)',
key)
if match and match.lastindex > 2:
if match.group(2) in self.dash_meta:
# The entry will have been created when the first
# port in the record was encountered so just
# populate the metrics and port info.
# TODO: The keys below are the names of the metrics
# that are in the Kafka record. This auto-discovery
# is fine if all that's needed are raw metrics. If
# metrics are "cooked" by a downstream process and
# subsequently fed to graphite/carbon without being
# re-posted to Kafka, discovery becomes impossible.
# In those cases and in cases where finer grain
# control of what's displayed is required, a config
# file would be necessary.
self.dash_meta[match.group(2)]['ports'][match.group(3)] = \
need_dash[key]['metrics'].keys()
else:
# Not there, create a meta-data record for the
# device and add this port.
#print("Adding meta data for", match.group(1),
# match.group(2))
createList.append(match.group(2))
self.dash_meta[match.group(2)] = {}
self.dash_meta[match.group(2)]['timer'] = 600
self.dash_meta[match.group(
2)]['device'] = match.group(1)
self.dash_meta[match.group(2)]['created'] = False
self.dash_meta[match.group(2)]['ports'] = {}
#print("Adding port", match.group(3), "to", match.group(1),
# match.group(2))
self.dash_meta[match.group(2)]['ports'][match.group(3)] = \
need_dash[key]['metrics'].keys()
# Now go ahead and create the dashboards using the meta data that
# wwas just populated for them.
if len(createList) != 0: # Create any missing dashboards.
self.create_dashboards(createList)
except:
e = sys.exc_info()
log.error("error", error=e)
class ConsumerExample(object):
def __init__(self, consul_endpoint, topic="voltha.heartbeat", runtime=60):
self.topic = topic
self.runtime = runtime
self.kafka_endpoint = get_endpoint_from_consul(consul_endpoint,
'kafka')
self._client = KafkaClient(self.kafka_endpoint)
self._consumer_list = [] # List of consumers
# List of deferred returned from consumers' start() methods
self._consumer_d_list = []
@inlineCallbacks
def start(self):
partitions = []
try:
while not partitions:
yield self._client.load_metadata_for_topics(self.topic)
e = self._client.metadata_error_for_topic(self.topic)
if e:
log.warning('no-metadata-for-topic',
error=e,
topic=self.topic)
else:
partitions = self._client.topic_partitions[self.topic]
except KafkaUnavailableError:
log.error("unable-to-communicate-with-Kafka-brokers")
self.stop()
def _note_consumer_stopped(result, consumer):
log.info('consumer-stopped', consumer=consumer, result=result)
for partition in partitions:
c = Consumer(self._client, self.topic, partition,
self.msg_processor)
self._consumer_list.append(c)
# log.info('consumer-started', topic=self.topic, partition=partition)
d = c.start(OFFSET_LATEST)
d.addBoth(_note_consumer_stopped, c)
self._consumer_d_list.append(d)
# Stop ourselves after we've run the allotted time
reactor.callLater(self.runtime, self.stop)
def stop(self):
log.info("\n")
log.info('end-of-execution-stopping-consumers')
# Ask each of our consumers to stop. When a consumer fully stops, it
# fires the deferred returned from its start() method. We saved all
# those deferreds away (above, in start()) in self._consumer_d_list,
# so now we'll use a DeferredList to wait for all of them...
for consumer in self._consumer_list:
consumer.stop()
dl = DeferredList(self._consumer_d_list)
# Once the consumers are all stopped, then close our client
def _stop_client(result):
if isinstance(result, Failure):
log.error('error', result=result)
else:
log.info('all-consumers-stopped', client=self._client)
self._client.close()
return result
dl.addBoth(_stop_client)
# And once the client is shutdown, stop the reactor
def _stop_reactor(result):
reactor.stop()
return result
dl.addBoth(_stop_reactor)
def msg_processor(self, consumer, msglist):
for msg in msglist:
log.info(msg)
class IKafkaMessagingProxy(object):
_kafka_messaging_instance = None
def __init__(self, kafka_host_port, kv_store, default_topic, target_cls):
"""
Initialize the kafka proxy. This is a singleton (may change to
non-singleton if performance is better)
:param kafka_host_port: Kafka host and port
:param kv_store: Key-Value store
:param default_topic: Default topic to subscribe to
:param target_cls: target class - method of that class is invoked
when a message is received on the default_topic
"""
# return an exception if the object already exist
if IKafkaMessagingProxy._kafka_messaging_instance:
raise Exception('Singleton-exist', cls=IKafkaMessagingProxy)
log.debug("Initializing-KafkaProxy")
self.kafka_host_port = kafka_host_port
self.kv_store = kv_store
self.default_topic = default_topic
self.target_cls = target_cls
self.topic_target_cls_map = {}
self.topic_consumer_map = {}
self.topic_callback_map = {}
self.subscribers = {}
self.kafka_client = None
self.kafka_proxy = None
self.transaction_id_deferred_map = {}
self.received_msg_queue = DeferredQueue()
self.init_time = 0
self.init_received_time = 0
self.init_resp_time = 0
self.init_received_resp_time = 0
self.num_messages = 0
self.total_time = 0
self.num_responses = 0
self.total_time_responses = 0
log.debug("KafkaProxy-initialized")
def start(self):
try:
# Create the kafka client
# assert self.kafka_host is not None
# assert self.kafka_port is not None
# kafka_host_port = ":".join((self.kafka_host, self.kafka_port))
self.kafka_client = KafkaClient(self.kafka_host_port)
# Get the kafka proxy instance. If it does not exist then
# create it
self.kafka_proxy = get_kafka_proxy()
if self.kafka_proxy == None:
KafkaProxy(kafka_endpoint=self.kafka_host_port).start()
self.kafka_proxy = get_kafka_proxy()
# Subscribe the default topic and target_cls
self.topic_target_cls_map[self.default_topic] = self.target_cls
# Start the queue to handle incoming messages
reactor.callLater(0, self._received_message_processing_loop)
# Start listening for incoming messages
reactor.callLater(0,
self.subscribe,
self.default_topic,
target_cls=self.target_cls)
# Setup the singleton instance
IKafkaMessagingProxy._kafka_messaging_instance = self
except Exception as e:
log.exception("Failed-to-start-proxy", e=e)
def stop(self):
"""
Invoked to stop the kafka proxy
:return: None on success, Exception on failure
"""
log.debug("Stopping-messaging-proxy ...")
try:
# Stop all the consumers
deferred_list = []
for key, values in self.topic_consumer_map.iteritems():
deferred_list.extend([c.stop() for c in values])
if not deferred_list:
d = gatherResults(deferred_list)
d.addCallback(lambda result: self.kafka_client.close())
log.debug("Messaging-proxy-stopped.")
except Exception as e:
log.exception("Exception-when-stopping-messaging-proxy:", e=e)
@inlineCallbacks
def create_topic(self, topic):
yield self._wait_until_topic_is_ready(self.kafka_client, topic)
@inlineCallbacks
def _wait_until_topic_is_ready(self, client, topic):
e = True
while e:
yield client.load_metadata_for_topics(topic)
e = client.metadata_error_for_topic(topic)
if e:
log.debug("Topic-not-ready-retrying...", topic=topic)
def _clear_backoff(self):
if self.retries:
log.info('reconnected-to-consul', after_retries=self.retries)
self.retries = 0
def get_target_cls(self):
return self.target_cls
def get_default_topic(self):
return self.default_topic
@inlineCallbacks
def _subscribe(self, topic, offset, callback=None, target_cls=None):
try:
log.debug("subscribing-to-topic-start", topic=topic)
yield self._wait_until_topic_is_ready(self.kafka_client, topic)
partitions = self.kafka_client.topic_partitions[topic]
consumers = []
# First setup the generic callback - all received messages will
# go through that queue
if topic not in self.topic_consumer_map:
log.debug("topic-not-in-consumer-map", topic=topic)
consumers = [
Consumer(self.kafka_client, topic, partition,
self._enqueue_received_message)
for partition in partitions
]
self.topic_consumer_map[topic] = consumers
log.debug("_subscribe",
topic=topic,
consumermap=self.topic_consumer_map)
if target_cls is not None and callback is None:
# Scenario #1
if topic not in self.topic_target_cls_map:
self.topic_target_cls_map[topic] = target_cls
elif target_cls is None and callback is not None:
# Scenario #2
log.debug("custom-callback",
topic=topic,
callback_map=self.topic_callback_map)
if topic not in self.topic_callback_map:
self.topic_callback_map[topic] = [callback]
else:
self.topic_callback_map[topic].extend([callback])
else:
log.warn("invalid-parameters")
def cb_closed(result):
"""
Called when a consumer cleanly stops.
"""
log.debug("Consumers-cleanly-stopped")
def eb_failed(failure):
"""
Called when a consumer fails due to an uncaught exception in the
processing callback or a network error on shutdown. In this case we
simply log the error.
"""
log.warn("Consumers-failed", failure=failure)
for c in consumers:
c.start(offset).addCallbacks(cb_closed, eb_failed)
log.debug("subscribed-to-topic", topic=topic)
returnValue(True)
except Exception as e:
log.exception("Exception-during-subscription", e=e)
returnValue(False)
@inlineCallbacks
def subscribe(self,
topic,
callback=None,
target_cls=None,
max_retry=3,
offset=OFFSET_LATEST):
"""
Scenario 1: invoked to subscribe to a specific topic with a
target_cls to invoke when a message is received on that topic. This
handles the case of request/response where this library performs the
heavy lifting. In this case the m_callback must to be None
Scenario 2: invoked to subscribe to a specific topic with a
specific callback to invoke when a message is received on that topic.
This handles the case where the caller wants to process the message
received itself. In this case the target_cls must to be None
:param topic: topic to subscribe to
:param callback: Callback to invoke when a message is received on
the topic. Either one of callback or target_cls needs can be none
:param target_cls: Target class to use when a message is
received on the topic. There can only be 1 target_cls per topic.
Either one of callback or target_cls needs can be none
:param max_retry: the number of retries before reporting failure
to subscribe. This caters for scenario where the kafka topic is not
ready.
:return: True on success, False on failure
"""
RETRY_BACKOFF = [0.05, 0.1, 0.2, 0.5, 1, 2, 5]
def _backoff(msg, retries):
wait_time = RETRY_BACKOFF[min(retries, len(RETRY_BACKOFF) - 1)]
log.info(msg, retry_in=wait_time)
return asleep(wait_time)
retry = 0
subscribed = False
while not subscribed:
subscribed = yield self._subscribe(topic,
callback=callback,
target_cls=target_cls,
offset=offset)
if subscribed:
returnValue(True)
elif retry > max_retry:
returnValue(False)
else:
_backoff("subscription-not-complete", retry)
retry += 1
# while not self._subscribe(topic, callback=callback,
# target_cls=target_cls):
# if retry > max_retry:
# return False
# else:
# _backoff("subscription-not-complete", retry)
# retry += 1
# return True
def unsubscribe(self, topic):
"""
Invoked when unsubscribing to a topic
:param topic: topic to unsubscibe from
:return: None on success or Exception on failure
"""
log.debug("Unsubscribing-to-topic", topic=topic)
def remove_topic(topic):
if topic in self.topic_consumer_map:
del self.topic_consumer_map[topic]
try:
if topic in self.topic_consumer_map:
consumers = self.topic_consumer_map[topic]
d = gatherResults([c.stop() for c in consumers])
d.addCallback(remove_topic, topic)
log.debug("Unsubscribed-to-topic.", topic=topic)
else:
log.debug("Topic-does-not-exist.", topic=topic)
except Exception as e:
log.exception("Exception-when-stopping-messaging-proxy:", e=e)
@inlineCallbacks
def _enqueue_received_message(self, reactor, message_list):
"""
Internal method to continuously queue all received messaged
irrespective of topic
:param reactor: A requirement by the Twisted Python kafka library
:param message_list: Received list of messages
:return: None on success, Exception on failure
"""
try:
for m in message_list:
log.debug("received-msg", msg=m)
yield self.received_msg_queue.put(m)
except Exception as e:
log.exception("Failed-enqueueing-received-message", e=e)
@inlineCallbacks
def _received_message_processing_loop(self):
"""
Internal method to continuously process all received messages one
at a time
:return: None on success, Exception on failure
"""
while True:
try:
message = yield self.received_msg_queue.get()
yield self._process_message(message)
except Exception as e:
log.exception("Failed-dequeueing-received-message", e=e)
def _to_string(self, unicode_str):
if unicode_str is not None:
if type(unicode_str) == unicode:
return unicode_str.encode('ascii', 'ignore')
else:
return unicode_str
else:
return None
def _format_request(self, rpc, to_topic, reply_topic, **kwargs):
"""
Format a request to send over kafka
:param rpc: Requested remote API
:param to_topic: Topic to send the request
:param reply_topic: Topic to receive the resulting response, if any
:param kwargs: Dictionary of key-value pairs to pass as arguments to
the remote rpc API.
:return: A InterContainerMessage message type on success or None on
failure
"""
try:
transaction_id = uuid4().hex
request = InterContainerMessage()
request_body = InterContainerRequestBody()
request.header.id = transaction_id
request.header.type = MessageType.Value("REQUEST")
request.header.from_topic = reply_topic
request.header.to_topic = to_topic
response_required = False
if reply_topic:
request_body.reply_to_topic = reply_topic
request_body.response_required = True
response_required = True
request.header.timestamp = int(round(time.time() * 1000))
request_body.rpc = rpc
for a, b in kwargs.iteritems():
arg = Argument()
arg.key = a
try:
arg.value.Pack(b)
request_body.args.extend([arg])
except Exception as e:
log.exception("Failed-parsing-value", e=e)
request.body.Pack(request_body)
return request, transaction_id, response_required
except Exception as e:
log.exception("formatting-request-failed",
rpc=rpc,
to_topic=to_topic,
reply_topic=reply_topic,
args=kwargs)
return None, None, None
def _format_response(self, msg_header, msg_body, status):
"""
Format a response
:param msg_header: The header portion of a received request
:param msg_body: The response body
:param status: True is this represents a successful response
:return: a InterContainerMessage message type
"""
try:
assert isinstance(msg_header, Header)
response = InterContainerMessage()
response_body = InterContainerResponseBody()
response.header.id = msg_header.id
response.header.timestamp = int(round(time.time() * 1000))
response.header.type = MessageType.Value("RESPONSE")
response.header.from_topic = msg_header.to_topic
response.header.to_topic = msg_header.from_topic
if msg_body is not None:
response_body.result.Pack(msg_body)
response_body.success = status
response.body.Pack(response_body)
return response
except Exception as e:
log.exception("formatting-response-failed",
header=msg_header,
body=msg_body,
status=status,
e=e)
return None
def _parse_response(self, msg):
try:
message = InterContainerMessage()
message.ParseFromString(msg)
resp = InterContainerResponseBody()
if message.body.Is(InterContainerResponseBody.DESCRIPTOR):
message.body.Unpack(resp)
else:
log.debug("unsupported-msg", msg_type=type(message.body))
return None
log.debug("parsed-response", input=message, output=resp)
return resp
except Exception as e:
log.exception("parsing-response-failed", msg=msg, e=e)
return None
@inlineCallbacks
def _process_message(self, m):
"""
Default internal method invoked for every batch of messages received
from Kafka.
"""
def _toDict(args):
"""
Convert a repeatable Argument type into a python dictionary
:param args: Repeatable core_adapter.Argument type
:return: a python dictionary
"""
if args is None:
return None
result = {}
for arg in args:
assert isinstance(arg, Argument)
result[arg.key] = arg.value
return result
current_time = int(round(time.time() * 1000))
# log.debug("Got Message", message=m)
try:
val = m.message.value
# print m.topic
# Go over customized callbacks first
if m.topic in self.topic_callback_map:
for c in self.topic_callback_map[m.topic]:
yield c(val)
# Check whether we need to process request/response scenario
if m.topic not in self.topic_target_cls_map:
return
# Process request/response scenario
message = InterContainerMessage()
message.ParseFromString(val)
if message.header.type == MessageType.Value("REQUEST"):
# Get the target class for that specific topic
targetted_topic = self._to_string(message.header.to_topic)
msg_body = InterContainerRequestBody()
if message.body.Is(InterContainerRequestBody.DESCRIPTOR):
message.body.Unpack(msg_body)
else:
log.debug("unsupported-msg", msg_type=type(message.body))
return
if targetted_topic in self.topic_target_cls_map:
if msg_body.args:
log.debug("message-body-args-present", body=msg_body)
(status, res) = yield getattr(
self.topic_target_cls_map[targetted_topic],
self._to_string(
msg_body.rpc))(**_toDict(msg_body.args))
else:
log.debug("message-body-args-absent",
body=msg_body,
rpc=msg_body.rpc)
(status, res) = yield getattr(
self.topic_target_cls_map[targetted_topic],
self._to_string(msg_body.rpc))()
if msg_body.response_required:
response = self._format_response(
msg_header=message.header,
msg_body=res,
status=status,
)
if response is not None:
res_topic = self._to_string(
response.header.to_topic)
self._send_kafka_message(res_topic, response)
log.debug("Response-sent",
response=response.body,
to_topic=res_topic)
elif message.header.type == MessageType.Value("RESPONSE"):
trns_id = self._to_string(message.header.id)
if trns_id in self.transaction_id_deferred_map:
resp = self._parse_response(val)
self.transaction_id_deferred_map[trns_id].callback(resp)
else:
log.error("!!INVALID-TRANSACTION-TYPE!!")
except Exception as e:
log.exception("Failed-to-process-message", message=m, e=e)
@inlineCallbacks
def _send_kafka_message(self, topic, msg):
try:
yield self.kafka_proxy.send_message(topic, msg.SerializeToString())
except Exception, e:
log.exception("Failed-sending-message", message=msg, e=e)