def heartbeat_source(n):
last_heartbeat_timestamp_s = get_latest_message(
heartbeat_topic_name_source, heartbeat_config).timestamp / 1000
last_heartbeat_date = datetime.datetime.fromtimestamp(
last_heartbeat_timestamp_s, timezone('US/Pacific'))
now_s = round(time.time())
time_since_heartbeat_s = now_s - last_heartbeat_timestamp_s
label = "Source Connector Status - Last Heartbeat Timestamp: %s - Minutes Ago: %s" % (
str(last_heartbeat_date), str(round(time_since_heartbeat_s / 60)))
if time_since_heartbeat_s <= 5 * 60:
# heard from within 5 minutes. all is well
color = colors['alive']
value = True
else:
# haven't heard from in 5 minutes. pronounced dead.
color = colors['dead']
value = False
return label, color, value
def run_forever(self):
while True:
# poll kafka topic until consumer partition is automatically generated
message = get_latest_message(self.input_topic_name,
self.consumer_config)
if message is None:
continue
# push latest scores to s3, log this push
last_push_filename, last_push_timestamp = self.__push_to_s3(
message)
self.__log_push_to_s3(last_push_filename)
# wait until the next push interval
next_push_timestamp = last_push_timestamp + self.min_push_interval
while datetime.datetime.now() < next_push_timestamp:
sleep_duration = max(
(next_push_timestamp - datetime.datetime.now()).seconds, 1)
print("sleeping until %s" % next_push_timestamp)
print("sleeping for %s seconds" % sleep_duration)
time.sleep(sleep_duration)
def update_graph_live(n, hours_window):
message = get_latest_message(report_topic_name, report_config)
df = pd.read_json(json.dumps(message.value), orient='index')
df['date'] = [
datetime.datetime.fromtimestamp(ts / 1000) for ts in df.POST_TIMESTAMP
]
max_ts = max(df.POST_TIMESTAMP)
df['tsnorm'] = [(ts - max_ts) / 1000 / 60 / 60 for ts in df.POST_TIMESTAMP]
figure1 = {
'data': [{
'x': df['tsnorm'],
'y': df['coldness_score'],
'type': 'bar',
'name': 'COLD score',
'width': 0.02,
'marker': {
"color": colors['cold']
}
}, {
'x':
df['tsnorm'],
'y':
df['hotness_score'],
'type':
'bar',
'name':
'HOT score',
'width':
0.02,
'marker': {
"color": colors['hot']
},
'hovertext': [
'Post ID: %s\nPreviews: %s\nFull Views: %s\nCTR: %s' %
(post_id, previews, full_views, full_views / max(previews, 1))
for post_id, previews, full_views in zip(
df['PROPERTIES_SHOPPABLE_POST_ID'], df['PREVIEW'],
df['FULL_VIEW'])
]
}],
'layout': {
'title':
'Post Scores. Last Updates: %s' %
str(datetime.datetime.now().astimezone(timezone('US/Pacific'))),
'barmode':
'stack',
'xaxis': {
'title': 'Hours Ago',
'range': [-hours_window, 0]
},
'yaxis': {
'title': 'Score'
}
}
}
# figure2 = {
# 'data': [{'x': df['tsnorm'], 'y': df['coldness_score'], 'type': 'bar', 'name': 'COLD score', 'width': 0.04,
# 'marker': {"color": colors['cold']}},
# {'x': df['tsnorm'], 'y': df['hotness_score'], 'type': 'bar', 'name': 'HOT score', 'width': 0.04,
# 'marker': {"color": colors['hot']},
# 'hovertext': ['Post ID: %s\nPreviews: %s\nFull Views: %s\nCTR: %s'
# % (post_id, previews, full_views, full_views / max(previews, 1))
# for post_id, previews, full_views in
# zip(df['PROPERTIES_SHOPPABLE_POST_ID'], df['PREVIEW'], df['FULL_VIEW'])]}
# ],
# 'layout': {
# 'title': 'Post Scores. Last Updates: %s' % str(datetime.datetime.now().astimezone(timezone('US/Pacific'))),
# 'barmode': 'stack',
# 'xaxis': {'title': 'Hours Ago', 'range': [-24, 0]},
# 'yaxis': {'title': 'Score'}
# }
# }
return figure1
'enable_auto_commit': True,
'value_deserializer': lambda x: json.loads(x.decode('utf-8'))
}
heartbeat_config = {
'bootstrap_servers': bootstrap_servers,
'auto_offset_reset': 'latest',
'enable_auto_commit': True
}
report_topic_name = 'recent_posts_scores_snapshot'
heartbeat_topic_name_sink = 'heartbeat_conn_s3_sink'
heartbeat_topic_name_source = 'heartbeat_conn_segment_source'
heartbeat_topic_name_table_generator = 'heartbeat_table_generator'
s3_topic_name = 'connector_s3_sink_push_log'
# try:
current_scoring_fn_kwargs = get_latest_message('scores_config_update').value
print(current_scoring_fn_kwargs)
# except:
# current_scoring_fn_kwargs = {'max_coldness_score':40, 'min_previews_threshold':20, 'cold_threshold_steepness': 0.15,
# 'max_hotness_score':60, 'ctr_hotness_threshold':0.14, 'hot_threshold_steepness':13, 'score_offset':0}
app.layout = html.Div([
html.H1(
children='Real-Time Scoring Dashboard',
style={
'textAlign': 'center',
'colors': colors['text']
},
),
dcc.Graph(id='bar_graph'),
html.Label("Time Window (Hours)"),