def dashboard():
if not session:
return redirect(url_for('index'))
else:
if session['user'] == 'rebecca':
more_tweets_word = keen.count("more_tweets", group_by="word")
players_last_hr = keen.count('game_play', group_by="user", timeframe="previous_60_minutes")
return render_template('dashboard.html', more_tweets_word=more_tweets_word, players_last_hr=players_last_hr)
else:
return redirect(url_for('index'))
def dashboard():
if not session:
return redirect(url_for('index'))
else:
if session['user'] == 'rebecca':
more_tweets_word = keen.count("more_tweets", group_by="word")
players_last_hr = keen.count('game_play',
group_by="user",
timeframe="previous_60_minutes")
return render_template('dashboard.html',
more_tweets_word=more_tweets_word,
players_last_hr=players_last_hr)
else:
return redirect(url_for('index'))
def has_build_id(repo=None, build_id=None):
"""
Check if build_id exists in Keen.io database.
Parameters :
- repo : repo name (fe. buildtimetrend/python-lib)
- build_id : ID of the build
"""
if repo is None or build_id is None:
logger.error("Repo or build_id is not set")
raise ValueError("Repo or build_id is not set")
if not is_readable():
raise SystemError("Keen.io Project ID or API Read Key is not set")
try:
count = keen.count(
"build_jobs",
filters=[get_repo_filter(repo), {
"property_name": "job.build",
"operator": "eq",
"property_value": str(build_id)}]
)
except requests.ConnectionError:
logger.error("Connection to Keen.io API failed")
raise SystemError("Connection to Keen.io API failed")
except keen.exceptions.KeenApiError as msg:
logger.error("Error in keenio.has_build_id : " + str(msg))
raise SystemError(msg)
return count > 0
def test_order_by(self, get):
get.return_value = self.LIST_RESPONSE_DESCENDING
collection = "query_test"
limit = 2
order_by = {
"property_name": "result",
"direction": keen.direction.DESCENDING
}
resp = keen.count(collection,
timeframe="today",
group_by="number",
order_by=order_by,
limit=limit)
self.assertTrue("https://api.keen.io/3.0/projects/{0}/queries/count".
format(keen.project_id) in get.call_args[0][0])
self.assertEqual(2, get.call_args[1]["params"]["limit"])
self.assertEqual(collection,
get.call_args[1]["params"]["event_collection"])
# Order by should always be wrapped in a list, and stringified.
# This test is broken because JSON. It thinks double quoted strings != single quoted strings.
# ... annoying, to say the least.
# self.assertTrue(str([order_by]), get.call_args[1]["params"]["order_by"])
# So instead let's use this only slightly useful test:
self.assertTrue("order_by" in get.call_args[1]["params"])
self.assertTrue(
keen.read_key in get.call_args[1]["headers"]["Authorization"])
self.assertEqual(resp, self.LIST_RESPONSE_DESCENDING.json()["result"])
def popular():
choose_method = ["popular", "trending"]
method = random.choice(choose_method)
if method == "popular":
timeframe = "this_1_years"
if method == "trending":
timeframe = "this_7_days"
keen.add_event("view", { "page": method })
popular_videolist = keen.count("video_view", timeframe=timeframe, group_by="page")
return render_template('popular.html', popular_videolist=popular_videolist)
def index():
choose_method = ["popular", "trending"]
method = random.choice(choose_method)
if method == "popular":
timeframe = "this_1_years"
if method == "trending":
timeframe = "this_7_days"
keen.add_event("view", { "page": method })
videos = keen.count("video_view", timeframe=timeframe, group_by="page", order_by={"property_name": "result", "direction": keen.direction.ASCENDING})
return render_template('home.html', videos=videos, playlist_url=playlist_url)
def test_order_by(self, get):
get.return_value = self.LIST_RESPONSE_DESCENDING
collection = "query_test"
limit = 2
order_by = {"property_name": "result", "direction": keen.direction.DESCENDING}
resp = keen.count(collection, timeframe="today", group_by="number", order_by=order_by, limit=limit)
self.assertTrue("https://api.keen.io/3.0/projects/{0}/queries/count".format(keen.project_id) in
get.call_args[0][0])
self.assertEqual(2, get.call_args[1]["params"]["limit"])
self.assertEqual(collection, get.call_args[1]["params"]["event_collection"])
# Order by should always be wrapped in a list, and stringified.
# This test is broken because JSON. It thinks double quoted strings != single quoted strings.
# ... annoying, to say the least.
# self.assertTrue(str([order_by]), get.call_args[1]["params"]["order_by"])
# So instead let's use this only slightly useful test:
self.assertTrue("order_by" in get.call_args[1]["params"])
self.assertTrue(keen.read_key in get.call_args[1]["headers"]["Authorization"])
self.assertEqual(resp, self.LIST_RESPONSE_DESCENDING.json()["result"])
def test_multi_group_by(self):
resp = keen.count("query test", timeframe="today", group_by=["number", "string"])
assert type(resp) is list
assert len(resp) == 1
def test_group_by(self):
resp = keen.count("query test", timeframe="today", group_by="number")
assert type(resp) is list
def test_count(self):
resp = keen.count("query test", timeframe="today", filters=self.get_filter())
assert type(resp) is int
def test_interval(self):
resp = keen.count("query test", timeframe="this_2_days", interval="daily")
self.assertEqual(type(resp), list)
def test_group_by(self):
resp = keen.count("query test", timeframe="today", group_by="number")
self.assertEqual(type(resp), list)
def test_multi_group_by(self):
resp = keen.count("query test",
timeframe="today",
group_by=["number", "string"])
assert type(resp) is list
assert len(resp) == 1
def test_interval(self, get):
get.return_value = self.LIST_RESPONSE
resp = keen.count("query test",
timeframe="this_2_days",
interval="daily")
self.assertEqual(type(resp), list)
def test_count(self, get):
get.return_value = self.INT_RESPONSE
resp = keen.count("query test",
timeframe="today",
filters=self.get_filter())
self.assertEqual(type(resp), int)
def test_count(self):
resp = keen.count("query test",
timeframe="today",
filters=self.get_filter())
assert type(resp) is int
def test_interval(self):
resp = keen.count("query test",
timeframe="this_2_days",
interval="daily")
assert type(resp) is list
def test_interval(self):
resp = keen.count("query test", timeframe="this_2_days", interval="daily")
assert type(resp) is list
async def keenlog_users(self, ctx, *args):
"""Message count by users.
Params:
--time TIME, -t
Time in ES notation. 7d for 7 days, 1h for 1 hour
Default: 7d (7 days)
--count COUNT, -c
Number of results to show
Default: 10
--excludechannels EXCLUDECHANNEL [EXCLUDECHANNEL ...], -ec
List of channels to exclude
--includechannels INCLUDECHANNEL [INCLUDECHANNEL ...], -ic
List of channels to include (multiples are interpreted as OR)
--excluderoles EXCLUDEROLE [EXCLUDEROLE ...], -er
List of roles to exclude
--includeroles INCLUDEROLE [INCLUDEROLE ...], -ir
List of roles to include (multiples are interpreted as AND)
--excludebot, -eb
Exclude bot accounts
--excludebotcommands, -ebc
Exclude bot commands.
--split {none, channel}, -s
Split chart
Example:
[p]keenlog user --time 2d --count 20 --include general some-channel
Counts number of messages sent by authors within last 2 days in channels #general and #some-channel
Note:
It might take a few minutes to process for servers which have many users and activity.
"""
parser = KeenLogger.parser()
try:
p_args = parser.parse_args(args)
except SystemExit:
await send_cmd_help(ctx)
return
await self.bot.type()
server = ctx.message.server
# s = MessageEventModel.search()
# s = s.filter('match', **{'server.id': server.id})
#
# if p_args.time is not None:
# s = s.filter('range', timestamp={'gte': 'now-{}/m'.format(p_args.time), 'lte': 'now/m'})
# if p_args.includechannels is not None:
# for channel in p_args.includechannels:
# s = s.filter('match', **{'channel.name.keyword': channel})
# if p_args.excludechannels is not None:
# for channel in p_args.excludechannels:
# s = s.query('bool', must_not=[Q('match', **{'channel.name.keyword': channel})])
# if p_args.includeroles is not None:
# for role in p_args.includeroles:
# s = s.filter('match', **{'author.roles.name.keyword': role})
# if p_args.excluderoles is not None:
# for role in p_args.excluderoles:
# s = s.query('bool', must_not=[Q('match', **{'author.roles.name.keyword': role})])
# if p_args.excludebot:
# s = s.filter('match', **{'author.bot': False})
# s = self.message_search.server_messages(server, p_args)
resp = keen.count("message",
filters=[{
"property_name": "server.id",
"operator": "eq",
"property_value": server.id
}],
timeframe='this_7_days',
group_by=['author.id'])
resp = sorted(resp, key=lambda r: r["result"], reverse=True)
# Embed
em = discord.Embed(title="{}: User activity by messages".format(
server.name),
description=KeenLogView.description(p_args),
color=random_discord_color())
total_message_count = sum([r["result"] for r in resp])
# em.add_field(
# name="Rank",
# value="{} / {} (active) / {} (all)".format(
# rank, active_members, len(member.server.members)
# )
# )
em.add_field(name="Messages", value=total_message_count)
# last_seen_str = last_seen.strftime("%Y-%m-%d %H:%M:%S UTC")
# em.add_field(name="Last seen", value=last_seen_str)
max_count = None
for r in resp[:p_args.count]:
count = r["result"]
if max_count is None:
max_count = count
print(r)
member = server.get_member(r["author.id"])
chart = KeenLogView.inline_barchart(count, max_count)
if member is not None:
member_name = member.display_name
else:
member_name = 'User {}'.format(r["author.id"])
em.add_field(name='{}: {} message'.format(member_name, count),
value=chart,
inline=False)
await self.bot.say(embed=em)
def test_group_by(self):
resp = keen.count("query test", timeframe="today", group_by="number")
assert isinstance(resp, list)
def test_group_by(self):
resp = keen.count("query test", timeframe="today", group_by="number")
assert type(resp) is list
def test_multi_group_by(self, get):
get.return_value = self.LIST_RESPONSE
resp = keen.count("query test",
timeframe="today",
group_by=["number", "string"])
self.assertEqual(type(resp), list)
def test_interval(self):
resp = keen.count("query test", timeframe="this_2_days", interval="daily")
assert isinstance(resp, list)
async def keenlog_user(self, ctx, member: Member = None, *args):
"""Message statistics by user.
Params:
--time TIME, -t
Time in ES notation. 7d for 7 days, 1h for 1 hour
Default: 7d (7 days)
--count COUNT, -c
Number of results to show
Default: 10
--excludechannels EXCLUDECHANNEL [EXCLUDECHANNEL ...], -ec
List of channels to exclude
--includechannels INCLUDECHANNEL [INCLUDECHANNEL ...], -ic
List of channels to include (multiples are interpreted as OR)
--excludebotcommands, -ebc
Exclude bot commands.
Example:
[p]keenlog user --time 2d --count 20 --include general some-channel
Counts number of messages sent by authors within last 2 days in channels #general and #some-channel
Note:
It might take a few minutes to process for servers which have many users and activity.
"""
parser = KeenLogger.parser()
try:
p_args = parser.parse_args(args)
except SystemExit:
await send_cmd_help(ctx)
return
author = ctx.message.author
if member is None:
member = author
await self.bot.type()
# mds = self.message_search
time = p_args.time
# rank = mds.author_rank(member, time)
# channels = mds.author_channels(member, time)
# active_members = mds.active_members(member.server, time)
# message_count = mds.author_messages_count(member, time)
# last_seen = mds.author_lastseen(member)
resp = keen.count("message",
filters=[{
"property_name": "author.id",
"operator": "eq",
"property_value": member.id
}, {
"property_name": "server.id",
"operator": "eq",
"property_value": ctx.message.server.id
}],
timeframe='this_7_days',
group_by='channel.id')
resp = sorted(resp, key=lambda r: r["result"], reverse=True)
# Embed
em = discord.Embed(title="Member Activity: {}".format(
member.display_name),
description=KeenLogView.description(p_args,
show_top=False),
color=random_discord_color())
total_message_count = sum([r["result"] for r in resp])
# em.add_field(
# name="Rank",
# value="{} / {} (active) / {} (all)".format(
# rank, active_members, len(member.server.members)
# )
# )
em.add_field(name="Messages", value=total_message_count)
# last_seen_str = last_seen.strftime("%Y-%m-%d %H:%M:%S UTC")
# em.add_field(name="Last seen", value=last_seen_str)
max_count = None
for r in resp[:p_args.count]:
count = r["result"]
if max_count is None:
max_count = count
print(r)
channel = member.server.get_channel(r["channel.id"])
chart = KeenLogView.inline_barchart(count, max_count)
if channel is not None:
channel_name = channel.name
else:
channel_name = 'None'
em.add_field(name='{}: {} message'.format(channel_name, count),
value=chart,
inline=False)
await self.bot.say(embed=em)
def test_count(self, get):
get.return_value = self.INT_RESPONSE
resp = keen.count("query test", timeframe="today", filters=self.get_filter())
self.assertEqual(type(resp), int)
def test_count(self):
resp = keen.count("query test", timeframe="today", filters=self.get_filter())
self.assertEqual(type(resp), int)
def test_multi_group_by(self, get):
get.return_value = self.LIST_RESPONSE
resp = keen.count("query test", timeframe="today", group_by=["number", "string"])
self.assertEqual(type(resp), list)
def test_multi_group_by(self):
resp = keen.count("query test", timeframe="today", group_by=["number", "string"])
self.assertEqual(type(resp), list)
def test_interval(self, get):
get.return_value = self.LIST_RESPONSE
resp = keen.count("query test", timeframe="this_2_days", interval="daily")
self.assertEqual(type(resp), list)
import keen
import pandas as pd
import json
keen.project_id = "[xxxxx]"
keen.write_key = "[xxxxx]"
keen.read_key = "[xxxxx]"
df = pd.read_csv('filename.csv', low_memory=False)
collname = 'filename'
recs = len(df)
cols = df.columns
i = 0
recs = len(df)
while i < recs:
y = {}
j = 0
while j < len(cols):
# need to put stuff in the appropriate double quotes where required....
y[cols[j]] = df[cols[j]][i]
j += 1
# yz = json.dumps(y)
keen.add_event(collname, y)
i += 1
x = keen.count(collname)
print x
},
{
"start": f"20{fy}-06-01T00:00:00.000Z",
"end": f"20{fy}-07-01T00:00:00.000Z"
},
]
dfs_year = []
print("looping through months")
for index, month in enumerate(months_tf):
downloads = keen.count("s3_seshat.datasd.org_logs",
timeframe=month,
group_by=["log.key", "log.user_agent"],
interval="daily",
filters=[{
'property_name': 'log.operation',
'operator': 'contains',
'property_value': 'GET.OBJECT'
}])
df = pd.io.json.json_normalize(
downloads, 'value', [['timeframe', 'end'], ['timeframe', 'start']])
dfs_year.append(df)
print(f"appended month {index}")
# In[10]:
print("Concatting months")
df_year_all = pd.concat(dfs_year, ignore_index=True)
df_year_process = process_results(df_year_all)
