def dashboard(): if not session: return redirect(url_for('index')) else: if session['user'] == 'rebecca': more_tweets_word = keen.count("more_tweets", group_by="word") players_last_hr = keen.count('game_play', group_by="user", timeframe="previous_60_minutes") return render_template('dashboard.html', more_tweets_word=more_tweets_word, players_last_hr=players_last_hr) else: return redirect(url_for('index'))
def has_build_id(repo=None, build_id=None): """ Check if build_id exists in Keen.io database. Parameters : - repo : repo name (fe. buildtimetrend/python-lib) - build_id : ID of the build """ if repo is None or build_id is None: logger.error("Repo or build_id is not set") raise ValueError("Repo or build_id is not set") if not is_readable(): raise SystemError("Keen.io Project ID or API Read Key is not set") try: count = keen.count( "build_jobs", filters=[get_repo_filter(repo), { "property_name": "job.build", "operator": "eq", "property_value": str(build_id)}] ) except requests.ConnectionError: logger.error("Connection to Keen.io API failed") raise SystemError("Connection to Keen.io API failed") except keen.exceptions.KeenApiError as msg: logger.error("Error in keenio.has_build_id : " + str(msg)) raise SystemError(msg) return count > 0
def test_order_by(self, get): get.return_value = self.LIST_RESPONSE_DESCENDING collection = "query_test" limit = 2 order_by = { "property_name": "result", "direction": keen.direction.DESCENDING } resp = keen.count(collection, timeframe="today", group_by="number", order_by=order_by, limit=limit) self.assertTrue("https://api.keen.io/3.0/projects/{0}/queries/count". format(keen.project_id) in get.call_args[0][0]) self.assertEqual(2, get.call_args[1]["params"]["limit"]) self.assertEqual(collection, get.call_args[1]["params"]["event_collection"]) # Order by should always be wrapped in a list, and stringified. # This test is broken because JSON. It thinks double quoted strings != single quoted strings. # ... annoying, to say the least. # self.assertTrue(str([order_by]), get.call_args[1]["params"]["order_by"]) # So instead let's use this only slightly useful test: self.assertTrue("order_by" in get.call_args[1]["params"]) self.assertTrue( keen.read_key in get.call_args[1]["headers"]["Authorization"]) self.assertEqual(resp, self.LIST_RESPONSE_DESCENDING.json()["result"])
def popular(): choose_method = ["popular", "trending"] method = random.choice(choose_method) if method == "popular": timeframe = "this_1_years" if method == "trending": timeframe = "this_7_days" keen.add_event("view", { "page": method }) popular_videolist = keen.count("video_view", timeframe=timeframe, group_by="page") return render_template('popular.html', popular_videolist=popular_videolist)
def index(): choose_method = ["popular", "trending"] method = random.choice(choose_method) if method == "popular": timeframe = "this_1_years" if method == "trending": timeframe = "this_7_days" keen.add_event("view", { "page": method }) videos = keen.count("video_view", timeframe=timeframe, group_by="page", order_by={"property_name": "result", "direction": keen.direction.ASCENDING}) return render_template('home.html', videos=videos, playlist_url=playlist_url)
def test_order_by(self, get): get.return_value = self.LIST_RESPONSE_DESCENDING collection = "query_test" limit = 2 order_by = {"property_name": "result", "direction": keen.direction.DESCENDING} resp = keen.count(collection, timeframe="today", group_by="number", order_by=order_by, limit=limit) self.assertTrue("https://api.keen.io/3.0/projects/{0}/queries/count".format(keen.project_id) in get.call_args[0][0]) self.assertEqual(2, get.call_args[1]["params"]["limit"]) self.assertEqual(collection, get.call_args[1]["params"]["event_collection"]) # Order by should always be wrapped in a list, and stringified. # This test is broken because JSON. It thinks double quoted strings != single quoted strings. # ... annoying, to say the least. # self.assertTrue(str([order_by]), get.call_args[1]["params"]["order_by"]) # So instead let's use this only slightly useful test: self.assertTrue("order_by" in get.call_args[1]["params"]) self.assertTrue(keen.read_key in get.call_args[1]["headers"]["Authorization"]) self.assertEqual(resp, self.LIST_RESPONSE_DESCENDING.json()["result"])
def test_multi_group_by(self): resp = keen.count("query test", timeframe="today", group_by=["number", "string"]) assert type(resp) is list assert len(resp) == 1
def test_group_by(self): resp = keen.count("query test", timeframe="today", group_by="number") assert type(resp) is list
def test_count(self): resp = keen.count("query test", timeframe="today", filters=self.get_filter()) assert type(resp) is int
def test_interval(self): resp = keen.count("query test", timeframe="this_2_days", interval="daily") self.assertEqual(type(resp), list)
def test_group_by(self): resp = keen.count("query test", timeframe="today", group_by="number") self.assertEqual(type(resp), list)
def test_interval(self, get): get.return_value = self.LIST_RESPONSE resp = keen.count("query test", timeframe="this_2_days", interval="daily") self.assertEqual(type(resp), list)
def test_count(self, get): get.return_value = self.INT_RESPONSE resp = keen.count("query test", timeframe="today", filters=self.get_filter()) self.assertEqual(type(resp), int)
def test_interval(self): resp = keen.count("query test", timeframe="this_2_days", interval="daily") assert type(resp) is list
async def keenlog_users(self, ctx, *args): """Message count by users. Params: --time TIME, -t Time in ES notation. 7d for 7 days, 1h for 1 hour Default: 7d (7 days) --count COUNT, -c Number of results to show Default: 10 --excludechannels EXCLUDECHANNEL [EXCLUDECHANNEL ...], -ec List of channels to exclude --includechannels INCLUDECHANNEL [INCLUDECHANNEL ...], -ic List of channels to include (multiples are interpreted as OR) --excluderoles EXCLUDEROLE [EXCLUDEROLE ...], -er List of roles to exclude --includeroles INCLUDEROLE [INCLUDEROLE ...], -ir List of roles to include (multiples are interpreted as AND) --excludebot, -eb Exclude bot accounts --excludebotcommands, -ebc Exclude bot commands. --split {none, channel}, -s Split chart Example: [p]keenlog user --time 2d --count 20 --include general some-channel Counts number of messages sent by authors within last 2 days in channels #general and #some-channel Note: It might take a few minutes to process for servers which have many users and activity. """ parser = KeenLogger.parser() try: p_args = parser.parse_args(args) except SystemExit: await send_cmd_help(ctx) return await self.bot.type() server = ctx.message.server # s = MessageEventModel.search() # s = s.filter('match', **{'server.id': server.id}) # # if p_args.time is not None: # s = s.filter('range', timestamp={'gte': 'now-{}/m'.format(p_args.time), 'lte': 'now/m'}) # if p_args.includechannels is not None: # for channel in p_args.includechannels: # s = s.filter('match', **{'channel.name.keyword': channel}) # if p_args.excludechannels is not None: # for channel in p_args.excludechannels: # s = s.query('bool', must_not=[Q('match', **{'channel.name.keyword': channel})]) # if p_args.includeroles is not None: # for role in p_args.includeroles: # s = s.filter('match', **{'author.roles.name.keyword': role}) # if p_args.excluderoles is not None: # for role in p_args.excluderoles: # s = s.query('bool', must_not=[Q('match', **{'author.roles.name.keyword': role})]) # if p_args.excludebot: # s = s.filter('match', **{'author.bot': False}) # s = self.message_search.server_messages(server, p_args) resp = keen.count("message", filters=[{ "property_name": "server.id", "operator": "eq", "property_value": server.id }], timeframe='this_7_days', group_by=['author.id']) resp = sorted(resp, key=lambda r: r["result"], reverse=True) # Embed em = discord.Embed(title="{}: User activity by messages".format( server.name), description=KeenLogView.description(p_args), color=random_discord_color()) total_message_count = sum([r["result"] for r in resp]) # em.add_field( # name="Rank", # value="{} / {} (active) / {} (all)".format( # rank, active_members, len(member.server.members) # ) # ) em.add_field(name="Messages", value=total_message_count) # last_seen_str = last_seen.strftime("%Y-%m-%d %H:%M:%S UTC") # em.add_field(name="Last seen", value=last_seen_str) max_count = None for r in resp[:p_args.count]: count = r["result"] if max_count is None: max_count = count print(r) member = server.get_member(r["author.id"]) chart = KeenLogView.inline_barchart(count, max_count) if member is not None: member_name = member.display_name else: member_name = 'User {}'.format(r["author.id"]) em.add_field(name='{}: {} message'.format(member_name, count), value=chart, inline=False) await self.bot.say(embed=em)
def test_group_by(self): resp = keen.count("query test", timeframe="today", group_by="number") assert isinstance(resp, list)
def test_multi_group_by(self, get): get.return_value = self.LIST_RESPONSE resp = keen.count("query test", timeframe="today", group_by=["number", "string"]) self.assertEqual(type(resp), list)
def test_interval(self): resp = keen.count("query test", timeframe="this_2_days", interval="daily") assert isinstance(resp, list)
async def keenlog_user(self, ctx, member: Member = None, *args): """Message statistics by user. Params: --time TIME, -t Time in ES notation. 7d for 7 days, 1h for 1 hour Default: 7d (7 days) --count COUNT, -c Number of results to show Default: 10 --excludechannels EXCLUDECHANNEL [EXCLUDECHANNEL ...], -ec List of channels to exclude --includechannels INCLUDECHANNEL [INCLUDECHANNEL ...], -ic List of channels to include (multiples are interpreted as OR) --excludebotcommands, -ebc Exclude bot commands. Example: [p]keenlog user --time 2d --count 20 --include general some-channel Counts number of messages sent by authors within last 2 days in channels #general and #some-channel Note: It might take a few minutes to process for servers which have many users and activity. """ parser = KeenLogger.parser() try: p_args = parser.parse_args(args) except SystemExit: await send_cmd_help(ctx) return author = ctx.message.author if member is None: member = author await self.bot.type() # mds = self.message_search time = p_args.time # rank = mds.author_rank(member, time) # channels = mds.author_channels(member, time) # active_members = mds.active_members(member.server, time) # message_count = mds.author_messages_count(member, time) # last_seen = mds.author_lastseen(member) resp = keen.count("message", filters=[{ "property_name": "author.id", "operator": "eq", "property_value": member.id }, { "property_name": "server.id", "operator": "eq", "property_value": ctx.message.server.id }], timeframe='this_7_days', group_by='channel.id') resp = sorted(resp, key=lambda r: r["result"], reverse=True) # Embed em = discord.Embed(title="Member Activity: {}".format( member.display_name), description=KeenLogView.description(p_args, show_top=False), color=random_discord_color()) total_message_count = sum([r["result"] for r in resp]) # em.add_field( # name="Rank", # value="{} / {} (active) / {} (all)".format( # rank, active_members, len(member.server.members) # ) # ) em.add_field(name="Messages", value=total_message_count) # last_seen_str = last_seen.strftime("%Y-%m-%d %H:%M:%S UTC") # em.add_field(name="Last seen", value=last_seen_str) max_count = None for r in resp[:p_args.count]: count = r["result"] if max_count is None: max_count = count print(r) channel = member.server.get_channel(r["channel.id"]) chart = KeenLogView.inline_barchart(count, max_count) if channel is not None: channel_name = channel.name else: channel_name = 'None' em.add_field(name='{}: {} message'.format(channel_name, count), value=chart, inline=False) await self.bot.say(embed=em)
def test_count(self): resp = keen.count("query test", timeframe="today", filters=self.get_filter()) self.assertEqual(type(resp), int)
def test_multi_group_by(self): resp = keen.count("query test", timeframe="today", group_by=["number", "string"]) self.assertEqual(type(resp), list)
import keen import pandas as pd import json keen.project_id = "[xxxxx]" keen.write_key = "[xxxxx]" keen.read_key = "[xxxxx]" df = pd.read_csv('filename.csv', low_memory=False) collname = 'filename' recs = len(df) cols = df.columns i = 0 recs = len(df) while i < recs: y = {} j = 0 while j < len(cols): # need to put stuff in the appropriate double quotes where required.... y[cols[j]] = df[cols[j]][i] j += 1 # yz = json.dumps(y) keen.add_event(collname, y) i += 1 x = keen.count(collname) print x
}, { "start": f"20{fy}-06-01T00:00:00.000Z", "end": f"20{fy}-07-01T00:00:00.000Z" }, ] dfs_year = [] print("looping through months") for index, month in enumerate(months_tf): downloads = keen.count("s3_seshat.datasd.org_logs", timeframe=month, group_by=["log.key", "log.user_agent"], interval="daily", filters=[{ 'property_name': 'log.operation', 'operator': 'contains', 'property_value': 'GET.OBJECT' }]) df = pd.io.json.json_normalize( downloads, 'value', [['timeframe', 'end'], ['timeframe', 'start']]) dfs_year.append(df) print(f"appended month {index}") # In[10]: print("Concatting months") df_year_all = pd.concat(dfs_year, ignore_index=True) df_year_process = process_results(df_year_all)