def add_favorite(bot, update, item: Bot, callback_alert=None): user = User.from_update(update) uid = util.uid_from_update(update) mid = util.mid_from_update(update) from components.basic import main_menu_buttons main_menu_markup = ReplyKeyboardMarkup(main_menu_buttons(uid in settings.MODERATORS)) fav, created = Favorite.add(user=user, item=item) if created: Statistic.of(user, 'add-favorite', item.username) text = mdformat.love("{} added to your {}favorites.".format(fav.bot, '' if callback_alert else '/')) if callback_alert: update.callback_query.answer(text=text, show_alert=False) else: msg = util.send_md_message(bot, uid, text, to_edit=mid, reply_markup=main_menu_markup) mid = msg.message_id util.wait(bot, update) send_favorites_list(bot, update, to_edit=mid) else: text = mdformat.none_action( "{} is already a favorite of yours.{}".format(fav.bot, '' if callback_alert else ' /favorites')) if callback_alert: update.callback_query.answer(text=text, show_alert=False) else: util.send_md_message(bot, uid, text, reply_markup=main_menu_markup) return ConversationHandler.END
def send_category(bot, update, chat_data, category): uid = util.uid_from_update(update) cid = update.effective_chat.id bots = Bot.of_category_without_new( category)[:settings.MAX_BOTS_PER_MESSAGE] bots_with_description = [b for b in bots if b.description is not None] detailed_buttons_enabled = len( bots_with_description) > 0 and util.is_private_message(update) callback = CallbackActions.SEND_BOT_DETAILS if detailed_buttons_enabled: buttons = [ InlineKeyboardButton( x.username, callback_data=util.callback_for_action(callback, {"id": x.id}), ) for x in bots_with_description ] else: buttons = [] menu = util.build_menu(buttons, 2) menu.insert( 0, [ InlineKeyboardButton( captions.BACK, callback_data=util.callback_for_action( CallbackActions.SELECT_CATEGORY), ), InlineKeyboardButton( "Show in BotList", url="http://t.me/botlist/{}".format( category.current_message_id), ), InlineKeyboardButton("Share", switch_inline_query=category.name), ], ) txt = "There are *{}* bots in the category *{}*:\n\n".format( len(bots), str(category)) if uid in settings.MODERATORS and util.is_private_message(update): # append admin edit buttons txt += "\n".join(["{} — /edit{} 🛃".format(b, b.id) for b in bots]) else: txt += "\n".join([str(b) for b in bots]) if detailed_buttons_enabled: txt += "\n\n" + util.action_hint( "Press a button below to get a detailed description.") reply_markup = InlineKeyboardMarkup(menu) reply_markup, callback = botlistchat.append_restricted_delete_button( update, chat_data, reply_markup) msg = bot.formatter.send_or_edit(cid, txt, to_edit=util.mid_from_update(update), reply_markup=reply_markup) callback(msg) Statistic.of(update, "menu", "of category {}".format(str(category)), Statistic.ANALYSIS)
def add_keyword(bot, update, chat_data): user = User.from_telegram_object(update.effective_user) if check_suggestion_limit(bot, update, user): return kw = update.message.text bot_to_edit = chat_data.get('edit_bot') kw = helpers.format_keyword(kw) # Sanity checks if kw in settings.FORBIDDEN_KEYWORDS: update.message.reply_text('The keyword {} is forbidden.'.format(kw)) return if len(kw) <= 1: update.message.reply_text('Keywords must be longer than 1 character.') return if len(kw) >= 20: update.message.reply_text( 'Keywords must not be longer than 20 characters.') # Ignore duplicates try: Keyword.get((Keyword.name == kw) & (Keyword.entity == bot_to_edit)) return except Keyword.DoesNotExist: pass Suggestion.add_or_update(user=user, action='add_keyword', subject=bot_to_edit, value=kw) set_keywords(bot, update, chat_data, bot_to_edit) Statistic.of(update, 'added keyword to'.format(kw), bot_to_edit.username)
def add_or_update(user, action, subject, value): from models import Statistic # value may be None already_exists = Suggestion.get_pending(action, subject, user, value) if already_exists: # Does the new suggestion reset the value? if action == 'remove_keyword': try: kw = Keyword.get(entity=subject, name=value) kw.delete_instance() except Keyword.DoesNotExist: pass elif action == 'add_keyword': return # TODO: is this right? elif value == getattr(already_exists.subject, action): already_exists.delete_instance() return None already_exists.value = value already_exists.save() Statistic.of(user, 'made changes to their suggestion: ', str(already_exists)) return already_exists else: new_suggestion = Suggestion(user=user, action=action, date=datetime.date.today(), subject=subject, value=value) new_suggestion.save() Statistic.of(user, 'suggestion', new_suggestion._md_plaintext()) return new_suggestion
def test_field_stats(client): create_statistics() by_field = Aggregate.get_info_by_field("model", "cucumber", "model", "country") assert len(by_field.keys()) == 3 assert by_field['total'] == 1 print(by_field) Statistic.objects().delete() Aggregate.objects().delete()
def delete_bot(bot, update, to_edit: Bot): username = to_edit.username to_edit.disable(Bot.DisabledReason.banned) to_edit.save() bot.formatter.send_or_edit(update.effective_user.id, "Bot has been disabled and banned.", to_edit=util.mid_from_update(update)) Statistic.of(update, 'disable', username, Statistic.IMPORTANT)
def apply_all_changes(bot, update, chat_data, to_edit): user = User.from_update(update) user_suggestions = Suggestion.select_all_of_user(user) for suggestion in user_suggestions: suggestion.apply() refreshed_bot = Bot.get(id=to_edit.id) edit_bot(bot, update, chat_data, refreshed_bot) Statistic.of(update, "apply", refreshed_bot.username)
def _input_failed(bot, update, chat_data, text): chat_id = util.uid_from_update(update) bot.formatter.send_failure(chat_id, text) Statistic.of( update, "error", "input failed in admin menu for {}".format(text), Statistic.ANALYSIS, ) chat_data["add_bot_message"] = None
def check_suggestion_limit(bot, update, user): cid = update.effective_chat.id if Suggestion.over_limit(user): bot.formatter.send_failure( cid, "You have reached the limit of {} suggestions. Please wait for " "the Moderators to approve of some of them.".format( settings.SUGGESTION_LIMIT)) Statistic.of(update, 'hit the suggestion limit') return True return False
def _too_many_favorites_handler(bot, update, user): uid = util.uid_from_update(update) any_removed = False while too_many_favorites(user): oldest = Favorite.get_oldest(user) oldest.delete_instance() any_removed = True Statistic.of(update, 'had to lose a favorite because HE HAD TOO F****N MANY 😬') if any_removed: txt = "You have too many favorites, _they do not fit into a single message_. That's why I removed your " \ "oldest bot, *{}*, from your list of favorites.".format(oldest.bot if oldest.bot else oldest.custom_bot) util.send_md_message(bot, uid, txt)
def test_get(client): create_statistics() expected = { 'model': Aggregate.get_most_popular('model', 90), 'country': Aggregate.get_most_popular('country', 90), 'total': Aggregate.get_count(90) } result = client.simulate_get('/api/v1/stats') assert result.status_code == 200 assert result.json == expected Statistic.objects().delete() Aggregate.objects().delete()
def create_season_and_stat_objects(): for player in data: player_search = Player.query.filter_by(name=player['Player']).first() team_search = Team.query.filter_by(abbreviation=player['Team']).first() stat_line = Statistic(end_yr_team=player['Team'], games_played=int(player['GP']), time_on_ice=round(float(player['TOI']),2), goals=int(player['G']), assists=int(player['A']), points=int(player['P']), primary_points=int(player['P1']), pts_per_60=round(float(player['P/60']),2), p_pts_per_60=round(float(player['P1/60']),2), cf=int(player['CF']), ca=int(player['CA']), corsi_plus_minus=int(player['C+/-']), cf_percentage=round(float(player['CF%']),2), rel_cf=round(float(player['Rel CF%']),2), gf=int(player['GF']), ga=int(player['GA']), plus_minus=int(player['G+/-']), pdo=round(float(player['PDO']),2), zsr=round(float(player['ZSR']),2), weighted_cf=create_weighted_cf(player), weighted_ca=create_weighted_ca(player), weighted_corsi_percentage=create_weighted_corsi_percent(player)) season = Season(year=player['Season'], player=player_search, team=team_search, statistic=stat_line) db.session.add(season) db.session.commit()
def set_notifications(bot, update, value: bool): cid = update.effective_chat.id try: notifications = Notifications.get(Notifications.chat_id == cid) except Notifications.DoesNotExist: notifications = Notifications(chat_id=cid) notifications.enabled = value notifications.save() Statistic.of(update, ('enabled' if value else 'disabled') + ' notifications for their group {}'.format(cid)) msg = util.success("Nice! Notifications enabled." ) if value else "Ok, notifications disabled." msg += '\nYou can always adjust this setting with the /subscribe command.' bot.formatter.send_or_edit(cid, msg, to_edit=util.mid_from_update(update)) return ConversationHandler.END
def create_statistics(): stats = [] devices = ['cucumber', 'pumpkin', 'tomato', 'avocado', 'toast'] for i in range(len(devices)): for date in ['20170101', '20170102', '20170103', '20170104', '20170105']: for version in ['13.0', '14.1']: for t in ['NIGHTLY', 'UNOFFICIAL']: Statistic(d=str(i), m=devices[i], v='{}-{}-{}-{}'.format(version, date, t, devices[i]), u='US', c='Carrier', c_id='0').save() Aggregate.add_stat(d=str(i), m=devices[i], v='{}-{}-{}-{}'.format(version, date, t, devices[i]), u='US', c='Carrier', c_id='0')
def updateNodeAccessStat(self, updateData): print "Update node accesses stat" stat = Statistic.query.filter(Statistic.statId == self.STATISTICS_STATID_NODE_ACCESSES).first() if stat == None: newStat = Statistic("Accesses per node", self.STATISTICS_STATID_NODE_ACCESSES, Statistic.STATTYPE_DOUGHNUT_CLASSES, 0, 0) db.session.add(newStat) db.session.commit() for nodeName in updateData: self.incrementalEntryAddOrUpdate(Statistic.STATTYPE_DOUGHNUT_CLASSES, self.STATISTICS_STATID_NODE_ACCESSES,nodeName,updateData[nodeName],0,0,0,0)
def ban_user(_bot, update, user: User, ban_state: bool): if user.banned and ban_state is True: update.message.reply_text( mdformat.none_action("User {} is already banned.".format(user)), parse_mode="markdown", ) raise DispatcherHandlerStop if not user.banned and ban_state is False: update.message.reply_text( mdformat.none_action("User {} is not banned.".format(user)), parse_mode="markdown", ) raise DispatcherHandlerStop user.banned = ban_state if ban_state is True: with db.atomic(): user_submissions = Bot.select().where( (Bot.approved == False) & (Bot.submitted_by == user) # TODO: does this need to include `Bot.deleted == True`? ) for b in user_submissions: b.delete_instance() users_suggestions = Suggestion.select().where( (Suggestion.executed == False) & (Suggestion.user == user)) for s in users_suggestions: s.delete_instance() update.message.reply_text( mdformat.success( "User {} banned, all bot submissions and suggestions removed.". format(user)), parse_mode="markdown", ) Statistic.of(update, "ban", user.markdown_short) else: update.message.reply_text(mdformat.success( "User {} unbanned.".format(user)), parse_mode="markdown") Statistic.of(update, "unban", user.markdown_short) user.save()
def send_botlist(bot, update, resend=False, silent=False): log.info("Re-sending BotList..." if resend else "Updating BotList...") channel = helpers.get_channel() revision = Revision.get_instance() revision.nr += 1 revision.save() all_categories = Category.select_all() botlist = BotList(bot, update, channel, resend, silent) if resend: botlist.delete_full_botlist() botlist.update_intro() botlist.update_categories(all_categories) botlist.update_new_bots_list() botlist.update_category_list() botlist.send_footer() botlist.finish() channel.save() Statistic.of(update, 'send', 'botlist (resend: {})'.format(str(resend)), Statistic.IMPORTANT)
def updateAccessesStat(self, updateData): print "Update Accesses stat" stat = Statistic.query.filter(Statistic.statId == self.STATISTICS_STATID_ACCESSES).first() if stat == None: newStat = Statistic("Accesses total", self.STATISTICS_STATID_ACCESSES, Statistic.STATTYPE_LINE_SERIES, 0, 2, "", 0, "Card auth.", "Web auth.") db.session.add(newStat) db.session.commit() for year in updateData: for month in updateData[year]: for seriesIndex in range(len(updateData[year][month])): self.incrementalEntryAddOrUpdate(Statistic.STATTYPE_LINE_SERIES, self.STATISTICS_STATID_ACCESSES, str(month) + "/" + str(year % 1000), updateData[year][month][seriesIndex], month, year, self.BINNING_NONE, seriesIndex)
def computeIndicators(self,indicator_type,sensor,day_of_computation,ts_from,ts_to): self.__dataFetcher.compileUrl("viabasse",sensor,ts_from,ts_to) if self.__dataFetcher.call() == 1 and self.__dataFetcher.getStatusCode() == 200: data = self.__dataFetcher.getJson() min_value = Statistic(day=day_of_computation,k="min_temp",v=10000.0,sensor=sensor,type=indicator_type) max_value = Statistic(day=day_of_computation,k="max_temp",v=-10000.0,sensor=sensor,type=indicator_type) sum_value = 0 count_value = 0 for line in data: if line["temperature"] < min_value.v: min_value.v = float(line["temperature"]) min_value.when = datetime.fromtimestamp((int(line["timestamp"])/1000)) if line["temperature"] > max_value.v: max_value.v = float(line["temperature"]) max_value.when = datetime.fromtimestamp((int(line["timestamp"])/1000)) sum_value = sum_value + line["temperature"] count_value = count_value + 1 avg_value = Statistic(day=day_of_computation,k="avg_temp",v=round(sum_value/count_value,2),sensor=sensor,type=indicator_type) range_value = Statistic(day=day_of_computation,k="range_temp",v=max_value.v-min_value.v,sensor=sensor,type=indicator_type) return {'min':min_value,'max':max_value,'avg':avg_value,'range':range_value} else: return None
def updateWeekdaysStat(self, updateData): print "Update Weekdate stat" stat = Statistic.query.filter(Statistic.statId == self.STATISTICS_STATID_WEEKDAYS).first() if stat == None: newStat = Statistic("Accesses per weekday", self.STATISTICS_STATID_WEEKDAYS, Statistic.STATTYPE_RADAR_SERIES, 7, 1, "", 0, "Weekdays") db.session.add(newStat) db.session.commit() for day in range(0,7): daynamesList = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] dayname = daynamesList[day] #dayEntry = StatisticEntry(self.STATISTICS_STATID_WEEKDAYS,dayname,random.randrange(10,100), 0, 0, 0, day) self.incrementalEntryAddOrUpdate(Statistic.STATTYPE_RADAR_SERIES,self.STATISTICS_STATID_WEEKDAYS,dayname,updateData[day], 0, 0, day, 0)
def updateUserCountStat(self,updateData): print "Update UserCount stat" stat = Statistic.query.filter(Statistic.statId == self.STATISTICS_STATID_USERCOUNT).first() if stat == None: newStat = Statistic("User total", self.STATISTICS_STATID_USERCOUNT, Statistic.STATTYPE_LINE_SERIES, 0, 3, "", 0, "Users", "Supervisors", "Admins") db.session.add(newStat) db.session.commit() now = datetime.datetime.now() #- datetime.timedelta(days=30) month = now.month year = now.year for i in range(0, 3): self.staticEntryAddOrUpdate(Statistic.STATTYPE_LINE_SERIES, self.STATISTICS_STATID_USERCOUNT, str(month) + "/" + str(year % 1000), updateData[i], month, year, self.BINNING_NONE, i)
def share_with_moderator(bot, update, bot_in_question, moderator): user = User.from_update(update) buttons = [[ InlineKeyboardButton( "Yea, let me take this one!", callback_data=util.callback_for_action( CallbackActions.APPROVE_REJECT_BOTS, {"id": bot_in_question.id}), ) ]] reply_markup = InlineKeyboardMarkup(buttons) text = "{} thinks that you have the means to inspect this bot submission:\n▶️ {}".format( user.markdown_short, bot_in_question) try: util.send_md_message( bot, moderator.chat_id, text, reply_markup=reply_markup, disable_web_page_preview=True, ) answer_text = mdformat.success( "I will ask {} to have a look at this submission.".format( moderator.plaintext)) except Exception as e: answer_text = mdformat.failure( f"Could not contact {moderator.plaintext}: {e}") if update.callback_query: update.callback_query.answer(text=answer_text) Statistic.of( update, "share", "submission {} with {}".format(bot_in_question.username, moderator.plaintext), )
def updateUserActivityGroups(userData): print "Update User Groups stat" stat = Statistic.query.filter( Statistic.statId == StatisticsManager.STATISTICS_STATID_ACTIVITY_USER_GROUPS).first() if stat == None: newStat = Statistic( "User activity groups", StatisticsManager.STATISTICS_STATID_ACTIVITY_USER_GROUPS, Statistic.STATTYPE_YEARLY_BAR_SERIES, 0, 4, "", Statistic.STATDISPLAY_CONFIG_NO_TOTAL, "Zero activity users", "Low activity users", "Medium activity users", "High activity users") db.session.add(newStat) db.session.commit() result = StatisticsManager.evaluateUserGroups(userData) now = datetime.datetime.now() month = now.month year = now.year StatisticsManager.staticEntryAddOrUpdate( Statistic.STATTYPE_YEARLY_BAR_SERIES, StatisticsManager.STATISTICS_STATID_ACTIVITY_USER_GROUPS, str(month) + "/" + str(year % 1000), result[0], month, year, StatisticsManager.BINNING_NONE, StatisticsManager.SERIES_GROUPS_NO_ACTIVITY) StatisticsManager.staticEntryAddOrUpdate( Statistic.STATTYPE_YEARLY_BAR_SERIES, StatisticsManager.STATISTICS_STATID_ACTIVITY_USER_GROUPS, str(month) + "/" + str(year % 1000), result[1], month, year, StatisticsManager.BINNING_NONE, StatisticsManager.SERIES_GROUPS_LOW_ACTIVITY) StatisticsManager.staticEntryAddOrUpdate( Statistic.STATTYPE_YEARLY_BAR_SERIES, StatisticsManager.STATISTICS_STATID_ACTIVITY_USER_GROUPS, str(month) + "/" + str(year % 1000), result[2], month, year, StatisticsManager.BINNING_NONE, StatisticsManager.SERIES_GROUPS_MEDIUM_ACTIVITY) StatisticsManager.staticEntryAddOrUpdate( Statistic.STATTYPE_YEARLY_BAR_SERIES, StatisticsManager.STATISTICS_STATID_ACTIVITY_USER_GROUPS, str(month) + "/" + str(year % 1000), result[3], month, year, StatisticsManager.BINNING_NONE, StatisticsManager.SERIES_GROUPS_HIGH_ACTIVITY)
def updateUserActivityGroupAverages(self, userData): print "Update User Group Accesses stat" stat = Statistic.query.filter(Statistic.statId == self.STATISTICS_STATID_USER_GROUPS_AVERAGE).first() if stat == None: newStat = Statistic("Total accesses of user groups", self.STATISTICS_STATID_USER_GROUPS_AVERAGE, Statistic.STATTYPE_YEARLY_BAR_SERIES, 0, 3, "", Statistic.STATDISPLAY_CONFIG_NO_TOTAL, "Low activity users", "Medium activity users", "High activity users") db.session.add(newStat) db.session.commit() result = self.evaluateUserGroups(userData) now = datetime.datetime.now() month = now.month year = now.year self.staticEntryAddOrUpdate(Statistic.STATTYPE_LINE_SERIES, self.STATISTICS_STATID_USER_GROUPS_AVERAGE, str(month) + "/" + str(year % 1000), result[7], month, year, self.BINNING_NONE, self.SERIES_GROUP_BY_LOW_ACTIVITY) self.staticEntryAddOrUpdate(Statistic.STATTYPE_LINE_SERIES, self.STATISTICS_STATID_USER_GROUPS_AVERAGE, str(month) + "/" + str(year % 1000), result[8], month, year, self.BINNING_NONE, self.SERIES_GROUP_BY_MEDIUM_ACTIVITY) self.staticEntryAddOrUpdate(Statistic.STATTYPE_LINE_SERIES, self.STATISTICS_STATID_USER_GROUPS_AVERAGE, str(month) + "/" + str(year % 1000), result[9], month, year, self.BINNING_NONE, self.SERIES_GROUP_BY_HIGH_ACTIVITY)
def generate_stats(filtered_docs, filters): # create set of common keys and a local dict to hold parsed doc data keys = None docs = dict() for doc in filtered_docs: # transfer doc data to local dict docs[doc.id] = doc.to_dict() # update keys if keys == None: keys = set(docs[doc.id].keys()) else: keys = keys.intersection(docs[doc.id].keys()) # create dictionary with all values for each key field_values = dict() for key in keys: field_values[key] = [] for doc_id, doc_data in docs.items(): datapoint = doc_data[key] if datapoint is not None: field_values[key].append(datapoint) else: field_values.pop(key) break # generate dict with freq of each value freq_dict = dict() for key in field_values.keys(): freq_dict[key] = dict() for value in set(field_values[key]): freq_dict[key][value] = field_values[key].count( value) # / len(field_values[key]) # convert data into Statistic objects stats = [] for key, value in freq_dict.items(): if key not in filters.keys(): stats.append( Statistic(getStatName(key), getStatDescription(key), value)) # return stats return stats
def send_activity_logs(bot, update, args=None, level=Statistic.INFO): num = 200 if args: try: num = int(args[0]) num = min(num, 500) except: pass uid = update.effective_user.id recent_statistic = Statistic.select().order_by( Statistic.date.desc()).limit(num) recent_statistic = list(reversed(recent_statistic)) step_size = 30 for i in range(0, len(recent_statistic), step_size): items = recent_statistic[i:i + step_size] text = "\n".join(x.md_str() for x in items) bot.formatter.send_message(uid, text)
def send_statistic(bot, update): interesting_actions = [ "explore", "menu", "command", "request", "made changes to their suggestion:", "issued deletion of conversation in BotListChat", ] stats = (Statistic.select( Statistic, fn.COUNT(Statistic.entity).alias("count")).where( Statistic.action << interesting_actions).group_by( Statistic.action, Statistic.entity)) maxlen = max(len(str(x.count)) for x in stats) text = "\n".join("`{}▪️` {} {}".format( str(s.count).ljust(maxlen), s.action.title(), s.entity) for s in stats) bot.formatter.send_message(update.effective_chat.id, text, parse_mode="markdown")
def submit_form(): if request.method == 'OPTIONS': # CORS preflight return _build_cors_prelight_response() elif request.method == 'POST': # The actual request following the preflight name = request.get_json().get('name') email = request.get_json().get('email') subject = request.get_json().get('subject') message = request.get_json().get('message') try: statistic = Statistic(name=name, email=email, subject=subject, message=message) db.session.add(statistic) db.session.commit() return _corsify_actual_response( jsonify("Statistic added. statistic id={}".format( statistic.id))) except Exception as e: return str(e)
def updateUserActivityGroupAccesses(userData): print "Update User Group Accesses stat" stat = Statistic.query.filter( Statistic.statId == StatisticsManager.STATISTICS_STATID_USER_GROUPS_ACCESSES).first() if stat == None: newStat = Statistic( "Average accesses of user groups", StatisticsManager.STATISTICS_STATID_USER_GROUPS_ACCESSES, Statistic.STATTYPE_LINE_SERIES, 0, 3, "", Statistic.STATDISPLAY_CONFIG_NO_TOTAL, "Low activity users", "Medium activity users", "High activity users") db.session.add(newStat) db.session.commit() result = StatisticsManager.evaluateUserGroups(userData) now = datetime.datetime.now() month = now.month year = now.year StatisticsManager.staticEntryAddOrUpdate( Statistic.STATTYPE_LINE_SERIES, StatisticsManager.STATISTICS_STATID_USER_GROUPS_ACCESSES, str(month) + "/" + str(year % 1000), result[4], month, year, StatisticsManager.BINNING_NONE, StatisticsManager.SERIES_GROUP_BY_LOW_ACTIVITY) StatisticsManager.staticEntryAddOrUpdate( Statistic.STATTYPE_LINE_SERIES, StatisticsManager.STATISTICS_STATID_USER_GROUPS_ACCESSES, str(month) + "/" + str(year % 1000), result[5], month, year, StatisticsManager.BINNING_NONE, StatisticsManager.SERIES_GROUP_BY_MEDIUM_ACTIVITY) StatisticsManager.staticEntryAddOrUpdate( Statistic.STATTYPE_LINE_SERIES, StatisticsManager.STATISTICS_STATID_USER_GROUPS_ACCESSES, str(month) + "/" + str(year % 1000), result[6], month, year, StatisticsManager.BINNING_NONE, StatisticsManager.SERIES_GROUP_BY_HIGH_ACTIVITY)
def extract_frequency(region): stats = Statistic.select(Statistic.video).where( Statistic.trending_region == region ).group_by(Statistic.video) #.limit(10000) unique_video = defaultdict() for s in tqdm(stats): if s.video.id not in unique_video: unique_video[s.video.id] = s.video valid_tag = [] print('videos {}'.format(len(unique_video))) for _, video in unique_video.items(): tags = extract_video_unique_keyword(video) for tag in tags: _tag = clean_tag(tag) if _tag and len(_tag) > 1: valid_tag.append(_tag) print('total tag {}, videos {}'.format(len(valid_tag), len(unique_video))) return Counter(valid_tag)
def stats(request, sensor): yesterday = date.today()-timedelta(1) tmp_ts_from = int(mktime(yesterday.timetuple())) print tmp_ts_from tmp_ts_to = tmp_ts_from + 24*60*60 ts_from = tmp_ts_from * 1000 ts_to = tmp_ts_to * 1000 result = makeRequest( sensor, ts_from, ts_to ) if result.status_code == 200: data = json.loads(result.content) min_value = Statistic(day=datetime.fromtimestamp(tmp_ts_from).date(),k="min_temp",v=10000.0,sensor=sensor) max_value = Statistic(day=datetime.fromtimestamp(tmp_ts_from).date(),k="max_temp",v=-10000.0,sensor=sensor) sum_value = 0 count_value = 0 for line in data: if line["temperature"] < min_value.v: min_value.v = float(line["temperature"]) min_value.when = datetime.fromtimestamp((int(line["timestamp"])/1000)+settings.TMZDIFF) if line["temperature"] > max_value.v: max_value.v = float(line["temperature"]) max_value.when = datetime.fromtimestamp((int(line["timestamp"])/1000)+settings.TMZDIFF) sum_value = sum_value + line["temperature"] count_value = count_value + 1 avg_value = Statistic(day=datetime.fromtimestamp(tmp_ts_from).date(),k="avg_temp",v=round(sum_value/count_value,2),sensor=sensor) db.put(min_value) db.put(max_value) db.put(avg_value) return HttpResponse("min: {}, max: {}, avg: {}".format(min_value.v,max_value.v,avg_value.v))
def insert_statistic(kwargs): '''method used to insert a statistic in DB''' stats = kwargs stats_to_insert = [] for stat in stats: message = "inserimento statistica: %s" % stat logger.debug(message) host = stat.get('host') user = stat.get('user') message = "conversione stat_time" logger.debug(message) stat_type = stat.get('stat_type', '') print "stat_type:" + stat_type stat_time = datetime_conversion(stat.get('stat_time')) timestamp = datetime_conversion(stat.get('timestamp')) stat_element = stat.get('stat_element') stat_value = stat.get('stat_value') stat_extra = stat.get('stat_extra', '') for pe in ProductionEnvironment.objects.filter(host__name=host, user=user): st = StatType.objects.filter(name=stat_type) if st.count() == 0: message = "la statistica di tipo %s non e' configurata" % stat_type print message logger.error(message) continue s = Statistic(stat_time=stat_time, timestamp=timestamp, \ stat_element=stat_element, stat_value=stat_value, stat_extra=stat_extra, \ productionEnvironment=pe, statType=st[0]) stats_to_insert.append(s) try: s.save() except : message = sys.exc_info() logger.error(message) #print sys.exc_info() #exc_type, exc_obj, exc_tb = sys.exc_info() #message = 'Exception: %s' % str(e) #message = sys.exc_info() #logger.error(message) #print message # try: # s.save() # except IntegrityError: # message = "statistica scartata in quanto gia' inserita %s" % s # print message # logger.debug(message) # continue # except: # exc_type, exc_obj, exc_tb = sys.exc_info() # message = 'Exception: %s' % str(e) # message = sys.exc_info() # logger.error(message) # print message # message = "statistica scartata: %s" %stat # print message # logger.debug(message) # continue message = "inserita statistica %s" % s print message logger.debug(message)
def main(): global statistic all_events = Context.events.get_all_evensts() statistic = Statistic(all_events) app.run()