def handle_be_stat_message(rc, stat_data): """ :param rc: connection to redis :type rc: StrictRedis :param stat_data: stats from backend :type stat_data: dict """ app.logger.debug('Got stat data: {}'.format(stat_data)) ts_from = int(stat_data['ts_from']) ts_to = int(stat_data['ts_to']) hits = stat_data['hits'] if not ts_from or not ts_to or ts_from > ts_to or not hits: raise Exception("Invalid or empty data received.") ts_from_stored = int(rc.get('handle_be_stat_message_ts_from') or 0) ts_to_stored = int(rc.get('handle_be_stat_message_ts_to') or 0) app.logger.debug('ts_from: {}'.format(ts_from)) app.logger.debug('ts_to: {}'.format(ts_to)) app.logger.debug('ts_from_stored: {}'.format(ts_from_stored)) app.logger.debug('ts_to_stored: {}'.format(ts_to_stored)) if (ts_from < ts_to_stored and ts_to > ts_from_stored): app.logger.debug('Time overlap with already stored data. Skipping.') return hits_formatted = defaultdict(int) for key_str, count in hits.items(): key = key_str.split('|') if key[0] == 'chroot_repo_metadata_dl_stat': redis_key = CHROOT_REPO_MD_DL_STAT_FMT.format( copr_user=key[1], copr_project_name=key[2], copr_chroot=key[3]) elif key[0] == 'chroot_rpms_dl_stat': redis_key = CHROOT_RPMS_DL_STAT_FMT.format( copr_user=key[1], copr_project_name=key[2], copr_chroot=key[3]) elif key[0] == 'project_rpms_dl_stat': redis_key = PROJECT_RPMS_DL_STAT_FMT.format( copr_user=key[1], copr_project_name=key[2]) else: raise Exception('Unknown key {}'.format(key[0])) hits_formatted[redis_key] += count for redis_key, count in hits_formatted.items(): TimedStatEvents.add_event(rc, redis_key, count=count, timestamp=ts_to) rc.set('handle_be_stat_message_ts_from', ts_from) rc.set('handle_be_stat_message_ts_to', ts_to)
def test_timed_stats_events(self): if self.disabled: return TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, timestamp=self.time_now, ) assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 1 TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, timestamp=self.time_now, count=2) assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3 TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, timestamp=self.time_now - 1000000, count=2) TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, timestamp=self.time_now - 3000000, count=3) assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3 assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, day_min=self.time_now - 2000000) == 5 assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, day_min=self.time_now - 5000000) == 8 TimedStatEvents.trim_before(self.rc, name="foobar", prefix=self.prefix, threshold_timestamp=self.time_now - 200000) assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, day_min=self.time_now - 5000000) == 3
def handle_logstash(rc, ls_data): """ :param rc: connection to redis :type rc: StrictRedis :param ls_data: log stash record :type ls_data: dict """ dt_unixtime = string_dt_to_unixtime(ls_data["@timestamp"]) app.logger.debug("got ls_data: {}".format(ls_data)) # don't count statistics from builders if "clientip" in ls_data: if is_ip_from_builder_net(ls_data["clientip"]): return if "tags" not in ls_data: return tags = set(ls_data["tags"]) if "frontend" in tags and "repo_dl": name = REPO_DL_STAT_FMT.format(**ls_data) CounterStatLogic.incr(name=name, counter_type=CounterStatType.REPO_DL) db.session.commit() if "backend" in tags and "repomdxml" in tags: key = CHROOT_REPO_MD_DL_STAT_FMT.format(**ls_data) TimedStatEvents.add_event(rc, key, timestamp=dt_unixtime) if "backend" in tags and "rpm" in tags: key_chroot = CHROOT_RPMS_DL_STAT_FMT.format(**ls_data) key_project = PROJECT_RPMS_DL_STAT_FMT.format(**ls_data) TimedStatEvents.add_event(rc, key_chroot, timestamp=dt_unixtime) TimedStatEvents.add_event(rc, key_project, timestamp=dt_unixtime)
def test_timed_stats_events(self): if self.disabled: return TimedStatEvents.add_event( self.rc, name="foobar", prefix=self.prefix, timestamp=self.time_now, ) assert TimedStatEvents.get_count( self.rc, name="foobar", prefix=self.prefix, ) == 1 TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, timestamp=self.time_now, count=2) assert TimedStatEvents.get_count( self.rc, name="foobar", prefix=self.prefix, ) == 3 TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, timestamp=self.time_now - 1000000, count=2) TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, timestamp=self.time_now - 3000000, count=3) assert TimedStatEvents.get_count( self.rc, name="foobar", prefix=self.prefix, ) == 3 assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, day_min=self.time_now - 2000000) == 5 assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, day_min=self.time_now - 5000000) == 8 TimedStatEvents.trim_before(self.rc, name="foobar", prefix=self.prefix, threshold_timestamp=self.time_now - 200000) assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, day_min=self.time_now - 5000000) == 3