def handle_be_stat_message(rc, stat_data):
"""
:param rc: connection to redis
:type rc: StrictRedis
:param stat_data: stats from backend
:type stat_data: dict
"""
app.logger.debug('Got stat data: {}'.format(stat_data))
ts_from = int(stat_data['ts_from'])
ts_to = int(stat_data['ts_to'])
hits = stat_data['hits']
if not ts_from or not ts_to or ts_from > ts_to or not hits:
raise Exception("Invalid or empty data received.")
ts_from_stored = int(rc.get('handle_be_stat_message_ts_from') or 0)
ts_to_stored = int(rc.get('handle_be_stat_message_ts_to') or 0)
app.logger.debug('ts_from: {}'.format(ts_from))
app.logger.debug('ts_to: {}'.format(ts_to))
app.logger.debug('ts_from_stored: {}'.format(ts_from_stored))
app.logger.debug('ts_to_stored: {}'.format(ts_to_stored))
if (ts_from < ts_to_stored and ts_to > ts_from_stored):
app.logger.debug('Time overlap with already stored data. Skipping.')
return
hits_formatted = defaultdict(int)
for key_str, count in hits.items():
key = key_str.split('|')
if key[0] == 'chroot_repo_metadata_dl_stat':
redis_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
copr_user=key[1],
copr_project_name=key[2],
copr_chroot=key[3])
elif key[0] == 'chroot_rpms_dl_stat':
redis_key = CHROOT_RPMS_DL_STAT_FMT.format(
copr_user=key[1],
copr_project_name=key[2],
copr_chroot=key[3])
elif key[0] == 'project_rpms_dl_stat':
redis_key = PROJECT_RPMS_DL_STAT_FMT.format(
copr_user=key[1],
copr_project_name=key[2])
else:
raise Exception('Unknown key {}'.format(key[0]))
hits_formatted[redis_key] += count
for redis_key, count in hits_formatted.items():
TimedStatEvents.add_event(rc, redis_key, count=count, timestamp=ts_to)
rc.set('handle_be_stat_message_ts_from', ts_from)
rc.set('handle_be_stat_message_ts_to', ts_to)
def render_copr_detail(copr):
repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
form = forms.CoprLegalFlagForm()
repos_info = {}
for chroot in copr.active_chroots:
chroot_rpms_dl_stat_key = CHROOT_RPMS_DL_STAT_FMT.format(
copr_user=copr.owner_name,
copr_project_name=copr.name,
copr_chroot=chroot.name,
)
chroot_rpms_dl_stat = TimedStatEvents.get_count(
rconnect=rcp.get_connection(),
name=chroot_rpms_dl_stat_key,
)
logoset = set()
logodir = app.static_folder + "/chroot_logodir"
for logo in os.listdir(logodir):
# glob.glob() uses listdir() and fnmatch anyways
if fnmatch.fnmatch(logo, "*.png"):
logoset.add(logo[:-4])
if chroot.name_release not in repos_info:
logo = None
if chroot.name_release in logoset:
logo = chroot.name_release + ".png"
elif chroot.os_release in logoset:
logo = chroot.os_release + ".png"
repos_info[chroot.name_release] = {
"name_release": chroot.name_release,
"os_release": chroot.os_release,
"os_version": chroot.os_version,
"logo": logo,
"arch_list": [chroot.arch],
"repo_file": "{}-{}.repo".format(copr.repo_id,
chroot.name_release),
"dl_stat": repo_dl_stat[chroot.name_release],
"rpm_dl_stat": {
chroot.arch: chroot_rpms_dl_stat
}
}
else:
repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
repos_info[chroot.name_release]["rpm_dl_stat"][
chroot.arch] = chroot_rpms_dl_stat
repos_info_list = sorted(repos_info.values(),
key=lambda rec: rec["name_release"])
builds = builds_logic.BuildsLogic.get_multiple_by_copr(
copr=copr).limit(1).all()
return flask.render_template(
"coprs/detail/overview.html",
copr=copr,
user=flask.g.user,
form=form,
repo_dl_stat=repo_dl_stat,
repos_info_list=repos_info_list,
latest_build=builds[0] if len(builds) == 1 else None,
)
def render_copr_detail(copr):
repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
form = forms.CoprLegalFlagForm()
repos_info = {}
for chroot in copr.active_chroots:
# chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
# copr_user=copr.owner.name,
# copr_project_name=copr.name,
# copr_chroot=chroot.name,
# )
chroot_rpms_dl_stat_key = CHROOT_RPMS_DL_STAT_FMT.format(
copr_user=copr.owner.name,
copr_project_name=copr.name,
copr_chroot=chroot.name,
)
chroot_rpms_dl_stat = TimedStatEvents.get_count(
rconnect=rcp.get_connection(),
name=chroot_rpms_dl_stat_key,
)
if chroot.name_release not in repos_info:
repos_info[chroot.name_release] = {
"name_release":
chroot.name_release,
"name_release_human":
chroot.name_release_human,
"os_release":
chroot.os_release,
"os_version":
chroot.os_version,
"arch_list": [chroot.arch],
"repo_file":
"{}-{}-{}.repo".format(copr.owner.name, copr.name,
chroot.name_release),
"dl_stat":
repo_dl_stat[chroot.name_release],
"rpm_dl_stat": {
chroot.arch: chroot_rpms_dl_stat
}
}
else:
repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
repos_info[chroot.name_release]["rpm_dl_stat"][
chroot.arch] = chroot_rpms_dl_stat
repos_info_list = sorted(repos_info.values(),
key=lambda rec: rec["name_release"])
builds = builds_logic.BuildsLogic.get_multiple_by_copr(
copr=copr).limit(1).all()
return flask.render_template(
"coprs/detail/overview.html",
copr=copr,
user=flask.g.user,
form=form,
repo_dl_stat=repo_dl_stat,
repos_info_list=repos_info_list,
latest_build=builds[0] if len(builds) == 1 else None,
)
def handle_logstash(rc, ls_data):
"""
:param rc: connection to redis
:type rc: StrictRedis
:param ls_data: log stash record
:type ls_data: dict
"""
dt_unixtime = string_dt_to_unixtime(ls_data["@timestamp"])
app.logger.debug("got ls_data: {}".format(ls_data))
# don't count statistics from builders
if "clientip" in ls_data:
if is_ip_from_builder_net(ls_data["clientip"]):
return
if "tags" not in ls_data:
return
tags = set(ls_data["tags"])
if "frontend" in tags and "repo_dl":
name = REPO_DL_STAT_FMT.format(**ls_data)
CounterStatLogic.incr(name=name, counter_type=CounterStatType.REPO_DL)
db.session.commit()
if "backend" in tags and "repomdxml" in tags:
key = CHROOT_REPO_MD_DL_STAT_FMT.format(**ls_data)
TimedStatEvents.add_event(rc, key, timestamp=dt_unixtime)
if "backend" in tags and "rpm" in tags:
key_chroot = CHROOT_RPMS_DL_STAT_FMT.format(**ls_data)
key_project = PROJECT_RPMS_DL_STAT_FMT.format(**ls_data)
TimedStatEvents.add_event(rc, key_chroot, timestamp=dt_unixtime)
TimedStatEvents.add_event(rc, key_project, timestamp=dt_unixtime)
def render_copr_detail(copr):
repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
form = forms.CoprLegalFlagForm()
repos_info = {}
for chroot in copr.active_chroots:
# chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
# copr_user=copr.user.name,
# copr_project_name=copr.name,
# copr_chroot=chroot.name,
# )
chroot_rpms_dl_stat_key = CHROOT_RPMS_DL_STAT_FMT.format(
copr_user=copr.user.name,
copr_project_name=copr.name,
copr_chroot=chroot.name,
)
chroot_rpms_dl_stat = TimedStatEvents.get_count(
rconnect=rcp.get_connection(),
name=chroot_rpms_dl_stat_key,
)
if chroot.name_release not in repos_info:
repos_info[chroot.name_release] = {
"name_release": chroot.name_release,
"name_release_human": chroot.name_release_human,
"os_release": chroot.os_release,
"os_version": chroot.os_version,
"arch_list": [chroot.arch],
"repo_file": "{}-{}-{}.repo".format(copr.group.name if copr.is_a_group_project else copr.user.name, copr.name, chroot.name_release),
"dl_stat": repo_dl_stat[chroot.name_release],
"rpm_dl_stat": {
chroot.arch: chroot_rpms_dl_stat
}
}
else:
repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
repos_info[chroot.name_release]["rpm_dl_stat"][chroot.arch] = chroot_rpms_dl_stat
repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"])
builds = builds_logic.BuildsLogic.get_multiple_by_copr(copr=copr).limit(1).all()
return flask.render_template(
"coprs/detail/overview.html",
copr=copr,
user=flask.g.user,
form=form,
repo_dl_stat=repo_dl_stat,
repos_info_list=repos_info_list,
latest_build=builds[0] if len(builds) == 1 else None,
)
def copr_detail(username, coprname):
query = coprs_logic.CoprsLogic.get(
flask.g.user, username, coprname, with_mock_chroots=True)
form = forms.CoprLegalFlagForm()
try:
copr = query.one()
except sqlalchemy.orm.exc.NoResultFound:
return page_not_found(
"Copr with name {0} does not exist.".format(coprname))
repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
repos_info = {}
for chroot in copr.active_chroots:
chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
copr_user=copr.owner.name,
copr_project_name=copr.name,
copr_chroot=chroot.name,
)
chroot_rpms_dl_stat = TimedStatEvents.get_count(
rconnect=rcp.get_connection(),
name=chroot_rpms_dl_stat_key,
)
if chroot.name_release not in repos_info:
repos_info[chroot.name_release] = {
"name_release": chroot.name_release,
"name_release_human": chroot.name_release_human,
"arch_list": [chroot.arch],
"repo_file": "{}-{}-{}.repo".format(copr.owner.name, copr.name, chroot.name_release),
"dl_stat": repo_dl_stat[chroot.name_release],
"rpm_dl_stat": {
chroot.arch: chroot_rpms_dl_stat
}
}
else:
repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
repos_info[chroot.name_release]["rpm_dl_stat"][chroot.arch] = chroot_rpms_dl_stat
repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"])
return flask.render_template("coprs/detail/overview.html",
copr=copr,
form=form,
repo_dl_stat=repo_dl_stat,
repos_info_list=repos_info_list,
)
def test_timed_stats_events(self):
if self.disabled:
return
TimedStatEvents.add_event(
self.rc,
name="foobar",
prefix=self.prefix,
timestamp=self.time_now,
)
assert TimedStatEvents.get_count(
self.rc,
name="foobar",
prefix=self.prefix,
) == 1
TimedStatEvents.add_event(self.rc,
name="foobar",
prefix=self.prefix,
timestamp=self.time_now,
count=2)
assert TimedStatEvents.get_count(
self.rc,
name="foobar",
prefix=self.prefix,
) == 3
TimedStatEvents.add_event(self.rc,
name="foobar",
prefix=self.prefix,
timestamp=self.time_now - 1000000,
count=2)
TimedStatEvents.add_event(self.rc,
name="foobar",
prefix=self.prefix,
timestamp=self.time_now - 3000000,
count=3)
assert TimedStatEvents.get_count(
self.rc,
name="foobar",
prefix=self.prefix,
) == 3
assert TimedStatEvents.get_count(self.rc,
name="foobar",
prefix=self.prefix,
day_min=self.time_now - 2000000) == 5
assert TimedStatEvents.get_count(self.rc,
name="foobar",
prefix=self.prefix,
day_min=self.time_now - 5000000) == 8
TimedStatEvents.trim_before(self.rc,
name="foobar",
prefix=self.prefix,
threshold_timestamp=self.time_now - 200000)
assert TimedStatEvents.get_count(self.rc,
name="foobar",
prefix=self.prefix,
day_min=self.time_now - 5000000) == 3
def test_timed_stats_events(self):
if self.disabled:
return
TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
timestamp=self.time_now, )
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 1
TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
timestamp=self.time_now, count=2)
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3
TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
timestamp=self.time_now - 1000000, count=2)
TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
timestamp=self.time_now - 3000000, count=3)
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
day_min=self.time_now - 2000000) == 5
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
day_min=self.time_now - 5000000) == 8
TimedStatEvents.trim_before(self.rc, name="foobar",
prefix=self.prefix, threshold_timestamp=self.time_now - 200000)
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
day_min=self.time_now - 5000000) == 3