Exemplo n.º 1
0
def handle_be_stat_message(rc, stat_data):
    """
    :param rc: connection to redis
    :type rc: StrictRedis

    :param stat_data: stats from backend
    :type stat_data: dict
    """
    app.logger.debug('Got stat data: {}'.format(stat_data))

    ts_from = int(stat_data['ts_from'])
    ts_to = int(stat_data['ts_to'])
    hits = stat_data['hits']

    if not ts_from or not ts_to or ts_from > ts_to or not hits:
        raise Exception("Invalid or empty data received.")

    ts_from_stored = int(rc.get('handle_be_stat_message_ts_from') or 0)
    ts_to_stored = int(rc.get('handle_be_stat_message_ts_to') or 0)

    app.logger.debug('ts_from: {}'.format(ts_from))
    app.logger.debug('ts_to: {}'.format(ts_to))
    app.logger.debug('ts_from_stored: {}'.format(ts_from_stored))
    app.logger.debug('ts_to_stored: {}'.format(ts_to_stored))

    if (ts_from < ts_to_stored and ts_to > ts_from_stored):
        app.logger.debug('Time overlap with already stored data. Skipping.')
        return

    hits_formatted = defaultdict(int)
    for key_str, count in hits.items():
        key = key_str.split('|')
        if key[0] == 'chroot_repo_metadata_dl_stat':
            redis_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
                copr_user=key[1],
                copr_project_name=key[2],
                copr_chroot=key[3])
        elif key[0] == 'chroot_rpms_dl_stat':
            redis_key = CHROOT_RPMS_DL_STAT_FMT.format(
                copr_user=key[1],
                copr_project_name=key[2],
                copr_chroot=key[3])
        elif key[0] == 'project_rpms_dl_stat':
            redis_key = PROJECT_RPMS_DL_STAT_FMT.format(
                copr_user=key[1],
                copr_project_name=key[2])
        else:
            raise Exception('Unknown key {}'.format(key[0]))

        hits_formatted[redis_key] += count

    for redis_key, count in hits_formatted.items():
        TimedStatEvents.add_event(rc, redis_key, count=count, timestamp=ts_to)

    rc.set('handle_be_stat_message_ts_from', ts_from)
    rc.set('handle_be_stat_message_ts_to', ts_to)
Exemplo n.º 2
0
def render_copr_detail(copr):
    repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
    form = forms.CoprLegalFlagForm()
    repos_info = {}
    for chroot in copr.active_chroots:
        chroot_rpms_dl_stat_key = CHROOT_RPMS_DL_STAT_FMT.format(
            copr_user=copr.owner_name,
            copr_project_name=copr.name,
            copr_chroot=chroot.name,
        )
        chroot_rpms_dl_stat = TimedStatEvents.get_count(
            rconnect=rcp.get_connection(),
            name=chroot_rpms_dl_stat_key,
        )

        logoset = set()
        logodir = app.static_folder + "/chroot_logodir"
        for logo in os.listdir(logodir):
            # glob.glob() uses listdir() and fnmatch anyways
            if fnmatch.fnmatch(logo, "*.png"):
                logoset.add(logo[:-4])

        if chroot.name_release not in repos_info:
            logo = None
            if chroot.name_release in logoset:
                logo = chroot.name_release + ".png"
            elif chroot.os_release in logoset:
                logo = chroot.os_release + ".png"

            repos_info[chroot.name_release] = {
                "name_release": chroot.name_release,
                "os_release": chroot.os_release,
                "os_version": chroot.os_version,
                "logo": logo,
                "arch_list": [chroot.arch],
                "repo_file": "{}-{}.repo".format(copr.repo_id,
                                                 chroot.name_release),
                "dl_stat": repo_dl_stat[chroot.name_release],
                "rpm_dl_stat": {
                    chroot.arch: chroot_rpms_dl_stat
                }
            }
        else:
            repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
            repos_info[chroot.name_release]["rpm_dl_stat"][
                chroot.arch] = chroot_rpms_dl_stat
    repos_info_list = sorted(repos_info.values(),
                             key=lambda rec: rec["name_release"])
    builds = builds_logic.BuildsLogic.get_multiple_by_copr(
        copr=copr).limit(1).all()

    return flask.render_template(
        "coprs/detail/overview.html",
        copr=copr,
        user=flask.g.user,
        form=form,
        repo_dl_stat=repo_dl_stat,
        repos_info_list=repos_info_list,
        latest_build=builds[0] if len(builds) == 1 else None,
    )
Exemplo n.º 3
0
def render_copr_detail(copr):
    repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
    form = forms.CoprLegalFlagForm()
    repos_info = {}
    for chroot in copr.active_chroots:
        # chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
        #     copr_user=copr.owner.name,
        #     copr_project_name=copr.name,
        #     copr_chroot=chroot.name,
        # )
        chroot_rpms_dl_stat_key = CHROOT_RPMS_DL_STAT_FMT.format(
            copr_user=copr.owner.name,
            copr_project_name=copr.name,
            copr_chroot=chroot.name,
        )
        chroot_rpms_dl_stat = TimedStatEvents.get_count(
            rconnect=rcp.get_connection(),
            name=chroot_rpms_dl_stat_key,
        )

        if chroot.name_release not in repos_info:
            repos_info[chroot.name_release] = {
                "name_release":
                chroot.name_release,
                "name_release_human":
                chroot.name_release_human,
                "os_release":
                chroot.os_release,
                "os_version":
                chroot.os_version,
                "arch_list": [chroot.arch],
                "repo_file":
                "{}-{}-{}.repo".format(copr.owner.name, copr.name,
                                       chroot.name_release),
                "dl_stat":
                repo_dl_stat[chroot.name_release],
                "rpm_dl_stat": {
                    chroot.arch: chroot_rpms_dl_stat
                }
            }
        else:
            repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
            repos_info[chroot.name_release]["rpm_dl_stat"][
                chroot.arch] = chroot_rpms_dl_stat
    repos_info_list = sorted(repos_info.values(),
                             key=lambda rec: rec["name_release"])
    builds = builds_logic.BuildsLogic.get_multiple_by_copr(
        copr=copr).limit(1).all()
    return flask.render_template(
        "coprs/detail/overview.html",
        copr=copr,
        user=flask.g.user,
        form=form,
        repo_dl_stat=repo_dl_stat,
        repos_info_list=repos_info_list,
        latest_build=builds[0] if len(builds) == 1 else None,
    )
Exemplo n.º 4
0
def handle_logstash(rc, ls_data):
    """
    :param rc: connection to redis
    :type rc: StrictRedis

    :param ls_data: log stash record
    :type ls_data: dict
    """
    dt_unixtime = string_dt_to_unixtime(ls_data["@timestamp"])
    app.logger.debug("got ls_data: {}".format(ls_data))

    # don't count statistics from builders
    if "clientip" in ls_data:
        if is_ip_from_builder_net(ls_data["clientip"]):
            return

    if "tags" not in ls_data:
        return

    tags = set(ls_data["tags"])
    if "frontend" in tags and "repo_dl":
        name = REPO_DL_STAT_FMT.format(**ls_data)
        CounterStatLogic.incr(name=name, counter_type=CounterStatType.REPO_DL)
        db.session.commit()

    if "backend" in tags and "repomdxml" in tags:
        key = CHROOT_REPO_MD_DL_STAT_FMT.format(**ls_data)
        TimedStatEvents.add_event(rc, key, timestamp=dt_unixtime)

    if "backend" in tags and "rpm" in tags:
        key_chroot = CHROOT_RPMS_DL_STAT_FMT.format(**ls_data)
        key_project = PROJECT_RPMS_DL_STAT_FMT.format(**ls_data)
        TimedStatEvents.add_event(rc, key_chroot, timestamp=dt_unixtime)
        TimedStatEvents.add_event(rc, key_project, timestamp=dt_unixtime)
Exemplo n.º 5
0
def render_copr_detail(copr):
    repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
    form = forms.CoprLegalFlagForm()
    repos_info = {}
    for chroot in copr.active_chroots:
        # chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
        #     copr_user=copr.user.name,
        #     copr_project_name=copr.name,
        #     copr_chroot=chroot.name,
        # )
        chroot_rpms_dl_stat_key = CHROOT_RPMS_DL_STAT_FMT.format(
            copr_user=copr.user.name,
            copr_project_name=copr.name,
            copr_chroot=chroot.name,
        )
        chroot_rpms_dl_stat = TimedStatEvents.get_count(
            rconnect=rcp.get_connection(),
            name=chroot_rpms_dl_stat_key,
        )

        if chroot.name_release not in repos_info:
            repos_info[chroot.name_release] = {
                "name_release": chroot.name_release,
                "name_release_human": chroot.name_release_human,
                "os_release": chroot.os_release,
                "os_version": chroot.os_version,
                "arch_list": [chroot.arch],
                "repo_file": "{}-{}-{}.repo".format(copr.group.name if copr.is_a_group_project else copr.user.name, copr.name, chroot.name_release),
                "dl_stat": repo_dl_stat[chroot.name_release],
                "rpm_dl_stat": {
                    chroot.arch: chroot_rpms_dl_stat
                }
            }
        else:
            repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
            repos_info[chroot.name_release]["rpm_dl_stat"][chroot.arch] = chroot_rpms_dl_stat
    repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"])
    builds = builds_logic.BuildsLogic.get_multiple_by_copr(copr=copr).limit(1).all()

    return flask.render_template(
        "coprs/detail/overview.html",
        copr=copr,
        user=flask.g.user,
        form=form,
        repo_dl_stat=repo_dl_stat,
        repos_info_list=repos_info_list,
        latest_build=builds[0] if len(builds) == 1 else None,
    )
Exemplo n.º 6
0
def copr_detail(username, coprname):
    query = coprs_logic.CoprsLogic.get(
        flask.g.user, username, coprname, with_mock_chroots=True)
    form = forms.CoprLegalFlagForm()
    try:
        copr = query.one()
    except sqlalchemy.orm.exc.NoResultFound:
        return page_not_found(
            "Copr with name {0} does not exist.".format(coprname))

    repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)

    repos_info = {}
    for chroot in copr.active_chroots:
        chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
            copr_user=copr.owner.name,
            copr_project_name=copr.name,
            copr_chroot=chroot.name,
        )
        chroot_rpms_dl_stat = TimedStatEvents.get_count(
            rconnect=rcp.get_connection(),
            name=chroot_rpms_dl_stat_key,
        )

        if chroot.name_release not in repos_info:
            repos_info[chroot.name_release] = {
                "name_release": chroot.name_release,
                "name_release_human": chroot.name_release_human,
                "arch_list": [chroot.arch],
                "repo_file": "{}-{}-{}.repo".format(copr.owner.name, copr.name, chroot.name_release),
                "dl_stat": repo_dl_stat[chroot.name_release],
                "rpm_dl_stat": {
                    chroot.arch: chroot_rpms_dl_stat
                }
            }
        else:
            repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
            repos_info[chroot.name_release]["rpm_dl_stat"][chroot.arch] = chroot_rpms_dl_stat

    repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"])

    return flask.render_template("coprs/detail/overview.html",
                                 copr=copr,
                                 form=form,
                                 repo_dl_stat=repo_dl_stat,
                                 repos_info_list=repos_info_list,
                                 )
Exemplo n.º 7
0
    def test_timed_stats_events(self):
        if self.disabled:
            return

        TimedStatEvents.add_event(
            self.rc,
            name="foobar",
            prefix=self.prefix,
            timestamp=self.time_now,
        )

        assert TimedStatEvents.get_count(
            self.rc,
            name="foobar",
            prefix=self.prefix,
        ) == 1
        TimedStatEvents.add_event(self.rc,
                                  name="foobar",
                                  prefix=self.prefix,
                                  timestamp=self.time_now,
                                  count=2)

        assert TimedStatEvents.get_count(
            self.rc,
            name="foobar",
            prefix=self.prefix,
        ) == 3

        TimedStatEvents.add_event(self.rc,
                                  name="foobar",
                                  prefix=self.prefix,
                                  timestamp=self.time_now - 1000000,
                                  count=2)
        TimedStatEvents.add_event(self.rc,
                                  name="foobar",
                                  prefix=self.prefix,
                                  timestamp=self.time_now - 3000000,
                                  count=3)

        assert TimedStatEvents.get_count(
            self.rc,
            name="foobar",
            prefix=self.prefix,
        ) == 3
        assert TimedStatEvents.get_count(self.rc,
                                         name="foobar",
                                         prefix=self.prefix,
                                         day_min=self.time_now - 2000000) == 5
        assert TimedStatEvents.get_count(self.rc,
                                         name="foobar",
                                         prefix=self.prefix,
                                         day_min=self.time_now - 5000000) == 8

        TimedStatEvents.trim_before(self.rc,
                                    name="foobar",
                                    prefix=self.prefix,
                                    threshold_timestamp=self.time_now - 200000)

        assert TimedStatEvents.get_count(self.rc,
                                         name="foobar",
                                         prefix=self.prefix,
                                         day_min=self.time_now - 5000000) == 3
Exemplo n.º 8
0
    def test_timed_stats_events(self):
        if self.disabled:
            return

        TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
                                  timestamp=self.time_now, )

        assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 1
        TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
                                  timestamp=self.time_now, count=2)

        assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3

        TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
                                  timestamp=self.time_now - 1000000, count=2)
        TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
                                  timestamp=self.time_now - 3000000, count=3)

        assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3
        assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
                                         day_min=self.time_now - 2000000) == 5
        assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
                                         day_min=self.time_now - 5000000) == 8

        TimedStatEvents.trim_before(self.rc, name="foobar",
                                    prefix=self.prefix, threshold_timestamp=self.time_now - 200000)

        assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
                                         day_min=self.time_now - 5000000) == 3