def main():
    settings = startup.read_settings()
    Log.start(settings.debug)
    try:
        for repo in settings.param.repos:
            with DB(settings.database) as db:
                try:
                    pull_repo(repo)

                    #GET LATEST DATE
                    existing_range = db.query("""
                        SELECT
                            max(`date`) `max`,
                            min(`date`) `min`,
                            min(revision) min_rev,
                            max(revision) max_rev
                        FROM
                            changesets
                        WHERE
                            repo={{repo}}
                    """, {"repo": repo.name})[0]

                    ranges = struct.wrap([
                        {"min": nvl(existing_range.max, CNV.milli2datetime(0)) + timedelta(0, 1)},
                        {"max": existing_range.min}
                    ])

                    for r in ranges:
                        for g, docs in Q.groupby(get_changesets(date_range=r, repo=repo), size=100):
                            for doc in docs:
                                doc.file_changes = None
                                doc.file_adds = None
                                doc.file_dels = None
                                doc.description = doc.description[0:16000]

                            db.insert_list("changesets", docs)
                            db.flush()

                    missing_revisions = sql.find_holes(db, "changesets", "revision", {"term":{"repo":repo.name}}, {"min": 0, "max": existing_range.max_rev + 1})
                    for _range in missing_revisions:
                        for g, docs in Q.groupby(get_changesets(revision_range=_range, repo=repo), size=100):
                            for doc in docs:
                                doc.file_changes = None
                                doc.file_adds = None
                                doc.file_dels = None
                                doc.description = doc.description[0:16000]

                            db.insert_list("changesets", docs)
                            db.flush()



                except Exception, e:
                    Log.warning("Failure to pull from {{repo.name}}", {"repo":repo}, e)
    finally:
        Log.stop()
Ejemplo n.º 2
0
    def help_send_alerts(self, to_list):
        try:
            self.setup(to_list)

            ########################################################################
            # TEST
            ########################################################################
            send_alerts(
                settings=struct.wrap({"param": {"debug": True}}),
                db=self.db
            )

            ########################################################################
            # VERIFY
            ########################################################################
            emails = self.get_new_emails() # id, to, body

            if len(to_list) == 0:
                assert len(emails) == 0
                return

            #VERIFY ONE MAIL SENT
            assert len(emails) == 1
            #VERIFY to MATCHES WHAT WAS PASSED TO THIS FUNCTION
            assert set(emails[0].to) == set(to_list), "mail.delivery not matching what's send"

            #VERIFY last_sent IS WRITTEN
            alert_state = self.db.query("""
                SELECT
                    id
                FROM
                    alerts
                WHERE
                    reason={{reason}} AND
                    last_sent>={{send_time}}
            """, {
                "reason": self.reason,
                "send_time": self.now
            })
            expected_marked = set([d.id for d in self.test_data if CNV.JSON2object(d.details).expect == 'pass'])
            actual_marked = set(Q.select(alert_state, "id"))
            assert expected_marked == actual_marked, expand_template(
                "Expecting only id in {{expected}}, but instead got {{actual}}", {
                    "expected": str(expected_marked),
                    "actual": str(actual_marked)
                })

            #VERIFY BODY HAS THE CORRECT ALERTS
            expecting_alerts = set([d.id for d in map(lambda d: CNV.JSON2object(d.details), self.test_data) if d.expect == 'pass'])
            actual_alerts_sent = set([
                CNV.value2int(between(b, ">>>>", "<<<<"))
                for b in emails[0].body.split(dzAlerts.daemons.alert.SEPARATOR)
                if CNV.value2int(between(b, ">>>>", "<<<<")) != None
            ])
            assert expecting_alerts == actual_alerts_sent
        except Exception, e:
            Log.error("Test failure", e)
Ejemplo n.º 3
0
def _es_terms2(es, mvel, query):
    """
    WE ASSUME THERE ARE JUST TWO EDGES, AND EACH HAS A SIMPLE value
    """

    # REQUEST VALUES IN FIRST DIMENSION
    q1 = query.copy()
    q1.edges = query.edges[0:1:]
    values1 = es_terms(es, mvel, q1).edges[0].domain.partitions.value

    select = struct.listwrap(query.select)
    esQuery = buildESQuery(query)
    for s in select:
        for i, v in enumerate(values1):
            esQuery.facets[s.name + "," + str(i)] = {
                "terms": {
                    "field": query.edges[1].value,
                    "size": nvl(query.limit, 200000)
                },
                "facet_filter": simplify({"and": [
                    query.where,
                    {"term": {query.edges[0].value: v}}
                ]})
            }

    data = es_query_util.post(es, esQuery, query.limit)

    # UNION ALL TERMS FROM SECOND DIMENSION
    values2 = set()
    for k, f in data.facets.items():
        values2.update(f.terms.term)
    values2 = Q.sort(values2)
    term2index = {v: i for i, v in enumerate(values2)}
    query.edges[1].domain.partitions = StructList([{"name": v, "value": v} for v in values2])

    # MAKE CUBE
    output = {}
    dims = [len(values1), len(values2)]
    for s in select:
        output[s.name] = Matrix(*dims)

    # FILL CUBE
    # EXPECTING ONLY SELECT CLAUSE FACETS
    for facetName, facet in data.facets.items():
        coord = facetName.split(",")
        s = [s for s in select if s.name == coord[0]][0]
        i1 = int(coord[1])
        for term in facet.terms:
            i2 = term2index[term.term]
            output[s.name][(i1, i2)] = term[aggregates[s.aggregate]]

    cube = Cube(query.select, query.edges, output)
    cube.query = query
    return cube
Ejemplo n.º 4
0
def send_alerts(settings, db):
    """
    BLINDLY SENDS ALERTS FROM THE ALERTS TABLE, ASSUMING ALL HAVE THE SAME STRUCTURE.
    """
    debug = settings.param.debug
    db.debug = debug

    #TODO: REMOVE, LEAVE IN DB
    if db.debug:
        db.execute("update reasons set email_subject={{subject}}, email_template={{template}} where code={{reason}}", {
            "template": CNV.object2JSON(TEMPLATE),
            "subject": CNV.object2JSON(SUBJECT),
            "reason": REASON
        })
        db.flush()

    try:
        new_alerts = db.query("""
            SELECT
                a.id alert_id,
                a.reason,
                r.description,
                a.details,
                a.severity,
                a.confidence,
                a.revision,
                r.email_template,
                r.email_subject
            FROM
                alerts a
            JOIN
                reasons r on r.code = a.reason
            WHERE
                a.last_sent IS NULL AND
                a.status <> 'obsolete' AND
                math.bayesian_add(a.severity, a.confidence) > {{alert_limit}} AND
                a.solution IS NULL AND
                a.reason in {{reasons}} AND
                a.create_time > {{min_time}}
            ORDER BY
                math.bayesian_add(a.severity, a.confidence) DESC,
                json.number(left(details, 65000), "diff_percent") DESC
            LIMIT
                10
        """, {
            "last_sent": datetime.utcnow() - RESEND_AFTER,
            "alert_limit": ALERT_LIMIT - EPSILON,
            "min_time": datetime.utcnow()-LOOK_BACK,
            "reasons": SQL("("+", ".join(db.quote_value(v) for v in SEND_REASONS)+")")
        })

        if not new_alerts:
            if debug:
                Log.note("Nothing important to email")
            return

        for alert in new_alerts:
            #poor souls that signed up for emails
            listeners = ";".join(db.query("SELECT email FROM listeners WHERE reason={{reason}}", {"reason": alert.reason}).email)

            body = [HEADER]
            if alert.confidence >= 1:
                alert.confidence = 0.999999

            alert.details = CNV.JSON2object(alert.details)
            try:
                alert.revision = CNV.JSON2object(alert.revision)
            except Exception, e:
                pass
            alert.score = str(-log(1.0-Math.bayesian_add(alert.severity, alert.confidence), 10))  #SHOW NUMBER OF NINES
            alert.details.url = alert.details.page_url
            example = alert.details.example
            for e in alert.details.tests.example + [example]:
                if e.push_date_min:
                    e.push_date_max = (2 * e.push_date) - e.push_date_min
                    e.date_range = (datetime.utcnow()-CNV.milli2datetime(e.push_date_min)).total_seconds()/(24*60*60)  #REQUIRED FOR DATAZILLA B2G CHART REFERENCE
                    e.date_range = nvl(nvl(*[v for v in (7, 30, 60) if v > e.date_range]), 90)  #PICK FIRST v > CURRENT VALUE

            subject = expand_template(CNV.JSON2object(alert.email_subject), alert)
            body.append(expand_template(CNV.JSON2object(alert.email_template), alert))
            body = "".join(body)+FOOTER

            if debug:
                Log.note("EMAIL: {{email}}", {"email": body})

            if len(body) > MAX_EMAIL_LENGTH:
                Log.note("Truncated the email body")
                suffix = "... (has been truncated)"
                body = body[0:MAX_EMAIL_LENGTH - len(suffix)] + suffix   #keep it reasonable

            db.call("mail.send", (
                listeners, #to
                subject,
                body, #body
                None
            ))

            #I HOPE I CAN SEND ARRAYS OF NUMBERS
            db.execute(
                "UPDATE alerts SET last_sent={{time}} WHERE {{where}}", {
                    "time": datetime.utcnow(),
                    "where": esfilter2sqlwhere(db, {"terms": {"id": Q.select(new_alerts, "alert_id")}})
                })

    except Exception, e:
        Log.error("Could not send alerts", e)
    def insert_test_results(self, test_data):
        ## diff_time IS REQUIRED TO TRANSLATE THE TEST DATE DATES TO SOMETHING MORE CURRENT
        now_time = CNV.datetime2unix(datetime.utcnow())
        max_time = max(Q.select(test_data, "timestamp"))
        diff_time = now_time - max_time

        ## INSERT THE TEST RESULTS
        for t in test_data:
            time = t.timestamp
            time += diff_time

            self.db.insert("test_data_all_dimensions", {
                "id": SQL("util.newid()"),
                "test_run_id": SQL("util.newid()"),
                "product_id": 0,
                "operating_system_id": 0,
                "test_id": 0,
                "page_id": self.page_id,
                "date_received": time,
                "revision": "ba928cbd5191",
                "product": "Firefox",
                "branch": "Mozilla-Inbound",
                "branch_version": "23.0a1",
                "operating_system_name": "mac",
                "operating_system_version": "OS X 10.8",
                "processor": "x86_64",
                "build_type": "opt",
                "machine_name": "talos-mtnlion-r5-049",
                "pushlog_id": 19998363,
                "push_date": time,
                "test_name": "tp5o",
                "page_url": self.url,
                "mean": float(t.mean),
                "std": sqrt(t.variance),
                "h0_rejected": 0,
                "p": None,
                "n_replicates": t.count,
                "fdr": 0,
                "trend_mean": None,
                "trend_std": None,
                "test_evaluation": 0,
                "status": 1
            })

        self.db.flush()
        self.db.execute("""
            INSERT INTO objectstore (id, test_run_id, date_loaded, processed_exception, branch, json_blob)
            SELECT
                {{id}},
                test_run_id,
                {{now}},
                'complete',
                branch,
                '{}'
            FROM
                test_data_all_dimensions
            GROUP BY
                test_run_id
        """, {
            "id": SQL("util.newid()"),
            "now": CNV.datetime2unix(datetime.utcnow())
        })
def alert_sustained_median(settings, qb, alerts_db):
    """
    find single points that deviate from the trend
    """
    # OBJECTSTORE = settings.objectstore.schema + ".objectstore"
    # TDAD = settings.perftest.schema + ".test_data_all_dimensions"
    TDAD = settings.query["from"]
    PUSH_DATE = "datazilla.date_loaded"

    debug = nvl(settings.param.debug, DEBUG)
    query = settings.query

    def is_bad(r):
        if settings.param.sustained_median.trigger < r.result.confidence:
            test_param = nvl(settings.param.test[literal_field(r.Talos.Test.name)], settings.param.suite[literal_field(r.Talos.Test.suite)])

            if test_param == None:
                return True

            if test_param.disable:
                return False

            if test_param.better == "higher":
                diff = -r.diff
            elif test_param.better == "lower":
                diff = r.diff
            else:
                diff = abs(r.diff)  # DEFAULT = ANY DIRECTION IS BAD

            if test_param.min_regression:
                if unicode(test_param.min_regression.strip()[-1]) == "%":
                    min_diff = Math.abs(r.past_stats.mean * float(test_param.min_regression.strip()[:-1]) / 100.0)
                else:
                    min_diff = Math.abs(float(test_param.min_regression))
            else:
                min_diff = Math.abs(r.past_stats.mean * 0.01)

            if diff > min_diff:
                return True

        return False

    with Timer("pull combinations"):
        disabled_suites = [s for s, p in settings.param.suite.items() if p.disable]
        disabled_tests = [t for t, p in settings.param.test.items() if p.disable]

        temp = Query({
            "from": TDAD,
            "select": {"name": "min_push_date", "value": PUSH_DATE, "aggregate": "min"},
            "edges": query.edges,
            "where": {"and": [
                True if settings.args.restart else {"missing": {"field": settings.param.mark_complete}},
                {"exists": {"field": "result.test_name"}},
                {"range": {PUSH_DATE: {"gte": OLDEST_TS}}},
                {"not": {"terms": {"Talos.Test.suite": disabled_suites}}},
                {"not": {"terms": {"Talos.Test.name": disabled_tests}}}
                # {"term": {"testrun.suite": "cart"}},
                # {"term": {"result.test_name": "1-customize-enter.error.TART"}},
                # {"term": {"test_machine.osversion": "OS X 10.8"}}
                #FOR DEBUGGING SPECIFIC SERIES
                # {"term": {"test_machine.type": "hamachi"}},
                # {"term": {"test_machine.platform": "Gonk"}},
                # {"term": {"test_machine.os": "Firefox OS"}},
                # {"term": {"test_build.branch": "master"}},
                # {"term": {"testrun.suite": "communications/ftu"}},
                # {"term": {"result.test_name": "startup_time"}}
            ]},
            "limit": nvl(settings.param.combo_limit, 1000)
        }, qb)

        new_test_points = qb.query(temp)

    #BRING IN ALL NEEDED DATA
    if debug:
        Log.note("Pull all data for {{num}} groups:\n{{groups.name}}", {
            "num": len(new_test_points),
            "groups": query.edges
        })

    # all_min_date = Null
    all_touched = set()
    evaled_tests = set()
    alerts = []   # PUT ALL THE EXCEPTION ITEMS HERE
    for g, test_points in Q.groupby(new_test_points, query.edges):
        if not test_points.min_push_date:
            continue
        try:
            if settings.args.restart:
                first_sample = OLDEST_TS
            else:
                first_sample = MAX(MIN(test_points.min_push_date), OLDEST_TS)
            # FOR THIS g, HOW FAR BACK IN TIME MUST WE GO TO COVER OUR WINDOW_SIZE?
            first_in_window = qb.query({
                "select": {"name": "min_date", "value": "push_date", "aggregate": "min"},
                "from": {
                    "from": TDAD,
                    "select": {"name": "push_date", "value": PUSH_DATE},
                    "where": {"and": [
                        {"term": g},
                        {"range": {PUSH_DATE: {"lt": first_sample}}}
                    ]},
                    "sort": {"field": PUSH_DATE, "sort": -1},
                    "limit": settings.param.sustained_median.window_size * 2
                }
            })
            if len(first_in_window) > settings.param.sustained_median.window_size * 2:
                do_all = False
            else:
                do_all = True

            min_date = MIN(first_sample, first_in_window.min_date)

            #LOAD TEST RESULTS FROM DATABASE
            test_results = qb.query({
                "from": {
                    "from": "talos",
                    "select": [{"name": "push_date", "value": PUSH_DATE}] +
                              query.select +
                              query.edges,
                    "where": {"and": [
                        {"term": g},
                        {"range": {PUSH_DATE: {"gte": min_date}}}
                    ]},
                },
                "sort": "push_date"
            })

            Log.note("{{num}} test results found for {{group}} dating back no further than {{start_date}}", {
                "num": len(test_results),
                "group": g,
                "start_date": CNV.milli2datetime(min_date)
            })

            if debug:
                Log.note("Find sustained_median exceptions")

            #APPLY WINDOW FUNCTIONS
            stats = Q.run({
                "from": {
                    "from": test_results,
                    "where": {"exists": {"field": "value"}}
                },
                "window": [
                    {
                        # WE DO NOT WANT TO CONSIDER THE POINTS BEFORE FULL WINDOW SIZE
                        "name": "ignored",
                        "value": lambda r, i: False if do_all or i > settings.param.sustained_median.window_size else True
                    }, {
                        # SO WE CAN SHOW A DATAZILLA WINDOW
                        "name": "push_date_min",
                        "value": lambda r: r.push_date,
                        "sort": "push_date",
                        "aggregate": windows.Min,
                        "range": {"min": -settings.param.sustained_median.window_size, "max": 0}
                    }, {
                        # SO WE CAN SHOW A DATAZILLA WINDOW
                        "name": "push_date_max",
                        "value": lambda r: r.push_date,
                        "sort": "push_date",
                        "aggregate": windows.Max,
                        "range": {"min": 0, "max": settings.param.sustained_median.window_size}
                    }, {
                        "name": "past_revision",
                        "value": lambda r, i, rows: rows[i - 1].Talos.Revision,
                        "sort": "push_date"
                    }, {
                        "name": "past_stats",
                        "value": lambda r: r.value,
                        "sort": "push_date",
                        "aggregate": windows.Stats(middle=0.60),
                        "range": {"min": -settings.param.sustained_median.window_size, "max": 0}
                    }, {
                        "name": "future_stats",
                        "value": lambda r: r.value,
                        "sort": "push_date",
                        "aggregate": windows.Stats(middle=0.60),
                        "range": {"min": 0, "max": settings.param.sustained_median.window_size}
                    }, {
                        "name": "result",
                        "value": lambda r, i, rows: median_test(
                            rows[-settings.param.sustained_median.window_size + i:i:].value,
                            rows[i:settings.param.sustained_median.window_size + i:].value,
                            interpolate=False
                        ),
                        "sort": "push_date"
                    }, {
                        "name": "diff",
                        "value": lambda r: r.future_stats.mean - r.past_stats.mean
                    }, {
                        "name": "diff_percent",
                        "value": lambda r: (r.future_stats.mean - r.past_stats.mean) / r.past_stats.mean
                    }, {
                        "name": "is_diff",
                        "value": is_bad
                    }, {
                        #USE THIS TO FILL CONFIDENCE HOLES
                        #WE CAN MARK IT is_diff KNOWING THERE IS A HIGHER CONFIDENCE
                        "name": "future_is_diff",
                        "value": lambda r, i, rows: rows[i - 1].is_diff and r.result.confidence < rows[i - 1].result.confidence,
                        "sort": "push_date"
                    }, {
                        #WE CAN MARK IT is_diff KNOWING THERE IS A HIGHER CONFIDENCE
                        "name": "past_is_diff",
                        "value": lambda r, i, rows: rows[i - 1].is_diff and r.result.confidence < rows[i - 1].result.confidence,
                        "sort": {"value": "push_date", "sort": -1}
                    },
                ]
            })

            #PICK THE BEST SCORE FOR EACH is_diff==True REGION
            for g2, data in Q.groupby(stats, "is_diff", contiguous=True):
                if g2.is_diff:
                    best = Q.sort(data, ["result.confidence", "diff"]).last()
                    best["pass"] = True

            all_touched.update(Q.select(test_results, ["test_run_id", "Talos.Test"]))

            # TESTS THAT HAVE BEEN (RE)EVALUATED GIVEN THE NEW INFORMATION
            evaled_tests.update(Q.run({
                "from": test_results,
                "select": ["test_run_id", "Talos.Test"],
                "where": {"term": {"ignored": False}}
            }))

            File("test_values.txt").write(CNV.list2tab(Q.select(stats, [
                {"name": "push_date", "value": lambda x: CNV.datetime2string(CNV.milli2datetime(x.push_date), "%d-%b-%Y %H:%M:%S")},
                "value",
                {"name": "revision", "value": "Talos.Revision"},
                {"name": "confidence", "value": "result.confidence"},
                "pass"
            ])))

            #TESTS THAT HAVE SHOWN THEMSELVES TO BE EXCEPTIONAL
            new_exceptions = Q.filter(stats, {"term": {"pass": True}})
            for v in new_exceptions:
                if v.ignored:
                    continue
                alert = Struct(
                    status="new",
                    create_time=CNV.milli2datetime(v.push_date),
                    tdad_id={"test_run_id": v.test_run_id, "Talos": {"Test": v.Talos.Test}},
                    reason=REASON,
                    revision=v.Talos.Revision,
                    details=v,
                    severity=SEVERITY,
                    confidence=v.result.confidence
                )
                alerts.append(alert)

            if debug:
                Log.note("{{num}} new exceptions found", {"num": len(new_exceptions)})

        except Exception, e:
            Log.warning("Problem with alert identification, continue to log existing alerts and stop cleanly", e)
                "id",
                "tdad_id",
                "status",
                "last_updated",
                "severity",
                "confidence",
                "details",
                "solution"
            ],
            "where": {"and": [
                {"terms": {"tdad_id": evaled_tests}},
                {"term": {"reason": REASON}}
            ]}
        })

    found_alerts = Q.unique_index(alerts, "tdad_id")
    current_alerts = Q.unique_index(current_alerts, "tdad_id")

    new_alerts = found_alerts - current_alerts
    changed_alerts = current_alerts & found_alerts
    obsolete_alerts = Q.filter(current_alerts - found_alerts, {"not": {"term": {"status": "obsolete"}}})

    if debug:
        Log.note("Update Alerts: ({{num_new}} new, {{num_change}} changed, {{num_delete}} obsoleted)", {
            "num_new": len(new_alerts),
            "num_change": len(changed_alerts),
            "num_delete": len(obsolete_alerts)
        })

    if new_alerts:
        for a in new_alerts:
def page_threshold_limit(db, debug):
    """
    simplest of rules to test the dataflow from test_run, to alert, to email
    may prove slightly useful also!
    #point out any pages that are breaking human-set threshold limits
    """
    db.debug = debug

    try:
        # CALCULATE HOW FAR BACK TO LOOK
        lasttime = db.query("SELECT last_run, description FROM reasons WHERE code={{type}}", {"type": REASON})[0]
        lasttime = nvl(lasttime.last_run, datetime.utcnow())
        min_date = lasttime + LOOK_BACK

        # FIND ALL PAGES THAT HAVE LIMITS TO TEST
        # BRING BACK ONES THAT BREAK LIMITS
        # BUT DO NOT ALREADY HAVE AN ALERTS EXISTING
        pages = db.query(
            """
            SELECT
                t.id tdad_id,
                t.n_replicates,
                t.mean,
                t.std,
                h.threshold,
                h.severity,
                h.reason,
                m.id alert_id
            FROM
                alert_page_thresholds h
            JOIN
                test_data_all_dimensions t ON t.page_id=h.page
            LEFT JOIN
                alerts m on m.tdad_id=t.test_run_id AND m.reason={{type}}
            WHERE
                h.threshold<t.mean AND
                t.push_date>{{min_date}} AND
                (m.id IS NULL OR m.status='obsol11ete')
        """,
            {"type": REASON, "min_date": min_date},
        )

        # FOR EACH PAGE THAT BREAKS LIMITS
        for page in pages:
            if page.alert_id != None:
                break

            alert = {
                "id": SQL("util.newid()"),
                "status": "new",
                "create_time": datetime.utcnow(),
                "last_updated": datetime.utcnow(),
                "tdad_id": page.tdad_id,
                "reason": REASON,
                "details": CNV.object2JSON(
                    {"expected": float(page.threshold), "actual": float(page.mean), "reason": page.reason}
                ),
                "severity": page.severity,
                "confidence": 1.0  # USING NORMAL DIST ASSUMPTION WE CAN ADJUST
                # CONFIDENCE EVEN BEFORE THRESHOLD IS HIT!
                # FOR NOW WE KEEP IT SIMPLE
            }

            db.insert("alerts", alert)

        for page in pages:
            if page.alert_id == None:
                break
            db.update("alerts", None)  # ERROR FOR NOW

        # OBSOLETE THE ALERTS THAT SHOULD NO LONGER GET SENT
        obsolete = db.query(
            """
            SELECT
                m.id,
                m.tdad_id
            FROM
                alerts m
            JOIN
                test_data_all_dimensions t ON m.tdad_id=t.id
            JOIN
                alert_page_thresholds h on t.page_id=h.page
            WHERE
                m.reason={{reason}} AND
                h.threshold>=t.mean AND
                t.push_date>{{time}}
        """,
            {"reason": REASON, "time": min_date},
        )

        if obsolete:
            db.execute(
                "UPDATE alerts SET status='obsolete' WHERE {{where}}",
                {"where": esfilter2sqlwhere(db, {"terms": {"id": Q.select(obsolete, "id")}})},
            )

        db.execute(
            "UPDATE reasons SET last_run={{now}} WHERE code={{reason}}", {"now": datetime.utcnow(), "reason": REASON}
        )

    except Exception, e:

        Log.error("Could not perform threshold comparisons", e)
def talos_alert_revision(settings):
    assert settings.alerts != None
    settings.db.debug = settings.param.debug
    with DB(settings.alerts) as db:
        with ESQuery(ElasticSearch(settings.query["from"])) as esq:

            dbq = DBQuery(db)
            esq.addDimension(CNV.JSON2object(File(settings.dimension.filename).read()))

            #TODO: REMOVE, LEAVE IN DB
            if db.debug:
                db.execute("update reasons set email_subject={{subject}}, email_template={{template}} where code={{reason}}", {
                    "template": CNV.object2JSON(TEMPLATE),
                    "subject": CNV.object2JSON(SUBJECT),
                    "reason": REASON
                })
                db.flush()

            #EXISTING SUSTAINED EXCEPTIONS
            existing_sustained_alerts = dbq.query({
                "from": "alerts",
                "select": "*",
                "where": {"and": [
                    {"term": {"reason": talos_sustained_median.REASON}},
                    {"not": {"term": {"status": "obsolete"}}},
                    {"range": {"create_time": {"gte": NOW - LOOK_BACK}}}
                ]}
            })

            tests = Q.index(existing_sustained_alerts, ["revision", "details.Talos.Test"])

            #EXISTING REVISION-LEVEL ALERTS
            old_alerts = dbq.query({
                "from": "alerts",
                "select": "*",
                "where": {"and": [
                    {"term": {"reason": REASON}},
                    {"or": [
                        {"terms": {"revision": set(existing_sustained_alerts.revision)}},

                        {"term": {"reason": talos_sustained_median.REASON}},
                        {"term": {"status": "obsolete"}},
                        {"range": {"create_time": {"gte": NOW - LOOK_BACK}}}
                    ]}
                ]}
            })
            old_alerts = Q.unique_index(old_alerts, "revision")

            #SUMMARIZE
            known_alerts = StructList()
            for revision in set(existing_sustained_alerts.revision):
            #FIND TOTAL TDAD FOR EACH INTERESTING REVISION
                total_tests = esq.query({
                    "from": "talos",
                    "select": {"name": "count", "aggregate": "count"},
                    "where": {"and":[
                        {"terms": {"Talos.Revision": revision}}
                    ]}
                })
                total_exceptions = tests[(revision, )]  # FILTER BY revision

                parts = StructList()
                for g, exceptions in Q.groupby(total_exceptions, ["details.Talos.Test"]):
                    worst_in_test = Q.sort(exceptions, ["confidence", "details.diff_percent"]).last()
                    example = worst_in_test.details
                    # ADD SOME DATAZILLA SPECIFIC URL PARAMETERS
                    branch = example.Talos.Branch.replace("-Non-PGO", "")
                    example.tbpl.url.branch = TBPL_PATH.get(branch, branch)
                    example.mercurial.url.branch = MECURIAL_PATH.get(branch, branch)
                    example.datazilla.url.branch = example.Talos.Branch #+ ("" if worst_in_test.Talos.Branch.pgo else "-Non-PGO")
                    example.datazilla.url.x86 = "true" if example.Talos.Platform == "x86" else "false"
                    example.datazilla.url.x86_64 = "true" if example.Talos.Platform == "x86_64" else "false"
                    example.datazilla.url.stop = nvl(example.push_date_max, (2*example.push_date) - example.push_date_min)

                    num_except = len(exceptions)
                    if num_except == 0:
                        continue

                    part = {
                        "test": g.details.Talos.Test,
                        "num_exceptions": num_except,
                        "num_tests": total_tests,
                        "confidence": worst_in_test.confidence,
                        "example": example
                    }
                    parts.append(part)

                parts = Q.sort(parts, [{"field": "confidence", "sort": -1}])
                worst_in_revision = parts[0].example

                known_alerts.append({
                    "status": "new",
                    "create_time": CNV.milli2datetime(worst_in_revision.push_date),
                    "reason": REASON,
                    "revision": revision,
                    "tdad_id": revision,
                    "details": {
                        "revision": revision,
                        "total_tests": total_tests,
                        "total_exceptions": len(total_exceptions),
                        "tests": parts,
                        "example": worst_in_revision
                    },
                    "severity": SEVERITY,
                    "confidence": worst_in_revision.result.confidence
                })

            known_alerts = Q.unique_index(known_alerts, "revision")

            #NEW ALERTS, JUST INSERT
            new_alerts = known_alerts - old_alerts
            if new_alerts:
                for revision in new_alerts:
                    revision.id = SQL("util.newid()")
                    revision.last_updated = NOW
                db.insert_list("alerts", new_alerts)

            #SHOW SUSTAINED ALERTS ARE COVERED
            db.execute("""
                INSERT INTO hierarchy (parent, child)
                SELECT
                    r.id parent,
                    p.id child
                FROM
                    alerts p
                LEFT JOIN
                    hierarchy h on h.child=p.id
                LEFT JOIN
                    alerts r on r.revision=p.revision AND r.reason={{parent_reason}}
                WHERE
                    {{where}}
            """, {
                "where": esfilter2sqlwhere(db, {"and": [
                    {"term": {"p.reason": talos_sustained_median.REASON}},
                    {"terms": {"p.revision": Q.select(existing_sustained_alerts, "revision")}},
                    {"missing": "h.parent"}
                ]}),
                "parent_reason": REASON
            })

            #CURRENT ALERTS, UPDATE IF DIFFERENT
            for known_alert in known_alerts & old_alerts:
                if len(nvl(known_alert.solution, "").strip()) != 0:
                    continue  # DO NOT TOUCH SOLVED ALERTS

                old_alert = old_alerts[known_alert]
                if old_alert.status == 'obsolete' or significant_difference(known_alert.severity, old_alert.severity) or significant_difference(known_alert.confidence, old_alert.confidence):
                    known_alert.last_updated = NOW
                    db.update("alerts", {"id": old_alert.id}, known_alert)

            #OLD ALERTS, OBSOLETE
            for old_alert in old_alerts - known_alerts:
                if old_alert.status == 'obsolete':
                    continue
                db.update("alerts", {"id": old_alert.id}, {"status": "obsolete", "last_updated": NOW, "details":None})
Ejemplo n.º 10
0
 def verify_delivery(self, mail_content, to_list):
     #VERIFY DELIVERY IN DATABASE IS SAME AS LIST
     mail_delivery = self.db.query("SELECT id, deliver_to FROM mail.delivery WHERE content={{content_id}}", {"content_id": mail_content.id})
     mail_delivery = set(Q.select(mail_delivery, "deliver_to"))
     assert mail_delivery == set(to_list)