Esempio n. 1
0
 def lookup_ticket(self, id):
     path = "ticket/?" + urllib.urlencode({'raw': True, 'query': json.dumps({'id': id})})
     res = self.load_json_from_server(path)
     if res:
         return res[0]
     else:
         return scrape(id)
Esempio n. 2
0
def lookup_ticket(server, id):
    url = server + "/ticket/?" + urllib.urlencode({'raw': True, 'query': json.dumps({'id': id})})
    res = json.load(urllib2.urlopen(url))
    if res:
        return res[0]
    else:
        return scrape(id)
Esempio n. 3
0
def post_report(ticket_id):
    """
    Posting a report to the database of reports.
    """
    try:
        ticket = tickets.find_one({'id': ticket_id})
        if ticket is None:
            ticket = scrape(ticket_id)
        if 'reports' not in ticket:
            ticket['reports'] = []
        report = json.loads(request.form.get('report'))
        assert (isinstance(report, dict)), "report is not a dict"
        for fld in ['status', 'spkgs', 'base', 'machine', 'time']:
            assert (fld in report), "{} missing in report".format(fld)

        machine_name = report['machine'][-1]
        if machine_name in BLACKLIST:
            msg = 'machine {} is blacklisted'.format(machine_name)
            raise RuntimeError(msg)

        prune_pending(ticket, report['machine'])
        ticket['reports'].append(report)
        db.logs.put(request.files.get('log'), _id=log_name(ticket_id, report))
        if 'retry' in ticket:
            ticket['retry'] = False
        ticket['last_activity'] = now_str()
        db.save_ticket(ticket)
        return "ok (report successfully posted)"
    except:
        traceback.print_exc()
        return "error in posting the report"
Esempio n. 4
0
def render_ticket_status_svg(ticket):
    """
    Return the svg status image for the given ticket.
    """
    try:
        if 'fast' in request.args:
            info = tickets.find_one({'id': ticket})
        else:
            info = scrape(ticket, db=db)
    except:
        info = tickets.find_one({'id': ticket})

    if 'base' in request.args:
        base = request.args.get('base')
    else:
        base = latest_version(info.get('reports', []))

    status = get_ticket_status(info, base=base)[1]  # single status
    path = status_image_path(status, image_type='svg')

    # with no base
    response = make_response(open(path).read())
    response.headers['Content-type'] = 'image/svg+xml'
    response.headers['Cache-Control'] = 'no-cache'
    return response
Esempio n. 5
0
def render_ticket_status(ticket):
    """
    Return the status image for the given ticket.

    This now only renders a png image with the base.
    """
    try:
        if 'fast' in request.args:
            info = tickets.find_one({'id': ticket})
        else:
            info = scrape(ticket, db=db)
    except:
        info = tickets.find_one({'id': ticket})

    if 'base' in request.args:
        base = request.args.get('base')
    else:
        base = latest_version(info.get('reports', []))

    # status = get_ticket_status(info, base=base)[2]  # composite status

    if 'fast' in request.args:
        display_base = None
    else:
        display_base = base

    response = make_response(create_base_image(base=display_base))
    response.headers['Content-type'] = 'image/png'
    response.headers['Cache-Control'] = 'no-cache'
    return response
Esempio n. 6
0
def render_ticket_status(ticket):
    try:
        info = trac.scrape(ticket, db=db)
    except:
        info = tickets.find_one({'id': ticket})
    if 'reports' in info:
        base = latest_version(current_reports(info))
    else:
        base = None
    status = get_ticket_status(info, base=base)[2]
    response = make_response(create_status_image(status, base=base))
    response.headers['Content-type'] = 'image/png'
    response.headers['Cache-Control'] = 'no-cache'
    return response
Esempio n. 7
0
def render_ticket(ticket):
    try:
        info = trac.scrape(ticket, db=db, force='force' in request.args)
    except:
        info = tickets.find_one({'id': ticket})
    if info is None:
        return "No such ticket."
    if 'kick' in request.args:
        info['retry'] = True
        db.save_ticket(info)
    if 'reports' in info:
        info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
    else:
        info['reports'] = []

    old_reports = list(info['reports'])
    patchbot.prune_pending(info)
    if old_reports != info['reports']:
        db.save_ticket(info)

    def format_info(info):
        new_info = {}
        for key, value in info.items():
            if key == 'patches':
                new_info['patches'] = format_patches(ticket, value)
            elif key == 'reports' or key == 'pending':
                pass
            elif key == 'depends_on':
                new_info[key] = ', '.join("<a href='/ticket/%s'>%s</a>" % (a, a) for a in value)
            elif key == 'authors':
                new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a,a) for a in value)
            elif key == 'participants':
                new_info[key] = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a,a) for a in value)
            elif isinstance(value, list):
                new_info[key] = ', '.join(value)
            elif key not in ('id', '_id'):
                new_info[key] = value
        return new_info
    def preprocess_reports(all):
        for item in all:
            if 'patches' in item:
                required = info['depends_on'] + info['patches']
                item['patch_list'] = format_patches(ticket, item['patches'], item.get('deps'), required)
            if item['base'] != base:
                item['base'] = "<span style='color: red'>%s</span>" % item['base']
            if 'time' in item:
                item['log'] = log_name(info['id'], item)
            yield item
    return render_template("ticket.html", reports=preprocess_reports(info['reports']), ticket=ticket, info=format_info(info), status=get_ticket_status(info, base=base)[2])
Esempio n. 8
0
def render_ticket_status(ticket):
    try:
        if 'fast' in request.args:
            info = tickets.find_one({'id': ticket})
        else:
            info = trac.scrape(ticket, db=db)
    except:
        info = tickets.find_one({'id': ticket})
    if 'base' in request.args:
        base = request.args.get('base')
    else:
        base = latest_version(info['reports'] or [])
    status = get_ticket_status(info, base=base)[2]
    if 'fast' in request.args:
        display_base = None
    else:
        display_base = base
    response = make_response(create_status_image(status, base=display_base))
    response.headers['Content-type'] = 'image/png'
    response.headers['Cache-Control'] = 'no-cache'
    return response
Esempio n. 9
0
def post_report(ticket_id):
    try:
        ticket = db.lookup_ticket(ticket_id)
        if ticket is None:
            ticket = trac.scrape(ticket_id)
        if 'reports' not in ticket:
            ticket['reports'] = []
        report = json.loads(request.form.get('report'))
        assert isinstance(report, dict)
        for fld in ['status', 'patches', 'spkgs', 'base', 'machine', 'time']:
            assert fld in report
        patchbot.prune_pending(ticket, report['machine'])
        ticket['reports'].append(report)
        db.logs.put(request.files.get('log'), _id=log_name(ticket_id, report))
        if 'retry' in ticket:
            ticket['retry'] = False
        ticket['last_activity'] = now_str()
        db.save_ticket(ticket)
        return "ok"
    except:
        traceback.print_exc()
        return "error"
Esempio n. 10
0
def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
    base = get_base(sage_root)
    if ticket is None:
        ticket = get_ticket(base=base, server=server, **conf)
    else:
        ticket = None, scrape(int(ticket))
    if not ticket:
        print "No more tickets."
        if random.random() < 0.01:
            cleanup(sage_root, server)
        time.sleep(conf['idle'])
        return
    rating, ticket = ticket
    print "\n" * 2
    print "=" * 30, ticket['id'], "=" * 30
    print ticket['title']
    print "score", rating
    print "\n" * 2
    log_dir = sage_root + "/logs"
    if not os.path.exists(log_dir):
        os.mkdir(log_dir)
    log = '%s/%s-log.txt' % (log_dir, ticket['id'])
    report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], user=conf['user'], log=None)
    plugins_results = []
    try:
        with Tee(log, time=True, timeout=conf['timeout']):
            t = Timer()
            start_time = time.time()

            state = 'started'
            os.environ['MAKE'] = "make -j%s" % conf['parallelism']
            os.environ['SAGE_ROOT'] = sage_root
            # TODO: Ensure that sage-main is pristine.
            pull_from_trac(sage_root, ticket['id'], force=True)
            t.finish("Apply")
            state = 'applied'
            
            do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id'])
            t.finish("Build")
            state = 'built'
            
            working_dir = "%s/devel/sage-%s" % (sage_root, ticket['id'])
            # Only the ones on this ticket.
            patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
            kwds = {
                "original_dir": "%s/devel/sage-0" % sage_root,
                "patched_dir": working_dir,
                "patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches if p],
            }
            for name, plugin in conf['plugins']:
                try:
                    print plugin_boundary(name)
                    plugin(ticket, **kwds)
                    passed = True
                except Exception:
                    traceback.print_exc()
                    passed = False
                finally:
                    t.finish(name)
                    print plugin_boundary(name, end=True)
                    plugins_results.append((name, passed))
                    
            test_dirs = ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
            if conf['parallelism'] > 1:
                test_cmd = "-tp %s" % conf['parallelism']
            else:
                test_cmd = "-t"
            do_or_die("$SAGE_ROOT/sage %s -sagenb %s" % (test_cmd, ' '.join(test_dirs)))
            #do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
            #do_or_die('sage -testall')
            t.finish("Tests")
            state = 'tested'
            
            if not all(passed for name, passed in plugins_results):
                state = 'failed_plugin'

            print
            t.print_all()
    except Exception:
        traceback.print_exc()
    
    for _ in range(5):
        try:
            print "Reporting", ticket['id'], status[state]
            report_ticket(server, ticket, status=status[state], base=base, machine=conf['machine'], user=conf['user'], log=log, plugins=plugins_results)
            print "Done reporting", ticket['id']
            break
        except urllib2.HTTPError:
            traceback.print_exc()
            time.sleep(conf['idle'])
    else:
        print "Error reporting", ticket['id']
    return status[state]
Esempio n. 11
0
def render_ticket(ticket):
    try:
        info = trac.scrape(ticket, db=db, force='force' in request.args)
    except:
        info = tickets.find_one({'id': ticket})
    if info is None:
        return "No such ticket."
    if 'kick' in request.args:
        info['retry'] = True
        db.save_ticket(info)
    if 'reports' in info:
        info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
    else:
        info['reports'] = []
    
    base_reports = base_reports_by_machine_and_base()
        
    old_reports = list(info['reports'])
    patchbot.prune_pending(info)
    if old_reports != info['reports']:
        db.save_ticket(info)

    def format_info(info):
        new_info = {}
        for key, value in info.items():
            if key == 'patches':
                new_info['patches'] = format_patches(ticket, value)
            elif key == 'reports' or key == 'pending':
                pass
            elif key == 'depends_on':
                deps_status = {}
                for dep in tickets.find({'id': {'$in': [int(a) for a in value]}}, ['status', 'id']):
                    if 'closed' in dep['status']:
                        dep['style'] = 'text-decoration: line-through'
                    else:
                        dep['style'] = ''
                    deps_status[dep['id']] = dep
                new_info[key] = ', '.join("<img src='/ticket/%s/status.png?fast' height=16><a href='/ticket/%s' style='%s'>%s</a>" % (a, a, deps_status[a]['style'], a) for a in value)
            elif key == 'authors':
                new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a,a) for a in value)
            elif key == 'participants':
                new_info[key] = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a,a) for a in value)
            elif isinstance(value, list):
                new_info[key] = ', '.join(value)
            elif key not in ('id', '_id'):
                new_info[key] = value
        return new_info
    def preprocess_reports(all):
        for item in all:
            base_report = base_reports.get(item['base'] + "/" + "/".join(item['machine']), base_reports.get(item['base']))
            if base_report:
                item['base_log'] = urllib.quote(log_name(0, base_report))
            if 'patches' in item:
                required = info['depends_on'] + info['patches']
                item['patch_list'] = format_patches(ticket, item['patches'], item.get('deps'), required)
            item['raw_base'] = item['base']
            if item['base'] != base:
                item['base'] = "<span style='color: red'>%s</span>" % item['base']
            if 'time' in item:
                item['log'] = log_name(info['id'], item)
            yield item
    def normalize_plugin(plugin):
        while len(plugin) < 3:
            plugin.append(None)
        return plugin
    return render_template("ticket.html", reports=preprocess_reports(info['reports']), ticket=ticket, info=format_info(info), status=get_ticket_status(info, base=base)[2], normalize_plugin=normalize_plugin)
Esempio n. 12
0
def render_ticket(ticket):
    try:
        info = trac.scrape(ticket, db=db, force='force' in request.args)
    except:
        info = tickets.find_one({'id': ticket})
    if info is None:
        return "No such ticket."
    if 'kick' in request.args:
        info['retry'] = True
        db.save_ticket(info)
    if 'reports' in info:
        info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
    else:
        info['reports'] = []

    base_reports = base_reports_by_machine_and_base()

    old_reports = list(info['reports'])
    patchbot.prune_pending(info)
    if old_reports != info['reports']:
        db.save_ticket(info)

    def format_info(info):
        new_info = {}
        for key, value in info.items():
            if key == 'patches':
                new_info['patches'] = format_patches(ticket, value)
            elif key == 'reports' or key == 'pending':
                pass
            elif key == 'depends_on':
                deps_status = {}
                def is_int(a):
                    try:
                        int(a)
                        return True
                    except ValueError:
                        return False
                for dep in tickets.find({'id': {'$in': [int(a) for a in value if is_int(a)]}}, ['status', 'id']):
                    if 'closed' in dep['status']:
                        dep['style'] = 'text-decoration: line-through'
                    else:
                        dep['style'] = ''
                    deps_status[dep['id']] = dep
                new_info[key] = ', '.join("<img src='/ticket/%s/status.png?fast' height=16><a href='/ticket/%s' style='%s'>%s</a>" % (a, a, deps_status[a]['style'], a) for a in value)
            elif key == 'authors':
                new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a,a) for a in value)
            elif key == 'participants':
                new_info[key] = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a,a) for a in value)
            elif isinstance(value, list):
                new_info[key] = ', '.join(value)
            elif key not in ('id', '_id'):
                new_info[key] = value
        return new_info
    base = latest_base()
    def format_git_describe(res):
        if res:
            if '-' in res:
                tag, commits = res.split('-')[:2]
                return "%s + %s commits" % (tag, commits)
            elif 'commits' in res:
                # old style
                return res
            else:
                return res + " + 0 commits"
        else:
            return '?'
    def preprocess_reports(all):
        for item in all:
            base_report = base_reports.get(item['base'] + "/" + "/".join(item['machine']), base_reports.get(item['base']))
            if base_report:
                item['base_log'] = urllib.quote(log_name(0, base_report))
            if 'patches' in item:
                required = info['depends_on'] + info['patches']
                item['patch_list'] = format_patches(ticket, item['patches'], item.get('deps'), required)
            if 'git_base' in item:
                git_log = item.get('git_log')
                item['git_log_len'] = '?' if git_log is None else len(git_log)
            item['raw_base'] = item['base']
            if compare_version(item['base'], base) < 0:
                item['base'] = "<span style='color: red'>%s</span>" % item['base']
            if 'time' in item:
                item['log'] = log_name(info['id'], item)
            if 'git_commit_human' not in item:
                item['git_commit_human'] = "%s new commits" % len(item['log'])
            for x in ('commit', 'base', 'merge'):
                field = 'git_%s_human' % x
                item[field] = format_git_describe(item.get(field, None))
            yield item
    def normalize_plugin(plugin):
        while len(plugin) < 3:
            plugin.append(None)
        return plugin
    def sort_fields(items):
        return sorted(items, key=(lambda x: (x[0] != 'title', x)))
    return render_template("ticket.html", reports=preprocess_reports(info['reports']), ticket=ticket, info=format_info(info), status=get_ticket_status(info, base=base)[2], normalize_plugin=normalize_plugin, sort_fields=sort_fields)
Esempio n. 13
0
    def test_a_ticket(self, ticket=None):

        self.reload_config()

        if ticket is None:
            ticket = self.get_ticket()
        else:
            ticket = None, scrape(int(ticket))
        if not ticket:
            print "No more tickets."
            time.sleep(self.config['idle'])
            return

        rating, ticket = ticket
        print "\n" * 2
        print "=" * 30, ticket['id'], "=" * 30
        print ticket['title']
        print "score", rating
        print "\n" * 2
        self.log_dir = self.sage_root + "/logs/patchbot"
        if not os.path.exists(self.log_dir):
            os.makedirs(self.log_dir)
        log = '%s/%s-log.txt' % (self.log_dir, ticket['id'])
        history = open("%s/history.txt" % self.log_dir, "a")
        history.write("%s %s\n" % (datetime(), ticket['id']))
        history.close()
        if not self.plugin_only:
            self.report_ticket(ticket, status='Pending', log=log)
        plugins_results = []
        try:
            t = Timer()
            with Tee(log, time=True, timeout=self.config['timeout'], timer=t):
                print "Sage Patchbot", patchbot_version.get_version()

                if ticket['spkgs']:
                    state = 'spkg'
                    print "\n".join(ticket['spkgs'])
                    print
                    for spkg in ticket['spkgs']:
                        print
                        print '+' * 10, spkg, '+' * 10
                        print
                        try:
                            self.check_spkg(spkg)
                        except Exception:
                            traceback.print_exc()
                        t.finish(spkg)

                if not ticket['spkgs']:
                    state = 'started'
                    os.environ['MAKE'] = "make -j%s" % self.config['parallelism']
                    os.environ['SAGE_ROOT'] = self.sage_root
                    os.environ['GIT_AUTHOR_NAME'] = os.environ['GIT_COMMITTER_NAME'] = 'patchbot'
                    os.environ['GIT_AUTHOR_EMAIL'] = os.environ['GIT_COMMITTER_EMAIL'] = 'patchbot@localhost'
                    os.environ['GIT_AUTHOR_DATE'] = os.environ['GIT_COMMITTER_DATE'] = '1970-01-01T00:00:00'
                    pull_from_trac(self.sage_root, ticket['id'], force=True, use_ccache=self.config['use_ccache'])
                    t.finish("Apply")
                    state = 'applied'
                    if not self.plugin_only:
                        self.report_ticket(ticket, status='Pending', log=log, pending_status=state)

                    do_or_die("$MAKE")
                    t.finish("Build")
                    state = 'built'
                    if not self.plugin_only:
                        self.report_ticket(ticket, status='Pending', log=log, pending_status=state)

                    # TODO: Exclude dependencies.
                    patch_dir = tempfile.mkdtemp()
                    if ticket['id'] != 0:
                        do_or_die("git format-patch -o '%s' patchbot/base..patchbot/ticket_merged" % patch_dir)

                    kwds = {
                        "original_dir": self.sage_root,
                        "patched_dir": os.getcwd(),
                        "patches": [os.path.join(patch_dir, p) for p in os.listdir(patch_dir)],
                        "sage_binary": os.path.join(os.getcwd(), 'sage'),
                        "dry_run": self.dry_run,
                    }

                    for name, plugin in self.config['plugins']:
                        try:
                            if ticket['id'] != 0 and os.path.exists(os.path.join(self.log_dir, '0', name)):
                                baseline = pickle.load(open(os.path.join(self.log_dir, '0', name)))
                            else:
                                baseline = None
                            print plugin_boundary(name)
                            do_or_die("git checkout patchbot/ticket_merged")
                            res = plugin(ticket, is_git=True, baseline=baseline, **kwds)
                            passed = True
                        except Exception:
                            traceback.print_exc()
                            passed = False
                            res = None
                        finally:
                            if isinstance(res, PluginResult):
                                if res.baseline is not None:
                                    plugin_dir = os.path.join(self.log_dir, str(ticket['id']))
                                    if not os.path.exists(plugin_dir):
                                        os.mkdir(plugin_dir)
                                    pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w'))
                                    passed = res.status == PluginResult.Passed
                                    print name, res.status
                                    plugins_results.append((name, passed, res.data))
                            else:
                                plugins_results.append((name, passed, None))
                            t.finish(name)
                            print plugin_boundary(name, end=True)
                    plugins_passed = all(passed for (name, passed, data) in plugins_results)
                    self.report_ticket(ticket, status='Pending', log=log, pending_status='plugins_passed' if plugins_passed else 'plugins_failed')

                    if self.plugin_only:
                        state = 'plugins' if plugins_passed else 'plugins_failed'
                    else:
                        if self.dry_run:
                            test_target = "$SAGE_ROOT/src/sage/misc/a*.py"
                            # TODO: Remove
                            test_target = "$SAGE_ROOT/src/sage/doctest/*.py"
                        else:
                            test_target = "--all --long"
                        if self.config['parallelism'] > 1:
                            test_cmd = "-tp %s" % self.config['parallelism']
                        else:
                            test_cmd = "-t"
                        do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, test_target))
                        t.finish("Tests")
                        state = 'tested'

                        if not plugins_passed:
                            state = 'tests_passed_plugins_failed'

        except (urllib2.HTTPError, socket.error):
            # Don't report failure because the network/trac died...
            print
            t.print_all()
            traceback.print_exc()
            state = 'network_error'
        except Exception:
            traceback.print_exc()

        for _ in range(5):
            try:
                print "Reporting", ticket['id'], status[state]
                self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results, dry_run=self.dry_run)
                print "Done reporting", ticket['id']
                break
            except IOError:
                traceback.print_exc()
                time.sleep(self.config['idle'])
        else:
            print "Error reporting", ticket['id']
        maybe_temp_root = os.environ['SAGE_ROOT']
        if maybe_temp_root.endswith("-sage-git-temp-%s" % ticket['id']):
            shutil.rmtree(maybe_temp_root)
        return status[state]
Esempio n. 14
0
def render_ticket(ticket):
    """
    reports on a given ticket

    possible options: ?force and ?kick

    ?force will refresh the info in the patchbot-server database

    ?kick will tell the patchbot-clients to retry the ticket

    ?base to select reports according to their base
    """
    latest = latest_base()

    if 'base' in request.args:
        chosen_base = request.args.get('base')
    else:
        chosen_base = 'all' if ticket != 0 else 'develop'

    if chosen_base == 'latest' or chosen_base == 'develop':
        chosen_base = latest

    try:
        info = scrape(ticket, db=db, force='force' in request.args)
    except:
        info = tickets.find_one({'id': ticket})

    if info is None:
        return "No such ticket."
    if 'kick' in request.args:
        info['retry'] = True
        db.save_ticket(info)
    if 'reports' in info:
        info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
    else:
        info['reports'] = []

    base_reports = base_reports_by_machine_and_base()

    old_reports = list(info['reports'])
    prune_pending(info)
    if old_reports != info['reports']:
        db.save_ticket(info)

    def format_info(info):
        new_info = {}
        for key, value in info.items():
            if key in ['patches', 'reports', 'pending']:
                pass
            elif key == 'depends_on':
                deps_status = {}

                def is_int(a):
                    try:
                        int(a)
                        return True
                    except ValueError:
                        return False
                for dep in tickets.find({'id': {'$in': [int(a) for a in value
                                                        if is_int(a)]}},
                                        ['status', 'id']):
                    if 'closed' in dep['status']:
                        dep['style'] = 'text-decoration: line-through'
                    else:
                        dep['style'] = ''
                    deps_status[dep['id']] = dep
                new_info[key] = ', '.join("<img src='/ticket/%s/status.svg?fast' height=16><a href='/ticket/%s' style='%s'>%s</a>" % (a, a, deps_status[a]['style'], a) for a in value)
            elif key == 'authors':
                new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a, a) for a in value)
            elif key == 'authors_fullnames':
                link = u"<a href='http://git.sagemath.org/sage.git/log/?qt=author&amp;q={}'>{}</a>"
                auths = u", ".join(link.format(a.replace(u" ", u"%20"), a)
                                   for a in value)
                trust_check = u"(<a href='/trust_check?who="
                trust_check += u",".join(u"{}".format(a) for a in value)
                trust_check += u"'>Check trust</a>) "
                new_info[key] = trust_check + auths
            elif key == 'participants':
                parts = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a, a) for a in value)
                new_info[key] = parts
            elif key == 'git_branch':
                new_info[key] = '<a href="http://git.sagemath.org/sage.git/log/?h=%s">%s</a>' % (value, value)
            elif key == 'spkgs':
                new_info[key] = ', '.join("<a href='%s'>%s</a>" % (a, a) for a in value)
            elif isinstance(value, list):
                new_info[key] = ', '.join(value)
            elif key not in ('id', '_id'):
                new_info[key] = value
        return new_info

    def format_git_describe(res):
        if res:
            if '-' in res:
                tag, commits = res.split('-')[:2]
                return "%s + %s commits" % (tag, commits)
            elif 'commits' in res:
                # old style
                return res
            else:
                return res + " + 0 commits"
        else:
            return '?'

    def preprocess_reports(all):
        for item in all:
            base_of_this_report = item['base']
            base_report = base_reports.get(item['base'] + "/" + "/".join(item['machine']), base_reports.get(item['base']))
            if base_report:
                item['base_log'] = quote(log_name(0, base_report))
            if 'git_base' in item:
                git_log = item.get('git_log')
                item['git_log_len'] = '?' if git_log is None else len(git_log)
            item['raw_base'] = item['base']
            if compare_version(item['base'], latest) < 0:
                item['base'] = "<span style='color: red'>%s</span>" % item['base']
            if 'time' in item:
                item['log'] = log_name(info['id'], item)
            if 'git_commit_human' not in item:
                item['git_commit_human'] = "%s new commits" % len(item['log'])
            for x in ('commit', 'base', 'merge'):
                field = 'git_%s_human' % x
                item[field] = format_git_describe(item.get(field, None))
            if chosen_base == 'all' or chosen_base == base_of_this_report:
                yield item

    def normalize_plugin(plugin):
        while len(plugin) < 3:
            plugin.append(None)
        return plugin

    def sort_fields(items):
        return sorted(items, key=(lambda x: (x[0] != 'title', x)))

    status_data = get_ticket_status(info, base=latest)[1]  # single status

    return render_template("ticket.html",
                           reports=preprocess_reports(info['reports']),
                           ticket=ticket, info=format_info(info),
                           status=status_data,
                           normalize_plugin=normalize_plugin,
                           sort_fields=sort_fields)
Esempio n. 15
0
    def test_a_ticket(self, ticket=None):
    
        self.reload_config()

        if ticket is None:
            ticket = self.get_ticket()
        else:
            ticket = None, scrape(int(ticket))
        if not ticket:
            print "No more tickets."
            time.sleep(conf['idle'])
            return

        rating, ticket = ticket
        print "\n" * 2
        print "=" * 30, ticket['id'], "=" * 30
        print ticket['title']
        print "score", rating
        print "\n" * 2
        log_dir = self.sage_root + "/logs"
        if not os.path.exists(log_dir):
            os.mkdir(log_dir)
        log = '%s/%s-log.txt' % (log_dir, ticket['id'])
        if not self.plugin_only:
            self.report_ticket(ticket, status='Pending', log=None)
        plugins_results = []
        try:
            with Tee(log, time=True, timeout=self.config['timeout']):
                t = Timer()
                start_time = time.time()

                if ticket['spkgs']:
                    state = 'spkg'
                    print "\n".join(ticket['spkgs'])
                    print
                    for spkg in ticket['spkgs']:
                        print
                        print '+' * 10, spkg, '+' * 10
                        print
                        try:
                            self.check_spkg(spkg)
                        except Exception:
                            traceback.print_exc()
                        t.finish(spkg)

                else:
                    state = 'started'
                    os.environ['MAKE'] = "make -j%s" % self.config['parallelism']
                    os.environ['SAGE_ROOT'] = self.sage_root
                # TODO: Ensure that sage-main is pristine.
                    pull_from_trac(self.sage_root, ticket['id'], force=True)
                    t.finish("Apply")
                    state = 'applied'
                
                    do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id'])
                    t.finish("Build")
                    state = 'built'
                
                    working_dir = "%s/devel/sage-%s" % (self.sage_root, ticket['id'])
                # Only the ones on this ticket.
                    patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
                    kwds = {
                        "original_dir": "%s/devel/sage-0" % self.sage_root,
                        "patched_dir": working_dir,
                        "patches": ["%s/devel/sage-%s/.hg/patches/%s" % (self.sage_root, ticket['id'], p) for p in patches if p],
                        "sage_binary": os.path.join(self.sage_root, 'sage')
                        }
                
                    for name, plugin in self.config['plugins']:
                        try:
                            if ticket['id'] != 0 and os.path.exists(os.path.join(log_dir, '0', name)):
                                baseline = pickle.load(open(os.path.join(log_dir, '0', name)))
                            else:
                                baseline = None
                            print plugin_boundary(name)
                            res = plugin(ticket, baseline=baseline, **kwds)
                            passed = True
                        except Exception:
                            traceback.print_exc()
                            passed = False
                            res = None
                        finally:
                            if isinstance(res, PluginResult):
                                if res.baseline is not None:
                                    plugin_dir = os.path.join(log_dir, str(ticket['id']))
                                    if not os.path.exists(plugin_dir):
                                        os.mkdir(plugin_dir)
                                    pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w'))
                                    passed = res.status == PluginResult.Passed
                                    print name, res.status
                                    plugins_results.append((name, passed, res.data))
                                else:
                                    plugins_results.append((name, passed, None))
                            t.finish(name)
                            print plugin_boundary(name, end=True)
                    plugins_passed = all(passed for (name, passed, data) in plugins_results)
                
                    if self.plugin_only:
                        state = 'plugins' if plugins_passed else 'plugins_failed'
                    else: 
                        if self.dry_run:
                            test_dirs = ["$SAGE_ROOT/devel/sage-%s/sage/misc/a*.py" % (ticket['id'])]
                        else: 
                            test_dirs = ["--sagenb"] + ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
                        if conf['parallelism'] > 1:
                            test_cmd = "-tp %s" % conf['parallelism']
                        else: 
                            test_cmd = "-t"
                        do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, ' '.join(test_dirs)))
                        t.finish("Tests")
                        state = 'tested'
                    
                        if not plugins_passed:
                            state = 'tests_passed_plugins_failed'

                print
                t.print_all()
        except urllib2.HTTPError:
            # Don't report failure because the network/trac died...
            traceback.print_exc()
            return 'Pending'
        except Exception:
            traceback.print_exc()
        
        for _ in range(5):
            try:
                print "Reporting", ticket['id'], status[state]
                if not self.dry_run:
                    self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results)
                print "Done reporting", ticket['id']
                break
            except urllib2.HTTPError:
                traceback.print_exc()
                time.sleep(conf['idle'])
        else:
            print "Error reporting", ticket['id']
        if not conf['keep_open_branches'] and str(ticket['id']) != '0' and not ticket['spkgs']:
            shutil.rmtree(os.path.join(self.sage_root, "devel", "sage-%s" %ticket['id']))
        return status[state]