def test_transplant_using_at_bookmark(self):
        inbound_repo_path = tempfile.mkdtemp()
        local_repo_path = tempfile.mkdtemp()
        mozreview_repo_path = tempfile.mkdtemp()
        upstream_repo_path = tempfile.mkdtemp()

        try:
            # Configure 'upstream' repo
            cmds = [['hg', 'init']]
            for cmd in cmds:
                subprocess.check_call(cmd, cwd=upstream_repo_path)

            # Configure 'mozreview' repo
            shutil.copy(os.path.join(HERE, 'test-data', 'initial.patch'),
                        mozreview_repo_path)
            cmds = [['hg', 'init'],
                    ['hg', 'import', 'initial.patch']]
            for cmd in cmds:
                # Use check_output to suppress the output from hg import
                subprocess.check_output(cmd, stderr=subprocess.STDOUT,
                                        cwd=mozreview_repo_path)
            with open(os.path.join(mozreview_repo_path, '.hg', 'hgrc'), 'w') as f:
                f.write('[phases]\npublish = False\n')

            # Configure 'inbound' repo
            cmds = [['hg', 'init']]
            for cmd in cmds:
                subprocess.check_call(cmd, cwd=inbound_repo_path)

            # Configure 'local' repo
            cmds = [['hg', 'init'],
                    ['hg', 'bookmark', 'upstream']]
            for cmd in cmds:
                subprocess.check_call(cmd, cwd=local_repo_path)

            with open(os.path.join(local_repo_path, '.hg', 'hgrc'), 'w') as f:
                f.write('[paths]\nversion-control-tools = ')
                f.write(inbound_repo_path)
                f.write('\nmozreview = ')
                f.write(mozreview_repo_path)
                f.write('\nupstream= ')
                f.write(mozreview_repo_path)
                f.write('\n')

            def get_repo_path(tree):
                return local_repo_path
            transplant.get_repo_path = get_repo_path

            landed, result = transplant.transplant('mozreview',
                                                   'version-control-tools',
                                                   '0', '', '@')
            self.assertEqual(landed, True)
            self.assertEqual(len(result), 12)
            self.assertIsNotNone(re.match('([a-f0-9]+)', result))

        finally:
            shutil.rmtree(local_repo_path)
            shutil.rmtree(mozreview_repo_path)
            shutil.rmtree(inbound_repo_path)
            shutil.rmtree(upstream_repo_path)
Пример #2
0
def handle_pending_transplants(logger, dbconn):
    cursor = dbconn.cursor()
    query = """
        select tree,revision,bugid from Autoland
        where can_be_landed is true and landed is null
    """
    cursor.execute(query)

    landed = []
    for row in cursor.fetchall():
        tree, rev, bugid = row

        pushlog = mercurial.get_pushlog(tree, rev)
        if not pushlog:
            logger.debug('could not get pushlog for tree: %s rev %s' %
                        (tree, rev))
            return

        changesets = []
        for key in pushlog:
            for changeset in pushlog[key]['changesets']:
                # we assume by convention head revision is empty and should
                # not be landed
                if changeset != rev:
                    changesets.append(changeset)

        # TODO: allow for transplant to other trees than 'mozilla-inbound'
        result = transplant.transplant(tree, 'mozilla-inbound', changesets)

        if not result:
            logger.debug("""could not connect to transplant server:
                            tree: %s rev %s""" % (tree, rev))
            continue

        if 'error' in result:
            succeeded = False
            logger.info('transplant failed: tree: %s rev: %s error: %s' %
                        (tree, rev, json.dumps(result)))
            comment = 'Autoland request failed: could not transplant: %s'
            add_bugzilla_comment(dbconn, bugid, comment % result['error'])
        else:
            succeeded = True
            comment = 'Autoland request succeeded: mozilla-inbound tip: %s'
            add_bugzilla_comment(dbconn, bugid, comment % result['tip'])

        landed.append([succeeded, json.dumps(result), datetime.datetime.now(),
                      tree, rev])

    if landed:
        query = """
            update Autoland set landed=%s,transplant_result=%s,last_updated=%s
            where tree=%s and revision=%s
        """
        cursor.executemany(query, landed)
        dbconn.commit()
    def test_rewrite_descriptions(self):
        inbound_repo_path = tempfile.mkdtemp()
        local_repo_path = tempfile.mkdtemp()
        mozreview_repo_path = tempfile.mkdtemp()
        upstream_repo_path = tempfile.mkdtemp()

        try:
            # Configure 'upstream' repo
            cmds = [['hg', 'init']]
            for cmd in cmds:
                subprocess.check_call(cmd, cwd=upstream_repo_path)

            # Configure 'mozreview' repo
            for f in ['initial.patch', 'added-blah.patch']:
                shutil.copy(os.path.join(HERE, 'test-data', f),
                            mozreview_repo_path)
            cmds = [['hg', 'init'],
                    ['hg', 'import', 'initial.patch'],
                    ['hg', 'import', 'added-blah.patch'],
                    ['hg', 'log', '-r', '.', '--template', '{node|short}']]
            for cmd in cmds:
                # Use check_output to suppress the output from hg import
                rev = subprocess.check_output(cmd, stderr=subprocess.STDOUT,
                                              cwd=mozreview_repo_path)
            with open(os.path.join(mozreview_repo_path, '.hg', 'hgrc'), 'w') as f:
                f.write('[phases]\npublish = False\n')

            # Configure 'inbound' repo
            shutil.copy(os.path.join(HERE, 'test-data', 'initial.patch'),
                        inbound_repo_path)
            cmds = [['hg', 'init'],
                    ['hg', 'import', 'initial.patch']]
            for cmd in cmds:
                subprocess.check_call(cmd, cwd=inbound_repo_path)

            # Configure 'local' repo
            cmds = [['hg', 'init'],
                    ['hg', 'bookmark', 'upstream']]
            for cmd in cmds:
                subprocess.check_call(cmd, cwd=local_repo_path)

            with open(os.path.join(local_repo_path, '.hg', 'hgrc'), 'w') as f:
                f.write('[paths]\ninbound = ')
                f.write(inbound_repo_path)
                f.write('\nmozreview = ')
                f.write(mozreview_repo_path)
                f.write('\nupstream= ')
                f.write(upstream_repo_path)
                f.write('\n')

                ext = os.path.join(os.path.split(HERE)[0], 'hgext', 'rewritecommitdescriptions.py')
                f.write('[extensions]\n')
                f.write('rewritecommitdescriptions = %s\n\n' % ext)

            def get_repo_path(tree):
                return local_repo_path
            transplant.get_repo_path = get_repo_path

            commit_descriptions = {'1cf8d88f5d98': 'please rewrite me!'}

            landed, result = transplant.transplant('mozreview', 'inbound',
                                                   '1cf8d88f5d98',
                                                   commit_descriptions=commit_descriptions)
            self.assertEqual(landed, True)
            self.assertEqual(len(result), 12)
            self.assertIsNotNone(re.match('([a-f0-9]+)', result))

            # Check that we rewrote as expected
            cmds = [['hg', 'log', '-r', 'tip']]
            for cmd in cmds:
                result = subprocess.check_output(cmd, cwd=inbound_repo_path)
            self.assertTrue('please rewrite me!' in result)

        finally:
            shutil.rmtree(inbound_repo_path)
            shutil.rmtree(local_repo_path)
            shutil.rmtree(mozreview_repo_path)
            shutil.rmtree(upstream_repo_path)
Пример #4
0
def handle_pending_transplants(logger, dbconn):
    cursor = dbconn.cursor()
    now = datetime.datetime.now()
    query = """
        select id, destination, request
        from Transplant
        where landed is null and (last_updated is null
            or last_updated<=%(time)s)
    """
    transplant_retry_delay = TRANSPLANT_RETRY_DELAY
    if config.testing():
        transplant_retry_delay = datetime.timedelta(seconds=1)

    cursor.execute(query, ({'time': now - transplant_retry_delay}))

    current_treestatus = {}
    finished_revisions = []
    mozreview_updates = []
    retry_revisions = []

    def handle_treeclosed(transplant_id, tree, rev, destination, trysyntax,
                          pingback_url):
        retry_revisions.append((now, transplant_id))

        data = {
            'request_id': transplant_id,
            'tree': tree,
            'rev': rev,
            'destination': destination,
            'trysyntax': trysyntax,
            'landed': False,
            'error_msg': '',
            'result': 'Tree %s is closed - retrying later.' % tree
        }
        mozreview_updates.append([transplant_id, json.dumps(data)])

    # This code is a bit messy because we have to deal with the fact that the
    # the tree could close between the call to tree_is_open and when we
    # actually attempt the revision.
    #
    # We keep a list of revisions to retry called retry_revisions which we
    # append to whenever we detect a closed tree. These revisions have their
    # last_updated field updated so we will retry them after a suitable delay.
    #
    # The other list we keep is for transplant attempts that either succeeded
    # or failed due to a reason other than a closed tree, which is called
    # finished_revisions. Successful or not, we're finished with them, they
    # will not be retried.
    for row in cursor.fetchall():
        transplant_id, destination, request = row

        requester = request['ldap_username']
        tree = request['tree']
        rev = request['rev']
        trysyntax = request.get('trysyntax', '')
        push_bookmark = request.get('push_bookmark', '')
        pingback_url = request.get('pingback_url', '')
        commit_descriptions = request.get('commit_descriptions')
        tree_open = current_treestatus.setdefault(destination,
                                                  treestatus.tree_is_open(destination))

        if not tree_open:
            handle_treeclosed(transplant_id, tree, rev, destination,
                              trysyntax, pingback_url)
            continue

        attempts = 0
        logger.info('initiating transplant from tree: %s rev: %s '
                    'to destination: %s' % (tree, rev, destination))
        started = datetime.datetime.now()
        while attempts < MAX_TRANSPLANT_ATTEMPTS:
            # TODO: We should break the transplant call into two steps, one
            #       to pull down the commits to transplant, and another
            #       one to rebase it and attempt to push so we don't
            #       duplicate work unnecessarily if we have to rebase more
            #       than once.
            os.environ['AUTOLAND_REQUEST_USER'] = requester
            landed, result = transplant.transplant(logger, tree,
                                                   destination, rev,
                                                   trysyntax, push_bookmark,
                                                   commit_descriptions)
            del os.environ['AUTOLAND_REQUEST_USER']

            if landed or 'abort: push creates new remote head' not in result:
                break

            attempts += 1

        if landed:
            logger.info('transplant successful - new revision: %s' % result)
        else:
            if 'is CLOSED!' in result:
                logger.info('transplant failed: tree: %s is closed - '
                            ' retrying later.' % tree)
                current_treestatus[destination] = False
                handle_treeclosed(transplant_id, tree, rev, destination,
                                  trysyntax, pingback_url)
                continue
            elif 'abort: push creates new remote head' in result:
                logger.info('transplant failed: we lost a push race')
                retry_revisions.append((now, transplant_id))
                continue
            elif 'unresolved conflicts (see hg resolve' in result:
                logger.info('transplant failed - manual rebase required: '
                            'tree: %s rev: %s destination: %s error: %s' %
                            (tree, rev, destination, result))
                # This is the only autoland error for which we expect the
                # user to take action. We should make things nicer than the
                # raw mercurial error.
                # TODO: sad trombone sound
                header = ('We\'re sorry, Autoland could not rebase your '
                          'commits for you automatically. Please manually '
                          'rebase your commits and try again.\n\n')
                result = header + result
            else:
                logger.info('transplant failed: tree: %s rev: %s '
                            'destination: %s error: %s' %
                            (tree, rev, destination, result))

        completed = datetime.datetime.now()
        logger.info('elapsed transplant time: %s' % (completed - started))

        # set up data to be posted back to mozreview
        data = {
            'request_id': transplant_id,
            'tree': tree,
            'rev': rev,
            'destination': destination,
            'trysyntax': trysyntax,
            'landed': landed,
            'error_msg': '',
            'result': ''
        }

        if landed:
            data['result'] = result
        else:
            data['error_msg'] = result

        mozreview_updates.append([transplant_id, json.dumps(data)])

        finished_revisions.append([landed, result, transplant_id])

    if retry_revisions:
        query = """
            update Transplant set last_updated=%s
            where id=%s
        """
        cursor.executemany(query, retry_revisions)
        dbconn.commit()

    if finished_revisions:
        query = """
            update Transplant set landed=%s,result=%s
            where id=%s
        """
        cursor.executemany(query, finished_revisions)
        dbconn.commit()

    if mozreview_updates:
        query = """
            insert into MozreviewUpdate(transplant_id,data)
            values(%s,%s)
        """
        cursor.executemany(query, mozreview_updates)
        dbconn.commit()
Пример #5
0
def handle_pending_transplants(logger, dbconn):
    cursor = dbconn.cursor()
    now = datetime.datetime.now()
    query = """
        SELECT id, destination, request
        FROM Transplant
        WHERE landed IS NULL
              AND (last_updated IS NULL OR last_updated<=%(time)s)
        ORDER BY created
    """
    transplant_retry_delay = TRANSPLANT_RETRY_DELAY
    if config.testing():
        transplant_retry_delay = datetime.timedelta(seconds=1)

    cursor.execute(query, ({'time': now - transplant_retry_delay}))

    current_treestatus = {}
    finished_revisions = []
    mozreview_updates = []
    retry_revisions = []

    def handle_treeclosed(transplant_id, tree, rev, destination, trysyntax,
                          pingback_url):
        retry_revisions.append((now, transplant_id))

        data = {
            'request_id': transplant_id,
            'tree': tree,
            'rev': rev,
            'destination': destination,
            'trysyntax': trysyntax,
            'landed': False,
            'error_msg': '',
            'result': 'Tree %s is closed - retrying later.' % tree
        }
        mozreview_updates.append([transplant_id, json.dumps(data)])

    # This code is a bit messy because we have to deal with the fact that the
    # the tree could close between the call to tree_is_open and when we
    # actually attempt the revision.
    #
    # We keep a list of revisions to retry called retry_revisions which we
    # append to whenever we detect a closed tree. These revisions have their
    # last_updated field updated so we will retry them after a suitable delay.
    #
    # The other list we keep is for transplant attempts that either succeeded
    # or failed due to a reason other than a closed tree, which is called
    # finished_revisions. Successful or not, we're finished with them, they
    # will not be retried.
    for row in cursor.fetchall():
        transplant_id, destination, request = row

        # Many of these values are used as command arguments. So convert
        # to binary because command arguments aren't unicode.
        destination = destination.encode('ascii')
        requester = request['ldap_username']
        tree = request['tree'].encode('ascii')
        rev = request['rev'].encode('ascii')
        trysyntax = request.get('trysyntax', '')
        push_bookmark = request.get('push_bookmark', '').encode('ascii')
        pingback_url = request.get('pingback_url', '').encode('ascii')
        commit_descriptions = request.get('commit_descriptions')
        tree_open = current_treestatus.setdefault(
            destination, treestatus.tree_is_open(destination))

        if not tree_open:
            handle_treeclosed(transplant_id, tree, rev, destination,
                              trysyntax, pingback_url)
            continue

        attempts = 0
        started = datetime.datetime.now()
        while attempts < MAX_TRANSPLANT_ATTEMPTS:
            logger.info('initiating transplant from tree: %s rev: %s '
                        'to destination: %s, attempt %s' % (
                            tree, rev, destination, attempts + 1))

            # TODO: We should break the transplant call into two steps, one
            #       to pull down the commits to transplant, and another
            #       one to rebase it and attempt to push so we don't
            #       duplicate work unnecessarily if we have to rebase more
            #       than once.
            os.environ['AUTOLAND_REQUEST_USER'] = requester
            landed, result = transplant.transplant(logger, tree,
                                                   destination, rev,
                                                   trysyntax, push_bookmark,
                                                   commit_descriptions)
            del os.environ['AUTOLAND_REQUEST_USER']

            logger.info('transplant from tree: %s rev: %s attempt: %s: %s' % (
                tree, rev, attempts + 1, result))

            if landed or 'abort: push creates new remote head' not in result:
                break

            attempts += 1

        if landed:
            logger.info('transplant successful - new revision: %s' % result)
        else:
            if 'is CLOSED!' in result:
                logger.info('transplant failed: tree: %s is closed - '
                            ' retrying later.' % tree)
                current_treestatus[destination] = False
                handle_treeclosed(transplant_id, tree, rev, destination,
                                  trysyntax, pingback_url)
                continue
            elif 'abort: push creates new remote head' in result:
                logger.info('transplant failed: we lost a push race')
                retry_revisions.append((now, transplant_id))
                continue
            elif 'unresolved conflicts (see hg resolve' in result:
                logger.info('transplant failed - manual rebase required: '
                            'tree: %s rev: %s destination: %s error: %s' %
                            (tree, rev, destination, result))
                # This is the only autoland error for which we expect the
                # user to take action. We should make things nicer than the
                # raw mercurial error.
                # TODO: sad trombone sound
                header = ('We\'re sorry, Autoland could not rebase your '
                          'commits for you automatically. Please manually '
                          'rebase your commits and try again.\n\n')
                result = header + result
            else:
                logger.info('transplant failed: tree: %s rev: %s '
                            'destination: %s error: %s' %
                            (tree, rev, destination, result))

        completed = datetime.datetime.now()
        logger.info('elapsed transplant time: %s' % (completed - started))

        # set up data to be posted back to mozreview
        data = {
            'request_id': transplant_id,
            'tree': tree,
            'rev': rev,
            'destination': destination,
            'trysyntax': trysyntax,
            'landed': landed,
            'error_msg': '',
            'result': ''
        }

        if landed:
            data['result'] = result
        else:
            data['error_msg'] = result

        mozreview_updates.append([transplant_id, json.dumps(data)])

        finished_revisions.append([landed, result, transplant_id])

    if retry_revisions:
        query = """
            update Transplant set last_updated=%s
            where id=%s
        """
        cursor.executemany(query, retry_revisions)
        dbconn.commit()

    if finished_revisions:
        query = """
            update Transplant set landed=%s,result=%s
            where id=%s
        """
        cursor.executemany(query, finished_revisions)
        dbconn.commit()

    if mozreview_updates:
        query = """
            insert into MozreviewUpdate(transplant_id,data)
            values(%s,%s)
        """
        cursor.executemany(query, mozreview_updates)
        dbconn.commit()
Пример #6
0
def handle_pending_transplants(logger, dbconn):
    cursor = dbconn.cursor()

    query = """
        select id,tree,rev,destination,trysyntax,push_bookmark,pingback_url
        from Transplant
        where landed is null
    """
    cursor.execute(query)

    landed_revisions = []
    mozreview_updates = []
    for row in cursor.fetchall():
        (transplant_id, tree, rev, destination, trysyntax, push_bookmark,
            pingback_url) = row

        if not treestatus.tree_is_open(destination):
            continue

        landed, result = transplant.transplant(tree, destination, rev,
                                               trysyntax, push_bookmark)
        if landed:
            logger.info(('transplanted from tree: %s rev: %s'
                         ' to destination: %s new revision: %s') %
                        (tree, rev, destination, result))
        else:
            if 'is CLOSED!' in result:
                logger.info('transplant failed: tree: %s is closed - '
                             ' retrying later.' % tree)

                # continuing here will skip updating the autoland request
                # so we will attempt to land it again later.
                continue
            else:
                logger.info('transplant failed: tree: %s rev: %s '
                            'destination: %s error: %s' %
                            (tree, rev, destination, result))

        # set up data to be posted back to mozreview
        data = {
            'request_id': transplant_id,
            'tree': tree,
            'rev': rev,
            'destination': destination,
            'trysyntax': trysyntax,
            'landed': landed,
            'error_msg': '',
            'result': ''
        }

        if landed:
            data['result'] = result
        else:
            data['error_msg'] = result

        mozreview_updates.append([transplant_id, pingback_url, json.dumps(data)])

        landed_revisions.append([landed, result, transplant_id])

    if landed_revisions:
        query = """
            update Transplant set landed=%s,result=%s
            where id=%s
        """
        cursor.executemany(query, landed_revisions)
        dbconn.commit()

    if mozreview_updates:
        query = """
            insert into MozreviewUpdate(request_id,pingback_url,data)
            values(%s,%s,%s)
        """
        cursor.executemany(query, mozreview_updates)
        dbconn.commit()