Beispiel #1
0
def fetch_commits_and_submit(repository, server_config, max_fetch_count):
    assert 'name' in repository, 'The repository name should be specified'
    assert 'url' in repository, 'The SVN repository URL should be specified'

    if 'revisionToFetch' not in repository:
        print "Determining the stating revision for %s" % repository['name']
        repository['revisionToFecth'] = determine_first_revision_to_fetch(server_config['server']['url'], repository['name'])

    if 'useServerAuth' in repository:
        repository['username'] = server_config['server']['auth']['username']
        repository['password'] = server_config['server']['auth']['password']

    pending_commits = []
    for unused in range(max_fetch_count):
        commit = fetch_commit_and_resolve_author(repository, repository.get('accountNameFinderScript', None), repository['revisionToFecth'])
        if not commit:
            break
        pending_commits += [commit]
        repository['revisionToFecth'] += 1

    if not pending_commits:
        print "No new revision found for %s (waiting for r%d)" % (repository['name'], repository['revisionToFecth'])
        return

    revision_list = 'r' + ', r'.join(map(lambda commit: str(commit['revision']), pending_commits))
    print "Submitting revisions %s for %s to %s" % (revision_list, repository['name'], server_config['server']['url'])
    submit_commits(pending_commits, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password'])
    print "Successfully submitted."
    print
Beispiel #2
0
    def fetch_commits_and_submit(self, server_config, max_fetch_count):
        if not self._last_fetched:
            print "Determining the stating revision for %s" % self._name
            self._last_fetched = self.determine_last_reported_revision(server_config)

        pending_commits = []
        for unused in range(max_fetch_count):
            commit = self.fetch_commit(server_config, self._last_fetched)
            if not commit:
                break
            pending_commits += [commit]
            self._last_fetched = commit['revision']

        if not pending_commits:
            print "No new revision found for %s (last fetched: %s)" % (self._name, self.format_revision(self._last_fetched))
            return

        revision_list = ', '.join([self.format_revision(commit['revision']) for commit in pending_commits])

        print "Submitting revisions %s for %s to %s" % (revision_list, self._name, server_config['server']['url'])

        submit_commits(pending_commits, server_config['server']['url'],
            server_config['slave']['name'], server_config['slave']['password'])

        print "Successfully submitted."
        print
Beispiel #3
0
    def fetch_and_report_new_builds(self, server_config):
        available_builds = self._fetch_available_builds()
        reported_revisions = self._reported_revisions
        print 'Found %d builds' % len(available_builds)

        available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
        self._assign_order(available_builds)

        print "Submitting %d builds" % len(available_builds)
        submit_commits(available_builds, server_config['server']['url'], server_config['worker']['name'], server_config['worker']['password'])
        reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
Beispiel #4
0
    def fetch_and_report_new_builds(self, server_config):
        available_builds = self._fetch_available_builds()
        reported_revisions = self._reported_revisions
        print 'Found %d builds' % len(available_builds)

        available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
        self._assign_fake_timestamps(available_builds)

        print "Submitting %d builds" % len(available_builds)
        submit_commits(available_builds, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password'])
        reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
Beispiel #5
0
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('--config',
                        required=True,
                        help='Path to a config JSON file')
    args = parser.parse_args()

    with open(args.config) as config_file:
        config = json.load(config_file)

    setup_auth(config['server'])

    submission_size = config['submissionSize']
    reported_revisions = set()

    while True:
        if 'customCommands' in config:
            available_builds = []
            for command in config['customCommands']:
                print "Executing", ' '.join(command['command'])
                available_builds += available_builds_from_command(
                    config['repositoryName'], command['command'],
                    command['linesToIgnore'])
                print "Got %d builds" % len(available_builds)
        else:
            url = config['buildSourceURL']
            print "Fetching available builds from", url
            available_builds = fetch_available_builds(
                config['repositoryName'], url, config['trainVersionMap'])

        available_builds = filter(
            lambda commit: commit['revision'] not in reported_revisions,
            available_builds)
        print "%d builds available" % len(available_builds)

        while available_builds:
            commits_to_submit = available_builds[:submission_size]
            revisions_to_report = map(lambda commit: commit['revision'],
                                      commits_to_submit)
            print "Submitting builds (%d remaining):" % len(
                available_builds), json.dumps(revisions_to_report)
            available_builds = available_builds[submission_size:]

            submit_commits(commits_to_submit, config['server']['url'],
                           config['slave']['name'],
                           config['slave']['password'])
            reported_revisions |= set(revisions_to_report)

            time.sleep(config['submissionInterval'])

        print "Sleeping for %d seconds" % config['fetchInterval']
        time.sleep(config['fetchInterval'])
Beispiel #6
0
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', required=True, help='Path to a config JSON file')
    args = parser.parse_args()

    with open(args.config) as config_file:
        config = json.load(config_file)

    setup_auth(config['server'])

    submission_size = config['submissionSize']
    reported_revisions = set()

    while True:
        if 'customCommands' in config:
            available_builds = []
            for command in config['customCommands']:
                print "Executing", ' '.join(command['command'])
                available_builds += available_builds_from_command(config['repositoryName'], command['command'], command['linesToIgnore'])
                print "Got %d builds" % len(available_builds)
        else:
            url = config['buildSourceURL']
            print "Fetching available builds from", url
            available_builds = fetch_available_builds(config['repositoryName'], url, config['trainVersionMap'])

        available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
        print "%d builds available" % len(available_builds)

        while available_builds:
            commits_to_submit = available_builds[:submission_size]
            revisions_to_report = map(lambda commit: commit['revision'], commits_to_submit)
            print "Submitting builds (%d remaining):" % len(available_builds), json.dumps(revisions_to_report)
            available_builds = available_builds[submission_size:]

            submit_commits(commits_to_submit, config['server']['url'], config['slave']['name'], config['slave']['password'])
            reported_revisions |= set(revisions_to_report)

            time.sleep(config['submissionInterval'])

        print "Sleeping for %d seconds" % config['fetchInterval']
        time.sleep(config['fetchInterval'])
Beispiel #7
0
    def fetch_commits_and_submit(self, server_config, max_fetch_count,
                                 max_ancestor_fetch_count):
        if not self._last_fetched:
            print "Determining the starting revision for %s" % self._name
            self._last_fetched = self.determine_last_reported_revision(
                server_config)

        pending_commits = []
        for unused in range(max_fetch_count):
            commit = self.fetch_next_commit(server_config, self._last_fetched)
            if not commit:
                break
            pending_commits += [commit]
            self._last_fetched = commit['revision']

        if not pending_commits:
            print "No new revision found for %s (last fetched: %s)" % (
                self._name, self.format_revision(self._last_fetched))
            return

        for unused in range(max_ancestor_fetch_count):
            revision_list = ', '.join([
                self.format_revision(commit['revision'])
                for commit in pending_commits
            ])
            print "Submitting revisions %s for %s to %s" % (
                revision_list, self._name, server_config['server']['url'])

            result = submit_commits(pending_commits,
                                    server_config['server']['url'],
                                    server_config['slave']['name'],
                                    server_config['slave']['password'],
                                    ['OK', 'FailedToFindPreviousCommit'])

            if result.get('status') == 'OK':
                break

            if result.get('status') == 'FailedToFindPreviousCommit':
                previous_commit = self.fetch_commit(
                    server_config, result['commit']['previousCommit'])
                if not previous_commit:
                    raise Exception(
                        'Could not find the previous commit %s of %s' %
                        (result['commit']['previousCommit'],
                         result['commit']['revision']))
                pending_commits = [previous_commit] + pending_commits

        if result.get('status') != 'OK':
            raise Exception(result)

        print "Successfully submitted."
        print
Beispiel #8
0
def main(argv):
    if len(argv) < 7:
        sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <slave-name> <slave-password> <seconds-to-sleep> [<account-to-name-helper>]')

    repository_name = argv[1]
    repository_url = argv[2]
    dashboard_url = argv[3]
    slave_name = argv[4]
    slave_password = argv[5]
    seconds_to_sleep = float(argv[6])
    account_to_name_helper = argv[7] if len(argv) > 7 else None

    print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url)

    revision_to_fetch = determine_first_revision_to_fetch(dashboard_url, repository_name)
    print "Start fetching commits at r%d" % revision_to_fetch

    pending_commits_to_send = []

    while True:
        commit = fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch)

        if commit:
            print "Fetched r%d." % revision_to_fetch
            pending_commits_to_send += [commit]
            revision_to_fetch += 1
        else:
            print "Revision %d not found" % revision_to_fetch

        if not commit or len(pending_commits_to_send) >= 10:
            if pending_commits_to_send:
                print "Submitting the above commits to %s..." % dashboard_url
                submit_commits(pending_commits_to_send, dashboard_url, slave_name, slave_password)
                print "Successfully submitted."
            pending_commits_to_send = []
            time.sleep(seconds_to_sleep)