def fetch_commits_and_submit(repository, server_config, max_fetch_count): assert 'name' in repository, 'The repository name should be specified' assert 'url' in repository, 'The SVN repository URL should be specified' if 'revisionToFetch' not in repository: print "Determining the stating revision for %s" % repository['name'] repository['revisionToFecth'] = determine_first_revision_to_fetch(server_config['server']['url'], repository['name']) if 'useServerAuth' in repository: repository['username'] = server_config['server']['auth']['username'] repository['password'] = server_config['server']['auth']['password'] pending_commits = [] for unused in range(max_fetch_count): commit = fetch_commit_and_resolve_author(repository, repository.get('accountNameFinderScript', None), repository['revisionToFecth']) if not commit: break pending_commits += [commit] repository['revisionToFecth'] += 1 if not pending_commits: print "No new revision found for %s (waiting for r%d)" % (repository['name'], repository['revisionToFecth']) return revision_list = 'r' + ', r'.join(map(lambda commit: str(commit['revision']), pending_commits)) print "Submitting revisions %s for %s to %s" % (revision_list, repository['name'], server_config['server']['url']) submit_commits(pending_commits, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password']) print "Successfully submitted." print
def fetch_commits_and_submit(self, server_config, max_fetch_count): if not self._last_fetched: print "Determining the stating revision for %s" % self._name self._last_fetched = self.determine_last_reported_revision(server_config) pending_commits = [] for unused in range(max_fetch_count): commit = self.fetch_commit(server_config, self._last_fetched) if not commit: break pending_commits += [commit] self._last_fetched = commit['revision'] if not pending_commits: print "No new revision found for %s (last fetched: %s)" % (self._name, self.format_revision(self._last_fetched)) return revision_list = ', '.join([self.format_revision(commit['revision']) for commit in pending_commits]) print "Submitting revisions %s for %s to %s" % (revision_list, self._name, server_config['server']['url']) submit_commits(pending_commits, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password']) print "Successfully submitted." print
def fetch_and_report_new_builds(self, server_config): available_builds = self._fetch_available_builds() reported_revisions = self._reported_revisions print 'Found %d builds' % len(available_builds) available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds) self._assign_order(available_builds) print "Submitting %d builds" % len(available_builds) submit_commits(available_builds, server_config['server']['url'], server_config['worker']['name'], server_config['worker']['password']) reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
def fetch_and_report_new_builds(self, server_config): available_builds = self._fetch_available_builds() reported_revisions = self._reported_revisions print 'Found %d builds' % len(available_builds) available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds) self._assign_fake_timestamps(available_builds) print "Submitting %d builds" % len(available_builds) submit_commits(available_builds, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password']) reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--config', required=True, help='Path to a config JSON file') args = parser.parse_args() with open(args.config) as config_file: config = json.load(config_file) setup_auth(config['server']) submission_size = config['submissionSize'] reported_revisions = set() while True: if 'customCommands' in config: available_builds = [] for command in config['customCommands']: print "Executing", ' '.join(command['command']) available_builds += available_builds_from_command( config['repositoryName'], command['command'], command['linesToIgnore']) print "Got %d builds" % len(available_builds) else: url = config['buildSourceURL'] print "Fetching available builds from", url available_builds = fetch_available_builds( config['repositoryName'], url, config['trainVersionMap']) available_builds = filter( lambda commit: commit['revision'] not in reported_revisions, available_builds) print "%d builds available" % len(available_builds) while available_builds: commits_to_submit = available_builds[:submission_size] revisions_to_report = map(lambda commit: commit['revision'], commits_to_submit) print "Submitting builds (%d remaining):" % len( available_builds), json.dumps(revisions_to_report) available_builds = available_builds[submission_size:] submit_commits(commits_to_submit, config['server']['url'], config['slave']['name'], config['slave']['password']) reported_revisions |= set(revisions_to_report) time.sleep(config['submissionInterval']) print "Sleeping for %d seconds" % config['fetchInterval'] time.sleep(config['fetchInterval'])
def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--config', required=True, help='Path to a config JSON file') args = parser.parse_args() with open(args.config) as config_file: config = json.load(config_file) setup_auth(config['server']) submission_size = config['submissionSize'] reported_revisions = set() while True: if 'customCommands' in config: available_builds = [] for command in config['customCommands']: print "Executing", ' '.join(command['command']) available_builds += available_builds_from_command(config['repositoryName'], command['command'], command['linesToIgnore']) print "Got %d builds" % len(available_builds) else: url = config['buildSourceURL'] print "Fetching available builds from", url available_builds = fetch_available_builds(config['repositoryName'], url, config['trainVersionMap']) available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds) print "%d builds available" % len(available_builds) while available_builds: commits_to_submit = available_builds[:submission_size] revisions_to_report = map(lambda commit: commit['revision'], commits_to_submit) print "Submitting builds (%d remaining):" % len(available_builds), json.dumps(revisions_to_report) available_builds = available_builds[submission_size:] submit_commits(commits_to_submit, config['server']['url'], config['slave']['name'], config['slave']['password']) reported_revisions |= set(revisions_to_report) time.sleep(config['submissionInterval']) print "Sleeping for %d seconds" % config['fetchInterval'] time.sleep(config['fetchInterval'])
def fetch_commits_and_submit(self, server_config, max_fetch_count, max_ancestor_fetch_count): if not self._last_fetched: print "Determining the starting revision for %s" % self._name self._last_fetched = self.determine_last_reported_revision( server_config) pending_commits = [] for unused in range(max_fetch_count): commit = self.fetch_next_commit(server_config, self._last_fetched) if not commit: break pending_commits += [commit] self._last_fetched = commit['revision'] if not pending_commits: print "No new revision found for %s (last fetched: %s)" % ( self._name, self.format_revision(self._last_fetched)) return for unused in range(max_ancestor_fetch_count): revision_list = ', '.join([ self.format_revision(commit['revision']) for commit in pending_commits ]) print "Submitting revisions %s for %s to %s" % ( revision_list, self._name, server_config['server']['url']) result = submit_commits(pending_commits, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password'], ['OK', 'FailedToFindPreviousCommit']) if result.get('status') == 'OK': break if result.get('status') == 'FailedToFindPreviousCommit': previous_commit = self.fetch_commit( server_config, result['commit']['previousCommit']) if not previous_commit: raise Exception( 'Could not find the previous commit %s of %s' % (result['commit']['previousCommit'], result['commit']['revision'])) pending_commits = [previous_commit] + pending_commits if result.get('status') != 'OK': raise Exception(result) print "Successfully submitted." print
def main(argv): if len(argv) < 7: sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <slave-name> <slave-password> <seconds-to-sleep> [<account-to-name-helper>]') repository_name = argv[1] repository_url = argv[2] dashboard_url = argv[3] slave_name = argv[4] slave_password = argv[5] seconds_to_sleep = float(argv[6]) account_to_name_helper = argv[7] if len(argv) > 7 else None print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url) revision_to_fetch = determine_first_revision_to_fetch(dashboard_url, repository_name) print "Start fetching commits at r%d" % revision_to_fetch pending_commits_to_send = [] while True: commit = fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch) if commit: print "Fetched r%d." % revision_to_fetch pending_commits_to_send += [commit] revision_to_fetch += 1 else: print "Revision %d not found" % revision_to_fetch if not commit or len(pending_commits_to_send) >= 10: if pending_commits_to_send: print "Submitting the above commits to %s..." % dashboard_url submit_commits(pending_commits_to_send, dashboard_url, slave_name, slave_password) print "Successfully submitted." pending_commits_to_send = [] time.sleep(seconds_to_sleep)