def fetch_commits_and_submit(repository, server_config, max_fetch_count):
assert 'name' in repository, 'The repository name should be specified'
assert 'url' in repository, 'The SVN repository URL should be specified'
if 'revisionToFetch' not in repository:
print "Determining the stating revision for %s" % repository['name']
repository['revisionToFecth'] = determine_first_revision_to_fetch(server_config['server']['url'], repository['name'])
if 'useServerAuth' in repository:
repository['username'] = server_config['server']['auth']['username']
repository['password'] = server_config['server']['auth']['password']
pending_commits = []
for unused in range(max_fetch_count):
commit = fetch_commit_and_resolve_author(repository, repository.get('accountNameFinderScript', None), repository['revisionToFecth'])
if not commit:
break
pending_commits += [commit]
repository['revisionToFecth'] += 1
if not pending_commits:
print "No new revision found for %s (waiting for r%d)" % (repository['name'], repository['revisionToFecth'])
return
revision_list = 'r' + ', r'.join(map(lambda commit: str(commit['revision']), pending_commits))
print "Submitting revisions %s for %s to %s" % (revision_list, repository['name'], server_config['server']['url'])
submit_commits(pending_commits, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password'])
print "Successfully submitted."
print
def fetch_commits_and_submit(self, server_config, max_fetch_count):
if not self._last_fetched:
print "Determining the stating revision for %s" % self._name
self._last_fetched = self.determine_last_reported_revision(server_config)
pending_commits = []
for unused in range(max_fetch_count):
commit = self.fetch_commit(server_config, self._last_fetched)
if not commit:
break
pending_commits += [commit]
self._last_fetched = commit['revision']
if not pending_commits:
print "No new revision found for %s (last fetched: %s)" % (self._name, self.format_revision(self._last_fetched))
return
revision_list = ', '.join([self.format_revision(commit['revision']) for commit in pending_commits])
print "Submitting revisions %s for %s to %s" % (revision_list, self._name, server_config['server']['url'])
submit_commits(pending_commits, server_config['server']['url'],
server_config['slave']['name'], server_config['slave']['password'])
print "Successfully submitted."
print
def fetch_and_report_new_builds(self, server_config):
available_builds = self._fetch_available_builds()
reported_revisions = self._reported_revisions
print 'Found %d builds' % len(available_builds)
available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
self._assign_order(available_builds)
print "Submitting %d builds" % len(available_builds)
submit_commits(available_builds, server_config['server']['url'], server_config['worker']['name'], server_config['worker']['password'])
reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
def fetch_and_report_new_builds(self, server_config):
available_builds = self._fetch_available_builds()
reported_revisions = self._reported_revisions
print 'Found %d builds' % len(available_builds)
available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
self._assign_fake_timestamps(available_builds)
print "Submitting %d builds" % len(available_builds)
submit_commits(available_builds, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password'])
reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--config',
required=True,
help='Path to a config JSON file')
args = parser.parse_args()
with open(args.config) as config_file:
config = json.load(config_file)
setup_auth(config['server'])
submission_size = config['submissionSize']
reported_revisions = set()
while True:
if 'customCommands' in config:
available_builds = []
for command in config['customCommands']:
print "Executing", ' '.join(command['command'])
available_builds += available_builds_from_command(
config['repositoryName'], command['command'],
command['linesToIgnore'])
print "Got %d builds" % len(available_builds)
else:
url = config['buildSourceURL']
print "Fetching available builds from", url
available_builds = fetch_available_builds(
config['repositoryName'], url, config['trainVersionMap'])
available_builds = filter(
lambda commit: commit['revision'] not in reported_revisions,
available_builds)
print "%d builds available" % len(available_builds)
while available_builds:
commits_to_submit = available_builds[:submission_size]
revisions_to_report = map(lambda commit: commit['revision'],
commits_to_submit)
print "Submitting builds (%d remaining):" % len(
available_builds), json.dumps(revisions_to_report)
available_builds = available_builds[submission_size:]
submit_commits(commits_to_submit, config['server']['url'],
config['slave']['name'],
config['slave']['password'])
reported_revisions |= set(revisions_to_report)
time.sleep(config['submissionInterval'])
print "Sleeping for %d seconds" % config['fetchInterval']
time.sleep(config['fetchInterval'])
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--config', required=True, help='Path to a config JSON file')
args = parser.parse_args()
with open(args.config) as config_file:
config = json.load(config_file)
setup_auth(config['server'])
submission_size = config['submissionSize']
reported_revisions = set()
while True:
if 'customCommands' in config:
available_builds = []
for command in config['customCommands']:
print "Executing", ' '.join(command['command'])
available_builds += available_builds_from_command(config['repositoryName'], command['command'], command['linesToIgnore'])
print "Got %d builds" % len(available_builds)
else:
url = config['buildSourceURL']
print "Fetching available builds from", url
available_builds = fetch_available_builds(config['repositoryName'], url, config['trainVersionMap'])
available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
print "%d builds available" % len(available_builds)
while available_builds:
commits_to_submit = available_builds[:submission_size]
revisions_to_report = map(lambda commit: commit['revision'], commits_to_submit)
print "Submitting builds (%d remaining):" % len(available_builds), json.dumps(revisions_to_report)
available_builds = available_builds[submission_size:]
submit_commits(commits_to_submit, config['server']['url'], config['slave']['name'], config['slave']['password'])
reported_revisions |= set(revisions_to_report)
time.sleep(config['submissionInterval'])
print "Sleeping for %d seconds" % config['fetchInterval']
time.sleep(config['fetchInterval'])
def fetch_commits_and_submit(self, server_config, max_fetch_count,
max_ancestor_fetch_count):
if not self._last_fetched:
print "Determining the starting revision for %s" % self._name
self._last_fetched = self.determine_last_reported_revision(
server_config)
pending_commits = []
for unused in range(max_fetch_count):
commit = self.fetch_next_commit(server_config, self._last_fetched)
if not commit:
break
pending_commits += [commit]
self._last_fetched = commit['revision']
if not pending_commits:
print "No new revision found for %s (last fetched: %s)" % (
self._name, self.format_revision(self._last_fetched))
return
for unused in range(max_ancestor_fetch_count):
revision_list = ', '.join([
self.format_revision(commit['revision'])
for commit in pending_commits
])
print "Submitting revisions %s for %s to %s" % (
revision_list, self._name, server_config['server']['url'])
result = submit_commits(pending_commits,
server_config['server']['url'],
server_config['slave']['name'],
server_config['slave']['password'],
['OK', 'FailedToFindPreviousCommit'])
if result.get('status') == 'OK':
break
if result.get('status') == 'FailedToFindPreviousCommit':
previous_commit = self.fetch_commit(
server_config, result['commit']['previousCommit'])
if not previous_commit:
raise Exception(
'Could not find the previous commit %s of %s' %
(result['commit']['previousCommit'],
result['commit']['revision']))
pending_commits = [previous_commit] + pending_commits
if result.get('status') != 'OK':
raise Exception(result)
print "Successfully submitted."
print
def main(argv):
if len(argv) < 7:
sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <slave-name> <slave-password> <seconds-to-sleep> [<account-to-name-helper>]')
repository_name = argv[1]
repository_url = argv[2]
dashboard_url = argv[3]
slave_name = argv[4]
slave_password = argv[5]
seconds_to_sleep = float(argv[6])
account_to_name_helper = argv[7] if len(argv) > 7 else None
print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url)
revision_to_fetch = determine_first_revision_to_fetch(dashboard_url, repository_name)
print "Start fetching commits at r%d" % revision_to_fetch
pending_commits_to_send = []
while True:
commit = fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch)
if commit:
print "Fetched r%d." % revision_to_fetch
pending_commits_to_send += [commit]
revision_to_fetch += 1
else:
print "Revision %d not found" % revision_to_fetch
if not commit or len(pending_commits_to_send) >= 10:
if pending_commits_to_send:
print "Submitting the above commits to %s..." % dashboard_url
submit_commits(pending_commits_to_send, dashboard_url, slave_name, slave_password)
print "Successfully submitted."
pending_commits_to_send = []
time.sleep(seconds_to_sleep)