def build_slave(src_feed, archive_file, archive_dir_public_url, target_feed):
try:
COMPILE = [os.environ['0COMPILE']]
except KeyError:
# (build slave has an old 0release)
COMPILE = [
'0launch', '--not-before=1.2',
'http://0install.net/2006/interfaces/0compile.xml'
]
feed = support.load_feed(src_feed)
src_feed = os.path.abspath(src_feed)
archive_file = os.path.abspath(archive_file)
target_feed = os.path.abspath(target_feed)
impl, = feed.implementations.values()
tmpdir = tempfile.mkdtemp(prefix='0release-')
try:
os.chdir(tmpdir)
depdir = os.path.join(tmpdir, 'dependencies')
os.mkdir(depdir)
support.unpack_tarball(archive_file)
portable_rename(impl.download_sources[0].extract,
os.path.join(depdir, impl.id))
config = ConfigParser.RawConfigParser()
config.add_section('compile')
config.set('compile', 'download-base-url', archive_dir_public_url)
config.set('compile', 'version-modifier', '')
config.set('compile', 'interface', src_feed)
config.set('compile', 'selections', '')
config.set('compile', 'metadir', '0install')
stream = open(os.path.join(tmpdir, '0compile.properties'), 'w')
try:
config.write(stream)
finally:
stream.close()
support.check_call(COMPILE + ['build'], cwd=tmpdir)
support.check_call(COMPILE + ['publish', '--target-feed', target_feed],
cwd=tmpdir)
# TODO: run unit-tests
feed = support.load_feed(target_feed)
impl = support.get_singleton_impl(feed)
archive_file = support.get_archive_basename(impl)
shutil.move(archive_file,
os.path.join(os.path.dirname(target_feed), archive_file))
except:
print "\nLeaving temporary directory %s for inspection...\n" % tmpdir
raise
else:
shutil.rmtree(tmpdir)
def build_slave(src_feed, archive_file, archive_dir_public_url, target_feed):
try:
COMPILE = [os.environ['0COMPILE']]
except KeyError:
# (build slave has an old 0release)
COMPILE = ['0launch', '--not-before=0.30', 'http://0install.net/2006/interfaces/0compile.xml']
feed = support.load_feed(src_feed)
src_feed = os.path.abspath(src_feed)
archive_file = os.path.abspath(archive_file)
target_feed = os.path.abspath(target_feed)
impl, = feed.implementations.values()
tmpdir = tempfile.mkdtemp(prefix = '0release-')
try:
os.chdir(tmpdir)
depdir = os.path.join(tmpdir, 'dependencies')
os.mkdir(depdir)
support.unpack_tarball(archive_file)
portable_rename(impl.download_sources[0].extract, os.path.join(depdir, impl.id))
config = ConfigParser.RawConfigParser()
config.add_section('compile')
config.set('compile', 'download-base-url', archive_dir_public_url)
config.set('compile', 'version-modifier', '')
config.set('compile', 'interface', src_feed)
config.set('compile', 'selections', '')
config.set('compile', 'metadir', '0install')
stream = open(os.path.join(tmpdir, '0compile.properties'), 'w')
try:
config.write(stream)
finally:
stream.close()
support.check_call(COMPILE + ['build'], cwd = tmpdir)
support.check_call(COMPILE + ['publish', '--target-feed', target_feed], cwd = tmpdir)
# TODO: run unit-tests
feed = support.load_feed(target_feed)
impl = support.get_singleton_impl(feed)
archive_file = support.get_archive_basename(impl)
shutil.move(archive_file, os.path.join(os.path.dirname(target_feed), archive_file))
except:
print "\nLeaving temporary directory %s for inspection...\n" % tmpdir
raise
else:
shutil.rmtree(tmpdir)
def build_binaries(self):
if not self.targets: return
print "Source package, so generating binaries..."
archive_file = support.get_archive_basename(self.src_impl)
for target in self.targets:
start = self.get('builder-' + target, 'start', None)
command = self.config.get('builder-' + target, 'build')
stop = self.get('builder-' + target, 'stop', None)
binary_feed = 'binary-' + target + '.xml'
if os.path.exists(binary_feed):
print "Feed %s already exists; not rebuilding" % binary_feed
else:
print "\nBuilding binary with builder '%s' ...\n" % target
if start: support.show_and_run(start, [])
try:
args = [
os.path.basename(self.src_feed_name), archive_file,
self.archive_dir_public_url, binary_feed + '.new'
]
if not command:
assert target == 'host', 'Missing build command'
support.check_call(
[sys.executable, sys.argv[0], '--build-slave'] +
args)
else:
support.show_and_run(command, args)
finally:
if stop: support.show_and_run(stop, [])
bin_feed = support.load_feed(binary_feed + '.new')
bin_impl = support.get_singleton_impl(bin_feed)
bin_archive_file = support.get_archive_basename(bin_impl)
bin_size = bin_impl.download_sources[0].size
assert os.path.exists(
bin_archive_file
), "Compiled binary '%s' not found!" % os.path.abspath(
bin_archive_file)
assert os.path.getsize(
bin_archive_file
) == bin_size, "Compiled binary '%s' has wrong size!" % os.path.abspath(
bin_archive_file)
portable_rename(binary_feed + '.new', binary_feed)
def __init__(self, options, src_feed_name, release_version):
self.src_feed_name = src_feed_name
self.src_feed = support.load_feed(src_feed_name)
self.archive_dir_public_url = support.get_archive_url(
options, release_version, '')
self.config = ConfigParser.RawConfigParser()
# Start with a default configuration
self.config.add_section('global')
self.config.set('global', 'builders', 'host')
self.config.add_section('builder-host')
#self.config.set('builder-host', 'build', '0launch --not-before 0.10 http://0install.net/2007/interfaces/0release.xml --build-slave "$@"')
self.config.set('builder-host', 'build', '')
self.src_impl = support.get_singleton_impl(self.src_feed)
if self.src_impl.arch and self.src_impl.arch.endswith('-src'):
path = basedir.load_first_config('0install.net', '0release',
'builders.conf')
if path:
info("Loading configuration file '%s'", path)
self.config.read(path)
else:
info(
"No builders.conf configuration; will build a binary for this host only"
)
if options.builders is not None:
builders = options.builders
else:
builders = self.config.get('global', 'builders').strip()
if builders:
self.targets = [x.strip() for x in builders.split(',')]
info("%d build targets configured: %s", len(self.targets),
self.targets)
else:
self.targets = []
info("No builders set; no binaries will be built")
else:
self.targets = []
def build_binaries(self):
if not self.targets: return
print "Source package, so generating binaries..."
archive_file = support.get_archive_basename(self.src_impl)
for target in self.targets:
start = self.get('builder-' + target, 'start', None)
command = self.config.get('builder-' + target, 'build')
stop = self.get('builder-' + target, 'stop', None)
binary_feed = 'binary-' + target + '.xml'
if os.path.exists(binary_feed):
print "Feed %s already exists; not rebuilding" % binary_feed
else:
print "\nBuilding binary with builder '%s' ...\n" % target
if start: support.show_and_run(start, [])
try:
args = [os.path.basename(self.src_feed_name), archive_file, self.archive_dir_public_url, binary_feed + '.new']
if not command:
assert target == 'host', 'Missing build command'
support.check_call([sys.executable, sys.argv[0], '--build-slave'] + args)
else:
support.show_and_run(command, args)
finally:
if stop: support.show_and_run(stop, [])
bin_feed = support.load_feed(binary_feed + '.new')
bin_impl = support.get_singleton_impl(bin_feed)
bin_archive_file = support.get_archive_basename(bin_impl)
bin_size = bin_impl.download_sources[0].size
assert os.path.exists(bin_archive_file), "Compiled binary '%s' not found!" % os.path.abspath(bin_archive_file)
assert os.path.getsize(bin_archive_file) == bin_size, "Compiled binary '%s' has wrong size!" % os.path.abspath(bin_archive_file)
portable_rename(binary_feed + '.new', binary_feed)
def __init__(self, options, src_feed_name, release_version):
self.src_feed_name = src_feed_name
self.src_feed = support.load_feed(src_feed_name)
self.archive_dir_public_url = support.get_archive_url(options, release_version, '')
self.config = ConfigParser.RawConfigParser()
# Start with a default configuration
self.config.add_section('global')
self.config.set('global', 'builders', 'host')
self.config.add_section('builder-host')
#self.config.set('builder-host', 'build', '0launch --not-before 0.10 http://0install.net/2007/interfaces/0release.xml --build-slave "$@"')
self.config.set('builder-host', 'build', '')
self.src_impl = support.get_singleton_impl(self.src_feed)
if self.src_impl.arch and self.src_impl.arch.endswith('-src'):
path = basedir.load_first_config('0install.net', '0release', 'builders.conf')
if path:
info("Loading configuration file '%s'", path)
self.config.read(path)
else:
info("No builders.conf configuration; will build a binary for this host only")
if options.builders is not None:
builders = options.builders
else:
builders = self.config.get('global', 'builders').strip()
if builders:
self.targets = [x.strip() for x in builders.split(',')]
info("%d build targets configured: %s", len(self.targets), self.targets)
else:
self.targets = []
info("No builders set; no binaries will be built")
else:
self.targets = []
def do_release(local_feed, options):
if options.master_feed_file or options.archive_dir_public_url or options.archive_upload_command or options.master_feed_upload_command:
print(legacy_warning)
if options.master_feed_file:
options.master_feed_file = os.path.abspath(options.master_feed_file)
if not local_feed.feed_for:
raise SafeException("Feed %s missing a <feed-for> element" % local_feed.local_path)
status = support.Status()
local_impl = support.get_singleton_impl(local_feed)
local_impl_dir = local_impl.id
assert os.path.isabs(local_impl_dir)
local_impl_dir = os.path.realpath(local_impl_dir)
assert os.path.isdir(local_impl_dir)
if not local_feed.local_path.startswith(local_impl_dir + os.sep):
raise SafeException("Local feed path '%s' does not start with '%s'" %
(local_feed.local_path, local_impl_dir + os.sep))
# From the impl directory to the feed
# NOT relative to the archive root (in general)
local_iface_rel_path = local_feed.local_path[len(local_impl_dir) + 1:]
assert not local_iface_rel_path.startswith('/')
assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
phase_actions = {}
for phase in valid_phases:
phase_actions[phase] = [] # List of <release:action> elements
version_substitutions = []
add_toplevel_dir = None
release_management = local_feed.get_metadata(XMLNS_RELEASE, 'management')
if len(release_management) == 1:
info("Found <release:management> element.")
release_management = release_management[0]
for x in release_management.childNodes:
if x.uri == XMLNS_RELEASE and x.name == 'action':
phase = x.getAttribute('phase')
if phase not in valid_phases:
raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_feed.local_path, '\n'.join(valid_phases)))
phase_actions[phase].append(x.content)
elif x.uri == XMLNS_RELEASE and x.name == 'update-version':
version_substitutions.append((x.getAttribute('path'), re.compile(x.content, re.MULTILINE)))
elif x.uri == XMLNS_RELEASE and x.name == 'add-toplevel-directory':
add_toplevel_dir = local_feed.get_name()
else:
warn("Unknown <release:management> element: %s", x)
elif len(release_management) > 1:
raise SafeException("Multiple <release:management> sections in %s!" % local_feed)
else:
info("No <release:management> element found in local feed.")
scm = get_scm(local_feed, options)
# Path relative to the archive / SCM root
local_iface_rel_root_path = local_feed.local_path[len(scm.root_dir) + 1:]
def run_hooks(phase, cwd, env):
info("Running hooks for phase '%s'" % phase)
full_env = os.environ.copy()
full_env.update(env)
for x in phase_actions[phase]:
print "[%s]: %s" % (phase, x)
support.check_call(x, shell = True, cwd = cwd, env = full_env)
def set_to_release():
print "Snapshot version is " + local_impl.get_version()
release_version = options.release_version
if release_version is None:
suggested = support.suggest_release_version(local_impl.get_version())
release_version = raw_input("Version number for new release [%s]: " % suggested)
if not release_version:
release_version = suggested
scm.ensure_no_tag(release_version)
status.head_before_release = scm.get_head_revision()
status.save()
working_copy = local_impl.id
do_version_substitutions(local_impl_dir, version_substitutions, release_version)
run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
print "Releasing version", release_version
support.publish(local_feed.local_path, set_released = 'today', set_version = release_version)
support.backup_if_exists(release_version)
os.mkdir(release_version)
os.chdir(release_version)
status.old_snapshot_version = local_impl.get_version()
status.release_version = release_version
status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD')
status.save()
def set_to_snapshot(snapshot_version):
assert snapshot_version.endswith('-post')
support.publish(local_feed.local_path, set_released = '', set_version = snapshot_version)
do_version_substitutions(local_impl_dir, version_substitutions, snapshot_version)
scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME)
status.new_snapshot_version = scm.get_head_revision()
status.save()
def ensure_ready_to_release():
#if not options.master_feed_file:
# raise SafeException("Master feed file not set! Check your configuration")
scm.ensure_committed()
scm.ensure_versioned(os.path.abspath(local_feed.local_path))
info("No uncommitted changes. Good.")
# Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
#run_unit_tests(local_impl)
scm.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>')
branch = scm.get_current_branch()
if branch != "refs/heads/master":
print "\nWARNING: you are currently on the '%s' branch.\nThe release will be made from that branch.\n" % branch
def create_feed(target_feed, local_iface_path, archive_file, archive_name, main):
shutil.copyfile(local_iface_path, target_feed)
support.publish(target_feed,
set_main = main,
archive_url = support.get_archive_url(options, status.release_version, os.path.basename(archive_file)),
archive_file = archive_file,
archive_extract = archive_name)
def get_previous_release(this_version):
"""Return the highest numbered verison in the master feed before this_version.
@return: version, or None if there wasn't one"""
parsed_release_version = model.parse_version(this_version)
versions = [model.parse_version(version) for version in scm.get_tagged_versions()]
versions = [version for version in versions if version < parsed_release_version]
if versions:
return model.format_version(max(versions))
return None
def export_changelog(previous_release):
changelog = file('changelog-%s' % status.release_version, 'w')
try:
try:
scm.export_changelog(previous_release, status.head_before_release, changelog)
except SafeException, ex:
print "WARNING: Failed to generate changelog: " + str(ex)
else:
status.created_archive = 'true'
status.save()
if need_set_snapshot:
set_to_snapshot(status.release_version + '-post')
# Revert back to the original revision, so that any fixes the user makes
# will get applied before the tag
scm.reset_hard(scm.get_current_branch())
#backup_if_exists(archive_name)
support.unpack_tarball(archive_file)
extracted_feed_path = os.path.abspath(os.path.join(export_prefix, local_iface_rel_root_path))
assert os.path.isfile(extracted_feed_path), "Local feed not in archive! Is it under version control?"
extracted_feed = support.load_feed(extracted_feed_path)
extracted_impl = support.get_singleton_impl(extracted_feed)
if extracted_impl.main:
# Find main executable, relative to the archive root
abs_main = os.path.join(os.path.dirname(extracted_feed_path), extracted_impl.id, extracted_impl.main)
main = os.path.relpath(abs_main, archive_name + os.sep)
if main != extracted_impl.main:
print "(adjusting main: '%s' for the feed inside the archive, '%s' externally)" % (extracted_impl.main, main)
# XXX: this is going to fail if the feed uses the new <command> syntax
if not os.path.exists(abs_main):
raise SafeException("Main executable '%s' not found after unpacking archive!" % abs_main)
if main == extracted_impl.main:
main = None # Don't change the main attribute
else:
main = None
def do_release(local_feed, options):
if options.master_feed_file or options.archive_dir_public_url or options.archive_upload_command or options.master_feed_upload_command:
print(legacy_warning)
if options.master_feed_file:
options.master_feed_file = os.path.abspath(options.master_feed_file)
if not local_feed.feed_for:
raise SafeException("Feed %s missing a <feed-for> element" % local_feed.local_path)
status = support.Status()
local_impl = support.get_singleton_impl(local_feed)
local_impl_dir = local_impl.id
assert os.path.isabs(local_impl_dir)
local_impl_dir = os.path.realpath(local_impl_dir)
assert os.path.isdir(local_impl_dir)
assert local_feed.local_path.startswith(local_impl_dir + os.sep)
# From the impl directory to the feed
# NOT relative to the archive root (in general)
local_iface_rel_path = local_feed.local_path[len(local_impl_dir) + 1:]
assert not local_iface_rel_path.startswith('/')
assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
phase_actions = {}
for phase in valid_phases:
phase_actions[phase] = [] # List of <release:action> elements
version_substitutions = []
add_toplevel_dir = None
release_management = local_feed.get_metadata(XMLNS_RELEASE, 'management')
if len(release_management) == 1:
info("Found <release:management> element.")
release_management = release_management[0]
for x in release_management.childNodes:
if x.uri == XMLNS_RELEASE and x.name == 'action':
phase = x.getAttribute('phase')
if phase not in valid_phases:
raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_feed.local_path, '\n'.join(valid_phases)))
phase_actions[phase].append(x.content)
elif x.uri == XMLNS_RELEASE and x.name == 'update-version':
version_substitutions.append((x.getAttribute('path'), re.compile(x.content, re.MULTILINE)))
elif x.uri == XMLNS_RELEASE and x.name == 'add-toplevel-directory':
add_toplevel_dir = local_feed.get_name()
else:
warn("Unknown <release:management> element: %s", x)
elif len(release_management) > 1:
raise SafeException("Multiple <release:management> sections in %s!" % local_feed)
else:
info("No <release:management> element found in local feed.")
scm = get_scm(local_feed, options)
# Path relative to the archive / SCM root
local_iface_rel_root_path = local_feed.local_path[len(scm.root_dir) + 1:]
def run_hooks(phase, cwd, env):
info("Running hooks for phase '%s'" % phase)
full_env = os.environ.copy()
full_env.update(env)
for x in phase_actions[phase]:
print "[%s]: %s" % (phase, x)
support.check_call(x, shell = True, cwd = cwd, env = full_env)
def set_to_release():
print "Snapshot version is " + local_impl.get_version()
release_version = options.release_version
if release_version is None:
suggested = support.suggest_release_version(local_impl.get_version())
release_version = raw_input("Version number for new release [%s]: " % suggested)
if not release_version:
release_version = suggested
scm.ensure_no_tag(release_version)
status.head_before_release = scm.get_head_revision()
status.save()
working_copy = local_impl.id
do_version_substitutions(local_impl_dir, version_substitutions, release_version)
run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
print "Releasing version", release_version
support.publish(local_feed.local_path, set_released = 'today', set_version = release_version)
support.backup_if_exists(release_version)
os.mkdir(release_version)
os.chdir(release_version)
status.old_snapshot_version = local_impl.get_version()
status.release_version = release_version
status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD')
status.save()
def set_to_snapshot(snapshot_version):
assert snapshot_version.endswith('-post')
support.publish(local_feed.local_path, set_released = '', set_version = snapshot_version)
do_version_substitutions(local_impl_dir, version_substitutions, snapshot_version)
scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME)
status.new_snapshot_version = scm.get_head_revision()
status.save()
def ensure_ready_to_release():
#if not options.master_feed_file:
# raise SafeException("Master feed file not set! Check your configuration")
scm.ensure_committed()
scm.ensure_versioned(os.path.abspath(local_feed.local_path))
info("No uncommitted changes. Good.")
# Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
#run_unit_tests(local_impl)
scm.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>')
def create_feed(target_feed, local_iface_path, archive_file, archive_name, main):
shutil.copyfile(local_iface_path, target_feed)
support.publish(target_feed,
set_main = main,
archive_url = support.get_archive_url(options, status.release_version, os.path.basename(archive_file)),
archive_file = archive_file,
archive_extract = archive_name)
def get_previous_release(this_version):
"""Return the highest numbered verison in the master feed before this_version.
@return: version, or None if there wasn't one"""
parsed_release_version = model.parse_version(this_version)
versions = [model.parse_version(version) for version in scm.get_tagged_versions()]
versions = [version for version in versions if version < parsed_release_version]
if versions:
return model.format_version(max(versions))
return None
def export_changelog(previous_release):
changelog = file('changelog-%s' % status.release_version, 'w')
try:
try:
scm.export_changelog(previous_release, status.head_before_release, changelog)
except SafeException, ex:
print "WARNING: Failed to generate changelog: " + str(ex)
else:
status.created_archive = 'true'
status.save()
if need_set_snapshot:
set_to_snapshot(status.release_version + '-post')
# Revert back to the original revision, so that any fixes the user makes
# will get applied before the tag
scm.reset_hard(scm.get_current_branch())
#backup_if_exists(archive_name)
support.unpack_tarball(archive_file)
extracted_feed_path = os.path.abspath(os.path.join(export_prefix, local_iface_rel_root_path))
assert os.path.isfile(extracted_feed_path), "Local feed not in archive! Is it under version control?"
extracted_feed = support.load_feed(extracted_feed_path)
extracted_impl = support.get_singleton_impl(extracted_feed)
if extracted_impl.main:
# Find main executable, relative to the archive root
abs_main = os.path.join(os.path.dirname(extracted_feed_path), extracted_impl.id, extracted_impl.main)
main = os.path.relpath(abs_main, archive_name + os.sep)
if main != extracted_impl.main:
print "(adjusting main: '%s' for the feed inside the archive, '%s' externally)" % (extracted_impl.main, main)
# XXX: this is going to fail if the feed uses the new <command> syntax
if not os.path.exists(abs_main):
raise SafeException("Main executable '%s' not found after unpacking archive!" % abs_main)
if main == extracted_impl.main:
main = None # Don't change the main attribute
else:
main = None
