def build_slave(src_feed, archive_file, archive_dir_public_url, target_feed): try: COMPILE = [os.environ['0COMPILE']] except KeyError: # (build slave has an old 0release) COMPILE = [ '0launch', '--not-before=1.2', 'http://0install.net/2006/interfaces/0compile.xml' ] feed = support.load_feed(src_feed) src_feed = os.path.abspath(src_feed) archive_file = os.path.abspath(archive_file) target_feed = os.path.abspath(target_feed) impl, = feed.implementations.values() tmpdir = tempfile.mkdtemp(prefix='0release-') try: os.chdir(tmpdir) depdir = os.path.join(tmpdir, 'dependencies') os.mkdir(depdir) support.unpack_tarball(archive_file) portable_rename(impl.download_sources[0].extract, os.path.join(depdir, impl.id)) config = ConfigParser.RawConfigParser() config.add_section('compile') config.set('compile', 'download-base-url', archive_dir_public_url) config.set('compile', 'version-modifier', '') config.set('compile', 'interface', src_feed) config.set('compile', 'selections', '') config.set('compile', 'metadir', '0install') stream = open(os.path.join(tmpdir, '0compile.properties'), 'w') try: config.write(stream) finally: stream.close() support.check_call(COMPILE + ['build'], cwd=tmpdir) support.check_call(COMPILE + ['publish', '--target-feed', target_feed], cwd=tmpdir) # TODO: run unit-tests feed = support.load_feed(target_feed) impl = support.get_singleton_impl(feed) archive_file = support.get_archive_basename(impl) shutil.move(archive_file, os.path.join(os.path.dirname(target_feed), archive_file)) except: print "\nLeaving temporary directory %s for inspection...\n" % tmpdir raise else: shutil.rmtree(tmpdir)
def build_slave(src_feed, archive_file, archive_dir_public_url, target_feed): try: COMPILE = [os.environ['0COMPILE']] except KeyError: # (build slave has an old 0release) COMPILE = ['0launch', '--not-before=0.30', 'http://0install.net/2006/interfaces/0compile.xml'] feed = support.load_feed(src_feed) src_feed = os.path.abspath(src_feed) archive_file = os.path.abspath(archive_file) target_feed = os.path.abspath(target_feed) impl, = feed.implementations.values() tmpdir = tempfile.mkdtemp(prefix = '0release-') try: os.chdir(tmpdir) depdir = os.path.join(tmpdir, 'dependencies') os.mkdir(depdir) support.unpack_tarball(archive_file) portable_rename(impl.download_sources[0].extract, os.path.join(depdir, impl.id)) config = ConfigParser.RawConfigParser() config.add_section('compile') config.set('compile', 'download-base-url', archive_dir_public_url) config.set('compile', 'version-modifier', '') config.set('compile', 'interface', src_feed) config.set('compile', 'selections', '') config.set('compile', 'metadir', '0install') stream = open(os.path.join(tmpdir, '0compile.properties'), 'w') try: config.write(stream) finally: stream.close() support.check_call(COMPILE + ['build'], cwd = tmpdir) support.check_call(COMPILE + ['publish', '--target-feed', target_feed], cwd = tmpdir) # TODO: run unit-tests feed = support.load_feed(target_feed) impl = support.get_singleton_impl(feed) archive_file = support.get_archive_basename(impl) shutil.move(archive_file, os.path.join(os.path.dirname(target_feed), archive_file)) except: print "\nLeaving temporary directory %s for inspection...\n" % tmpdir raise else: shutil.rmtree(tmpdir)
def build_binaries(self): if not self.targets: return print "Source package, so generating binaries..." archive_file = support.get_archive_basename(self.src_impl) for target in self.targets: start = self.get('builder-' + target, 'start', None) command = self.config.get('builder-' + target, 'build') stop = self.get('builder-' + target, 'stop', None) binary_feed = 'binary-' + target + '.xml' if os.path.exists(binary_feed): print "Feed %s already exists; not rebuilding" % binary_feed else: print "\nBuilding binary with builder '%s' ...\n" % target if start: support.show_and_run(start, []) try: args = [ os.path.basename(self.src_feed_name), archive_file, self.archive_dir_public_url, binary_feed + '.new' ] if not command: assert target == 'host', 'Missing build command' support.check_call( [sys.executable, sys.argv[0], '--build-slave'] + args) else: support.show_and_run(command, args) finally: if stop: support.show_and_run(stop, []) bin_feed = support.load_feed(binary_feed + '.new') bin_impl = support.get_singleton_impl(bin_feed) bin_archive_file = support.get_archive_basename(bin_impl) bin_size = bin_impl.download_sources[0].size assert os.path.exists( bin_archive_file ), "Compiled binary '%s' not found!" % os.path.abspath( bin_archive_file) assert os.path.getsize( bin_archive_file ) == bin_size, "Compiled binary '%s' has wrong size!" % os.path.abspath( bin_archive_file) portable_rename(binary_feed + '.new', binary_feed)
def __init__(self, options, src_feed_name, release_version): self.src_feed_name = src_feed_name self.src_feed = support.load_feed(src_feed_name) self.archive_dir_public_url = support.get_archive_url( options, release_version, '') self.config = ConfigParser.RawConfigParser() # Start with a default configuration self.config.add_section('global') self.config.set('global', 'builders', 'host') self.config.add_section('builder-host') #self.config.set('builder-host', 'build', '0launch --not-before 0.10 http://0install.net/2007/interfaces/0release.xml --build-slave "$@"') self.config.set('builder-host', 'build', '') self.src_impl = support.get_singleton_impl(self.src_feed) if self.src_impl.arch and self.src_impl.arch.endswith('-src'): path = basedir.load_first_config('0install.net', '0release', 'builders.conf') if path: info("Loading configuration file '%s'", path) self.config.read(path) else: info( "No builders.conf configuration; will build a binary for this host only" ) if options.builders is not None: builders = options.builders else: builders = self.config.get('global', 'builders').strip() if builders: self.targets = [x.strip() for x in builders.split(',')] info("%d build targets configured: %s", len(self.targets), self.targets) else: self.targets = [] info("No builders set; no binaries will be built") else: self.targets = []
def build_binaries(self): if not self.targets: return print "Source package, so generating binaries..." archive_file = support.get_archive_basename(self.src_impl) for target in self.targets: start = self.get('builder-' + target, 'start', None) command = self.config.get('builder-' + target, 'build') stop = self.get('builder-' + target, 'stop', None) binary_feed = 'binary-' + target + '.xml' if os.path.exists(binary_feed): print "Feed %s already exists; not rebuilding" % binary_feed else: print "\nBuilding binary with builder '%s' ...\n" % target if start: support.show_and_run(start, []) try: args = [os.path.basename(self.src_feed_name), archive_file, self.archive_dir_public_url, binary_feed + '.new'] if not command: assert target == 'host', 'Missing build command' support.check_call([sys.executable, sys.argv[0], '--build-slave'] + args) else: support.show_and_run(command, args) finally: if stop: support.show_and_run(stop, []) bin_feed = support.load_feed(binary_feed + '.new') bin_impl = support.get_singleton_impl(bin_feed) bin_archive_file = support.get_archive_basename(bin_impl) bin_size = bin_impl.download_sources[0].size assert os.path.exists(bin_archive_file), "Compiled binary '%s' not found!" % os.path.abspath(bin_archive_file) assert os.path.getsize(bin_archive_file) == bin_size, "Compiled binary '%s' has wrong size!" % os.path.abspath(bin_archive_file) portable_rename(binary_feed + '.new', binary_feed)
def __init__(self, options, src_feed_name, release_version): self.src_feed_name = src_feed_name self.src_feed = support.load_feed(src_feed_name) self.archive_dir_public_url = support.get_archive_url(options, release_version, '') self.config = ConfigParser.RawConfigParser() # Start with a default configuration self.config.add_section('global') self.config.set('global', 'builders', 'host') self.config.add_section('builder-host') #self.config.set('builder-host', 'build', '0launch --not-before 0.10 http://0install.net/2007/interfaces/0release.xml --build-slave "$@"') self.config.set('builder-host', 'build', '') self.src_impl = support.get_singleton_impl(self.src_feed) if self.src_impl.arch and self.src_impl.arch.endswith('-src'): path = basedir.load_first_config('0install.net', '0release', 'builders.conf') if path: info("Loading configuration file '%s'", path) self.config.read(path) else: info("No builders.conf configuration; will build a binary for this host only") if options.builders is not None: builders = options.builders else: builders = self.config.get('global', 'builders').strip() if builders: self.targets = [x.strip() for x in builders.split(',')] info("%d build targets configured: %s", len(self.targets), self.targets) else: self.targets = [] info("No builders set; no binaries will be built") else: self.targets = []
def do_release(local_feed, options): if options.master_feed_file or options.archive_dir_public_url or options.archive_upload_command or options.master_feed_upload_command: print(legacy_warning) if options.master_feed_file: options.master_feed_file = os.path.abspath(options.master_feed_file) if not local_feed.feed_for: raise SafeException("Feed %s missing a <feed-for> element" % local_feed.local_path) status = support.Status() local_impl = support.get_singleton_impl(local_feed) local_impl_dir = local_impl.id assert os.path.isabs(local_impl_dir) local_impl_dir = os.path.realpath(local_impl_dir) assert os.path.isdir(local_impl_dir) if not local_feed.local_path.startswith(local_impl_dir + os.sep): raise SafeException("Local feed path '%s' does not start with '%s'" % (local_feed.local_path, local_impl_dir + os.sep)) # From the impl directory to the feed # NOT relative to the archive root (in general) local_iface_rel_path = local_feed.local_path[len(local_impl_dir) + 1:] assert not local_iface_rel_path.startswith('/') assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path)) phase_actions = {} for phase in valid_phases: phase_actions[phase] = [] # List of <release:action> elements version_substitutions = [] add_toplevel_dir = None release_management = local_feed.get_metadata(XMLNS_RELEASE, 'management') if len(release_management) == 1: info("Found <release:management> element.") release_management = release_management[0] for x in release_management.childNodes: if x.uri == XMLNS_RELEASE and x.name == 'action': phase = x.getAttribute('phase') if phase not in valid_phases: raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_feed.local_path, '\n'.join(valid_phases))) phase_actions[phase].append(x.content) elif x.uri == XMLNS_RELEASE and x.name == 'update-version': version_substitutions.append((x.getAttribute('path'), re.compile(x.content, re.MULTILINE))) elif x.uri == XMLNS_RELEASE and x.name == 'add-toplevel-directory': add_toplevel_dir = local_feed.get_name() else: warn("Unknown <release:management> element: %s", x) elif len(release_management) > 1: raise SafeException("Multiple <release:management> sections in %s!" % local_feed) else: info("No <release:management> element found in local feed.") scm = get_scm(local_feed, options) # Path relative to the archive / SCM root local_iface_rel_root_path = local_feed.local_path[len(scm.root_dir) + 1:] def run_hooks(phase, cwd, env): info("Running hooks for phase '%s'" % phase) full_env = os.environ.copy() full_env.update(env) for x in phase_actions[phase]: print "[%s]: %s" % (phase, x) support.check_call(x, shell = True, cwd = cwd, env = full_env) def set_to_release(): print "Snapshot version is " + local_impl.get_version() release_version = options.release_version if release_version is None: suggested = support.suggest_release_version(local_impl.get_version()) release_version = raw_input("Version number for new release [%s]: " % suggested) if not release_version: release_version = suggested scm.ensure_no_tag(release_version) status.head_before_release = scm.get_head_revision() status.save() working_copy = local_impl.id do_version_substitutions(local_impl_dir, version_substitutions, release_version) run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version}) print "Releasing version", release_version support.publish(local_feed.local_path, set_released = 'today', set_version = release_version) support.backup_if_exists(release_version) os.mkdir(release_version) os.chdir(release_version) status.old_snapshot_version = local_impl.get_version() status.release_version = release_version status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD') status.save() def set_to_snapshot(snapshot_version): assert snapshot_version.endswith('-post') support.publish(local_feed.local_path, set_released = '', set_version = snapshot_version) do_version_substitutions(local_impl_dir, version_substitutions, snapshot_version) scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME) status.new_snapshot_version = scm.get_head_revision() status.save() def ensure_ready_to_release(): #if not options.master_feed_file: # raise SafeException("Master feed file not set! Check your configuration") scm.ensure_committed() scm.ensure_versioned(os.path.abspath(local_feed.local_path)) info("No uncommitted changes. Good.") # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful. #run_unit_tests(local_impl) scm.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>') branch = scm.get_current_branch() if branch != "refs/heads/master": print "\nWARNING: you are currently on the '%s' branch.\nThe release will be made from that branch.\n" % branch def create_feed(target_feed, local_iface_path, archive_file, archive_name, main): shutil.copyfile(local_iface_path, target_feed) support.publish(target_feed, set_main = main, archive_url = support.get_archive_url(options, status.release_version, os.path.basename(archive_file)), archive_file = archive_file, archive_extract = archive_name) def get_previous_release(this_version): """Return the highest numbered verison in the master feed before this_version. @return: version, or None if there wasn't one""" parsed_release_version = model.parse_version(this_version) versions = [model.parse_version(version) for version in scm.get_tagged_versions()] versions = [version for version in versions if version < parsed_release_version] if versions: return model.format_version(max(versions)) return None def export_changelog(previous_release): changelog = file('changelog-%s' % status.release_version, 'w') try: try: scm.export_changelog(previous_release, status.head_before_release, changelog) except SafeException, ex: print "WARNING: Failed to generate changelog: " + str(ex) else:
status.created_archive = 'true' status.save() if need_set_snapshot: set_to_snapshot(status.release_version + '-post') # Revert back to the original revision, so that any fixes the user makes # will get applied before the tag scm.reset_hard(scm.get_current_branch()) #backup_if_exists(archive_name) support.unpack_tarball(archive_file) extracted_feed_path = os.path.abspath(os.path.join(export_prefix, local_iface_rel_root_path)) assert os.path.isfile(extracted_feed_path), "Local feed not in archive! Is it under version control?" extracted_feed = support.load_feed(extracted_feed_path) extracted_impl = support.get_singleton_impl(extracted_feed) if extracted_impl.main: # Find main executable, relative to the archive root abs_main = os.path.join(os.path.dirname(extracted_feed_path), extracted_impl.id, extracted_impl.main) main = os.path.relpath(abs_main, archive_name + os.sep) if main != extracted_impl.main: print "(adjusting main: '%s' for the feed inside the archive, '%s' externally)" % (extracted_impl.main, main) # XXX: this is going to fail if the feed uses the new <command> syntax if not os.path.exists(abs_main): raise SafeException("Main executable '%s' not found after unpacking archive!" % abs_main) if main == extracted_impl.main: main = None # Don't change the main attribute else: main = None
def do_release(local_feed, options): if options.master_feed_file or options.archive_dir_public_url or options.archive_upload_command or options.master_feed_upload_command: print(legacy_warning) if options.master_feed_file: options.master_feed_file = os.path.abspath(options.master_feed_file) if not local_feed.feed_for: raise SafeException("Feed %s missing a <feed-for> element" % local_feed.local_path) status = support.Status() local_impl = support.get_singleton_impl(local_feed) local_impl_dir = local_impl.id assert os.path.isabs(local_impl_dir) local_impl_dir = os.path.realpath(local_impl_dir) assert os.path.isdir(local_impl_dir) assert local_feed.local_path.startswith(local_impl_dir + os.sep) # From the impl directory to the feed # NOT relative to the archive root (in general) local_iface_rel_path = local_feed.local_path[len(local_impl_dir) + 1:] assert not local_iface_rel_path.startswith('/') assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path)) phase_actions = {} for phase in valid_phases: phase_actions[phase] = [] # List of <release:action> elements version_substitutions = [] add_toplevel_dir = None release_management = local_feed.get_metadata(XMLNS_RELEASE, 'management') if len(release_management) == 1: info("Found <release:management> element.") release_management = release_management[0] for x in release_management.childNodes: if x.uri == XMLNS_RELEASE and x.name == 'action': phase = x.getAttribute('phase') if phase not in valid_phases: raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_feed.local_path, '\n'.join(valid_phases))) phase_actions[phase].append(x.content) elif x.uri == XMLNS_RELEASE and x.name == 'update-version': version_substitutions.append((x.getAttribute('path'), re.compile(x.content, re.MULTILINE))) elif x.uri == XMLNS_RELEASE and x.name == 'add-toplevel-directory': add_toplevel_dir = local_feed.get_name() else: warn("Unknown <release:management> element: %s", x) elif len(release_management) > 1: raise SafeException("Multiple <release:management> sections in %s!" % local_feed) else: info("No <release:management> element found in local feed.") scm = get_scm(local_feed, options) # Path relative to the archive / SCM root local_iface_rel_root_path = local_feed.local_path[len(scm.root_dir) + 1:] def run_hooks(phase, cwd, env): info("Running hooks for phase '%s'" % phase) full_env = os.environ.copy() full_env.update(env) for x in phase_actions[phase]: print "[%s]: %s" % (phase, x) support.check_call(x, shell = True, cwd = cwd, env = full_env) def set_to_release(): print "Snapshot version is " + local_impl.get_version() release_version = options.release_version if release_version is None: suggested = support.suggest_release_version(local_impl.get_version()) release_version = raw_input("Version number for new release [%s]: " % suggested) if not release_version: release_version = suggested scm.ensure_no_tag(release_version) status.head_before_release = scm.get_head_revision() status.save() working_copy = local_impl.id do_version_substitutions(local_impl_dir, version_substitutions, release_version) run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version}) print "Releasing version", release_version support.publish(local_feed.local_path, set_released = 'today', set_version = release_version) support.backup_if_exists(release_version) os.mkdir(release_version) os.chdir(release_version) status.old_snapshot_version = local_impl.get_version() status.release_version = release_version status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD') status.save() def set_to_snapshot(snapshot_version): assert snapshot_version.endswith('-post') support.publish(local_feed.local_path, set_released = '', set_version = snapshot_version) do_version_substitutions(local_impl_dir, version_substitutions, snapshot_version) scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME) status.new_snapshot_version = scm.get_head_revision() status.save() def ensure_ready_to_release(): #if not options.master_feed_file: # raise SafeException("Master feed file not set! Check your configuration") scm.ensure_committed() scm.ensure_versioned(os.path.abspath(local_feed.local_path)) info("No uncommitted changes. Good.") # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful. #run_unit_tests(local_impl) scm.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>') def create_feed(target_feed, local_iface_path, archive_file, archive_name, main): shutil.copyfile(local_iface_path, target_feed) support.publish(target_feed, set_main = main, archive_url = support.get_archive_url(options, status.release_version, os.path.basename(archive_file)), archive_file = archive_file, archive_extract = archive_name) def get_previous_release(this_version): """Return the highest numbered verison in the master feed before this_version. @return: version, or None if there wasn't one""" parsed_release_version = model.parse_version(this_version) versions = [model.parse_version(version) for version in scm.get_tagged_versions()] versions = [version for version in versions if version < parsed_release_version] if versions: return model.format_version(max(versions)) return None def export_changelog(previous_release): changelog = file('changelog-%s' % status.release_version, 'w') try: try: scm.export_changelog(previous_release, status.head_before_release, changelog) except SafeException, ex: print "WARNING: Failed to generate changelog: " + str(ex) else: