def do_include_deps(args):
"""include-deps"""
buildenv = BuildEnv()
depdir = os.path.realpath('dependencies')
ensure_dir(depdir)
dirs_to_copy = []
sels = buildenv.get_selections()
for needed_iface in sels.selections:
impl = buildenv.chosen_impl(needed_iface)
assert impl
if impl.local_path is not None:
raise SafeException("Can't export '%s' as it's a local implementation (not supported yet; sorry)" % impl)
if not impl.id.startswith('package:'):
dirs_to_copy.append(lookup(impl))
copied = 0
for cached in dirs_to_copy:
required_digest = os.path.basename(cached)
target_impl_dir = os.path.join(depdir, required_digest)
if not os.path.isdir(target_impl_dir):
if required_digest.startswith('sha1='):
shutil.copytree(cached, target_impl_dir)
else:
with open(os.path.join(cached, '.manifest'), 'rb') as stream:
manifest_data = stream.read()
manifest.copy_tree_with_verify(cached, depdir, manifest_data, required_digest)
copied += 1
print("Copied %d dependencies to %s (%d already there)" % (copied, depdir, len(dirs_to_copy) - copied))
def do_include_deps(args):
"""include-deps"""
buildenv = BuildEnv()
depdir = os.path.realpath('dependencies')
ensure_dir(depdir)
dirs_to_copy = []
sels = buildenv.get_selections()
for needed_iface in sels.selections:
impl = buildenv.chosen_impl(needed_iface)
assert impl
if impl.local_path is not None:
raise SafeException("Can't export '%s' as it's a local implementation (not supported yet; sorry)" % impl)
if not impl.id.startswith('package:'):
dirs_to_copy.append(lookup(impl))
copied = 0
for cached in dirs_to_copy:
required_digest = os.path.basename(cached)
target_impl_dir = os.path.join(depdir, required_digest)
if not os.path.isdir(target_impl_dir):
if required_digest.startswith('sha1='):
shutil.copytree(cached, target_impl_dir)
else:
manifest_data = file(os.path.join(cached, '.manifest')).read()
manifest.copy_tree_with_verify(cached, depdir, manifest_data, required_digest)
copied += 1
print "Copied %d dependencies to %s (%d already there)" % (copied, depdir, len(dirs_to_copy) - copied)
def do_build(args):
"""build [ --no-sandbox ] [ --shell | --force | --clean ]"""
buildenv = BuildEnv()
sels = buildenv.get_selections()
parser = OptionParser(usage="usage: %prog build [options]")
parser.add_option('', "--no-sandbox", help="disable use of sandboxing", action='store_true')
parser.add_option("-s", "--shell", help="run a shell instead of building", action='store_true')
parser.add_option("-c", "--clean", help="remove the build directories", action='store_true')
parser.add_option("-f", "--force", help="build even if dependencies have changed", action='store_true')
parser.disable_interspersed_args()
(options, args2) = parser.parse_args(args)
builddir = os.path.realpath('build')
changes = buildenv.get_build_changes()
if changes:
if not (options.force or options.clean):
raise SafeException("Build dependencies have changed:\n" +
'\n'.join(changes) + "\n\n" +
"To build anyway, use: 0compile build --force\n" +
"To do a clean build: 0compile build --clean")
if not options.no_sandbox:
print("Build dependencies have changed:\n" + '\n'.join(changes))
ensure_dir(builddir, options.clean)
ensure_dir(buildenv.distdir, options.clean)
if options.no_sandbox:
return do_build_internal(options, args2)
tmpdir = tempfile.mkdtemp(prefix = '0compile-')
try:
my_dir = os.path.dirname(__file__)
readable = ['.', my_dir]
writable = ['build', buildenv.distdir, tmpdir]
env('TMPDIR', tmpdir)
for selection in list(sels.selections.values()):
if not is_package_impl(selection):
readable.append(lookup(selection))
options = []
if __main__.options.verbose:
options.append('--verbose')
readable.append('/etc') # /etc/ld.*
spawn_and_check_maybe_sandboxed(readable, writable, tmpdir, sys.executable, ['-u', sys.argv[0]] + options + ['build', '--no-sandbox'] + args)
finally:
info("Deleting temporary directory '%s'" % tmpdir)
shutil.rmtree(tmpdir)
def do_build(args):
"""build [ --no-sandbox ] [ --shell | --force | --clean ]"""
buildenv = BuildEnv()
sels = buildenv.get_selections()
parser = OptionParser(usage="usage: %prog build [options]")
parser.add_option('', "--no-sandbox", help="disable use of sandboxing", action='store_true')
parser.add_option("-s", "--shell", help="run a shell instead of building", action='store_true')
parser.add_option("-c", "--clean", help="remove the build directories", action='store_true')
parser.add_option("-f", "--force", help="build even if dependencies have changed", action='store_true')
parser.disable_interspersed_args()
(options, args2) = parser.parse_args(args)
builddir = os.path.realpath('build')
changes = buildenv.get_build_changes()
if changes:
if not (options.force or options.clean):
raise SafeException("Build dependencies have changed:\n" +
'\n'.join(changes) + "\n\n" +
"To build anyway, use: 0compile build --force\n" +
"To do a clean build: 0compile build --clean")
if not options.no_sandbox:
print "Build dependencies have changed:\n" + '\n'.join(changes)
ensure_dir(builddir, options.clean)
ensure_dir(buildenv.distdir, options.clean)
if options.no_sandbox:
return do_build_internal(options, args2)
tmpdir = tempfile.mkdtemp(prefix = '0compile-')
try:
my_dir = os.path.dirname(__file__)
readable = ['.', my_dir]
writable = ['build', buildenv.distdir, tmpdir]
env('TMPDIR', tmpdir)
for selection in sels.selections.values():
if not is_package_impl(selection):
readable.append(lookup(selection))
options = []
if __main__.options.verbose:
options.append('--verbose')
readable.append('/etc') # /etc/ld.*
spawn_and_check_maybe_sandboxed(readable, writable, tmpdir, sys.executable, ['-u', sys.argv[0]] + options + ['build', '--no-sandbox'] + args)
finally:
info("Deleting temporary directory '%s'" % tmpdir)
shutil.rmtree(tmpdir)
def do_build_internal(options, args):
"""build-internal"""
# If a sandbox is being used, we're in it now.
import getpass, socket
buildenv = BuildEnv()
sels = buildenv.get_selections()
builddir = os.path.realpath('build')
ensure_dir(buildenv.metadir)
build_env_xml = join(buildenv.metadir, 'build-environment.xml')
buildenv_doc = sels.toDOM()
# Create build-environment.xml file
root = buildenv_doc.documentElement
info = buildenv_doc.createElementNS(XMLNS_0COMPILE, 'build-info')
root.appendChild(info)
info.setAttributeNS(None, 'time', time.strftime('%Y-%m-%d %H:%M').strip())
info.setAttributeNS(None, 'host', socket.getfqdn())
info.setAttributeNS(None, 'user', getpass.getuser())
info.setAttributeNS(None, 'arch', '%s-%s' % (uname[0], uname[4]))
stream = file(build_env_xml, 'w')
buildenv_doc.writexml(stream, addindent=" ", newl="\n")
stream.close()
# Create local binary interface file.
# We use the main feed for the interface as the template for the name,
# summary, etc (note: this is not necessarily the feed that contained
# the source code).
master_feed = iface_cache.get_feed(buildenv.interface)
src_impl = buildenv.chosen_impl(buildenv.interface)
write_sample_feed(buildenv, master_feed, src_impl)
# Check 0compile is new enough
min_version = model.parse_version(
src_impl.attrs.get(XMLNS_0COMPILE + ' min-version', None))
if min_version and min_version > model.parse_version(__main__.version):
raise SafeException(
"%s-%s requires 0compile >= %s, but we are only version %s" %
(master_feed.get_name(), src_impl.version,
model.format_version(min_version), __main__.version))
# Create the patch
patch_file = join(buildenv.metadir, 'from-%s.patch' % src_impl.version)
if buildenv.user_srcdir:
with open(patch_file, 'w') as stream:
# (ignore errors; will already be shown on stderr)
try:
subprocess.call(["diff", "-urN", buildenv.orig_srcdir, 'src'],
stdout=stream)
except OSError as ex:
print >> sys.stderr, "WARNING: Failed to run 'diff': ", ex
if os.path.getsize(patch_file) == 0:
os.unlink(patch_file)
elif os.path.exists(patch_file):
os.unlink(patch_file)
env('BUILDDIR', builddir)
env('DISTDIR', buildenv.distdir)
env('SRCDIR', buildenv.user_srcdir or buildenv.orig_srcdir)
env('BINARYFEED', buildenv.local_iface_file)
os.chdir(builddir)
print "cd", builddir
setup = CompileSetup(iface_cache.stores, sels)
setup.prepare_env()
# These mappings are needed when mixing Zero Install -dev packages with
# native package binaries.
mappings = {}
for impl in sels.selections.values():
# Add mappings that have been set explicitly...
new_mappings = impl.attrs.get(XMLNS_0COMPILE + ' lib-mappings', '')
if new_mappings:
new_mappings = new_mappings.split(' ')
for mapping in new_mappings:
assert ':' in mapping, "lib-mappings missing ':' in '%s' from '%s'" % (
mapping, impl.feed)
name, major_version = mapping.split(':', 1)
assert '/' not in mapping, "lib-mappings '%s' contains a / in the version number (from '%s')!" % (
mapping, impl.feed)
if sys.platform == 'darwin':
mappings[name] = 'lib%s.%s.dylib' % (name, major_version)
else:
mappings[name] = 'lib%s.so.%s' % (name, major_version)
# Auto-detect required mappings where possible...
# (if the -dev package is native, the symlinks will be OK)
if not is_package_impl(impl):
impl_path = lookup(impl)
for libdirname in ['lib', 'usr/lib', 'lib64', 'usr/lib64']:
libdir = os.path.join(impl_path, libdirname)
if os.path.isdir(libdir):
find_broken_version_symlinks(libdir, mappings)
if mappings:
set_up_mappings(mappings)
overrides_dir = os.path.join(os.environ['TMPDIR'], PKG_CONFIG_OVERRIDES)
if os.path.isdir(overrides_dir):
add_overrides = model.EnvironmentBinding('PKG_CONFIG_PATH',
PKG_CONFIG_OVERRIDES)
do_env_binding(add_overrides, os.environ['TMPDIR'])
# Some programs want to put temporary build files in the source directory.
# Make a copy of the source if needed.
dup_src_type = src_impl.attrs.get(XMLNS_0COMPILE + ' dup-src', None)
if dup_src_type == 'true':
dup_src(copy_file)
env('SRCDIR', builddir)
elif dup_src_type:
raise Exception("Unknown dup-src value '%s'" % dup_src_type)
if options.shell:
spawn_and_check(find_in_path('cmd' if os.name == 'nt' else 'sh'), [])
else:
command = sels.commands[0].qdom.attrs.get('shell-command', None)
if command is None:
# New style <command>
prog_args = setup.build_command(sels.interface,
sels.command) + args
else:
# Old style shell-command='...'
if os.name == 'nt':
prog_args = [
os.environ['0COMPILE_BASH'], '-eux', '-c', command
] + args
else:
prog_args = ['/bin/sh', '-c', command + ' "$@"', '-'] + args
assert len(sels.commands) == 1
# Remove any existing log files
for log in ['build.log', 'build-success.log', 'build-failure.log']:
if os.path.exists(log):
os.unlink(log)
# Run the command, copying output to a new log
with open('build.log', 'w') as log:
print >> log, "Build log for %s-%s" % (master_feed.get_name(),
src_impl.version)
print >> log, "\nBuilt using 0compile-%s" % __main__.version
print >> log, "\nBuild system: " + ', '.join(uname)
print >> log, "\n%s:\n" % ENV_FILE
with open(os.path.join(os.pardir, ENV_FILE)) as properties_file:
shutil.copyfileobj(properties_file, log)
log.write('\n')
if os.path.exists(patch_file):
print >> log, "\nPatched with:\n"
shutil.copyfileobj(file(patch_file), log)
log.write('\n')
if command:
print "Executing: " + command, args
print >> log, "Executing: " + command, args
else:
print "Executing: " + str(prog_args)
print >> log, "Executing: " + str(prog_args)
# Tee the output to the console and to the log
child = subprocess.Popen(prog_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
while True:
data = os.read(child.stdout.fileno(), 100)
if not data: break
sys.stdout.write(data)
log.write(data)
status = child.wait()
failure = None
if status == 0:
print >> log, "Build successful"
shorten_dynamic_library_install_names()
fixup_generated_pkgconfig_files()
remove_la_files()
elif status > 0:
failure = "Build failed with exit code %d" % status
else:
failure = "Build failure: exited due to signal %d" % (-status)
if failure:
print >> log, failure
if failure:
os.rename('build.log', 'build-failure.log')
raise SafeException("Command '%s': %s" % (prog_args, failure))
else:
os.rename('build.log', 'build-success.log')
def do_build_internal(options, args):
"""build-internal"""
# If a sandbox is being used, we're in it now.
import getpass, socket
buildenv = BuildEnv()
sels = buildenv.get_selections()
builddir = os.path.realpath('build')
ensure_dir(buildenv.metadir)
build_env_xml = join(buildenv.metadir, 'build-environment.xml')
buildenv_doc = sels.toDOM()
# Create build-environment.xml file
root = buildenv_doc.documentElement
info = buildenv_doc.createElementNS(XMLNS_0COMPILE, 'build-info')
root.appendChild(info)
info.setAttributeNS(None, 'time', time.strftime('%Y-%m-%d %H:%M').strip())
info.setAttributeNS(None, 'host', socket.getfqdn())
info.setAttributeNS(None, 'user', getpass.getuser())
info.setAttributeNS(None, 'arch', '%s-%s' % (uname[0], uname[4]))
stream = file(build_env_xml, 'w')
buildenv_doc.writexml(stream, addindent=" ", newl="\n")
stream.close()
# Create local binary interface file.
# We use the main feed for the interface as the template for the name,
# summary, etc (note: this is not necessarily the feed that contained
# the source code).
master_feed = iface_cache.get_feed(buildenv.interface)
src_impl = buildenv.chosen_impl(buildenv.interface)
write_sample_feed(buildenv, master_feed, src_impl)
# Check 0compile is new enough
min_version = model.parse_version(src_impl.attrs.get(XMLNS_0COMPILE + ' min-version', None))
if min_version and min_version > model.parse_version(__main__.version):
raise SafeException("%s-%s requires 0compile >= %s, but we are only version %s" %
(master_feed.get_name(), src_impl.version, model.format_version(min_version), __main__.version))
# Create the patch
patch_file = join(buildenv.metadir, 'from-%s.patch' % src_impl.version)
if buildenv.user_srcdir:
with open(patch_file, 'w') as stream:
# (ignore errors; will already be shown on stderr)
try:
subprocess.call(["diff", "-urN", buildenv.orig_srcdir, 'src'], stdout = stream)
except OSError as ex:
print >>sys.stderr, "WARNING: Failed to run 'diff': ", ex
if os.path.getsize(patch_file) == 0:
os.unlink(patch_file)
elif os.path.exists(patch_file):
os.unlink(patch_file)
env('BUILDDIR', builddir)
env('DISTDIR', buildenv.distdir)
env('SRCDIR', buildenv.user_srcdir or buildenv.orig_srcdir)
env('BINARYFEED', buildenv.local_iface_file)
os.chdir(builddir)
print "cd", builddir
setup = CompileSetup(iface_cache.stores, sels)
setup.prepare_env()
# These mappings are needed when mixing Zero Install -dev packages with
# native package binaries.
mappings = {}
for impl in sels.selections.values():
# Add mappings that have been set explicitly...
new_mappings = impl.attrs.get(XMLNS_0COMPILE + ' lib-mappings', '')
if new_mappings:
new_mappings = new_mappings.split(' ')
for mapping in new_mappings:
assert ':' in mapping, "lib-mappings missing ':' in '%s' from '%s'" % (mapping, impl.feed)
name, major_version = mapping.split(':', 1)
assert '/' not in mapping, "lib-mappings '%s' contains a / in the version number (from '%s')!" % (mapping, impl.feed)
if sys.platform == 'darwin':
mappings[name] = 'lib%s.%s.dylib' % (name, major_version)
else:
mappings[name] = 'lib%s.so.%s' % (name, major_version)
# Auto-detect required mappings where possible...
# (if the -dev package is native, the symlinks will be OK)
if not is_package_impl(impl):
impl_path = lookup(impl)
for libdirname in ['lib', 'usr/lib', 'lib64', 'usr/lib64']:
libdir = os.path.join(impl_path, libdirname)
if os.path.isdir(libdir):
find_broken_version_symlinks(libdir, mappings)
if mappings:
set_up_mappings(mappings)
overrides_dir = os.path.join(os.environ['TMPDIR'], PKG_CONFIG_OVERRIDES)
if os.path.isdir(overrides_dir):
add_overrides = model.EnvironmentBinding('PKG_CONFIG_PATH', PKG_CONFIG_OVERRIDES)
do_env_binding(add_overrides, os.environ['TMPDIR'])
# Some programs want to put temporary build files in the source directory.
# Make a copy of the source if needed.
dup_src_type = src_impl.attrs.get(XMLNS_0COMPILE + ' dup-src', None)
if dup_src_type == 'true':
dup_src(shutil.copy2)
env('SRCDIR', builddir)
elif dup_src_type:
raise Exception("Unknown dup-src value '%s'" % dup_src_type)
if options.shell:
spawn_and_check(find_in_path('cmd' if os.name == 'nt' else 'sh'), [])
else:
command = sels.commands[0].qdom.attrs.get('shell-command', None)
if command is None:
# New style <command>
prog_args = setup.build_command(sels.interface, sels.command) + args
else:
# Old style shell-command='...'
if os.name == 'nt':
prog_args = ['cmd', '/c', command] + args
else:
prog_args = ['/bin/sh', '-c', command + ' "$@"', '-'] + args
assert len(sels.commands) == 1
# Remove any existing log files
for log in ['build.log', 'build-success.log', 'build-failure.log']:
if os.path.exists(log):
os.unlink(log)
# Run the command, copying output to a new log
with open('build.log', 'w') as log:
print >>log, "Build log for %s-%s" % (master_feed.get_name(),
src_impl.version)
print >>log, "\nBuilt using 0compile-%s" % __main__.version
print >>log, "\nBuild system: " + ', '.join(uname)
print >>log, "\n%s:\n" % ENV_FILE
with open(os.path.join(os.pardir, ENV_FILE)) as properties_file:
shutil.copyfileobj(properties_file, log)
log.write('\n')
if os.path.exists(patch_file):
print >>log, "\nPatched with:\n"
shutil.copyfileobj(file(patch_file), log)
log.write('\n')
if command:
print "Executing: " + command, args
print >>log, "Executing: " + command, args
else:
print "Executing: " + str(prog_args)
print >>log, "Executing: " + str(prog_args)
# Tee the output to the console and to the log
child = subprocess.Popen(prog_args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
while True:
data = os.read(child.stdout.fileno(), 100)
if not data: break
sys.stdout.write(data)
log.write(data)
status = child.wait()
failure = None
if status == 0:
print >>log, "Build successful"
shorten_dynamic_library_install_names()
fixup_generated_pkgconfig_files()
remove_la_files()
elif status > 0:
failure = "Build failed with exit code %d" % status
else:
failure = "Build failure: exited due to signal %d" % (-status)
if failure:
print >>log, failure
if failure:
os.rename('build.log', 'build-failure.log')
raise SafeException("Command '%s': %s" % (prog_args, failure))
else:
os.rename('build.log', 'build-success.log')