def create_pkg_file(self):
"""Create and pkg file for use with the FreeBSD pkg tool.
Create a package from the result of the package's InstallStep.
"""
install_dir = self.get_install_location()
if not os.path.exists(install_dir):
log('Skiping pkg creation. Install dir not found: %s' %
install_dir)
return
# Strip all elf or pexe files in the install directory (except .o files
# since we don't want to strip, for example, crt1.o)
if not self.config.debug and self.config.toolchain != 'emscripten':
strip = util.get_strip(self.config)
for root, _, files in os.walk(install_dir):
for filename in files:
fullname = os.path.join(root, filename)
if (os.path.isfile(fullname) and util.is_elf_file(fullname)
and os.path.splitext(fullname)[1] != '.o'):
log('stripping: %s %s' % (strip, fullname))
subprocess.check_call([strip, fullname])
abi = 'pkg_' + self.config.toolchain
if self.config.arch != self.config.toolchain:
abi += "_" + util.arch_to_pkgarch[self.config.arch]
abi_dir = os.path.join(paths.PUBLISH_ROOT, abi)
pkg_file = os.path.join(abi_dir,
'%s-%s.tbz' % (self.NAME, self.VERSION))
util.makedirs(abi_dir)
deps = self.DEPENDS
if self.config.toolchain != 'glibc':
deps = []
bsd_pkg.create_pkg_file(self.NAME, self.VERSION, self.config.arch,
self.get_install_location(), pkg_file, deps)
def create_pkg_file(self):
"""Create and pkg file for use with the FreeBSD pkg tool.
Create a package from the result of the package's InstallStep.
"""
install_dir = self.get_install_location()
if not os.path.exists(install_dir):
log('Skiping pkg creation. Install dir not found: %s' % install_dir)
return
# Strip all elf or pexe files in the install directory (except .o files
# since we don't want to strip, for example, crt1.o)
if not self.config.debug and self.config.toolchain != 'emscripten':
strip = util.get_strip(self.config)
for root, _, files in os.walk(install_dir):
for filename in files:
fullname = os.path.join(root, filename)
if (os.path.isfile(fullname) and util.is_elf_file(fullname) and
os.path.splitext(fullname)[1] != '.o'):
log('stripping: %s %s' % (strip, fullname))
subprocess.check_call([strip, fullname])
abi = 'pkg_' + self.config.toolchain
if self.config.arch != self.config.toolchain:
abi += "_" + util.arch_to_pkgarch[self.config.arch]
abi_dir = os.path.join(paths.PUBLISH_ROOT, abi)
pkg_file = os.path.join(abi_dir, '%s-%s.tbz' % (self.NAME, self.VERSION))
util.makedirs(abi_dir)
deps = self.DEPENDS
if self.config.toolchain != 'glibc':
deps = []
bsd_pkg.create_pkg_file(self.NAME, self.VERSION, self.config.arch,
self.get_install_location(), pkg_file, deps)
def write_stamp(stamp_file, stamp_contents):
"""Write a stamp file to disk with the given file contents."""
stamp_dir = os.path.dirname(stamp_file)
util.makedirs(stamp_dir)
with open(stamp_file, 'w') as f:
f.write(stamp_contents)
log('Wrote stamp: %s' % stamp_file)
def write_stamp(stamp_file, stamp_contents):
"""Write a stamp file to disk with the given file contents."""
stamp_dir = os.path.dirname(stamp_file)
util.makedirs(stamp_dir)
with open(stamp_file, 'w') as f:
f.write(stamp_contents)
log('Wrote stamp: %s' % stamp_file)
def _install_files(self, force):
dest = util.get_install_root(self.config)
dest_tmp = os.path.join(dest, 'install_tmp')
if os.path.exists(dest_tmp):
shutil.rmtree(dest_tmp)
if self.is_any_version_installed():
raise error.Error('package already installed: %s' %
self.info_string())
self.log_status('Installing')
util.log_verbose('installing from: %s' % self.filename)
util.makedirs(dest_tmp)
names = []
try:
with tarfile.open(self.filename) as tar:
for info in tar:
if info.isdir():
continue
name = posixpath.normpath(info.name)
if name == 'pkg_info':
continue
if not name.startswith(PAYLOAD_DIR + '/'):
raise error.PkgFormatError(
'invalid file in package: %s' % name)
name = name[len(PAYLOAD_DIR) + 1:]
names.append(name)
if not force:
for name in names:
full_name = os.path.join(dest, name)
if os.path.exists(full_name):
raise error.Error('file already exists: %s' %
full_name)
tar.extractall(dest_tmp)
payload_tree = os.path.join(dest_tmp, PAYLOAD_DIR)
names = filter_out_executables(names, payload_tree)
for name in names:
install_file(name, payload_tree, dest)
finally:
shutil.rmtree(dest_tmp)
for name in names:
relocate_file(name, dest)
self.write_file_list(names)
def build(self, build_deps, force=None):
self.check_buildable()
if build_deps:
self.install_deps(force)
if not force and self.is_built():
self.log_status('Already built')
return
log_root = os.path.join(paths.OUT_DIR, 'logs')
util.makedirs(log_root)
self.log_status('Building')
if util.log_level > util.LOG_INFO:
log_filename = None
else:
log_filename = os.path.join(
log_root,
'%s_%s.log' % (self.NAME, str(self.config).replace('/', '_')))
if os.path.exists(log_filename):
os.remove(log_filename)
start = time.time()
with util.DirLock(self.root):
try:
with redirect_stdout_stderr(log_filename):
old_log_level = util.log_level
util.log_level = util.LOG_VERBOSE
try:
self.download()
self.extract()
self.patch()
self.run_build_sh()
self.create_pkg_file()
finally:
util.log_level = old_log_level
except KeyboardInterrupt:
# Treat KeyboardInterrupt as special, and not an actual failure. This
# avoid log spew to stdout when they user interupts a quit build.
raise
except:
if log_filename:
with open(log_filename) as log_file:
sys.stdout.write(log_file.read())
raise
duration = format_time_delta(time.time() - start)
util.log_heading('Build complete', ' [took %s]' % duration)
def build(self, build_deps, force=None):
self.check_buildable()
if build_deps:
self.install_deps(force)
if not force and self.is_built():
self.log_status('Already built')
return
log_root = os.path.join(paths.OUT_DIR, 'logs')
util.makedirs(log_root)
self.log_status('Building')
if util.log_level > util.LOG_INFO:
log_filename = None
else:
log_filename = os.path.join(log_root, '%s_%s.log' %
(self.NAME,
str(self.config).replace('/', '_')))
if os.path.exists(log_filename):
os.remove(log_filename)
start = time.time()
with util.DirLock(self.root):
try:
with redirect_stdout_stderr(log_filename):
old_log_level = util.log_level
util.log_level = util.LOG_VERBOSE
try:
self.download()
self.extract()
self.patch()
self.run_build_sh()
self.create_pkg_file()
finally:
util.log_level = old_log_level
except KeyboardInterrupt:
# Treat KeyboardInterrupt as special, and not an actual failure. This
# avoid log spew to stdout when they user interupts a quit build.
raise
except:
if log_filename:
with open(log_filename) as log_file:
sys.stdout.write(log_file.read())
raise
duration = format_time_delta(time.time() - start)
util.log_heading('Build complete', ' [took %s]' % duration)
def download(self, package_name, config):
if not os.path.exists(PREBUILT_ROOT):
util.makedirs(PREBUILT_ROOT)
info = self.packages[(package_name, config)]
filename = os.path.join(PREBUILT_ROOT, os.path.basename(info['BIN_URL']))
if os.path.exists(filename):
try:
util.verify_hash(filename, info['BIN_SHA1'])
return filename
except util.HashVerificationError:
pass
util.log('Downloading prebuilt binary ...')
util.download_file(filename, info['BIN_URL'])
util.verify_hash(filename, info['BIN_SHA1'])
return filename
def _install_files(self, force):
dest = util.get_install_root(self.config)
dest_tmp = os.path.join(dest, 'install_tmp')
if os.path.exists(dest_tmp):
shutil.rmtree(dest_tmp)
if self.is_any_version_installed():
raise error.Error('package already installed: %s' % self.info_string())
self.log_status('Installing')
util.log_verbose('installing from: %s' % self.filename)
util.makedirs(dest_tmp)
names = []
try:
with tarfile.open(self.filename) as tar:
for info in tar:
if info.isdir():
continue
name = posixpath.normpath(info.name)
if name == 'pkg_info':
continue
if not name.startswith(PAYLOAD_DIR + '/'):
raise error.PkgFormatError('invalid file in package: %s' % name)
name = name[len(PAYLOAD_DIR) + 1:]
names.append(name)
if not force:
for name in names:
full_name = os.path.join(dest, name)
if os.path.exists(full_name):
raise error.Error('file already exists: %s' % full_name)
tar.extractall(dest_tmp)
payload_tree = os.path.join(dest_tmp, PAYLOAD_DIR)
names = filter_out_executables(names, payload_tree)
for name in names:
install_file(name, payload_tree, dest)
finally:
shutil.rmtree(dest_tmp)
for name in names:
relocate_file(name, dest)
self.write_file_list(names)
def extract(self):
"""Extract the package archive into its build location.
This method assumes the package has already been downloaded.
"""
if self.is_git_upstream():
self.git_clone()
return
archive = self.download_location()
if not archive:
self.log('Skipping extract; No upstream archive')
return
dest = self.get_build_location()
output_path, new_foldername = os.path.split(dest)
util.makedirs(output_path)
# Check existing stamp file contents
stamp_file = self.get_extract_stamp()
stamp_contents = self.get_extract_stamp_content()
if os.path.exists(dest):
if stamp_contents_match(stamp_file, stamp_contents):
log('Already up-to-date: %s' % util.rel_path(dest))
return
raise Error("Upstream archive or patch has changed.\n" +
"Please remove existing checkout and try again: '%s'" %
dest)
util.log_heading('Extracting')
util.makedirs(paths.OUT_DIR)
tmp_output_path = tempfile.mkdtemp(dir=paths.OUT_DIR)
try:
extract_archive(archive, tmp_output_path)
src = os.path.join(tmp_output_path, new_foldername)
if not os.path.isdir(src):
raise Error('Archive contents not found: %s' % src)
log_verbose("renaming '%s' -> '%s'" % (src, dest))
os.rename(src, dest)
finally:
util.remove_tree(tmp_output_path)
self.remove_stamps()
write_stamp(stamp_file, stamp_contents)
def extract(self):
"""Extract the package archive into its build location.
This method assumes the package has already been downloaded.
"""
if self.is_git_upstream():
self.git_clone()
return
archive = self.download_location()
if not archive:
self.log('Skipping extract; No upstream archive')
return
dest = self.get_build_location()
output_path, new_foldername = os.path.split(dest)
util.makedirs(output_path)
# Check existing stamp file contents
stamp_file = self.get_extract_stamp()
stamp_contents = self.get_extract_stamp_content()
if os.path.exists(dest):
if stamp_contents_match(stamp_file, stamp_contents):
log('Already up-to-date: %s' % util.rel_path(dest))
return
raise Error("Upstream archive or patch has changed.\n" +
"Please remove existing checkout and try again: '%s'" % dest)
util.log_heading('Extracting')
util.makedirs(paths.OUT_DIR)
tmp_output_path = tempfile.mkdtemp(dir=paths.OUT_DIR)
try:
extract_archive(archive, tmp_output_path)
src = os.path.join(tmp_output_path, new_foldername)
if not os.path.isdir(src):
raise Error('Archive contents not found: %s' % src)
log_verbose("renaming '%s' -> '%s'" % (src, dest))
os.rename(src, dest)
finally:
util.remove_tree(tmp_output_path)
self.remove_stamps()
write_stamp(stamp_file, stamp_contents)
def download_archive(self, force_mirror):
"""Download the archive to the local cache directory.
Args:
force_mirror: force downloading from mirror only.
"""
filename = self.download_location()
if not filename or os.path.exists(filename):
return
util.makedirs(os.path.dirname(filename))
# Always try the mirror URL first
mirror_url = self.get_mirror_url()
try:
util.download_file(filename, mirror_url)
except Error as e:
if not force_mirror:
# Fall back to the original upstream URL
util.download_file(filename, self.URL)
else:
raise e
def download_archive(self, force_mirror):
"""Download the archive to the local cache directory.
Args:
force_mirror: force downloading from mirror only.
"""
filename = self.download_location()
if not filename or os.path.exists(filename):
return
util.makedirs(os.path.dirname(filename))
# Always try the mirror URL first
mirror_url = self.get_mirror_url()
try:
util.download_file(filename, mirror_url)
except Error as e:
if not force_mirror:
# Fall back to the original upstream URL
util.download_file(filename, self.URL)
else:
raise e
def install_file(filename, old_root, new_root):
"""Install a single file by moving it into a new location.
Args:
filename: Relative name of file to install.
old_root: The current location of the file.
new_root: The new desired root for the file.
"""
oldname = os.path.join(old_root, filename)
util.log_verbose('install: %s' % filename)
newname = os.path.join(new_root, filename)
dirname = os.path.dirname(newname)
if not os.path.isdir(dirname):
util.makedirs(dirname)
os.rename(oldname, newname)
# When install binaries ELF files into the toolchain direcoties, remove
# the X bit so that they do not found when searching the PATH.
if util.is_elf_file(newname) or util.is_pexe_file(newname):
mode = os.stat(newname).st_mode
mode = mode & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
os.chmod(newname, mode)
def make_dir_if_needed(filename):
dirname = os.path.dirname(filename)
if not os.path.isdir(dirname):
util.makedirs(dirname)
