def relocate_osx_libraries(self):
'''
Make OSX libraries relocatable
'''
relocator = OSXRelocator(self.config.prefix, self.config.prefix, True)
for f in self.files_list():
if f.split('/')[0] in ['lib', 'bin', 'libexec']:
relocator.relocate_file(os.path.join(self.config.prefix, f))
def _relocate_binaries(self):
prefix = self.config.prefix
if prefix[-1] == '/':
prefix = prefix[:-1]
for path in ['bin', 'lib', 'libexec']:
relocator = OSXRelocator(
os.path.join(self.appdir, 'Contents', 'Home', path),
self.config.prefix, '@executable_path/../', True)
relocator.relocate()
def _relocate_binaries(self):
prefix = self.config.prefix
if prefix[-1] == '/':
prefix = prefix[:-1]
for path in ['bin', 'lib', 'libexec']:
relocator = OSXRelocator(
os.path.join(self.appdir, 'Contents', 'Home', path),
self.config.prefix, '@executable_path/../', True)
relocator.relocate()
def _relocate_binaries(self, tmp_dir):
if not self.package.relocate_osx_binaries:
return
prefix = self.config.prefix
if prefix[-1] == '/':
prefix = prefix[:-1]
for path in ['bin', 'lib', 'libexec']:
relocator = OSXRelocator(os.path.join(tmp_dir, path),
self.config.prefix, '@executable_path/../', True)
relocator.relocate()
def _relocate_binaries(self, tmp_dir):
if not self.package.relocate_osx_binaries:
return
prefix = self.config.prefix
if prefix[-1] == '/':
prefix = prefix[:-1]
for path in ['bin', 'lib', 'libexec']:
relocator = OSXRelocator(os.path.join(tmp_dir,
path), self.config.prefix,
'@executable_path/../', True)
relocator.relocate()
def _relocate_binaries(self):
if not self.package.relocate_osx_binaries:
return
prefix = self.config.prefix
if prefix[-1] == "/":
prefix = prefix[:-1]
for path in ["bin", "lib", "libexec"]:
relocator = OSXRelocator(
os.path.join(self.appdir, "Contents", "Home", path), self.config.prefix, "@executable_path/../", True
)
relocator.relocate()
class RelocatableTarOSX(RelocatableTar):
def extractAndRelocate(self, extract_to_path):
self.relocator = OSXRelocator(extract_to_path, extract_to_path, True)
RelocatableTar.extractAndRelocate(self, extract_to_path)
def _isDyLibFile(self, file):
return os.path.splitext(file)[1] in ['.dylib']
def _hasRelocatableExtension(self, file):
return (RelocatableTar._hasRelocatableExtension(self, file) or
self._isDyLibFile(file))
def _relocate(self, file, subst_path):
if self._isDyLibFile(file):
self.relocator.change_id(file, file)
else:
RelocatableTar._relocate(self, file, subst_path)
def relocate_osx_libraries(self):
'''
Make OSX libraries relocatable
'''
relocator = OSXRelocator(self.config.prefix, self.config.prefix, True)
def get_real_path(fp):
return os.path.realpath(os.path.join(self.config.prefix, fp))
def file_is_relocatable(fp):
return fp.split('/')[0] in ['lib', 'bin', 'libexec'] and \
os.path.splitext(fp)[1] not in ['.a', '.pc', '.la']
# Only relocate files are that are potentially relocatable and
# remove duplicates by symbolic links so we relocate libs only
# once.
for f in set([get_real_path(x) for x in self.files_list() \
if file_is_relocatable(x)]):
relocator.relocate_file(f)
def create_universal_file(self, output, inputlist, dirs):
tmp_inputs = []
# relocate all files with the prefix of the merged file.
# which must be done before merging them.
for f in inputlist:
# keep the filename in the suffix to preserve the filename extension
tmp = tempfile.NamedTemporaryFile(suffix=os.path.basename(f))
tmp_inputs.append(tmp)
shutil.copy(f, tmp.name)
prefix_to_replace = [d for d in dirs if d in f][0]
relocator = OSXRelocator (self.output_root, prefix_to_replace, self.output_root,
False)
# since we are using a temporary file, we must force the library id
# name to real one and not based on the filename
relocator.relocate_file(tmp.name,
id=f.replace(prefix_to_replace, self.output_root))
cmd = '%s -create %s -output %s' % (self.LIPO_CMD,
' '.join([f.name for f in tmp_inputs]), output)
self._call(cmd)
def create_universal_file(self, output, inputlist, dirs):
tmp_inputs = []
# relocate all files with the prefix of the merged file.
# which must be done before merging them.
for f in inputlist:
# keep the filename in the suffix to preserve the filename extension
tmp = tempfile.NamedTemporaryFile(suffix=os.path.basename(f))
tmp_inputs.append(tmp)
shutil.copy(f, tmp.name)
prefix_to_replace = [d for d in dirs if d in f][0]
relocator = OSXRelocator (self.output_root, prefix_to_replace, self.output_root,
False)
# since we are using a temporary file, we must force the library id
# name to real one and not based on the filename
relocator.relocate_file(tmp.name,
id=f.replace(prefix_to_replace, self.output_root))
cmd = '%s -create %s -output %s' % (self.LIPO_CMD,
' '.join([f.name for f in tmp_inputs]), output)
self._call(cmd)
def __init__(self, store, force=False, dry_run=False):
self.store = store
self.cookbook = store.cookbook
self.config = self.cookbook.get_config()
self.force = force
shell.DRY_RUN = dry_run
if not self.config.binaries:
raise FatalError(_('Configuration without binaries path'))
self.binaries = os.path.join(self.config.binaries,
self.config.get_md5())
if not self.config.binary_repo:
raise FatalError(_('Configuration without binary repo'))
self.binary_repo = os.path.join(self.config.binary_repo,
self.config.get_md5())
m.message('Using config MD5: %s' % self.config.get_md5())
if not os.path.exists(self.binaries):
os.makedirs(self.binaries)
if self.config.target_platform == Platform.DARWIN:
self.relocator = OSXRelocator(self.config.prefix,
self.config.prefix, True)
def _create_tarball(self, filename, files, package_prefix, relocatable):
tar = tarfile.open(filename, "w:bz2")
for f in files:
filepath = os.path.join(self.prefix, f)
arcname = os.path.join(package_prefix, f)
if relocatable and not os.path.islink(filepath):
if os.path.splitext(f)[1] in [
'.la', '.pc'
] or ('bin' in os.path.splitext(f)[0]
and is_text_file(filepath)):
with open(filepath, 'r') as fo:
content = fo.read()
content = replace_prefix(self.config.prefix, content,
"CERBERO_PREFIX")
rewritten = tempfile.NamedTemporaryFile()
rewritten.write(content)
rewritten.flush()
rewritten.seek(0)
tinfo = tar.gettarinfo(arcname=arcname, fileobj=fo)
tinfo.size = len(content)
tinfo.name = os.path.join(package_prefix, f)
tar.addfile(tinfo, rewritten)
rewritten.close()
elif os.path.splitext(f)[1] in [
'.dylib'
] and self.config.target_platform == Platform.DARWIN:
tempdir = tempfile.mkdtemp()
os.makedirs(os.path.join(tempdir, os.path.dirname(f)))
rewritten = os.path.join(tempdir, f)
shutil.copy(filepath, rewritten)
relocator = OSXRelocator(self.config.prefix, tempdir, True)
relocator.change_id(rewritten)
tar.add(rewritten, arcname)
shutil.rmtree(tempdir)
else:
tar.add(filepath, arcname)
else:
tar.add(filepath, arcname)
tar.close()
def testMergedLibraryPaths(self):
def check_prefix(path):
if self.tmp not in path:
return
self.assertTrue(uni_prefix in path)
self.assertTrue(x86_prefix not in path)
self.assertTrue(x86_64_prefix not in path)
self._compile(Architecture.X86)
self._compile(Architecture.X86_64)
self._check_compiled_files()
uni_prefix = os.path.join(self.tmp, Architecture.UNIVERSAL)
x86_prefix = os.path.join(self.tmp, Architecture.X86)
x86_64_prefix = os.path.join(self.tmp, Architecture.X86_64)
gen = OSXUniversalGenerator(uni_prefix)
gen.merge_dirs([x86_prefix, x86_64_prefix])
libfoo = os.path.join(self.tmp, Architecture.UNIVERSAL, 'lib', 'libfoo.so')
libname = OSXRelocator.library_id_name(libfoo)
check_prefix(libname)
for p in OSXRelocator.list_shared_libraries(libfoo):
check_prefix(p)
async def create_universal_file(self, output, inputlist, dirs):
tmp_inputs = []
# relocate all files with the prefix of the merged file.
# which must be done before merging them.
for f in inputlist:
# keep the filename in the suffix to preserve the filename extension
tmp = tempfile.NamedTemporaryFile(suffix=os.path.basename(f))
tmp_inputs.append(tmp)
shutil.copy(f, tmp.name)
prefix_to_replace = [d for d in dirs if d in f][0]
relocator = OSXRelocator(self.output_root,
prefix_to_replace,
False,
logfile=self.logfile)
# since we are using a temporary file, we must force the library id
# name to real one and not based on the filename
relocator.relocate_file(tmp.name)
relocator.change_id(tmp.name,
id=f.replace(prefix_to_replace,
self.output_root))
cmd = [self.LIPO_CMD, '-create'] + [f.name for f in tmp_inputs
] + ['-output', output]
shell.new_call(cmd)
for tmp in tmp_inputs:
tmp.close()
def testMergedLibraryPaths(self):
def check_prefix(path):
if self.tmp not in path:
return
self.assertTrue(uni_prefix in path)
self.assertTrue(x86_prefix not in path)
self.assertTrue(x86_64_prefix not in path)
self._compile(Architecture.X86)
self._compile(Architecture.X86_64)
self._check_compiled_files()
uni_prefix = os.path.join(self.tmp, Architecture.UNIVERSAL)
x86_prefix = os.path.join(self.tmp, Architecture.X86)
x86_64_prefix = os.path.join(self.tmp, Architecture.X86_64)
gen = OSXUniversalGenerator(uni_prefix)
gen.merge_dirs([x86_prefix, x86_64_prefix])
libfoo = os.path.join(self.tmp, Architecture.UNIVERSAL, 'lib',
'libfoo.so')
libname = OSXRelocator.library_id_name(libfoo)
check_prefix(libname)
for p in OSXRelocator.list_shared_libraries(libfoo):
check_prefix(p)
def extractAndRelocate(self, extract_to_path):
self.relocator = OSXRelocator(extract_to_path, extract_to_path, True)
RelocatableTar.extractAndRelocate(self, extract_to_path)
def _relocate_binaries(self, tmp_dir):
if not self.package.relocate_osx_binaries:
return
relocator = OSXRelocator(tmp_dir, self.config.prefix, True)
for path in ['bin', 'lib', 'libexec']:
relocator.relocate_dir(path)
class Fridge(object):
'''
This fridge unfreezes or freezes a cook from a recipe
'''
# Freeze/Unfreeze steps
FETCH_BINARY = (N_('Fetch Binary'), 'fetch_binary')
EXTRACT_BINARY = (N_('Extract Binary'), 'extract_binary')
GEN_BINARY = (N_('Generate Binary'), 'generate_binary')
UPLOAD_BINARY = (N_('Upload Binary'), 'upload_binary')
def __init__(self, store, force=False, dry_run=False):
self.store = store
self.cookbook = store.cookbook
self.config = self.cookbook.get_config()
self.force = force
shell.DRY_RUN = dry_run
if not self.config.binaries:
raise FatalError(_('Configuration without binaries path'))
self.binaries = os.path.join(self.config.binaries,
self.config.get_md5())
if not self.config.binary_repo:
raise FatalError(_('Configuration without binary repo'))
self.binary_repo = os.path.join(self.config.binary_repo,
self.config.get_md5())
m.message('Using config MD5: %s' % self.config.get_md5())
if not os.path.exists(self.binaries):
os.makedirs(self.binaries)
if self.config.target_platform == Platform.DARWIN:
self.relocator = OSXRelocator(self.config.prefix,
self.config.prefix, True)
def unfreeze_recipe(self, recipe_name, count, total):
recipe = self.cookbook.get_recipe(recipe_name)
if not recipe.allow_package_creation:
raise RecipeNotFreezableError(recipe_name)
steps = [self.FETCH_BINARY, self.EXTRACT_BINARY]
self._apply_steps(recipe, steps, count, total)
def freeze_recipe(self, recipe_name, count, total):
recipe = self.cookbook.get_recipe(recipe_name)
if not recipe.allow_package_creation:
raise RecipeNotFreezableError(recipe_name)
steps = [self.GEN_BINARY, self.UPLOAD_BINARY]
self._apply_steps(recipe, steps, count, total)
def fetch_binary(self, recipe):
packages_names = self._get_packages_names(recipe)
# TODO we need to fetch the ${filename}.md5 file first
# compare the md5 with the current md5 and then download
# or not
for filename in packages_names.itervalues():
if filename:
download_curl(os.path.join(self.binary_repo, filename),
os.path.join(self.binaries, filename),
user=self.config.binary_repo_username,
password=self.config.binary_repo_password,
overwrite=True)
def extract_binary(self, recipe):
packages_names = self._get_packages_names(recipe)
# There is a weird bug where the links in the devel package are overwriting the
# file it's linking instaed of just creating the link.
# For example libmonosgen-2.0.dylib will be extracted creating # a link
# libmonosgen-2.0.dylib -> libmonosgen-2.0.1.dylib and copying
# libmonosgen-2.0.dylib to libmonosgen-2.0.1.dylib
# As a workaround we extract first the devel package and finally the runtime
for filename in [
packages_names[PackageType.DEVEL],
packages_names[PackageType.RUNTIME]
]:
if filename:
tar = tarfile.open(os.path.join(self.binaries, filename),
'r:bz2')
tar.extractall(self.config.prefix)
for member in tar.getmembers():
# Simple sed for .la and .pc files
if os.path.splitext(member.name)[1] in [
'.la', '.pc'
] or ('bin' in os.path.splitext(member.name)[0]
and is_text_file(
os.path.join(self.config.prefix, member.name))):
shell.replace(
os.path.join(self.config.prefix, member.name),
{"CERBERO_PREFIX": self.config.prefix})
if os.path.splitext(member.name)[1] in [
'.dylib'
] and self.config.target_platform == Platform.DARWIN:
extracted_object = os.path.join(
self.config.prefix, member.name)
# When extracting, we change the install_name of the library to match the path
if not os.path.islink(extracted_object):
self.relocator.change_id(extracted_object,
extracted_object)
tar.close()
def generate_binary(self, recipe):
p = self.store.get_package('%s-pkg' % recipe.name)
tar = DistArchive(self.config, p, self.store, ArchiveType.TARBALL)
p.pre_package()
paths = tar.pack(self.binaries,
devel=True,
force=True,
force_empty=True,
relocatable=True)
p.post_package(paths)
def upload_binary(self, recipe):
packages_names = self._get_packages_names(recipe)
# TODO we need to upload the .md5 files too for a smart cache system
for filename in packages_names.itervalues():
if filename:
upload_curl(os.path.join(self.binaries, filename),
os.path.join(self.binary_repo, filename),
user=self.config.binary_repo_username,
password=self.config.binary_repo_password)
def _get_packages_names(self, recipe):
ret = {PackageType.RUNTIME: None, PackageType.DEVEL: None}
p = self.store.get_package('%s-pkg' % recipe.name)
tar = DistArchive(self.config, p, self.store, ArchiveType.TARBALL)
# use the package (not the packager) to avoid the warnings
ret[PackageType.RUNTIME] = tar.get_name(PackageType.RUNTIME)
ret[PackageType.DEVEL] = tar.get_name(PackageType.DEVEL)
return ret
def _apply_steps(self, recipe, steps, count, total):
for desc, step in steps:
m.build_step(count, total, recipe.name, step)
# check if the current step needs to be done
if self.cookbook.step_done(recipe.name, step) and not self.force:
m.action(_("Step done"))
continue
# call step function
stepfunc = getattr(self, step)
if not stepfunc:
raise FatalError(_('Step %s not found') % step)
try:
stepfunc(recipe)
# update status successfully
self.cookbook.update_step_status(recipe.name, step)
except Exception as e:
m.warning(str(e))
raise BuildStepError(recipe, step, traceback.format_exc())
# Update the recipe status
p = self.store.get_package('%s-pkg' % recipe.name)
v = p.version.rsplit('-')[0]
self.cookbook.update_build_status(recipe.name, v)
def _relocate_binaries(self, tmp_dir):
if not self.package.relocate_osx_binaries:
return
relocator = OSXRelocator(tmp_dir, self.config.prefix, True)
for path in ['bin', 'lib', 'libexec']:
relocator.relocate_dir(path)
