def _load_main_config(self): if os.path.exists(DEFAULT_CONFIG_FILE): self._parse(DEFAULT_CONFIG_FILE) else: msg = _('Using default configuration because %s is missing') % \ DEFAULT_CONFIG_FILE m.warning(msg)
def _search_libraries(self, files): ''' Search libraries in the prefix. Unfortunately the filename might vary depending on the platform and we need to match the library name and it's extension. There is a corner case on windows where a libray might be named foo.dll, foo-1.dll, libfoo.dll, or libfoo-1.dll NOTE: Unlike other searchfuncs which return lists, this returns a dict with a mapping from the libname to the actual on-disk file. We use the libname (the key) in gen_library_file so we don't have to guess (sometimes incorrectly) based on the dll filename. ''' libdir = self.extensions['sdir'] libext = self.extensions['srext'] libregex = self.extensions['sregex'] libsmatch = {} notfound = [] for f in files: libsmatch[f] = list( find_shlib_regex(f[3:], self.config.prefix, libdir, libext, libregex)) if not libsmatch[f]: notfound.append(f) if notfound: msg = "Some libraries weren't found while searching!" for each in notfound: msg += '\n' + each m.warning(msg) return libsmatch
def pack(self, output_dir, devel=True, force=False, keep_temp=False, split=True, package_prefix=''): try: dist_files = self.files_list(PackageType.RUNTIME, force) except EmptyPackageError: m.warning(_("The runtime package is empty")) dist_files = [] if devel: try: devel_files = self.files_list(PackageType.DEVEL, force) except EmptyPackageError: m.warning(_("The development package is empty")) devel_files = [] else: devel_files = [] if not split: dist_files += devel_files if not dist_files and not devel_files: raise EmptyPackageError(self.package.name) filenames = [] if dist_files: runtime = self._create_tarball(output_dir, PackageType.RUNTIME, dist_files, force, package_prefix) filenames.append(runtime) if split and devel and len(devel_files) != 0: devel = self._create_tarball(output_dir, PackageType.DEVEL, devel_files, force, package_prefix) filenames.append(devel) return filenames
def pack(self, output_dir, devel=True, force=False, keep_temp=False, split=True, package_prefix=""): try: dist_files = self.files_list(PackageType.RUNTIME, force) except EmptyPackageError: m.warning(_("The runtime package is empty")) dist_files = [] if devel: try: devel_files = self.files_list(PackageType.DEVEL, force) except EmptyPackageError: m.warning(_("The development package is empty")) devel_files = [] else: devel_files = [] if not split: dist_files += devel_files if not dist_files and not devel_files: raise EmptyPackageError(self.package.name) filenames = [] if dist_files: runtime = self._create_tarball(output_dir, PackageType.RUNTIME, dist_files, force, package_prefix) filenames.append(runtime) if split and devel and len(devel_files) != 0: devel = self._create_tarball(output_dir, PackageType.DEVEL, devel_files, force, package_prefix) filenames.append(devel) return filenames
def run(self, config, args): cookbook = CookBook(config) failed = [] p_name = args.package[0] store = PackagesStore(config) p = store.get_package(p_name) ordered_recipes = map(lambda x: cookbook.get_recipe(x), p.recipes_dependencies()) for recipe in ordered_recipes: if cookbook.recipe_needs_build(recipe.name): raise CommandError( _("Recipe %s is not built yet" % recipe.name)) for recipe in ordered_recipes: # call step function stepfunc = None try: stepfunc = getattr(recipe, 'check') except: m.message('%s has no check step, skipped' % recipe.name) if stepfunc: try: m.message('Running checks for recipe %s' % recipe.name) stepfunc() except Exception as ex: failed.append(recipe.name) m.warning(_("%s checks failed: %s") % (recipe.name, ex)) if failed: raise CommandError( _("Error running %s checks on:\n " + "\n ".join(failed)) % p_name)
def __init__(self, variants): # Set default values for v in self.__enabled_variants: setattr(self, v, True) for v in self.__disabled_variants: setattr(self, v, False) for v, choices in self.__mapping_variants.items(): setattr(self, v, choices[0]) # Set the configured values for v in variants: if '=' in v: key, value = v.split('=', 1) key = key.replace('-', '_') if key not in self.__mapping_variants: raise AttributeError( 'Mapping variant {!r} is unknown'.format(key)) if value not in self.__mapping_variants[key]: raise AttributeError( 'Mapping variant {!r} value {!r} is unknown'.format( key, value)) setattr(self, key, value) elif v.startswith('no'): if v[2:] not in self.__bool_variants: m.warning('Variant {!r} is unknown or obsolete'.format( v[2:])) setattr(self, v[2:], False) else: if v not in self.__bool_variants: m.warning('Variant {!r} is unknown or obsolete'.format(v)) setattr(self, v, True) # Set auto mapping values based on other values if self.vscrt == 'auto': self.vscrt = 'md' if self.debug and not self.optimization: self.vscrt = 'mdd'
async def fetch_dep(self, config, dep, namespace): try: dep_path = os.path.join(config.home_dir, os.path.basename(dep['url'])) await shell.download(dep['url'], dep_path, check_cert=True, overwrite=True) if dep['checksum'] == self.checksum(dep_path): await shell.unpack(dep_path, config.home_dir) else: m.warning("Corrupted dependency file, ignoring.") os.remove(dep_path) # Don't need to relocate on Windows and macOS since we build # pkg-config with --enable-define-prefix. # In case this needs to be re-enabled at some point, note that the # current self.build_dir value is hard-coded and is wrong on macOS # and Windows. It should instead be derived from CI env vars. if config.platform == Platform.LINUX: origin = self.build_dir % namespace m.message("Relocating from %s to %s" % (origin, config.home_dir)) # FIXME: Just a quick hack for now shell.call(("grep -lnrIU %(origin)s | xargs " "sed \"s#%(origin)s#%(dest)s#g\" -i") % { 'origin': origin, 'dest': config.home_dir }, config.home_dir) except FatalError as e: m.warning("Could not retrieve dependencies for commit %s: %s" % (dep['commit'], e.msg))
def run(self, config, args): cookbook = CookBook(config) failed = [] p_name = args.package[0] store = PackagesStore(config) p = store.get_package(p_name) ordered_recipes = map(lambda x: cookbook.get_recipe(x), p.recipes_dependencies()) for recipe in ordered_recipes: if cookbook.recipe_needs_build(recipe.name): raise CommandError(_("Recipe %s is not built yet" % recipe.name)) for recipe in ordered_recipes: # call step function stepfunc = None try: stepfunc = getattr(recipe, 'check') except: m.message('%s has no check step, skipped' % recipe.name) if stepfunc: try: m.message('Running checks for recipe %s' % recipe.name) stepfunc() except Exception, ex: failed.append(recipe.name) m.warning(_("%s checks failed: %s") % (recipe.name, ex))
def create(self, libname, dllpath, arch, outputdir): bindir, dllname = os.path.split(dllpath) # Create the .def file shell.call('gendef %s' % dllpath, outputdir) defname = dllname.replace('.dll', '.def') implib = '%s.lib' % libname[3:] # Create the import library vc_path = self._get_vc_tools_path() # Prefer LIB.exe over dlltool: # http://sourceware.org/bugzilla/show_bug.cgi?id=12633 if vc_path is not None: # Spaces msys and shell are a beautiful combination lib_path = to_unixpath(os.path.join(vc_path, 'lib.exe')) lib_path = lib_path.replace('\\', '/') lib_path = lib_path.replace('(', '\\\(').replace(')', '\\\)') lib_path = lib_path.replace(' ', '\\\\ ') if arch == Architecture.X86: arch = 'x86' else: arch = 'x64' shell.call(self.LIB_TPL % (lib_path, defname, implib, arch), outputdir) else: m.warning("Using dlltool instead of lib.exe! Resulting .lib files" " will have problems with Visual Studio, see " " http://sourceware.org/bugzilla/show_bug.cgi?id=12633") shell.call(self.DLLTOOL_TPL % (defname, implib, dllname), outputdir) return os.path.join(outputdir, implib)
def __new__(klass, config, build_tools_only): bs = [] bs.append(BuildTools(config)) if build_tools_only: return bs target_distro = config.target_distro distro = config.distro target_distro_version = config.target_distro_version distro_version = config.distro_version # Try to find a bootstrapper for the distro-distro_version combination, # both for the target host and the build one. For instance, when # bootstraping to cross-compile for windows we also need to bootstrap # the build host. target = (target_distro, target_distro_version) build = (distro, distro_version) if target == build: blist = [target] else: blist = [target, build] for d, v in blist: if d not in bootstrappers: raise FatalError(_("No bootstrapper for the distro %s" % d)) if v not in bootstrappers[d]: # Be tolerant with the distro version m.warning(_("No bootstrapper for the distro version %s" % v)) v = None bs.insert(0, bootstrappers[d][v](config)) return bs
def _make_paths_relative(self): sofiles = shell.find_files('*.so', self.tmp_install_dir) for sof in sofiles: try: shell.call("chrpath -d %s" % sof, self.tmp_install_dir, fail=False) except FatalError: m.warning("Could not 'chrpath' %s" % sof) shell.call("ln -s . usr", self.tmp_install_dir, fail=False) # Make gd-pixbuf loader.cache file use relative paths cache = os.path.join(self.tmp_install_dir, 'lib', 'gdk-pixbuf-2.0', '2.10.0', 'loaders.cache') shell.replace(cache, {self.config.install_dir: '.'}) for icondir in os.listdir(os.path.join(self.tmp_install_dir, "share/icons/")): if os.path.exists(os.path.join(icondir, "index.theme")): shell.call("gtk-update-icon-cache %s" % icondir, fail=False) shell.call("update-mime-database %s" % os.path.join(self.tmp_install_dir, "share", "mime"), fail=False) # Use system wide applications in case the bundle needs to open apps not included in # the bundle (to show the documentation most probably) shell.call("rm -rf %s" % os.path.join(self.tmp_install_dir, "share", "applications"), fail=False) shell.call("ln -s %s %s" % (os.path.join("/usr", "share", "applications"), os.path.join(self.tmp_install_dir, "share", "applications")), fail=False)
def find_dll_implib(config, libname, prefix, libdir, ext, regex): implibdir = 'lib' implibs = [ 'lib{}.dll.a'.format(libname), libname + '.lib', 'lib{}.lib'.format(libname) ] implib_notfound = [] for implib in implibs: path = os.path.join(prefix, implibdir, implib) if not os.path.exists(path): implib_notfound.append(implib) continue dllname = get_implib_dllname(config, path) if dllname == 0: continue dllname = dllname.strip() if dllname == '': continue return [os.path.join(libdir, dllname)] # If import libraries aren't found, look for a DLL by exactly the specified # name. This is to cover cases like libgcc_s_sjlj-1.dll which don't have an # import library since they're only used at runtime. dllname = 'lib{}.dll'.format(libname) path = os.path.join(prefix, libdir, dllname) if os.path.exists(path): return [os.path.join(libdir, dllname)] if len(implib_notfound) == len(implibs): m.warning("No import libraries found for {!r}".format(libname)) else: implibs = ', '.join(set(implibs) - set(implib_notfound)) m.warning("No dllname found from implibs: {}".format(implibs)) # This will trigger an error in _search_libraries() return []
def insert_python_site(self): try: import setuptools.version as stv except ImportError: return version = stv.__version__.split('.', 1) if len(version) < 1 or int(version[0]) < 49: return m.warning('detected setuptools >= 49.0.0, installing fallback site.py file. ' 'See https://github.com/pypa/setuptools/issues/2295') # Since python-setuptools 49.0.0, site.py is not installed by # easy_install/setup.py anymore which breaks python installs outside # the system prefix. # https://github.com/pypa/setuptools/issues/2295 # # Install the previously installed site.py ourselves as a workaround config = self.cookbook.get_config() py_prefix = sysconfig.get_path('purelib', vars={'base': ''}) # Must strip \/ to ensure that the path is relative py_prefix = PurePath(config.prefix) / PurePath(py_prefix.strip('\\/')) src_file = os.path.join(os.path.dirname(__file__), 'site-patch.py') shutil.copy(src_file, py_prefix / 'site.py')
def _create_bundle(self, files, package_type): ''' Moves all the files that are going to be packaged to a temporary directory to create the bundle ''' tmp = tempfile.mkdtemp() root = os.path.join(tmp, 'Root') resources = os.path.join(tmp, 'Resources') for f in files: in_path = os.path.join(self.config.prefix, f) if not os.path.exists(in_path): m.warning("File %s is missing and won't be added to the " "package" % in_path) continue out_path = os.path.join(root, f) out_dir = os.path.split(out_path)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) shutil.copy(in_path, out_path) if package_type == PackageType.DEVEL: self._create_framework_headers(self.config.prefix, self.include_dirs, root) # Copy scripts to the Resources directory os.makedirs(resources) if os.path.exists(self.package.resources_preinstall): shutil.copy(os.path.join(self.package.resources_preinstall), os.path.join(resources, 'preinstall')) if os.path.exists(self.package.resources_postinstall): shutil.copy(os.path.join(self.package.resources_postinstall), os.path.join(resources, 'postinstall')) return root, resources
def create(self, libname, dllpath, platform, target_arch, outputdir): # foo.lib must not start with 'lib' if libname.startswith('lib'): self.filename = libname[3:] + '.lib' else: self.filename = libname + '.lib' bindir, dllname = os.path.split(dllpath) # Create the .def file defname = self.gendef(dllpath, outputdir, libname) # Create the import library lib_path, paths = self._get_lib_exe_path(target_arch, platform) # Prefer LIB.exe over dlltool: # http://sourceware.org/bugzilla/show_bug.cgi?id=12633 if lib_path is not None: if target_arch == Architecture.X86: arch = 'x86' else: arch = 'x64' env = self.config.env.copy() env['PATH'] = paths + ';' + env['PATH'] cmd = [ lib_path, '/DEF:' + defname, '/OUT:' + self.filename, '/MACHINE:' + arch ] shell.new_call(cmd, outputdir, logfile=self.logfile, env=env) else: m.warning("Using dlltool instead of lib.exe! Resulting .lib files" " will have problems with Visual Studio, see " " http://sourceware.org/bugzilla/show_bug.cgi?id=12633") self.dlltool(defname, dllname, outputdir) return os.path.join(outputdir, self.filename)
def run(self, config, args): cookbook = CookBook(config) if args.recipe == 'all': recipes = cookbook.get_recipes_list() else: recipes = [cookbook.get_recipe(args.recipe)] if len(recipes) == 0: m.message(_("No recipes found")) tagname = args.tagname tagdescription = args.tagdescription force = args.force for recipe in recipes: if recipe.stype != SourceType.GIT and \ recipe.stype != SourceType.GIT_TARBALL: m.message( _("Recipe '%s' has a custom source repository, " "skipping") % recipe.name) continue recipe.fetch(checkout=False) tags = git.list_tags(recipe.repo_dir) exists = (tagname in tags) if exists: if not force: m.warning( _("Recipe '%s' tag '%s' already exists, " "not updating" % (recipe.name, tagname))) continue git.delete_tag(recipe.repo_dir, tagname) commit = 'origin/sdk-%s' % recipe.version git.create_tag(recipe.repo_dir, tagname, tagdescription, commit)
def _create_bundle(self, files, package_type): ''' Moves all the files that are going to be packaged to a temporary directory to create the bundle ''' tmp = tempfile.mkdtemp() root = os.path.join(tmp, 'Root') resources = os.path.join(tmp, 'Resources') for f in files: in_path = os.path.join(self.config.prefix, f) if not os.path.exists(in_path): m.warning("File %s is missing and won't be added to the " "package" % in_path) continue out_path = os.path.join(root, f) out_dir = os.path.split(out_path)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) shutil.copy(in_path, out_path) if package_type == PackageType.DEVEL: self._create_framework_headers(self.config.prefix, self.include_dirs, root) # Copy scripts to the Resources directory os.makedirs(resources) if os.path.exists(self.package.resources_preinstall): shutil.copy(os.path.join(self.package.resources_preinstall), os.path.join(resources, 'preinstall')) if os.path.exists(self.package.resources_postinstall): shutil.copy(os.path.join(self.package.resources_postinstall), os.path.join(resources, 'postinstall')) return tmp, root, resources
def generate_gst_la_files(self): ''' Generate .la files for all libraries and plugins packaged by this Meson recipe using the pkg-config files installed by our Meson build files. ''' pluginpcdir = os.path.join(self.config.libdir, 'gstreamer-1.0', 'pkgconfig') env = os.environ.copy() env['PKG_CONFIG_LIBDIR'] += os.pathsep + pluginpcdir if self.use_system_libs: add_system_libs(self.config, env) # Get la file -> pkg-config name mapping libs_la_files = {} plugin_la_files = {} for f in self.devel_files_list(): if not f.endswith('.a') or not f.startswith('lib/'): continue if f.startswith('lib/gstreamer-1.0/'): libtype = 'plugin' else: libtype = 'library' fpath = os.path.join(self._get_arch_prefix(), f) if not os.path.isfile(fpath): arch = self.config.target_arch m.warning('{} {} {!r} not found'.format(arch, libtype, fpath)) continue pcname = os.path.basename(f)[3:-2] la = os.path.splitext(f)[0] + '.la' if libtype == 'plugin': self._write_gst_la_file(la, pcname, None, None, None, env) else: pcname = pcname.replace('gst', 'gstreamer-') # Same versioning as gstreamer minor, micro = (map(int, self.version.split('.')[1:3])) minor = minor * 100 + micro self._write_gst_la_file(la, pcname, 0, minor, 0, env)
def _load_packages_from_file(self, filepath, custom=None): packages = [] d = { 'Platform': Platform, 'Architecture': Architecture, 'Distro': Distro, 'DistroVersion': DistroVersion, 'License': License, 'package': package, 'PackageType': PackageType, 'custom': custom } d_keys = set(list(d.keys())) try: new_d = d.copy() parse_file(filepath, new_d) # List new objects parsed added to the globals dict diff_vals = [new_d[x] for x in set(new_d.keys()) - d_keys] # Find all objects inheriting from Package for package_cls in [ x for x in diff_vals if self._is_package_class(x) ]: pkg = self._load_package_from_file(package_cls, filepath, custom) if pkg is not None: packages.append(pkg) except Exception: m.warning("Error loading package from file %s" % filepath) traceback.print_exc() return packages
def _load_release(self): rls = os.path.join(self.args.repo, 'bundle.yaml') if os.path.exists(self.args.repo) and not os.path.exists(rls): pkgs = Package(self.config).packages() pkgs['build-tools'] = BuildTools(self.config).get() for name, profile in pkgs.viewitems(): filename = '%(name)s-%(platform)s-%(arch)s-%(version)s.yaml' % profile path = os.path.join(self.args.repo, filename) if os.path.exists(path): pro = yaml.load(open(path)) pro['__file__'] = path if pro['version'] != profile['version']: m.warning('skip %s since %s != %s' % (name, pro['version'], profile['version'])) else: self.profile[profile['name']] = pro else: self.profile[profile['name']] = profile else: url = os.path.join(self.args.repo, 'bundle.yaml') path = cache(url, self.args.cache_dir) self._cache_bundle(path)
def _load_package_from_file(self, filepath): mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) try: d = {'Platform': Platform, 'Architecture': Architecture, 'Distro': Distro, 'DistroVersion': DistroVersion, 'License': License, 'package': package, 'PackageType': PackageType} execfile(filepath, d) if 'Package' in d: p = d['Package'](self._config, self, self.cookbook) elif 'SDKPackage' in d: p = d['SDKPackage'](self._config, self) elif 'InstallerPackage' in d: p = d['InstallerPackage'](self._config, self) elif 'App' in d: p = d['App'](self._config, self, self.cookbook) else: raise Exception('Package, SDKPackage, InstallerPackage or App ' 'class not found') p.prepare() # reload files from package now that we called prepare that # may have changed it p.load_files() return p except Exception, ex: import traceback traceback.print_exc() m.warning("Error loading package %s" % ex)
def _search_libraries(self, files): """ Search libraries in the prefix. Unfortunately the filename might vary depending on the platform and we need to match the library name and it's extension. There is a corner case on windows where a libray might be named foo.dll, foo-1.dll, libfoo.dll, or libfoo-1.dll NOTE: Unlike other searchfuncs which return lists, this returns a dict with a mapping from the libname to the actual on-disk file. We use the libname (the key) in gen_library_file so we don't have to guess (sometimes incorrectly) based on the dll filename. """ libdir = self.extensions["sdir"] libext = self.extensions["srext"] libregex = self.extensions["sregex"] libsmatch = {} notfound = [] for f in files: libsmatch[f] = list(find_shlib_regex(f[3:], self.config.prefix, libdir, libext, libregex)) if not libsmatch[f]: notfound.append(f) if notfound: msg = "Some libraries weren't found while searching!" for each in notfound: msg += "\n" + each m.warning(msg) return libsmatch
def _restore_cache(self): try: with open(self._cache_file(self.get_config()), 'rb') as f: self.status = pickle.load(f) except Exception: self.status = {} m.warning(_("Could not recover status"))
def override(self, variants): if not isinstance(variants, list): variants = [variants] # Set the configured values for v in variants: if '=' in v: key, value = v.split('=', 1) key = key.replace('-', '_') if key not in self.__mapping_variants: raise AttributeError( 'Mapping variant {!r} is unknown'.format(key)) if value not in self.__mapping_variants[key]: raise AttributeError( 'Mapping variant {!r} value {!r} is unknown'.format( key, value)) setattr(self, key, value) elif v.startswith('no'): if v[2:] not in self.__bool_variants: m.warning('Variant {!r} is unknown or obsolete'.format( v[2:])) setattr(self, v[2:], False) else: if v not in self.__bool_variants: m.warning('Variant {!r} is unknown or obsolete'.format(v)) setattr(self, v, True) # Auto-set vscrt variant if it wasn't set explicitly if self.vscrt == 'auto': self.vscrt = 'md' if self.debug and not self.optimization: self.vscrt = 'mdd'
def _make_paths_relative(self): sofiles = shell.find_files('*.so', self.tmp_install_dir) for sof in sofiles: try: shell.call("chrpath -d %s" % sof, self.tmp_install_dir, fail=False) except FatalError: m.warning("Could not 'chrpath' %s" % sof) shell.call("ln -s . usr", self.tmp_install_dir, fail=False) for icondir in os.listdir( os.path.join(self.tmp_install_dir, "share/icons/")): if os.path.exists(os.path.join(icondir, "index.theme")): shell.call("gtk-update-icon-cache %s" % icondir, fail=False) shell.call("update-mime-database %s" % os.path.join(self.tmp_install_dir, "share", "mime"), fail=False) shell.call("glib-compile-schemas %s/share/glib-2.0/schemas" % self.tmp_install_dir) # Use system wide applications in case the bundle needs to open apps not included in # the bundle (to show the documentation most probably) shell.call("rm -rf %s" % os.path.join(self.tmp_install_dir, "share", "applications"), fail=False) shell.call( "ln -s %s %s" % (os.path.join("/usr", "share", "applications"), os.path.join(self.tmp_install_dir, "share", "applications")), fail=False)
def _load_recipes_from_file(self, filepath, custom=None): recipes = [] d = {'Platform': Platform, 'Architecture': Architecture, 'BuildType': BuildType, 'SourceType': SourceType, 'Distro': Distro, 'DistroVersion': DistroVersion, 'License': License, 'recipe': crecipe, 'os': os, 'BuildSteps': crecipe.BuildSteps, 'InvalidRecipeError': InvalidRecipeError, 'FatalError': FatalError, 'custom': custom, '_': _, 'shell': shell} d_keys = set(list(d.keys())) try: new_d = d.copy () parse_file(filepath, new_d) # List new objects parsed added to the globals dict diff_keys = [x for x in set(new_d.keys()) - d_keys] # Find all objects inheriting from Recipe for recipe_cls_key in [x for x in diff_keys if self._is_recipe_class(new_d[x])]: if self._config.target_arch != Architecture.UNIVERSAL: recipe = self._load_recipe_from_class( new_d[recipe_cls_key], self._config, filepath, custom) else: recipe = self._load_universal_recipe(d, new_d[recipe_cls_key], recipe_cls_key, filepath) if recipe is not None: recipes.append(recipe) except Exception: m.warning("Error loading recipe in file %s" % (filepath)) print(traceback.format_exc()) return recipes
def run(self, config, args): cookbook = CookBook(config) if args.recipe == 'all': recipes = cookbook.get_recipes_list() else: recipes = [cookbook.get_recipe(args.recipe)] if len(recipes) == 0: m.message(_("No recipes found")) tagname = args.tagname tagdescription = args.tagdescription force = args.force for recipe in recipes: if recipe.stype != SourceType.GIT and \ recipe.stype != SourceType.GIT_TARBALL: m.message(_("Recipe '%s' has a custom source repository, " "skipping") % recipe.name) continue recipe.fetch(checkout=False) tags = git.list_tags(recipe.repo_dir) exists = (tagname in tags) if exists: if not force: m.warning(_("Recipe '%s' tag '%s' already exists, " "not updating" % (recipe.name, tagname))) continue git.delete_tag(recipe.repo_dir, tagname) commit = 'origin/sdk-%s' % recipe.version git.create_tag(recipe.repo_dir, tagname, tagdescription, commit)
def _load_recipes_from_dir(self, repo, skip_errors): recipes = {} recipes_files = shell.find_files('*%s' % self.RECIPE_EXT, repo) recipes_files.extend(shell.find_files('*/*%s' % self.RECIPE_EXT, repo)) custom = None # If a manifest is being used or if recipes_commits is defined, disable # usage of tarballs when tagged for release. We need to do this before # `custom.py` is loaded, so we can't set it on the module afterwards. # We need to set it on the parent class. if self._config.manifest or self._config.recipes_commits: crecipe.Recipe._using_manifest_force_git = True m_path = os.path.join(repo, 'custom.py') if os.path.exists(m_path): custom = imp.load_source('custom', m_path) for f in recipes_files: # Try to load recipes with the custom.py module located in the # recipes dir which can contain private classes and methods with # common code for gstreamer recipes. try: recipes_from_file = self._load_recipes_from_file( f, skip_errors, custom) except RecipeNotFoundError: m.warning(_("Could not found a valid recipe in %s") % f) if recipes_from_file is None: continue for recipe in recipes_from_file: recipes[recipe.name] = recipe return recipes
def _get_vswhere_vs_install(vswhere, vs_versions): import json vswhere_exe = str(vswhere) # Get a list of installation paths for all installed Visual Studio # instances, from VS 2013 to the latest one, sorted from newest to # oldest, and including preview releases. # Will not include BuildTools installations. out = check_output([vswhere_exe, '-legacy', '-prerelease', '-format', 'json', '-utf8']) installs = _sort_vs_installs(json.loads(out)) program_files = get_program_files_dir() for install in installs: version = install['installationVersion'] vs_version = 'vs' + version.split('.', maxsplit=1)[0] if vs_version not in vs_versions: continue prefix = install['installationPath'] suffix = VCVARSALLS[vs_version][1] path = program_files / prefix / suffix # Find the location of the Visual Studio installation if path.is_file(): return path.as_posix(), vs_version m.warning('vswhere.exe could not find Visual Studio version(s) {}. Falling ' 'back to manual searching...' .format(', '.join(vs_versions))) return None
def find_dep(self, deps, sha): for dep in deps: if dep['commit'] == sha: return dep m.warning("Did not find cache for commit %s" % sha) return None
def run(self, config, args): self.store = PackagesStore(config, offline=args.offline) p = self.store.get_package(args.package[0]) if args.skip_deps_build and args.only_build_deps: raise UsageError( _("Cannot use --skip-deps-build together with " "--only-build-deps")) if not args.skip_deps_build: self._build_deps(config, p, args.no_devel, args.offline, args.dry_run, args.jobs) if args.only_build_deps or args.dry_run: return if args.compress_method != 'default': m.message('Forcing tarball compression method as ' + args.compress_method) config.package_tarball_compression = args.compress_method if p is None: raise PackageNotFoundError(args.package[0]) p.pre_package() packager_class = Packager if args.tarball: if config.target_platform == Platform.ANDROID and \ config.target_arch == Architecture.UNIVERSAL: packager_class = AndroidPackager else: packager_class = DistTarball elif config.variants.uwp: # Split devel/runtime packages are useless for UWP since we will # need both when building the package, and all needed runtime DLLs # are packaged with the app as assets. m.warning('Forcing single-tarball output for UWP package') args.no_split = True packager_class = DistTarball m.action(_("Creating package for %s") % p.name) pkg = packager_class(config, p, self.store) output_dir = os.path.abspath(args.output_dir) if isinstance(pkg, DistTarball): paths = pkg.pack(output_dir, args.no_devel, args.force, args.keep_temp, split=not args.no_split, strip_binaries=p.strip) else: paths = pkg.pack(output_dir, args.no_devel, args.force, args.keep_temp) if None in paths: paths.remove(None) paths = p.post_package(paths, output_dir) or paths m.action( _("Package successfully created in %s") % ' '.join([os.path.abspath(x) for x in paths]))
def install_xz(self): msys_xz = shutil.which('xz') if not msys_xz: m.warning('xz not found, are you not using an MSYS shell?') msys_bindir = os.path.dirname(msys_xz) src = os.path.join(self.xz_tmp_prefix.name, 'bin_x86-64') for b in ('xz.exe', 'xzdec.exe', 'lzmadec.exe', 'lzmainfo.exe'): shutil.copy2(os.path.join(src, b), os.path.join(msys_bindir, b))
def save(self): try: cache_file = self._cache_file(self.get_config()) if not os.path.exists(os.path.dirname(cache_file)): os.makedirs(os.path.dirname(cache_file)) with open(cache_file, 'wb') as f: pickle.dump(self.status, f) except IOError as ex: m.warning(_("Could not cache the CookBook: %s") % ex)
def _qmake_or_pkgdir(qmake): qmake_path = Path(qmake) if not qmake_path.is_file(): m.warning('QMAKE={!r} does not exist'.format(str(qmake_path))) return (None, None) pkgdir = (qmake_path.parent.parent / 'lib/pkgconfig') if pkgdir.is_dir(): return (pkgdir.as_posix(), qmake_path.as_posix()) return (None, qmake_path.as_posix())
def strip_dir(self, dir_path): if not self.strip_cmd: m.warning('Strip command is not defined') return tasks = [] for dirpath, dirnames, filenames in os.walk(dir_path): for f in filenames: tasks.append(self._async_strip_file(os.path.join(dirpath, f))) run_until_complete(tasks)
def get_deps(self, config, args): url = self.make_url(config, args, self.log_filename) deps = [] try: deps = self.json_get(url) except FatalError as e: m.warning("Could not get cache list: %s" % e.msg) return deps
def __init__(self, args): if user_is_root(): m.warning(_("Running as root")) self.check_in_cerbero_shell() self.init_logging() self.create_parser() self.load_commands() self.parse_arguments(args) self.load_config() self.run_command()
def _load_packages_from_dir(self, repo): packages_dict = {} packages = shell.find_files('*%s' % self.PKG_EXT, repo) packages.extend(shell.find_files('*/*%s' % self.PKG_EXT, repo)) for f in packages: p = self._load_package_from_file(f) if p is None: m.warning(_("Could not found a valid package in %s") % f) continue packages_dict[p.name] = p return packages_dict
def load_commands(subparsers): import os commands_dir = os.path.abspath(os.path.dirname(__file__)) for name in os.listdir(commands_dir): name, extension = os.path.splitext(name) if extension != '.py': continue try: __import__('cerbero.commands.%s' % name) except ImportError, e: m.warning("Error importing command %s:\n %s" % (name, e))
def gen_library_file(self, output_dir=None): ''' Generates library files (.lib) for the dll's provided by this recipe ''' genlib = GenLib() for dllpath in self.libraries(): try: implib = genlib.create(os.path.join(self.config.prefix, dllpath), os.path.join(self.config.prefix, 'lib')) logging.debug('Created %s' % implib) except: m.warning("Could not create .lib, gendef might be missing")
def run(self, config, args): name = args.name[0] version = args.version[0] filename = os.path.join(config.recipes_dir, '%s.recipe' % name) if not args.force and os.path.exists(filename): m.warning(_("Recipe '%s' (%s) already exists, " "use -f to replace" % (name, filename))) return template_args = {} template = RECEIPT_TPL template_args['name'] = name template_args['version'] = version if args.licenses: licenses = args.licenses.split(',') self.validate_licenses(licenses) template += LICENSES_TPL template_args['licenses'] = ', '.join( ['License.' + self.supported_licenses[l] \ for l in licenses]) if args.commit: template += COMMIT_TPL template_args['commit'] = args.commit if args.origin: template += ORIGIN_TPL template_args['origin'] = args.origin if args.deps: template += DEPS_TPL deps = args.deps.split(',') cookbook = CookBook(config) for dname in deps: try: recipe = cookbook.get_recipe(dname) except RecipeNotFoundError as ex: raise UsageError(_("Error creating recipe: " "dependant recipe %s does not exist") % dname) template_args['deps'] = deps try: f = open(filename, 'w') f.write(template % template_args) f.close() m.action(_("Recipe '%s' successfully created in %s") % (name, filename)) except IOError as ex: raise FatalError(_("Error creating recipe: %s") % ex)
def __init__(self, args): if user_is_root(): m.warning(_("Running as root")) self.check_in_cerbero_shell() self.init_logging() # Create a argument parser with none config .cbc base module argparse self.create_parser() self.load_commands() self.parse_arguments(args) self.load_config() self.run_command()
def __new__(klass, config, package, store): d = config.target_distro v = config.target_distro_version if d not in _packagers: raise FatalError(_("No packager available for the distro %s" % d)) if v not in _packagers[d]: # Be tolerant with the distro version m.warning(_("No specific packager available for the distro " "version %s, using generic packager for distro %s" % (v, d))) v = None return _packagers[d][v](config, package, store)
def run(self, config, args): name = args.name[0] version = args.version[0] store = PackagesStore(config) filename = os.path.join(config.packages_dir, '%s.package' % name) if not args.force and os.path.exists(filename): m.warning(_("Package '%s' (%s) already exists, " "use -f to replace" % (name, filename))) return template_args = {} template = RECEIPT_TPL template_args['name'] = name template_args['version'] = version if args.short_desc: template_args['shortdesc'] = args.short_desc else: template_args['shortdesc'] = name if args.codename: template += CODENAME_TPL template_args['codename'] = args.codename if args.vendor: template += VENDOR_TPL template_args['vendor'] = args.vendor if args.url: template += URL_TPL template_args['url'] = args.url if args.license: self.validate_licenses([args.license]) template += LICENSE_TPL template_args['license'] = \ 'License.' + self.supported_licenses[args.license] deps = [] if args.deps: template += DEPS_TPL deps = args.deps.split(',') for dname in deps: try: package = store.get_package(dname) except Exception, ex: raise UsageError(_("Error creating package: " "dependant package %s does not exist") % dname) template_args['deps'] = deps
def _create_merge_modules(self, package_type): packagedeps = {} for package in self.packagedeps: package.set_mode(package_type) m.action("Creating Merge Module for %s" % package) packager = MergeModulePackager(self.config, package, self.store) try: path = packager.create_merge_module(self.output_dir, package_type, self.force, self.package.version, self.keep_temp) packagedeps[package] = path except EmptyPackageError: m.warning("Package %s is empty" % package) self.packagedeps = packagedeps self.merge_modules[package_type] = packagedeps.values()
def _copy_installdir(self): ''' Copy all the files that are going to be packaged to the bundle's temporary directory ''' os.makedirs(self.tmp_install_dir) for f in set(self.package.files_list()): in_path = os.path.join(self.config.prefix, f) if not os.path.exists(in_path): m.warning("File %s is missing and won't be added to the " "package" % in_path) continue out_path = os.path.join(self.tmp_install_dir, f) odir = os.path.split(out_path)[0] if not os.path.exists(odir): os.makedirs(odir) shutil.copy(in_path, out_path)
def _create_bundle(self): """ Moves all the files that are going to be packaged to the bundle's temporary directory """ out_dir = os.path.join(self.appdir, "Contents", "Home") os.makedirs(out_dir) for f in self.package.files_list(): in_path = os.path.join(self.config.prefix, f) if not os.path.exists(in_path): m.warning("File %s is missing and won't be added to the " "package" % in_path) continue out_path = os.path.join(out_dir, f) odir = os.path.split(out_path)[0] if not os.path.exists(odir): os.makedirs(odir) shutil.copy(in_path, out_path)
def copy_files(origdir, destdir, files, extensions, target_platform): for f in files: f = f % extensions install_dir = os.path.dirname(os.path.join(destdir, f)) if not os.path.exists(install_dir): os.makedirs(install_dir) if destdir[1] == ':': # windows path relprefix = to_unixpath(destdir)[2:] else: relprefix = destdir[1:] orig = os.path.join(origdir, relprefix, f) dest = os.path.join(destdir, f) m.action("copying %s to %s" % (orig, dest)) try: shutil.copy(orig, dest) except IOError: m.warning("Could not copy %s to %s" % (orig, dest))
def __new__(klass, config, package, store): d = config.target_distro v = config.target_distro_version if d not in _packagers: raise FatalError(_("No packager available for the distro %s" % d)) if v not in _packagers[d]: # Be tolerant with the distro version m.warning(_("No specific packager available for the distro " "version %s, using generic packager for distro %s" % (v, d))) v = None if (d == Distro.WINDOWS and config.platform == Platform.LINUX): m.warning("Cross-compiling for Windows, overriding Packager") d = Distro.NONE return _packagers[d][v](config, package, store)
def _create_merge_modules(self, package_type, wix_use_fragment): packagedeps = {} for package in self.packagedeps: package.set_mode(package_type) package.wix_use_fragment = wix_use_fragment m.action("Creating Merge Module for %s" % package) packager = MergeModulePackager(self.config, package, self.store) if self.wix_wine_prefix: packager.wix_wine_prefix = self.wix_wine_prefix try: path = packager.create_merge_module(self.output_dir, package_type, self.force, self.package.version, self.keep_temp) packagedeps[package] = path except EmptyPackageError: m.warning("Package %s is empty" % package) self.packager_tmpdirs.append(packager._tmpdir()) self.packagedeps = packagedeps self.merge_modules[package_type] = packagedeps.values()
def __init__(self): Source.__init__(self) self.remotes = {} if self.remotes is None else self.remotes.copy() if 'origin' in self.remotes: url = self.replace_name_and_version(self.remotes['origin']) o = urllib.parse.urlparse(url) if o.scheme in ('http', 'git'): raise FatalError('git remote origin URL {!r} must use HTTPS not {!r}' ''.format(url, o.scheme)) if o.scheme in ('file', 'ssh'): m.warning('git remote origin URL {!r} uses {!r}, please only use this ' 'for testing'.format(url, o.scheme)) self.remotes['origin'] = url else: # XXX: When is this used? self.remotes['origin'] = '%s/%s.git' % \ (self.config.git_root, self.name) self.repo_dir = os.path.join(self.config.local_sources, self.name) self._previous_env = None
def files_list(self, package_type, force): if package_type == PackageType.DEVEL: files = self.package.devel_files_list() else: files = self.package.files_list() real_files = [] for f in files: if os.path.exists(os.path.join(self.config.prefix, f)): real_files.append(f) diff = list(set(files) - set(real_files)) if len(diff) != 0: if force: m.warning(_("Some files required by this package are missing " "in the prefix:\n%s" % '\n'.join(diff))) else: raise MissingPackageFilesError(diff) if len(real_files) == 0: raise EmptyPackageError(self.package.name) return real_files
def _load_packages(self): self._packages = {} packages = defaultdict(dict) repos = self._config.get_packages_repos() for reponame, (repodir, priority) in repos.iteritems(): packages[int(priority)].update( self._load_packages_from_dir(repodir)) # Add packages by ascending pripority for key in sorted(packages.keys()): self._packages.update(packages[key]) # Add a package for every recipe for recipe in self.cookbook.get_recipes_list(): if not recipe.allow_package_creation: continue p = self._package_from_recipe(recipe) if p.name in self._packages.keys(): m.warning("Package with name '%s' already exists, not including it", p.name) else: self._packages[p.name] = p