def gen_dep(self, config, args, deps, sha):
deps_filename = self.get_deps_filename(config)
if os.path.exists(deps_filename):
os.remove(deps_filename)
log_filename = self.get_log_filename(config)
if os.path.exists(log_filename):
os.remove(log_filename)
# Workaround special mangling for windows hidden in the config
arch = os.path.basename(config.sources)
try:
shell.new_call([
'tar', '-C', config.home_dir,
'--use-compress-program=xz --threads=0', '--exclude=var/tmp',
'-cf', deps_filename, 'build-tools', config.build_tools_cache,
os.path.join('dist', arch), config.cache_file
])
url = self.make_url(config, args,
'%s-%s' % (sha, self.deps_filename))
deps.insert(
0, {
'commit': sha,
'checksum': self.checksum(deps_filename),
'url': url
})
deps = deps[0:self.log_size]
with open(log_filename, 'w') as outfile:
json.dump(deps, outfile, indent=1)
except FatalError:
os.remove(deps_filename)
os.remove(log_filename)
raise
m.message('build-dep cache generated as {}'.format(deps_filename))
def run(self, config, args):
cookbook = CookBook(config)
failed = []
p_name = args.package[0]
store = PackagesStore(config)
p = store.get_package(p_name)
ordered_recipes = map(lambda x: cookbook.get_recipe(x),
p.recipes_dependencies())
for recipe in ordered_recipes:
if cookbook.recipe_needs_build(recipe.name):
raise CommandError(
_("Recipe %s is not built yet" % recipe.name))
for recipe in ordered_recipes:
# call step function
stepfunc = None
try:
stepfunc = getattr(recipe, 'check')
except:
m.message('%s has no check step, skipped' % recipe.name)
if stepfunc:
try:
m.message('Running checks for recipe %s' % recipe.name)
stepfunc()
except Exception as ex:
failed.append(recipe.name)
m.warning(_("%s checks failed: %s") % (recipe.name, ex))
if failed:
raise CommandError(
_("Error running %s checks on:\n " + "\n ".join(failed))
% p_name)
async def fetch_dep(self, config, dep, namespace):
try:
dep_path = os.path.join(config.home_dir,
os.path.basename(dep['url']))
await shell.download(dep['url'],
dep_path,
check_cert=True,
overwrite=True)
if dep['checksum'] == self.checksum(dep_path):
await shell.unpack(dep_path, config.home_dir)
else:
m.warning("Corrupted dependency file, ignoring.")
os.remove(dep_path)
# Don't need to relocate on Windows and macOS since we build
# pkg-config with --enable-define-prefix.
# In case this needs to be re-enabled at some point, note that the
# current self.build_dir value is hard-coded and is wrong on macOS
# and Windows. It should instead be derived from CI env vars.
if config.platform == Platform.LINUX:
origin = self.build_dir % namespace
m.message("Relocating from %s to %s" %
(origin, config.home_dir))
# FIXME: Just a quick hack for now
shell.call(("grep -lnrIU %(origin)s | xargs "
"sed \"s#%(origin)s#%(dest)s#g\" -i") % {
'origin': origin,
'dest': config.home_dir
}, config.home_dir)
except FatalError as e:
m.warning("Could not retrieve dependencies for commit %s: %s" %
(dep['commit'], e.msg))
def cmd(cmd, cmd_dir='.', fail=True, verbose=False):
'''
Run a shell (dos cmd.exe) command
@param cmd: the command to run
@type cmd: str
@param cmd_dir: directory where the command will be run
@param cmd_dir: str
@param fail: whether or not to raise an exception if the command fails
@type fail: bool
'''
try:
if LOGFILE is None:
if verbose:
m.message("Running command '%s'" % cmd)
else:
LOGFILE.write("Running command '%s'\n" % cmd)
LOGFILE.flush()
shell = True
stream = LOGFILE or sys.stdout
if DRY_RUN:
# write to sdterr so it's filtered more easilly
m.error("cd %s && %s && cd %s" % (cmd_dir, cmd, os.getcwd()))
ret = 0
else:
ret = subprocess.check_call(cmd, cwd=cmd_dir,
stderr=subprocess.STDOUT,
stdout=StdOut(stream),
env=os.environ.copy(), shell=shell)
except subprocess.CalledProcessError:
if fail:
raise FatalError(_("Error running command: %s") % cmd)
else:
ret = 0
return ret
def run(self, config, args):
cookbook = CookBook(config)
recipe_name = args.recipe[0]
all_deps = args.all
graph = args.graph
if all_deps:
recipes = cookbook.list_recipe_deps(recipe_name)
else:
recipes = [
cookbook.get_recipe(x)
for x in cookbook.get_recipe(recipe_name).list_deps()
]
if len(recipes) == 0:
m.message(_('%s has 0 dependencies') % recipe_name)
return
if not graph:
for recipe in recipes:
# Don't print the recipe we asked for
if recipe.name == recipe_name:
continue
m.message(recipe.name)
else:
def print_dep(cookbook, recipe, level=0, already_shown=[]):
m.message("%s%s" % (" " * 3 * level, recipe.name))
already_shown.append(recipe)
for r in [cookbook.get_recipe(x) for x in recipe.list_deps()]:
if not r in already_shown:
print_dep(cookbook, r, level + 1, already_shown)
elif not r.name == recipe.name:
m.message("%s(%s)" % (" " * 3 * (level + 1), r.name))
print_dep(cookbook, cookbook.get_recipe(recipe_name))
def start_cooking(self):
'''
Cooks the recipe and all its dependencies
'''
recipes = [self.cookbook.get_recipe(x) for x in self.recipes]
if self.no_deps:
ordered_recipes = recipes
else:
ordered_recipes = []
for recipe in self.recipes:
deps = self.cookbook.list_recipe_deps(recipe)
# remove recipes already scheduled to be built
deps = [x for x in deps if x not in ordered_recipes]
ordered_recipes.extend(deps)
if self.deps_only:
ordered_recipes = [x for x in ordered_recipes if x not in recipes]
m.message(
_("Building the following recipes: %s") %
' '.join([x.name for x in ordered_recipes]))
steps = [step[1] for step in recipes[0].steps]
self._build_status_printer = BuildStatusPrinter(
steps, self.interactive)
self._static_libraries_built = []
run_until_complete(self._cook_recipes(ordered_recipes))
def run(self, config, args):
store = PackagesStore(config)
packages = store.get_packages_list()
if len(packages) == 0:
m.message(_("No packages found"))
for p in packages:
m.message("%s - %s" % (p.name, p.version))
def run(self, config, args):
cookbook = CookBook(config)
if args.recipe == 'all':
recipes = cookbook.get_recipes_list()
else:
recipes = [cookbook.get_recipe(args.recipe)]
if len(recipes) == 0:
m.message(_("No recipes found"))
tagname = args.tagname
tagdescription = args.tagdescription
force = args.force
for recipe in recipes:
if recipe.stype != SourceType.GIT and \
recipe.stype != SourceType.GIT_TARBALL:
m.message(
_("Recipe '%s' has a custom source repository, "
"skipping") % recipe.name)
continue
recipe.fetch(checkout=False)
tags = git.list_tags(recipe.repo_dir)
exists = (tagname in tags)
if exists:
if not force:
m.warning(
_("Recipe '%s' tag '%s' already exists, "
"not updating" % (recipe.name, tagname)))
continue
git.delete_tag(recipe.repo_dir, tagname)
commit = 'origin/sdk-%s' % recipe.version
git.create_tag(recipe.repo_dir, tagname, tagdescription, commit)
def run(self, config, args):
cookbook = CookBook(config)
if args.recipe == 'all':
recipes = cookbook.get_recipes_list()
else:
recipes = [cookbook.get_recipe(args.recipe)]
if len(recipes) == 0:
m.message(_("No recipes found"))
tagname = args.tagname
tagdescription = args.tagdescription
force = args.force
for recipe in recipes:
if recipe.stype != SourceType.GIT and \
recipe.stype != SourceType.GIT_TARBALL:
m.message(_("Recipe '%s' has a custom source repository, "
"skipping") % recipe.name)
continue
recipe.fetch(checkout=False)
tags = git.list_tags(recipe.repo_dir)
exists = (tagname in tags)
if exists:
if not force:
m.warning(_("Recipe '%s' tag '%s' already exists, "
"not updating" % (recipe.name, tagname)))
continue
git.delete_tag(recipe.repo_dir, tagname)
commit = 'origin/sdk-%s' % recipe.version
git.create_tag(recipe.repo_dir, tagname, tagdescription,
commit)
def run(self, config, args):
cookbook = CookBook(config)
recipe_name = args.recipe[0]
all_deps = args.all
graph = args.graph
if all_deps:
recipes = cookbook.list_recipe_deps(recipe_name)
else:
recipes = [cookbook.get_recipe(x) for x in
cookbook.get_recipe(recipe_name).list_deps()]
if len(recipes) == 0:
m.message(_('%s has 0 dependencies') % recipe_name)
return
if not graph:
for recipe in recipes:
# Don't print the recipe we asked for
if recipe.name == recipe_name:
continue
m.message(recipe.name)
else:
def print_dep(cookbook, recipe, level=0, already_shown=[]):
m.message("%s%s" %( " " * 3 * level, recipe.name))
already_shown.append(recipe)
for r in [cookbook.get_recipe(x) for x in recipe.list_deps()]:
if not r in already_shown:
print_dep(cookbook, r, level + 1, already_shown)
elif not r.name == recipe.name:
m.message("%s(%s)" % ( " " * 3 * (level + 1), r.name))
print_dep(cookbook, cookbook.get_recipe(recipe_name))
def get_closest_recipe(self, name):
'''
Gets the closest recipe name from name in
the cookbook if only one recipe name
matches.
@param recipe_name: name of the recipe
@type recipe_name: str
@return: the closest recipe name
@rtype: str
'''
# If there's an exact match, just return it
if name in self.recipes:
return name
recipe_name = ''
for r in self.recipes:
if name in r:
if recipe_name:
m.message(
"Name '%s' matches two or more recipes: [%s, %s]" %
(name, r, recipe_name))
return ''
recipe_name = r
if recipe_name:
m.message("Found a recipe %s matching name %s" %
(recipe_name, name))
return recipe_name
def new_call(cmd,
cmd_dir=None,
fail=True,
logfile=None,
env=None,
verbose=False):
cmd = _cmd_string_to_array(cmd, env)
if logfile:
logfile.write('Running command {!r}\n'.format(cmd))
logfile.flush()
if verbose:
m.message('Running {!r}\n'.format(cmd))
try:
subprocess.check_call(cmd,
cwd=cmd_dir,
env=env,
stdout=logfile,
stderr=subprocess.STDOUT,
stdin=subprocess.DEVNULL)
except SUBPROCESS_EXCEPTIONS as e:
returncode = getattr(e, 'returncode', -1)
if not fail:
stream = logfile or sys.stderr
if isinstance(e, FileNotFoundError):
stream.write('{}: file not found\n'.format(cmd[0]))
if isinstance(e, PermissionError):
stream.write('{!r}: permission error\n'.format(cmd))
return returncode
msg = ''
if logfile:
msg = 'Output in logfile {}'.format(logfile.name)
raise CommandError(msg, cmd, returncode)
return 0
def fetch(self, cookbook, recipes, no_deps, reset_rdeps):
fetch_recipes = []
if not recipes:
fetch_recipes = cookbook.get_recipes_list()
elif no_deps:
fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
else:
for recipe in recipes:
fetch_recipes += cookbook.list_recipe_deps(recipe)
fetch_recipes = remove_list_duplicates (fetch_recipes)
m.message(_("Fetching the following recipes: %s") %
' '.join([x.name for x in fetch_recipes]))
to_rebuild = []
for i in range(len(fetch_recipes)):
recipe = fetch_recipes[i]
m.build_step(i + 1, len(fetch_recipes), recipe, 'Fetch')
recipe.fetch()
bv = cookbook.recipe_built_version(recipe.name)
cv = recipe.built_version()
if bv != cv:
to_rebuild.append(recipe)
cookbook.reset_recipe_status(recipe.name)
if reset_rdeps:
for r in cookbook.list_recipe_reverse_deps(recipe.name):
to_rebuild.append(r)
cookbook.reset_recipe_status(r.name)
if to_rebuild:
m.message(_("These recipes have been updated and will "
"be rebuilt:\n%s") %
'\n'.join([x.name for x in to_rebuild]))
def run(self, config, args):
cookbook = CookBook(config)
recipe_name = args.recipe[0]
recursive = args.recursive
recipe = cookbook.get_recipe(recipe_name)
if recursive:
ordered_recipes = cookbook.list_recipe_deps(recipe_name)
else:
ordered_recipes = [recipe]
for recipe in ordered_recipes:
if cookbook.recipe_needs_build(recipe.name):
raise FatalError(_("Recipe %s is not built yet" % recipe.name))
for recipe in ordered_recipes:
# call step function
stepfunc = None
try:
stepfunc = getattr(recipe, 'check')
except:
m.message('%s has no check step, skipped' % recipe.name)
if stepfunc:
try:
if asyncio.iscoroutinefunction(stepfunc):
run_until_complete(stepfunc())
else:
stepfunc()
except FatalError as e:
raise e
except Exception as ex:
raise FatalError(
_("Error running %s checks: %s") % (recipe.name, ex))
def run(self, config, args):
store = PackagesStore(config)
p_name = args.package[0]
if args.list_files:
m.message('\n'.join(store.get_package_files_list(p_name)))
else:
p = store.get_package(p_name)
licenses = [p.license]
if not isinstance(p, MetaPackage):
recipes_licenses = p.recipes_licenses()
recipes_licenses.update(p.devel_recipes_licenses())
for recipe_name, categories_licenses in \
six.iteritems(recipes_licenses):
for category_licenses in six.itervalues(
categories_licenses):
licenses.extend(category_licenses)
licenses = sorted(list(set(licenses)))
d = {
'name':
p.name,
'version':
p.version,
'url':
p.url,
'licenses':
' and '.join([l.acronym for l in licenses]),
'desc':
p.shortdesc,
'deps':
', '.join(
[p.name for p in store.get_package_deps(p_name, True)])
}
m.message(INFO_TPL % d)
def run(self, config, args):
store = PackagesStore(config)
packages = store.get_packages_list()
if len(packages) == 0:
m.message(_("No packages found"))
for p in packages:
m.message("%s - %s" % (p.name, p.version))
def run(self, config, args):
cookbook = CookBook(config)
recipe_name = args.recipe[0]
recursive = args.recursive
recipe = cookbook.get_recipe(recipe_name)
if recursive:
ordered_recipes = cookbook.list_recipe_deps(recipe_name)
else:
ordered_recipes = [recipe]
for recipe in ordered_recipes:
if cookbook.recipe_needs_build(recipe.name):
raise FatalError(_("Recipe %s is not built yet" % recipe.name))
for recipe in ordered_recipes:
# call step function
stepfunc = None
try:
stepfunc = getattr(recipe, 'check')
except:
m.message('%s has no check step, skipped' % recipe.name)
if stepfunc:
try:
stepfunc()
except FatalError as e:
raise e
except Exception as ex:
raise FatalError(_("Error running %s checks: %s") %
(recipe.name, ex))
def run(self, config, args):
cookbook = CookBook(config)
recipes = cookbook.get_recipes_list()
if len(recipes) == 0:
m.message(_("No recipes found"))
for recipe in recipes:
m.message("%s - %s" % (recipe.name, recipe.version))
def run(self, config, args):
cookbook = CookBook(config)
failed = []
p_name = args.package[0]
store = PackagesStore(config)
p = store.get_package(p_name)
ordered_recipes = map(lambda x: cookbook.get_recipe(x),
p.recipes_dependencies())
for recipe in ordered_recipes:
if cookbook.recipe_needs_build(recipe.name):
raise CommandError(_("Recipe %s is not built yet" % recipe.name))
for recipe in ordered_recipes:
# call step function
stepfunc = None
try:
stepfunc = getattr(recipe, 'check')
except:
m.message('%s has no check step, skipped' % recipe.name)
if stepfunc:
try:
m.message('Running checks for recipe %s' % recipe.name)
stepfunc()
except Exception, ex:
failed.append(recipe.name)
m.warning(_("%s checks failed: %s") % (recipe.name, ex))
def run(self, config, args):
self.store = PackagesStore(config, offline=args.offline)
p = self.store.get_package(args.package[0])
if args.skip_deps_build and args.only_build_deps:
raise UsageError(
_("Cannot use --skip-deps-build together with "
"--only-build-deps"))
if not args.skip_deps_build:
self._build_deps(config, p, args.no_devel, args.offline,
args.dry_run, args.jobs)
if args.only_build_deps or args.dry_run:
return
if args.compress_method != 'default':
m.message('Forcing tarball compression method as ' +
args.compress_method)
config.package_tarball_compression = args.compress_method
if p is None:
raise PackageNotFoundError(args.package[0])
p.pre_package()
packager_class = Packager
if args.tarball:
if config.target_platform == Platform.ANDROID and \
config.target_arch == Architecture.UNIVERSAL:
packager_class = AndroidPackager
else:
packager_class = DistTarball
elif config.variants.uwp:
# Split devel/runtime packages are useless for UWP since we will
# need both when building the package, and all needed runtime DLLs
# are packaged with the app as assets.
m.warning('Forcing single-tarball output for UWP package')
args.no_split = True
packager_class = DistTarball
m.action(_("Creating package for %s") % p.name)
pkg = packager_class(config, p, self.store)
output_dir = os.path.abspath(args.output_dir)
if isinstance(pkg, DistTarball):
paths = pkg.pack(output_dir,
args.no_devel,
args.force,
args.keep_temp,
split=not args.no_split,
strip_binaries=p.strip)
else:
paths = pkg.pack(output_dir, args.no_devel, args.force,
args.keep_temp)
if None in paths:
paths.remove(None)
paths = p.post_package(paths, output_dir) or paths
m.action(
_("Package successfully created in %s") %
' '.join([os.path.abspath(x) for x in paths]))
def call(cmd, cmd_dir='.', fail=True, verbose=False, logfile=None, env=None):
'''
Run a shell command
DEPRECATED: Use new_call and a cmd array wherever possible
@param cmd: the command to run
@type cmd: str
@param cmd_dir: directory where the command will be run
@param cmd_dir: str
@param fail: whether or not to raise an exception if the command fails
@type fail: bool
'''
try:
if logfile is None:
if verbose:
m.message("Running command '%s'" % cmd)
stream = None
else:
logfile.write("Running command '%s'\n" % cmd)
logfile.flush()
stream = logfile
shell = True
if PLATFORM == Platform.WINDOWS:
# windows do not understand ./
if cmd.startswith('./'):
cmd = cmd[2:]
# run all processes through sh.exe to get scripts working
cmd = '%s "%s"' % ('sh -c', cmd)
# fix paths with backslashes
cmd = _fix_mingw_cmd(cmd)
# Disable shell which uses cmd.exe
shell = False
if DRY_RUN:
# write to sdterr so it's filtered more easilly
m.error("cd %s && %s && cd %s" % (cmd_dir, cmd, os.getcwd()))
ret = 0
else:
if env is not None:
env = env.copy()
else:
env = os.environ.copy()
# Force python scripts to print their output on newlines instead
# of on exit. Ensures that we get continuous output in log files.
env['PYTHONUNBUFFERED'] = '1'
ret = subprocess.check_call(cmd,
cwd=cmd_dir,
bufsize=1,
stderr=subprocess.STDOUT,
stdout=stream,
universal_newlines=True,
env=env,
shell=shell)
except subprocess.CalledProcessError:
if fail:
raise FatalError(_("Error running command: %s") % cmd)
else:
ret = 0
return ret
def print_dep(cookbook, recipe, level=0, already_shown=[]):
m.message("%s%s" %( " " * 3 * level, recipe.name))
already_shown.append(recipe)
for r in [cookbook.get_recipe(x) for x in recipe.list_deps()]:
if not r in already_shown:
print_dep(cookbook, r, level + 1, already_shown)
elif not r.name == recipe.name:
m.message("%s(%s)" % ( " " * 3 * (level + 1), r.name))
def print_dep(cookbook, recipe, level=0, already_shown=[]):
m.message("%s%s" % (" " * 3 * level, recipe.name))
already_shown.append(recipe)
for r in [cookbook.get_recipe(x) for x in recipe.list_deps()]:
if not r in already_shown:
print_dep(cookbook, r, level + 1, already_shown)
elif not r.name == recipe.name:
m.message("%s(%s)" % (" " * 3 * (level + 1), r.name))
def _install_perl_deps(self):
cpanm_installer = os.path.join(self.config.local_sources, 'cpanm')
shell.new_call(['chmod', '+x', cpanm_installer])
# Install XML::Parser, required for intltool
cmd = ['sudo', cpanm_installer, 'XML::Parser']
m.message(
"Installing XML::Parser, may require a password for running \'" +
" ".join(cmd) + "\'")
shell.new_call(cmd)
def run(self, config, args):
BaseCache.run(self, config, args)
sha = self.get_git_sha(args)
deps = self.get_deps(config, args)
dep = self.find_dep(deps, sha)
if dep:
run_until_complete(self.fetch_dep(config, dep, args.namespace))
m.message('All done!')
def run(self, config, args):
cookbook = CookBook(config)
recipe_name = args.recipe[0]
recipes = cookbook.list_recipe_reverse_deps(recipe_name)
if len(recipes) == 0:
m.error(_('%s has 0 reverse dependencies') % recipe_name)
return
for recipe in recipes:
m.message(recipe.name)
def run(self, config, args):
if args.bootstrap:
config.cache_file = config.build_tools_cache
cookbook = CookBook(config)
is_modifying = False or args.touch or args.reset
if not is_modifying:
for attr in self.recipe_attributes:
if getattr(args, attr) is not None:
is_modifying = True
break
global_status = cookbook.status
recipes = args.recipe or list(global_status.keys())
m.message('{} cache values for recipes: {}'.format(
'Showing' if not is_modifying else 'Modifying',
', '.join(recipes)))
for recipe in recipes:
if recipe not in global_status.keys():
m.error('Recipe {} not in cookbook'.format(recipe))
continue
status = global_status[recipe]
print('[{}]'.format(recipe))
text = ''
if is_modifying:
text = 'Before\n'
print('{}{}\n'.format(text, status))
if is_modifying:
if args.reset:
cookbook.reset_recipe_status(recipe)
m.message('Recipe {} reset'.format(recipe))
else:
if args.touch:
status.touch()
for attr in self.recipe_attributes:
var = getattr(args, attr)
if var is not None:
if isinstance(getattr(self.recipe_status, attr),
bool):
if var.lower() == 'true':
var = True
elif var.lower() == 'false':
var = False
else:
m.error(
'Error: Attribute "{}" needs to be either "True" or "False"'
.format(attr))
return
setattr(status, attr, var)
cookbook.save()
print('After\n{}\n'.format(status))
def run(self, config, args):
cookbook = CookBook(config)
recipes = cookbook.get_recipes_list()
if len(recipes) == 0:
m.message(_("No recipes found"))
for recipe in recipes:
try:
current = recipe.built_version().split("\n")[0]
except:
current = "Not checked out"
m.message("%s - %s (current checkout: %s)" % (recipe.name, recipe.version, current))
def __init__(self, steps, interactive):
self.steps = steps[:]
self.step_to_recipe = collections.defaultdict(list)
self.recipe_to_step = {}
self.total = 0
self.count = 0
self.interactive = interactive
# FIXME: Default MSYS shell doesn't handle ANSI escape sequences correctly
if os.environ.get('TERM') == 'cygwin':
m.message(
'Running under MSYS: reverting to basic build status output')
self.interactive = False
def run(self, config, args):
cookbook = CookBook(config)
recipes = cookbook.get_recipes_list()
if len(recipes) == 0:
m.message(_("No recipes found"))
for recipe in recipes:
try:
current = recipe.built_version().split("\n")[0]
except:
current = "Not checked out"
m.message("%s - %s (current checkout: %s)" % (recipe.name, recipe.version, current))
def start_cooking(self):
'''
Cooks the recipe and all its dependencies
'''
recipes = [self.cookbook.get_recipe(x) for x in self.recipes]
if self.no_deps:
ordered_recipes = recipes
else:
ordered_recipes = []
for recipe in self.recipes:
deps = self.cookbook.list_recipe_deps(recipe)
# remove recipes already scheduled to be built
deps = [x for x in deps if x not in ordered_recipes]
ordered_recipes.extend(deps)
if self.deps_only:
ordered_recipes = [x for x in ordered_recipes if x not in recipes]
m.message(
_("Building the following recipes: %s") %
' '.join([x.name for x in ordered_recipes]))
i = 1
self._static_libraries_built = []
for recipe in ordered_recipes:
try:
self._cook_recipe(recipe, i, len(ordered_recipes))
except BuildStepError as be:
if not self.interactive:
raise be
msg = be.msg
msg += _("Select an action to proceed:")
action = shell.prompt_multiple(msg, RecoveryActions())
if action == RecoveryActions.SHELL:
shell.enter_build_environment(self.config.target_platform,
be.arch,
recipe.get_for_arch(
be.arch, 'build_dir'),
env=recipe.config.env)
raise be
elif action == RecoveryActions.RETRY_ALL:
shutil.rmtree(recipe.get_for_arch(be.arch, 'build_dir'))
self.cookbook.reset_recipe_status(recipe.name)
self._cook_recipe(recipe, i, len(ordered_recipes))
elif action == RecoveryActions.RETRY_STEP:
self._cook_recipe(recipe, i, len(ordered_recipes))
elif action == RecoveryActions.SKIP:
i += 1
continue
elif action == RecoveryActions.ABORT:
raise AbortedError()
i += 1
def find_orphan_files(self, allfiles, prefix, excludes=[]):
cmd = ['find', '.', '-type', 'f']
for x in excludes:
cmd += ['(', '!', '-name', x, ')']
distfiles = shell.check_output(cmd, cmd_dir=prefix).splitlines()
# remove './' from the list of files
distfiles = [f[2:] for f in distfiles]
orphan = sorted(list((set(distfiles) - set(allfiles))))
if len(orphan) > 0:
m.message("Found orphan files:")
m.message('\n'.join(orphan))
def call(cmd, cmd_dir='.', fail=True, verbose=False):
'''
Run a shell command
@param cmd: the command to run
@type cmd: str
@param cmd_dir: directory where the command will be run
@param cmd_dir: str
@param fail: whether or not to raise an exception if the command fails
@type fail: bool
'''
global CALL_ENV
try:
if LOGFILE is None:
if verbose:
m.message("Running command '%s'" % cmd)
else:
LOGFILE.write("Running command '%s'\n" % cmd)
LOGFILE.flush()
shell = True
if PLATFORM == Platform.WINDOWS:
# windows do not understand ./
if cmd.startswith('./'):
cmd = cmd[2:]
# run all processes through sh.exe to get scripts working
cmd = '%s "%s"' % ('sh -c', cmd)
# fix paths with backslashes
cmd = _fix_mingw_cmd(cmd)
# Disable shell which uses cmd.exe
shell = False
stream = LOGFILE or sys.stdout
if DRY_RUN:
# write to sdterr so it's filtered more easilly
m.error("cd %s && %s && cd %s" % (cmd_dir, cmd, os.getcwd()))
ret = 0
else:
if CALL_ENV is not None:
env = CALL_ENV.copy()
else:
env = os.environ.copy()
ret = subprocess.check_call(cmd,
cwd=cmd_dir,
stderr=subprocess.STDOUT,
stdout=StdOut(stream),
env=env,
shell=shell)
except subprocess.CalledProcessError:
if fail:
raise FatalError(_("Error running command: %s") % cmd)
else:
ret = 0
return ret
def _hijack_download(url,
destination=None,
recursive=False,
check_cert=True,
overwrite=False):
mirror_url = cac.mirror(url)
if mirror_url:
m.message('%s has been redirect to %s.' % (url, mirror_url))
_old_shell_download(mirror_url, destination, recursive, check_cert,
overwrite)
else:
_old_shell_download(url, destination, recursive, check_cert,
overwrite)
def find_orphan_files(self, allfiles, prefix, excludes=[]):
cmd = 'find . -type f %s'
exc = map(lambda x: "\\( ! -name '%s' \\)" % x, excludes)
cmd = cmd % ' '.join(exc)
distfiles = shell.check_call(cmd, prefix).split('\n')
# remove './' from the list of files
distfiles = [f[2:] for f in distfiles]
orphan = sorted(list((set(distfiles) - set(allfiles))))
if len(orphan) > 0:
m.message("Found orphan files:")
m.message('\n'.join(orphan))
def find_orphan_files(self, allfiles, prefix, excludes=[]):
cmd = 'find . -type f %s'
exc = map(lambda x: "\\( ! -name '%s' \\)" % x, excludes)
cmd = cmd % ' '.join(exc)
distfiles = shell.check_call(cmd, prefix).split('\n')
# remove './' from the list of files
distfiles = [f[2:] for f in distfiles]
orphan = sorted(list((set(distfiles) - set(allfiles))))
if len(orphan) > 0:
m.message("Found orphan files:")
m.message('\n'.join(orphan))
def _unpack(self, name, cats=['devel', 'runtime'], version=None):
profile = self.profile[name]
#ppath = profile.get('__file__',None)
#if ppath is None or not os.path.exists( ppath ):
# m.error('package <%s> not exists!'%name)
# raise FatalError('install package failed')
path = profile.get('__file__', None)
if path:
if not os.path.exists(path):
m.error('package <%s> not exists!' % name)
raise FatalError('install package failed')
d = os.path.dirname(profile['__file__'])
else:
d = profile.get('__dir__', None)
assert d, '!!!!!!!'
prefix = self.args.prefix
if prefix is None:
if name == 'build-tools':
prefix = self.config.build_tools_prefix
else:
prefix = self.config.prefix
#profile path
ip = os.path.join(prefix, name + '.yaml')
if os.path.exists(ip):
pro = yaml.load(open(ip))
if version:
m.error(
'installed package %s version %s not consistent with require %s'
% (name, pro['version'], version))
raise FatalError("install package failed.")
else:
m.message('package %s (%s) already installed.' %
(name, pro['version']))
return
for cat, info in profile['tarball'].viewitems():
filename = info['filename']
url = os.path.join(d, filename)
path = cache(url, self.args.cache_dir)
assert os.path.exists(path)
shell.unpack(path, prefix)
yaml.dump(profile, open(ip, 'w'), default_flow_style=False)
m.message('package <%s> installation done.' % name)
def json_get(self, url):
m.message("GET %s" % url)
tmpdir = tempfile.mkdtemp()
tmpfile = os.path.join(tmpdir, 'deps.json')
run_until_complete(
shell.download(url,
destination=tmpfile,
logfile=open(os.devnull, 'w')))
with open(tmpfile, 'r') as f:
resp = f.read()
shutil.rmtree(tmpdir)
return json.loads(resp)
def __init__(self, store, force=False, dry_run=False):
self.store = store
self.cookbook = store.cookbook
self.config = self.cookbook.get_config()
self.force = force
shell.DRY_RUN = dry_run
if not self.config.binaries:
raise FatalError(_('Configuration without binaries path'))
self.binaries = os.path.join(self.config.binaries, self.config.get_md5())
if not self.config.binary_repo:
raise FatalError(_('Configuration without binary repo'))
self.binary_repo = os.path.join(self.config.binary_repo, self.config.get_md5())
m.message('Using config MD5: %s' % self.config.get_md5())
if not os.path.exists(self.binaries):
os.makedirs(self.binaries)
def run(self, config, args):
if config.target_platform != Platform.WINDOWS:
raise UsageError(_('%s command can only be used targetting '
'Windows platforms') % self.name)
if args.output_dir is not None and not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
cookbook = CookBook(config)
recipes = cookbook.get_recipes_list()
for recipe in recipes:
try:
recipe.gen_library_file(args.output_dir)
except Exception, e:
m.message(_("Error generaring library files for %s:\n %s") %
(recipe.name, e))
def runargs(self, config, output_dir, filename, libraries):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if len(libraries) == 0:
raise UsageError("You need to specify at least one library name")
filename = filename or libraries[0]
filepath = os.path.join(output_dir, '%s.xcconfig' % filename)
xcconfig = XCConfig(libraries)
xcconfig.create(filepath)
m.action('Created %s.xcconfig' % filename)
m.message('XCode config file were sucessfully created in %s' %
os.path.abspath(filepath))
def runargs(self, config, output_dir, prefix=DEFAULT_PREFIX_MACRO):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for pc in PkgConfig.list_all():
p2v = PkgConfig2VSProps(pc, prefix=config.prefix,
inherit_common=True,
prefix_replacement='$(%s)' % prefix)
p2v.create(output_dir)
m.action('Created %s.props' % pc)
common = CommonProps(prefix)
common.create(output_dir)
m.message('Property sheets files were sucessfully created in %s' %
os.path.abspath(output_dir))
def call(cmd, cmd_dir='.', fail=True, env=None, verbose=False):
'''
Run a shell command
@param cmd: the command to run
@type cmd: str
@param cmd_dir: directory where the command will be run
@param cmd_dir: str
@param fail: whether or not to raise an exception if the command fails
@type fail: bool
'''
try:
if LOGFILE is None:
if verbose:
m.message("Running command '%s'" % cmd)
else:
LOGFILE.write("Running command '%s'\n" % cmd)
shell = True
if PLATFORM == Platform.WINDOWS:
# windows do not understand ./
if cmd.startswith('./'):
cmd = cmd[2:]
# run all processes through sh.exe to get scripts working
cmd = '%s "%s"' % ('sh -c', cmd)
# fix paths with backslashes
cmd = _fix_mingw_cmd(cmd)
# Disable shell which uses cmd.exe
shell = False
stream = LOGFILE or sys.stdout
if DRY_RUN:
# write to sdterr so it's filtered more easilly
m.error("cd %s && %s && cd %s" % (cmd_dir, cmd, os.getcwd()))
ret = 0
else:
if not env:
env = os.environ.copy()
ret = subprocess.check_call(cmd, cwd=cmd_dir,
stderr=subprocess.STDOUT,
stdout=StdOut(stream),
env=env, shell=shell)
except subprocess.CalledProcessError:
if fail:
raise FatalError(_("Error running command: %s") % cmd)
else:
ret = 0
return ret
def run(self, config, args):
cookbook = CookBook(config)
recipe_name = args.recipe[0]
all_deps = args.all
if all_deps:
recipes = cookbook.list_recipe_deps(recipe_name)
else:
recipes = [cookbook.get_recipe(x) for x in
cookbook.get_recipe(recipe_name).list_deps()]
if len(recipes) == 0:
m.message(_('%s has 0 dependencies') % recipe_name)
return
for recipe in recipes:
# Don't print the recipe we asked for
if recipe.name == recipe_name:
continue
m.message(recipe.name)
def start_cooking(self):
'''
Cooks the recipe and all its dependencies
'''
if self.no_deps:
ordered_recipes = [self.cookbook.get_recipe(x) for x in
self.recipes]
else:
ordered_recipes = []
for recipe in self.recipes:
recipes = self.cookbook.list_recipe_deps(recipe)
# remove recipes already scheduled to be built
recipes = [x for x in recipes if x not in ordered_recipes]
ordered_recipes.extend(recipes)
m.message(_("Building the following recipes: %s") %
' '.join([x.name for x in ordered_recipes]))
i = 1
for recipe in ordered_recipes:
try:
self._cook_recipe(recipe, i, len(ordered_recipes))
except BuildStepError as be:
if not self.interactive:
raise be
msg = be.msg
msg += _("Select an action to proceed:")
action = shell.prompt_multiple(msg, RecoveryActions())
if action == RecoveryActions.SHELL:
shell.enter_build_environment(self.config.target_platform,
self.config.target_arch, recipe.build_dir)
break
elif action == RecoveryActions.RETRY_ALL:
shutil.rmtree(recipe.build_dir)
self.cookbook.reset_recipe_status(recipe.name)
self._cook_recipe(recipe, i, len(ordered_recipes))
elif action == RecoveryActions.RETRY_STEP:
self._cook_recipe(recipe, i, len(ordered_recipes))
elif action == RecoveryActions.SKIP:
continue
elif action == RecoveryActions.ABORT:
raise AbortedError()
i += 1
def _check_duplicated_symbols(self, files, tmpdir):
for f in files:
syms = defaultdict(list)
symbols = shell.check_call('nm -UA %s' % f, tmpdir).split('\n')
# nm output is: test.o: 00000000 T _gzwrite
# (filename, address, symbol type, symbols_name)
for s in symbols:
s = s.split(' ')
if len(s) == 4 and s[2] == 'T':
syms[s[3]].append(s)
dups = {}
for k,v in syms.items():
if len(v) > 1:
dups[k] = v
if dups:
m.warning ("The static library contains duplicated symbols")
for k, v in dups.items():
m.message (k) # symbol name
for l in v:
m.message (" %s" % l[0]) # file
def start_cooking(self, recipes, use_binaries=False, upload_binaries=False,
build_missing=True):
'''
Cooks the provided recipe names
'''
if isinstance(recipes, str):
recipes = [recipes]
m.message(_("Building the following recipes: %s") %
' '.join([x for x in recipes]))
if use_binaries or upload_binaries:
def _build(recipe, i, length):
self.cook_recipe(recipe, i, length)
if upload_binaries:
try:
fridge.freeze_recipe(recipe, i, length)
except RecipeNotFreezableError:
pass
if not self.store:
self.store = PackagesStore(self.cookbook.get_config())
fridge = Fridge(self.store, force=self.force, dry_run=self.dry_run)
i = 1
for recipe in recipes:
if use_binaries:
try:
fridge.unfreeze_recipe(recipe, i, len(recipes))
except (RecipeNotFreezableError, BuildStepError) as e:
if build_missing or isinstance(e, RecipeNotFreezableError):
_build(recipe, i, len(recipes))
else:
raise e
else:
_build(recipe, i, len(recipes))
i += 1
else:
i = 1
for recipe in recipes:
self.cook_recipe(recipe, i, len(recipes))
i += 1
def start_cooking(self):
'''
Cooks the recipe and all its dependencies
'''
if self.no_deps:
ordered_recipes = [self.cookbook.get_recipe(x) for x in
self.recipes]
else:
ordered_recipes = []
for recipe in self.recipes:
recipes = self.cookbook.list_recipe_deps(recipe)
# remove recipes already scheduled to be built
recipes = [x for x in recipes if x not in ordered_recipes]
ordered_recipes.extend(recipes)
m.message(_("Building the following recipes: %s") %
' '.join([x.name for x in ordered_recipes]))
i = 1
for recipe in ordered_recipes:
self._cook_recipe(recipe, i, len(ordered_recipes))
i += 1
def run(self, config, args):
store = PackagesStore(config)
p_name = args.package[0]
if args.list_files:
m.message('\n'.join(store.get_package_files_list(p_name)))
else:
p = store.get_package(p_name)
licenses = [p.license]
if not isinstance(p, MetaPackage):
recipes_licenses = p.recipes_licenses()
recipes_licenses.update(p.devel_recipes_licenses())
for recipe_name, categories_licenses in \
recipes_licenses.items():
for category_licenses in categories_licenses.values():
licenses.extend(category_licenses)
licenses = sorted(list(set(licenses)))
d = {'name': p.name, 'version': p.version, 'url': p.url,
'licenses': ' and '.join([l.acronym for l in licenses]),
'desc': p.shortdesc,
'deps': ', '.join([p.name for p in
store.get_package_deps(p_name, True)])}
m.message(INFO_TPL % d)
def _print_missing_files(self, recipe, tmp):
recipe_files = set(recipe.files_list())
installed_files = set(shell.find_newer_files(recipe.config.prefix,
tmp.name))
not_in_recipe = list(installed_files - recipe_files)
not_installed = list(recipe_files - installed_files)
if len(not_in_recipe) != 0:
m.message(_("The following files where installed, but are not "
"listed in the recipe:"))
m.message('\n'.join(sorted(not_in_recipe)))
if len(not_installed) != 0:
m.message(_("The following files are listed in the recipe, but "
"where not installed:"))
m.message('\n'.join(sorted(not_installed)))
def fetch(self, cookbook, recipes, no_deps, reset_rdeps, full_reset):
fetch_recipes = []
if not recipes:
fetch_recipes = cookbook.get_recipes_list()
elif no_deps:
fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
else:
for recipe in recipes:
fetch_recipes += cookbook.list_recipe_deps(recipe)
fetch_recipes = remove_list_duplicates (fetch_recipes)
m.message(_("Fetching the following recipes: %s") %
' '.join([x.name for x in fetch_recipes]))
to_rebuild = []
for i in range(len(fetch_recipes)):
recipe = fetch_recipes[i]
m.build_step(i + 1, len(fetch_recipes), recipe, 'Fetch')
recipe.fetch()
bv = cookbook.recipe_built_version(recipe.name)
cv = recipe.built_version()
if bv != cv:
# On different versions, only reset recipe if:
# * forced
# * OR it was fully built already
if full_reset or not cookbook.recipe_needs_build(recipe.name):
to_rebuild.append(recipe)
cookbook.reset_recipe_status(recipe.name)
if reset_rdeps:
for r in cookbook.list_recipe_reverse_deps(recipe.name):
to_rebuild.append(r)
cookbook.reset_recipe_status(r.name)
if to_rebuild:
to_rebuild = sorted(list(set(to_rebuild)), key=lambda r:r.name)
m.message(_("These recipes have been updated and will "
"be rebuilt:\n%s") %
'\n'.join([x.name for x in to_rebuild]))
def run(self, config, args):
store = PackagesStore(config)
p_name = args.package[0]
if args.list_files:
m.message("\n".join(store.get_package_files_list(p_name)))
else:
p = store.get_package(p_name)
licenses = [p.license]
if not isinstance(p, MetaPackage):
recipes_licenses = p.recipes_licenses()
recipes_licenses.update(p.devel_recipes_licenses())
for recipe_name, categories_licenses in recipes_licenses.iteritems():
for category_licenses in categories_licenses.itervalues():
licenses.extend(category_licenses)
licenses = sorted(list(set(licenses)))
d = {
"name": p.name,
"version": p.version,
"url": p.url,
"licenses": " and ".join([l.acronym for l in licenses]),
"desc": p.shortdesc,
"deps": ", ".join([p.name for p in store.get_package_deps(p_name, True)]),
}
m.message(INFO_TPL % d)
def find_duplicates(self, allfiles):
count = collections.Counter(allfiles)
duplicates = [x for x in count if count[x] > 1]
if len(duplicates) > 0:
m.message("Found duplicates files in packages:")
m.message("%r" % duplicates)