class native_package_base(with_metaclass(ABCMeta, object)):
def __init__(self, bl):
bl = bl or blurber()
check.check_blurber(bl)
self._blurber = bl
@abstractmethod
def installed_packages(self):
'Return a list of packages on this computer.'
raise NotImplemented('installed_packages')
@abstractmethod
def package_files(self, package_name):
'Return a list of files installed for package.'
raise NotImplemented('package_files')
@abstractmethod
def package_dirs(self, package_name):
'Return a list of dirs installed for package.'
raise NotImplemented('package_dirs')
@abstractmethod
def is_installed(self, package_name):
'Return True if package is installed.'
raise NotImplemented('is_installed')
@abstractmethod
def owner(self, filename):
'Return the package that owns filename.'
raise NotImplemented('owner')
@abstractmethod
def package_info(self, filename):
'Return info structure about the package.'
raise NotImplemented('package_info')
@abstractmethod
def remove(self, package_name, force_package_root):
'Remove a package.'
raise NotImplemented('remove')
@abstractmethod
def install(self, package_filename):
'Install a package.'
raise NotImplemented('install')
def has_package(self, package_name):
'Return True if package_name is installed.'
check.check_string(package_name)
return package_name in self.installed_packages()
def has_any_package(self, package_names):
'Return True if any of package_names is installed.'
check.check_string_seq(package_names)
for package_name in package_names:
if self.has_package(package_name):
return True
return False
class computer_setup_task(with_metaclass(_computer_setup_register_meta,
object)):
@abstractmethod
def name(self):
'Name for task.'
raise NotImplemented('name')
@abstractmethod
def description(self):
'Description for task.'
raise NotImplemented('description')
@abstractmethod
def average_duration(self):
'Average duration in seconds.'
raise NotImplemented('average_duration')
@abstractmethod
def is_needed(self):
'Return True of the task needs to run.'
raise NotImplemented('is_needed')
@abstractmethod
def run(self, options):
'Run the task.'
raise NotImplemented('run')
class vmware_command_interpreter(
with_metaclass(_command_interpreter_register_meta, object)):
@abstractmethod
def name(self):
'Name for this interpreter.'
raise NotImplemented('name')
@abstractmethod
def is_default(self):
'Return True if this command interpreter is the default.'
raise NotImplemented('is_default')
@abstractmethod
def supported_systems(self):
'Return a tuple of supported systems.'
raise NotImplemented('supported_systems')
@abstractmethod
def full_path(self):
'Return the full path of the interpreter in the vm'
raise NotImplemented('interpreter_path')
command = namedtuple('command', 'interpreter_path, script_text')
@abstractmethod
def build_command(self, script_text):
'Build a command and return a command object for it'
raise NotImplemented('build_command')
class filesystem_base(with_metaclass(ABCMeta, object)):
'Abstract interface for dealing with system specific non portable filesystem stuff.'
@classmethod
@abstractmethod
def free_disk_space(self, directory):
'Return the free space for directory in bytes.'
raise NotImplemented('free_disk_space')
@classmethod
@abstractmethod
def sync(self):
'Sync the filesystem. Only works for both unix and windows in python3. Otherwise only unix.'
raise NotImplemented('sync')
@classmethod
@abstractmethod
def has_symlinks(self):
'Return True if this system has support for symlinks.'
raise NotImplemented('has_symlinks')
@classmethod
@abstractmethod
def remove_directory(self, d):
'Recursively remove a directory.'
raise NotImplemented('remove_directory')
class archive_operation_base(with_metaclass(ABCMeta, object)):
'An archive operation interface.'
@abstractmethod
def execute(self, temp_dir):
'Execute this operation in a temp_dir of the unpacked archive.'
raise NotImplementedError()
class recipe_load_env_base(with_metaclass(ABCMeta, object)):
@abstractmethod
def get_build_target(self):
raise NotImplementedError
@abstractmethod
def get_downloads_manager(self):
raise NotImplementedError
@abstractmethod
def get_source_finder(self):
raise NotImplementedError
@property
def build_target(self):
return self.get_build_target()
@property
def downloads_manager(self):
return self.get_downloads_manager()
@property
def source_finder(self):
return self.get_source_finder()
class credentials_source(with_metaclass(ABCMeta, object)):
@abstractmethod
def is_valid(self):
pass
@abstractmethod
def credentials(self):
pass
class python_source_base(with_metaclass(ABCMeta, object)):
'''
Abstract interface for dealing with the source of a python exe
and other platform specific python information
'''
@classmethod
@abstractmethod
def exe_source(clazz, exe):
'Return the source of the python executable. Stuff like brew, xcode, system, python.org.'
raise NotImplemented('exe_source')
@classmethod
@abstractmethod
def possible_python_bin_dirs(clazz):
'Return a list of possible dirs where the python executable might be.'
raise NotImplemented('possible_python_bin_dirs')
@classmethod
@abstractmethod
def possible_python_exe_patterns(clazz):
'Return a list of possible python exe fnmatch patters.'
raise NotImplemented('possible_python_exe_patterns')
@classmethod
@abstractmethod
def possible_python_dir_should_be_ignored(clazz, dirname):
'Return True if dirname should be ignored as a possible python bin dir.'
raise NotImplemented('possible_python_dir_should_be_ignored')
@classmethod
@abstractmethod
def exe_name(clazz, exe):
'Return the name of a python exe. without possible extensions or absolute paths.'
raise NotImplemented('exe_name')
@classmethod
@abstractmethod
def possible_python_dot_org_installer_filenames(clazz, full_version):
'Return a list of possible python.org installer filenames for full version.'
raise NotImplemented('possible_python_dot_org_installer_filenames')
@classmethod
@abstractmethod
def versioned_python_exe(clazz, root_dir, version):
'Return the absolute path the python exe with major.minor version in a virtual env.'
raise NotImplemented('versioned_python_exe')
@classmethod
@abstractmethod
def python_exe(clazz, root_dir, version):
'Return the absolute path the python exe with major version in a virtual env.'
raise NotImplemented('python_exe')
@classmethod
@abstractmethod
def activate_script(clazz, root_dir, variant):
'Return the absolute path the the acitivate script of a virtual env.'
raise NotImplemented('activate_script')
class source_finder(with_metaclass(ABCMeta, object)):
@abstractmethod
def find_tarball(self, filename):
pass
@abstractmethod
def ensure_source(self, filename):
pass
@classmethod
def _find_by_filename(self, where, filename):
return tarball_finder.find_by_filename(where, filename)
class _file_mime_type_detector_base(with_metaclass(ABCMeta, object)):
@classmethod
@abstractmethod
def is_supported(clazz):
'Return True if this class is supported on the current platform.'
raise NotImplemented('is_supported')
@classmethod
@abstractmethod
def detect_mime_type(clazz, filename):
'Detect the mime type for file.'
raise NotImplemented('detect_mime_type')
class process_lister_base(with_metaclass(ABCMeta, object)):
'Abstract interface for listing processes.'
@classmethod
@abstractmethod
def list_processes(clazz):
'List all processes.'
raise NotImplemented('list_processes')
@classmethod
@abstractmethod
def open_files(clazz, pid):
'Return a list of open files for pid or None if pid not found.'
raise NotImplemented('open_files')
class vmware_app_base(with_metaclass(ABCMeta, object)):
'Abstract interface for interacting with the vmware workstation or fusion main application.'
@classmethod
@abstractmethod
def is_installed(clazz):
'Return True if vmware is installed.'
raise NotImplemented('is_installed')
@classmethod
@abstractmethod
def is_running(clazz):
'Return True if vmware is running.'
raise NotImplemented('is_running')
@classmethod
@abstractmethod
def ensure_running(clazz):
'Ensure vmware is running.'
raise NotImplemented('ensure_running')
@classmethod
@abstractmethod
def ensure_stopped(clazz):
'Ensure vmware is stopped.'
raise NotImplemented('ensure_stopped')
@classmethod
@abstractmethod
def host_type(clazz):
'Host type form vmrun authentication.'
raise NotImplemented('host_type')
@classmethod
@abstractmethod
def preferences_filename(clazz):
'The full path to the preferences filename.'
raise NotImplemented('preferences_filename')
@classmethod
@abstractmethod
def inventory_filename(clazz):
'The full path to the inventory filename.'
raise NotImplemented('inventory_filename')
@classmethod
@abstractmethod
def vmrun_exe_path(clazz):
'The full path to the vmrun executable.'
raise NotImplemented('vmrun_exe_path')
class file_metadata_getter_base(with_metaclass(ABCMeta, object)):
@classmethod
@abstractmethod
def name(clazz):
'Return the name of this getter.'
raise NotImplemented('name')
@abstractmethod
def get_value(self, filename):
'Get a metadata value from filename and return it encoded as bytes.'
raise NotImplemented('get_value')
@abstractmethod
def decode_value(self, value):
'Decode a value given as bytes.'
raise NotImplemented('decode_value')
class properties_file_formatter_base(with_metaclass(ABCMeta, object)):
@abstractmethod
def delimiter(self):
raise NotImplemented('delimiter')
@abstractmethod
def parse_value(self, key, value):
raise NotImplemented('parse_value')
@abstractmethod
def value_to_text(self, key, value):
raise NotImplemented('value_to_text')
@abstractmethod
def key_value_to_text(self, key, value):
raise NotImplemented('key_value_to_text')
class criteria(with_metaclass(ABCMeta, object)):
'criteria for finding files.'
FILTER = 1
STOP = 2
VALID_ACTIONS = [FILTER, STOP]
FILE = 0x1
DIR = 0x2
ANY = FILE | DIR
VALID_TARGETS = [FILE, DIR, ANY]
def __init__(self, action=FILTER, target=ANY):
self.action = self.check_action(action)
self.target = self.check_target(target)
@abstractmethod
def matches(self, variables):
pass
def targets_files(self):
return (self.target & self.FILE) != 0
def targets_dirs(self):
return (self.target & self.DIR) != 0
@classmethod
def action_is_valid(clazz, action):
return action in clazz.VALID_ACTIONS
@classmethod
def check_action(clazz, action):
if not clazz.action_is_valid(action):
raise ValueError('invalid action: %s' % (str(action)))
return action
@classmethod
def target_is_valid(clazz, target):
return target in clazz.VALID_TARGETS
@classmethod
def check_target(clazz, target):
if not clazz.target_is_valid(target):
raise ValueError('invalid target: %s' % (str(target)))
return target
class log_writer(with_metaclass(ABCMeta, object)):
'Abstract base class for writing logs.'
@abstractmethod
def write(self, text):
'same as file.write.'
raise NotImplementedError('write')
@abstractmethod
def close(self):
'same as file.close.'
raise NotImplementedError('close')
@abstractmethod
def flush(self):
'same as file.flush.'
raise NotImplementedError('flush')
class cli_options_base(with_metaclass(ABCMeta, object)):
@classmethod
@abstractmethod
def default_values(clazz):
'Return a dict of default values for these options.'
raise NotImplemented('default_values')
@classmethod
@abstractmethod
def sensitive_keys(clazz):
'Return a tuple of keys that are secrets and should be protected from __str__.'
raise NotImplemented('sensitive_keys')
@classmethod
@abstractmethod
def value_type_hints(clazz):
raise NotImplemented('morph_value_types')
@classmethod
@abstractmethod
def config_file_key(clazz):
raise NotImplemented('config_file_key')
@classmethod
@abstractmethod
def config_file_env_var_name(clazz):
raise NotImplemented('config_file_env_var_name')
@classmethod
@abstractmethod
def config_file_section(clazz):
raise NotImplemented('config_file_section')
@classmethod
@abstractmethod
def error_class(clazz):
raise NotImplemented('error_class')
@abstractmethod
def check_value_types(self):
'Check the type of each option.'
raise NotImplemented('check_value_types')
@classmethod
def ignore_config_file_variables(clazz):
return False
class vmware_command_interpreter_base(with_metaclass(ABCMeta, object)):
'Abstract interface for dealing with command interpreters in vms.'
@classmethod
@abstractmethod
def interpreters(clazz):
'Return a tuple of all the available interpreters in the system'
raise NotImplemented('interpreters')
@classmethod
@abstractmethod
def default_interpreter(clazz):
'Return the default interpreter'
raise NotImplemented('default_interpreter')
@classmethod
@abstractmethod
def interpreter_path(clazz, name):
'Return the full path to the named interpreter'
raise NotImplemented('interpreter_path')
command = namedtuple('command', 'interpreter_path, script_text')
@classmethod
@abstractmethod
def build_command(clazz, name, script_text):
'Build a command and return a command object for it'
raise NotImplemented('build_command')
@classmethod
def interpreter_is_valid(clazz, name):
'Return True if this system interpreter is valid.'
check.check_string(name)
return name in clazz.interpreters()
@classmethod
def check_interpreter(clazz, name):
'Raise an exception if the interpreter is not valid.'
check.check_string(name)
if not clazz.interpreter_is_valid(name):
raise vmware_error(
'Invalid interpreter: "{}" Should be one of: {}"'.format(
name, ' '.join(clazz.interpreters())))
class library_base(with_metaclass(ABCMeta, object)):
def __init__(self):
pass
@abstractmethod
def is_shared_library(self, filename):
'Return True if filename is a shared library.'
pass
@abstractmethod
def is_static_library(self, filename):
'Return True if filename is a shared library.'
pass
@abstractmethod
def dependencies(self, filename):
'Return a list of dependencies for filename (executable or shared lib) or None if not applicable.'
pass
@abstractmethod
def shared_extension(self):
'Return the shared library extension.'
pass
@abstractmethod
def static_extension(self):
'Return the static library extension.'
pass
@abstractmethod
def shared_prefix(self):
'Return the shared library prefix.'
pass
@abstractmethod
def static_prefix(self):
'Return the static library prefix.'
pass
@abstractmethod
def binary_format_name(self):
'The name of the binary format (usually elf or macho).'
pass
class file_cache_item_base(with_metaclass(ABCMeta, object)):
@abstractmethod
def __init__(self):
pass
@abstractmethod
def save(self, info):
assert False, 'not implemented'
@abstractmethod
def load(self, cached_filename):
assert False, 'not implemented'
@abstractmethod
def name(self):
assert False, 'not implemented'
@abstractmethod
def checksum(self):
assert False, 'not implemented'
class something_base(with_metaclass(ABCMeta, object)):
def __init__(self):
pass
@abstractmethod
def creator(self):
'Return the creator name.'
pass
@abstractmethod
def suck_level(self):
'Return a number between 0 and 10 indicating how much this something sucks.'
pass
# @abstractmethod
def caca(self):
'Return a number between 0 and 10 indicating how much this something sucks.'
pass
class dependency_provider(with_metaclass(ABCMeta, object)):
def __init__(self):
pass
@abstractmethod
def provided(self):
'Return a list of dependencies provided by this provider.'
pass
@classmethod
def determine_provided(clazz, o):
'Determine the list of dependencies provided by o if it is a single or list of dependency provider(s).'
if check.is_dependency_provider(o):
return o.provided()
elif check.is_dependency_provider_seq(o):
#assert False
result = []
for item in o:
result.extend(item.provided())
return result
else:
return []
class native_package_manager_base(with_metaclass(ABCMeta, object)):
def __init__(self):
pass
@abstractmethod
def installed_packages(self, interface):
'Return a list of packages on this computer.'
pass
@abstractmethod
def package_files(self, package_name):
'Return a list of installed files for the given package.'
pass
@abstractmethod
def package_dirs(self, package_name):
'Return a list of installed dirs for the given package.'
pass
@abstractmethod
def package_contents(self, package_name):
'Return a list of contents for the given package.'
pass
@abstractmethod
def is_installed(self, package_name):
'Return True if package is installed.'
pass
@abstractmethod
def owner(self, filename):
'Return the package that owns filename.'
pass
@abstractmethod
def package_info(self, filename):
'Return info structure about the package.'
pass
class platform_determiner_base(with_metaclass(ABCMeta, object)):
'Abstract base class for determining what platform we are on.'
@abstractmethod
def system(self):
'system.'
pass
@abstractmethod
def distro(self):
'distro.'
pass
@abstractmethod
def codename(self):
'codename.'
pass
@abstractmethod
def family(self):
'distro family.'
pass
@abstractmethod
def version_major(self):
'distro version major number.'
pass
@abstractmethod
def version_minor(self):
'distro version minor number.'
pass
@abstractmethod
def arch(self):
'arch.'
pass
class python_installer_base(with_metaclass(ABCMeta, object)):
_log = logger('python_installer')
def __init__(self, options):
check.check_python_installer_options(options)
self.options = options
@abstractmethod
def available_versions(self, num):
'Return a list of python versions available to install.'
raise NotImplemented('available_versions')
@abstractmethod
def installed_versions(self):
'Return a list of installed python versions.'
raise NotImplemented('installed_versions')
@abstractmethod
def install(self, version):
'Install the major.minor.revision or major.minor version of python.'
raise NotImplemented('install')
@abstractmethod
def update(self, version):
'Update to the latest major.minor version of python.'
raise NotImplemented('update')
@abstractmethod
def needs_update(self, version):
'Return True if python version major.minor needs update.'
raise NotImplemented('needs_update')
@abstractmethod
def install_package(self, package_filename):
'Install a python package directly. Not always supported.'
raise NotImplemented('install_package')
@abstractmethod
def uninstall(self, full_version):
'Uninstall the major.minor.revision full version of python.'
raise NotImplemented('uninstall_full_version')
@abstractmethod
def download(self, full_version):
'Download the major.minor.revision full version of python to a temporary file.'
raise NotImplemented('download')
@abstractmethod
def supports_full_version(self):
'Return True if this installer supports installing by full version.'
raise NotImplemented('supports_full_version')
def blurb(self, message, output = None, fit = False):
'Print a blurb'
self.options.blurber.blurb(message, output = output, fit = fit)
def blurb_verbose(self, message, output = None, fit = False):
'Print a blurb but only in verbose mode'
self.options.blurber.blurb_verbose(message, output = output, fit = fit)
def installed_versions_matching(self, version):
'Return installed versions matching version only.'
version = python_version.check_version_or_full_version(version)
installed_versions = self.installed_versions()
self._log.log_d('installed_versions_matching: installed_versions: {}'.format(installed_versions.to_string()))
matching_versions = installed_versions.filter_by_version(version)
matching_versions.sort()
self._log.log_d('installed_versions_matching: matching_versions: {}'.format(matching_versions.to_string()))
return matching_versions
def is_installed(self, version):
'Return True if a python matching version is installed.'
version = python_version.check_version_or_full_version(version)
self._log.log_d('is_installed: version={}'.format(version))
matching_versions = self.installed_versions_matching(version)
self._log.log_d('is_installed: matching_versions={}'.format(matching_versions))
result = len(matching_versions) > 0
self._log.log_d('is_installed: result={}'.format(result))
return result
class step(with_metaclass(step_register_meta, object)):
result = step_result
def __init__(self):
self._recipe = None
self._args = {}
self._values = None
@property
def values(self):
if self._values is None:
raise ValueError('values have not been set yet.')
return self._values
@values.setter
def values(self, values):
check.check_dict(values)
if self._values is not None:
raise ValueError('values can only be set once.')
self._values = values
@classmethod
def args_definition(clazz):
if not hasattr(clazz, '_args_definition'):
setattr(clazz, '_args_definition', clazz._determine_args_definition())
return getattr(clazz, '_args_definition')
@classmethod
def _determine_args_definition(clazz):
defs = clazz.define_args()
if check.is_string(defs):
try:
defs = value_definition.parse_many(defs)
except RuntimeError as ex:
filename = inspect.getfile(clazz.define_args)
line_number = inspect.getsourcelines(clazz.define_args)[1]
raise RuntimeError('%s: %s:%s' % (ex.message, filename, line_number))
check.check_dict(defs)
return defs
@abstractmethod
def execute(self, script, env, values, inputs):
'Execute the step.'
pass
def on_tag_changed(self):
'Called when the tag changes.'
pass
@classmethod
@abstractmethod
def define_args(clazz):
'Return a list of arg specs.'
return {}
def update_args(self, args):
dict_util.update(self.args, args)
@property
def recipe(self):
assert self._recipe != None
return self._recipe
@recipe.setter
def recipe(self, recipe):
assert recipe != None
assert self._recipe == None
self._recipe = recipe
@property
def tag(self):
return self.bes_log_tag__
@tag.setter
def tag(self, tag):
log.add_logging(self, tag)
build_blurb.add_blurb(self, tag)
self.on_tag_changed()
@property
def args(self):
return self._args
@args.setter
def args(self, args):
if not isinstance(args, dict):
raise RuntimeError('args needs to be a dict')
self._args = args
@classmethod
def create_command_env(clazz, script):
env = os_env.make_clean_env(keep_keys = [ 'PYTHONPATH' ])
STATIC = True
export_compilation_flags_requirements = clazz.export_compilation_flags_requirements(script, script.build_target.system)
if STATIC:
env['REBBE_PKG_CONFIG_STATIC'] = '1'
if export_compilation_flags_requirements:
cflags, libs = script.requirements_manager.compilation_flags(export_compilation_flags_requirements, static = STATIC)
else:
cflags = []
libs = []
cflags += script.descriptor.extra_cflags(script.build_target.system)
env['REBUILD_REQUIREMENTS_CFLAGS'] = ' '.join(cflags)
env['REBUILD_REQUIREMENTS_CXXFLAGS'] = ' '.join(cflags)
env['REBUILD_REQUIREMENTS_LDFLAGS'] = ' '.join(libs)
env.update(script.substitutions)
tool_deps = script.resolve_deps(['TOOL'], False)
run_build_deps = script.resolve_deps(['RUN', 'BUILD'], False)
env = script.env.tools_manager.transform_env(env, tool_deps)
env = script.requirements_manager.transform_env(env, run_build_deps.names())
staged_files_env = os_env.make_shell_env(script.staged_files_dir)
os_env.update(env, staged_files_env)
env['REBUILD_PYTHON_VERSION'] = '2.7'
env['PYTHON'] = 'python%s' % (env['REBUILD_PYTHON_VERSION'])
tc = toolchain.get_toolchain(script.build_target)
compiler_flags = tc.compiler_flags()
env['REBUILD_COMPILE_CFLAGS'] = ' '.join(compiler_flags.get('CFLAGS', []))
env['REBUILD_COMPILE_LDFLAGS'] = ' '.join(compiler_flags.get('LDFLAGS', []))
env['REBUILD_COMPILE_CXXFLAGS'] = ' '.join(compiler_flags.get('CXXFLAGS', []))
env['REBUILD_COMPILE_ARCH_FLAGS'] = ' '.join(compiler_flags.get('REBUILD_COMPILE_ARCH_FLAGS', []))
env['REBUILD_COMPILE_OPT_FLAGS'] = ' '.join(compiler_flags.get('REBUILD_COMPILE_OPT_FLAGS', []))
ce = tc.compiler_environment()
os_env.update(env, ce)
env['REBUILD_COMPILE_ENVIRONMENT'] = toolchain.flatten_for_shell(ce)
return env
@classmethod
def env_dump(clazz, env, package_name, label):
for key, value in sorted(env.items()):
build_blurb.blurb_verbose('rebuild', '%s(%s): %s=%s' % (label, package_name, key, value))
@classmethod
def call_shell(clazz, command, script, env, shell_env = None, save_logs = None, execution_dir = None):
command = execute.listify_command(command)
command = [ part for part in command if part ]
shell_env = shell_env or key_value_list()
save_logs = save_logs or []
check.check_key_value_list(shell_env)
log.log_i('rebuild', 'call_shell(command=%s)' % (command))
build_blurb.blurb_verbose('rebuild', 'call_shell(cmd=%s, shell_env=%s)' % (command, shell_env))
env = clazz.create_command_env(script)
env.update(shell_env)
#clazz.env_dump(env, script.descriptor.name, 'PRE ENVIRONMENT')
clazz._env_substitite(env)
rogue_key = clazz._env_find_roque_dollar_sign(env)
if rogue_key:
raise RuntimeError('Rogue dollar sign (\"$\") found in %s: %s' % (rogue_key, env[rogue_key]))
command = [ variable.substitute(part, env) for part in object_util.listify(command) ]
# Check that no rogue dollar signs are in the command
for part in command:
if variable.has_rogue_dollar_signs(part):
raise RuntimeError('Rogue dollar sign (\"$\") found in: %s' % (part))
build_blurb.blurb('rebuild', '%s - %s' % (script.descriptor.name, ' '.join(command)))
# file_util.mkdir(script.build_dir)
retry_script = clazz._write_retry_script(command, env, script)
#clazz.env_dump(env, script.descriptor.name, clazz.__name__ + ' : ' + 'POST ENVIRONMENT')
for k,v in env.items():
if string_util.is_quoted(v):
env[k] = string_util.unquote(v)
if execution_dir:
cwd = path.join(script.build_dir, execution_dir)
else:
cwd = script.build_dir
rv = execute.execute(command,
cwd = cwd,
env = env,
shell = True,
non_blocking = build_blurb.verbose,
stderr_to_stdout = build_blurb.verbose,
raise_error = False)
message = rv.stdout
if rv.stderr:
message = message + '\n' + rv.stderr
result = step_result(rv.exit_code == 0, message)
clazz.save_build_dir_logs(script, save_logs)
return result
RETRY_SCRIPT_FILENAME = 'rebbe_retry.sh'
@classmethod
def _write_retry_script(clazz, command, env, script):
from bes.compat import StringIO
s = StringIO()
s.write('#!/bin/bash\n')
s.write('mkdir -p %s\n' % (script.staged_files_dir))
items = sorted(env.items())
last_item = items.pop(-1)
def _item_str(key, value, slash):
return '%s=\"%s\"%s\n' % (key, value, slash)
for key, value in items:
s.write(_item_str(key, value, '\\'))
s.write(_item_str(last_item[0], last_item[1], ''))
if string_util.is_string(command):
s.write(command)
else:
s.write(' '.join(command))
content = s.getvalue()
file_path = path.join(script.build_dir, clazz.RETRY_SCRIPT_FILENAME)
file_util.save(file_path, content = content, mode = 0o755)
return file_path
@classmethod
def call_hooks(clazz, hooks, script, env):
check.check_value_hook_list(hooks)
for hook in hooks:
rv = hook.execute(script, env)
check.is_hook_result(rv)
if not rv.success:
return step_result(rv.success, rv.message)
return step_result(True, None)
@classmethod
def save_build_dir_logs(clazz, env, filenames):
'Save any logs that exists to the logs/ dir'
for filename in filenames:
src = path.join(env.build_dir, filename)
dst = path.join(env.logs_dir, path.basename(src))
if path.isfile(src):
file_util.copy(src, dst)
@classmethod
def export_compilation_flags_requirements(clazz, script, system):
config = script.descriptor.properties.get('export_compilation_flags_requirements', [])
if config:
if check.is_masked_value_list(config):
resolved = config.resolve(system, value_type.STRING_LIST)
else:
raise RuntimeError('not a valid masked_value_list: %s' % (config))
else:
resolved = []
requirements = script.descriptor.requirements.filter_by_hardness(['RUN', 'BUILD']).filter_by_system(system)
deps_names = requirements.names()
export_names = resolved
if export_names == dependency_resolver.ALL_DEPS:
export_names = deps_names
delta = (set(export_names) - set(deps_names))
if delta:
raise RuntimeError('Trying to export deps that are not specified by %s: %s' % (script.descriptor.name, ' '.join(delta)))
return export_names
@classmethod
def _env_find_roque_dollar_sign(clazz, env):
for key in sorted(env.keys()):
if variable.has_rogue_dollar_signs(env[key]):
return key
return None
@classmethod
def _env_substitite(clazz, env):
for key in sorted(env.keys()):
env[key] = variable.substitute(str(env[key]), env)
class file_checksum_getter_base(with_metaclass(ABCMeta, object)):
@abstractmethod
def checksum(self, algorithm, filename):
'Return the checksum for filename using algorithm.'
pass
class system_command(with_metaclass(ABCMeta, object)):
'Abstract base class for dealing with system commands.'
_log = logger('system_command')
@classmethod
@abstractmethod
def exe_name(clazz):
'The name of the executable.'
raise NotImplemented('exe_name')
@classmethod
@abstractmethod
def extra_path(clazz):
'List of extra paths where to find the command.'
raise NotImplemented('extra_path')
@classmethod
@abstractmethod
def error_class(clazz):
'The error exception class to raise when errors happen.'
raise NotImplemented('error_class')
@classmethod
@abstractmethod
def static_args(clazz):
'List of static arg for all calls of the command.'
raise NotImplemented('static_args')
@classmethod
@abstractmethod
def supported_systems(clazz):
'Return a list of supported systems.'
raise NotImplemented('supported_systems')
@classmethod
def _find_exe(clazz):
'Find the exe'
extra_path = clazz.extra_path()
exe_name = clazz.exe_name()
if path.isabs(exe_name):
exe = exe_name
else:
exe = which.which(exe_name, extra_path = clazz.extra_path())
if exe:
return exe
error_class = clazz.error_class()
if not isinstance(error_class, Exception.__class__):
raise TypeError('Return value of error_clas() should be an Exception type: {} - {}'.format(error_class,
type(error_class)))
raise error_class('Failed to find {}'.format(exe_name))
@classmethod
def call_command(clazz,
args,
raise_error = True,
env = None,
use_sudo = False,
stderr_to_stdout = False,
check_python_script = True,
input_data = None,
non_blocking = False,
output_encoding = None,
output_function = None):
'Call the command'
check.check_string_seq(args)
check.check_bool(raise_error)
check.check_dict(env, check.STRING_TYPES, check.STRING_TYPES, allow_none = True)
check.check_bool(use_sudo)
check.check_bytes(input_data, allow_none = True)
check.check_bool(non_blocking)
check.check_string(output_encoding, allow_none = True)
check.check_callable(output_function, allow_none = True)
clazz.check_supported()
clazz._log.log_d('call_command: args={}'.format(' '.join(args)))
if isinstance(args, ( list, tuple )):
parsed_args = list(args)
elif isinstance(args, compat.STRING_TYPES):
parsed_args = command_line.parse_args(args)
else:
raise TypeError('Invalid args type. Should be tuple, list or string: {} - {}'.format(args,
type(args)))
clazz._log.log_d('call_command: parsed_args={}'.format(' '.join(parsed_args)))
exe = clazz._find_exe()
static_args = clazz.static_args() or []
if not isinstance(static_args, ( list, tuple )):
raise TypeError('Return value of static_args() should be list or tuple: {} - {}'.format(static_args,
type(static_args)))
cmd = []
if use_sudo:
cmd.append('sudo')
cmd.append(exe)
cmd.extend(list(static_args))
cmd.extend(args)
clazz._log.log_d('call_command: cmd={} env={}'.format(' '.join(cmd), env))
return execute.execute(cmd,
raise_error = raise_error,
env = env,
stderr_to_stdout = stderr_to_stdout,
check_python_script = check_python_script,
input_data = input_data,
non_blocking = non_blocking,
output_encoding = output_encoding)
@classmethod
def call_command_parse_lines(clazz, args, sort = False):
'Call a command that returns a list of lines'
rv = clazz.call_command(args, raise_error = True)
result = clazz.split_lines(rv.stdout)
if sort:
result = sorted(result)
return result
@classmethod
def is_supported(clazz):
'Return True if this command is support on the current system'
return host.SYSTEM in clazz.supported_systems()
@classmethod
def check_supported(clazz):
'Check that the current system supports this command otherwise raise an error'
if clazz.is_supported():
return
raise clazz.error_class('{} is not supported on {} - only {}'.format(clazz.exe_name(),
host.SYSTEM,
' '.join(clazz.supported_systems())))
@classmethod
def has_command(clazz):
'Return True if the command is found'
if not clazz.is_supported():
return False
try:
exe = clazz._find_exe()
return True
except clazz.error_class() as ex:
pass
return False
@classmethod
def split_lines(clazz, text):
lines = text.splitlines()
lines = [ line.strip() for line in lines ]
return [ line for line in lines if line ]
@classmethod
def split_by_white_space(clazz, line):
parts = []
for part in re.split(r'\s+', line):
part = part.strip()
if part:
parts.append(part)
return parts
@classmethod
def check_result(clazz, result, message = None):
'Check that a result is successful or raise an exceptions from it.'
check.check_string(message, allow_none = True)
check.check_execute_result(result)
if result.exit_code == 0:
return
message = message or 'Failed to execute: {}'.format(' '.join(result.command))
outputs = [ o.strip() for o in [ result.stdout, result.stderr ] if o.strip() ]
error_message = '{} - {}'.format(message, ' - '.join(outputs))
raise clazz.error_class()(error_message)
class value_hook(with_metaclass(hook_register_meta, value_base)):
result = hook_result
def __init__(self, origin=None, properties=None):
'Create a new hook.'
super(value_hook, self).__init__(origin, properties=properties)
def __eq__(self, other):
return self.filename == other.filename
#@abstractmethod
def value_to_string(self, quote, include_properties=True):
buf = StringIO()
buf.write(self.__class__.__name__)
self._append_properties_string(buf, include_properties)
return buf.getvalue()
@classmethod
#@abstractmethod
def default_value(clazz, class_name):
'Return the default value to use for this class.'
return value_hook_list()
@property
def filename(self):
filename = getattr(self, '__load_file__', None)
if not filename:
raise RuntimeError('filename not set')
return path.abspath(filename)
#@abstractmethod
def sources(self, recipe_env):
'Return a list of sources this caca provides or None if no sources.'
return [self.filename]
#@abstractmethod
def substitutions_changed(self):
pass
@classmethod
#@abstractmethod
def parse(clazz, origin, value, node):
if origin:
check.check_value_origin(origin)
check.check_node(node)
hook_name, _, rest = string_util.partition_by_white_space(value)
hook_class = hook_registry.get(hook_name)
if not hook_class:
raise TypeError('%s: hook class not found: %s' %
(origin, hook_name))
properties = clazz.parse_properties(rest)
hook_instance = hook_class(origin=origin, properties=properties)
return hook_instance
@classmethod
#@abstractmethod
def resolve(clazz, values, class_name):
check.check_value_hook_seq(values)
assert class_name == value_type.HOOK_LIST
result_hooks = []
for value in values:
check.check_value_hook(value)
result_hooks.append(value)
result = value_hook_list(values=result_hooks)
result.remove_dups()
return result
@abstractmethod
def execute(self, script, env):
'Execute the hook. Same semantics as step.execute.'
pass
class deleter(with_metaclass(ABCMeta, object)):
@abstractmethod
def delete_file(self, filename):
'Delete a file or directory.'
raise NotImplementedError('not implemented')
