def cmake_high(
building_site, options, arguments, environment, environment_mode, source_subdir, build_in_separate_dir, log=None
):
building_site = wayround_org.utils.path.abspath(building_site)
own_log = False
if log is None:
own_log = True
log = wayround_org.utils.log.Log(wayround_org.aipsetup.build.getDIR_BUILD_LOGS(building_site), "cmake")
env = wayround_org.utils.osutils.env_vars_edit(environment, environment_mode)
if len(environment) > 0:
log.info("Environment modifications:\n{}".format(pprint.pformat(environment)))
script_path = determine_abs_configure_dir(building_site, source_subdir)
working_dir = determine_building_dir(building_site, source_subdir, build_in_separate_dir)
ret = cmake_low(
log=log,
build_dir=working_dir,
src_dir=script_path,
working_dir=working_dir,
opts=options,
args=arguments,
env=env,
)
if own_log:
log.close()
return ret
def builder_action_bld_cleanup(self, called_as, log):
"""
Standard building dir cleanup
"""
if os.path.isdir(self.get_bld_dir()):
log.info("cleaningup building dir")
wayround_org.utils.file.cleanup_dir(self.get_bld_dir())
return 0
def builder_action_dst_cleanup(self, called_as, log):
"""
Standard destdir cleanup
"""
if os.path.isdir(self.get_dst_dir()):
log.info("cleaningup destination dir")
wayround_org.utils.file.cleanup_dir(self.get_dst_dir())
return 0
def builder_action_src_cleanup(self, called_as, log):
"""
Standard sources cleanup
"""
if os.path.isdir(self.get_src_dir()):
log.info("cleaningup source dir")
wayround_org.utils.file.cleanup_dir(self.get_src_dir())
return 0
def cmake_low(log, build_dir, src_dir, working_dir, opts, args, env):
ret = 0
cmd = ["cmake"] + opts + ["--build=" + build_dir] + args + [src_dir]
log.info("directory: {}".format(working_dir))
log.info("src: {}".format(src_dir))
log.info("build dir: {}".format(build_dir))
log.info("command:")
for i in cmd:
log.info(" {}".format(i))
# log.info("command(joined): {}".format(' '.join(cmd)))
p = None
try:
p = subprocess.Popen(cmd, env=env, stdout=log.stdout, stderr=log.stderr, cwd=working_dir)
except:
log.error(
"exception while starting cmake\n"
" command line was:\n"
" " + repr(cmd) + wayround_org.utils.error.return_exception_info(sys.exc_info())
)
ret = 100
else:
try:
p.wait()
except:
log.error(
"Exception occurred while waiting for cmake\n{}".format(
wayround_org.utils.error.return_exception_info(sys.exc_info())
)
)
ret = 100
else:
tmp_s = "cmake return code was: {}".format(p.returncode)
if p.returncode == 0:
log.info(tmp_s)
else:
log.error(tmp_s)
ret = p.returncode
return ret
def builder_action_configure(self, called_as, log):
log.info(
"crossbuild?: {}, crossbuilder?: {}".format(
self.get_is_crossbuild(),
self.get_is_crossbuilder()
)
)
self.check_deprecated_methods(called_as, log)
envs = {}
if hasattr(self, 'builder_action_configure_define_environment'):
envs = self.builder_action_configure_define_environment(
called_as,
log
)
opts = []
if hasattr(self, 'builder_action_configure_define_opts'):
opts = self.builder_action_configure_define_opts(
called_as,
log
)
args = []
if hasattr(self, 'builder_action_configure_define_args'):
args = self.builder_action_configure_define_args(
called_as,
log
)
ret = autotools.configure_high(
self.buildingsite_path,
log=log,
options=opts,
arguments=args,
environment=envs,
environment_mode='copy',
source_configure_reldir=self.source_configure_reldir,
use_separate_buildding_dir=self.separate_build_dir,
script_name=self.builder_action_configure_define_script_name(
called_as,
log
),
run_script_not_bash=(
self.builder_action_configure_define_run_script_not_bash(
called_as,
log
)
),
relative_call=(
self.builder_action_configure_define_relative_call(
called_as,
log
)
)
)
#sleeptime = 5
#log.info("sleep: \033[0;1m{}\033[0m seconds".format(sleeptime))
# time.sleep(sleeptime)
return ret
def builder_action_autogen(self, called_as, log):
cfg_script_name = self.builder_action_configure_define_script_name(
called_as,
log
)
ret = 0
do_work = False
if os.path.isfile(
wayround_org.utils.path.join(
self.get_src_dir(),
self.source_configure_reldir,
cfg_script_name
)
):
log.info("configurer found. no generator use presumed")
else:
log.info("configurer not found. generator use presumed")
do_work = True
if self.forced_autogen:
log.info(
"generator use is forced".format(
cfg_script_name
)
)
do_work = True
if do_work:
log.info(
"trying to find and use generator mesures"
)
for i in [
('autogen.sh', ['./autogen.sh']),
('bootstrap.sh', ['./bootstrap.sh']),
('bootstrap', ['./bootstrap']),
('genconfig.sh', ['./genconfig.sh']),
('configure.ac', ['autoreconf', '-i']),
('configure.in', ['autoreconf', '-i']),
]:
if os.path.isfile(
wayround_org.utils.path.join(
self.get_src_dir(),
self.source_configure_reldir,
i[0]
)
):
log.info(
"found `{}'. trying to execute: {}".format(
i[0],
' '.join(i[1])
)
)
wd = wayround_org.utils.path.join(
self.get_src_dir(),
self.source_configure_reldir
)
if '/' in i[1][0]:
tgt_file = wayround_org.utils.path.join(wd, i[1][0])
log.info("changing mode (+x) for: {}".format(tgt_file))
chmod_p = subprocess.Popen(
['chmod', '+x', tgt_file],
cwd=wd
)
chmod_p.wait()
if i[1][0].endswith('.sh'):
i[1].insert(0, 'bash')
p = subprocess.Popen(
i[1],
cwd=wd,
stdout=log.stdout,
stderr=log.stderr
)
ret = p.wait()
break
else:
log.error(
"./{} not found and no generators found".format(
cfg_script_name
)
)
ret = 2
return ret
def clean_find_so_problems(command_name, opts, args, adds):
"""
Find so libraries missing in system and write package names requiring those
missing libraries.
"""
import wayround_org.aipsetup.controllers
config = adds['config']
ret = 0
basedir = '/'
pkg_client = \
wayround_org.aipsetup.controllers.pkg_client_by_config(
config
)
system = wayround_org.aipsetup.controllers.sys_ctl_by_config(
config, pkg_client, basedir
)
problems = system.find_so_problems_in_system(
verbose=True
)
libs = sorted(problems.keys())
log = wayround_org.utils.log.Log(
os.getcwd(), 'problems'
)
print("Writing log to {}".format(log.log_filename))
logging.info("Gathering asps file tree. Please wait...")
tree = system.list_installed_asps_and_their_files(mute=False)
logging.info("Now working")
total_problem_packages_list = set()
count_checked = 0
libs_c = len(libs)
for i in libs:
log.info("Library `{}' required by following files:".format(i))
files = problems[i]
files.sort()
for j in files:
log.info(" {}".format(j))
pkgs2 = system.find_file_in_files_installed_by_asps(
files, mode='end', mute=False, predefined_asp_tree=tree
)
pkgs2_l = sorted(pkgs2.keys())
count_checked += 1
log.info(" Contained in problem packages:")
for j in pkgs2_l:
log.info(" {}".format(j))
total_problem_packages_list |= set(pkgs2_l)
logging.info(
"Checked libraries: {} of {}".format(count_checked, libs_c)
)
log.info('---------------------------------')
pkgs = system.find_file_in_files_installed_by_asps(
libs, mode='end', mute=False, predefined_asp_tree=tree
)
pkgs_l = list(pkgs.keys())
pkgs_l.sort()
log.info('')
log.info("Libs found in packages:")
for i in pkgs_l:
log.info(" {}".format(i))
log.info('')
log.info("Total Problem Packages List:")
total_problem_packages_list = list(total_problem_packages_list)
total_problem_packages_list.sort()
for i in total_problem_packages_list:
log.info(" {}".format(i))
log.stop()
print("Log written to {}".format(log.log_filename))
return ret
def clean_packages_with_broken_files(command_name, opts, args, adds):
"""
Find packages with broken files
"""
import wayround_org.aipsetup.controllers
config = adds['config']
pkg_client = \
wayround_org.aipsetup.controllers.pkg_client_by_config(
config
)
system = wayround_org.aipsetup.controllers.sys_ctl_by_config(
config,
pkg_client,
basedir='/'
)
r = system.list_installed_asps_and_their_sums(mute=False)
logging.info("Checking Packages")
asps = list(r.keys())
asps_c = len(asps)
problems = {}
b = 0
m = 0
for i in range(asps_c):
asp_name = asps[i]
asp = r[asp_name]
if isinstance(asp, dict):
problems[asp_name] = {'missing': [], 'broken': []}
files = list(asp.keys())
fc = len(files)
fi = 0
perc = 0
if i != 0:
perc = (100.0 / (asps_c / i))
for j in files:
if not os.path.exists(j):
problems[asp_name]['missing'].append(j)
m += 1
else:
_sum = wayround_org.utils.checksum.make_file_checksum(
j, method='sha512'
)
if _sum != asp[j]:
problems[asp_name]['broken'].append(j)
b += 1
fi += 1
wayround_org.utils.terminal.progress_write(
" ({perc:5.2f}%) {p} packages of {pc}, {f} files of "
"{fc}. found {b} broken, {m} missing".format(
perc=perc,
p=i,
pc=asps_c,
f=fi,
fc=fc,
m=m,
b=b
)
)
for i in list(problems.keys()):
if (len(
problems[i]['missing']
) == 0
and len(problems[i]['broken']) == 0):
del problems[i]
print()
log = wayround_org.utils.log.Log(
os.getcwd(), 'problems'
)
log.info(pprint.pformat(problems))
log_name = log.log_filename
log.close()
logging.info("Log saved to {}".format(log_name))
return 0
def clean_find_garbage(command_name, opts, args, adds):
"""
Search system for garbage making log and cleaning script
[-h=all|cpu-vend-os-triplet] [-a=all|cpu-vend-os-triplet]
-b=BASENAME - system root path
--script-type=bash - system cleaning script language (only bash supported)
--so - look only for .so files garbage in /usr/lib directory
"""
import wayround_org.aipsetup.controllers
import wayround_org.aipsetup.package_name_parser
import wayround_org.aipsetup.client_pkg
import wayround_org.aipsetup.system
config = adds['config']
ret = 0
if wayround_org.utils.getopt.check_options(
opts,
opts_list=[
'-b=',
'--script-type=',
'--so',
'-h=',
'-a='
]
) != 0:
ret = 1
if ret == 0:
timestamp = wayround_org.utils.time.currenttime_stamp()
basedir = '/'
script = 'system_garbage_remove_{}.sh'.format(timestamp)
script_type = 'bash'
only_lib = False
down_script = 'get_required_sources_{}.sh'.format(timestamp)
if '-b' in opts:
basedir = opts['-b']
if '--script-type' in opts:
script_type = opts['--script-type']
only_lib = '--so' in opts
host, arch = _process_h_and_a_opts_specific(opts, config)
log = wayround_org.utils.log.Log(
os.getcwd(), 'system_garbage', timestamp=timestamp
)
if not script_type in ['bash']:
logging.error("Invalid --script-type value")
ret = 1
else:
pkg_client = \
wayround_org.aipsetup.controllers.pkg_client_by_config(
config
)
system = wayround_org.aipsetup.controllers.sys_ctl_by_config(
config,
pkg_client,
basedir=basedir
)
log.info("Searching for garbage")
res = system.find_system_garbage(
mute=False,
only_lib=only_lib,
host=host
)
if not isinstance(res, list):
log.error("Some error while searching for garbage")
ret = 1
else:
log.info("Garbage search complete")
log.info(
"Separating garbage .so files to know "
"which packages depending on them"
)
libs = wayround_org.utils.path.exclude_files_not_in_dirs(
res,
system.library_paths(host=host),
)
libs = wayround_org.aipsetup.system.filter_so_files(
libs,
verbose=True
)
if only_lib:
res = libs
libs = wayround_org.utils.path.bases(libs)
asp_deps = system.load_asp_deps_all(mute=False)
asps_lkd_to_garbage = {}
log.info("Calculating garbage dependencies")
for asp_name in list(asp_deps.keys()):
if not asp_name in asps_lkd_to_garbage:
asps_lkd_to_garbage[asp_name] = dict()
for file_name in list(asp_deps[asp_name].keys()):
file_name_with_dest_dir = \
wayround_org.utils.path.insert_base(
file_name, basedir
)
if (not file_name_with_dest_dir
in asps_lkd_to_garbage[asp_name]):
asps_lkd_to_garbage[
asp_name
][file_name_with_dest_dir] = set()
asps_lkd_to_garbage[
asp_name
][file_name_with_dest_dir] |= \
(set(libs) & set(asp_deps[asp_name][file_name]))
if len(
asps_lkd_to_garbage[
asp_name][file_name_with_dest_dir]
) == 0:
del asps_lkd_to_garbage[
asp_name][file_name_with_dest_dir]
if len(asps_lkd_to_garbage[asp_name]) == 0:
del asps_lkd_to_garbage[asp_name]
s = open(script, 'w')
s.write("""\
#!/bin/bash
# This is fuse to ensure You are know what You are doing
exit 1
""")
log.info("Writing report and cleaning script")
res.sort()
for i in res:
try:
log.info(" {}".format(i), echo=False)
except:
log.error("Error logging {}".format(repr(i)))
try:
s.write("rm {}\n".format(shlex.quote(i)))
except:
log.error("Error writing {}".format(repr(i)))
log.info(
"Packages linked to garbage libraries:\n{}".format(
pprint.pformat(asps_lkd_to_garbage)
),
echo=False
)
log.info("Generating download script")
required_packages = set()
for i in list(asps_lkd_to_garbage.keys()):
p = wayround_org.aipsetup.package_name_parser.\
package_name_parse(i)
if not p:
log.error(
"Can't parse ASP name `{}' to add it to download script".format(
i)
)
else:
required_packages.add(p['groups']['name'])
log.info("Writing download script")
ds = open(down_script, 'w')
ds.write(
"""\
#!/bin/bash
aipsetup3 src getl {}
""".format(' '.join(required_packages))
)
ds.close()
s.close()
logging.warning("""
Do not run cleaning script at once!
Check everything is correct!
Wrong cleaning can ruin your system
"""
)
log.close()
return ret
def waf(
cwd,
options,
arguments,
environment,
environment_mode,
log
):
ret = 0
cwd = wayround_org.utils.path.abspath(cwd)
env = wayround_org.utils.osutils.env_vars_edit(
environment,
environment_mode
)
if len(environment) > 0:
log.info(
"Environment modifications: {}".format(
repr(environment)
)
)
if 'PYTHON' in environment:
python = environment['PYTHON']
else:
if 'PATH' in environment:
PATH = environment['PATH']
else:
PATH = os.environ['PATH']
PATH = PATH.split(':')
python = wayround_org.utils.file.which('python2', PATH)
del PATH
cmd = [
python,
wayround_org.utils.path.join(cwd, 'waf')
] + options + arguments
log.info("directory: {}".format(cwd))
log.info("command: {}".format(cmd))
log.info("command(joined): {}".format(' '.join(cmd)))
p = None
try:
p = subprocess.Popen(
args=cmd,
cwd=cwd,
stdout=log.stdout,
stderr=log.stderr,
env=env
)
except:
log.error(
"exception while starting waf script\n"
" command line was:\n"
" " + repr(cmd) +
wayround_org.utils.error.return_exception_info(
sys.exc_info()
)
)
ret = 100
else:
try:
p.wait()
except:
log.error(
"Exception occurred while waiting for configure\n" +
wayround_org.utils.error.return_exception_info(
sys.exc_info()
)
)
ret = 100
else:
tmp_s = "configure return code was: {}".format(p.returncode)
if p.returncode == 0:
log.info(tmp_s)
else:
log.error(tmp_s)
ret = p.returncode
return ret