예제 #1
0
def cmake_high(
    building_site, options, arguments, environment, environment_mode, source_subdir, build_in_separate_dir, log=None
):

    building_site = wayround_org.utils.path.abspath(building_site)

    own_log = False
    if log is None:
        own_log = True
        log = wayround_org.utils.log.Log(wayround_org.aipsetup.build.getDIR_BUILD_LOGS(building_site), "cmake")

    env = wayround_org.utils.osutils.env_vars_edit(environment, environment_mode)

    if len(environment) > 0:
        log.info("Environment modifications:\n{}".format(pprint.pformat(environment)))

    script_path = determine_abs_configure_dir(building_site, source_subdir)

    working_dir = determine_building_dir(building_site, source_subdir, build_in_separate_dir)

    ret = cmake_low(
        log=log,
        build_dir=working_dir,
        src_dir=script_path,
        working_dir=working_dir,
        opts=options,
        args=arguments,
        env=env,
    )

    if own_log:
        log.close()

    return ret
예제 #2
0
    def builder_action_bld_cleanup(self, called_as, log):
        """
        Standard building dir cleanup
        """

        if os.path.isdir(self.get_bld_dir()):
            log.info("cleaningup building dir")
            wayround_org.utils.file.cleanup_dir(self.get_bld_dir())

        return 0
예제 #3
0
    def builder_action_dst_cleanup(self, called_as, log):
        """
        Standard destdir cleanup
        """

        if os.path.isdir(self.get_dst_dir()):
            log.info("cleaningup destination dir")
            wayround_org.utils.file.cleanup_dir(self.get_dst_dir())

        return 0
예제 #4
0
    def builder_action_src_cleanup(self, called_as, log):
        """
        Standard sources cleanup
        """

        if os.path.isdir(self.get_src_dir()):
            log.info("cleaningup source dir")
            wayround_org.utils.file.cleanup_dir(self.get_src_dir())

        return 0
예제 #5
0
def cmake_low(log, build_dir, src_dir, working_dir, opts, args, env):

    ret = 0

    cmd = ["cmake"] + opts + ["--build=" + build_dir] + args + [src_dir]

    log.info("directory: {}".format(working_dir))
    log.info("src: {}".format(src_dir))
    log.info("build dir: {}".format(build_dir))
    log.info("command:")
    for i in cmd:
        log.info("    {}".format(i))
    # log.info("command(joined): {}".format(' '.join(cmd)))

    p = None
    try:
        p = subprocess.Popen(cmd, env=env, stdout=log.stdout, stderr=log.stderr, cwd=working_dir)
    except:
        log.error(
            "exception while starting cmake\n"
            "    command line was:\n"
            "    " + repr(cmd) + wayround_org.utils.error.return_exception_info(sys.exc_info())
        )
        ret = 100

    else:

        try:
            p.wait()
        except:
            log.error(
                "Exception occurred while waiting for cmake\n{}".format(
                    wayround_org.utils.error.return_exception_info(sys.exc_info())
                )
            )
            ret = 100
        else:
            tmp_s = "cmake return code was: {}".format(p.returncode)

            if p.returncode == 0:
                log.info(tmp_s)
            else:
                log.error(tmp_s)

            ret = p.returncode

    return ret
예제 #6
0
    def builder_action_configure(self, called_as, log):

        log.info(
            "crossbuild?: {}, crossbuilder?: {}".format(
                self.get_is_crossbuild(),
                self.get_is_crossbuilder()
                )
            )

        self.check_deprecated_methods(called_as, log)

        envs = {}
        if hasattr(self, 'builder_action_configure_define_environment'):
            envs = self.builder_action_configure_define_environment(
                called_as,
                log
                )

        opts = []
        if hasattr(self, 'builder_action_configure_define_opts'):
            opts = self.builder_action_configure_define_opts(
                called_as,
                log
                )

        args = []
        if hasattr(self, 'builder_action_configure_define_args'):
            args = self.builder_action_configure_define_args(
                called_as,
                log
                )

        ret = autotools.configure_high(
            self.buildingsite_path,
            log=log,
            options=opts,
            arguments=args,
            environment=envs,
            environment_mode='copy',
            source_configure_reldir=self.source_configure_reldir,
            use_separate_buildding_dir=self.separate_build_dir,
            script_name=self.builder_action_configure_define_script_name(
                called_as,
                log
                ),
            run_script_not_bash=(
                self.builder_action_configure_define_run_script_not_bash(
                    called_as,
                    log
                    )
                ),
            relative_call=(
                self.builder_action_configure_define_relative_call(
                    called_as,
                    log
                    )
                )
            )

        #sleeptime = 5
        #log.info("sleep: \033[0;1m{}\033[0m seconds".format(sleeptime))
        # time.sleep(sleeptime)
        return ret
예제 #7
0
    def builder_action_autogen(self, called_as, log):

        cfg_script_name = self.builder_action_configure_define_script_name(
            called_as,
            log
            )

        ret = 0

        do_work = False

        if os.path.isfile(
                wayround_org.utils.path.join(
                    self.get_src_dir(),
                    self.source_configure_reldir,
                    cfg_script_name
                    )
                ):

            log.info("configurer found. no generator use presumed")
        else:
            log.info("configurer not found. generator use presumed")
            do_work = True

        if self.forced_autogen:
            log.info(
                "generator use is forced".format(
                    cfg_script_name
                )
            )
            do_work = True

        if do_work:

            log.info(
                "trying to find and use generator mesures"
                )

            for i in [
                    ('autogen.sh', ['./autogen.sh']),
                    ('bootstrap.sh', ['./bootstrap.sh']),
                    ('bootstrap', ['./bootstrap']),
                    ('genconfig.sh', ['./genconfig.sh']),
                    ('configure.ac', ['autoreconf', '-i']),
                    ('configure.in', ['autoreconf', '-i']),
                    ]:

                if os.path.isfile(
                        wayround_org.utils.path.join(
                            self.get_src_dir(),
                            self.source_configure_reldir,
                            i[0]
                            )
                        ):

                    log.info(
                        "found `{}'. trying to execute: {}".format(
                            i[0],
                            ' '.join(i[1])
                            )
                        )

                    wd = wayround_org.utils.path.join(
                        self.get_src_dir(),
                        self.source_configure_reldir
                        )
                    if '/' in i[1][0]:
                        tgt_file = wayround_org.utils.path.join(wd, i[1][0])
                        log.info("changing mode (+x) for: {}".format(tgt_file))
                        chmod_p = subprocess.Popen(
                            ['chmod', '+x', tgt_file],
                            cwd=wd
                            )
                        chmod_p.wait()

                    if i[1][0].endswith('.sh'):
                        i[1].insert(0, 'bash')

                    p = subprocess.Popen(
                        i[1],
                        cwd=wd,
                        stdout=log.stdout,
                        stderr=log.stderr
                        )
                    ret = p.wait()
                    break
            else:
                log.error(
                    "./{} not found and no generators found".format(
                        cfg_script_name
                        )
                    )
                ret = 2
        return ret
def clean_find_so_problems(command_name, opts, args, adds):
    """
    Find so libraries missing in system and write package names requiring those
    missing libraries.
    """

    import wayround_org.aipsetup.controllers

    config = adds['config']

    ret = 0

    basedir = '/'

    pkg_client = \
        wayround_org.aipsetup.controllers.pkg_client_by_config(
            config
            )

    system = wayround_org.aipsetup.controllers.sys_ctl_by_config(
        config, pkg_client, basedir
        )

    problems = system.find_so_problems_in_system(
        verbose=True
        )

    libs = sorted(problems.keys())

    log = wayround_org.utils.log.Log(
        os.getcwd(), 'problems'
        )

    print("Writing log to {}".format(log.log_filename))

    logging.info("Gathering asps file tree. Please wait...")
    tree = system.list_installed_asps_and_their_files(mute=False)
    logging.info("Now working")

    total_problem_packages_list = set()

    count_checked = 0
    libs_c = len(libs)
    for i in libs:
        log.info("Library `{}' required by following files:".format(i))

        files = problems[i]
        files.sort()

        for j in files:
            log.info("    {}".format(j))

        pkgs2 = system.find_file_in_files_installed_by_asps(
            files, mode='end', mute=False, predefined_asp_tree=tree
            )

        pkgs2_l = sorted(pkgs2.keys())

        count_checked += 1

        log.info("  Contained in problem packages:")
        for j in pkgs2_l:
            log.info("    {}".format(j))

        total_problem_packages_list |= set(pkgs2_l)

        logging.info(
            "Checked libraries: {} of {}".format(count_checked, libs_c)
            )

        log.info('---------------------------------')

    pkgs = system.find_file_in_files_installed_by_asps(
        libs, mode='end', mute=False, predefined_asp_tree=tree
        )

    pkgs_l = list(pkgs.keys())
    pkgs_l.sort()

    log.info('')
    log.info("Libs found in packages:")
    for i in pkgs_l:
        log.info("    {}".format(i))

    log.info('')

    log.info("Total Problem Packages List:")
    total_problem_packages_list = list(total_problem_packages_list)
    total_problem_packages_list.sort()
    for i in total_problem_packages_list:
        log.info("    {}".format(i))

    log.stop()
    print("Log written to {}".format(log.log_filename))

    return ret
def clean_packages_with_broken_files(command_name, opts, args, adds):
    """
    Find packages with broken files
    """

    import wayround_org.aipsetup.controllers

    config = adds['config']

    pkg_client = \
        wayround_org.aipsetup.controllers.pkg_client_by_config(
            config
            )

    system = wayround_org.aipsetup.controllers.sys_ctl_by_config(
        config,
        pkg_client,
        basedir='/'
        )

    r = system.list_installed_asps_and_their_sums(mute=False)

    logging.info("Checking Packages")

    asps = list(r.keys())
    asps_c = len(asps)

    problems = {}

    b = 0
    m = 0

    for i in range(asps_c):

        asp_name = asps[i]

        asp = r[asp_name]

        if isinstance(asp, dict):

            problems[asp_name] = {'missing': [], 'broken': []}

            files = list(asp.keys())
            fc = len(files)
            fi = 0

            perc = 0
            if i != 0:
                perc = (100.0 / (asps_c / i))

            for j in files:

                if not os.path.exists(j):
                    problems[asp_name]['missing'].append(j)
                    m += 1

                else:

                    _sum = wayround_org.utils.checksum.make_file_checksum(
                        j, method='sha512'
                        )

                    if _sum != asp[j]:
                        problems[asp_name]['broken'].append(j)
                        b += 1

                fi += 1

                wayround_org.utils.terminal.progress_write(
                    "    ({perc:5.2f}%) {p} packages of {pc}, {f} files of "
                    "{fc}. found {b} broken, {m} missing".format(
                        perc=perc,
                        p=i,
                        pc=asps_c,
                        f=fi,
                        fc=fc,
                        m=m,
                        b=b
                        )
                    )

    for i in list(problems.keys()):

        if (len(
                problems[i]['missing']
                ) == 0
                and len(problems[i]['broken']) == 0):
            del problems[i]

    print()

    log = wayround_org.utils.log.Log(
        os.getcwd(), 'problems'
        )

    log.info(pprint.pformat(problems))

    log_name = log.log_filename

    log.close()

    logging.info("Log saved to {}".format(log_name))

    return 0
def clean_find_garbage(command_name, opts, args, adds):
    """
    Search system for garbage making log and cleaning script

    [-h=all|cpu-vend-os-triplet] [-a=all|cpu-vend-os-triplet]

    -b=BASENAME        - system root path
    --script-type=bash - system cleaning script language (only bash supported)
    --so               - look only for .so files garbage in /usr/lib directory
    """

    import wayround_org.aipsetup.controllers
    import wayround_org.aipsetup.package_name_parser
    import wayround_org.aipsetup.client_pkg
    import wayround_org.aipsetup.system

    config = adds['config']

    ret = 0

    if wayround_org.utils.getopt.check_options(
            opts,
            opts_list=[
                '-b=',
                '--script-type=',
                '--so',
                '-h=',
                '-a='
                ]
            ) != 0:
        ret = 1

    if ret == 0:

        timestamp = wayround_org.utils.time.currenttime_stamp()

        basedir = '/'
        script = 'system_garbage_remove_{}.sh'.format(timestamp)
        script_type = 'bash'
        only_lib = False
        down_script = 'get_required_sources_{}.sh'.format(timestamp)

        if '-b' in opts:
            basedir = opts['-b']

        if '--script-type' in opts:
            script_type = opts['--script-type']

        only_lib = '--so' in opts

        host, arch = _process_h_and_a_opts_specific(opts, config)

        log = wayround_org.utils.log.Log(
            os.getcwd(), 'system_garbage', timestamp=timestamp
            )

        if not script_type in ['bash']:
            logging.error("Invalid --script-type value")
            ret = 1
        else:

            pkg_client = \
                wayround_org.aipsetup.controllers.pkg_client_by_config(
                    config
                    )

            system = wayround_org.aipsetup.controllers.sys_ctl_by_config(
                config,
                pkg_client,
                basedir=basedir
                )

            log.info("Searching for garbage")
            res = system.find_system_garbage(
                mute=False,
                only_lib=only_lib,
                host=host
                )

            if not isinstance(res, list):
                log.error("Some error while searching for garbage")
                ret = 1
            else:

                log.info("Garbage search complete")
                log.info(
                    "Separating garbage .so files to know "
                    "which packages depending on them"
                    )

                libs = wayround_org.utils.path.exclude_files_not_in_dirs(
                    res,
                    system.library_paths(host=host),
                    )

                libs = wayround_org.aipsetup.system.filter_so_files(
                    libs,
                    verbose=True
                    )

                if only_lib:
                    res = libs

                libs = wayround_org.utils.path.bases(libs)

                asp_deps = system.load_asp_deps_all(mute=False)

                asps_lkd_to_garbage = {}

                log.info("Calculating garbage dependencies")

                for asp_name in list(asp_deps.keys()):

                    if not asp_name in asps_lkd_to_garbage:
                        asps_lkd_to_garbage[asp_name] = dict()

                    for file_name in list(asp_deps[asp_name].keys()):

                        file_name_with_dest_dir = \
                            wayround_org.utils.path.insert_base(
                                file_name, basedir
                                )

                        if (not file_name_with_dest_dir
                                in asps_lkd_to_garbage[asp_name]):
                            asps_lkd_to_garbage[
                                asp_name
                                ][file_name_with_dest_dir] = set()

                        asps_lkd_to_garbage[
                            asp_name
                            ][file_name_with_dest_dir] |= \
                            (set(libs) & set(asp_deps[asp_name][file_name]))

                        if len(
                                asps_lkd_to_garbage[
                                    asp_name][file_name_with_dest_dir]
                                ) == 0:
                            del asps_lkd_to_garbage[
                                asp_name][file_name_with_dest_dir]

                    if len(asps_lkd_to_garbage[asp_name]) == 0:
                        del asps_lkd_to_garbage[asp_name]

                s = open(script, 'w')

                s.write("""\
#!/bin/bash

# This is fuse to ensure You are know what You are doing
exit 1


""")

                log.info("Writing report and cleaning script")

                res.sort()

                for i in res:
                    try:
                        log.info("    {}".format(i), echo=False)
                    except:
                        log.error("Error logging {}".format(repr(i)))

                    try:
                        s.write("rm {}\n".format(shlex.quote(i)))
                    except:
                        log.error("Error writing {}".format(repr(i)))

                log.info(
                    "Packages linked to garbage libraries:\n{}".format(
                        pprint.pformat(asps_lkd_to_garbage)
                        ),
                    echo=False
                    )

                log.info("Generating download script")
                required_packages = set()

                for i in list(asps_lkd_to_garbage.keys()):
                    p = wayround_org.aipsetup.package_name_parser.\
                        package_name_parse(i)

                    if not p:
                        log.error(
                            "Can't parse ASP name `{}' to add it to download script".format(
                                i)
                            )
                    else:
                        required_packages.add(p['groups']['name'])

                log.info("Writing download script")
                ds = open(down_script, 'w')
                ds.write(
                    """\
#!/bin/bash

aipsetup3 src getl {}
""".format(' '.join(required_packages))
                    )

                ds.close()

                s.close()

                logging.warning("""
Do not run cleaning script at once!
Check everything is correct!
Wrong cleaning can ruin your system
"""
                                )

            log.close()

    return ret
예제 #11
0
def waf(
        cwd,
        options,
        arguments,
        environment,
        environment_mode,
        log
        ):

    ret = 0

    cwd = wayround_org.utils.path.abspath(cwd)

    env = wayround_org.utils.osutils.env_vars_edit(
        environment,
        environment_mode
        )

    if len(environment) > 0:
        log.info(
            "Environment modifications: {}".format(
                repr(environment)
                )
            )

    if 'PYTHON' in environment:
        python = environment['PYTHON']
    else:
        if 'PATH' in environment:
            PATH = environment['PATH']
        else:
            PATH = os.environ['PATH']

        PATH = PATH.split(':')

        python = wayround_org.utils.file.which('python2', PATH)

        del PATH

    cmd = [
        python,
        wayround_org.utils.path.join(cwd, 'waf')
        ] + options + arguments

    log.info("directory: {}".format(cwd))
    log.info("command: {}".format(cmd))
    log.info("command(joined): {}".format(' '.join(cmd)))

    p = None
    try:
        p = subprocess.Popen(
            args=cmd,
            cwd=cwd,
            stdout=log.stdout,
            stderr=log.stderr,
            env=env
            )
    except:
        log.error(
            "exception while starting waf script\n"
            "    command line was:\n"
            "    " + repr(cmd) +
            wayround_org.utils.error.return_exception_info(
                sys.exc_info()
                )
            )
        ret = 100
    else:

        try:
            p.wait()
        except:
            log.error(
                "Exception occurred while waiting for configure\n" +
                wayround_org.utils.error.return_exception_info(
                    sys.exc_info()
                    )
                )
            ret = 100
        else:
            tmp_s = "configure return code was: {}".format(p.returncode)

            if p.returncode == 0:
                log.info(tmp_s)
            else:
                log.error(tmp_s)

            ret = p.returncode

    return ret