Beispiel #1
0
def run(param):
    version = param['version']

    tar_filename = param['config_package']['source']['file'].format(
        version=version)
    tar_file = os.path.join(param['package_path']['misc_dir'], 'download',
                            tar_filename)
    main_dir = param['config_package']['source'].get(
        'main', '').format(version=version)

    dst_dir = param['config_package'].get('path', {}).get('source')
    if not dst_dir:
        return {'success': False, 'message': 'Path "source" is not specified'}

    safe_rmdir(dst_dir)
    safe_mkdir(dst_dir)

    if main_dir:
        strip_number = main_dir.strip(os.sep).count(os.sep) + 1
        cmd = [
            'tar', '--strip-components',
            str(strip_number), '-xvf', tar_file, main_dir
        ]
    else:
        cmd = ['tar', '-xvf', tar_file]

    with open(param['log_file'], 'w') as f:
        ret = call_and_log(cmd, log=f, cwd=dst_dir)

    return {'success': ret == 0, 'message': 'Tar exit code: {0}'.format(ret)}
Beispiel #2
0
    def execute(self,
                package,
                category,
                subdir,
                version,
                category_origin,
                subdir_origin,
                version_origin,
                from_install=False):
        config_package_name = 'package_install' if from_install else 'package_runtime'

        pkg_cfg_origin = copy.deepcopy(
            self._config[config_package_name].package_config(
                category_origin, subdir_origin, package,
                version_origin)['config_origin'])
        pkg_cfg_origin['version'] = version

        pkg_path = package_path(self._config['app'], self._config['category'],
                                category, subdir, package, version)

        ctg_cf, sd_cf, pkg_cf, ver_cf = check_conflict_package(
            pkg_path['main_dir'], self._config['package_runtime'])
        if ctg_cf:
            raise InstallPackageConfigError(
                'Package path conflicts with package "{0}", category "{1}", subdir "{2}", version "{3}"'
                .format(pkg_cf, ctg_cf, sd_cf, ver_cf))

        safe_mkdir(pkg_path['config_dir'])
        dump_config(pkg_cfg_origin, pkg_path['config_file'])

        self._config.reset()
Beispiel #3
0
def run(param):
    build_dir = param['config_package'].get('path', {}).get('build')
    if not build_dir:
        return {'success': False, 'message': 'Path "build" is not specified'}

    install_dir = param['config_package'].get('path', {}).get('install')
    if not install_dir:
        return {'success': False, 'message': 'Path "install" is not specified'}

    safe_mkdir(install_dir)

    install_args = param['config_package'].get('make_install',
                                               {}).get('args', ['install'])
    install_args = ensure_list(install_args)

    env = param.get('env')
    env_install = env.copy()
    for k, v in param['config_package'].get('make_install',
                                            {}).get('env', {}).items():
        env_install[k] = v.format(**param['config_package'].get('path', {}))

    with open(param['log_file'], 'w') as f:
        cmd = ['make'] + install_args
        ret = call_and_log(cmd, log=f, cwd=build_dir, env=env_install)

    return {
        'success': ret == 0,
        'message': 'Make install exit code: {0}'.format(ret)
    }
Beispiel #4
0
def run(param):
    release_version = param['config_scenario']['version']

    if len(param['command']) > 1:
        destination = param['command'][1]
    else:
        destination = os.getcwd()

    destination = expand_path(destination)
    pack_dir = os.path.join(destination, 'pack_' + release_version)

    for category in param['config_package_install']:
        for subdir in param['config_package_install'][category]:
            for package in param['config_package_install'][category][subdir]:
                for version, value in param['config_package_install'][
                        category][subdir][package].items():
                    pkg_cfg = value['config_origin']

                    download = pkg_cfg.get('install', {}).get('download')
                    if not download:
                        continue

                    url = pkg_cfg.get('source', {}).get('url')
                    if not url:
                        continue

                    pkg_dir = os.path.join(pack_dir, package, version)
                    safe_mkdir(pkg_dir)

                    _logger.info('Packing {0}...'.format(package))

                    if download == 'http':
                        url = url.format(version=version)
                        pkg_file = _download_http(url, pkg_dir)
                    elif download == 'svn':
                        url = url.format(version=version)
                        pkg_file = _download_svn(url, pkg_dir, package,
                                                 version)
                    elif download == 'git':
                        tag = pkg_cfg.get('source', {}).get('tag', 'master')
                        tag = tag.format(version=version)
                        pkg_file = _download_git(url, tag, pkg_dir, package,
                                                 version)
                    else:
                        _logger.warn('Package {0} not packed'.format(package))
                        continue

                    with open(os.path.join(pkg_dir, 'md5sum.txt'), 'w') as f:
                        call(['md5sum', pkg_file], cwd=pkg_dir, stdout=f)
                    with open(os.path.join(pkg_dir, 'sha1sum.txt'), 'w') as f:
                        call(['sha1sum', pkg_file], cwd=pkg_dir, stdout=f)
                    with open(os.path.join(pkg_dir, 'url.txt'), 'w') as f:
                        f.write(url + '\n')

                    _logger.info('Package {0} packed'.format(package))

    _logger.info('All packages in version {0} packed in {1}'.format(
        release_version, pack_dir))
Beispiel #5
0
    def execute(self, param):
        pkg = param['package']
        step = param['step']
        sub_step = param['sub_step']

        if sub_step == 0:
            step_full_name = '{0} - {1}'.format(pkg, step)
        else:
            step_full_name = '{0} - {1} - {2}'.format(pkg, step, sub_step)

        if not param['action_install']:
            _logger.debug('Skip step: {0}'.format(step_full_name))
            return {'success': True, 'skip': True}

        result = {}

        result['start'] = datetime.datetime.utcnow()

        safe_mkdir(os.path.dirname(param['log_file']))

        try:
            with Handler() as h:
                result_action = h.run('install', param)
        except HandlerNotFoundError as e:
            _logger.error('Install handler "{0}" not found for "{1}"'.format(
                param['action'], step_full_name))
            raise
        except Exception as e:
            _logger.error('Install handler "{0}" error for {1}: {2}'.format(
                param['action'], step_full_name, e))
            if param['config_output']['verbose']:
                _logger.error('\n{0}'.format(traceback.format_exc()))
            raise

        if isinstance(result_action, bool):
            result['success'] = result_action
        elif isinstance(result_action, dict):
            result['success'] = result_action.get('success', False)
        else:
            result['success'] = False

        if not result['success']:
            if isinstance(result_action, dict) and 'message' in result_action:
                _logger.error('"{0}" execution error: {1}'.format(
                    step_full_name, result_action['message']))
            _logger.error('"{0}" execution error. Find log in "{1}"'.format(
                step_full_name, param['log_file']))
            raise InstallExecutorError(
                '"{0}" execution error'.format(step_full_name))

        result['action'] = result_action
        result['end'] = datetime.datetime.utcnow()

        return result
Beispiel #6
0
    def execute(self, config_user, config_version, destination):
        release_version = config_version.get('version')

        try:
            config_release = ConfigRelease(config_version)
        except ConfigReleaseError:
            _logger.debug(
                'Install release definition: {0}'.format(release_version))
            install = Install(config_user, config_version)
            config_release = install.config_release()

        self.__pkg_mgr = PackageManager(config_version, config_release)

        if not destination:
            destination = os.getcwd()
        destination = expand_path(destination)
        pack_dir = os.path.join(destination, 'pack_' + release_version)

        for pkg, pkg_info in self.__pkg_mgr.package_all().items():
            version = pkg_info.get('package', {}).get('version')
            if not version:
                continue

            download = pkg_info.get('install', {}).get('download')
            if not download:
                continue

            url = download.get('param', {}).get('url')
            if not url:
                continue
            url = url.format(version=version)

            pkg_dir = os.path.join(pack_dir, pkg, version)
            safe_mkdir(pkg_dir)

            _logger.info('Packing {0}...'.format(pkg))

            if download.get('handler') == 'http':
                pkg_file = _download_http(url, pkg_dir)
            elif download.get('handler') == 'svn':
                pkg_file = _download_svn(url, pkg_dir, pkg, version)

            with open(os.path.join(pkg_dir, 'md5sum.txt'), 'w') as f:
                call(['md5sum', pkg_file], cwd=pkg_dir, stdout=f)
            with open(os.path.join(pkg_dir, 'sha1sum.txt'), 'w') as f:
                call(['sha1sum', pkg_file], cwd=pkg_dir, stdout=f)
            with open(os.path.join(pkg_dir, 'url.txt'), 'w') as f:
                f.write(url + '\n')

            _logger.info('Package {0} packed'.format(pkg))

        _logger.info('All packages in version {0} packed in {1}'.format(
            release_version, pack_dir))
Beispiel #7
0
    def execute(self, param):
        pkg = param['package']
        action = param['action']
        sub_action = param['sub_action']

        if sub_action == 0:
            action_full_name = '{0} - {1}'.format(pkg, action)
        else:
            action_full_name = '{0} - {1} - {2}'.format(
                pkg, action, sub_action)

        result = {}

        result['start'] = datetime.datetime.utcnow()

        safe_mkdir(param['pkg_info']['dir']['log'])

        try:
            result_action = run_handler('', 'install', param['action_name'],
                                        param)
        except Exception as e:
            _logger.critical('"{0}" install handler error: {1}'.format(
                action_full_name, e))
            if param['config_user']['verbose']:
                _logger.critical('\n{0}'.format(traceback.format_exc()))
            raise

        result['success'] = False
        if isinstance(result_action, bool) and result_action:
            result['success'] = True
        if isinstance(
                result_action, dict
        ) and 'success' in result_action and result_action['success']:
            result['success'] = True

        if not result['success']:
            if isinstance(result_action, dict) and 'message' in result_action:
                _logger.error('"{0}" execution error: {1}'.format(
                    action_full_name, result_action['message']))
            _logger.critical('"{0}" execution error. Find log in "{1}"'.format(
                action_full_name, param['log_file']))
            raise InstallExecutorError(
                '"{0}" execution error'.format(action_full_name))

        result['action'] = result_action
        result['end'] = datetime.datetime.utcnow()

        return result
Beispiel #8
0
    def save_release_status(self, end_time):
        if not self['category'].get('install'):
            _logger.warn('Will not save release status for: {0}'.format(
                self['name']))
            return

        release_status = self.__load_status(self.__release_status_file())

        if 'version' not in release_status:
            release_status['version'] = []
        release_version = self.__config_release.get('version')
        if release_version and release_version not in release_status['version']:
            release_status['version'].append(release_version)

        safe_mkdir(self['dir']['status'])
        dump_config(release_status, self.__release_status_file())
Beispiel #9
0
def run(param):
    version = param['version']
    url = param['config_package']['source']['url'].format(version=version)

    source_dir = param['config_package'].get('path', {}).get('source')
    if not source_dir:
        return {'success': False, 'message': 'Path "source" is not specified'}

    safe_rmdir(source_dir)
    safe_mkdir(source_dir)

    cmd = ['svn', 'export', '--force', url, source_dir]
    with open(param['log_file'], 'w') as f:
        ret = call_and_log(cmd, log=f)

    return {'success': ret == 0, 'message': 'Svn exit code: {0}'.format(ret)}
Beispiel #10
0
    def execute(self, package, category, subdir, version):
        pkg_path = package_path(self._config['app'], self._config['category'],
                                category, subdir, package, version)

        ctg_cf, sd_cf, pkg_cf, ver_cf = check_conflict_package(
            pkg_path['main_dir'], self._config['package_runtime'])
        if ctg_cf:
            raise CreatePackageConfigError(
                'Package path conflicts with package "{0}", category "{1}", subdir "{2}", version "{3}"'
                .format(pkg_cf, ctg_cf, sd_cf, ver_cf))

        safe_mkdir(pkg_path['config_dir'])

        if not os.path.exists(pkg_path['config_file']):
            open(pkg_path['config_file'], 'w').close()

        self._config.reset()
Beispiel #11
0
    def save_release_status(self, pkg, end_time):
        if pkg not in self.__pkgs:
            raise PackageNotFoundError('Package {0} not found'.format(pkg))
        if not self.__pkgs[pkg]['config_category'].get('install'):
            _logger.warn('Will not save release status for: {0}'.format(pkg))
            return

        release_status = self.__load_status(self.__release_status_file(pkg))

        if 'version' not in release_status:
            release_status['version'] = []
        release_version = self.__config_release.get('version')
        if release_version and release_version not in release_status['version']:
            release_status['version'].append(release_version)

        safe_mkdir(self.__pkgs[pkg]['dir']['status'])
        dump_config(release_status, self.__release_status_file(pkg))
Beispiel #12
0
def run(param):
    version = param['version']

    url = param['config_package']['source']['url'].format(version=version)
    filename = param['config_package']['source']['file'].format(
        version=version)
    md5url = param['config_package']['source'].get('md5url',
                                                   '').format(version=version)
    dst_dir = os.path.join(param['package_path']['misc_dir'], 'download')

    safe_rmdir(dst_dir)
    safe_mkdir(dst_dir)

    with open(param['log_file'], 'w') as f:
        cmd = ['curl', '-v', '-f', '-L', '-s', '-S', '-R', '-O', url]
        ret = call_and_log(cmd, log=f, cwd=dst_dir)
        if ret != 0:
            return {
                'success': False,
                'message':
                'File download error, CURL exit code: {0}'.format(ret)
            }

        if md5url:
            cmd = [
                'curl', '-v', '-f', '-L', '-s', '-S', '-R', '-o', 'md5sum.txt',
                md5url
            ]
            ret = call_and_log(cmd, log=f, cwd=dst_dir)
            if ret != 0:
                return {
                    'success':
                    False,
                    'message':
                    'md5sum download error, CURL exit code: {0}'.format(ret)
                }

            cmd = ['md5sum', '-c', 'md5sum.txt']
            ret = call_and_log(cmd, log=f, cwd=dst_dir)
            if ret != 0:
                return {
                    'success': False,
                    'message': 'md5 checksum error: {0}'.format(ret)
                }

    return {'success': ret == 0, 'message': 'CURL exit code: {0}'.format(ret)}
Beispiel #13
0
    def save_action_status(self, action, start, end):
        if not self['category'].get('install'):
            _logger.warn('Will not save install status for: {0}.{1}'.format(
                self['name'], action))
            return

        install_status = self.__load_status(self.__install_status_file())

        if 'steps' not in install_status:
            install_status['steps'] = {}
        if action not in install_status['steps']:
            install_status['steps'][action] = {}
        install_status['steps'][action]['finished'] = True
        install_status['steps'][action]['start'] = start
        install_status['steps'][action]['end'] = end

        safe_mkdir(self['dir']['status'])
        dump_config(install_status, self.__install_status_file())
Beispiel #14
0
    def save_action_status(self, pkg, action, start, end):
        if pkg not in self.__pkgs:
            raise PackageNotFoundError('Package {0} not found'.format(pkg))
        if not self.__pkgs[pkg]['config_category'].get('install'):
            _logger.warn('Will not save install status for: {0}.{1}'.format(
                pkg, action))
            return

        install_status = self.__load_status(self.__install_status_file(pkg))

        if 'steps' not in install_status:
            install_status['steps'] = {}
        if action not in install_status['steps']:
            install_status['steps'][action] = {}
        install_status['steps'][action]['finished'] = True
        install_status['steps'][action]['start'] = start
        install_status['steps'][action]['end'] = end

        safe_mkdir(self.__pkgs[pkg]['dir']['status'])
        dump_config(install_status, self.__install_status_file(pkg))
Beispiel #15
0
def run(param):
    source_dir = param['config_package'].get('path', {}).get('source')
    if not source_dir:
        return {'success': False, 'message': 'Path "source" is not specified'}

    build_dir = param['config_package'].get('path', {}).get('build')
    if not build_dir:
        return {'success': False, 'message': 'Path "build" is not specified'}

    install_dir = param['config_package'].get('path', {}).get('install')
    if not install_dir:
        return {'success': False, 'message': 'Path "install" is not specified'}


    if source_dir != build_dir:
        safe_rmdir(build_dir)
    safe_mkdir(build_dir)


    cmake_args = param['config_package'].get('cmake', {}).get('args', [])
    cmake_args = ensure_list(cmake_args)
    cmake_args = [p.format(**param['config_package_install_path']) for p in cmake_args]

    if not param['config_package'].get('cmake', {}).get('ignore_install_prefix', False):
        cmake_args.insert(0, '-DCMAKE_INSTALL_PREFIX='+install_dir)

    cmake_var = param['config_package'].get('cmake', {}).get('var', {})
    for k, v in cmake_var.items():
        full_value = v.format(**param['config_package_install_path'])
        full_arg = '-D{0}={1}'.format(k, full_value)
        cmake_args.append(full_arg)


    env = param.get('env')

    with open(param['log_file'], 'w') as f:
        cmd = ['cmake', source_dir] + cmake_args
        ret = call_and_log(cmd, log=f, cwd=build_dir, env=env)

    return {'success': ret==0, 'message': 'CMake exit code: {0}'.format(ret)}
Beispiel #16
0
def run(param):
    version = param['version']
    url = param['config_package']['source']['url']
    tag = param['config_package']['source'].get('tag',
                                                '').format(version=version)
    keep_dotgit = param['config_package']['source'].get('keep_dotgit', False)
    branch = param['config_package']['source'].get('branch', 'develop')

    if not tag:
        return {'success': False, 'message': 'Git tag is not specified'}

    source_dir = param['config_package'].get('path', {}).get('source')
    if not source_dir:
        return {'success': False, 'message': 'Path "source" is not specified'}

    safe_rmdir(source_dir)
    safe_mkdir(source_dir)

    with open(param['log_file'], 'w') as f:
        cmd = ['git', 'clone', url, source_dir]
        ret = call_and_log(cmd, log=f)
        if ret != 0:
            return {
                'success': False,
                'message': 'Git clone exit code: {0}'.format(ret)
            }

        cmd = ['git', 'checkout', tag, '-b', branch]
        ret = call_and_log(cmd, log=f, cwd=source_dir)
        if ret != 0:
            return {
                'success': False,
                'message': 'Git checkout tag exit code: {0}'.format(ret)
            }

    if not keep_dotgit:
        safe_rmdir(os.path.join(source_dir, '.git'))

    return {'success': ret == 0, 'message': 'Git OK'}
Beispiel #17
0
def run(param):
    source_dir = param['config_package'].get('path', {}).get('source')
    if not source_dir:
        return {'success': False, 'message': 'Path "source" is not specified'}

    build_dir = param['config_package'].get('path', {}).get('build')
    if not build_dir:
        return {'success': False, 'message': 'Path "build" is not specified'}

    install_dir = param['config_package'].get('path', {}).get('install')
    if not install_dir:
        return {'success': False, 'message': 'Path "install" is not specified'}


    if source_dir != build_dir:
        safe_rmdir(build_dir)
    safe_mkdir(build_dir)


    configure_args = param['config_package'].get('configure', {}).get('args', [])
    configure_args = ensure_list(configure_args)
    configure_args = [p.format(**param['config_package_install_path']) for p in configure_args]

    if not param['config_package'].get('configure', {}).get('ignore_install_prefix', False):
        configure_args.insert(0, '--prefix='+install_dir)

    env = param.get('env')
    env_configure = env.copy()
    for k, v in param['config_package'].get('configure', {}).get('env', {}).items():
        env_configure[k] = v.format(**param['config_package_install_path'])

    configure_path = os.path.join(source_dir, 'configure')


    with open(param['log_file'], 'w') as f:
        cmd = [configure_path] + configure_args
        ret = call_and_log(cmd, log=f, cwd=build_dir, env=env_configure)

    return {'success': ret==0, 'message': 'Configure exit code: {0}'.format(ret)}
Beispiel #18
0
    def __dag_run(self):
        # Get clean environment
        env = Env(initial_env=self._env.env_final(),
                  env_prefix=self._config['app']['env_prefix'])
        env.unload_packages()
        env.unload_release()
        env.unload_app()
        env.load_app(self._config['app'])
        env.load_release(self._config['scenario'], self._config['option'],
                         self._config['release'])

        selector = InstallSelector(self._config)
        processor = MultiThreadProcessor()
        executor = InstallExecutor(self._config, env.env_final())

        start_time = datetime.datetime.utcnow()

        with Handler(self._config['release_path']['handler_python_dir']):
            dag_run(self.__dag,
                    selector=selector,
                    processor=processor,
                    executor=executor)

        end_time = datetime.datetime.utcnow()

        self._config['release_status']['install'] = {}
        self._config['release_status']['install']['start'] = start_time
        self._config['release_status']['install']['end'] = end_time
        self._config['release_status']['install']['finished'] = True
        self._config['release_status']['option'] = {}
        for opt in self._config['release_install']['options_to_save']:
            if opt in self._config['option']:
                self._config['release_status']['option'][opt] = self._config[
                    'option'][opt]

        safe_mkdir(self._config['release_path']['status_dir'])
        self._config['release_status'].save_to_file(
            self._config['release_path']['status_file'])
Beispiel #19
0
 def save_package_config(self, category, subdir, name, version):
     pkg_install_cfg = self.package_config(category, subdir, name, version)
     safe_mkdir(pkg_install_cfg['package_path']['config_dir'])
     dump_config(pkg_install_cfg['config_origin'],
                 pkg_install_cfg['package_path']['config_file'])
Beispiel #20
0
 def save_install_status(self, category, subdir, name, version):
     pkg_install_cfg = self.package_config(category, subdir, name, version)
     safe_mkdir(pkg_install_cfg['package_path']['status_dir'])
     dump_config(pkg_install_cfg['install_status'],
                 pkg_install_cfg['package_path']['status_install_file'])