示例#1
0
 def add_config_module(self, config, module):
     if config not in self.get_client_info("build_configs"):
         raise BaseClient.ClientException(
             'config {} is not a valid build config'.format(config))
     if not isinstance(module, str):
         raise BaseClient.ClientException('module must be a str')
     if config not in self.get_client_info("config_modules"):
         self.client_info["config_modules"][config] = []
     self.client_info["config_modules"][config].append(module)
示例#2
0
    def remove_build_root(self):
        """
        Removes the build root.
        Raises:
          BaseClient.ClientException: If BUILD_ROOT does not exist, or the directory removal failed.
        """
        build_root = self.get_build_root()

        if self.build_root_exists():
            logger.info('Removing BUILD_ROOT {}'.format(build_root))
            shutil.rmtree(build_root)
        else:
            raise BaseClient.ClientException('Failed to remove BUILD_ROOT {}; it does not exist'.format(build_root))
示例#3
0
 def create_build_root(self):
     """
     Creates the build root.
     Raises:
       BaseClient.ClientException: If the BUILD_ROOT directory could not be created or if it already exists.
     """
     build_root = self.get_build_root()
     if self.build_root_exists():
         raise BaseClient.ClientException('Failed to create BUILD_ROOT {}; it already exists'.format(build_root))
     else:
         try:
             os.mkdir(build_root)
             logger.info('Created BUILD_ROOT {}'.format(build_root))
         except:
             logger.exception('Failed to create BUILD_ROOT {}'.format(build_root))
             raise
示例#4
0
def commandline_client(args):
    parser = argparse.ArgumentParser()
    parser.add_argument("--url", dest='url', help="The URL of the CIVET site.", required=True)
    parser.add_argument("--build-key", dest='build_key', help="Your build_key", required=True)
    parser.add_argument("--configs",
            dest='configs',
            nargs='+',
            help="The configurations this client supports (eg 'linux-gnu')")
    parser.add_argument("--name", dest='name', help="The name for this particular client. Should be unique.", required=True)
    parser.add_argument("--single-shot",
            dest='single_shot',
            action='store_true',
            help="Execute one test (if there is one) and then exit")
    parser.add_argument("--poll",
            dest='poll',
            type=int,
            default=30,
            help="Number of seconds to wait before polling for more jobs in continuous mode")
    parser.add_argument("--daemon", dest='daemon', choices=['start', 'stop', 'restart'], help="Start a UNIX daemon.")
    parser.add_argument("--log-dir",
            dest='log_dir',
            default='.',
            help="Where to write the log file.  The log will be written as ci_PID.log")
    parser.add_argument("--log-file", dest='log_file', help="Filename to append the log to")
    parser.add_argument("--insecure", dest='insecure', action='store_false', help="Turns off SSL certificate verification")
    parser.add_argument("--ssl-cert",
            dest='ssl_cert',
            help="An crt file to be used when doing SSL certificate verification. This will override --insecure.")
    parser.add_argument("--env",
            dest='env',
            nargs=2,
            action='append',
            help="Sets a client environment variable (example: VAR_NAME VALUE)")
    #parsed, unknown = parser.parse_known_args(args)
    parsed = parser.parse_args(args)

    client_info = {"url": parsed.url,
        "client_name": parsed.name,
        "server": parsed.url,
        "servers": [parsed.url],
        "ssl_verify": parsed.insecure,
        "ssl_cert": parsed.ssl_cert,
        "log_file": parsed.log_file,
        "log_dir": parsed.log_dir,
        "build_key": parsed.build_key,
        "single_shot": parsed.single_shot,
        "poll": parsed.poll,
        "daemon_cmd": parsed.daemon,
        "request_timeout": 30,
        "update_step_time": 20,
        "server_update_interval": 20,
        "server_update_timeout": 5,
        "max_output_size": 5*1024*1024
        }

    c = BaseClient.BaseClient(client_info)

    if parsed.daemon == 'start' or parsed.daemon == 'restart' or platform.system() == "Windows":
        if not parsed.configs:
            raise BaseClient.ClientException('--configs must be provided')

        for config in parsed.configs:
            c.add_config(config)

        if parsed.env:
            for var, value in parsed.env:
                c.set_environment(var, value)

        # Add the BUILD_ROOT to the client environment if it exists in the global environment
        # This is to preserve old behavior for folks that are setting the variable before running the client
        if (not parsed.env or 'BUILD_ROOT' not in parsed.env) and 'BUILD_ROOT' in os.environ:
            c.set_environment('BUILD_ROOT', os.environ.get('BUILD_ROOT'))

    return c, parsed.daemon
示例#5
0
    def run(self, exit_if=None):
        """
        Main client loop. Polls the server for jobs and runs them.
        Loads the proper environment for each config.
        Inputs:
          exit_if: Optional function with a single parameter (which is passed as self; this client)
                   that returns a bool. If the function returns True, exit the poll loop at that
                   time (checked at the end of the poll loop). Used in testing.
        Returns:
          None
        """
        logger.info('Starting client {}'.format(self.get_client_info('client_name')))
        logger.info('Build root: {}'.format(self.get_build_root()))

        if exit_if is not None and (not callable(exit_if) or len(signature(exit_if).parameters) != 1):
            raise BaseClient.ClientException('exit_if must be callable with a single parameter (the client)')

        # Deprecated environment setting; you should start the client with the vars set instead
        if hasattr(settings, 'ENVIRONMENT') and settings.ENVIRONMENT is not None:
            logger.info('DEPRECATED: Set environment variables manually instead of using settings.ENVIRONMENT')
            for k, v in settings.ENVIRONMENT.items():
                self.set_environment(k, v)

        logger.info('Available configs: {}'.format(' '.join([config for config in self.get_client_info("build_configs")])))

        while True:
            if self.get_client_info('manage_build_root') and self.build_root_exists():
                logger.warning("BUILD_ROOT {} already exists at beginning of poll loop; removing"
                               .format(self.get_build_root()))
                self.remove_build_root()

            ran_job = False
            for server in settings.SERVERS:
                if self.cancel_signal.triggered or self.graceful_signal.triggered or self.runner_error:
                    break
                try:
                    if self.check_server(server):
                        ran_job = True
                except Exception:
                    logger.debug("Error: %s" % traceback.format_exc())
                    break

            if self.cancel_signal.triggered or self.graceful_signal.triggered:
                logger.info("Received signal...exiting")
                break
            if self.runner_error:
                logger.info("Error in runner...exiting")
                break
            if exit_if is not None:
                should_exit = exit_if(self)
                if type(should_exit) != bool:
                    raise BaseClient.ClientException('exit_if must return type bool')
                if should_exit:
                    break
            if not ran_job:
                time.sleep(self.get_client_info('poll'))

        if self.get_client_info('manage_build_root') and self.build_root_exists():
            logger.warning("BUILD_ROOT {} still exists after exiting poll loop; removing"
                           .format(self.get_build_root()))
            self.remove_build_root()
示例#6
0
def commandline_client(args):
    parser = argparse.ArgumentParser()
    parser.add_argument('--client',
                        dest='client',
                        type=int,
                        help='The number of the client.',
                        required=True)
    parser.add_argument('--daemon',
                        dest='daemon',
                        choices=['start', 'stop', 'restart', 'none'],
                        help="Start a UNIX daemon.",
                        required=True)
    parser.add_argument(
        "--configs",
        dest='configs',
        nargs='+',
        help="The configurations this client supports (eg 'linux-gnu')")
    parser.add_argument(
        "--env",
        dest='env',
        nargs=2,
        action='append',
        help="Sets a client environment variable (example: VAR_NAME VALUE)")
    parser.add_argument("--build-root",
                        type=str,
                        dest='build_root',
                        help="Sets the build root")
    parser.add_argument(
        "--user-client-suffix",
        action='store_true',
        dest='user_client_suffix',
        help=
        'Adds the user to client name as a suffix, i.e, sets the name to <hostname>_<user>_<client number>'
    )

    parsed = parser.parse_args(args)
    home = os.environ.get("CIVET_HOME",
                          os.path.join(os.environ["HOME"], "civet"))

    log_dir = '{}/logs'.format(home)
    client_name = socket.gethostname()
    if parsed.user_client_suffix:
        client_name += '_{}'.format(pwd.getpwuid(os.getuid())[0])
    client_name += '_{}'.format(parsed.client)
    client_info = {
        "url": "",
        "client_name": client_name,
        "server": "",
        "servers": [],
        "ssl_verify": False,
        "ssl_cert": "",
        "log_file": "",
        "log_dir": log_dir,
        "build_key": "",
        "single_shot": False,
        "poll": 60,
        "daemon_cmd": parsed.daemon,
        "request_timeout": 120,
        "update_step_time": 30,
        "server_update_timeout": 5,
        # This needs to be bigger than update_step_time so that
        # the ping message doesn't become the default message
        "server_update_interval": 50,
        "max_output_size": 5 * 1024 * 1024
    }

    c = INLClient.INLClient(client_info)

    if parsed.daemon == 'start' or parsed.daemon == 'restart' or platform.system(
    ) == "Windows":
        if not parsed.configs:
            raise BaseClient.ClientException('--configs must be provided')

        if parsed.build_root:
            build_root = parsed.build_root
        else:
            build_root = '{}/build_{}'.format(home, parsed.client)

        for config in parsed.configs:
            c.add_config(config)

        c.set_environment('BUILD_ROOT', build_root)
        c.set_environment('CIVET_HOME', home)
        c.set_environment('CIVET_CLIENT_NUMBER', parsed.client)
        if parsed.env:
            for var, value in parsed.env:
                c.set_environment(var, value)

    return c, parsed.daemon