def add_config_module(self, config, module):
if config not in self.get_client_info("build_configs"):
raise BaseClient.ClientException(
'config {} is not a valid build config'.format(config))
if not isinstance(module, str):
raise BaseClient.ClientException('module must be a str')
if config not in self.get_client_info("config_modules"):
self.client_info["config_modules"][config] = []
self.client_info["config_modules"][config].append(module)
def remove_build_root(self):
"""
Removes the build root.
Raises:
BaseClient.ClientException: If BUILD_ROOT does not exist, or the directory removal failed.
"""
build_root = self.get_build_root()
if self.build_root_exists():
logger.info('Removing BUILD_ROOT {}'.format(build_root))
shutil.rmtree(build_root)
else:
raise BaseClient.ClientException('Failed to remove BUILD_ROOT {}; it does not exist'.format(build_root))
def create_build_root(self):
"""
Creates the build root.
Raises:
BaseClient.ClientException: If the BUILD_ROOT directory could not be created or if it already exists.
"""
build_root = self.get_build_root()
if self.build_root_exists():
raise BaseClient.ClientException('Failed to create BUILD_ROOT {}; it already exists'.format(build_root))
else:
try:
os.mkdir(build_root)
logger.info('Created BUILD_ROOT {}'.format(build_root))
except:
logger.exception('Failed to create BUILD_ROOT {}'.format(build_root))
raise
def commandline_client(args):
parser = argparse.ArgumentParser()
parser.add_argument("--url", dest='url', help="The URL of the CIVET site.", required=True)
parser.add_argument("--build-key", dest='build_key', help="Your build_key", required=True)
parser.add_argument("--configs",
dest='configs',
nargs='+',
help="The configurations this client supports (eg 'linux-gnu')")
parser.add_argument("--name", dest='name', help="The name for this particular client. Should be unique.", required=True)
parser.add_argument("--single-shot",
dest='single_shot',
action='store_true',
help="Execute one test (if there is one) and then exit")
parser.add_argument("--poll",
dest='poll',
type=int,
default=30,
help="Number of seconds to wait before polling for more jobs in continuous mode")
parser.add_argument("--daemon", dest='daemon', choices=['start', 'stop', 'restart'], help="Start a UNIX daemon.")
parser.add_argument("--log-dir",
dest='log_dir',
default='.',
help="Where to write the log file. The log will be written as ci_PID.log")
parser.add_argument("--log-file", dest='log_file', help="Filename to append the log to")
parser.add_argument("--insecure", dest='insecure', action='store_false', help="Turns off SSL certificate verification")
parser.add_argument("--ssl-cert",
dest='ssl_cert',
help="An crt file to be used when doing SSL certificate verification. This will override --insecure.")
parser.add_argument("--env",
dest='env',
nargs=2,
action='append',
help="Sets a client environment variable (example: VAR_NAME VALUE)")
#parsed, unknown = parser.parse_known_args(args)
parsed = parser.parse_args(args)
client_info = {"url": parsed.url,
"client_name": parsed.name,
"server": parsed.url,
"servers": [parsed.url],
"ssl_verify": parsed.insecure,
"ssl_cert": parsed.ssl_cert,
"log_file": parsed.log_file,
"log_dir": parsed.log_dir,
"build_key": parsed.build_key,
"single_shot": parsed.single_shot,
"poll": parsed.poll,
"daemon_cmd": parsed.daemon,
"request_timeout": 30,
"update_step_time": 20,
"server_update_interval": 20,
"server_update_timeout": 5,
"max_output_size": 5*1024*1024
}
c = BaseClient.BaseClient(client_info)
if parsed.daemon == 'start' or parsed.daemon == 'restart' or platform.system() == "Windows":
if not parsed.configs:
raise BaseClient.ClientException('--configs must be provided')
for config in parsed.configs:
c.add_config(config)
if parsed.env:
for var, value in parsed.env:
c.set_environment(var, value)
# Add the BUILD_ROOT to the client environment if it exists in the global environment
# This is to preserve old behavior for folks that are setting the variable before running the client
if (not parsed.env or 'BUILD_ROOT' not in parsed.env) and 'BUILD_ROOT' in os.environ:
c.set_environment('BUILD_ROOT', os.environ.get('BUILD_ROOT'))
return c, parsed.daemon
def run(self, exit_if=None):
"""
Main client loop. Polls the server for jobs and runs them.
Loads the proper environment for each config.
Inputs:
exit_if: Optional function with a single parameter (which is passed as self; this client)
that returns a bool. If the function returns True, exit the poll loop at that
time (checked at the end of the poll loop). Used in testing.
Returns:
None
"""
logger.info('Starting client {}'.format(self.get_client_info('client_name')))
logger.info('Build root: {}'.format(self.get_build_root()))
if exit_if is not None and (not callable(exit_if) or len(signature(exit_if).parameters) != 1):
raise BaseClient.ClientException('exit_if must be callable with a single parameter (the client)')
# Deprecated environment setting; you should start the client with the vars set instead
if hasattr(settings, 'ENVIRONMENT') and settings.ENVIRONMENT is not None:
logger.info('DEPRECATED: Set environment variables manually instead of using settings.ENVIRONMENT')
for k, v in settings.ENVIRONMENT.items():
self.set_environment(k, v)
logger.info('Available configs: {}'.format(' '.join([config for config in self.get_client_info("build_configs")])))
while True:
if self.get_client_info('manage_build_root') and self.build_root_exists():
logger.warning("BUILD_ROOT {} already exists at beginning of poll loop; removing"
.format(self.get_build_root()))
self.remove_build_root()
ran_job = False
for server in settings.SERVERS:
if self.cancel_signal.triggered or self.graceful_signal.triggered or self.runner_error:
break
try:
if self.check_server(server):
ran_job = True
except Exception:
logger.debug("Error: %s" % traceback.format_exc())
break
if self.cancel_signal.triggered or self.graceful_signal.triggered:
logger.info("Received signal...exiting")
break
if self.runner_error:
logger.info("Error in runner...exiting")
break
if exit_if is not None:
should_exit = exit_if(self)
if type(should_exit) != bool:
raise BaseClient.ClientException('exit_if must return type bool')
if should_exit:
break
if not ran_job:
time.sleep(self.get_client_info('poll'))
if self.get_client_info('manage_build_root') and self.build_root_exists():
logger.warning("BUILD_ROOT {} still exists after exiting poll loop; removing"
.format(self.get_build_root()))
self.remove_build_root()
def commandline_client(args):
parser = argparse.ArgumentParser()
parser.add_argument('--client',
dest='client',
type=int,
help='The number of the client.',
required=True)
parser.add_argument('--daemon',
dest='daemon',
choices=['start', 'stop', 'restart', 'none'],
help="Start a UNIX daemon.",
required=True)
parser.add_argument(
"--configs",
dest='configs',
nargs='+',
help="The configurations this client supports (eg 'linux-gnu')")
parser.add_argument(
"--env",
dest='env',
nargs=2,
action='append',
help="Sets a client environment variable (example: VAR_NAME VALUE)")
parser.add_argument("--build-root",
type=str,
dest='build_root',
help="Sets the build root")
parser.add_argument(
"--user-client-suffix",
action='store_true',
dest='user_client_suffix',
help=
'Adds the user to client name as a suffix, i.e, sets the name to <hostname>_<user>_<client number>'
)
parsed = parser.parse_args(args)
home = os.environ.get("CIVET_HOME",
os.path.join(os.environ["HOME"], "civet"))
log_dir = '{}/logs'.format(home)
client_name = socket.gethostname()
if parsed.user_client_suffix:
client_name += '_{}'.format(pwd.getpwuid(os.getuid())[0])
client_name += '_{}'.format(parsed.client)
client_info = {
"url": "",
"client_name": client_name,
"server": "",
"servers": [],
"ssl_verify": False,
"ssl_cert": "",
"log_file": "",
"log_dir": log_dir,
"build_key": "",
"single_shot": False,
"poll": 60,
"daemon_cmd": parsed.daemon,
"request_timeout": 120,
"update_step_time": 30,
"server_update_timeout": 5,
# This needs to be bigger than update_step_time so that
# the ping message doesn't become the default message
"server_update_interval": 50,
"max_output_size": 5 * 1024 * 1024
}
c = INLClient.INLClient(client_info)
if parsed.daemon == 'start' or parsed.daemon == 'restart' or platform.system(
) == "Windows":
if not parsed.configs:
raise BaseClient.ClientException('--configs must be provided')
if parsed.build_root:
build_root = parsed.build_root
else:
build_root = '{}/build_{}'.format(home, parsed.client)
for config in parsed.configs:
c.add_config(config)
c.set_environment('BUILD_ROOT', build_root)
c.set_environment('CIVET_HOME', home)
c.set_environment('CIVET_CLIENT_NUMBER', parsed.client)
if parsed.env:
for var, value in parsed.env:
c.set_environment(var, value)
return c, parsed.daemon
