def run(self,
log_level,
master_url,
remote_file=None,
build_type=None,
**request_params):
"""
Execute a build and wait for it to complete.
:param log_level: the log level at which to do application logging (or None for default log level)
:type log_level: str | None
:param master_url: the url (specified by the user) of the master to which we should send the build
:type master_url: str | None
:param remote_file: a list of remote files where each element contains the output file name and the resource URL
:type remote_file: list[list[str]] | None
:param build_type: the build type of the request to be sent (e.g., "git", "directory"). If not specified
will default to the "directory" project type.
:type build_type: str | None
:param request_params: key-value pairs to be provided as build parameters in the build request
:type request_params: dict
"""
log_level = log_level or Configuration['log_level']
log.configure_logging(log_level=log_level,
simplified_console_logs=True)
request_params['type'] = build_type or request_params.get(
'type') or 'directory'
if remote_file:
request_params['remote_files'] = {
name: url
for name, url in remote_file
}
operational_master_url = master_url or '{}:{}'.format(
Configuration['hostname'], Configuration['port'])
# If running a single master, single slave--both on localhost--we need to launch services locally.
if master_url is None and Network.are_hosts_same(Configuration['master_hostname'], 'localhost') \
and len(Configuration['slaves']) == 1 \
and Network.are_hosts_same(Configuration['slaves'][0], 'localhost'):
self._start_local_services_if_needed(operational_master_url)
if request_params['type'] == 'directory':
request_params['project_directory'] = request_params.get(
'project_directory') or os.getcwd()
runner = BuildRunner(master_url=operational_master_url,
request_params=request_params,
secret=Secret.get())
if not runner.run():
sys.exit(1)
def _deploy_binaries_and_conf(self, host, username, current_executable, binaries_tar_path, in_use_conf_path):
"""
Move binaries and conf to single host.
:param host: host to deploy to
:type host: str
:param username: current username
:param current_executable: path to the executable (ie: /usr/bin/python, ./clusterrunner, etc)
:type current_executable: str
:param binaries_tar_path: path to tar.gz file of clusterrunner binaries
:type binaries_tar_path: str
:param in_use_conf_path: path toe currently used conf file
:type in_use_conf_path: str
"""
clusterrunner_dir = join(os.path.expanduser('~'), '.clusterrunner')
clusterrunner_executable_dir = join(clusterrunner_dir, 'dist')
clusterrunner_executable_deploy_target = join(clusterrunner_executable_dir, 'clusterrunner')
clusterrunner_conf_deploy_target = join(clusterrunner_dir, 'clusterrunner.conf')
deploy_target = DeployTarget(host, username)
if Network.are_hosts_same(host, 'localhost'):
# Do not want to overwrite the currently running executable.
if current_executable != clusterrunner_executable_deploy_target:
deploy_target.deploy_binary(binaries_tar_path, clusterrunner_executable_dir)
# Do not want to overwrite the currently used conf.
if in_use_conf_path != clusterrunner_conf_deploy_target:
deploy_target.deploy_conf(in_use_conf_path, clusterrunner_conf_deploy_target)
else:
deploy_target.deploy_binary(binaries_tar_path, clusterrunner_executable_dir)
deploy_target.deploy_conf(in_use_conf_path, clusterrunner_conf_deploy_target)
def run(self, log_level, master_url, remote_file=None, build_type=None, **request_params):
"""
Execute a build and wait for it to complete.
:param log_level: the log level at which to do application logging (or None for default log level)
:type log_level: str | None
:param master_url: the url (specified by the user) of the master to which we should send the build
:type master_url: str | None
:param remote_file: a list of remote files where each element contains the output file name and the resource URL
:type remote_file: list[list[str]] | None
:param build_type: the build type of the request to be sent (e.g., "git", "directory"). If not specified
will default to the "directory" project type.
:type build_type: str | None
:param request_params: key-value pairs to be provided as build parameters in the build request
:type request_params: dict
"""
log_level = log_level or Configuration['log_level']
log.configure_logging(log_level=log_level, simplified_console_logs=True)
request_params['type'] = build_type or request_params.get('type') or 'directory'
if remote_file:
request_params['remote_files'] = {name: url for name, url in remote_file}
operational_master_url = master_url or '{}:{}'.format(Configuration['hostname'], Configuration['port'])
# If running a single master, single slave--both on localhost--we need to launch services locally.
if master_url is None and Network.are_hosts_same(Configuration['master_hostname'], 'localhost') \
and len(Configuration['slaves']) == 1 \
and Network.are_hosts_same(Configuration['slaves'][0], 'localhost'):
self._start_local_services_if_needed(operational_master_url)
if request_params['type'] == 'directory':
request_params['project_directory'] = request_params.get('project_directory') or os.getcwd()
runner = BuildRunner(master_url=operational_master_url, request_params=request_params, secret=Secret.get())
if not runner.run():
sys.exit(1)
def test_are_hosts_same(
self,
host_to_id,
expect_hosts_are_same,
):
def side_effect(host):
host_id = host_to_id[host]
if host_id is None:
raise socket.gaierror
else:
return host_id
self._patch_socket_gethostbyname(side_effect=side_effect)
self.assertEqual(Network.are_hosts_same(*host_to_id), expect_hosts_are_same)
def test_are_hosts_same(
self,
host_to_id,
expect_hosts_are_same,
):
def side_effect(host):
host_id = host_to_id[host]
if host_id is None:
raise socket.gaierror
else:
return host_id
self._patch_socket_gethostbyname(side_effect=side_effect)
self.assertEqual(Network.are_hosts_same(*host_to_id),
expect_hosts_are_same)
def test_are_hosts_same_returns_false_if_rsa_keys_dont_match(self):
def popen_side_effect(*args, **kwargs):
if args[0] == 'ssh-keyscan -t rsa host_a':
mock_popen = Mock()
mock_popen.communicate.return_value = [b"a_host ssh-rsa the_value_a", None]
mock_popen.returncode = 0
return mock_popen
elif args[0] == 'ssh-keyscan -t rsa host_b':
mock_popen = Mock()
mock_popen.communicate.return_value = [b"a_host ssh-rsa the_other_value_b", None]
mock_popen.returncode = 0
return mock_popen
else:
return None
popen_patch = self.patch('subprocess.Popen')
popen_patch.side_effect = popen_side_effect
self.assertFalse(Network.are_hosts_same('host_a', 'host_b'))
def test_are_hosts_same_returns_false_if_rsa_keys_dont_match(self):
def popen_side_effect(*args, **kwargs):
if args[0] == 'ssh-keyscan -t rsa host_a':
mock_popen = Mock()
mock_popen.communicate.return_value = [
b"a_host ssh-rsa the_value_a", None
]
mock_popen.returncode = 0
return mock_popen
elif args[0] == 'ssh-keyscan -t rsa host_b':
mock_popen = Mock()
mock_popen.communicate.return_value = [
b"a_host ssh-rsa the_other_value_b", None
]
mock_popen.returncode = 0
return mock_popen
else:
return None
popen_patch = self.patch('subprocess.Popen')
popen_patch.side_effect = popen_side_effect
self.assertFalse(Network.are_hosts_same('host_a', 'host_b'))
def create(cls, host, user):
if Network.are_hosts_same(host, 'localhost'):
return LocalShellClient(host, user)
else:
return RemoteShellClient(host, user)
def test_are_hosts_same_returns_true_if_rsa_keys_match(self):
self._patch_popen_call_to_ssh_keyscan(0, b"a_host ssh-rsa the_same_byte_array", None)
self.assertTrue(Network.are_hosts_same('host1', 'host1_alias'))
def test_are_hosts_same_returns_false_if_rsa_key_is_none(self):
self._patch_popen_call_to_ssh_keyscan(1, 'some_output', 'some_error"')
self.assertFalse(Network.are_hosts_same('fail1', 'fail2'))
def test_are_hosts_same_returns_true_if_rsa_keys_match(self):
self._patch_popen_call_to_ssh_keyscan(
0, b"a_host ssh-rsa the_same_byte_array", None)
self.assertTrue(Network.are_hosts_same('host1', 'host1_alias'))
def test_are_hosts_same_returns_false_if_rsa_key_is_none(self):
self._patch_popen_call_to_ssh_keyscan(1, 'some_output', 'some_error"')
self.assertFalse(Network.are_hosts_same('fail1', 'fail2'))
