def run(self, log_level, master_url, remote_file=None, build_type=None, **request_params): """ Execute a build and wait for it to complete. :param log_level: the log level at which to do application logging (or None for default log level) :type log_level: str | None :param master_url: the url (specified by the user) of the master to which we should send the build :type master_url: str | None :param remote_file: a list of remote files where each element contains the output file name and the resource URL :type remote_file: list[list[str]] | None :param build_type: the build type of the request to be sent (e.g., "git", "directory"). If not specified will default to the "directory" project type. :type build_type: str | None :param request_params: key-value pairs to be provided as build parameters in the build request :type request_params: dict """ log_level = log_level or Configuration['log_level'] log.configure_logging(log_level=log_level, simplified_console_logs=True) request_params['type'] = build_type or request_params.get( 'type') or 'directory' if remote_file: request_params['remote_files'] = { name: url for name, url in remote_file } operational_master_url = master_url or '{}:{}'.format( Configuration['hostname'], Configuration['port']) # If running a single master, single slave--both on localhost--we need to launch services locally. if master_url is None and Network.are_hosts_same(Configuration['master_hostname'], 'localhost') \ and len(Configuration['slaves']) == 1 \ and Network.are_hosts_same(Configuration['slaves'][0], 'localhost'): self._start_local_services_if_needed(operational_master_url) if request_params['type'] == 'directory': request_params['project_directory'] = request_params.get( 'project_directory') or os.getcwd() runner = BuildRunner(master_url=operational_master_url, request_params=request_params, secret=Secret.get()) if not runner.run(): sys.exit(1)
def _deploy_binaries_and_conf(self, host, username, current_executable, binaries_tar_path, in_use_conf_path): """ Move binaries and conf to single host. :param host: host to deploy to :type host: str :param username: current username :param current_executable: path to the executable (ie: /usr/bin/python, ./clusterrunner, etc) :type current_executable: str :param binaries_tar_path: path to tar.gz file of clusterrunner binaries :type binaries_tar_path: str :param in_use_conf_path: path toe currently used conf file :type in_use_conf_path: str """ clusterrunner_dir = join(os.path.expanduser('~'), '.clusterrunner') clusterrunner_executable_dir = join(clusterrunner_dir, 'dist') clusterrunner_executable_deploy_target = join(clusterrunner_executable_dir, 'clusterrunner') clusterrunner_conf_deploy_target = join(clusterrunner_dir, 'clusterrunner.conf') deploy_target = DeployTarget(host, username) if Network.are_hosts_same(host, 'localhost'): # Do not want to overwrite the currently running executable. if current_executable != clusterrunner_executable_deploy_target: deploy_target.deploy_binary(binaries_tar_path, clusterrunner_executable_dir) # Do not want to overwrite the currently used conf. if in_use_conf_path != clusterrunner_conf_deploy_target: deploy_target.deploy_conf(in_use_conf_path, clusterrunner_conf_deploy_target) else: deploy_target.deploy_binary(binaries_tar_path, clusterrunner_executable_dir) deploy_target.deploy_conf(in_use_conf_path, clusterrunner_conf_deploy_target)
def run(self, log_level, master_url, remote_file=None, build_type=None, **request_params): """ Execute a build and wait for it to complete. :param log_level: the log level at which to do application logging (or None for default log level) :type log_level: str | None :param master_url: the url (specified by the user) of the master to which we should send the build :type master_url: str | None :param remote_file: a list of remote files where each element contains the output file name and the resource URL :type remote_file: list[list[str]] | None :param build_type: the build type of the request to be sent (e.g., "git", "directory"). If not specified will default to the "directory" project type. :type build_type: str | None :param request_params: key-value pairs to be provided as build parameters in the build request :type request_params: dict """ log_level = log_level or Configuration['log_level'] log.configure_logging(log_level=log_level, simplified_console_logs=True) request_params['type'] = build_type or request_params.get('type') or 'directory' if remote_file: request_params['remote_files'] = {name: url for name, url in remote_file} operational_master_url = master_url or '{}:{}'.format(Configuration['hostname'], Configuration['port']) # If running a single master, single slave--both on localhost--we need to launch services locally. if master_url is None and Network.are_hosts_same(Configuration['master_hostname'], 'localhost') \ and len(Configuration['slaves']) == 1 \ and Network.are_hosts_same(Configuration['slaves'][0], 'localhost'): self._start_local_services_if_needed(operational_master_url) if request_params['type'] == 'directory': request_params['project_directory'] = request_params.get('project_directory') or os.getcwd() runner = BuildRunner(master_url=operational_master_url, request_params=request_params, secret=Secret.get()) if not runner.run(): sys.exit(1)
def test_are_hosts_same( self, host_to_id, expect_hosts_are_same, ): def side_effect(host): host_id = host_to_id[host] if host_id is None: raise socket.gaierror else: return host_id self._patch_socket_gethostbyname(side_effect=side_effect) self.assertEqual(Network.are_hosts_same(*host_to_id), expect_hosts_are_same)
def test_are_hosts_same( self, host_to_id, expect_hosts_are_same, ): def side_effect(host): host_id = host_to_id[host] if host_id is None: raise socket.gaierror else: return host_id self._patch_socket_gethostbyname(side_effect=side_effect) self.assertEqual(Network.are_hosts_same(*host_to_id), expect_hosts_are_same)
def test_are_hosts_same_returns_false_if_rsa_keys_dont_match(self): def popen_side_effect(*args, **kwargs): if args[0] == 'ssh-keyscan -t rsa host_a': mock_popen = Mock() mock_popen.communicate.return_value = [b"a_host ssh-rsa the_value_a", None] mock_popen.returncode = 0 return mock_popen elif args[0] == 'ssh-keyscan -t rsa host_b': mock_popen = Mock() mock_popen.communicate.return_value = [b"a_host ssh-rsa the_other_value_b", None] mock_popen.returncode = 0 return mock_popen else: return None popen_patch = self.patch('subprocess.Popen') popen_patch.side_effect = popen_side_effect self.assertFalse(Network.are_hosts_same('host_a', 'host_b'))
def test_are_hosts_same_returns_false_if_rsa_keys_dont_match(self): def popen_side_effect(*args, **kwargs): if args[0] == 'ssh-keyscan -t rsa host_a': mock_popen = Mock() mock_popen.communicate.return_value = [ b"a_host ssh-rsa the_value_a", None ] mock_popen.returncode = 0 return mock_popen elif args[0] == 'ssh-keyscan -t rsa host_b': mock_popen = Mock() mock_popen.communicate.return_value = [ b"a_host ssh-rsa the_other_value_b", None ] mock_popen.returncode = 0 return mock_popen else: return None popen_patch = self.patch('subprocess.Popen') popen_patch.side_effect = popen_side_effect self.assertFalse(Network.are_hosts_same('host_a', 'host_b'))
def create(cls, host, user): if Network.are_hosts_same(host, 'localhost'): return LocalShellClient(host, user) else: return RemoteShellClient(host, user)
def test_are_hosts_same_returns_true_if_rsa_keys_match(self): self._patch_popen_call_to_ssh_keyscan(0, b"a_host ssh-rsa the_same_byte_array", None) self.assertTrue(Network.are_hosts_same('host1', 'host1_alias'))
def test_are_hosts_same_returns_false_if_rsa_key_is_none(self): self._patch_popen_call_to_ssh_keyscan(1, 'some_output', 'some_error"') self.assertFalse(Network.are_hosts_same('fail1', 'fail2'))
def test_are_hosts_same_returns_true_if_rsa_keys_match(self): self._patch_popen_call_to_ssh_keyscan( 0, b"a_host ssh-rsa the_same_byte_array", None) self.assertTrue(Network.are_hosts_same('host1', 'host1_alias'))
def test_are_hosts_same_returns_false_if_rsa_key_is_none(self): self._patch_popen_call_to_ssh_keyscan(1, 'some_output', 'some_error"') self.assertFalse(Network.are_hosts_same('fail1', 'fail2'))