def test_environment_provider():
"""Creating and resetting the environment."""
cluster1 = ClusterConfigImpl(host='host1',
port=22,
user='******',
auth=AuthMethod.ASK)
config = ClientConfig(clusters={'cluster1': cluster1})
environment = EnvironmentImpl(config=config)
EnvironmentProvider._state = None # pylint: disable=protected-access
environment_provider = EnvironmentProvider(initial_environment=environment)
assert EnvironmentProvider._state == environment_provider.__dict__ # noqa, pylint: disable=protected-access,line-too-long
assert environment_provider.environment is environment
cluster2 = ClusterConfigImpl(host='host2',
port=22,
user='******',
auth=AuthMethod.ASK)
config2 = ClientConfig(clusters={'cluster2': cluster2})
environment2 = EnvironmentImpl(config=config2)
environment_provider2 = EnvironmentProvider(
initial_environment=environment2)
assert environment_provider2.environment is environment
environment_provider2.environment = environment2
assert environment_provider2.environment is environment2
assert environment_provider.environment is environment2
EnvironmentProvider._state = None # pylint: disable=protected-access
def test_sanitize_cluster_config():
cluster = ClusterConfigImpl(
host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK,
key='key1',
install_key=False,
disable_sshd=True,
setup_actions=SetupActionsConfigImpl(
jupyter=['command1'],
dask=['command2']),
scratch='/scratch1')
sanitized = sanitize_cluster_config(remote=cluster)
# Cleared some fields.
assert sanitized == ClusterConfigImpl(
host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK,
key=None,
install_key=True,
disable_sshd=True,
setup_actions=SetupActionsConfigImpl(
jupyter=['command1'],
dask=['command2']),
scratch='/scratch1')
def test_client_config_deserialize():
LoggerProvider().log_level = DEBUG
input_json = {
'clusters': {
'cluster1': {
'host': 'abc',
'user': '******',
'port': 22,
'auth': 'ASK',
'key': None,
'installKey': True,
'disableSshd': False,
'setupActions': {
'jupyter': [],
'dask': []
},
'scratch': '$HOME',
'retries': {}
}
},
'logLevel': DEBUG
}
client_config = ClientConfig(clusters={
'cluster1':
ClusterConfigImpl(host='abc',
user='******',
port=22,
auth=AuthMethod.ASK)
},
log_level=DEBUG)
deserialized = deserialize_client_config_from_json(input_json)
pprint([i.__dict__ for i in deserialized.clusters.values()])
assert deserialize_client_config_from_json(input_json) == client_config
def test_worker_deployment_script():
config = ClusterConfigImpl(
host='host',
port=1234,
user='******',
auth=AuthMethod.ASK,
setup_actions=SetupActionsConfigImpl(dask=['echo ABC', 'echo DEF']))
script = get_scheduler_deployment_script(remote_port=1111,
bokeh_port=2222,
scratch_subdir='/scratch',
log_file='/home/user/log',
config=config)
expected = (
'#!/usr/bin/env bash\n'
'echo ABC\n'
'echo DEF\n'
'export PATH="$PATH:$(python -m site --user-base)/bin"\n'
'dask-scheduler --host 0.0.0.0 --port 1111 --bokeh --bokeh-port 2222'
' --no-show --local-directory /scratch'
' > /home/user/log 2>&1\n'
'exit $?')
print()
print(repr(script))
print(repr(expected))
assert script == expected
def test_scheduler_deployment_script():
config = ClusterConfigImpl(
host='host',
port=1234,
user='******',
auth=AuthMethod.ASK,
setup_actions=SetupActionsConfigImpl(dask=['echo ABC',
'echo DEF']))
script = get_worker_deployment_script(
scheduler_address='tcp://localhost:1111/',
bokeh_port=2222,
scratch_subdir='/scratch',
cores=12,
memory_limit=bitmath.GiB(10),
log_file='/home/user/log',
config=config)
expected = (
'#!/usr/bin/env bash\n'
'echo ABC\n'
'echo DEF\n'
'export PATH="$PATH:$(python -m site --user-base)/bin"\n'
'dask-worker tcp://localhost:1111/ --host 0.0.0.0 --bokeh'
' --bokeh-port 2222 --nanny --reconnect --nprocs 1 --nthreads 12'
' --memory-limit 10737418240 --local-directory /scratch'
' > /home/user/log 2>&1\n'
'exit $?')
print()
print(repr(script))
print(repr(expected))
assert script == expected
def test_client_cluster_config_validation_is_used():
with pytest.raises(ValueError):
ClusterConfigImpl(host='', port=22, user='******', auth=AuthMethod.ASK)
with pytest.raises(ValueError):
ClusterConfigImpl(host='abc',
port=-1,
user='******',
auth=AuthMethod.ASK)
with pytest.raises(ValueError):
ClusterConfigImpl(host='abc', port=22, user='', auth=AuthMethod.ASK)
with pytest.raises(ValueError):
ClusterConfigImpl(host='abc',
port=22,
user='******',
auth=AuthMethod.PUBLIC_KEY,
key='')
def reset_environment(user: str, auth: AuthMethod = AuthMethod.ASK):
"""Clears the environment and adds the testing cluster.
:param user: User to connect to the cluster as,
and whose home dir should be cleaned.
:param auth: Authentication method to use.
"""
# pylint: disable=protected-access
saved_state = EnvironmentProvider._state
EnvironmentProvider._state = None
os.environ['IDACT_KEY_LOCATION'] = get_test_key_location(user=user)
os.environ['IDACT_CONFIG_PATH'] = get_test_environment_file(user=user)
cluster = ClusterConfigImpl(host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=auth,
retries=get_default_retries_heavy_load())
cluster.retries[Retry.PORT_INFO] = set_retry(count=0)
EnvironmentProvider(initial_environment=EnvironmentImpl(
config=ClientConfig(clusters={TEST_CLUSTER: cluster})))
set_log_level(DEBUG)
try:
yield
finally:
EnvironmentProvider._state = saved_state
clear_home(user=user)
def deserialize_client_config_from_json(data: dict) -> ClientConfig:
"""Deserializes :class:`.ClientConfig` from json.
:param data: json to deserialize.
"""
log = get_logger(__name__)
log.debug("Loaded config: %s", data)
if use_defaults_in_missing_fields(data=data):
log.debug("Filled missing fields with None: %s", data)
clusters = {
name: ClusterConfigImpl(host=value['host'],
port=value['port'],
user=value['user'],
auth=AuthMethod[value['auth']],
key=value['key'],
install_key=value['installKey'],
disable_sshd=value['disableSshd'],
setup_actions=SetupActionsConfigImpl(
jupyter=value['setupActions']['jupyter'],
dask=value['setupActions']['dask']),
scratch=value['scratch'],
notebook_defaults=value['notebookDefaults'],
retries=provide_defaults_for_retries(
deserialize_retries(value['retries'])),
use_jupyter_lab=value['useJupyterLab'])
for name, value in data['clusters'].items()
}
return ClientConfig(clusters=clusters, log_level=data['logLevel'])
def get_data_for_test():
config = ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK)
uuid = 'uuid1'
return config, uuid
def get_data_for_test():
config = ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK)
access_node = NodeImpl(config=config)
return config, access_node
def test_entry_point_disabled_sshd():
config = ClusterConfigImpl(host='host',
port=123,
user='******',
auth=AuthMethod.PUBLIC_KEY,
disable_sshd=True)
formatted = get_entry_point_script_contents(config=config)
assert formatted == ("#!/usr/bin/env bash\n"
"trap : TERM INT; sleep infinity & wait")
def test_cluster_create():
"""Tests the construction of a cluster object from config."""
client_cluster_config = ClusterConfigImpl(host='abc',
port=22,
user='******',
auth=AuthMethod.ASK)
ClusterImpl(name='cluster', config=client_cluster_config)
assert str(client_cluster_config) == repr(client_cluster_config)
def test_third_try_succeeds_some_retries_with_config():
failures = []
config = ClusterConfigImpl(
host='h',
port=1,
user='******',
auth=AuthMethod.ASK,
retries={Retry.PORT_INFO: RetryConfigImpl(count=3, seconds_between=0)})
assert retry_with_config(
fun=lambda: failing_task(fail_times=2, failures=failures),
name=Retry.PORT_INFO,
config=config) == 123
assert failures == [0, 1]
def test_create_environment():
"""Creating the environment and adding a cluster."""
cluster1 = ClusterConfigImpl(host='host1',
port=22,
user='******',
auth=AuthMethod.ASK)
config = ClientConfig(clusters={'cluster1': cluster1})
environment = EnvironmentImpl(config=config)
assert environment.config is config
assert len(environment.config.clusters) == 1
assert len(environment.clusters) == 1
assert isinstance(environment.clusters['cluster1'], Cluster)
cluster2 = ClusterConfigImpl(host='host2',
port=22,
user='******',
auth=AuthMethod.ASK)
environment.add_cluster(name='cluster2', config=cluster2)
assert len(environment.config.clusters) == 2
assert len(environment.clusters) == 2
assert isinstance(environment.clusters['cluster2'], Cluster)
def get_data_for_test():
config = ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK)
uuid = 'uuid1'
deployment = GenericDeployment(node=NodeImpl(config=config),
pid=111,
runtime_dir='/dir1')
serialized_deployment = deployment.serialize()
return config, uuid, deployment, serialized_deployment
def test_client_cluster_config_create():
config = ClusterConfigImpl(host='abc',
port=22,
user='******',
auth=AuthMethod.ASK)
assert config.host == 'abc'
assert config.port == 22
assert config.user == 'user'
assert config.auth == AuthMethod.ASK
assert config.key is None
assert config.install_key
assert not config.disable_sshd
assert config.setup_actions.jupyter == []
assert config.setup_actions.dask == []
assert config.scratch == "$HOME"
assert config.retries == get_default_retries()
assert config.use_jupyter_lab
def get_deployment_for_test(uuid: str, job_id: int) -> NodesImpl:
config = ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK)
access_node = NodeImpl(config=config)
nodes = [NodeImpl(config=config),
NodeImpl(config=config)]
deployment = NodesImpl(nodes=nodes,
allocation=SlurmAllocation(
job_id=job_id,
access_node=access_node,
nodes=nodes,
entry_point_script_path='a',
parameters=AllocationParameters()),
uuid=uuid)
return deployment
def run_tunnel_test_for_bindings(user: str, bindings: List[Binding]):
"""Runs a tunneling test for a binding sequence.
Runs a Python server in a separate thread through ssh, then creates
a multi-hop tunnel, and finally performs a HTTP request to the local
address.
:param user: Test user.
:param bindings: Sequence of tunnel bindings.
"""
config = ClusterConfigImpl(host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK)
local_port = bindings[0].port
server_port = bindings[-1].port
with ExitStack() as stack:
tunnel = build_tunnel(config=config,
bindings=bindings,
ssh_password=get_test_user_password(user))
stack.enter_context(close_tunnel_on_exit(tunnel))
server = start_dummy_server_thread(user=user, server_port=server_port)
stack.enter_context(join_on_exit(server))
assert tunnel.here == local_port
assert tunnel.there == server_port
def access_dummy_server():
return requests.get(
"http://127.0.0.1:{local_port}".format(local_port=local_port))
request = retry(access_dummy_server,
retries=5 * get_testing_process_count(),
seconds_between_retries=2)
assert "text/html" in request.headers['Content-type']
def test_client_config_create_empty_and_add_cluster():
client_config = ClientConfig()
assert client_config.clusters == {}
cluster_cluster_config = VALID_CLIENT_CLUSTER_CONFIG
with pytest.raises(ValueError):
client_config.add_cluster(' Illegal Cluster Name',
cluster_cluster_config)
assert client_config.clusters == {}
client_config.add_cluster('cluster1', cluster_cluster_config)
assert client_config.clusters['cluster1'] is cluster_cluster_config
with pytest.raises(ValueError):
client_config.add_cluster(
'cluster1',
ClusterConfigImpl(host='another',
port=22,
user='******',
auth=AuthMethod.ASK))
assert client_config.clusters['cluster1'] is cluster_cluster_config
def test_entry_point_sshd():
config = ClusterConfigImpl(host='host',
port=123,
user='******',
auth=AuthMethod.PUBLIC_KEY,
disable_sshd=False)
formatted = get_entry_point_script_contents(config=config)
expected = (
"#!/usr/bin/env bash\n"
"SSHD_PORT=$(python -c 'import socket;"
" s=socket.socket();"
" s.bind((str(), 0)); print(s.getsockname()[1]);"
" s.close()')\n"
"mkdir -p ~/.idact/sshd_ports/alloc-$IDACT_ALLOCATION_ID\n"
"chmod 700 ~/.idact/sshd_ports/alloc-$IDACT_ALLOCATION_ID\n"
"touch ~/.idact/sshd_ports/alloc-$IDACT_ALLOCATION_ID/$(hostname):$SSHD_PORT\n" # noqa, pylint: disable=line-too-long
"export PATH=\"$PATH:/usr/sbin\"\n"
" $(which sshd)"
" -D "
" -f /dev/null"
" -oPort=$SSHD_PORT"
" -oListenAddress=0.0.0.0"
" -oHostKey=~/.ssh/ssh_host_rsa_key"
" -oProtocol=2"
" -oAllowUsers=$USER"
" -oPermitRootLogin=no"
" -oStrictModes=yes"
" -oPubkeyAuthentication=yes"
" -oAuthorizedKeysFile=.ssh/authorized_keys.idact"
" -oPasswordAuthentication=no"
" -oChallengeResponseAuthentication=no"
" -oKerberosAuthentication=no"
" -oGSSAPIAuthentication=no"
" -oUsePAM=no"
" -oSubsystem='sftp internal-sftp'"
" -oX11Forwarding=yes\n"
"exit $?")
assert formatted == expected
def test_get_password():
config = ClusterConfigImpl(
host='host',
port=1234,
user='******',
auth=AuthMethod.ASK)
PasswordCache()._password = None # pylint: disable=protected-access
output = []
def fake_getpass(prompt: str) -> str:
output.append(prompt)
return 'fakepass'
saved_getpass = None
try:
saved_getpass = getpass.getpass
getpass.getpass = fake_getpass
assert get_password(config) == 'fakepass'
assert output == ["Password for user@host:1234: "]
finally:
if saved_getpass is not None:
getpass.getpass = saved_getpass
def add_cluster(name: str,
user: str,
host: str,
port: int = 22,
auth: Optional[AuthMethod] = None,
key: Union[None, str, KeyType] = None,
install_key: bool = True,
disable_sshd: bool = False,
setup_actions: Optional[SetupActionsConfig] = None,
scratch: Optional[str] = None,
retries: Optional[Dict[Retry, RetryConfig]] = None,
use_jupyter_lab: bool = True) -> Cluster:
"""Adds a new cluster.
:param name:
Cluster name to identify it by.
:param user:
Remote cluster user to log in and run commands as.
:param host:
Cluster access node hostname.
:param port:
SSH port to access the cluster.
Default: 22.
:param auth:
Authentication method.
Default: :attr:`.AuthMethod.ASK` (password-based).
:param key:
Private key path (if applicable), or key type to generate.
Default: None
:param install_key:
True, if the public key should be installed on the cluster before
use (if applicable).
Default: True
:param disable_sshd:
Should be set to True, if the cluster allows ssh connection
to compute nodes out of the box.
Default: False
:param setup_actions:
Commands to run before deployment.
Default: None
:param setup_actions:
Commands to run before deployment.
Default: None
:param scratch:
Absolute path to a high-performance filesystem for temporary
computation data, or an environment variable that contains it.
Default: $HOME
:param retries:
Retry config by action name.
Defaults: see :func:`.get_default_retries`.
:param use_jupyter_lab:
Use Jupyter Lab instead of Jupyter Notebook.
Default: True.
"""
log = get_logger(__name__)
environment = EnvironmentProvider().environment
if auth is None:
log.info("No auth method specified, defaulting to password-based.")
auth = AuthMethod.ASK
if auth is AuthMethod.PUBLIC_KEY:
if isinstance(key, KeyType):
log.info("Generating public-private key pair.")
key = generate_key(host=host, key_type=key)
elif isinstance(key, str):
key = os.path.expanduser(key)
else:
raise ValueError("Invalid key argument for public key"
" authentication.")
config = ClusterConfigImpl(host=host,
port=port,
user=user,
auth=auth,
key=key,
install_key=install_key,
disable_sshd=disable_sshd,
setup_actions=setup_actions,
scratch=scratch,
retries=retries,
use_jupyter_lab=use_jupyter_lab)
return environment.add_cluster(name=name, config=config)
def test_merge_cluster_configs():
cluster_1 = ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK,
key='key1',
install_key=False,
disable_sshd=True,
setup_actions=SetupActionsConfigImpl(
jupyter=['command1'],
dask=['command2']),
scratch='/scratch1')
cluster_2 = ClusterConfigImpl(host='localhost2',
port=2,
user='******',
auth=AuthMethod.PUBLIC_KEY,
key='key2',
install_key=True,
disable_sshd=False,
setup_actions=SetupActionsConfigImpl(
jupyter=['command3'],
dask=['command4']),
scratch='/scratch2')
merged_1_2 = merge_cluster_configs(local=cluster_1,
remote=cluster_2)
merged_2_1 = merge_cluster_configs(local=cluster_2,
remote=cluster_1)
# Kept some fields from 1 after merge, replaced rest with 2.
assert merged_1_2 == ClusterConfigImpl(
host='localhost2',
port=2,
user='******',
auth=AuthMethod.PUBLIC_KEY,
key='key1',
install_key=False,
disable_sshd=False,
setup_actions=SetupActionsConfigImpl(
jupyter=['command3'],
dask=['command4']),
scratch='/scratch2')
# Kept some fields from 2 after merge, replaced rest with 1.
assert merged_2_1 == ClusterConfigImpl(
host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK,
key='key2',
install_key=True,
disable_sshd=True,
setup_actions=SetupActionsConfigImpl(
jupyter=['command1'],
dask=['command2']),
scratch='/scratch1')
# No changes.
assert cluster_1 == ClusterConfigImpl(
host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK,
key='key1',
install_key=False,
disable_sshd=True,
setup_actions=SetupActionsConfigImpl(
jupyter=['command1'],
dask=['command2']),
scratch='/scratch1')
# No changes.
assert cluster_2 == ClusterConfigImpl(
host='localhost2',
port=2,
user='******',
auth=AuthMethod.PUBLIC_KEY,
key='key2',
install_key=True,
disable_sshd=False,
setup_actions=SetupActionsConfigImpl(
jupyter=['command3'],
dask=['command4']),
scratch='/scratch2')
def get_config_for_test():
return ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=AuthMethod.ASK)
def check_able_to_pull_environment(user: str,
remote_environment_upload_path: str,
remote_environment_pull_path: Optional[
str]):
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
stack.enter_context(remove_remote_file(
user=user,
path=remote_environment_upload_path))
cluster = show_cluster(name=TEST_CLUSTER)
cluster.config.key = None
cluster.config.install_key = False
assert len(show_clusters()) == 1
fake_cluster = 'fake cluster'
remote_environment = EnvironmentImpl(
config=ClientConfig(
clusters={
TEST_CLUSTER: ClusterConfigImpl(
host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK,
key='key_remote',
install_key=True),
fake_cluster: ClusterConfigImpl(
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key='key_remote',
install_key=True)}))
remote_environment_serialized = serialize_environment(
environment=remote_environment)
node = cluster.get_access_node()
assert isinstance(node, NodeInternal)
put_file_on_node(node=node,
remote_path=remote_environment_upload_path,
contents=remote_environment_serialized)
pull_environment(cluster=cluster,
path=remote_environment_pull_path)
assert len(show_clusters()) == 2
# Local key was kept unchanged.
assert show_cluster(TEST_CLUSTER).config == ClusterConfigImpl(
host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=False)
# New cluster was sanitized
assert show_cluster(fake_cluster).config == ClusterConfigImpl(
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=True)
def check_able_to_merge_push_environment(
user: str, remote_environment_upload_path: str,
remote_environment_push_path: Optional[str]):
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
stack.enter_context(
remove_remote_file(user=user, path=remote_environment_upload_path))
cluster = show_cluster(name=TEST_CLUSTER)
cluster.config.key = None
cluster.config.install_key = False
cluster.config.retries[Retry.PORT_INFO] = \
get_default_retries_heavy_load()[Retry.PORT_INFO]
assert len(show_clusters()) == 1
node = cluster.get_access_node()
assert isinstance(node, NodeInternal)
# Upload an environment to be merged on push.
initial_remote_environment = EnvironmentImpl(config=ClientConfig(
clusters={
TEST_CLUSTER:
ClusterConfigImpl(host=get_testing_host(),
port=123,
user=user,
auth=AuthMethod.ASK,
key='key_remote',
install_key=True)
}))
initial_remote_environment_serialized = serialize_environment(
environment=initial_remote_environment)
put_file_on_node(node=node,
remote_path=remote_environment_upload_path,
contents=initial_remote_environment_serialized)
# Modify current environment.
fake_cluster = 'fake cluster'
add_cluster(name=fake_cluster,
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key='key_local',
install_key=False)
# Push with merge.
push_environment(cluster=cluster, path=remote_environment_push_path)
remote_environment_after_push_serialized = \
get_file_from_node(node=node,
remote_path=remote_environment_upload_path)
remote_environment_after_push = deserialize_environment(
text=remote_environment_after_push_serialized)
# Remote key was kept unchanged.
remote_clusters = remote_environment_after_push.config.clusters
assert len(remote_clusters) == 2
assert remote_clusters[TEST_CLUSTER].__dict__ == ClusterConfigImpl(
host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK,
key='key_remote',
install_key=True,
retries=get_default_retries_heavy_load()).__dict__
# New cluster was sanitized
assert remote_clusters[fake_cluster].__dict__ == ClusterConfigImpl(
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=True).__dict__
def check_able_to_push_new_environment(
user: str, remote_environment_expected_path: str,
remote_environment_push_path: Optional[str]):
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
stack.enter_context(
remove_remote_file(user=user,
path=remote_environment_expected_path))
cluster = show_cluster(name=TEST_CLUSTER)
cluster.config.key = None
cluster.config.install_key = False
cluster.config.retries[Retry.PORT_INFO] = \
get_default_retries_heavy_load()[Retry.PORT_INFO]
assert len(show_clusters()) == 1
node = cluster.get_access_node()
assert isinstance(node, NodeInternal)
# Modify current environment.
fake_cluster = 'fake cluster'
add_cluster(name=fake_cluster,
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key='key_local',
install_key=False)
# Target file does not exist.
with pytest.raises(RuntimeError):
node.run("cat {}".format(remote_environment_expected_path))
push_environment(cluster=cluster, path=remote_environment_push_path)
remote_environment_after_push_serialized = \
get_file_from_node(node=node,
remote_path=remote_environment_expected_path)
remote_environment_after_push = deserialize_environment(
text=remote_environment_after_push_serialized)
# Both clusters were sanitized.
remote_clusters = remote_environment_after_push.config.clusters
assert len(remote_clusters) == 2
assert remote_clusters[TEST_CLUSTER] == ClusterConfigImpl(
host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=True,
retries=get_default_retries_heavy_load())
assert remote_clusters[fake_cluster] == ClusterConfigImpl(
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=True)
def test_merge_common_clusters():
auth = AuthMethod.PUBLIC_KEY
target_clusters = {'a': ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=auth),
'b': ClusterConfigImpl(host='localhost2',
port=2,
user='******',
auth=auth),
'c': ClusterConfigImpl(host='localhost3',
port=3,
user='******',
auth=auth)}
remote_clusters = {'b': ClusterConfigImpl(host='localhost4',
port=4,
user='******',
auth=auth),
'c': ClusterConfigImpl(host='localhost5',
port=5,
user='******',
auth=auth),
'd': ClusterConfigImpl(host='localhost6',
port=6,
user='******',
auth=auth)}
merge_common_clusters(remote_clusters=remote_clusters,
target_clusters=target_clusters)
# Merged common clusters with remote.
assert target_clusters == {'a': ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=auth),
'b': ClusterConfigImpl(host='localhost4',
port=4,
user='******',
auth=auth),
'c': ClusterConfigImpl(host='localhost5',
port=5,
user='******',
auth=auth)}
# No changes.
assert remote_clusters == {'b': ClusterConfigImpl(host='localhost4',
port=4,
user='******',
auth=auth),
'c': ClusterConfigImpl(host='localhost5',
port=5,
user='******',
auth=auth),
'd': ClusterConfigImpl(host='localhost6',
port=6,
user='******',
auth=auth)}
def test_sanitize_new_clusters():
auth = AuthMethod.PUBLIC_KEY
target_clusters = {'a': ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=auth),
'b': ClusterConfigImpl(host='localhost2',
port=2,
user='******',
auth=auth),
'c': ClusterConfigImpl(host='localhost3',
port=3,
user='******',
auth=auth)}
remote_clusters = {'b': ClusterConfigImpl(host='localhost4',
port=4,
user='******',
auth=auth),
'c': ClusterConfigImpl(host='localhost5',
port=5,
user='******',
auth=auth),
'd': ClusterConfigImpl(host='localhost6',
port=6,
user='******',
auth=auth)}
sanitize_and_add_new_clusters(remote_clusters=remote_clusters,
target_clusters=target_clusters)
# Added new cluster.
assert target_clusters == {'a': ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=auth),
'b': ClusterConfigImpl(host='localhost2',
port=2,
user='******',
auth=auth),
'c': ClusterConfigImpl(host='localhost3',
port=3,
user='******',
auth=auth),
'd': ClusterConfigImpl(host='localhost6',
port=6,
user='******',
auth=auth)}
# No changes.
assert remote_clusters == {'b': ClusterConfigImpl(host='localhost4',
port=4,
user='******',
auth=auth),
'c': ClusterConfigImpl(host='localhost5',
port=5,
user='******',
auth=auth),
'd': ClusterConfigImpl(host='localhost6',
port=6,
user='******',
auth=auth)}
def test_merge_environments():
auth = AuthMethod.PUBLIC_KEY
environment_1 = EnvironmentImpl(
config=ClientConfig(clusters={'a': ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=auth,
key='key1'),
'b': ClusterConfigImpl(host='localhost2',
port=2,
user='******',
auth=auth,
key='key1'),
'c': ClusterConfigImpl(host='localhost3',
port=3,
user='******',
auth=auth)},
log_level=10))
environment_2 = EnvironmentImpl(
config=ClientConfig(clusters={'b': ClusterConfigImpl(host='localhost4',
port=4,
user='******',
auth=auth,
key='key2'),
'c': ClusterConfigImpl(host='localhost5',
port=5,
user='******',
auth=auth),
'd': ClusterConfigImpl(host='localhost6',
port=6,
user='******',
auth=auth,
key='key2')},
log_level=20))
merged_1_2 = merge_environments(local=environment_1,
remote=environment_2)
# Kept some fields from 1 after merge, replaced rest with 2.
# Sanitized some fields for new clusters from 2.
assert merged_1_2 == EnvironmentImpl(
config=ClientConfig(clusters={'a': ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=auth,
key='key1'),
'b': ClusterConfigImpl(host='localhost4',
port=4,
user='******',
auth=auth,
key='key1'),
'c': ClusterConfigImpl(host='localhost5',
port=5,
user='******',
auth=auth),
'd': ClusterConfigImpl(host='localhost6',
port=6,
user='******',
auth=auth,
key=None)},
log_level=10))
merged_2_1 = merge_environments(local=environment_2,
remote=environment_1)
# Kept some fields from 2 after merge, replaced rest with 1.
# Sanitized some fields for new clusters from 1.
assert merged_2_1 == EnvironmentImpl(
config=ClientConfig(clusters={'a': ClusterConfigImpl(host='localhost1',
port=1,
user='******',
auth=auth,
key=None),
'b': ClusterConfigImpl(host='localhost2',
port=2,
user='******',
auth=auth,
key='key2'),
'c': ClusterConfigImpl(host='localhost3',
port=3,
user='******',
auth=auth),
'd': ClusterConfigImpl(host='localhost6',
port=6,
user='******',
auth=auth,
key='key2')},
log_level=20))