def test_environment():
user = USER_2
test_environment_file = get_test_environment_file(user=user)
with ExitStack() as stack:
stack.enter_context(clear_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
clusters = show_clusters()
assert clusters == {}
cluster = add_cluster(name=TEST_CLUSTER,
user=user,
host='localhost',
port=2222)
clusters = show_clusters()
assert show_cluster(name=TEST_CLUSTER) is cluster
assert len(show_clusters()) == 1
assert clusters[TEST_CLUSTER] == cluster
assert cluster.name == TEST_CLUSTER
try:
save_environment(path=test_environment_file)
with clear_environment(user):
assert show_clusters() == {}
load_environment(path=test_environment_file)
cluster2 = show_cluster(name=TEST_CLUSTER)
assert cluster2 is not cluster
assert cluster2 == cluster
finally:
os.remove(test_environment_file)
def test_environment_missing_and_defaults():
user = USER_25
with ExitStack() as stack:
stack.enter_context(clear_environment(user))
EnvironmentProvider()._environment = None # noqa, pylint: disable=protected-access,line-too-long
assert show_clusters() == {}
assert not os.path.isfile(os.environ['IDACT_CONFIG_PATH'])
with pytest.raises(ValueError):
load_environment()
add_cluster(name=TEST_CLUSTER, user=user, host='localhost')
config = show_cluster(TEST_CLUSTER).config
set_log_level(logging.DEBUG)
check_config_is_default(config=config, user=user)
try:
save_environment()
with open(os.environ['IDACT_CONFIG_PATH'], 'r') as test_file:
contents = test_file.read().splitlines()
pprint(contents)
assert contents == get_default_config_contents(user=user)
finally:
os.remove(os.environ['IDACT_CONFIG_PATH'])
def test_environment_create_modify_save_load():
user = USER_19
test_environment_file = get_test_environment_file(user=user)
test_environment_file2 = get_test_environment_file(user=user) + '_2'
with ExitStack() as stack:
stack.enter_context(clear_environment(user))
assert show_clusters() == {}
add_cluster(name=TEST_CLUSTER, user=user, host='localhost')
config = show_cluster(TEST_CLUSTER).config
check_config_is_default(config=config, user=user)
try:
save_environment(path=test_environment_file)
with open(test_environment_file, 'r') as test_file:
contents = test_file.read().splitlines()
pprint(contents)
assert contents == get_default_config_contents(user=user)
config = show_cluster(TEST_CLUSTER).config
config.host = 'localhost2'
config.port = 2222
config.user = '******'
config.auth = AuthMethod.PUBLIC_KEY
config.key = './fake-key'
config.install_key = False
config.disable_sshd = True
config.setup_actions.jupyter = ['abc']
config.setup_actions.dask = ['abc', 'def']
config.scratch = '$HOME2'
config.use_jupyter_lab = False
set_log_level(logging.INFO)
check_config_is_modified(config=config)
save_environment(path=test_environment_file2)
with open(test_environment_file2, 'r') as test_file:
contents = test_file.read().splitlines()
pprint(contents)
assert contents == get_modified_config_contents()
load_environment(test_environment_file)
config = show_cluster(TEST_CLUSTER).config
check_config_is_default(config=config, user=user)
load_environment(test_environment_file2)
config = show_cluster(TEST_CLUSTER).config
check_config_is_modified(config=config)
finally:
try:
os.remove(test_environment_file)
finally:
os.remove(test_environment_file2)
def test_environment_add_cluster_and_remove():
user = USER_26
with ExitStack() as stack:
stack.enter_context(clear_environment(user))
assert show_clusters() == {}
add_cluster(name=TEST_CLUSTER, user=user, host='localhost')
add_cluster(name='fake cluster', user=user, host='localhost2')
assert len(show_clusters()) == 2
cluster = show_cluster('fake cluster')
assert cluster.config.host == 'localhost2'
remove_cluster('fake cluster')
assert len(show_clusters()) == 1
add_cluster(name='fake cluster', user=user, host='localhost3')
cluster2 = show_cluster('fake cluster')
assert cluster.config.host == 'localhost2'
assert cluster2.config.host == 'localhost3'
cluster3 = show_cluster(TEST_CLUSTER)
assert cluster3.config.host == 'localhost'
remove_cluster('fake cluster')
remove_cluster(TEST_CLUSTER)
assert show_clusters() == {}
with pytest.raises(KeyError):
remove_cluster('fake cluster')
def test_basic():
user = USER_1
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(set_up_key_location(user))
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
clusters = show_clusters()
print(clusters)
assert len(clusters) == 1
cluster = show_cluster(name=TEST_CLUSTER)
print(cluster)
assert clusters[TEST_CLUSTER] == cluster
nodes = cluster.allocate_nodes(nodes=2,
cores=1,
memory_per_node=MiB(100),
walltime=Walltime(minutes=30),
native_args={'--partition': 'debug'})
with cancel_on_exit(nodes):
assert len(nodes) == 2
assert nodes[0] in nodes
print(nodes)
assert str(nodes) == repr(nodes)
nodes.wait(timeout=SLURM_WAIT_TIMEOUT)
assert nodes.running()
print(nodes)
print(nodes[0])
assert nodes[0].run('whoami') == user
assert nodes[1].run('whoami') == user
assert not nodes.running()
with pytest.raises(RuntimeError):
nodes.wait()
with pytest.raises(RuntimeError):
nodes[0].run('whoami')
def check_able_to_pull_environment(user: str,
remote_environment_upload_path: str,
remote_environment_pull_path: Optional[
str]):
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
stack.enter_context(remove_remote_file(
user=user,
path=remote_environment_upload_path))
cluster = show_cluster(name=TEST_CLUSTER)
cluster.config.key = None
cluster.config.install_key = False
assert len(show_clusters()) == 1
fake_cluster = 'fake cluster'
remote_environment = EnvironmentImpl(
config=ClientConfig(
clusters={
TEST_CLUSTER: ClusterConfigImpl(
host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK,
key='key_remote',
install_key=True),
fake_cluster: ClusterConfigImpl(
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key='key_remote',
install_key=True)}))
remote_environment_serialized = serialize_environment(
environment=remote_environment)
node = cluster.get_access_node()
assert isinstance(node, NodeInternal)
put_file_on_node(node=node,
remote_path=remote_environment_upload_path,
contents=remote_environment_serialized)
pull_environment(cluster=cluster,
path=remote_environment_pull_path)
assert len(show_clusters()) == 2
# Local key was kept unchanged.
assert show_cluster(TEST_CLUSTER).config == ClusterConfigImpl(
host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=False)
# New cluster was sanitized
assert show_cluster(fake_cluster).config == ClusterConfigImpl(
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=True)
def check_able_to_merge_push_environment(
user: str, remote_environment_upload_path: str,
remote_environment_push_path: Optional[str]):
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
stack.enter_context(
remove_remote_file(user=user, path=remote_environment_upload_path))
cluster = show_cluster(name=TEST_CLUSTER)
cluster.config.key = None
cluster.config.install_key = False
cluster.config.retries[Retry.PORT_INFO] = \
get_default_retries_heavy_load()[Retry.PORT_INFO]
assert len(show_clusters()) == 1
node = cluster.get_access_node()
assert isinstance(node, NodeInternal)
# Upload an environment to be merged on push.
initial_remote_environment = EnvironmentImpl(config=ClientConfig(
clusters={
TEST_CLUSTER:
ClusterConfigImpl(host=get_testing_host(),
port=123,
user=user,
auth=AuthMethod.ASK,
key='key_remote',
install_key=True)
}))
initial_remote_environment_serialized = serialize_environment(
environment=initial_remote_environment)
put_file_on_node(node=node,
remote_path=remote_environment_upload_path,
contents=initial_remote_environment_serialized)
# Modify current environment.
fake_cluster = 'fake cluster'
add_cluster(name=fake_cluster,
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key='key_local',
install_key=False)
# Push with merge.
push_environment(cluster=cluster, path=remote_environment_push_path)
remote_environment_after_push_serialized = \
get_file_from_node(node=node,
remote_path=remote_environment_upload_path)
remote_environment_after_push = deserialize_environment(
text=remote_environment_after_push_serialized)
# Remote key was kept unchanged.
remote_clusters = remote_environment_after_push.config.clusters
assert len(remote_clusters) == 2
assert remote_clusters[TEST_CLUSTER].__dict__ == ClusterConfigImpl(
host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK,
key='key_remote',
install_key=True,
retries=get_default_retries_heavy_load()).__dict__
# New cluster was sanitized
assert remote_clusters[fake_cluster].__dict__ == ClusterConfigImpl(
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=True).__dict__
def check_able_to_push_new_environment(
user: str, remote_environment_expected_path: str,
remote_environment_push_path: Optional[str]):
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
stack.enter_context(
remove_remote_file(user=user,
path=remote_environment_expected_path))
cluster = show_cluster(name=TEST_CLUSTER)
cluster.config.key = None
cluster.config.install_key = False
cluster.config.retries[Retry.PORT_INFO] = \
get_default_retries_heavy_load()[Retry.PORT_INFO]
assert len(show_clusters()) == 1
node = cluster.get_access_node()
assert isinstance(node, NodeInternal)
# Modify current environment.
fake_cluster = 'fake cluster'
add_cluster(name=fake_cluster,
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key='key_local',
install_key=False)
# Target file does not exist.
with pytest.raises(RuntimeError):
node.run("cat {}".format(remote_environment_expected_path))
push_environment(cluster=cluster, path=remote_environment_push_path)
remote_environment_after_push_serialized = \
get_file_from_node(node=node,
remote_path=remote_environment_expected_path)
remote_environment_after_push = deserialize_environment(
text=remote_environment_after_push_serialized)
# Both clusters were sanitized.
remote_clusters = remote_environment_after_push.config.clusters
assert len(remote_clusters) == 2
assert remote_clusters[TEST_CLUSTER] == ClusterConfigImpl(
host=get_testing_host(),
port=get_testing_port(),
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=True,
retries=get_default_retries_heavy_load())
assert remote_clusters[fake_cluster] == ClusterConfigImpl(
host='fakehost',
port=2,
user=user,
auth=AuthMethod.ASK,
key=None,
install_key=True)