def test_cannot_set_socket_and_port():
workspace_yaml = '''
load_from:
- grpc_server:
socket: myname
port: 5678
'''
with pytest.raises(CheckError, match='must supply either a socket or a port'):
load_workspace_from_config(
yaml.safe_load(workspace_yaml),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, 'not_a_real.yaml'),
)
def test_grpc_multi_location_workspace(config_source):
with load_workspace_from_config(
yaml.safe_load(config_source(sys.executable)),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, "not_a_real.yaml"),
) as workspace:
assert isinstance(workspace, Workspace)
assert len(workspace.repository_location_handles) == 6
assert workspace.has_repository_location_handle("loaded_from_file")
assert workspace.has_repository_location_handle("loaded_from_module")
loaded_from_file_handle = workspace.get_repository_location_handle(
"loaded_from_file")
assert isinstance(loaded_from_file_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
assert loaded_from_file_handle.repository_names == {
"hello_world_repository"
}
loaded_from_module_handle = workspace.get_repository_location_handle(
"loaded_from_module")
assert isinstance(loaded_from_module_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
assert loaded_from_module_handle.repository_names == {
"hello_world_repository"
}
named_loaded_from_file_handle = workspace.get_repository_location_handle(
"named_loaded_from_file")
assert named_loaded_from_file_handle.repository_names == {
"hello_world_repository_name"
}
assert isinstance(named_loaded_from_file_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
named_loaded_from_module_handle = workspace.get_repository_location_handle(
"named_loaded_from_module")
assert named_loaded_from_module_handle.repository_names == {
"hello_world_repository_name"
}
assert isinstance(named_loaded_from_file_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
named_loaded_from_module_attribute_handle = workspace.get_repository_location_handle(
"named_loaded_from_module_attribute")
assert named_loaded_from_module_attribute_handle.repository_names == {
"hello_world_repository_name"
}
assert isinstance(named_loaded_from_module_attribute_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
named_loaded_from_file_attribute_handle = workspace.get_repository_location_handle(
"named_loaded_from_file_attribute")
assert named_loaded_from_file_attribute_handle.repository_names == {
"hello_world_repository_name"
}
assert isinstance(named_loaded_from_file_attribute_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
def test_cannot_set_socket_and_port():
workspace_yaml = """
load_from:
- grpc_server:
socket: myname
port: 5678
"""
with pytest.raises(CheckError,
match="must supply either a socket or a port"):
load_workspace_from_config(
yaml.safe_load(workspace_yaml),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, "not_a_real.yaml"),
UserProcessApi.CLI,
)
def test_grpc_socket_workspace():
first_server_process = GrpcServerProcess()
with first_server_process.create_ephemeral_client() as first_server:
second_server_process = GrpcServerProcess()
with second_server_process.create_ephemeral_client() as second_server:
first_socket = first_server.socket
second_socket = second_server.socket
workspace_yaml = """
load_from:
- grpc_server:
host: localhost
socket: {socket_one}
- grpc_server:
socket: {socket_two}
location_name: 'local_port_default_host'
""".format(socket_one=first_socket, socket_two=second_socket)
workspace = load_workspace_from_config(
yaml.safe_load(workspace_yaml),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, "not_a_real.yaml"),
)
assert isinstance(workspace, Workspace)
assert len(workspace.repository_location_handles) == 2
default_location_name = "grpc:localhost:{socket}".format(
socket=first_socket)
assert workspace.has_repository_location_handle(
default_location_name)
local_port = workspace.get_repository_location_handle(
default_location_name)
assert local_port.socket == first_socket
assert local_port.host == "localhost"
assert local_port.port is None
assert workspace.has_repository_location_handle(
"local_port_default_host")
local_port_default_host = workspace.get_repository_location_handle(
"local_port_default_host")
assert local_port_default_host.socket == second_socket
assert local_port_default_host.host == "localhost"
assert local_port_default_host.port is None
assert all(
map(lambda x: x.location_name,
workspace.repository_location_handles))
second_server_process.wait()
first_server_process.wait()
def test_grpc_socket_workspace():
with GrpcServerProcess() as first_server:
with GrpcServerProcess() as second_server:
first_socket = first_server.socket
second_socket = second_server.socket
workspace_yaml = '''
load_from:
- grpc_server:
host: localhost
socket: {socket_one}
- grpc_server:
socket: {socket_two}
location_name: 'local_port_default_host'
'''.format(
socket_one=first_socket, socket_two=second_socket
)
workspace = load_workspace_from_config(
yaml.safe_load(workspace_yaml),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, 'not_a_real.yaml'),
python_user_process_api=UserProcessApi.CLI, # doesn't affect grpc_server locations
)
assert isinstance(workspace, Workspace)
assert len(workspace.repository_location_handles) == 2
default_location_name = 'grpc:localhost:{socket}'.format(socket=first_socket)
assert workspace.has_repository_location_handle(default_location_name)
local_port = workspace.get_repository_location_handle(default_location_name)
assert local_port.socket == first_socket
assert local_port.host == 'localhost'
assert local_port.port is None
assert workspace.has_repository_location_handle('local_port_default_host')
local_port_default_host = workspace.get_repository_location_handle(
'local_port_default_host'
)
assert local_port_default_host.socket == second_socket
assert local_port_default_host.host == 'localhost'
assert local_port_default_host.port is None
assert all(map(lambda x: x.location_name, workspace.repository_location_handles))
def test_grpc_server_workspace():
with GrpcServerProcess(force_port=True).create_ephemeral_client() as first_server:
with GrpcServerProcess(force_port=True).create_ephemeral_client() as second_server:
first_port = first_server.port
second_port = second_server.port
workspace_yaml = '''
load_from:
- grpc_server:
host: localhost
port: {port_one}
- grpc_server:
port: {port_two}
location_name: 'local_port_default_host'
'''.format(
port_one=first_port, port_two=second_port
)
workspace = load_workspace_from_config(
yaml.safe_load(workspace_yaml),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, 'not_a_real.yaml'),
UserProcessApi.CLI,
)
assert isinstance(workspace, Workspace)
assert len(workspace.repository_location_handles) == 2
default_location_name = 'grpc:localhost:{port}'.format(port=first_port)
assert workspace.has_repository_location_handle(default_location_name)
local_port = workspace.get_repository_location_handle(default_location_name)
assert local_port.port == first_port
assert local_port.host == 'localhost'
assert local_port.socket is None
assert workspace.has_repository_location_handle('local_port_default_host')
local_port_default_host = workspace.get_repository_location_handle(
'local_port_default_host'
)
assert local_port_default_host.port == second_port
assert local_port_default_host.host == 'localhost'
assert local_port_default_host.socket is None
assert all(map(lambda x: x.location_name, workspace.repository_location_handles))
def test_multi_python_environment_workspace():
workspace_yaml = '''
load_from:
- python_environment:
executable_path: {executable}
target:
python_file:
relative_path: hello_world_repository.py
location_name: loaded_from_file
- python_environment:
executable_path: {executable}
target:
python_module:
module_name: dagster.utils.test.hello_world_repository
location_name: loaded_from_module
'''.format(executable=sys.executable)
workspace = load_workspace_from_config(
yaml.safe_load(workspace_yaml),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, 'not_a_real.yaml'),
)
assert isinstance(workspace, Workspace)
assert len(workspace.repository_location_handles) == 2
assert workspace.has_repository_location_handle('loaded_from_file')
assert workspace.has_repository_location_handle('loaded_from_module')
loaded_from_file_handle = workspace.get_repository_location_handle(
'loaded_from_file')
assert set(
loaded_from_file_handle.repository_code_pointer_dict.keys()) == {
'hello_world_repository'
}
loaded_from_module_handle = workspace.get_repository_location_handle(
'loaded_from_module')
assert set(
loaded_from_module_handle.repository_code_pointer_dict.keys()) == {
'hello_world_repository'
}
def test_multi_python_environment_workspace():
with load_workspace_from_config(
yaml.safe_load(_get_multi_location_workspace_yaml()),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, "not_a_real.yaml"),
UserProcessApi.CLI,
) as workspace:
assert isinstance(workspace, Workspace)
assert len(workspace.repository_location_handles) == 6
assert workspace.has_repository_location_handle("loaded_from_file")
assert workspace.has_repository_location_handle("loaded_from_module")
assert workspace.has_repository_location_handle(
"named_loaded_from_file")
assert workspace.has_repository_location_handle(
"named_loaded_from_module")
loaded_from_file_handle = workspace.get_repository_location_handle(
"loaded_from_file")
assert set(
loaded_from_file_handle.repository_code_pointer_dict.keys()) == {
"hello_world_repository"
}
assert isinstance(loaded_from_file_handle,
PythonEnvRepositoryLocationHandle)
loaded_from_module_handle = workspace.get_repository_location_handle(
"loaded_from_module")
assert set(
loaded_from_module_handle.repository_code_pointer_dict.keys()) == {
"hello_world_repository"
}
assert isinstance(loaded_from_file_handle,
PythonEnvRepositoryLocationHandle)
named_loaded_from_file_handle = workspace.get_repository_location_handle(
"named_loaded_from_file")
assert set(named_loaded_from_file_handle.repository_code_pointer_dict.
keys()) == {"hello_world_repository_name"}
assert isinstance(named_loaded_from_file_handle,
PythonEnvRepositoryLocationHandle)
named_loaded_from_module_handle = workspace.get_repository_location_handle(
"named_loaded_from_module")
assert set(
named_loaded_from_module_handle.repository_code_pointer_dict.keys(
)) == {"hello_world_repository_name"}
assert isinstance(named_loaded_from_file_handle,
PythonEnvRepositoryLocationHandle)
named_loaded_from_module_attribute_handle = workspace.get_repository_location_handle(
"named_loaded_from_module_attribute")
assert set(named_loaded_from_module_attribute_handle.
repository_code_pointer_dict.keys()) == {
"hello_world_repository_name"
}
assert isinstance(named_loaded_from_module_attribute_handle,
PythonEnvRepositoryLocationHandle)
named_loaded_from_file_attribute_handle = workspace.get_repository_location_handle(
"named_loaded_from_file_attribute")
assert set(named_loaded_from_file_attribute_handle.
repository_code_pointer_dict.keys()) == {
"hello_world_repository_name"
}
assert isinstance(named_loaded_from_file_attribute_handle,
PythonEnvRepositoryLocationHandle)
def test_grpc_multi_location_workspace():
workspace = load_workspace_from_config(
yaml.safe_load(_get_multi_location_workspace_yaml()),
# fake out as if it were loaded by a yaml file in this directory
file_relative_path(__file__, 'not_a_real.yaml'),
UserProcessApi.GRPC,
)
assert isinstance(workspace, Workspace)
assert len(workspace.repository_location_handles) == 6
assert workspace.has_repository_location_handle('loaded_from_file')
assert workspace.has_repository_location_handle('loaded_from_module')
loaded_from_file_handle = workspace.get_repository_location_handle(
'loaded_from_file')
assert isinstance(loaded_from_file_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
assert loaded_from_file_handle.repository_names == {
'hello_world_repository'
}
loaded_from_module_handle = workspace.get_repository_location_handle(
'loaded_from_module')
assert isinstance(loaded_from_module_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
assert loaded_from_module_handle.repository_names == {
'hello_world_repository'
}
named_loaded_from_file_handle = workspace.get_repository_location_handle(
'named_loaded_from_file')
assert named_loaded_from_file_handle.repository_names == {
'hello_world_repository_name'
}
assert isinstance(named_loaded_from_file_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
named_loaded_from_module_handle = workspace.get_repository_location_handle(
'named_loaded_from_module')
assert named_loaded_from_module_handle.repository_names == {
'hello_world_repository_name'
}
assert isinstance(named_loaded_from_file_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
named_loaded_from_module_attribute_handle = workspace.get_repository_location_handle(
'named_loaded_from_module_attribute')
assert named_loaded_from_module_attribute_handle.repository_names == {
'hello_world_repository_name'
}
assert isinstance(named_loaded_from_module_attribute_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
named_loaded_from_file_attribute_handle = workspace.get_repository_location_handle(
'named_loaded_from_file_attribute')
assert named_loaded_from_file_attribute_handle.repository_names == {
'hello_world_repository_name'
}
assert isinstance(named_loaded_from_file_attribute_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle)
