def test_cannot_set_socket_and_port(): workspace_yaml = ''' load_from: - grpc_server: socket: myname port: 5678 ''' with pytest.raises(CheckError, match='must supply either a socket or a port'): load_workspace_from_config( yaml.safe_load(workspace_yaml), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, 'not_a_real.yaml'), )
def test_grpc_multi_location_workspace(config_source): with load_workspace_from_config( yaml.safe_load(config_source(sys.executable)), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, "not_a_real.yaml"), ) as workspace: assert isinstance(workspace, Workspace) assert len(workspace.repository_location_handles) == 6 assert workspace.has_repository_location_handle("loaded_from_file") assert workspace.has_repository_location_handle("loaded_from_module") loaded_from_file_handle = workspace.get_repository_location_handle( "loaded_from_file") assert isinstance(loaded_from_file_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) assert loaded_from_file_handle.repository_names == { "hello_world_repository" } loaded_from_module_handle = workspace.get_repository_location_handle( "loaded_from_module") assert isinstance(loaded_from_module_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) assert loaded_from_module_handle.repository_names == { "hello_world_repository" } named_loaded_from_file_handle = workspace.get_repository_location_handle( "named_loaded_from_file") assert named_loaded_from_file_handle.repository_names == { "hello_world_repository_name" } assert isinstance(named_loaded_from_file_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) named_loaded_from_module_handle = workspace.get_repository_location_handle( "named_loaded_from_module") assert named_loaded_from_module_handle.repository_names == { "hello_world_repository_name" } assert isinstance(named_loaded_from_file_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) named_loaded_from_module_attribute_handle = workspace.get_repository_location_handle( "named_loaded_from_module_attribute") assert named_loaded_from_module_attribute_handle.repository_names == { "hello_world_repository_name" } assert isinstance(named_loaded_from_module_attribute_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) named_loaded_from_file_attribute_handle = workspace.get_repository_location_handle( "named_loaded_from_file_attribute") assert named_loaded_from_file_attribute_handle.repository_names == { "hello_world_repository_name" } assert isinstance(named_loaded_from_file_attribute_handle, ManagedGrpcPythonEnvRepositoryLocationHandle)
def test_cannot_set_socket_and_port(): workspace_yaml = """ load_from: - grpc_server: socket: myname port: 5678 """ with pytest.raises(CheckError, match="must supply either a socket or a port"): load_workspace_from_config( yaml.safe_load(workspace_yaml), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, "not_a_real.yaml"), UserProcessApi.CLI, )
def test_grpc_socket_workspace(): first_server_process = GrpcServerProcess() with first_server_process.create_ephemeral_client() as first_server: second_server_process = GrpcServerProcess() with second_server_process.create_ephemeral_client() as second_server: first_socket = first_server.socket second_socket = second_server.socket workspace_yaml = """ load_from: - grpc_server: host: localhost socket: {socket_one} - grpc_server: socket: {socket_two} location_name: 'local_port_default_host' """.format(socket_one=first_socket, socket_two=second_socket) workspace = load_workspace_from_config( yaml.safe_load(workspace_yaml), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, "not_a_real.yaml"), ) assert isinstance(workspace, Workspace) assert len(workspace.repository_location_handles) == 2 default_location_name = "grpc:localhost:{socket}".format( socket=first_socket) assert workspace.has_repository_location_handle( default_location_name) local_port = workspace.get_repository_location_handle( default_location_name) assert local_port.socket == first_socket assert local_port.host == "localhost" assert local_port.port is None assert workspace.has_repository_location_handle( "local_port_default_host") local_port_default_host = workspace.get_repository_location_handle( "local_port_default_host") assert local_port_default_host.socket == second_socket assert local_port_default_host.host == "localhost" assert local_port_default_host.port is None assert all( map(lambda x: x.location_name, workspace.repository_location_handles)) second_server_process.wait() first_server_process.wait()
def test_grpc_socket_workspace(): with GrpcServerProcess() as first_server: with GrpcServerProcess() as second_server: first_socket = first_server.socket second_socket = second_server.socket workspace_yaml = ''' load_from: - grpc_server: host: localhost socket: {socket_one} - grpc_server: socket: {socket_two} location_name: 'local_port_default_host' '''.format( socket_one=first_socket, socket_two=second_socket ) workspace = load_workspace_from_config( yaml.safe_load(workspace_yaml), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, 'not_a_real.yaml'), python_user_process_api=UserProcessApi.CLI, # doesn't affect grpc_server locations ) assert isinstance(workspace, Workspace) assert len(workspace.repository_location_handles) == 2 default_location_name = 'grpc:localhost:{socket}'.format(socket=first_socket) assert workspace.has_repository_location_handle(default_location_name) local_port = workspace.get_repository_location_handle(default_location_name) assert local_port.socket == first_socket assert local_port.host == 'localhost' assert local_port.port is None assert workspace.has_repository_location_handle('local_port_default_host') local_port_default_host = workspace.get_repository_location_handle( 'local_port_default_host' ) assert local_port_default_host.socket == second_socket assert local_port_default_host.host == 'localhost' assert local_port_default_host.port is None assert all(map(lambda x: x.location_name, workspace.repository_location_handles))
def test_grpc_server_workspace(): with GrpcServerProcess(force_port=True).create_ephemeral_client() as first_server: with GrpcServerProcess(force_port=True).create_ephemeral_client() as second_server: first_port = first_server.port second_port = second_server.port workspace_yaml = ''' load_from: - grpc_server: host: localhost port: {port_one} - grpc_server: port: {port_two} location_name: 'local_port_default_host' '''.format( port_one=first_port, port_two=second_port ) workspace = load_workspace_from_config( yaml.safe_load(workspace_yaml), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, 'not_a_real.yaml'), UserProcessApi.CLI, ) assert isinstance(workspace, Workspace) assert len(workspace.repository_location_handles) == 2 default_location_name = 'grpc:localhost:{port}'.format(port=first_port) assert workspace.has_repository_location_handle(default_location_name) local_port = workspace.get_repository_location_handle(default_location_name) assert local_port.port == first_port assert local_port.host == 'localhost' assert local_port.socket is None assert workspace.has_repository_location_handle('local_port_default_host') local_port_default_host = workspace.get_repository_location_handle( 'local_port_default_host' ) assert local_port_default_host.port == second_port assert local_port_default_host.host == 'localhost' assert local_port_default_host.socket is None assert all(map(lambda x: x.location_name, workspace.repository_location_handles))
def test_multi_python_environment_workspace(): workspace_yaml = ''' load_from: - python_environment: executable_path: {executable} target: python_file: relative_path: hello_world_repository.py location_name: loaded_from_file - python_environment: executable_path: {executable} target: python_module: module_name: dagster.utils.test.hello_world_repository location_name: loaded_from_module '''.format(executable=sys.executable) workspace = load_workspace_from_config( yaml.safe_load(workspace_yaml), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, 'not_a_real.yaml'), ) assert isinstance(workspace, Workspace) assert len(workspace.repository_location_handles) == 2 assert workspace.has_repository_location_handle('loaded_from_file') assert workspace.has_repository_location_handle('loaded_from_module') loaded_from_file_handle = workspace.get_repository_location_handle( 'loaded_from_file') assert set( loaded_from_file_handle.repository_code_pointer_dict.keys()) == { 'hello_world_repository' } loaded_from_module_handle = workspace.get_repository_location_handle( 'loaded_from_module') assert set( loaded_from_module_handle.repository_code_pointer_dict.keys()) == { 'hello_world_repository' }
def test_multi_python_environment_workspace(): with load_workspace_from_config( yaml.safe_load(_get_multi_location_workspace_yaml()), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, "not_a_real.yaml"), UserProcessApi.CLI, ) as workspace: assert isinstance(workspace, Workspace) assert len(workspace.repository_location_handles) == 6 assert workspace.has_repository_location_handle("loaded_from_file") assert workspace.has_repository_location_handle("loaded_from_module") assert workspace.has_repository_location_handle( "named_loaded_from_file") assert workspace.has_repository_location_handle( "named_loaded_from_module") loaded_from_file_handle = workspace.get_repository_location_handle( "loaded_from_file") assert set( loaded_from_file_handle.repository_code_pointer_dict.keys()) == { "hello_world_repository" } assert isinstance(loaded_from_file_handle, PythonEnvRepositoryLocationHandle) loaded_from_module_handle = workspace.get_repository_location_handle( "loaded_from_module") assert set( loaded_from_module_handle.repository_code_pointer_dict.keys()) == { "hello_world_repository" } assert isinstance(loaded_from_file_handle, PythonEnvRepositoryLocationHandle) named_loaded_from_file_handle = workspace.get_repository_location_handle( "named_loaded_from_file") assert set(named_loaded_from_file_handle.repository_code_pointer_dict. keys()) == {"hello_world_repository_name"} assert isinstance(named_loaded_from_file_handle, PythonEnvRepositoryLocationHandle) named_loaded_from_module_handle = workspace.get_repository_location_handle( "named_loaded_from_module") assert set( named_loaded_from_module_handle.repository_code_pointer_dict.keys( )) == {"hello_world_repository_name"} assert isinstance(named_loaded_from_file_handle, PythonEnvRepositoryLocationHandle) named_loaded_from_module_attribute_handle = workspace.get_repository_location_handle( "named_loaded_from_module_attribute") assert set(named_loaded_from_module_attribute_handle. repository_code_pointer_dict.keys()) == { "hello_world_repository_name" } assert isinstance(named_loaded_from_module_attribute_handle, PythonEnvRepositoryLocationHandle) named_loaded_from_file_attribute_handle = workspace.get_repository_location_handle( "named_loaded_from_file_attribute") assert set(named_loaded_from_file_attribute_handle. repository_code_pointer_dict.keys()) == { "hello_world_repository_name" } assert isinstance(named_loaded_from_file_attribute_handle, PythonEnvRepositoryLocationHandle)
def test_grpc_multi_location_workspace(): workspace = load_workspace_from_config( yaml.safe_load(_get_multi_location_workspace_yaml()), # fake out as if it were loaded by a yaml file in this directory file_relative_path(__file__, 'not_a_real.yaml'), UserProcessApi.GRPC, ) assert isinstance(workspace, Workspace) assert len(workspace.repository_location_handles) == 6 assert workspace.has_repository_location_handle('loaded_from_file') assert workspace.has_repository_location_handle('loaded_from_module') loaded_from_file_handle = workspace.get_repository_location_handle( 'loaded_from_file') assert isinstance(loaded_from_file_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) assert loaded_from_file_handle.repository_names == { 'hello_world_repository' } loaded_from_module_handle = workspace.get_repository_location_handle( 'loaded_from_module') assert isinstance(loaded_from_module_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) assert loaded_from_module_handle.repository_names == { 'hello_world_repository' } named_loaded_from_file_handle = workspace.get_repository_location_handle( 'named_loaded_from_file') assert named_loaded_from_file_handle.repository_names == { 'hello_world_repository_name' } assert isinstance(named_loaded_from_file_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) named_loaded_from_module_handle = workspace.get_repository_location_handle( 'named_loaded_from_module') assert named_loaded_from_module_handle.repository_names == { 'hello_world_repository_name' } assert isinstance(named_loaded_from_file_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) named_loaded_from_module_attribute_handle = workspace.get_repository_location_handle( 'named_loaded_from_module_attribute') assert named_loaded_from_module_attribute_handle.repository_names == { 'hello_world_repository_name' } assert isinstance(named_loaded_from_module_attribute_handle, ManagedGrpcPythonEnvRepositoryLocationHandle) named_loaded_from_file_attribute_handle = workspace.get_repository_location_handle( 'named_loaded_from_file_attribute') assert named_loaded_from_file_attribute_handle.repository_names == { 'hello_world_repository_name' } assert isinstance(named_loaded_from_file_attribute_handle, ManagedGrpcPythonEnvRepositoryLocationHandle)